refactor(subscribe): govern fallback site pressure

This commit is contained in:
jxxghp
2026-09-01 17:09:54 +08:00
parent 389131acd6
commit bab1d2862e
19 changed files with 1071 additions and 18 deletions
@@ -0,0 +1,50 @@
"""站点搜索调用的轻量结果观察上下文。"""
from __future__ import annotations
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from typing import Iterator, Optional
@dataclass(slots=True)
class SiteSearchObservation:
"""记录一次站点搜索是否发出请求及其可治理结果。"""
attempted: bool = False
outcome: str = "skipped"
error: Optional[str] = None
_current_observation: ContextVar[Optional[SiteSearchObservation]] = ContextVar(
"site_search_observation",
default=None,
)
@contextmanager
def capture_site_search_observation() -> Iterator[SiteSearchObservation]:
"""为当前同步 worker 捕获一次索引器搜索结果。"""
observation = SiteSearchObservation()
token = _current_observation.set(observation)
try:
yield observation
finally:
_current_observation.reset(token)
def report_site_search_outcome(
*,
attempted: bool,
outcome: str,
error: Optional[str] = None,
) -> None:
"""由索引器在不改变公开返回合同的前提下发布调用结果。"""
observation = _current_observation.get()
if observation is None:
return
observation.attempted = attempted
observation.outcome = outcome
observation.error = error
+27 -1
View File
@@ -3,6 +3,8 @@
from dataclasses import dataclass
from typing import Optional, Protocol
from app.application.subscription.sitebudget import SiteBudgetClaim
@dataclass(frozen=True, slots=True)
class SearchBatchSnapshot:
@@ -40,6 +42,7 @@ class SearchTaskSnapshot:
lease_token: Optional[str]
created_at: str
updated_at: str
available_at: Optional[str] = None
started_at: Optional[str] = None
finished_at: Optional[str] = None
last_error: Optional[str] = None
@@ -63,8 +66,9 @@ class SubscriptionSearchRepository(Protocol):
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str] = None,
) -> SearchEnqueueResult:
"""按订阅 ID 建立批次,并合并已存在的活动任务。"""
"""按订阅 ID 建立批次,在启动抖动后合并活动任务。"""
...
def claim_next(self, *, owner: str, lease_seconds: int = 900) -> Optional[SearchTaskSnapshot]:
@@ -103,3 +107,25 @@ class SubscriptionSearchRepository(Protocol):
def get_batch(self, batch_id: str) -> Optional[SearchBatchSnapshot]:
"""按稳定批次 ID 返回当前聚合状态。"""
...
def claim_site(
self,
*,
site_id: int,
owner: str,
lease_seconds: int,
) -> SiteBudgetClaim:
"""认领一个站点的唯一在途搜索预算。"""
...
def finish_site(
self,
*,
site_id: int,
lease_token: str,
outcome: str,
next_allowed_at: str,
error: Optional[str] = None,
) -> bool:
"""释放站点预算并写入间隔、冷却和恢复状态。"""
...
+156
View File
@@ -0,0 +1,156 @@
"""订阅兜底搜索的站点级容量、间隔与冷却治理。"""
from __future__ import annotations
import random
import time
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Optional, Protocol
from app.application.site.search_observation import SiteSearchObservation
from app.runtime.stop import StopState
class SubscriptionSearchCancelled(RuntimeError):
"""表示订阅搜索在可取消预算等待点终止。"""
class SubscriptionSiteBudgetUnavailable(RuntimeError):
"""表示站点预算超出本任务允许的短等待窗口。"""
def __init__(self, *, site_id: int, retry_at: str) -> None:
"""保存站点和下一次可尝试时间,供批次聚合失败展示。"""
super().__init__(f"站点 {site_id} 冷却或已有在途搜索,最早可重试:{retry_at}")
self.site_id = site_id
self.retry_at = retry_at
@dataclass(frozen=True, slots=True)
class SiteBudgetClaim:
"""一次站点预算认领结果或下一次可尝试时间。"""
site_id: int
acquired: bool
retry_at: str
consecutive_failures: int
lease_token: Optional[str] = None
class SubscriptionSiteBudgetRepository(Protocol):
"""站点预算租约和冷却状态的持久化端口。"""
def claim_site(
self,
*,
site_id: int,
owner: str,
lease_seconds: int,
) -> SiteBudgetClaim:
"""认领站点唯一在途租约,未就绪时返回重试时间。"""
...
def finish_site(
self,
*,
site_id: int,
lease_token: str,
outcome: str,
next_allowed_at: str,
error: Optional[str] = None,
) -> bool:
"""收口当前站点租约并持久化下次允许时间。"""
...
def _utc_now() -> datetime:
"""返回带时区的 UTC 当前时间。"""
return datetime.now(timezone.utc)
class SubscriptionSiteBudget:
"""在同步搜索 worker 内执行可取消的站点预算等待与反馈。"""
def __init__(
self,
*,
repository: SubscriptionSiteBudgetRepository,
owner: str,
cancelled: Callable[[], bool],
stop_state: StopState,
interval_range: tuple[float, float] = (60.0, 300.0),
lease_seconds: int = 900,
max_wait_seconds: float = 5.0,
random_uniform: Callable[[float, float], float] = random.uniform,
sleeper: Callable[[float], None] = time.sleep,
clock: Callable[[], datetime] = _utc_now,
) -> None:
"""保存持久化端口及可注入的时钟、随机数和等待实现。"""
self._repository = repository
self._owner = owner
self._cancelled = cancelled
self._stop_state = stop_state
self._interval_range = interval_range
self._lease_seconds = max(1, lease_seconds)
self._max_wait_seconds = max(0.0, max_wait_seconds)
self._random_uniform = random_uniform
self._sleeper = sleeper
self._clock = clock
def acquire(self, site_id: int) -> SiteBudgetClaim:
"""循环认领指定站点,并在每秒边界检查取消与停机。"""
deadline = time.monotonic() + self._max_wait_seconds
while True:
self._raise_if_cancelled()
claim = self._repository.claim_site(
site_id=site_id,
owner=self._owner,
lease_seconds=self._lease_seconds,
)
if claim.acquired:
return claim
retry_at = datetime.fromisoformat(claim.retry_at)
remaining = max(0.0, (retry_at - self._clock()).total_seconds())
if remaining > max(0.0, deadline - time.monotonic()):
raise SubscriptionSiteBudgetUnavailable(
site_id=site_id,
retry_at=claim.retry_at,
)
self._sleeper(min(max(remaining, 0.05), 1.0))
def finish(self, claim: SiteBudgetClaim, observation: SiteSearchObservation) -> bool:
"""依据调用结果计算随机间隔或错误冷却并释放租约。"""
if not claim.lease_token:
return False
outcome = observation.outcome if observation.attempted else "skipped"
delay = self._next_delay(outcome, claim.consecutive_failures)
next_allowed_at = (self._clock() + timedelta(seconds=delay)).isoformat(timespec="seconds")
return self._repository.finish_site(
site_id=claim.site_id,
lease_token=claim.lease_token,
outcome=outcome,
next_allowed_at=next_allowed_at,
error=observation.error,
)
def _next_delay(self, outcome: str, consecutive_failures: int) -> float:
"""为成功渐进恢复、错误退避和本地跳过计算下一次等待。"""
if outcome == "skipped":
return 0.0
if outcome == "success":
low, high = self._interval_range
recovery_factor = 1.0 + min(max(consecutive_failures, 0), 3) * 0.5
return self._random_uniform(low, high) * recovery_factor
exponent = min(max(consecutive_failures, 0), 5)
base, ceiling = {
"rate_limited": (900.0, 21600.0),
"forbidden": (900.0, 21600.0),
"login_invalid": (900.0, 21600.0),
"timeout": (300.0, 7200.0),
}.get(outcome, (180.0, 3600.0))
return min(base * (2**exponent), ceiling)
def _raise_if_cancelled(self) -> None:
"""在不持有业务锁的等待边界传播取消或停机。"""
if self._stop_state.is_system_stopped or self._cancelled():
raise SubscriptionSearchCancelled("订阅搜索已取消")
+3
View File
@@ -63,6 +63,7 @@ if TYPE_CHECKING:
_restore_original_indices: Callable[..., Any]
_save_results: Callable[..., Any]
_search_all_sites: Callable[..., Any]
_search_site_torrents_with_budget: Callable[..., Any]
_search_state: Callable[..., Any]
_select_indexers: Callable[..., Any]
_selected_site_ids: Callable[..., Any]
@@ -96,6 +97,8 @@ if TYPE_CHECKING:
matches_music_resource: Callable[..., Any]
music_site_keywords: Callable[..., Any]
process: Callable[..., Any]
record_subscription_site_budget_failure: Callable[..., Any]
consume_subscription_site_budget_failures: Callable[..., Any]
save_last_search_params: Callable[..., Any]
search_by_id: Callable[..., Any]
search_by_title: Callable[..., Any]
+30
View File
@@ -1,5 +1,6 @@
"""搜索处理链稳定 Facade。"""
import threading
from collections.abc import AsyncIterator, Callable
from typing import Any, Optional, TypeVar, cast
@@ -15,6 +16,7 @@ from app.chain.search.result import SearchResultOwner
from app.chain.search.site import SearchSiteOwner
from app.chain.search.subtitle import SearchSubtitleOwner
from app.chain.search.title import SearchTitleOwner
from app.application.subscription.sitebudget import SubscriptionSiteBudget
from app.domain.context import Context, MediaInfo, SubtitleInfo
from app.runtime.events import Event, eventmanager
from app.schemas.mediaserver import NotExistMediaInfo
@@ -40,6 +42,33 @@ class SearchChain(ChainBase):
_SEARCH_PARAMS_CACHE_KEY = "__search_params__"
_AI_INDICES_CACHE_KEY = "__ai_recommend_indices__"
def configure_subscription_site_budget(
self,
budget: Optional[SubscriptionSiteBudget],
) -> None:
"""仅为订阅搜索启用或清除站点预算,不影响其它搜索入口。"""
self._subscription_site_budget = budget
self._subscription_site_budget_failures: list[str] = []
self._subscription_site_budget_failure_lock = threading.Lock()
def record_subscription_site_budget_failure(self, error: str) -> None:
"""线程安全地记录一个未执行站点,供订阅任务暴露聚合失败。"""
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
if lock is None:
return
with lock:
self._subscription_site_budget_failures.append(error)
def consume_subscription_site_budget_failures(self) -> tuple[str, ...]:
"""读取并清空当前订阅搜索积累的站点预算失败。"""
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
if lock is None:
return ()
with lock:
failures = tuple(self._subscription_site_budget_failures)
self._subscription_site_budget_failures.clear()
return failures
# owner descriptor 经类访问后被 mypy 视为普通 Callable;运行时仍需取回原始
# classmethod 函数,才能保持 SearchChain 的直接 MRO 与既有绑定语义。
music_site_keywords = classmethod( # type: ignore[var-annotated]
@@ -456,6 +485,7 @@ class SearchChain(ChainBase):
_async_indexers = SearchProviderOwner._async_indexers
_torrent_keyword = staticmethod(SearchProviderOwner._torrent_keyword)
_torrent_type = staticmethod(SearchProviderOwner._torrent_type)
_search_site_torrents_with_budget = SearchProviderOwner._search_site_torrents_with_budget
_iter_provider_batches = SearchProviderOwner._iter_provider_batches
_iter_provider_events = SearchProviderOwner._iter_provider_events
_iter_torrent_events = SearchProviderOwner._iter_torrent_events
+63 -4
View File
@@ -10,7 +10,15 @@ from datetime import datetime
from typing import Any, AsyncGenerator, Awaitable, Callable, Dict, Iterator, List, Optional
from app.application.configuration import get_configured_system_config
from app.application.site.search_observation import (
capture_site_search_observation,
report_site_search_outcome,
)
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
from app.application.subscription.sitebudget import (
SubscriptionSiteBudget,
SubscriptionSiteBudgetUnavailable,
)
from app.chain.search.contract import _SearchOwnerBase
from app.domain.context import MediaInfo, SubtitleInfo, TorrentInfo
from app.runtime.log import logger
@@ -182,13 +190,64 @@ class SearchProviderOwner(_SearchOwnerBase):
"""向进程共享线程 owner 提交一页,并登记该站点的续页位置。"""
page_number = search_pages[page_index]
future = ThreadHelper().submit(
_search_site_page,
self,
site=site, keyword=search_keyword,
media_type=media_type, page=page_number,
self._search_site_torrents_with_budget,
site=site,
keyword=search_keyword,
mtype=media_type,
page=page_number,
)
pending[future] = (site, page_index, page_number)
def _search_site_torrents_with_budget(
self,
*,
site: SiteIndexer,
keyword: str,
mtype: Optional[MediaType],
page: int,
) -> List[TorrentInfo]:
"""在订阅专属站点预算内执行一页同步搜索。"""
budget = getattr(self, "_subscription_site_budget", None)
site_id = site.get("id")
if not isinstance(budget, SubscriptionSiteBudget) or not isinstance(site_id, int):
return _search_site_page(
self,
site=site,
keyword=keyword,
media_type=mtype,
page=page,
)
try:
claim = budget.acquire(site_id)
except SubscriptionSiteBudgetUnavailable as error:
self.record_subscription_site_budget_failure(str(error))
logger.info(str(error))
return []
with capture_site_search_observation() as observation:
try:
return _search_site_page(
self,
site=site,
keyword=keyword,
media_type=mtype,
page=page,
)
except Exception as error:
report_site_search_outcome(
attempted=True,
outcome="error",
error=str(error),
)
raise
finally:
try:
budget.finish(claim, observation)
except Exception as error: # noqa: BLE001
logger.error(
f"站点 {site.get('name') or site_id} 搜索预算收口失败:{str(error)}",
exc_info=True,
)
def _submit_next_sync_site(
self,
*,
+41 -1
View File
@@ -2,7 +2,7 @@
import random
import time
from datetime import datetime
from datetime import datetime, timedelta, timezone
from typing import Any, Callable, Optional, cast
from uuid import uuid4
@@ -15,6 +15,10 @@ from app.application.subscription.contract import (
)
from app.application.subscription.query import SubscriptionQueryService
from app.application.subscription.execution import SearchBatchSnapshot, SubscriptionSearchRepository
from app.application.subscription.sitebudget import (
SubscriptionSearchCancelled,
SubscriptionSiteBudget,
)
from app.chain.media import MediaChain
from app.chain.search.facade import SearchChain
from app.chain.subscribe.contract import _SubscribeOwnerBase
@@ -209,6 +213,7 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
subscription_ids=tuple(subscribe.id for subscribe in subscribes),
source=source,
priority=priority,
available_at=self._search_batch_available_at(source),
)
total = len(subscribes)
if progress_callback:
@@ -293,6 +298,14 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
)
continue
current = subscribe
searchchain.configure_subscription_site_budget(
SubscriptionSiteBudget(
repository=queue,
owner=f"{owner}:{task.task_id}",
cancelled=lambda task_id=task.task_id: queue.is_cancel_requested(task_id),
stop_state=getattr(self, "stop_state", runtime_stop_state),
)
)
try:
current = self._process_search_subscription(subscribe, searchchain)
if queue.is_cancel_requested(task.task_id):
@@ -308,6 +321,12 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
state="completed",
)
processed.add(subscribe.id)
except SubscriptionSearchCancelled:
queue.release_task(
task_id=task.task_id,
lease_token=task.lease_token,
cancelled=True,
)
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
queue.finish_task(
@@ -317,6 +336,7 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
error=str(err),
)
finally:
searchchain.configure_subscription_site_budget(None)
if current and current.state == "N":
try:
self._SubscribeChain__apply_subscribe_update(
@@ -379,8 +399,18 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
return "订阅搜索完成,部分任务失败"
if batch.state == "cancelled":
return "订阅搜索已取消"
if batch.state in {"queued", "running", "cancelling"}:
return "订阅搜索任务已排队"
return "订阅搜索完成"
@staticmethod
def _search_batch_available_at(source: str) -> str:
"""为自动兜底批次持久化一次全局启动抖动。"""
delay = random.randint(0, 60) if source == "fallback" else 0
return (
datetime.now(timezone.utc) + timedelta(seconds=delay)
).isoformat(timespec="seconds")
def cancel_search_batch(self, batch_id: str) -> bool:
"""请求取消持久搜索批次;未注入队列时返回失败。"""
queue = cast(
@@ -491,6 +521,7 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
custom_words=subscribe.custom_words.split("\n") if subscribe.custom_words else None,
filter_params=self.get_params(subscribe),
)
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
if not contexts:
logger.warning(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
self.finish_subscribe_or_not(
@@ -499,11 +530,13 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
mediainfo=mediainfo,
lefts=no_exists,
)
self._raise_site_budget_failures(site_budget_failures)
return subscribe
matched = self._filter_search_contexts(subscribe, contexts)
if not matched:
logger.warning(f"订阅 {subscribe.name} 没有符合过滤条件的资源")
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
self._raise_site_budget_failures(site_budget_failures)
return subscribe
downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first(
contexts=matched,
@@ -524,8 +557,15 @@ class SubscribeSearchOwner(_SubscribeOwnerBase):
downloads=downloads,
lefts=lefts,
)
self._raise_site_budget_failures(site_budget_failures)
return cast(Optional[SubscriptionSnapshot], current)
@staticmethod
def _raise_site_budget_failures(failures: tuple[str, ...]) -> None:
"""在成功站点结果完成处理后暴露未执行站点的聚合失败。"""
if failures:
raise RuntimeError("".join(failures))
def _filter_search_contexts(
self,
subscribe: SubscriptionSnapshot,
+55
View File
@@ -10,7 +10,9 @@ from app.application.subscription.execution import (
SearchEnqueueResult,
SearchTaskSnapshot,
)
from app.application.subscription.sitebudget import SiteBudgetClaim
from app.db.models.subscriptionsearch import (
SubscriptionSiteBudget,
SubscriptionSearchBatch,
SubscriptionSearchTask,
)
@@ -55,6 +57,7 @@ def _task(record: SubscriptionSearchTask) -> SearchTaskSnapshot:
lease_token=record.lease_token,
created_at=record.created_at,
updated_at=record.updated_at,
available_at=record.available_at,
started_at=record.started_at,
finished_at=record.finished_at,
last_error=record.last_error,
@@ -91,6 +94,7 @@ class TransactionalSubscriptionSearchRepository:
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str] = None,
) -> SearchEnqueueResult:
"""创建批次并返回 single-flight 合并计数。"""
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
@@ -99,6 +103,7 @@ class TransactionalSubscriptionSearchRepository:
subscription_ids=subscription_ids,
source=source,
priority=priority,
available_at=available_at,
)
return SearchEnqueueResult(
batch=_batch(record),
@@ -167,3 +172,53 @@ class TransactionalSubscriptionSearchRepository:
_batch(record) if (record := repository.get_batch(batch_id)) is not None else None
)
)
def claim_site(
self,
*,
site_id: int,
owner: str,
lease_seconds: int,
) -> SiteBudgetClaim:
"""认领单站点预算并投影等待或租约事实。"""
def operation(repository: SubscriptionSearchOper) -> SiteBudgetClaim:
"""在短事务中认领并复制站点预算状态。"""
record, acquired = repository.claim_site(
site_id=site_id,
owner=owner,
lease_seconds=lease_seconds,
)
retry_at = (
record.lease_expires_at
if record.lease_token and not acquired
else record.next_allowed_at
) or record.next_allowed_at
return SiteBudgetClaim(
site_id=record.site_id,
acquired=acquired,
retry_at=retry_at,
consecutive_failures=record.consecutive_failures,
lease_token=record.lease_token if acquired else None,
)
return self._write(operation)
def finish_site(
self,
*,
site_id: int,
lease_token: str,
outcome: str,
next_allowed_at: str,
error: Optional[str] = None,
) -> bool:
"""释放站点租约并持久化间隔或冷却。"""
return self._write(
lambda repository: repository.finish_site(
site_id=site_id,
lease_token=lease_token,
outcome=outcome,
next_allowed_at=next_allowed_at,
error=error,
)
)
+4
View File
@@ -43,6 +43,10 @@ _MODEL_EXPORTS = {
"app.db.models.subscriptionsearch",
"SubscriptionSearchTask",
),
"SubscriptionSiteBudget": (
"app.db.models.subscriptionsearch",
"SubscriptionSiteBudget",
),
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
"TransferExecutionStep": (
+28
View File
@@ -50,6 +50,7 @@ class SubscriptionSearchTask(Base):
lease_owner: Mapped[Optional[str]] = mapped_column(String(128))
lease_token: Mapped[Optional[str]] = mapped_column(String(64))
lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40))
available_at: Mapped[Optional[str]] = mapped_column(String(40))
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
started_at: Mapped[Optional[str]] = mapped_column(String(40))
@@ -63,6 +64,7 @@ class SubscriptionSearchTask(Base):
"ix_subscriptionsearchtask_claim",
"state",
"priority",
"available_at",
"lease_expires_at",
"created_at",
"id",
@@ -70,3 +72,29 @@ class SubscriptionSearchTask(Base):
Index("ix_subscriptionsearchtask_batch_position", "batch_id", "position", "id"),
Index("ix_subscriptionsearchtask_subscription", "subscription_id", "created_at", "id"),
)
class SubscriptionSiteBudget(Base):
"""记录兜底搜索对单个站点的唯一租约、间隔与错误冷却。"""
id = get_id_column()
site_id: Mapped[int] = mapped_column(Integer, nullable=False)
lease_owner: Mapped[Optional[str]] = mapped_column(String(128))
lease_token: Mapped[Optional[str]] = mapped_column(String(64))
lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40))
next_allowed_at: Mapped[str] = mapped_column(String(40), nullable=False)
consecutive_failures: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
success_streak: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_outcome: Mapped[Optional[str]] = mapped_column(String(32))
last_error: Mapped[Optional[str]] = mapped_column(Text)
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
__table_args__ = (
UniqueConstraint("site_id", name="uq_subscriptionsitebudget_site_id"),
Index(
"ix_subscriptionsitebudget_ready",
"next_allowed_at",
"lease_expires_at",
"site_id",
),
)
+159
View File
@@ -10,6 +10,7 @@ from sqlalchemy.orm import Session
from app.db.base import DbOper, execute_dml
from app.db.models.subscriptionsearch import (
SubscriptionSiteBudget,
SubscriptionSearchBatch,
SubscriptionSearchTask,
)
@@ -29,6 +30,7 @@ class SubscriptionSearchOper(DbOper):
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str],
) -> tuple[SubscriptionSearchBatch, int, int]:
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
if not isinstance(self._db, Session):
@@ -58,6 +60,7 @@ class SubscriptionSearchOper(DbOper):
priority=priority,
position=position,
state="queued",
available_at=available_at or now,
created_at=now,
updated_at=now,
)
@@ -77,6 +80,16 @@ class SubscriptionSearchOper(DbOper):
(SubscriptionSearchTask.priority < priority, priority),
else_=SubscriptionSearchTask.priority,
),
available_at=case(
(
or_(
SubscriptionSearchTask.available_at.is_(None),
SubscriptionSearchTask.available_at > (available_at or now),
),
available_at or now,
),
else_=SubscriptionSearchTask.available_at,
),
updated_at=now,
),
execution_options={"synchronize_session": False},
@@ -100,6 +113,10 @@ class SubscriptionSearchOper(DbOper):
select(SubscriptionSearchTask)
.where(
SubscriptionSearchTask.cancel_requested == 0,
or_(
SubscriptionSearchTask.available_at.is_(None),
SubscriptionSearchTask.available_at <= now,
),
or_(
SubscriptionSearchTask.state == "queued",
and_(
@@ -113,6 +130,7 @@ class SubscriptionSearchOper(DbOper):
)
.order_by(
SubscriptionSearchTask.priority.desc(),
SubscriptionSearchTask.available_at.asc(),
SubscriptionSearchTask.created_at.asc(),
SubscriptionSearchTask.position.asc(),
SubscriptionSearchTask.id.asc(),
@@ -128,6 +146,10 @@ class SubscriptionSearchOper(DbOper):
.where(
SubscriptionSearchTask.id == candidate.id,
SubscriptionSearchTask.cancel_requested == 0,
or_(
SubscriptionSearchTask.available_at.is_(None),
SubscriptionSearchTask.available_at <= now,
),
or_(
SubscriptionSearchTask.state == "queued",
and_(
@@ -334,6 +356,143 @@ class SubscriptionSearchOper(DbOper):
select(SubscriptionSearchBatch).where(SubscriptionSearchBatch.batch_id == batch_id)
).scalars().first()
def claim_site(
self,
*,
site_id: int,
owner: str,
lease_seconds: int,
) -> tuple[SubscriptionSiteBudget, bool]:
"""以 CAS 认领单站点租约,返回当前或已认领预算记录。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅站点预算认领需要调用方提供同步 Session")
now = utc_now_text()
record = self._ensure_site_budget(site_id=site_id, now=now)
lease_busy = bool(
record.lease_token
and record.lease_expires_at
and record.lease_expires_at > now
)
if lease_busy or record.next_allowed_at > now:
return record, False
lease_token = uuid4().hex
lease_expires_at = (
datetime.now(timezone.utc) + timedelta(seconds=max(1, lease_seconds))
).isoformat(timespec="seconds")
updated = execute_dml(
self._db,
update(SubscriptionSiteBudget)
.where(
SubscriptionSiteBudget.id == record.id,
or_(
SubscriptionSiteBudget.lease_token.is_(None),
SubscriptionSiteBudget.lease_expires_at.is_(None),
SubscriptionSiteBudget.lease_expires_at <= now,
),
SubscriptionSiteBudget.next_allowed_at <= now,
)
.values(
lease_owner=owner,
lease_token=lease_token,
lease_expires_at=lease_expires_at,
updated_at=now,
),
execution_options={"synchronize_session": False},
)
if not updated:
self._db.expire_all()
current = self._db.execute(
select(SubscriptionSiteBudget).where(
SubscriptionSiteBudget.site_id == site_id
)
).scalar_one()
return current, False
self._db.flush()
self._db.expire_all()
claimed = self._db.execute(
select(SubscriptionSiteBudget).where(
SubscriptionSiteBudget.site_id == site_id,
SubscriptionSiteBudget.lease_token == lease_token,
)
).scalar_one()
return claimed, True
def finish_site(
self,
*,
site_id: int,
lease_token: str,
outcome: str,
next_allowed_at: str,
error: Optional[str],
) -> bool:
"""释放当前站点租约,并按结果推进失败或恢复计数。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅站点预算收口需要调用方提供同步 Session")
record = self._db.execute(
select(SubscriptionSiteBudget).where(
SubscriptionSiteBudget.site_id == site_id,
SubscriptionSiteBudget.lease_token == lease_token,
)
).scalars().first()
if record is None:
return False
if outcome == "success":
failures = max(0, record.consecutive_failures - 1)
success_streak = record.success_streak + 1
elif outcome == "skipped":
failures = record.consecutive_failures
success_streak = record.success_streak
else:
failures = record.consecutive_failures + 1
success_streak = 0
now = utc_now_text()
return bool(execute_dml(
self._db,
update(SubscriptionSiteBudget)
.where(
SubscriptionSiteBudget.id == record.id,
SubscriptionSiteBudget.lease_token == lease_token,
)
.values(
lease_owner=None,
lease_token=None,
lease_expires_at=None,
next_allowed_at=next_allowed_at,
consecutive_failures=failures,
success_streak=success_streak,
last_outcome=outcome,
last_error=error,
updated_at=now,
),
execution_options={"synchronize_session": False},
))
def _ensure_site_budget(self, *, site_id: int, now: str) -> SubscriptionSiteBudget:
"""并发安全地创建站点预算初始记录。"""
record = self._db.execute(
select(SubscriptionSiteBudget).where(
SubscriptionSiteBudget.site_id == site_id
)
).scalars().first()
if record is not None:
return record
try:
with self._db.begin_nested():
self._db.add(SubscriptionSiteBudget(
site_id=site_id,
next_allowed_at=now,
updated_at=now,
))
self._db.flush()
except IntegrityError:
self._db.expire_all()
return self._db.execute(
select(SubscriptionSiteBudget).where(
SubscriptionSiteBudget.site_id == site_id
)
).scalar_one()
def _batch_cancel_requested(self, batch_id: str) -> bool:
"""在当前事务中读取批次取消标记。"""
return bool(self._db.execute(
+47 -2
View File
@@ -5,6 +5,7 @@ from typing import Any, Callable, List, Mapping, Optional, Tuple, Union, cast
from app.application.site.health import get_configured_site_health_service
from app.application.site.query import get_configured_site_query_service
from app.application.site.search_observation import report_site_search_outcome
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
from app.domain import site as site_rules
from app.domain.context import Context, SubtitleInfo, TorrentInfo
@@ -64,6 +65,24 @@ class _IndexerSearchOutcome:
error_flag: bool
result: List[dict[str, Any]]
seconds: int
error: Optional[Exception] = None
def _classify_search_failure(error: Optional[Exception]) -> str:
"""把索引器异常归一为站点预算可执行的冷却类别。"""
if error is None:
return "error"
status_code = getattr(error, "status_code", None)
text = f"{type(error).__name__}: {error}".lower()
if status_code == 429 or "429" in text or "rate limit" in text:
return "rate_limited"
if status_code == 403 or "403" in text or "forbidden" in text:
return "forbidden"
if any(token in text for token in ("未登录", "登录失效", "login", "cookie")):
return "login_invalid"
if isinstance(error, TimeoutError) or "timeout" in text or "timed out" in text or "超时" in text:
return "timeout"
return "error"
class IndexerModule(_ModuleBase):
@@ -359,12 +378,14 @@ class IndexerModule(_ModuleBase):
start_time: datetime,
error_flag: bool,
result: List[dict[str, Any]],
error: Optional[Exception] = None,
) -> _IndexerSearchOutcome:
"""把同步、异步 I/O 结果整理为共用的搜索完成状态"""
return _IndexerSearchOutcome(
error_flag=error_flag,
result=result,
seconds=(datetime.now() - start_time).seconds,
error=error,
)
@staticmethod
@@ -403,15 +424,18 @@ class IndexerModule(_ModuleBase):
page=page,
)
if not request:
report_site_search_outcome(attempted=False, outcome="skipped")
return []
# 开始搜索
error: Optional[Exception] = None
try:
error_flag, result = self.__execute_search(site, request)
except Exception as err:
error = err
self.__log_search_error(site, "torrents", err)
outcome = self.__create_search_outcome(start_time, error_flag, result)
outcome = self.__create_search_outcome(start_time, error_flag, result, error)
# 统计索引情况
self.__indexer_statistic(
@@ -419,6 +443,15 @@ class IndexerModule(_ModuleBase):
error_flag=outcome.error_flag,
seconds=outcome.seconds,
)
report_site_search_outcome(
attempted=True,
outcome=(
_classify_search_failure(outcome.error)
if outcome.error_flag or outcome.error is not None
else "success"
),
error=str(outcome.error) if outcome.error is not None else None,
)
# 返回结果
return self.__parse_result(
@@ -498,15 +531,18 @@ class IndexerModule(_ModuleBase):
page=page,
)
if not request:
report_site_search_outcome(attempted=False, outcome="skipped")
return []
# 开始搜索
error: Optional[Exception] = None
try:
error_flag, result = await self.__async_execute_search(site, request)
except Exception as err:
error = err
self.__log_search_error(site, "torrents", err)
outcome = self.__create_search_outcome(start_time, error_flag, result)
outcome = self.__create_search_outcome(start_time, error_flag, result, error)
# 统计索引情况
await self.__async_indexer_statistic(
@@ -514,6 +550,15 @@ class IndexerModule(_ModuleBase):
error_flag=outcome.error_flag,
seconds=outcome.seconds,
)
report_site_search_outcome(
attempted=True,
outcome=(
_classify_search_failure(outcome.error)
if outcome.error_flag or outcome.error is not None
else "success"
),
error=str(outcome.error) if outcome.error is not None else None,
)
# 返回结果
return self.__parse_result(
+95
View File
@@ -0,0 +1,95 @@
"""3.0.20 增加订阅搜索站点预算与批次启动抖动。
Revision ID: d2a7c5e9f1b4
Revises: c1f4a8d2e6b9
Create Date: 2026-09-01
"""
# Alembic 的 op 是运行期代理,静态分析无法看到实际操作方法。
# pylint: disable=no-member
import sqlalchemy as sa
from alembic import op
revision = "d2a7c5e9f1b4"
down_revision = "c1f4a8d2e6b9"
branch_labels = None
depends_on = None
_TASK_TABLE = "subscriptionsearchtask"
_BUDGET_TABLE = "subscriptionsitebudget"
def _table_names() -> set[str]:
"""返回当前数据库表名集合。"""
return set(sa.inspect(op.get_bind()).get_table_names())
def _column_names(table_name: str) -> set[str]:
"""返回指定表的当前字段名集合。"""
return {
column["name"]
for column in sa.inspect(op.get_bind()).get_columns(table_name)
}
def upgrade() -> None:
"""创建单站点预算,并让搜索任务支持持久启动抖动。"""
tables = _table_names()
if _TASK_TABLE in tables and "available_at" not in _column_names(_TASK_TABLE):
with op.batch_alter_table(_TASK_TABLE) as batch_op:
batch_op.add_column(sa.Column("available_at", sa.String(length=40), nullable=True))
batch_op.drop_index("ix_subscriptionsearchtask_claim")
batch_op.create_index(
"ix_subscriptionsearchtask_claim",
[
"state",
"priority",
"available_at",
"lease_expires_at",
"created_at",
"id",
],
)
op.execute(
sa.text(
"UPDATE subscriptionsearchtask "
"SET available_at = created_at WHERE available_at IS NULL"
)
)
if _BUDGET_TABLE not in tables:
op.create_table(
_BUDGET_TABLE,
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("site_id", sa.Integer(), nullable=False),
sa.Column("lease_owner", sa.String(length=128), nullable=True),
sa.Column("lease_token", sa.String(length=64), nullable=True),
sa.Column("lease_expires_at", sa.String(length=40), nullable=True),
sa.Column("next_allowed_at", sa.String(length=40), nullable=False),
sa.Column("consecutive_failures", sa.Integer(), nullable=False),
sa.Column("success_streak", sa.Integer(), nullable=False),
sa.Column("last_outcome", sa.String(length=32), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("updated_at", sa.String(length=40), nullable=False),
sa.UniqueConstraint("site_id", name="uq_subscriptionsitebudget_site_id"),
)
op.create_index(
"ix_subscriptionsitebudget_ready",
_BUDGET_TABLE,
["next_allowed_at", "lease_expires_at", "site_id"],
)
def downgrade() -> None:
"""移除站点预算与任务启动抖动。"""
tables = _table_names()
if _BUDGET_TABLE in tables:
op.drop_table(_BUDGET_TABLE)
if _TASK_TABLE in tables and "available_at" in _column_names(_TASK_TABLE):
with op.batch_alter_table(_TASK_TABLE) as batch_op:
batch_op.drop_index("ix_subscriptionsearchtask_claim")
batch_op.drop_column("available_at")
batch_op.create_index(
"ix_subscriptionsearchtask_claim",
["state", "priority", "lease_expires_at", "created_at", "id"],
)
+2 -2
View File
@@ -754,8 +754,8 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 925 |
| 内部导入边 | 7,728 |
| Python 模块 | 927 |
| 内部导入边 | 7,745 |
| 非平凡 SCC | 1(精确 containment 的 TMDB 移植包环) |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 53(债务已清零,53 条精确 containment |
+1 -1
View File
@@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
| 指标 | 当前值 | 解释 |
|---|---:|---|
| 宿主 Python 模块 / 内部依赖边 | 925 / 7,728 | `dependency-baseline.json` 当前快照 |
| 宿主 Python 模块 / 内部依赖边 | 927 / 7,745 | `dependency-baseline.json` 当前快照 |
| 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 |
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
@@ -1,7 +1,7 @@
# MoviePilot 订阅执行治理
> 状态:`active2026-09-01 已由 MoviePilot v3 接管)`
> 当前叶:`SUB-GOV-002B`
> 当前叶:`SUB-GOV-002C`
> 迁移来源:`V3-RDY-009A`、`V3-RDY-009A1`、`V3-RDY-009A1B`、
> `V3-RDY-009A1C`、`V3-RDY-009A1D`
> 适用范围:V3 订阅刷新、匹配、兜底搜索、下载提交与用户状态闭环
@@ -233,13 +233,13 @@ RSS/Spider 保持现有逐站点串行刷新,不通过并发化换取几秒收
| `SUB-GOV-001D` | 引入单轮新鲜事实租约,评估稳定元数据复用、轻量季集查询和单媒体服务器事实合并 | 001C | `completed2026-09-01` |
| `SUB-GOV-001E` | 若 001B–D 后仍有可重复等待热点,再通过隔离压测评估 2/4 worker 的有界准备;提交保持串行 | 001D | `not_activated2026-09-01` |
| `SUB-GOV-002A` | 将 24 小时兜底搜索移出 Match/提交长锁,建立批次、订阅 single-flight、可取消等待和恢复游标 | 001C | `completed2026-09-01` |
| `SUB-GOV-002B` | 为兜底搜索建立每站点并发、间隔和错误冷却预算;RSS/Spider 保持基线压力 | 002A | `in_progress` |
| `SUB-GOV-002C` | 联合运行日常 Match、手工搜索和兜底搜索,验证公平性、锁等待、站点压力和失败恢复 | 001D, 002B | `pending` |
| `SUB-GOV-002B` | 为兜底搜索建立每站点并发、间隔和错误冷却预算;RSS/Spider 保持基线压力 | 002A | `completed2026-09-01` |
| `SUB-GOV-002C` | 联合运行日常 Match、手工搜索和兜底搜索,验证公平性、锁等待、站点压力和失败恢复 | 001D, 002B | `in_progress` |
| `SUB-GOV-003A` | 建立跨入口的订阅级下载幂等、下载器不确定终态和取消补偿 | 001C, 002A | `pending` |
| `SUB-GOV-003B` | 后端提供订阅及批次业务状态,前端展示排队、匹配、搜索、提交、完成、失败和取消 | 002A, 003A | `pending` |
| `SUB-GOV-004` | 多条订阅记录指向同一媒体时的跨记录季集去重和产品规则 | 003A | `pending(最低优先级)` |
当前只激活 `SUB-GOV-002B`。001BD 是日常 Match 主线;002A–C 是 24 小时兜底搜索主线;003A/B
当前只激活 `SUB-GOV-002C`。001BD 是日常 Match 主线;002A–C 是 24 小时兜底搜索主线;003A/B
只有在前两条链路的身份和终态稳定后实施。每个叶子完成验收并更新本表后,才激活下一个满足依赖的叶子。
### 6.1 SUB-GOV-001A 验收证据
@@ -314,6 +314,22 @@ RSS/Spider 保持现有逐站点串行刷新,不通过并发化换取几秒收
- 验证:队列/编排/调度专项 `43 passed`,迁移与声明式模型 `35 passed`,复杂度/并发/架构/迁移组合
`119 passed`,错误级 Pylint 为 0Alembic 唯一 head 为 `c1f4a8d2e6b9`
### 6.7 SUB-GOV-002B 验收证据
- 新增持久 `SubscriptionSiteBudget`,以 `site_id` 唯一租约保证每站点最多一个在途兜底搜索;不同站点使用
独立记录,可由 Search provider 的现有 worker 并行推进;
- 每次真实站点页请求后写入 `next_allowed_at`:成功保持 `60300s` 随机间隔,历史失败按一级一级递减实现
渐进恢复;`429/403`、登录失效、超时和通用错误进入有上限的指数冷却;
- 自动 fallback 批次在任务 `available_at` 中持久化 `060s` 全局启动抖动;手工任务优先级更高且立即入队,
但进入 Search provider 后仍使用完全相同的站点租约、间隔和冷却,不能旁路;
- 站点预算只包装订阅队列的同步搜索实例,不改变 RSS/Spider 刷新入口和普通手工资源搜索;索引器列表返回
合同、同步/异步结果及既有站点健康统计语义保持不变;
- 等待每秒检查批次取消,超过 5 秒短窗口的冷却或在途站点不阻塞整批:该站点本轮返回空页,其他站点继续,
订阅在处理成功站点结果后以聚合失败收口,避免慢站点拖死后续订阅;
- 验证:站点预算/队列/编排/provider 专项 `38 passed`,搜索和索引器兼容 `91 passed`,迁移与声明式模型
`37 passed, 1 skipped`,架构合同组合 `149 passed`,错误级 Pylint 为 0Alembic 唯一 head 为
`d2a7c5e9f1b4`Host 架构基线与 `git diff --check` 通过。
## 7. 上线前验证与验收
### 7.1 场景
+22 -3
View File
@@ -1089,8 +1089,8 @@
"runtime_only": true
}
},
"edge_count": 7728,
"edge_sha256": "33cdb555e19d7a5d15dbed065c98908269c3528ec4d61c2ac5515420c0b4cfa4",
"edge_count": 7745,
"edge_sha256": "4c722884c3a228e2fc51d5389d4fec3e10436e797c4ae168b9306154a9179475",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -3442,6 +3442,9 @@
"app.application.subscription.delete -> app.schemas",
"app.application.subscription.delete -> app.schemas.common",
"app.application.subscription.delete -> app.schemas.event",
"app.application.subscription.execution -> app.application",
"app.application.subscription.execution -> app.application.subscription",
"app.application.subscription.execution -> app.application.subscription.sitebudget",
"app.application.subscription.facts -> app.application",
"app.application.subscription.facts -> app.application.subscription",
"app.application.subscription.facts -> app.application.subscription.contract",
@@ -3492,6 +3495,11 @@
"app.application.subscription.search -> app.application",
"app.application.subscription.search -> app.application.subscription",
"app.application.subscription.search -> app.application.subscription.contract",
"app.application.subscription.sitebudget -> app.application",
"app.application.subscription.sitebudget -> app.application.site",
"app.application.subscription.sitebudget -> app.application.site.search_observation",
"app.application.subscription.sitebudget -> app.runtime",
"app.application.subscription.sitebudget -> app.runtime.stop",
"app.application.subscription.write -> app.application",
"app.application.subscription.write -> app.application.outbox",
"app.application.subscription.write -> app.application.subscription",
@@ -4172,6 +4180,9 @@
"app.chain.search.cache -> app.schemas.types",
"app.chain.search.contract -> app.chain",
"app.chain.search.contract -> app.chain.base",
"app.chain.search.facade -> app.application",
"app.chain.search.facade -> app.application.subscription",
"app.chain.search.facade -> app.application.subscription.sitebudget",
"app.chain.search.facade -> app.chain",
"app.chain.search.facade -> app.chain.base",
"app.chain.search.facade -> app.chain.search",
@@ -4246,6 +4257,9 @@
"app.chain.search.provider -> app.application",
"app.chain.search.provider -> app.application.configuration",
"app.chain.search.provider -> app.application.site",
"app.chain.search.provider -> app.application.site.search_observation",
"app.chain.search.provider -> app.application.subscription",
"app.chain.search.provider -> app.application.subscription.sitebudget",
"app.chain.search.provider -> app.chain",
"app.chain.search.provider -> app.chain.search",
"app.chain.search.provider -> app.chain.search.contract",
@@ -4588,6 +4602,7 @@
"app.chain.subscribe.search -> app.application.subscription.contract",
"app.chain.subscribe.search -> app.application.subscription.execution",
"app.chain.subscribe.search -> app.application.subscription.query",
"app.chain.subscribe.search -> app.application.subscription.sitebudget",
"app.chain.subscribe.search -> app.chain",
"app.chain.subscribe.search -> app.chain.media",
"app.chain.subscribe.search -> app.chain.search",
@@ -5134,6 +5149,7 @@
"app.db.adapters.subscriptionsearch -> app.application",
"app.db.adapters.subscriptionsearch -> app.application.subscription",
"app.db.adapters.subscriptionsearch -> app.application.subscription.execution",
"app.db.adapters.subscriptionsearch -> app.application.subscription.sitebudget",
"app.db.adapters.subscriptionsearch -> app.db",
"app.db.adapters.subscriptionsearch -> app.db.models",
"app.db.adapters.subscriptionsearch -> app.db.models.subscriptionsearch",
@@ -6133,6 +6149,7 @@
"app.modules.indexer -> app.application.site",
"app.modules.indexer -> app.application.site.health",
"app.modules.indexer -> app.application.site.query",
"app.modules.indexer -> app.application.site.search_observation",
"app.modules.indexer -> app.domain",
"app.modules.indexer -> app.domain.context",
"app.modules.indexer -> app.domain.site",
@@ -8821,7 +8838,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 925,
"module_count": 927,
"modules": [
"app",
"app.adapters",
@@ -9106,6 +9123,7 @@
"app.application.site.health",
"app.application.site.mutation",
"app.application.site.query",
"app.application.site.search_observation",
"app.application.storage",
"app.application.subscription",
"app.application.subscription.candidates",
@@ -9119,6 +9137,7 @@
"app.application.subscription.priority",
"app.application.subscription.query",
"app.application.subscription.search",
"app.application.subscription.sitebudget",
"app.application.subscription.write",
"app.application.system",
"app.application.torrent",
@@ -73,6 +73,7 @@ def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch
"""R/P 兜底搜索在持久队列中执行,不受日常 Match 长锁阻塞。"""
subscribes = [_subscribe(1), _subscribe(2)]
chain = _chain(tmp_path, subscribes)
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
processed = []
monkeypatch.setattr(
chain,
@@ -94,6 +95,7 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
"""单订阅异常不得中止批次后续任务,聚合终态必须暴露失败。"""
subscribes = [_subscribe(3), _subscribe(4)]
chain = _chain(tmp_path, subscribes)
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
processed = []
def process(subscribe, _searchchain):
+266
View File
@@ -0,0 +1,266 @@
"""订阅兜底搜索的站点并发、冷却、恢复与取消测试。"""
from datetime import datetime, timedelta, timezone
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from app.application.site.search_observation import (
SiteSearchObservation,
report_site_search_outcome,
)
from app.application.subscription.sitebudget import (
SiteBudgetClaim,
SubscriptionSearchCancelled,
SubscriptionSiteBudget,
)
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
from app.db.base import Base
from app.db.models.subscriptionsearch import SubscriptionSiteBudget as SiteBudgetRecord
from app.chain.search.facade import SearchChain
from app.modules.indexer import _classify_search_failure
from app.runtime.stop import ProcessStopState
def _repository(tmp_path):
"""构造使用独立 SQLite 文件的站点预算仓储。"""
engine = create_engine(f"sqlite:///{tmp_path / 'site-budget.db'}")
Base.metadata.create_all(engine)
return TransactionalSubscriptionSearchRepository(sessionmaker(bind=engine)), engine
def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path):
"""同站点第二个调用必须等待,不同站点可立即并行。"""
repository, _engine = _repository(tmp_path)
first = repository.claim_site(site_id=1, owner="worker-a", lease_seconds=900)
same_site = repository.claim_site(site_id=1, owner="worker-b", lease_seconds=900)
other_site = repository.claim_site(site_id=2, owner="worker-b", lease_seconds=900)
assert first.acquired is True
assert same_site.acquired is False
assert same_site.lease_token is None
assert other_site.acquired is True
def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_path):
"""失败增加冷却计数,后续成功每次只恢复一级而非直接清零。"""
repository, engine = _repository(tmp_path)
claim = repository.claim_site(site_id=3, owner="worker-a", lease_seconds=900)
now = datetime.now(timezone.utc)
assert repository.finish_site(
site_id=3,
lease_token=claim.lease_token,
outcome="timeout",
next_allowed_at=(now + timedelta(minutes=5)).isoformat(timespec="seconds"),
error="request timeout",
) is True
with Session(engine) as session:
failed = session.execute(
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3)
).scalar_one()
assert failed.consecutive_failures == 1
assert failed.success_streak == 0
failed.next_allowed_at = (now - timedelta(seconds=1)).isoformat(timespec="seconds")
session.commit()
recovered = repository.claim_site(site_id=3, owner="worker-b", lease_seconds=900)
assert recovered.acquired is True
assert recovered.consecutive_failures == 1
assert repository.finish_site(
site_id=3,
lease_token=recovered.lease_token,
outcome="success",
next_allowed_at=(now + timedelta(minutes=1)).isoformat(timespec="seconds"),
) is True
with Session(engine) as session:
healthy = session.execute(
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3)
).scalar_one()
assert healthy.consecutive_failures == 0
assert healthy.success_streak == 1
assert healthy.last_outcome == "success"
@pytest.mark.parametrize(
("error", "outcome"),
[
(RuntimeError("HTTP 429"), "rate_limited"),
(RuntimeError("403 forbidden"), "forbidden"),
(RuntimeError("Cookie 登录失效"), "login_invalid"),
(TimeoutError("request timed out"), "timeout"),
],
)
def test_indexer_failures_map_to_site_cooldown_categories(error, outcome):
"""外站典型失败必须进入对应的站点级冷却类别。"""
assert _classify_search_failure(error) == outcome
class _WaitingRepository:
"""始终返回未来重试时间的可取消预算仓储。"""
def claim_site(self, *, site_id: int, owner: str, lease_seconds: int) -> SiteBudgetClaim:
"""返回未认领的未来预算。"""
del owner, lease_seconds
return SiteBudgetClaim(
site_id=site_id,
acquired=False,
retry_at=(datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(timespec="seconds"),
consecutive_failures=0,
)
def finish_site(self, **_kwargs) -> bool:
"""等待测试不会取得租约,因此不应调用收口。"""
raise AssertionError("未取得站点预算时不应收口")
def test_site_budget_wait_is_cancellable_without_business_lock():
"""批次取消应在一次短等待后终止预算获取。"""
cancelled = False
def sleeper(_seconds: float) -> None:
"""模拟等待一次后收到取消请求。"""
nonlocal cancelled
cancelled = True
budget = SubscriptionSiteBudget(
repository=_WaitingRepository(),
owner="worker-a",
cancelled=lambda: cancelled,
stop_state=ProcessStopState(),
sleeper=sleeper,
max_wait_seconds=600,
)
with pytest.raises(SubscriptionSearchCancelled):
budget.acquire(9)
def test_site_budget_delay_keeps_manual_requests_under_same_budget():
"""来源优先级不参与站点冷却计算,手工任务不能获得旁路。"""
budget = SubscriptionSiteBudget(
repository=_WaitingRepository(),
owner="manual-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
random_uniform=lambda _low, _high: 60.0,
)
assert budget._next_delay("success", consecutive_failures=0) == 60.0 # pylint: disable=protected-access
assert budget._next_delay("success", consecutive_failures=2) == 120.0 # pylint: disable=protected-access
assert budget._next_delay("rate_limited", consecutive_failures=0) == 900.0 # pylint: disable=protected-access
def test_skipped_search_releases_budget_without_external_interval():
"""本地限流等未发请求结果不应伪造成功或追加外站间隔。"""
captured = {}
class _Repository(_WaitingRepository):
"""记录预算收口参数。"""
def finish_site(self, **kwargs) -> bool:
"""保存收口事实供断言。"""
captured.update(kwargs)
return True
now = datetime(2026, 9, 1, tzinfo=timezone.utc)
budget = SubscriptionSiteBudget(
repository=_Repository(),
owner="worker-a",
cancelled=lambda: False,
stop_state=ProcessStopState(),
clock=lambda: now,
)
claim = SiteBudgetClaim(
site_id=10,
acquired=True,
retry_at=now.isoformat(timespec="seconds"),
consecutive_failures=0,
lease_token="lease-token",
)
assert budget.finish(claim, SiteSearchObservation()) is True
assert captured["outcome"] == "skipped"
assert captured["next_allowed_at"] == now.isoformat(timespec="seconds")
def test_search_provider_reports_cooled_site_without_blocking_other_results():
"""超出短等待窗口的站点返回空页并记录聚合失败,而非阻塞整个 provider。"""
repository = _WaitingRepository()
budget = SubscriptionSiteBudget(
repository=repository,
owner="fallback-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
max_wait_seconds=0,
)
chain = object.__new__(SearchChain)
chain.configure_subscription_site_budget(budget)
chain.search_site_torrents = lambda **_kwargs: ["unexpected"]
result = chain._search_site_torrents_with_budget( # pylint: disable=protected-access
site={"id": 11, "name": "Cooling"},
keyword="movie",
mtype=None,
page=0,
)
assert result == []
failures = chain.consume_subscription_site_budget_failures()
assert len(failures) == 1
assert "站点 11" in failures[0]
def test_search_provider_releases_successful_site_budget():
"""真实站点页完成后必须释放租约并写入成功间隔。"""
captured = {}
class _Repository(_WaitingRepository):
"""提供立即可用租约并记录收口参数。"""
def claim_site(self, *, site_id: int, owner: str, lease_seconds: int) -> SiteBudgetClaim:
"""返回当前调用独占的站点租约。"""
del owner, lease_seconds
return SiteBudgetClaim(
site_id=site_id,
acquired=True,
retry_at=datetime.now(timezone.utc).isoformat(timespec="seconds"),
consecutive_failures=0,
lease_token="lease-token",
)
def finish_site(self, **kwargs) -> bool:
"""记录成功收口事实。"""
captured.update(kwargs)
return True
budget = SubscriptionSiteBudget(
repository=_Repository(),
owner="manual-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
random_uniform=lambda _low, _high: 60.0,
)
chain = object.__new__(SearchChain)
chain.configure_subscription_site_budget(budget)
def search_site_torrents(**_kwargs):
"""模拟索引器成功并发布观察结果。"""
report_site_search_outcome(attempted=True, outcome="success")
return ["torrent"]
chain.search_site_torrents = search_site_torrents
result = chain._search_site_torrents_with_budget( # pylint: disable=protected-access
site={"id": 12, "name": "Healthy"},
keyword="movie",
mtype=None,
page=0,
)
assert result == ["torrent"]
assert captured["site_id"] == 12
assert captured["outcome"] == "success"
assert captured["lease_token"] == "lease-token"