From 68880dbbf19db13c7185ce8e98102c1d803e8c40 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Thu, 3 Sep 2026 11:26:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=AE=A2=E9=98=85=E6=89=A7?= =?UTF-8?q?=E8=A1=8C=E6=B2=BB=E7=90=86=E4=B8=8E=E8=A7=84=E6=A8=A1=E6=80=A7?= =?UTF-8?q?=E8=83=BD=20(#6557)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(subscription): govern concurrent search and match execution * refactor(subscription): consume complete candidate snapshots * refactor(subscription): remove download submission ledger * feat(subscription): stagger fallback searches without site gaps * refactor(subscription): remove unused retry capability * test(subscription): verify complete scale execution * fix(subscription): surface site search failures * test(subscription): verify site pressure through search * test(subscription): use static facade imports * fix(subscription): preserve completed downloads on shutdown * fix(subscription): classify execution expiry as failure * fix(subscription): preserve fallback search staggering * test(subscription): stabilize unfinished wrapper gate * fix(subscription): settle expired search returns * style(subscription): normalize governance imports * fix(subscription): align governance type contracts * docs(architecture): sync mypy debt metric * docs(architecture): sync startup module counts --- app/agent/policy/mcp.py | 2 - .../policy/resources/api_mcp_schema.json | 12 - app/application/chain/context.py | 2 - app/application/download/admission.py | 122 +- app/application/subscription/candidates.py | 56 +- app/application/subscription/execution.py | 103 +- app/application/subscription/sitebudget.py | 64 +- app/application/subscription/status.py | 81 +- app/chain/_contracts.py | 8 + app/chain/_music.py | 75 +- app/chain/base.py | 1 - app/chain/download/admission.py | 258 ----- app/chain/download/batch.py | 4 +- app/chain/download/contract.py | 7 - app/chain/download/facade.py | 2 - app/chain/download/submission.py | 172 +-- app/chain/search/facade.py | 2 +- app/chain/search/provider.py | 9 +- app/chain/subscribe/contract.py | 7 +- app/chain/subscribe/facade.py | 6 +- app/chain/subscribe/match.py | 1014 ++++++++++------- app/chain/subscribe/policy.py | 26 +- app/chain/subscribe/refresh.py | 14 +- app/chain/subscribe/search.py | 668 +++++++---- app/chain/torrents.py | 63 +- app/db/adapters/subscriptiondownload.py | 147 --- app/db/adapters/subscriptionsearch.py | 9 +- app/db/adapters/subscriptionstatus.py | 41 +- app/db/models/__init__.py | 4 - app/db/models/subscriptiondownload.py | 53 - app/db/models/subscriptionsearch.py | 3 +- app/db/oper/subscriptiondownload.py | 242 ---- app/db/oper/subscriptionsearch.py | 51 +- app/schemas/subscribe.py | 3 +- app/startup/composition/chain.py | 2 - database/versions/b6c1d9e4a7f2_3_0_24.py | 37 + database/versions/c8f2e6a1d4b9_3_0_25.py | 73 ++ docs/architecture-overview.md | 4 +- docs/architecture/optimization-checklist.md | 4 +- ...iepilot-subscription-governance-handoff.md | 4 + ...iepilot-subscription-governance-roadmap.md | 4 + scripts/generate_agent_skill_docs.py | 5 - .../subscription_governance_scale.py | 968 +++++++++++++--- skills/database-operation/SKILL.md | 8 +- skills/moviepilot-api/SKILL.md | 16 +- tests/conftest.py | 2 - .../architecture/dependency-baseline.json | 57 +- .../fixtures/architecture/mypy-baseline.json | 2 +- .../startup-performance-baseline.json | 6 +- tests/test_download_architecture.py | 1 - tests/test_music_subscribe.py | 104 +- tests/test_subscribe_chain.py | 29 +- tests/test_subscribe_search_state.py | 53 +- tests/test_subscription_candidate_batch.py | 28 +- ...st_subscription_cross_channel_admission.py | 129 +++ .../test_subscription_download_governance.py | 540 +++------ .../test_subscription_execution_admission.py | 138 +++ tests/test_subscription_execution_status.py | 73 +- tests/test_subscription_governance_joint.py | 2 - tests/test_subscription_governance_replay.py | 7 +- tests/test_subscription_governance_scale.py | 288 ++++- tests/test_subscription_match_admission.py | 231 ++++ tests/test_subscription_search_governance.py | 483 +++++++- tests/test_subscription_search_queue.py | 68 ++ ...t_subscription_search_skipped_migration.py | 61 + tests/test_subscription_site_budget.py | 153 ++- 66 files changed, 4262 insertions(+), 2649 deletions(-) delete mode 100644 app/chain/download/admission.py delete mode 100644 app/db/adapters/subscriptiondownload.py delete mode 100644 app/db/models/subscriptiondownload.py delete mode 100644 app/db/oper/subscriptiondownload.py create mode 100644 database/versions/b6c1d9e4a7f2_3_0_24.py create mode 100644 database/versions/c8f2e6a1d4b9_3_0_25.py create mode 100644 tests/test_subscription_cross_channel_admission.py create mode 100644 tests/test_subscription_execution_admission.py create mode 100644 tests/test_subscription_match_admission.py create mode 100644 tests/test_subscription_search_skipped_migration.py diff --git a/app/agent/policy/mcp.py b/app/agent/policy/mcp.py index 3e01aa47e..325ca95d5 100644 --- a/app/agent/policy/mcp.py +++ b/app/agent/policy/mcp.py @@ -467,7 +467,6 @@ FIELD_DESCRIPTIONS.update( "cat": "Exact site category identifier returned by site.category.", "check_only": "Validate or preview the recommendation without applying search-result filtering.", "can_cancel": "Whether the current subscription execution can be cancelled.", - "can_retry": "Whether the current subscription execution can be retried.", "concurrency_key": "Workflow expression used to serialize actions sharing the same runtime key.", "condition": "Workflow branch or flow condition expression evaluated at runtime.", "context": "Persisted workflow execution context available to later actions.", @@ -517,7 +516,6 @@ FIELD_DESCRIPTIONS.update( "rating": "Numeric plugin rating accepted by the endpoint's declared bounds.", "reason": "Human-readable justification recorded with a manual-review decision.", "recursive": "Apply media-aware renaming recursively to child files when true.", - "requires_reconciliation": "Whether the execution requires reconciliation before it can continue.", "release_year": "Release year matched by an automatic category rule.", "result": "Persisted workflow action result value.", "result_payload": "Structured external-operation result recorded with manual review.", diff --git a/app/agent/policy/resources/api_mcp_schema.json b/app/agent/policy/resources/api_mcp_schema.json index f3257819d..df43f26fe 100644 --- a/app/agent/policy/resources/api_mcp_schema.json +++ b/app/agent/policy/resources/api_mcp_schema.json @@ -3532,12 +3532,6 @@ "title": "Can Cancel", "type": "boolean" }, - "can_retry": { - "default": false, - "description": "Whether the current subscription execution can be retried.", - "title": "Can Retry", - "type": "boolean" - }, "current_site_id": { "anyOf": [ { @@ -3567,12 +3561,6 @@ "title": "Phase", "type": "string" }, - "requires_reconciliation": { - "default": false, - "description": "Whether the execution requires reconciliation before it can continue.", - "title": "Requires Reconciliation", - "type": "boolean" - }, "source": { "anyOf": [ { diff --git a/app/application/chain/context.py b/app/application/chain/context.py index 5d8e40935..f0869f0f2 100644 --- a/app/application/chain/context.py +++ b/app/application/chain/context.py @@ -12,7 +12,6 @@ from app.application.configuration import ChainRuntimeConfig from app.runtime.stop import StopState, runtime_stop_state if TYPE_CHECKING: - from app.application.download.admission import SubscriptionDownloadRepository from app.application.download.failures import DownloadFailureRepository from app.application.history import ( DownloadHistoryRepository, @@ -77,7 +76,6 @@ class ChainRuntimeContext: download_failure_repository: DownloadFailureRepository user_repository: ChainUserRepository subscription_search_repository: Optional[SubscriptionSearchRepository] = None - subscription_download_repository: Optional[SubscriptionDownloadRepository] = None legacy_transfer_command: Optional[LegacyTransferCommand] = None durable_event_writer: Optional[ChainDurableEventWriter] = None configuration: ChainRuntimeConfig = field( diff --git a/app/application/download/admission.py b/app/application/download/admission.py index 13174e144..ed59c6081 100644 --- a/app/application/download/admission.py +++ b/app/application/download/admission.py @@ -1,130 +1,14 @@ -"""订阅下载提交的持久幂等与不确定终态合同。""" +"""订阅下载提交前的可取消执行边界。""" from __future__ import annotations from dataclasses import dataclass -from typing import Callable, Optional, Protocol - - -class DownloadReconciliationRequired(RuntimeError): - """表示下载器可能已接受任务，必须先对账才能继续自动提交。""" +from typing import Callable, Optional @dataclass(frozen=True, slots=True) class SubscriptionDownloadGovernance: - """把订阅身份、入口任务与取消检查传入下载提交边界。""" + """把取消检查与副作用起点回传给订阅执行上下文。""" - subscription_id: int - mode: str - task_id: Optional[str] = None cancelled: Optional[Callable[[], bool]] = None mark_started: Optional[Callable[[], None]] = None - - -@dataclass(frozen=True, slots=True) -class SubscriptionDownloadRequest: - """一次订阅下载提交认领所需的规范身份。""" - - idempotency_key: str - legacy_idempotency_key: Optional[str] - subscription_id: int - task_id: Optional[str] - logical_identity: str - resource_key: str - coverage: str - mode: str - delivery_scope: str - - -@dataclass(frozen=True, slots=True) -class SubscriptionDownloadSnapshot: - """脱离 Session 的订阅下载提交状态快照。""" - - idempotency_key: str - subscription_id: int - task_id: Optional[str] - state: str - attempt_count: int - attempt_token: Optional[str] - downloader: Optional[str] - download_hash: Optional[str] - available_at: Optional[str] - last_error: Optional[str] - created_at: str - updated_at: str - - -@dataclass(frozen=True, slots=True) -class SubscriptionDownloadClaim: - """返回本次是否取得唯一提交权以及当前持久状态。""" - - acquired: bool - snapshot: SubscriptionDownloadSnapshot - - -class SubscriptionDownloadRepository(Protocol): - """订阅下载幂等账本所需的最小持久化端口。""" - - def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim: - """按唯一键认领提交；仅到期 retryable/cancelled 状态允许重新认领。""" - ... - - def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]: - """按幂等键读取现有提交，供键版本兼容检查。""" - ... - - def mark_accepted( - self, - *, - idempotency_key: str, - attempt_token: str, - downloader: Optional[str], - download_hash: str, - ) -> bool: - """记录下载器已明确接受任务，后续任何失败都不得自动重试。""" - ... - - def mark_succeeded( - self, - *, - idempotency_key: str, - attempt_token: str, - ) -> bool: - """在 canonical 本地结算完成后写入成功终态。""" - ... - - def mark_retryable( - self, - *, - idempotency_key: str, - attempt_token: str, - available_at: str, - error: Optional[str], - ) -> bool: - """记录下载器明确拒绝、尚未产生外部副作用的可重试状态。""" - ... - - def mark_reconcile_required( - self, - *, - idempotency_key: str, - attempt_token: str, - error: Optional[str], - downloader: Optional[str] = None, - download_hash: Optional[str] = None, - ) -> bool: - """冻结可能已产生外部副作用的提交，等待下载器对账。""" - ... - - def mark_cancelled( - self, - *, - idempotency_key: str, - attempt_token: str, - ) -> bool: - """仅在进入下载器副作用前收口取消。""" - ... - - def has_started_for_task(self, task_id: str) -> bool: - """判断搜索任务是否已有不能按未执行处理的下载提交。""" - ... diff --git a/app/application/subscription/candidates.py b/app/application/subscription/candidates.py index 5cb1b9c24..f2868d10a 100644 --- a/app/application/subscription/candidates.py +++ b/app/application/subscription/candidates.py @@ -1,10 +1,7 @@ -"""订阅候选批次与无损路由合同。""" +"""订阅候选的无损路由合同。""" import copy -from dataclasses import dataclass, field -from datetime import datetime, timezone from typing import Any, Dict, List, Optional, cast -from uuid import uuid4 from app.application.subscription.contract import SubscriptionSnapshot from app.domain.context import Context, MediaInfo @@ -15,57 +12,6 @@ from app.schemas.types import MediaSource, MediaType CandidateGroups = Dict[str, List[Context]] -@dataclass(slots=True) -class CandidateBatch: - """一次资源获取产生的完整候选、增量候选与重试候选边界。""" - - batch_id: str - source: str - candidates: CandidateGroups - fresh_candidates: CandidateGroups = field(default_factory=dict) - retry_candidates: CandidateGroups = field(default_factory=dict) - sites: tuple[str, ...] = () - started_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) - finished_at: Optional[datetime] = None - - @classmethod - def create( - cls, - *, - source: str, - candidates: CandidateGroups, - fresh_candidates: Optional[CandidateGroups] = None, - retry_candidates: Optional[CandidateGroups] = None, - sites: Optional[List[str]] = None, - started_at: Optional[datetime] = None, - ) -> "CandidateBatch": - """构造已完成获取的候选批次。""" - return cls( - batch_id=uuid4().hex, - source=source, - candidates=candidates, - fresh_candidates=fresh_candidates or {}, - retry_candidates=retry_candidates or {}, - sites=tuple(sites or candidates.keys()), - started_at=started_at or datetime.now(timezone.utc), - finished_at=datetime.now(timezone.utc), - ) - - @classmethod - def from_legacy(cls, candidates: CandidateGroups, source: str = "legacy") -> "CandidateBatch": - """把旧入口传入的完整候选包装为无增量声明的兼容批次。""" - return cls.create(source=source, candidates=candidates) - - @staticmethod - def count(groups: CandidateGroups) -> int: - """统计分站点候选总数。""" - return sum(len(contexts) for contexts in groups.values()) - - def build_index(self) -> "CandidateIndex": - """基于完整候选构建一次性无损索引。""" - return CandidateIndex(self.candidates) - - class CandidateIndex: """一次构建并保持原顺序的订阅候选身份索引。""" diff --git a/app/application/subscription/execution.py b/app/application/subscription/execution.py index 0454b743b..97563fe0e 100644 --- a/app/application/subscription/execution.py +++ b/app/application/subscription/execution.py @@ -1,11 +1,101 @@ -"""订阅搜索执行批次、任务与持久队列端口。""" +"""订阅执行准入、搜索上下文、批次任务与持久队列端口。""" +import threading +import time from dataclasses import dataclass -from typing import Optional, Protocol +from typing import Callable, Mapping, Optional, Protocol +from uuid import uuid4 from app.application.subscription.sitebudget import SiteBudgetClaim +@dataclass(frozen=True, slots=True) +class SubscriptionExecutionLease: + """一次订阅执行的进程内所有权及协作截止时间。""" + + subscription_id: int + operation: str + owner_token: str + expires_at: float + + +class SubscriptionExecutionAdmission: + """按订阅 ID 控制 Search 与 Match 的进程内互斥。""" + + def __init__(self, clock: Callable[[], float] = time.monotonic) -> None: + """保存单调时钟和当前活跃 owner。""" + self._clock = clock + self._lock = threading.Lock() + self._owners: dict[int, SubscriptionExecutionLease] = {} + + def try_acquire( + self, + *, + subscription_id: int, + operation: str, + ttl_seconds: float, + ) -> Optional[SubscriptionExecutionLease]: + """无等待取得订阅所有权；已有 owner 时直接返回空。""" + with self._lock: + if subscription_id in self._owners: + return None + lease = SubscriptionExecutionLease( + subscription_id=subscription_id, + operation=operation, + owner_token=uuid4().hex, + expires_at=self._clock() + max(1.0, ttl_seconds), + ) + self._owners[subscription_id] = lease + return lease + + def release(self, lease: SubscriptionExecutionLease) -> bool: + """仅允许当前 owner 释放订阅所有权。""" + with self._lock: + current = self._owners.get(lease.subscription_id) + if current is None or current.owner_token != lease.owner_token: + return False + self._owners.pop(lease.subscription_id) + return True + + def is_expired(self, lease: SubscriptionExecutionLease) -> bool: + """判断 owner 是否已超过协作执行截止时间。""" + return self._clock() >= lease.expires_at + + +@dataclass(slots=True) +class SubscriptionExecutionContext: + """一次订阅执行的显式取消、阶段和副作用边界。""" + + lease: SubscriptionExecutionLease + admission: SubscriptionExecutionAdmission + task_id: Optional[str] = None + cancel_requested: Optional[Callable[[], bool]] = None + phase_changed: Optional[Callable[[str, Optional[int]], None]] = None + download_started: bool = False + + def is_cancel_requested(self) -> bool: + """判断调用入口是否请求在下一个安全边界退出。""" + return bool(self.cancel_requested and self.cancel_requested()) + + def is_expired(self) -> bool: + """判断本次执行是否已经超过协作截止时间。""" + return self.admission.is_expired(self.lease) + + def should_stop(self) -> bool: + """在安全边界合并用户取消和执行 TTL。""" + return self.is_expired() or self.is_cancel_requested() + + def report_phase(self, phase: str, current_site_id: Optional[int] = None) -> None: + """向当前搜索任务报告业务阶段。""" + if self.phase_changed: + self.phase_changed(phase, current_site_id) + + def mark_download_started(self) -> None: + """标记执行已越过下载器副作用边界。""" + self.download_started = True + self.report_phase("submitting") + + @dataclass(frozen=True, slots=True) class SearchBatchSnapshot: """订阅搜索批次的持久业务状态快照。""" @@ -24,6 +114,7 @@ class SearchBatchSnapshot: started_at: Optional[str] = None finished_at: Optional[str] = None last_error: Optional[str] = None + skipped_count: int = 0 @dataclass(frozen=True, slots=True) @@ -68,9 +159,9 @@ class SubscriptionSearchRepository(Protocol): subscription_ids: tuple[int, ...], source: str, priority: int, - available_at: Optional[str] = None, + available_at_by_subscription: Optional[Mapping[int, str]] = None, ) -> SearchEnqueueResult: - """按订阅 ID 建立批次，在启动抖动后合并活动任务。""" + """按订阅 ID 和各自到期时间建立或合并活动任务。""" ... def claim_next(self, *, owner: str, lease_seconds: int = 900) -> Optional[SearchTaskSnapshot]: @@ -85,7 +176,7 @@ class SubscriptionSearchRepository(Protocol): state: str, error: Optional[str] = None, ) -> bool: - """以租约令牌收口任务，并推进所属批次聚合状态。""" + """以租约令牌收口任务，并推进所属批次聚合状态（含 skipped）。""" ... def update_task_phase( @@ -140,5 +231,5 @@ class SubscriptionSearchRepository(Protocol): next_allowed_at: str, error: Optional[str] = None, ) -> bool: - """释放站点预算并写入间隔、冷却和恢复状态。""" + """释放站点预算并写入错误冷却和恢复状态。""" ... diff --git a/app/application/subscription/sitebudget.py b/app/application/subscription/sitebudget.py index f3d15d24a..97f7948d4 100644 --- a/app/application/subscription/sitebudget.py +++ b/app/application/subscription/sitebudget.py @@ -2,8 +2,6 @@ from __future__ import annotations -import random -import time from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Callable, Optional, Protocol @@ -17,7 +15,7 @@ class SubscriptionSearchCancelled(RuntimeError): class SubscriptionSiteBudgetUnavailable(RuntimeError): - """表示站点预算超出本任务允许的短等待窗口。""" + """表示站点仍处于错误冷却或已有未释放租约。""" def __init__(self, *, site_id: int, retry_at: str) -> None: """保存站点和下一次可尝试时间，供批次聚合失败展示。""" @@ -69,7 +67,7 @@ def _utc_now() -> datetime: class SubscriptionSiteBudget: - """在同步搜索 worker 内执行可取消的站点预算等待与反馈。""" + """以非阻塞认领保护站点租约，并持久化外站错误冷却。""" def __init__( self, @@ -78,49 +76,35 @@ class SubscriptionSiteBudget: owner: str, cancelled: Callable[[], bool], stop_state: StopState, - interval_range: tuple[float, float] = (60.0, 300.0), lease_seconds: int = 900, - max_wait_seconds: float = 5.0, - random_uniform: Callable[[float, float], float] = random.uniform, - sleeper: Callable[[float], None] = time.sleep, clock: Callable[[], datetime] = _utc_now, phase_changed: Optional[Callable[[str, Optional[int]], None]] = None, ) -> None: - """保存持久化端口及可注入的时钟、随机数和等待实现。""" + """保存持久化端口及可注入的时钟和阶段回调。""" self._repository = repository self._owner = owner self._cancelled = cancelled self._stop_state = stop_state - self._interval_range = interval_range self._lease_seconds = max(1, lease_seconds) - self._max_wait_seconds = max(0.0, max_wait_seconds) - self._random_uniform = random_uniform - self._sleeper = sleeper self._clock = clock self._phase_changed = phase_changed def acquire(self, site_id: int) -> SiteBudgetClaim: - """循环认领指定站点，并在每秒边界检查取消与停机。""" - deadline = time.monotonic() + self._max_wait_seconds - while True: - self._raise_if_cancelled() - claim = self._repository.claim_site( - site_id=site_id, - owner=self._owner, - lease_seconds=self._lease_seconds, - ) - if claim.acquired: - self._report_phase("searching", site_id) - return claim - self._report_phase("waiting_site_budget", site_id) - retry_at = datetime.fromisoformat(claim.retry_at) - remaining = max(0.0, (retry_at - self._clock()).total_seconds()) - if remaining > max(0.0, deadline - time.monotonic()): - raise SubscriptionSiteBudgetUnavailable( - site_id=site_id, - retry_at=claim.retry_at, - ) - self._sleeper(min(max(remaining, 0.05), 1.0)) + """只认领一次指定站点，未就绪时留待下一次正常调度。""" + self._raise_if_cancelled() + claim = self._repository.claim_site( + site_id=site_id, + owner=self._owner, + lease_seconds=self._lease_seconds, + ) + if claim.acquired: + self._report_phase("searching", site_id) + return claim + self._report_phase("waiting_site_budget", site_id) + raise SubscriptionSiteBudgetUnavailable( + site_id=site_id, + retry_at=claim.retry_at, + ) def _report_phase(self, phase: str, site_id: Optional[int]) -> None: """向任务所有者报告不改变预算语义的业务阶段。""" @@ -128,7 +112,7 @@ class SubscriptionSiteBudget: self._phase_changed(phase, site_id) def finish(self, claim: SiteBudgetClaim, observation: SiteSearchObservation) -> bool: - """依据调用结果计算随机间隔或错误冷却并释放租约。""" + """正常结果立即释放站点，错误结果按类别写入冷却。""" if not claim.lease_token: return False outcome = observation.outcome if observation.attempted else "skipped" @@ -143,13 +127,9 @@ class SubscriptionSiteBudget: ) def _next_delay(self, outcome: str, consecutive_failures: int) -> float: - """为成功渐进恢复、错误退避和本地跳过计算下一次等待。""" - if outcome == "skipped": + """正常或本地跳过立即恢复，错误按连续失败次数退避。""" + if outcome in {"success", "skipped"}: return 0.0 - if outcome == "success": - low, high = self._interval_range - recovery_factor = 1.0 + min(max(consecutive_failures, 0), 3) * 0.5 - return self._random_uniform(low, high) * recovery_factor exponent = min(max(consecutive_failures, 0), 5) base, ceiling = { "rate_limited": (900.0, 21600.0), @@ -160,6 +140,6 @@ class SubscriptionSiteBudget: return float(min(base * (2**exponent), ceiling)) def _raise_if_cancelled(self) -> None: - """在不持有业务锁的等待边界传播取消或停机。""" + """在创建站点租约前传播取消或停机。""" if self._stop_state.is_system_stopped or self._cancelled(): raise SubscriptionSearchCancelled("订阅搜索已取消") diff --git a/app/application/subscription/status.py b/app/application/subscription/status.py index 7d202f824..c0e848d94 100644 --- a/app/application/subscription/status.py +++ b/app/application/subscription/status.py @@ -4,13 +4,12 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from typing import Optional, Protocol -from app.application.download.admission import SubscriptionDownloadSnapshot from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot @dataclass(frozen=True, slots=True) class SubscriptionExecutionStatus: - """一个订阅跨搜索与下载账本合并后的用户可见状态。""" + """一个订阅最近搜索任务的用户可见状态。""" state: str phase: str @@ -21,8 +20,6 @@ class SubscriptionExecutionStatus: current_site_id: Optional[int] = None error: Optional[str] = None can_cancel: bool = False - can_retry: bool = False - requires_reconciliation: bool = False @dataclass(frozen=True, slots=True) @@ -44,10 +41,11 @@ class SubscriptionBatchStatus: current_site_id: Optional[int] = None error: Optional[str] = None can_cancel: bool = False + skipped_count: int = 0 class SubscriptionExecutionReadRepository(Protocol): - """请求级读取搜索任务、批次与下载提交事实的端口。""" + """请求级读取搜索任务与批次事实的端口。""" async def latest_search_tasks( self, @@ -56,13 +54,6 @@ class SubscriptionExecutionReadRepository(Protocol): """返回每条订阅最近更新的搜索任务。""" ... - async def latest_download_submissions( - self, - subscription_ids: tuple[int, ...], - ) -> dict[int, SubscriptionDownloadSnapshot]: - """返回每条订阅最近更新的下载提交。""" - ... - async def list_batches( self, *, @@ -81,16 +72,8 @@ class SubscriptionExecutionReadRepository(Protocol): class SubscriptionExecutionStatusService: - """把搜索队列与下载幂等账本投影为稳定业务状态。""" + """把搜索队列投影为稳定业务状态。""" - _DOWNLOAD_STATES = { - "submitting": "submitting", - "accepted": "accepted", - "succeeded": "completed", - "retryable": "retryable", - "reconcile_required": "reconcile_required", - "cancelled": "cancelled", - } _ACTIVE_STATES = { "queued", "running", @@ -99,16 +82,8 @@ class SubscriptionExecutionStatusService: "waiting_site_budget", "preparing", "submitting", - "accepted", "cancelling", } - _DOWNLOAD_OVERRIDE_STATES = { - "submitting", - "accepted", - "retryable", - "reconcile_required", - } - def __init__( self, repository: SubscriptionExecutionReadRepository, @@ -133,14 +108,10 @@ class SubscriptionExecutionStatusService: if not ids: return {} tasks = await self._repository.latest_search_tasks(ids) - downloads = await self._repository.latest_download_submissions(ids) result: dict[int, SubscriptionExecutionStatus] = {} for subscription_id in ids: task = tasks.get(subscription_id) - download = downloads.get(subscription_id) - if download and self._download_wins(task, download): - result[subscription_id] = self._from_download(download, task) - elif task: + if task: result[subscription_id] = self._from_task(task) return result @@ -175,17 +146,6 @@ class SubscriptionExecutionStatusService: return None return self._from_batch(batch, tasks) - @classmethod - def _download_wins( - cls, - task: Optional[SearchTaskSnapshot], - download: SubscriptionDownloadSnapshot, - ) -> bool: - """下载风险状态优先，其余事实按更新时间选择。""" - if download.state in cls._DOWNLOAD_OVERRIDE_STATES: - return True - return task is None or download.updated_at >= task.updated_at - @classmethod def _from_task(cls, task: SearchTaskSnapshot) -> SubscriptionExecutionStatus: """把搜索任务状态归一为稳定业务词汇。""" @@ -205,29 +165,6 @@ class SubscriptionExecutionStatusService: updated_at=task.updated_at, error=cls._safe_error(task.last_error), can_cancel=state in cls._ACTIVE_STATES, - can_retry=state == "failed", - ) - - @classmethod - def _from_download( - cls, - download: SubscriptionDownloadSnapshot, - task: Optional[SearchTaskSnapshot], - ) -> SubscriptionExecutionStatus: - """把下载提交账本状态投影为业务状态并保留搜索来源。""" - state = cls._DOWNLOAD_STATES.get(download.state, download.state) - return SubscriptionExecutionStatus( - state=state, - phase=state, - source=task.source if task else None, - batch_id=task.batch_id if task else None, - task_id=download.task_id or (task.task_id if task else None), - current_site_id=task.current_site_id if task else None, - updated_at=download.updated_at, - error=cls._safe_error(download.last_error), - can_cancel=state == "submitting", - can_retry=state == "retryable", - requires_reconciliation=state == "reconcile_required", ) @classmethod @@ -240,7 +177,12 @@ class SubscriptionExecutionStatusService: current = next((task for task in tasks if task.state == "running"), None) if current is None: current = next((task for task in tasks if task.state == "queued"), None) - processed = batch.finished_count + batch.failed_count + batch.cancelled_count + processed = ( + batch.finished_count + + batch.failed_count + + batch.cancelled_count + + batch.skipped_count + ) phase = current.phase if current else batch.state return SubscriptionBatchStatus( batch_id=batch.batch_id, @@ -252,6 +194,7 @@ class SubscriptionExecutionStatusService: finished_count=batch.finished_count, failed_count=batch.failed_count, cancelled_count=batch.cancelled_count, + skipped_count=batch.skipped_count, current_subscription_id=current.subscription_id if current else None, current_site_id=current.current_site_id if current else None, created_at=batch.created_at, diff --git a/app/chain/_contracts.py b/app/chain/_contracts.py index 2ab7c6f06..b7bd9f77b 100644 --- a/app/chain/_contracts.py +++ b/app/chain/_contracts.py @@ -66,6 +66,14 @@ class MusicSubscribeMixinHost(Protocol): def get_subscribe_source_keyword(self, subscribe: Any) -> str: ... + def _SubscribeChain__candidate_contract_changed( + self, + prepared: Any, + current: Any, + ) -> bool: + """判断准备候选期间订阅身份或过滤合同是否已经变化。""" + ... + class InteractionMixinHost(Protocol): """交互委托 mixin 对业务 Chain 的最小要求。""" diff --git a/app/chain/_music.py b/app/chain/_music.py index aec945ffe..d0e868b42 100644 --- a/app/chain/_music.py +++ b/app/chain/_music.py @@ -1,15 +1,18 @@ import copy from collections.abc import Mapping -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple from app.application.configuration import get_configured_system_config +from app.application.download.admission import SubscriptionDownloadGovernance from app.application.subscription.contract import ( SubscriptionRepository, SubscriptionSnapshot, build_subscribe_meta, subscribe_media_key, ) +from app.application.subscription.execution import SubscriptionExecutionContext from app.application.subscription.mutation import SubscriptionActor +from app.application.subscription.sitebudget import SubscriptionSearchCancelled from app.application.torrent.download import TorrentHelper from app.chain._contracts import MusicSubscribeMixinHost from app.chain.download import DownloadChain @@ -44,6 +47,9 @@ class MusicSubscribeMixin: __mixin_host_protocol__ = MusicSubscribeMixinHost subscription_repository: SubscriptionRepository sync_subscription_mutation_scope: "SyncSubscriptionMutationScope" + _SubscribeChain__candidate_contract_changed: Callable[ + [SubscriptionSnapshot, SubscriptionSnapshot], bool + ] """ 音乐订阅功能域 mixin：单曲/专辑目标识别、实体快照同步、候选筛选、 择优下载与完成推进。 @@ -348,10 +354,32 @@ class MusicSubscribeMixin: subscribe: SubscriptionSnapshot, mediainfo: MusicInfo, contexts: List[Context], + execution_context: Optional[SubscriptionExecutionContext] = None, ) -> None: """批量择优下载音乐候选，并按单曲或整专完成语义推进订阅。""" if not contexts: return + self._ensure_music_execution_active(execution_context) + repository = self.subscription_repository + current_subscribe = repository.get(subscribe.id) + if current_subscribe is None: + logger.info(f"音乐订阅 {subscribe.id} 已删除，放弃本轮下载提交") + return + if current_subscribe.state == "S": + logger.info(f"音乐订阅 {current_subscribe.name} 已暂停，放弃本轮下载提交") + return + if self._SubscribeChain__candidate_contract_changed(subscribe, current_subscribe): + logger.info(f"音乐订阅 {current_subscribe.name} 的筛选或媒体身份已变化，放弃旧候选并等待下一轮") + return + subscribe = current_subscribe + governance = None + if execution_context: + governance = SubscriptionDownloadGovernance( + cancelled=execution_context.should_stop, + mark_started=execution_context.mark_download_started, + ) + execution_context.report_phase("preparing") + self._ensure_music_execution_active(execution_context) downloads, _ = DownloadChain().batch_download( contexts=contexts, username=subscribe.username, @@ -359,6 +387,7 @@ class MusicSubscribeMixin: downloader=subscribe.downloader, source=self.get_subscribe_source_keyword(subscribe), custom_words=subscribe.custom_words, + governance=governance, ) successful = [ context for context in downloads or [] @@ -370,7 +399,6 @@ class MusicSubscribeMixin: context for context in successful if context.confirmed_full_coverage ] - repository = self.subscription_repository current_subscribe = None if subscribe.best_version and quality_downloads: best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order) @@ -397,12 +425,18 @@ class MusicSubscribeMixin: downloads=downloads, ) - def _search_music_subscribe(self, subscribe: SubscriptionSnapshot) -> None: + def _search_music_subscribe( + self, + subscribe: SubscriptionSnapshot, + execution_context: Optional[SubscriptionExecutionContext] = None, + ) -> None: """复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。""" + self._ensure_music_execution_active(execution_context) target = self._prepare_music_subscribe(subscribe) if not target: return subscribe, mediainfo, _ = target + self._ensure_music_execution_active(execution_context) sites = self.get_sub_sites(subscribe) default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \ @@ -414,13 +448,17 @@ class MusicSubscribeMixin: searchchain = SearchChain() contexts: List[Context] = [] + if execution_context: + execution_context.report_phase("searching") for keyword in keywords: + self._ensure_music_execution_active(execution_context) contexts = searchchain.search_by_title( title=keyword, sites=sites, mtype=MediaType.MUSIC, rule_groups=rule_groups, ) + self._ensure_music_execution_active(execution_context) contexts = self._filter_music_subscribe_contexts( subscribe=subscribe, mediainfo=mediainfo, @@ -433,18 +471,28 @@ class MusicSubscribeMixin: logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源") return - self._download_music_subscribe(subscribe, mediainfo, contexts) + self._download_music_subscribe( + subscribe, + mediainfo, + contexts, + execution_context=execution_context, + ) def _match_music_subscribe( self, subscribe: SubscriptionSnapshot, contexts: List[Context], + execution_context: Optional[SubscriptionExecutionContext] = None, ) -> None: """直接匹配本轮 RSS 缓存中的音乐资源，避免再次调用站点搜索接口。""" + self._ensure_music_execution_active(execution_context) target = self._prepare_music_subscribe(subscribe) if not target: return subscribe, mediainfo, _ = target + if execution_context: + execution_context.report_phase("matching") + self._ensure_music_execution_active(execution_context) matched = self._filter_music_subscribe_contexts( subscribe=subscribe, mediainfo=mediainfo, @@ -453,4 +501,21 @@ class MusicSubscribeMixin: if not matched: logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源") return - self._download_music_subscribe(subscribe, mediainfo, matched) + self._download_music_subscribe( + subscribe, + mediainfo, + matched, + execution_context=execution_context, + ) + + @staticmethod + def _ensure_music_execution_active( + execution_context: Optional[SubscriptionExecutionContext], + ) -> None: + """在音乐搜索、匹配和提交前的安全边界执行取消与 TTL 检查。""" + if execution_context is None: + return + if execution_context.is_cancel_requested(): + raise SubscriptionSearchCancelled("订阅搜索已取消") + if execution_context.is_expired(): + raise TimeoutError("订阅执行已超过协作截止时间") diff --git a/app/chain/base.py b/app/chain/base.py index 573676b5c..9620dddc1 100644 --- a/app/chain/base.py +++ b/app/chain/base.py @@ -71,7 +71,6 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, self.transfer_execution_repository = context.transfer_execution_repository self.media_server_repository = context.media_server_repository self.download_failure_repository = context.download_failure_repository - self.subscription_download_repository = context.subscription_download_repository self.user_repository = context.user_repository self.runtime_config = context.configuration self.stop_state = context.stop_state diff --git a/app/chain/download/admission.py b/app/chain/download/admission.py deleted file mode 100644 index ea31abfcc..000000000 --- a/app/chain/download/admission.py +++ /dev/null @@ -1,258 +0,0 @@ -"""订阅下载提交幂等身份与状态转换 owner。""" - -from __future__ import annotations - -import hashlib -import json -from datetime import datetime, timedelta, timezone -from typing import Optional, Set, cast - -from app.application.download.admission import ( - DownloadReconciliationRequired, - SubscriptionDownloadClaim, - SubscriptionDownloadGovernance, - SubscriptionDownloadRepository, - SubscriptionDownloadRequest, -) -from app.chain.download.contract import _DownloadOwnerBase -from app.domain import episode as episode_rules -from app.domain.context import Context -from app.schemas.media import build_media_key, resolve_media_identity -from app.schemas.types import MediaType - - -class DownloadAdmissionOwner(_DownloadOwnerBase): - """计算订阅提交唯一键并通过持久账本控制外部副作用。""" - - def _build_subscription_download_request( - self, - *, - context: Context, - episodes: Optional[Set[int]], - governance: SubscriptionDownloadGovernance, - delivery_scope: str, - ) -> SubscriptionDownloadRequest: - """组合逻辑媒体、资源、覆盖、模式和交付目标生成规范幂等请求。""" - media = context.media_info - meta = context.meta_info - torrent = context.torrent_info - media_source, media_id = resolve_media_identity(media=media) - media_key = build_media_key(media_source, media_id) or ( - f"{getattr(media, 'title', '')}:{getattr(media, 'year', '')}" - ) - seasons = sorted(set(getattr(meta, "season_list", None) or [])) - selected = sorted(set(episodes or getattr(meta, "episode_list", None) or [])) - if selected: - coverage = f"episodes:{episode_rules.format_ranges(selected)}" - elif seasons: - coverage = "seasons:" + ",".join(str(season) for season in seasons) + ":full" - else: - coverage = "full" - media_type = getattr(media, "type", None) - media_type_value = getattr(media_type, "value", media_type) - media_season = getattr(media, "season", None) - meta_season = getattr(meta, "season", None) - logical_identity = json.dumps( - { - "media_key": str(media_key or ""), - "media_type": str(media_type_value or ""), - "season": media_season if media_season is not None else meta_season, - "episode_group": getattr(media, "episode_group", None), - "music_type": getattr(media, "music_type", None), - }, - ensure_ascii=False, - sort_keys=True, - ) - resource_key = self._torrent_resource_key(torrent) - canonical = json.dumps( - { - "logical_identity": logical_identity, - "resource_key": resource_key, - "coverage": coverage, - "mode": governance.mode, - "delivery_scope": delivery_scope, - }, - ensure_ascii=False, - sort_keys=True, - ) - legacy_logical_identity = json.dumps( - { - "subscription_id": governance.subscription_id, - "media_key": str(media_key or ""), - "media_type": str(media_type_value or ""), - "season": media_season if media_season is not None else meta_season, - "episode_group": getattr(media, "episode_group", None), - }, - ensure_ascii=False, - sort_keys=True, - ) - legacy_canonical = json.dumps( - { - "logical_identity": legacy_logical_identity, - "resource_key": resource_key, - "coverage": coverage, - "mode": governance.mode, - }, - ensure_ascii=False, - sort_keys=True, - ) - return SubscriptionDownloadRequest( - idempotency_key=hashlib.sha256(canonical.encode("utf-8")).hexdigest(), - legacy_idempotency_key=hashlib.sha256( - legacy_canonical.encode("utf-8") - ).hexdigest(), - subscription_id=governance.subscription_id, - task_id=governance.task_id, - logical_identity=logical_identity, - resource_key=resource_key, - coverage=coverage, - mode=governance.mode, - delivery_scope=delivery_scope, - ) - - def _claim_subscription_download( - self, - *, - context: Context, - episodes: Optional[Set[int]], - governance: Optional[SubscriptionDownloadGovernance], - downloader: Optional[str], - download_uri: str, - ) -> tuple[Optional[SubscriptionDownloadClaim], Optional[str]]: - """在下载器调用前认领唯一提交权，并返回已成功提交的历史 hash。""" - if governance is None: - return None, None - legacy_hash = self._legacy_subscription_download_hash( - context=context, - episodes=episodes, - governance=governance, - ) - if legacy_hash: - return None, legacy_hash - repository = getattr(self, "subscription_download_repository", None) - if repository is None: - raise RuntimeError("订阅下载幂等仓储尚未配置") - request = self._build_subscription_download_request( - context=context, - episodes=episodes, - governance=governance, - delivery_scope=self._subscription_delivery_scope( - downloader=downloader, - download_uri=download_uri, - ), - ) - if request.legacy_idempotency_key: - legacy = repository.get(request.legacy_idempotency_key) - if legacy and legacy.state == "succeeded" and legacy.download_hash: - return None, legacy.download_hash - if legacy and legacy.state in {"submitting", "accepted", "reconcile_required"}: - raise DownloadReconciliationRequired( - f"订阅下载提交 {legacy.idempotency_key} 当前为 {legacy.state}，需要先对账下载器" - ) - claim = repository.claim(request) - snapshot = claim.snapshot - if claim.acquired: - if not snapshot.attempt_token: - raise RuntimeError("订阅下载提交已认领但缺少尝试令牌") - return claim, None - if snapshot.state == "succeeded" and snapshot.download_hash: - return claim, snapshot.download_hash - if snapshot.state in {"submitting", "accepted", "reconcile_required"}: - raise DownloadReconciliationRequired( - f"订阅下载提交 {snapshot.idempotency_key} 当前为 {snapshot.state}，需要先对账下载器" - ) - return claim, None - - @staticmethod - def _subscription_delivery_scope( - *, - downloader: Optional[str], - download_uri: str, - ) -> str: - """规范化实际下载器和保存目标，限定跨记录去重的产品边界。""" - return json.dumps( - { - "downloader": downloader or "auto", - "download_uri": download_uri, - }, - ensure_ascii=False, - sort_keys=True, - ) - - def _legacy_subscription_download_hash( - self, - *, - context: Context, - episodes: Optional[Set[int]], - governance: SubscriptionDownloadGovernance, - ) -> Optional[str]: - """重读迁移前下载历史，兼容识别同订阅同 torrent 同覆盖的成功提交。""" - media = context.media_info - meta = context.meta_info - torrent = context.torrent_info - repository = getattr(self, "download_history_repository", None) - media_source, media_id = resolve_media_identity(media=media) - if repository is None or not media_source or not media_id or not torrent: - return None - histories = repository.get_by_media_identity( - media_source=media_source, - media_id=str(media_id), - music_type=getattr(media, "music_type", None), - ) - expected_episodes = episode_rules.format_ranges( - sorted(set(episodes or getattr(meta, "episode_list", None) or [])) - ) - expected_season = str(getattr(meta, "season", None) or "") - for history in histories: - if not history.download_hash: - continue - if history.torrent_name != torrent.title or history.torrent_site != torrent.site_name: - continue - if not self._history_matches_subscription(history.note, governance.subscription_id): - continue - if getattr(history, "episode_group", None) != getattr(media, "episode_group", None): - continue - if expected_episodes: - if history.episodes == expected_episodes: - return cast(str, history.download_hash) - continue - if getattr(media, "type", None) == MediaType.TV and str(history.seasons or "") != expected_season: - continue - return cast(str, history.download_hash) - return None - - @staticmethod - def _history_matches_subscription(note: object, subscription_id: int) -> bool: - """从下载历史来源快照确认记录属于同一订阅，而非同媒体其他记录。""" - if not isinstance(note, dict): - return False - source = note.get("source") - if not isinstance(source, str) or not source.startswith("Subscribe|"): - return False - try: - payload = json.loads(source.split("|", 1)[1]) - return int(payload.get("id")) == subscription_id - except (AttributeError, TypeError, ValueError, json.JSONDecodeError): - return False - - @staticmethod - def _subscription_download_cancelled( - governance: Optional[SubscriptionDownloadGovernance], - ) -> bool: - """在可安全取消边界调用入口提供的取消检查。""" - return bool(governance and governance.cancelled and governance.cancelled()) - - @staticmethod - def _subscription_download_retry_at(error: Optional[str], ttl_seconds: int) -> str: - """把明确拒绝的下载提交推迟到失败冷却到期后。""" - del error - return ( - datetime.now(timezone.utc) + timedelta(seconds=max(1, ttl_seconds)) - ).isoformat(timespec="seconds") - - def _subscription_download_repository(self) -> SubscriptionDownloadRepository: - """返回已配置的订阅下载仓储，缺失时拒绝越过外部副作用边界。""" - repository = getattr(self, "subscription_download_repository", None) - if repository is None: - raise RuntimeError("订阅下载幂等仓储尚未配置") - return cast(SubscriptionDownloadRepository, repository) diff --git a/app/chain/download/batch.py b/app/chain/download/batch.py index b0fb8ca80..f5351aba3 100644 --- a/app/chain/download/batch.py +++ b/app/chain/download/batch.py @@ -87,13 +87,13 @@ class DownloadBatchOwner(_DownloadOwnerBase): :param userid: 用户ID :param username: 调用下载的用户名/插件名 :param downloader: 下载器 - :param custom_words: 下载来源自定义词；governance: 订阅幂等、入口任务和取消边界 + :param custom_words: 下载来源自定义词 + :param governance: 订阅取消与下载器副作用边界 :return: 已下载资源列表及剩余缺集，键格式为 no_exists[source:id] """ no_exists_was_none = no_exists is None if no_exists is None: no_exists = {} - # 已下载的项目 downloaded_list: List[Context] = [] custom_word_list = custom_words.splitlines() if custom_words else None diff --git a/app/chain/download/contract.py b/app/chain/download/contract.py index c4f9c5a3f..c0c3b99a4 100644 --- a/app/chain/download/contract.py +++ b/app/chain/download/contract.py @@ -9,7 +9,6 @@ if TYPE_CHECKING: """向类型检查器声明同一 Facade 上的跨 owner 方法。""" _active_download_failure_fingerprints: Callable[..., Any] - _build_subscription_download_request: Callable[..., Any] _append_no_exists: Callable[..., None] _build_download_notification: Callable[..., Any] _build_download_failure_fingerprint: Callable[..., Any] @@ -23,20 +22,14 @@ if TYPE_CHECKING: _is_job_active: Callable[..., bool] _is_subscribe_source: Callable[..., bool] _log_download_failure_cooldown: Callable[..., None] - _legacy_subscription_download_hash: Callable[..., Any] - _history_matches_subscription: Callable[..., bool] _media_identity_keys: Callable[..., set[str]] _matches_media_identity: Callable[..., bool] _prepare_batch_download_contexts: Callable[..., Any] - _claim_subscription_download: Callable[..., Any] _record_download_failure: Callable[..., Any] _resolve_media_download_dir: Callable[..., Any] _settle_download_success: Callable[..., None] _submit_download_added_task: Callable[..., None] - _torrent_resource_key: Callable[..., str] _subscription_download_cancelled: Callable[..., bool] - _subscription_download_repository: Callable[..., Any] - _subscription_download_retry_at: Callable[..., str] _validate_music_album_resource: Callable[..., Any] batch_download: Callable[..., Any] download_single: Callable[..., Any] diff --git a/app/chain/download/facade.py b/app/chain/download/facade.py index 1681c68a7..780cde049 100644 --- a/app/chain/download/facade.py +++ b/app/chain/download/facade.py @@ -3,7 +3,6 @@ from collections.abc import Callable from typing import Any, TypeVar -from app.chain.download.admission import DownloadAdmissionOwner from app.chain.download.batch import DownloadBatchOwner from app.chain.download.existence import DownloadExistenceOwner from app.chain.download.failure import DownloadFailureOwner @@ -30,7 +29,6 @@ class DownloadChain( DownloadSubtitleOwner, DownloadSelectionOwner, DownloadFailureOwner, - DownloadAdmissionOwner, DownloadSubmissionOwner, DownloadBatchOwner, DownloadExistenceOwner, diff --git a/app/chain/download/submission.py b/app/chain/download/submission.py index d73d96749..de61e411a 100644 --- a/app/chain/download/submission.py +++ b/app/chain/download/submission.py @@ -10,10 +10,7 @@ from urllib.parse import urlencode, urljoin, urlparse from app.application.configuration import get_chain_runtime_config_snapshot from app.application.directory import validate_download_save_path -from app.application.download.admission import ( - DownloadReconciliationRequired, - SubscriptionDownloadGovernance, -) +from app.application.download.admission import SubscriptionDownloadGovernance from app.application.torrent.download import TorrentHelper from app.chain.download.contract import _DownloadOwnerBase from app.chain.download.ports import ( @@ -284,6 +281,13 @@ class _DownloadResourceOwner(_DownloadOwnerBase): class DownloadSubmissionOwner(_DownloadResourceOwner): """单任务下载准备、提交与结算 owner。""" + @staticmethod + def _subscription_download_cancelled( + governance: Optional[SubscriptionDownloadGovernance], + ) -> bool: + """在下载器副作用前读取订阅执行上下文的停止信号。""" + return bool(governance and governance.cancelled and governance.cancelled()) + def download_single(self, context: Context, torrent_file: Optional[Path] = None, torrent_content: Optional[Union[str, bytes]] = None, @@ -478,57 +482,20 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): custom_words: Optional[str], governance: Optional[SubscriptionDownloadGovernance], ) -> tuple[Optional[str], Optional[str]]: - """认领幂等提交权，调用下载器并分派成功或拒绝结算。""" + """在可取消边界后调用下载器，并分派成功或拒绝结算。""" if self._subscription_download_cancelled(governance): return None, "订阅下载在提交前已取消" - admission, duplicate_hash = self._claim_subscription_download( - context=context, - episodes=episodes, - governance=governance, - downloader=downloader or prepared.site_downloader, - download_uri=prepared.download_uri, - ) - if duplicate_hash: - if governance and governance.mark_started: - governance.mark_started() - logger.info(f"{prepared.torrent.title} 已由重叠订阅入口提交，复用任务 {duplicate_hash}") - return duplicate_hash, "下载任务已由重叠入口提交" - if admission is not None and not admission.acquired: - return None, ( - f"订阅下载提交当前为 {admission.snapshot.state}，" - f"最早可重试：{admission.snapshot.available_at or '待下一轮'}" - ) - attempt_token = admission.snapshot.attempt_token if admission is not None else None - admission_key = admission.snapshot.idempotency_key if admission is not None else None - if admission is not None and self._subscription_download_cancelled(governance): - self._subscription_download_repository().mark_cancelled( - idempotency_key=admission.snapshot.idempotency_key, - attempt_token=admission.snapshot.attempt_token or "", - ) - return None, "订阅下载在下载器调用前已取消" if governance and governance.mark_started: governance.mark_started() - try: - result = self.download( - content=prepared.torrent_content, - cookie=prepared.torrent.site_cookie, - episodes=cast(Set[int], episodes), - download_dir=prepared.download_dir, - category=prepared.media.category, - label=label, - downloader=downloader or prepared.site_downloader, - ) - except Exception as err: - if admission_key and attempt_token: - self._subscription_download_repository().mark_reconcile_required( - idempotency_key=admission_key, - attempt_token=attempt_token, - error=f"下载器调用异常：{str(err)}", - ) - raise DownloadReconciliationRequired( - f"{prepared.torrent.title} 下载器结果不确定，已冻结自动重试" - ) from err - raise + result = self.download( + content=prepared.torrent_content, + cookie=prepared.torrent.site_cookie, + episodes=cast(Set[int], episodes), + download_dir=prepared.download_dir, + category=prepared.media.category, + label=label, + downloader=downloader or prepared.site_downloader, + ) actual_downloader, download_hash, layout, error_msg = ( result if result else (None, None, None, "未找到下载器") ) @@ -545,8 +512,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): actual_downloader=actual_downloader, download_hash=download_hash, layout=layout, - admission_key=admission_key, - attempt_token=attempt_token, ) else: self._record_rejected_download( @@ -559,8 +524,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): userid=userid, actual_downloader=actual_downloader, error_msg=error_msg, - admission_key=admission_key, - attempt_token=attempt_token, ) return download_hash, error_msg @@ -578,71 +541,28 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): actual_downloader: Optional[str], download_hash: str, layout: Optional[str], - admission_key: Optional[str], - attempt_token: Optional[str], ) -> None: - """持久化下载器接受事实，执行本地结算并确认成功终态。""" - if admission_key and attempt_token: - accepted = self._subscription_download_repository().mark_accepted( - idempotency_key=admission_key, - attempt_token=attempt_token, - downloader=actual_downloader, - download_hash=download_hash, - ) - if not accepted: - raise DownloadReconciliationRequired( - f"{prepared.torrent.title} 已被下载器接受，但本地接受状态写入失败" - ) - try: - self._settle_download_success( - context=context, - media=prepared.media, - meta=prepared.meta, - torrent=prepared.torrent, - folder_name=prepared.folder_name, - file_list=prepared.file_list, - download_dir=prepared.download_dir, - layout=layout, - downloader=actual_downloader, - download_hash=download_hash, - download_episodes=prepared.download_episodes, - episodes=episodes, - channel=channel, - source=source, - userid=userid, - username=username, - torrent_content=prepared.torrent_content, - custom_words=custom_words, - ) - except Exception as err: - if admission_key and attempt_token: - self._subscription_download_repository().mark_reconcile_required( - idempotency_key=admission_key, - attempt_token=attempt_token, - error=f"下载器已接受但本地结算失败：{str(err)}", - downloader=actual_downloader, - download_hash=download_hash, - ) - raise DownloadReconciliationRequired( - f"{prepared.torrent.title} 已被下载器接受但本地结算失败，已转待对账" - ) from err - raise - if admission_key and attempt_token: - succeeded = self._subscription_download_repository().mark_succeeded( - idempotency_key=admission_key, - attempt_token=attempt_token, - ) - if not succeeded: - self._subscription_download_repository().mark_reconcile_required( - idempotency_key=admission_key, - attempt_token=attempt_token, - error="本地结算完成但幂等成功终态写入失败", - downloader=actual_downloader, - download_hash=download_hash, - ) - raise DownloadReconciliationRequired( - f"{prepared.torrent.title} 本地结算完成但提交终态未确认，已转待对账" - ) + """按普通下载合同结算下载器明确返回的成功结果。""" + self._settle_download_success( + context=context, + media=prepared.media, + meta=prepared.meta, + torrent=prepared.torrent, + folder_name=prepared.folder_name, + file_list=prepared.file_list, + download_dir=prepared.download_dir, + layout=layout, + downloader=actual_downloader, + download_hash=download_hash, + download_episodes=prepared.download_episodes, + episodes=episodes, + channel=channel, + source=source, + userid=userid, + username=username, + torrent_content=prepared.torrent_content, + custom_words=custom_words, + ) def _record_rejected_download( self, @@ -656,20 +576,8 @@ class DownloadSubmissionOwner(_DownloadResourceOwner): userid: Union[str, int, None], actual_downloader: Optional[str], error_msg: str, - admission_key: Optional[str], - attempt_token: Optional[str], ) -> None: - """记录无外部副作用的明确拒绝，并通知原调用渠道。""" - if admission_key and attempt_token: - self._subscription_download_repository().mark_retryable( - idempotency_key=admission_key, - attempt_token=attempt_token, - available_at=self._subscription_download_retry_at( - error_msg, - self._download_failure_ttl(error_msg), - ), - error=error_msg, - ) + """按普通失败合同记录下载器明确拒绝并通知原调用渠道。""" logger.error( f"{prepared.media.title_year} 添加下载任务失败：" f"{prepared.torrent.title} - {prepared.torrent.enclosure}，{error_msg}" diff --git a/app/chain/search/facade.py b/app/chain/search/facade.py index 1b2d19549..89992baed 100644 --- a/app/chain/search/facade.py +++ b/app/chain/search/facade.py @@ -52,7 +52,7 @@ class SearchChain(ChainBase): self._subscription_site_budget_failure_lock = threading.Lock() def record_subscription_site_budget_failure(self, error: str) -> None: - """线程安全地记录一个未执行站点，供订阅任务暴露聚合失败。""" + """线程安全地记录一个站点执行失败，供订阅任务暴露聚合失败。""" lock = getattr(self, "_subscription_site_budget_failure_lock", None) if lock is None: return diff --git a/app/chain/search/provider.py b/app/chain/search/provider.py index 8b45a76b6..c0f4b9f61 100644 --- a/app/chain/search/provider.py +++ b/app/chain/search/provider.py @@ -220,7 +220,7 @@ class _SearchProviderSyncOwner(_SearchOwnerBase): return [] with capture_site_search_observation() as observation: try: - return _search_site_page( + result = _search_site_page( self, site=site, keyword=keyword, @@ -234,6 +234,13 @@ class _SearchProviderSyncOwner(_SearchOwnerBase): error=str(error), ) raise + else: + if observation.attempted and observation.outcome not in {"success", "skipped"}: + failure = observation.error or observation.outcome + self.record_subscription_site_budget_failure( + f"站点 {site.get('name') or site_id} 搜索失败：{failure}" + ) + return result finally: try: budget.finish(claim, observation) diff --git a/app/chain/subscribe/contract.py b/app/chain/subscribe/contract.py index 4675478e5..c3de16a25 100644 --- a/app/chain/subscribe/contract.py +++ b/app/chain/subscribe/contract.py @@ -14,8 +14,10 @@ if TYPE_CHECKING: class _SubscribeOwnerHost: """声明 SubscribeChain 组合后向各 owner 提供的属性和兄弟职责。""" - _LOCK_TIMOUT: int - _rlock: Any + _SUBSCRIPTION_EXECUTION_TTL: int + _match_lock: Any + _search_queue_lock: Any + _subscription_execution_admission: Any download_history_repository: Any eventmanager: Any messagehelper: Any @@ -98,7 +100,6 @@ if TYPE_CHECKING: get_subscribed_sites: Callable[..., Any] has_music_subscribe: Callable[..., Any] match: Callable[..., Any] - match_batch: Callable[..., Any] media_exists: Callable[..., Any] media_files: Callable[..., Any] obtain_images: Callable[..., Any] diff --git a/app/chain/subscribe/facade.py b/app/chain/subscribe/facade.py index cfc161be4..d031d5b8a 100644 --- a/app/chain/subscribe/facade.py +++ b/app/chain/subscribe/facade.py @@ -4,6 +4,7 @@ import threading from typing import Any from app.application.messaging.subscribe import SubscribeInteractionHandler +from app.application.subscription.execution import SubscriptionExecutionAdmission from app.chain._interaction import InteractionChainMixin from app.chain._music import MusicSubscribeMixin from app.chain.base import ChainBase @@ -48,9 +49,10 @@ class SubscribeChain( """ _interaction_handler_type = SubscribeInteractionHandler - _rlock = threading.RLock() + _match_lock = threading.Lock() _search_queue_lock = threading.Lock() - _LOCK_TIMOUT = 3600 * 2 + _subscription_execution_admission = SubscriptionExecutionAdmission() + _SUBSCRIPTION_EXECUTION_TTL = 3600 * 2 @classmethod def _music_media_chain(cls) -> MediaChain: diff --git a/app/chain/subscribe/match.py b/app/chain/subscribe/match.py index 0a9f371fe..f3e8658f1 100644 --- a/app/chain/subscribe/match.py +++ b/app/chain/subscribe/match.py @@ -1,13 +1,20 @@ """订阅候选准备、身份复核与资源匹配""" import copy +from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Literal, Optional from app.application.configuration import get_configured_system_config -from app.application.subscription.candidates import CandidateBatch, CandidateIndex -from app.application.subscription.contract import build_subscribe_meta, subscribe_media_key +from app.application.subscription.candidates import CandidateIndex +from app.application.subscription.contract import ( + SubscriptionSnapshot, + build_subscribe_meta, + subscribe_media_key, +) +from app.application.subscription.execution import SubscriptionExecutionContext from app.application.subscription.facts import FreshFactLease +from app.application.subscription.sitebudget import SubscriptionSearchCancelled from app.application.torrent.download import TorrentHelper from app.chain.media import MediaChain from app.chain.subscribe.contract import _SubscribeOwnerBase @@ -28,10 +35,176 @@ from app.schemas.types import ( SystemConfigKey, ) +_MatchOutcome = Literal["completed", "skipped", "failed"] + + +@dataclass(slots=True) +class _MatchExecutionSummary: + """保存一次 Match 批次的实际订阅执行结果。""" + + total: int + completed: int = 0 + skipped: int = 0 + failed: int = 0 + + @property + def finished(self) -> int: + """返回已经收口的订阅数量。""" + return self.completed + self.skipped + self.failed + + def record(self, outcome: Optional[str]) -> None: + """将单订阅结果归入稳定的完成、跳过或失败计数。""" + if outcome == "completed": + self.completed += 1 + elif outcome == "skipped": + self.skipped += 1 + else: + self.failed += 1 + + def as_data(self, current: Optional[int] = None) -> dict[str, int]: + """构造供进度消费者读取的实际计数。""" + data = { + "total": self.total, + "finished": self.finished, + "completed": self.completed, + "skipped": self.skipped, + "failed": self.failed, + } + if current is not None: + data["current"] = current + return data + + +def _report_match_progress( + progress_callback: Optional[Callable[..., None]], + summary: _MatchExecutionSummary, + *, + value: float, + text: str, + current: Optional[int] = None, +) -> None: + """发布 Match 进度，并始终携带真实的完成、跳过和失败计数。""" + if progress_callback: + progress_callback( + value=value, + text=text, + data=summary.as_data(current=current), + ) + + +def _is_title_match_allowed( + mediainfo: MediaInfo, + torrent_meta: MetaBase, + torrent_info: TorrentInfo, +) -> bool: + """判断资源是否可以通过标题兜底匹配订阅目标。""" + if not TorrentHelper.match_torrent( + mediainfo=mediainfo, + torrent_meta=torrent_meta, + torrent=torrent_info, + ): + return False + if not TorrentHelper.requires_identity_disambiguation( + mediainfo=mediainfo, + torrent_meta=torrent_meta, + ): + return True + logger.info( + f"{torrent_info.site_name} - {torrent_info.title} 仅通过无年份别名命中且候选媒体身份无法确认，已跳过" + ) + return False + + +def _get_media_id_match_source(mediainfo: Optional[MediaInfo]) -> str: + """返回候选自身识别命中的明确媒体 ID 类型。""" + media_source, media_id = resolve_media_identity(media=mediainfo) + if media_source and media_id: + return str(media_source) + return "unknown" + + +def _reconcile_candidate_media( + target_mediainfo: MediaInfo, + candidate_mediainfo: MediaInfo, + torrent_meta: MetaBase, + torrent_info: TorrentInfo, + context: Context, +) -> Optional[MediaInfo]: + """ + 在推断媒体 ID 冲突时，以订阅目标对候选标题做严格复核。 + + 标题携带明确媒体身份或目标缺少同来源 ID 时保持严格拒绝；只有普通 + 标题推断产生冲突且标题、别名、类型和年份复核通过时才回填订阅目标。 + """ + candidate_source, candidate_id = resolve_media_identity(media=candidate_mediainfo) + target_source, target_id = resolve_media_identity(media=target_mediainfo) + conflicts = [] + if candidate_source and candidate_id and (candidate_source != target_source or candidate_id != target_id): + conflicts.append( + ( + str(candidate_source), + candidate_id, + f"{target_source}:{target_id}" if target_source and target_id else None, + ) + ) + if not conflicts: + return candidate_mediainfo + + conflict_text = "、".join( + f"{source} {candidate_id} != {target_id if target_id is not None else '缺失'}" + for source, candidate_id, target_id in conflicts + ) + if any(target_id is None for _, _, target_id in conflicts): + logger.debug( + f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID与订阅目标不可比较：{conflict_text}" + ) + return None + + explicit_source, explicit_id = resolve_media_identity(media=torrent_meta) + if explicit_source and explicit_id: + logger.debug( + f"{torrent_info.site_name} - {torrent_info.title} 标题含明确媒体ID " + f"{explicit_source}:{explicit_id}，保持严格校验：{conflict_text}" + ) + return None + + if not TorrentHelper.match_torrent( + mediainfo=target_mediainfo, + torrent_meta=torrent_meta, + torrent=torrent_info, + ): + logger.debug( + f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且标题复核失败：{conflict_text}" + ) + return None + + evidence_matched, evidence = TorrentHelper.match_same_work_evidence( + target_mediainfo=target_mediainfo, + candidate_mediainfo=candidate_mediainfo, + torrent_meta=torrent_meta, + ) + if not evidence_matched: + logger.debug( + f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且缺少同作品证据：" + f"{evidence}；{conflict_text}" + ) + return None + + context.media_info = target_mediainfo + context.match_source = "title" + context.candidate_recognized = False + context.media_info_is_target = True + logger.debug( + f"{target_mediainfo.title_year} 候选媒体ID冲突（{conflict_text}），" + f"经标题或别名及{evidence}复核匹配到订阅目标：" + f"{torrent_info.site_name} - {torrent_info.title}" + ) + return target_mediainfo + def _prepare_subscription_match( owner: Any, - subscribe: Any, + subscribe: SubscriptionSnapshot, candidate_index: CandidateIndex, fresh_fact_lease: FreshFactLease, ) -> Optional[ @@ -109,7 +282,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase): if re_mediainfo: re_mediainfo.clear() context.media_info = re_mediainfo - context.match_source = self._SubscribeChain__get_media_id_match_source(re_mediainfo) + context.match_source = _get_media_id_match_source(re_mediainfo) context.candidate_recognized = bool(resolve_media_identity(media=re_mediainfo)[1]) context.media_info_is_target = False context.media_recognize_fail_count = 0 @@ -142,23 +315,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase): return if progress_callback: progress_callback(value=0, text="正在预处理订阅资源 ...") - return self._execute_match( + return self._run_match( torrents=torrents, progress_callback=progress_callback, ) - def match_batch( - self, - batch: CandidateBatch, - progress_callback: Optional[Callable[..., None]] = None, - ) -> None: - """消费包含完整缓存与本轮增量边界的订阅候选批次。""" - return self._execute_match( - torrents=batch.candidates, - progress_callback=progress_callback, - ) - - def _execute_match( + def _run_match( self, torrents: Dict[str, List[Context]], progress_callback: Optional[Callable[..., None]] = None, @@ -190,366 +352,442 @@ class SubscribeMatchOwner(_SubscribeOwnerBase): # 所有订阅 subscribes = self.subscription_repository.list(self.get_states_for_search("R")) total_num = len(subscribes) + summary = _MatchExecutionSummary(total=total_num) if progress_callback: - progress_callback( + _report_match_progress( + progress_callback, + summary, value=20, text=f"资源预处理完成，开始匹配 {total_num} 个订阅 ...", - data={"total": total_num, "finished": 0}, ) try: - for index, subscribe in enumerate(subscribes, start=1): + for index, listed_subscribe in enumerate(subscribes, start=1): if runtime_stop_state.is_system_stopped: break if progress_callback: - progress_callback( + _report_match_progress( + progress_callback, + summary, value=20 + ((index - 1) / total_num * 80 if total_num else 80), - text=(f"正在匹配订阅（{index}/{total_num}）{subscribe.name} ..."), - data={ - "total": total_num, - "finished": index - 1, - "current": subscribe.id, - }, + text=(f"正在匹配订阅（{index}/{total_num}）{listed_subscribe.name} ..."), + current=listed_subscribe.id, ) - logger.info(f"开始匹配订阅，标题：{subscribe.name} ...") - if subscribe.type == MediaType.MUSIC.value: - music_contexts = [context for contexts in processed_torrents.values() for context in contexts] - self._match_music_subscribe(subscribe, music_contexts) - continue - mediakey = subscribe_media_key(subscribe) - prepared = _prepare_subscription_match( - self, subscribe, candidate_index, fresh_fact_lease + outcome: Optional[str] = "skipped" + lease = self._subscription_execution_admission.try_acquire( + subscription_id=listed_subscribe.id, + operation="match", + ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL, ) - if prepared is None: - continue - meta, mediainfo, routed_torrents, domains, sub_sites = prepared - - # 如果媒体已存在或已下载完毕，跳过当前订阅处理 - exist_flag, no_exists = self.check_and_handle_existing_media( - subscribe=subscribe, meta=meta, mediainfo=mediainfo, mediakey=mediakey - ) - if exist_flag: - continue - - # 匹配前聚合多来源别名；清理大字段时保留匹配所需的标题合集。 - mediainfo = MediaChain().supplement_media_info(mediainfo) or mediainfo - auxiliary_names = list(getattr(mediainfo, "names", None) or []) - mediainfo.clear() - mediainfo.names = auxiliary_names - - # 订阅识别词 - if subscribe.custom_words: - custom_words_list = subscribe.custom_words.split("\n") + if lease is None: + logger.info(f"订阅 {listed_subscribe.name} 正在由其他通道处理，本轮匹配已跳过") else: - custom_words_list = None - - # 遍历预识别后的种子 - _match_context = [] - torrenthelper = TorrentHelper() - systemconfig = get_configured_system_config() - wordsmatcher = WordsMatcher() - for domain, contexts in routed_torrents.items(): - if runtime_stop_state.is_system_stopped: - break - if domains and domain not in domains: - continue - logger.debug(f"开始匹配站点：{domain}，共缓存了 {len(contexts)} 个种子...") - for context in contexts: - if runtime_stop_state.is_system_stopped: - break - # 提取信息 - _context = copy.copy(context) - torrent_meta = _context.meta_info - torrent_mediainfo = _context.media_info - torrent_info = _context.torrent_info - - # 不在订阅站点范围的不处理 - if sub_sites and torrent_info.site not in sub_sites: - logger.debug(f"{torrent_info.site_name} - {torrent_info.title} 不符合订阅站点要求") - continue - - # 有自定义识别词时，需要判断是否需要重新识别 - if custom_words_list: - # 使用org_string，应用一次后理论上不能再次应用 - _, apply_words = wordsmatcher.prepare( - torrent_meta.org_string, custom_words=custom_words_list - ) - if apply_words: - logger.info( - f"{torrent_info.site_name} - {torrent_info.title} 因订阅存在自定义识别词，重新识别元数据..." - ) - # 重新识别元数据 - torrent_meta = MetaInfo( - title=torrent_info.title, - subtitle=torrent_info.description, - custom_words=custom_words_list, - ) - # 更新元数据缓存 - _context.meta_info = torrent_meta - # 重新识别媒体信息 - torrent_mediainfo = MediaChain().recognize_by_meta( - torrent_meta, - episode_group=subscribe.episode_group, - obtain_images=False, - ) - if torrent_mediainfo: - # 清理多余信息 - torrent_mediainfo.clear() - # 更新种子缓存 - _context.media_info = torrent_mediainfo - _context.match_source = self._SubscribeChain__get_media_id_match_source( - torrent_mediainfo - ) - _context.candidate_recognized = bool( - resolve_media_identity(media=torrent_mediainfo)[1] - ) - _context.media_info_is_target = False - - # 如果仍然没有识别到媒体信息，尝试标题匹配 - if not torrent_mediainfo or not resolve_media_identity(media=torrent_mediainfo)[1]: - logger.debug( - f"{torrent_info.site_name} - {torrent_info.title} 重新识别失败，尝试通过标题匹配..." - ) - if not self._SubscribeChain__is_title_match_allowed( - mediainfo=mediainfo, - torrent_meta=torrent_meta, - torrent_info=torrent_info, - ): - continue - # 匹配成功 + current_subscribe = None + execution_context = SubscriptionExecutionContext( + lease=lease, + admission=self._subscription_execution_admission, + cancel_requested=lambda: runtime_stop_state.is_system_stopped, + ) + try: + current_subscribe = self.subscription_repository.get(listed_subscribe.id) + if current_subscribe is None: + logger.info(f"订阅 {listed_subscribe.id} 已不存在，本轮匹配跳过") + elif current_subscribe.state not in {"R", "P"}: logger.info( - f"{mediainfo.title_year} 通过标题匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}" - ) - torrent_mediainfo = mediainfo - # 更新种子缓存 - _context.media_info = mediainfo - _context.match_source = "title" - _context.candidate_recognized = False - _context.media_info_is_target = True - - # 直接比对媒体信息 - if torrent_mediainfo and resolve_media_identity(media=torrent_mediainfo)[1]: - if torrent_mediainfo.type != mediainfo.type: - continue - torrent_mediainfo = self._SubscribeChain__reconcile_candidate_media( - target_mediainfo=mediainfo, - candidate_mediainfo=torrent_mediainfo, - torrent_meta=torrent_meta, - torrent_info=torrent_info, - context=_context, - ) - if not torrent_mediainfo: - continue - match_source = _context.match_source - if match_source == "title": - # 标题兜底使用的是订阅目标 media_info，不能标记为候选自身识别结果。 - _context.candidate_recognized = False - _context.media_info_is_target = True - match_label = "标题复核" - elif match_source == "unknown": - _context.match_source = self._SubscribeChain__get_media_id_match_source( - torrent_mediainfo - ) - _context.candidate_recognized = True - _context.media_info_is_target = False - match_label = "媒体ID" - else: - _context.candidate_recognized = True - _context.media_info_is_target = False - match_label = "媒体ID" - logger.info( - f"{mediainfo.title_year} 通过{match_label}匹配到可选资源：" - f"{torrent_info.site_name} - {torrent_info.title}" + f"订阅 {current_subscribe.name} 当前状态为 {current_subscribe.state}，本轮匹配跳过" ) + elif execution_context.should_stop(): + logger.info(f"订阅 {current_subscribe.name} 已取消或超时，本轮匹配跳过") else: - continue - - # 如果是电视剧 - if torrent_mediainfo.type == MediaType.TV: - # 有多季的不要 - if len(torrent_meta.season_list) > 1: - logger.debug(f"{torrent_info.title} 有多季，不处理") - continue - # 比对季 - if torrent_meta.begin_season is not None: - if meta.begin_season != torrent_meta.begin_season: - logger.debug(f"{torrent_info.title} 季不匹配") - continue - elif meta.begin_season != 1: - logger.debug(f"{torrent_info.title} 季不匹配") - continue - # 非洗版 - if not subscribe.best_version: - # 不是缺失的剧集不要 - if no_exists and no_exists.get(mediakey): - # 缺失集 - no_exists_info = no_exists.get(mediakey).get(subscribe.season) - if no_exists_info: - # 是否有交集 - if ( - no_exists_info.episodes - and torrent_meta.episode_list - and not set(no_exists_info.episodes).intersection( - set(torrent_meta.episode_list) - ) - ): - logger.debug( - f"{torrent_info.title} 对应剧集 {torrent_meta.episode_list} 未包含缺失的剧集" - ) - continue - else: - if not self._SubscribeChain__is_full_season_best_version_resource( - meta=torrent_meta, - subscribe=subscribe, - ): - logger.debug( - f"{subscribe.name} 正在全集洗版，{torrent_info.title} 不是全集资源" - ) - continue - # 洗版时，不符合订阅集数的不要 - if meta.type == MediaType.TV and not self._is_episode_range_covered( - meta=torrent_meta, - subscribe=subscribe, - ): - logger.debug( - f"{subscribe.name} 正在洗版，{torrent_info.title} 不符合订阅集数范围" - ) - continue - - # 匹配订阅附加参数 - if not torrenthelper.filter_torrent( - torrent_info=torrent_info, filter_params=self.get_params(subscribe) - ): - continue - - # 优先级过滤规则 - if subscribe.best_version: - rule_groups = subscribe.filter_groups or systemconfig.get( - SystemConfigKey.BestVersionFilterRuleGroups + outcome = self._match_subscription( + subscribe=current_subscribe, + processed_torrents=processed_torrents, + candidate_index=candidate_index, + fresh_fact_lease=fresh_fact_lease, + execution_context=execution_context, ) - else: - rule_groups = subscribe.filter_groups or systemconfig.get( - SystemConfigKey.SubscribeFilterRuleGroups - ) - result: List[TorrentInfo] = self.filter_torrents( - rule_groups=rule_groups, torrent_list=[torrent_info], mediainfo=torrent_mediainfo + except SubscriptionSearchCancelled as err: + outcome = "skipped" + logger.info(f"订阅 {listed_subscribe.name} 匹配已取消：{str(err)}") + except Exception as err: + outcome = "failed" + subscribe_name = ( + current_subscribe.name if current_subscribe is not None else listed_subscribe.name ) - if result is not None and not result: - # 不符合过滤规则 - logger.debug(f"{torrent_info.title} 不匹配过滤规则") - continue - - # 洗版时，优先级小于已下载优先级的不要 - if subscribe.best_version: - if meta.type == MediaType.TV: - if not self._SubscribeChain__prepare_best_version_tv_candidate( - subscribe=subscribe, - context=_context, - priority=torrent_info.pri_order, - ): - logger.info( - f"{subscribe.name} 正在洗版，{torrent_info.title} " - f"优先级未达到当前模式的升级条件" - ) - continue - if ( - meta.type != MediaType.TV - and subscribe.current_priority - and torrent_info.pri_order <= subscribe.current_priority - ): - logger.info( - f"{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级" - ) - continue - - # 匹配成功 - logger.info(f"{mediainfo.title_year} 匹配成功：{torrent_info.title}") - # 自定义属性 - if subscribe.media_category: - torrent_mediainfo.category = subscribe.media_category - if subscribe.episode_group: - torrent_mediainfo.episode_group = subscribe.episode_group - _match_context.append(_context) - - if not _match_context: - # 未匹配到资源 - logger.info(f"{mediainfo.title_year} 未匹配到符合条件的资源") - self.finish_subscribe_or_not( - subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists - ) - continue - - # 开始批量择优下载 - logger.info(f"{mediainfo.title_year} 匹配完成，共匹配到{len(_match_context)}个资源") - downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first( - contexts=_match_context, - no_exists=no_exists, - subscribe=subscribe, - mediakey=mediakey, - username=subscribe.username, - save_path=subscribe.save_path, - downloader=subscribe.downloader, - source=self.get_subscribe_source_keyword(subscribe), + logger.error(f"订阅 {subscribe_name} 匹配失败：{str(err)}", exc_info=True) + finally: + self._subscription_execution_admission.release(lease) + summary.record(outcome) + _report_match_progress( + progress_callback, + summary, + value=20 + (summary.finished / total_num * 80 if total_num else 80), + text=(f"订阅匹配（{index}/{total_num}）处理完成"), ) - - # 同步外部修改，更新订阅信息 - updated_subscribe = self.subscription_repository.get(subscribe.id) - - # 判断是否要完成订阅 - if updated_subscribe: - self.finish_subscribe_or_not( - subscribe=updated_subscribe, - meta=meta, - mediainfo=mediainfo, - downloads=downloads, - lefts=lefts, - ) finally: processed_torrents.clear() del processed_torrents subscribes.clear() del subscribes if progress_callback: - progress_callback( + final_text = "订阅资源匹配完成" + if summary.finished < summary.total: + final_text = "订阅资源匹配已停止，部分订阅未执行" + elif summary.failed: + final_text = "订阅资源匹配完成，部分订阅失败" + elif summary.skipped: + final_text = "订阅资源匹配完成，部分订阅跳过" + _report_match_progress( + progress_callback, + summary, value=100, - text="订阅资源匹配完成", - data={"total": total_num, "finished": total_num}, + text=final_text, ) finally: if lock_acquired: - self._rlock.release() + self._match_lock.release() logger.debug(f"match Lock released at {datetime.now()}") - @staticmethod - def _SubscribeChain__get_media_id_match_source(mediainfo: Optional[MediaInfo]) -> str: - """ - 返回候选自身识别命中的明确媒体 ID 类型。 - """ - media_source, media_id = resolve_media_identity(media=mediainfo) - if media_source and media_id: - return str(media_source) - return "unknown" - @staticmethod - def _SubscribeChain__is_title_match_allowed( - mediainfo: MediaInfo, - torrent_meta: MetaBase, - torrent_info: TorrentInfo, - ) -> bool: - """判断资源是否可以通过标题兜底匹配订阅目标。""" - if not TorrentHelper.match_torrent( - mediainfo=mediainfo, - torrent_meta=torrent_meta, - torrent=torrent_info, - ): - return False - if not TorrentHelper.requires_identity_disambiguation( - mediainfo=mediainfo, - torrent_meta=torrent_meta, - ): - return True - logger.info( - f"{torrent_info.site_name} - {torrent_info.title} 仅通过无年份别名命中且候选媒体身份无法确认，已跳过" + def _match_subscription( + self, + *, + subscribe: SubscriptionSnapshot, + processed_torrents: Dict[str, List[Context]], + candidate_index: CandidateIndex, + fresh_fact_lease: FreshFactLease, + execution_context: SubscriptionExecutionContext, + ) -> _MatchOutcome: + """在订阅准入边界调用单订阅 Match 执行。""" + return self._execute_match( + subscribe=subscribe, + processed_torrents=processed_torrents, + candidate_index=candidate_index, + fresh_fact_lease=fresh_fact_lease, + execution_context=execution_context, ) - return False + + def _execute_match( + self, + *, + subscribe: SubscriptionSnapshot, + processed_torrents: Dict[str, List[Context]], + candidate_index: CandidateIndex, + fresh_fact_lease: FreshFactLease, + execution_context: SubscriptionExecutionContext, + ) -> _MatchOutcome: + """在单条订阅准入期间完成匹配、下载和完成判定。""" + if execution_context.should_stop(): + return "skipped" + logger.info(f"开始匹配订阅，标题：{subscribe.name} ...") + if subscribe.type == MediaType.MUSIC.value: + music_contexts = [context for contexts in processed_torrents.values() for context in contexts] + execution_context.report_phase("matching") + if not execution_context.should_stop(): + self._match_music_subscribe( + subscribe, + music_contexts, + execution_context=execution_context, + ) + return "completed" + return "skipped" + mediakey = subscribe_media_key(subscribe) + prepared = _prepare_subscription_match( + self, subscribe, candidate_index, fresh_fact_lease + ) + if prepared is None: + return "completed" + if execution_context.should_stop(): + return "skipped" + meta, mediainfo, routed_torrents, domains, sub_sites = prepared + + # 如果媒体已存在或已下载完毕，跳过当前订阅处理 + exist_flag, no_exists = self.check_and_handle_existing_media( + subscribe=subscribe, meta=meta, mediainfo=mediainfo, mediakey=mediakey + ) + if exist_flag: + return "completed" + + # 匹配前聚合多来源别名；清理大字段时保留匹配所需的标题合集。 + mediainfo = MediaChain().supplement_media_info(mediainfo) or mediainfo + auxiliary_names = list(getattr(mediainfo, "names", None) or []) + mediainfo.clear() + mediainfo.names = auxiliary_names + + # 订阅识别词 + if subscribe.custom_words: + custom_words_list = subscribe.custom_words.split("\n") + else: + custom_words_list = None + + # 遍历预识别后的种子 + _match_context = [] + torrenthelper = TorrentHelper() + systemconfig = get_configured_system_config() + wordsmatcher = WordsMatcher() + execution_context.report_phase("matching") + for domain, contexts in routed_torrents.items(): + if runtime_stop_state.is_system_stopped or execution_context.should_stop(): + break + if domains and domain not in domains: + continue + logger.debug(f"开始匹配站点：{domain}，共缓存了 {len(contexts)} 个种子...") + for context in contexts: + if runtime_stop_state.is_system_stopped or execution_context.should_stop(): + break + # 提取信息 + _context = copy.copy(context) + torrent_meta = _context.meta_info + torrent_mediainfo = _context.media_info + torrent_info = _context.torrent_info + + # 不在订阅站点范围的不处理 + if sub_sites and torrent_info.site not in sub_sites: + logger.debug(f"{torrent_info.site_name} - {torrent_info.title} 不符合订阅站点要求") + continue + + # 有自定义识别词时，需要判断是否需要重新识别 + if custom_words_list: + # 使用org_string，应用一次后理论上不能再次应用 + _, apply_words = wordsmatcher.prepare( + torrent_meta.org_string, custom_words=custom_words_list + ) + if apply_words: + logger.info( + f"{torrent_info.site_name} - {torrent_info.title} 因订阅存在自定义识别词，重新识别元数据..." + ) + # 重新识别元数据 + torrent_meta = MetaInfo( + title=torrent_info.title, + subtitle=torrent_info.description, + custom_words=custom_words_list, + ) + # 更新元数据缓存 + _context.meta_info = torrent_meta + # 重新识别媒体信息 + torrent_mediainfo = MediaChain().recognize_by_meta( + torrent_meta, + episode_group=subscribe.episode_group, + obtain_images=False, + ) + if torrent_mediainfo: + # 清理多余信息 + torrent_mediainfo.clear() + # 更新种子缓存 + _context.media_info = torrent_mediainfo + _context.match_source = _get_media_id_match_source( + torrent_mediainfo + ) + _context.candidate_recognized = bool( + resolve_media_identity(media=torrent_mediainfo)[1] + ) + _context.media_info_is_target = False + + # 如果仍然没有识别到媒体信息，尝试标题匹配 + if not torrent_mediainfo or not resolve_media_identity(media=torrent_mediainfo)[1]: + logger.debug( + f"{torrent_info.site_name} - {torrent_info.title} 重新识别失败，尝试通过标题匹配..." + ) + if not _is_title_match_allowed( + mediainfo=mediainfo, + torrent_meta=torrent_meta, + torrent_info=torrent_info, + ): + continue + # 匹配成功 + logger.info( + f"{mediainfo.title_year} 通过标题匹配到可选资源：{torrent_info.site_name} - {torrent_info.title}" + ) + torrent_mediainfo = mediainfo + # 更新种子缓存 + _context.media_info = mediainfo + _context.match_source = "title" + _context.candidate_recognized = False + _context.media_info_is_target = True + + # 直接比对媒体信息 + if torrent_mediainfo and resolve_media_identity(media=torrent_mediainfo)[1]: + if torrent_mediainfo.type != mediainfo.type: + continue + torrent_mediainfo = self._SubscribeChain__reconcile_candidate_media( + target_mediainfo=mediainfo, + candidate_mediainfo=torrent_mediainfo, + torrent_meta=torrent_meta, + torrent_info=torrent_info, + context=_context, + ) + if not torrent_mediainfo: + continue + match_source = _context.match_source + if match_source == "title": + # 标题兜底使用的是订阅目标 media_info，不能标记为候选自身识别结果。 + _context.candidate_recognized = False + _context.media_info_is_target = True + match_label = "标题复核" + elif match_source == "unknown": + _context.match_source = _get_media_id_match_source( + torrent_mediainfo + ) + _context.candidate_recognized = True + _context.media_info_is_target = False + match_label = "媒体ID" + else: + _context.candidate_recognized = True + _context.media_info_is_target = False + match_label = "媒体ID" + logger.info( + f"{mediainfo.title_year} 通过{match_label}匹配到可选资源：" + f"{torrent_info.site_name} - {torrent_info.title}" + ) + else: + continue + + # 如果是电视剧 + if torrent_mediainfo.type == MediaType.TV: + # 有多季的不要 + if len(torrent_meta.season_list) > 1: + logger.debug(f"{torrent_info.title} 有多季，不处理") + continue + # 比对季 + if torrent_meta.begin_season is not None: + if meta.begin_season != torrent_meta.begin_season: + logger.debug(f"{torrent_info.title} 季不匹配") + continue + elif meta.begin_season != 1: + logger.debug(f"{torrent_info.title} 季不匹配") + continue + # 非洗版 + if not subscribe.best_version: + # 不是缺失的剧集不要 + if no_exists and no_exists.get(mediakey): + # 缺失集 + no_exists_info = no_exists.get(mediakey).get(subscribe.season) + if no_exists_info: + # 是否有交集 + if ( + no_exists_info.episodes + and torrent_meta.episode_list + and not set(no_exists_info.episodes).intersection( + set(torrent_meta.episode_list) + ) + ): + logger.debug( + f"{torrent_info.title} 对应剧集 {torrent_meta.episode_list} 未包含缺失的剧集" + ) + continue + else: + if not self._SubscribeChain__is_full_season_best_version_resource( + meta=torrent_meta, + subscribe=subscribe, + ): + logger.debug( + f"{subscribe.name} 正在全集洗版，{torrent_info.title} 不是全集资源" + ) + continue + # 洗版时，不符合订阅集数的不要 + if meta.type == MediaType.TV and not self._is_episode_range_covered( + meta=torrent_meta, + subscribe=subscribe, + ): + logger.debug( + f"{subscribe.name} 正在洗版，{torrent_info.title} 不符合订阅集数范围" + ) + continue + + # 匹配订阅附加参数 + if not torrenthelper.filter_torrent( + torrent_info=torrent_info, filter_params=self.get_params(subscribe) + ): + continue + + # 优先级过滤规则 + if subscribe.best_version: + rule_groups = subscribe.filter_groups or systemconfig.get( + SystemConfigKey.BestVersionFilterRuleGroups + ) + else: + rule_groups = subscribe.filter_groups or systemconfig.get( + SystemConfigKey.SubscribeFilterRuleGroups + ) + result: List[TorrentInfo] = self.filter_torrents( + rule_groups=rule_groups, torrent_list=[torrent_info], mediainfo=torrent_mediainfo + ) + if result is not None and not result: + # 不符合过滤规则 + logger.debug(f"{torrent_info.title} 不匹配过滤规则") + continue + + # 洗版时，优先级小于已下载优先级的不要 + if subscribe.best_version: + if meta.type == MediaType.TV: + if not self._SubscribeChain__prepare_best_version_tv_candidate( + subscribe=subscribe, + context=_context, + priority=torrent_info.pri_order, + ): + logger.info( + f"{subscribe.name} 正在洗版，{torrent_info.title} " + f"优先级未达到当前模式的升级条件" + ) + continue + if ( + meta.type != MediaType.TV + and subscribe.current_priority + and torrent_info.pri_order <= subscribe.current_priority + ): + logger.info( + f"{subscribe.name} 正在洗版，{torrent_info.title} 优先级低于或等于已下载优先级" + ) + continue + + # 匹配成功 + logger.info(f"{mediainfo.title_year} 匹配成功：{torrent_info.title}") + # 自定义属性 + if subscribe.media_category: + torrent_mediainfo.category = subscribe.media_category + if subscribe.episode_group: + torrent_mediainfo.episode_group = subscribe.episode_group + _match_context.append(_context) + + if execution_context.should_stop(): + return "skipped" + + if not _match_context: + # 未匹配到资源 + logger.info(f"{mediainfo.title_year} 未匹配到符合条件的资源") + self.finish_subscribe_or_not( + subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists + ) + return "completed" + + if execution_context.should_stop(): + return "skipped" + + # 开始批量择优下载 + logger.info(f"{mediainfo.title_year} 匹配完成，共匹配到{len(_match_context)}个资源") + downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first( + contexts=_match_context, + no_exists=no_exists, + subscribe=subscribe, + mediakey=mediakey, + username=subscribe.username, + save_path=subscribe.save_path, + downloader=subscribe.downloader, + source=self.get_subscribe_source_keyword(subscribe), + execution_context=execution_context, + ) + + # 同步外部修改，更新订阅信息 + updated_subscribe = self.subscription_repository.get(subscribe.id) + + # 判断是否要完成订阅 + if updated_subscribe: + self.finish_subscribe_or_not( + subscribe=updated_subscribe, + meta=meta, + mediainfo=mediainfo, + downloads=downloads, + lefts=lefts, + ) + return "completed" @staticmethod def _SubscribeChain__reconcile_candidate_media( @@ -559,80 +797,18 @@ class SubscribeMatchOwner(_SubscribeOwnerBase): torrent_info: TorrentInfo, context: Context, ) -> Optional[MediaInfo]: - """ - 在推断媒体 ID 冲突时，以订阅目标对候选标题做严格复核。 - - 标题携带明确媒体身份或目标缺少同来源 ID 时保持严格拒绝；只有普通 - 标题推断产生冲突且标题、别名、类型和年份复核通过时才回填订阅目标。 - - :param target_mediainfo: 订阅目标媒体信息 - :param candidate_mediainfo: RSS 或首页候选的识别结果 - :param torrent_meta: 候选标题解析信息 - :param torrent_info: 候选种子信息 - :param context: 当前订阅使用的候选上下文副本 - :return: 可继续匹配的媒体信息，拒绝时返回 None - """ - candidate_source, candidate_id = resolve_media_identity(media=candidate_mediainfo) - target_source, target_id = resolve_media_identity(media=target_mediainfo) - conflicts = [] - if candidate_source and candidate_id and (candidate_source != target_source or candidate_id != target_id): - conflicts.append( - ( - str(candidate_source), - candidate_id, - f"{target_source}:{target_id}" if target_source and target_id else None, - ) - ) - if not conflicts: - return candidate_mediainfo - - conflict_text = "、".join( - f"{source} {candidate_id} != {target_id if target_id is not None else '缺失'}" - for source, candidate_id, target_id in conflicts - ) - if any(target_id is None for _, _, target_id in conflicts): - logger.debug( - f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID与订阅目标不可比较：{conflict_text}" - ) - return None - - explicit_source, explicit_id = resolve_media_identity(media=torrent_meta) - if explicit_source and explicit_id: - logger.debug( - f"{torrent_info.site_name} - {torrent_info.title} 标题含明确媒体ID " - f"{explicit_source}:{explicit_id}，保持严格校验：{conflict_text}" - ) - return None - - if not TorrentHelper.match_torrent( - mediainfo=target_mediainfo, - torrent_meta=torrent_meta, - torrent=torrent_info, - ): - logger.debug( - f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且标题复核失败：{conflict_text}" - ) - return None - - evidence_matched, evidence = TorrentHelper.match_same_work_evidence( + """保留组合宿主的稳定私有调用入口。""" + return _reconcile_candidate_media( target_mediainfo=target_mediainfo, candidate_mediainfo=candidate_mediainfo, torrent_meta=torrent_meta, + torrent_info=torrent_info, + context=context, ) - if not evidence_matched: - logger.debug( - f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且缺少同作品证据：" - f"{evidence}；{conflict_text}" - ) - return None - context.media_info = target_mediainfo - context.match_source = "title" - context.candidate_recognized = False - context.media_info_is_target = True - logger.debug( - f"{target_mediainfo.title_year} 候选媒体ID冲突（{conflict_text}），" - f"经标题或别名及{evidence}复核匹配到订阅目标：" - f"{torrent_info.site_name} - {torrent_info.title}" - ) - return target_mediainfo + @staticmethod + def _SubscribeChain__get_media_id_match_source( + mediainfo: Optional[MediaInfo], + ) -> str: + """保留组合宿主解析媒体 ID 匹配来源的稳定入口。""" + return _get_media_id_match_source(mediainfo) diff --git a/app/chain/subscribe/policy.py b/app/chain/subscribe/policy.py index 4d32462a2..d56679cf5 100644 --- a/app/chain/subscribe/policy.py +++ b/app/chain/subscribe/policy.py @@ -8,6 +8,7 @@ from app.application.subscription import priority as _priority from app.application.subscription.contract import ( SubscriptionSnapshot, ) +from app.application.subscription.execution import SubscriptionExecutionContext from app.chain.download import DownloadChain from app.chain.subscribe.contract import _SubscribeOwnerBase from app.domain.context import ( @@ -281,6 +282,7 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner): save_path: Optional[str] = None, downloader: Optional[str] = None, source: Optional[str] = None, + execution_context: Optional[SubscriptionExecutionContext] = None, ) -> Tuple[List[Context], Dict[Union[int, str], Dict[int, _SchemaNotExistMediaInfo]]]: """ TV 分集洗版先尝试覆盖目标范围的全集资源，失败后回退到按集下载。 @@ -327,17 +329,8 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner): downloader = current.downloader source = self.get_subscribe_source_keyword(current) governance = SubscriptionDownloadGovernance( - subscription_id=current.id, - mode=self._SubscribeChain__download_governance_mode(current), - task_id=cast(Optional[str], getattr(self, "_subscription_download_task_id", None)), - cancelled=cast( - Optional[Callable[[], bool]], - getattr(self, "_subscription_download_cancelled", None), - ), - mark_started=cast( - Optional[Callable[[], None]], - getattr(self, "_subscription_download_mark_started", None), - ), + cancelled=execution_context.should_stop if execution_context else None, + mark_started=execution_context.mark_download_started if execution_context else None, ) full_pack_no_exists = self._SubscribeChain__build_full_pack_first_no_exists( subscribe=subscribe, mediakey=mediakey @@ -465,17 +458,6 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner): accepted.append(context) return accepted - @staticmethod - def _SubscribeChain__download_governance_mode(subscribe: SubscriptionSnapshot) -> str: - """把订阅模式归一为稳定幂等键组成部分。""" - if not subscribe.best_version: - return "normal" - if subscribe.type == MediaType.TV.value and subscribe.best_version_full: - return "best_version_full" - if subscribe.type == MediaType.TV.value: - return "best_version_episode" - return "best_version" - @classmethod def _is_episode_range_covered(cls, meta: MetaBase, subscribe: SubscriptionSnapshot) -> bool: """ diff --git a/app/chain/subscribe/refresh.py b/app/chain/subscribe/refresh.py index e190ba43e..2edb0838c 100644 --- a/app/chain/subscribe/refresh.py +++ b/app/chain/subscribe/refresh.py @@ -5,7 +5,6 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from app.application.subscription import priority as _priority -from app.application.subscription.candidates import CandidateBatch from app.application.subscription.contract import ( SubscriptionSnapshot, build_subscribe_meta, @@ -78,21 +77,14 @@ class SubscribeRefreshOwner(SubscribeMetadataOwner): ) torrents_chain = TorrentsChain() - candidate_batch = torrents_chain.refresh_batch( + candidates = torrents_chain.refresh( sites=sites, progress_callback=_update_refresh_progress if progress_callback else None, # 存在音乐订阅时额外抓取站点音乐专用入口，音乐不一定在默认种子首页 include_music=self.has_music_subscribe(), ) - if not isinstance(candidate_batch, CandidateBatch): - legacy_candidates = torrents_chain.refresh( - sites=sites, - progress_callback=_update_refresh_progress if progress_callback else None, - include_music=self.has_music_subscribe(), - ) - candidate_batch = CandidateBatch.from_legacy(legacy_candidates or {}, source="refresh") - self.match_batch( - candidate_batch, + self.match( + candidates, progress_callback=_update_match_progress if progress_callback else None, ) if progress_callback: diff --git a/app/chain/subscribe/search.py b/app/chain/subscribe/search.py index f728ca8fb..fd9160b80 100644 --- a/app/chain/subscribe/search.py +++ b/app/chain/subscribe/search.py @@ -14,7 +14,12 @@ from app.application.subscription.contract import ( build_subscribe_meta, subscribe_media_key, ) -from app.application.subscription.execution import SearchBatchSnapshot, SubscriptionSearchRepository +from app.application.subscription.execution import ( + SearchBatchSnapshot, + SearchTaskSnapshot, + SubscriptionExecutionContext, + SubscriptionSearchRepository, +) from app.application.subscription.query import SubscriptionQueryService from app.application.subscription.sitebudget import ( SubscriptionSearchCancelled, @@ -36,6 +41,198 @@ from app.schemas.types import ( ) +def _ensure_execution_active( + execution_context: Optional[SubscriptionExecutionContext], +) -> None: + """在可安全停止的边界区分用户取消与执行超时。""" + if execution_context is None: + return + if execution_context.is_cancel_requested(): + raise SubscriptionSearchCancelled("订阅搜索已取消") + if execution_context.is_expired(): + raise TimeoutError("订阅执行已超过协作截止时间") + + +def _update_search_task_phase( + queue: SubscriptionSearchRepository, + task_id: str, + lease_token: str, + phase: str, + current_site_id: Optional[int] = None, +) -> None: + """以当前任务租约持久化业务阶段，过期执行者不得覆盖新状态。""" + queue.update_task_phase( + task_id=task_id, + lease_token=lease_token, + phase=phase, + current_site_id=current_site_id, + ) + + +def _search_source_and_priority( + *, + sid: Optional[int], + sids: Optional[tuple[int, ...]], + state: Optional[str], + manual: Optional[bool], +) -> tuple[str, int]: + """把兼容入口归一为持久来源和公平队列优先级。""" + if manual: + return "manual", 100 + if sid or sids is not None: + return "targeted", 80 + if state in {"R", "P"}: + return "fallback", 10 + return "new", 50 + + +def _search_task_available_at( + source: str, + subscription_ids: tuple[int, ...], + *, + now: Optional[datetime] = None, +) -> dict[int, str]: + """把兜底搜索的随机节奏持久化为逐订阅到期时间。""" + ordered_ids = tuple(dict.fromkeys(subscription_ids)) + if not ordered_ids: + return {} + cursor = now or datetime.now(timezone.utc) + if source == "fallback": + cursor += timedelta(seconds=random.randint(0, 60)) + available_at: dict[int, str] = {} + for position, subscription_id in enumerate(ordered_ids): + if source == "fallback" and position: + cursor += timedelta(seconds=random.randint(60, 300)) + available_at[subscription_id] = cursor.isoformat(timespec="seconds") + return available_at + + +def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str: + """把批次聚合终态转为兼容进度文案。""" + if batch is None: + return "订阅搜索任务已提交" + if batch.state == "failed": + return "订阅搜索完成，部分任务失败" + if batch.state == "cancelled": + return "订阅搜索已取消" + if batch.state == "skipped": + return "订阅搜索完成，部分任务本轮已跳过" + if batch.state in {"queued", "running", "cancelling"}: + return "订阅搜索任务已排队" + if batch.skipped_count: + return "订阅搜索完成，部分任务本轮已跳过" + return "订阅搜索完成" + + +def _inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]: + """返回兼容搜索的真实终态文案与计数。""" + text = ( + "订阅搜索完成" + if finished == total + else "订阅搜索结束，部分订阅本轮未执行或未完成" + ) + return text, {"total": total, "finished": finished} + + +def _batch_finished_count( + batch: Optional[SearchBatchSnapshot], + fallback: int, +) -> int: + """返回批次所有终态任务数；批次暂不可读时使用本轮实际完成数。""" + if batch is None: + return fallback + return ( + batch.finished_count + + batch.failed_count + + batch.cancelled_count + + batch.skipped_count + ) + + +def _skip_search_task( + queue: SubscriptionSearchRepository, + task: SearchTaskSnapshot, + reason: str, +) -> bool: + """以 skipped 终态收口未执行任务，并保留可见原因。""" + if task.lease_token is None: + return False + return queue.finish_task( + task_id=task.task_id, + lease_token=task.lease_token, + state="skipped", + error=reason, + ) + + +def _release_cancelled_or_stopped_search_task( + queue: SubscriptionSearchRepository, + task_id: str, + lease_token: str, + system_stopped: bool, +) -> bool: + """用户取消落终态，系统停机仅释放任务以供重启恢复。""" + return queue.release_task( + task_id=task_id, + lease_token=lease_token, + cancelled=not system_stopped, + ) + + +def _finish_returned_search_task( + *, + queue: SubscriptionSearchRepository, + task_id: str, + lease_token: str, + subscription_id: int, + execution_context: SubscriptionExecutionContext, + system_stopped: bool, + cancel_requested: bool, +) -> Optional[int]: + """按 TTL、停机和取消边界收口正常返回的搜索任务。""" + download_started = execution_context.download_started + if execution_context.is_expired() and not (system_stopped or cancel_requested): + if download_started: + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="completed", + error="执行截止时间晚于下载提交边界，已按实际结果完成", + ) + return subscription_id + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="failed", + error="订阅执行已超过协作截止时间", + ) + return None + if system_stopped: + if download_started: + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="completed", + error="停机请求晚于下载提交边界，已按实际结果完成", + ) + return subscription_id + queue.release_task(task_id=task_id, lease_token=lease_token) + return None + if cancel_requested: + if download_started: + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="completed", + error="取消请求晚于下载提交边界，已按实际结果完成", + ) + return subscription_id + queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True) + return None + queue.finish_task(task_id=task_id, lease_token=lease_token, state="completed") + return subscription_id + + class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): """订阅主动搜索入口与持久队列消费 owner。""" @@ -64,8 +261,9 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): operation: str, progress_callback: Optional[Callable[..., None]], ) -> bool: - """获取订阅任务锁，超时时统一记录并结束本轮进度。""" - if self._rlock.acquire(blocking=True, timeout=self._LOCK_TIMOUT): + """获取 Search 或 Match 通道锁，保持各通道内部串行。""" + lock = self._match_lock if operation == "match" else self._search_queue_lock + if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL): logger.debug(f"{operation} lock acquired at {datetime.now()}") return True operation_label = {"search": "搜索", "match": "匹配"}[operation] @@ -148,7 +346,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): manual: Optional[bool], progress_callback: Optional[Callable[..., None]], ) -> None: - """保留未注入持久队列宿主的旧串行锁语义。""" + """在独立 Search 通道内按订阅准入执行兼容搜索。""" lock_acquired = self._acquire_run_lock("search", progress_callback) if not lock_acquired: return @@ -167,28 +365,70 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): for index, subscribe in enumerate(subscribes, start=1): if runtime_stop_state.is_system_stopped: break - processed.append(subscribe) self._report_search_progress(progress_callback, subscribe, index, total) if self._defer_recent_subscription(subscribe): continue self._wait_before_scheduled_search(sid, sids, state, progress_callback) - current = subscribe + lease = self._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="search", + ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL, + ) + if lease is None: + logger.info(f"订阅 {subscribe.name} 正在由其他通道处理，本轮搜索已跳过") + continue + execution_context = SubscriptionExecutionContext( + lease=lease, + admission=self._subscription_execution_admission, + cancel_requested=lambda: runtime_stop_state.is_system_stopped, + ) + current = None try: - current = self._process_search_subscription(subscribe, searchchain) + current = self.subscription_repository.get(subscribe.id) + if current is None or current.state == "S": + if current and current.state == "S": + logger.info(f"订阅 {current.name} 已暂停，本轮搜索已跳过") + continue + processed_result = self._process_search_subscription( + current, + searchchain, + execution_context=execution_context, + ) + processed.append(processed_result or current) + current = processed_result + except SubscriptionSearchCancelled: + logger.info(f"订阅 {subscribe.name} 搜索已在安全边界取消") + except Exception as err: + logger.error(f"订阅 {subscribe.name} 搜索失败：{str(err)}", exc_info=True) finally: - if current and current.state == "N": - self._SubscribeChain__apply_subscribe_update( - current, - {"state": "R"}, - scene="search_reset", + try: + if current and current.state == "N": + self._SubscribeChain__apply_subscribe_update( + current, + {"state": "R"}, + scene="search_reset", + ) + except Exception as err: + logger.error( + f"订阅 {subscribe.name} 搜索后状态重置失败：{str(err)}", + exc_info=True, + ) + finally: + self._subscription_execution_admission.release(lease) + self._report_search_progress( + progress_callback, + subscribe, + index, + total, + finished=True, ) - self._report_search_progress(progress_callback, subscribe, index, total, finished=True) self._notify_manual_search(manual, sid, sids, subscribes, processed) if progress_callback: - progress_callback(value=100, text="订阅搜索完成") + text, data = _inline_search_result(total, len(processed)) + progress_callback(value=100, text=text, data=data) finally: subscribes.clear() - self._rlock.release() + self._search_queue_lock.release() logger.debug(f"search Lock released at {datetime.now()}") def _execute_queued_search( @@ -203,17 +443,21 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): ) -> str: """将搜索转为持久任务并在无 Match 长锁的短租约中串行消费。""" subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state) - source, priority = self._search_source_and_priority( + source, priority = _search_source_and_priority( sid=sid, sids=sids, state=state, manual=manual, ) + subscription_ids = tuple(subscribe.id for subscribe in subscribes) enqueued = queue.enqueue( - subscription_ids=tuple(subscribe.id for subscribe in subscribes), + subscription_ids=subscription_ids, source=source, priority=priority, - available_at=self._search_batch_available_at(source), + available_at_by_subscription=_search_task_available_at( + source, + subscription_ids, + ), ) total = len(subscribes) if progress_callback: @@ -238,11 +482,11 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): batch = queue.get_batch(enqueued.batch.batch_id) progress_callback( value=100, - text=self._batch_progress_text(batch), + text=_batch_progress_text(batch), data={ "batch_id": enqueued.batch.batch_id, "total": total, - "finished": len(processed_subscribes), + "finished": _batch_finished_count(batch, len(processed_subscribes)), "coalesced": enqueued.coalesced_count, }, ) @@ -256,8 +500,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): progress_callback: Optional[Callable[..., None]], ) -> set[int]: """有界消费可恢复任务；单任务失败不得阻止后续订阅。""" - queue_lock = getattr(self, "_search_queue_lock", None) - if queue_lock is not None and not queue_lock.acquire(blocking=False): + if not self._search_queue_lock.acquire(blocking=False): logger.debug("订阅搜索队列已有消费者，本轮仅保留持久任务") return set() owner = f"subscribe-search:{uuid4().hex}" @@ -270,147 +513,183 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): task = queue.claim_next(owner=owner) if task is None: break - if not task.lease_token: - logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌，跳过执行") - continue - task_id = str(task.task_id) - cancelled = partial(queue.is_cancel_requested, task_id) - if queue.is_cancel_requested(task.task_id): - queue.release_task( - task_id=task.task_id, - lease_token=task.lease_token, - cancelled=True, - ) - continue - subscribe = self.subscription_repository.get(task.subscription_id) - if subscribe is None: - queue.finish_task( - task_id=task.task_id, - lease_token=task.lease_token, - state="cancelled", - error="订阅已不存在", - ) - continue - self._report_search_progress(progress_callback, subscribe, index, limit) - if self._defer_recent_subscription(subscribe): - queue.finish_task( - task_id=task.task_id, - lease_token=task.lease_token, - state="completed", - ) - continue - current = subscribe - phase_changed = partial( - self._update_search_task_phase, - queue, - task_id, - task.lease_token, + subscription_id = self._execute_search_task( + queue=queue, + task=task, + owner=owner, + searchchain=searchchain, + index=index, + limit=limit, + progress_callback=progress_callback, ) - searchchain.configure_subscription_site_budget( - SubscriptionSiteBudget( - repository=queue, - owner=f"{owner}:{task_id}", - cancelled=cancelled, - stop_state=getattr(self, "stop_state", runtime_stop_state), - phase_changed=phase_changed, - ) - ) - self._subscription_download_task_id = task_id - self._subscription_download_cancelled = cancelled - self._subscription_download_crossed_boundary = False - self._subscription_download_mark_started = self._mark_subscription_download_started - self._subscription_execution_phase = phase_changed - try: - current = self._process_search_subscription(subscribe, searchchain) - if queue.is_cancel_requested(task.task_id): - if self._subscription_download_started_for_task(task.task_id): - queue.finish_task( - task_id=task.task_id, - lease_token=task.lease_token, - state="completed", - error="取消请求晚于下载提交边界，已按实际结果完成", - ) - else: - queue.release_task( - task_id=task.task_id, - lease_token=task.lease_token, - cancelled=True, - ) - else: - queue.finish_task( - task_id=task.task_id, - lease_token=task.lease_token, - state="completed", - ) - processed.add(subscribe.id) - except SubscriptionSearchCancelled: - queue.release_task( - task_id=task.task_id, - lease_token=task.lease_token, - cancelled=True, - ) - except Exception as err: - logger.error(f"订阅 {subscribe.name} 搜索失败：{str(err)}", exc_info=True) - queue.finish_task( - task_id=task.task_id, - lease_token=task.lease_token, - state="failed", - error=str(err), - ) - finally: - delattr(self, "_subscription_download_task_id") - delattr(self, "_subscription_download_cancelled") - delattr(self, "_subscription_download_mark_started") - delattr(self, "_subscription_execution_phase") - self._subscription_download_crossed_boundary = False - searchchain.configure_subscription_site_budget(None) - if current and current.state == "N": - try: - self._SubscribeChain__apply_subscribe_update( - current, - {"state": "R"}, - scene="search_reset", - ) - except Exception as err: - logger.error( - f"订阅 {current.name} 搜索后状态重置失败：{str(err)}", - exc_info=True, - ) - self._report_search_progress(progress_callback, subscribe, index, limit, finished=True) + if subscription_id is not None: + processed.add(subscription_id) finally: - if queue_lock is not None: - queue_lock.release() + self._search_queue_lock.release() return processed - def _subscription_download_started_for_task(self, task_id: str) -> bool: - """判断取消是否已晚于下载器副作用边界。""" - if bool(getattr(self, "_subscription_download_crossed_boundary", False)): - return True - repository = getattr(self, "subscription_download_repository", None) - return bool(repository and repository.has_started_for_task(task_id)) - - def _mark_subscription_download_started(self) -> None: - """记录当前搜索任务已提交或复用了真实下载结果。""" - self._subscription_download_crossed_boundary = True - phase_changed = getattr(self, "_subscription_execution_phase", None) - if phase_changed: - phase_changed("submitting", None) - - @staticmethod - def _update_search_task_phase( + def _execute_search_task( + self, + *, queue: SubscriptionSearchRepository, - task_id: str, - lease_token: str, - phase: str, - current_site_id: Optional[int] = None, - ) -> None: - """以当前任务租约持久化业务阶段，过期执行者不得覆盖新状态。""" - queue.update_task_phase( - task_id=task_id, - lease_token=lease_token, - phase=phase, - current_site_id=current_site_id, + task: SearchTaskSnapshot, + owner: str, + searchchain: SearchChain, + index: int, + limit: int, + progress_callback: Optional[Callable[..., None]], + ) -> Optional[int]: + """执行一条已认领任务，返回实际进入搜索处理的订阅 ID。""" + if task.lease_token is None: + logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌，跳过执行") + return None + task_id = str(task.task_id) + lease_token = task.lease_token + cancelled = partial(queue.is_cancel_requested, task_id) + stop_state = getattr(self, "stop_state", runtime_stop_state) + if cancelled(): + queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True) + return None + subscribe = self.subscription_repository.get(task.subscription_id) + if subscribe is None: + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="cancelled", + error="订阅已不存在", + ) + return None + self._report_search_progress(progress_callback, subscribe, index, limit) + if self._defer_recent_subscription(subscribe): + _skip_search_task(queue, task, "订阅仍在新增保护期，本轮搜索已跳过") + return None + lease = self._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="search", + ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL, ) + if lease is None: + logger.info(f"订阅 {subscribe.name} 正在由其他通道处理，本轮搜索已跳过") + _skip_search_task(queue, task, "同一订阅正在由其他通道处理，本轮搜索已跳过") + return None + phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token) + execution_context = SubscriptionExecutionContext( + lease=lease, + admission=self._subscription_execution_admission, + task_id=task_id, + cancel_requested=lambda: cancelled() or stop_state.is_system_stopped, + phase_changed=phase_changed, + ) + current = subscribe + try: + current = self.subscription_repository.get(task.subscription_id) + if current is None: + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="cancelled", + error="订阅已不存在", + ) + return None + if current.state == "S": + _skip_search_task(queue, task, "订阅已暂停，本轮搜索已跳过") + return None + searchchain.configure_subscription_site_budget( + SubscriptionSiteBudget( + repository=queue, + owner=f"{owner}:{task_id}", + cancelled=execution_context.should_stop, + stop_state=stop_state, + phase_changed=phase_changed, + ) + ) + current = self._process_search_subscription( + current, + searchchain, + execution_context=execution_context, + ) + system_stopped = stop_state.is_system_stopped + cancel_requested = False if system_stopped else cancelled() + return _finish_returned_search_task( + queue=queue, + task_id=task_id, + lease_token=lease_token, + subscription_id=task.subscription_id, + execution_context=execution_context, + system_stopped=system_stopped, + cancel_requested=cancel_requested, + ) + except SubscriptionSearchCancelled: + if execution_context.is_expired() and not execution_context.is_cancel_requested(): + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="failed", + error="订阅执行已超过协作截止时间", + ) + else: + _release_cancelled_or_stopped_search_task( + queue, + task_id, + lease_token, + stop_state.is_system_stopped, + ) + except Exception as err: + logger.error(f"订阅 {subscribe.name} 搜索失败：{str(err)}", exc_info=True) + queue.finish_task( + task_id=task_id, + lease_token=lease_token, + state="failed", + error=str(err), + ) + finally: + self._cleanup_search_task( + queue=queue, + searchchain=searchchain, + subscribe=subscribe, + current=current, + lease=lease, + progress_callback=progress_callback, + index=index, + limit=limit, + ) + return None + + def _cleanup_search_task( + self, + *, + queue: SubscriptionSearchRepository, + searchchain: SearchChain, + subscribe: SubscriptionSnapshot, + current: Optional[SubscriptionSnapshot], + lease: Any, + progress_callback: Optional[Callable[..., None]], + index: int, + limit: int, + ) -> None: + """清理站点预算和订阅状态，并在所有异常路径释放 owner。""" + try: + searchchain.configure_subscription_site_budget(None) + except Exception as err: + logger.error(f"订阅 {subscribe.name} 搜索站点预算清理失败：{str(err)}", exc_info=True) + try: + if current and current.state == "N": + self._SubscribeChain__apply_subscribe_update( + current, + {"state": "R"}, + scene="search_reset", + ) + except Exception as err: + logger.error(f"订阅 {subscribe.name} 搜索后状态重置失败：{str(err)}", exc_info=True) + finally: + self._subscription_execution_admission.release(lease) + self._report_search_progress( + progress_callback, + subscribe, + index, + limit, + finished=True, + ) def resume_search_queue( self, @@ -429,44 +708,6 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase): progress_callback=progress_callback, ) - @staticmethod - def _search_source_and_priority( - *, - sid: Optional[int], - sids: Optional[tuple[int, ...]], - state: Optional[str], - manual: Optional[bool], - ) -> tuple[str, int]: - """把兼容入口归一为持久来源和公平队列优先级。""" - if manual: - return "manual", 100 - if sid or sids is not None: - return "targeted", 80 - if state in {"R", "P"}: - return "fallback", 10 - return "new", 50 - - @staticmethod - def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str: - """把批次聚合终态转为兼容进度文案。""" - if batch is None: - return "订阅搜索任务已提交" - if batch.state == "failed": - return "订阅搜索完成，部分任务失败" - if batch.state == "cancelled": - return "订阅搜索已取消" - if batch.state in {"queued", "running", "cancelling"}: - return "订阅搜索任务已排队" - return "订阅搜索完成" - - @staticmethod - def _search_batch_available_at(source: str) -> str: - """为自动兜底批次持久化一次全局启动抖动。""" - delay = random.randint(0, 60) if source == "fallback" else 0 - return ( - datetime.now(timezone.utc) + timedelta(seconds=delay) - ).isoformat(timespec="seconds") - def cancel_search_batch(self, batch_id: str) -> bool: """请求取消持久搜索批次；未注入队列时返回失败。""" queue: Optional[SubscriptionSearchRepository] = getattr( @@ -518,8 +759,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): state: Optional[str], progress_callback: Optional[Callable[..., None]], ) -> None: - """为自动搜索增加随机间隔，手动和定向批次不等待。""" - if sid or sids is not None or state not in ["R", "P"]: + """未使用持久队列时为自动兜底搜索保留逐订阅随机间隔。""" + if sid or sids is not None or state not in {"R", "P"}: return sleep_time = random.randint(60, 300) logger.info(f"订阅搜索随机休眠 {sleep_time} 秒 ...") @@ -531,11 +772,13 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): self, subscribe: SubscriptionSnapshot, searchchain: SearchChain, + execution_context: Optional[SubscriptionExecutionContext] = None, ) -> Optional[SubscriptionSnapshot]: """处理单个订阅，并返回下载后重新读取的状态快照。""" + _ensure_execution_active(execution_context) logger.info(f"开始搜索订阅，标题：{subscribe.name} ...") if subscribe.type == MediaType.MUSIC.value: - self._search_music_subscribe(subscribe) + self._search_music_subscribe(subscribe, execution_context=execution_context) return subscribe try: meta = build_subscribe_meta(subscribe) @@ -555,6 +798,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): f"媒体来源：{subscribe.media_source}，媒体 ID：{subscribe.media_id}" ) return subscribe + _ensure_execution_active(execution_context) mediakey = subscribe_media_key(subscribe) exists, no_exists = self.check_and_handle_existing_media( subscribe=subscribe, @@ -569,9 +813,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): if subscribe.best_version else SystemConfigKey.SubscribeFilterRuleGroups ) - phase_changed = getattr(self, "_subscription_execution_phase", None) - if phase_changed: - phase_changed("searching", None) + if execution_context: + execution_context.report_phase("searching") contexts = searchchain.process( mediainfo=mediainfo, keyword=subscribe.keyword, @@ -583,6 +826,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): filter_params=self.get_params(subscribe), ) site_budget_failures = searchchain.consume_subscription_site_budget_failures() + _ensure_execution_active(execution_context) if not contexts: logger.warning(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源") self.finish_subscribe_or_not( @@ -599,8 +843,9 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists) self._raise_site_budget_failures(site_budget_failures) return subscribe - if phase_changed: - phase_changed("preparing", None) + if execution_context: + execution_context.report_phase("preparing") + _ensure_execution_active(execution_context) downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first( contexts=matched, no_exists=no_exists, @@ -610,6 +855,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): save_path=subscribe.save_path, downloader=subscribe.downloader, source=self.get_subscribe_source_keyword(subscribe), + execution_context=execution_context, ) current = self.subscription_repository.get(subscribe.id) if current: @@ -625,7 +871,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): @staticmethod def _raise_site_budget_failures(failures: tuple[str, ...]) -> None: - """在成功站点结果完成处理后暴露未执行站点的聚合失败。""" + """在成功站点结果完成处理后暴露其余站点的聚合失败。""" if failures: raise RuntimeError("；".join(failures)) @@ -713,9 +959,19 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner): if not subscribes: self.messagehelper.put("没有找到订阅！", title="订阅搜索", role="system") elif sid: - self.messagehelper.put(f"{subscribes[0].name} 搜索完成！", title="订阅搜索", role="system") + message = ( + f"{subscribes[0].name} 搜索完成！" + if processed + else f"{subscribes[0].name} 本轮未执行或未完成，将等待下一次正常调度。" + ) + self.messagehelper.put(message, title="订阅搜索", role="system") elif sids is not None: for subscribe in processed: self.messagehelper.put(f"{subscribe.name} 搜索完成！", title="订阅搜索", role="system") else: - self.messagehelper.put("所有订阅搜索完成！", title="订阅搜索", role="system") + message = ( + "所有订阅搜索完成！" + if len(processed) == len(subscribes) + else "订阅搜索完成，部分订阅本轮未执行或未完成。" + ) + self.messagehelper.put(message, title="订阅搜索", role="system") diff --git a/app/chain/torrents.py b/app/chain/torrents.py index bd2b1984a..3a580863e 100644 --- a/app/chain/torrents.py +++ b/app/chain/torrents.py @@ -1,12 +1,11 @@ import re import traceback -from datetime import datetime, timezone from typing import Callable, Dict, List, Optional, Union from app.application.configuration import get_configured_system_config from app.application.rss import RssHelper from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module -from app.application.subscription.candidates import CandidateBatch, CandidateIndex +from app.application.subscription.candidates import CandidateIndex from app.application.torrent.download import TorrentHelper from app.chain.base import ChainBase from app.chain.media import MediaChain @@ -422,8 +421,6 @@ class TorrentsChain(ChainBase): include_music: bool, torrents_cache: Dict[str, List[Context]], music_cache: Dict[str, List[Context]], - fresh_torrents: Dict[str, List[Context]], - fresh_music: Dict[str, List[Context]], ) -> str: """抓取并写入单个站点的影视、音乐资源缓存。""" domain = site_rules.extract_domain(indexer.get("domain")) @@ -483,9 +480,7 @@ class TorrentsChain(ChainBase): continue context = self._build_refresh_context(torrent, stype) target_cache = music_cache if torrent.category == MediaType.MUSIC.value else torrents_cache - target_fresh = fresh_music if torrent.category == MediaType.MUSIC.value else fresh_torrents target_cache.setdefault(domain, []).append(context) - target_fresh.setdefault(domain, []).append(context) if len(target_cache[domain]) > self.runtime_config.torrent_cache_size: target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:] return domain @@ -548,31 +543,14 @@ class TorrentsChain(ChainBase): progress_callback: Optional[Callable[..., None]] = None, include_music: bool = False, ) -> Dict[str, List[Context]]: - """兼容旧调用返回完整候选字典，批次语义由 ``refresh_batch`` 提供。""" - return self.refresh_batch( - stype=stype, - sites=sites, - progress_callback=progress_callback, - include_music=include_music, - ).candidates - - def refresh_batch( - self, - stype: Optional[str] = None, - sites: List[int] = None, - progress_callback: Optional[Callable[..., None]] = None, - include_music: bool = False, - ) -> CandidateBatch: """ - 刷新站点最新资源并返回完整缓存与本轮新增候选。 + 刷新站点最新资源并返回本轮可匹配的完整候选缓存。 :param stype: 强制指定缓存类型，spider:爬虫缓存，rss:rss缓存 :param sites: 强制指定站点ID列表，为空则读取设置的订阅站点 :param progress_callback: 资源刷新进度更新回调 :param include_music: 是否额外抓取站点的音乐专用浏览入口，服务音乐订阅 """ - started_at = datetime.now(timezone.utc) - # 刷新类型 if not stype: stype = self.runtime_config.subscribe_mode @@ -590,8 +568,6 @@ class TorrentsChain(ChainBase): music_cache = self.load_cache(self._music_rss_file) or {} self._ensure_context_compatibility(torrents_cache, stype=stype) self._ensure_context_compatibility(music_cache, stype=stype) - fresh_torrents: Dict[str, List[Context]] = {} - fresh_music: Dict[str, List[Context]] = {} # 缓存过滤掉无效种子（影视与音乐缓存分别处理） for _cache in (torrents_cache, music_cache): @@ -632,8 +608,6 @@ class TorrentsChain(ChainBase): include_music=include_music, torrents_cache=torrents_cache, music_cache=music_cache, - fresh_torrents=fresh_torrents, - fresh_music=fresh_music, )) # 保存缓存到本地，影视与音乐分别存储 @@ -649,10 +623,6 @@ class TorrentsChain(ChainBase): torrents_cache = {k: v for k, v in torrents_cache.items() if k in domains} if sites and music_cache: music_cache = {k: v for k, v in music_cache.items() if k in domains} - if sites and fresh_torrents: - fresh_torrents = {k: v for k, v in fresh_torrents.items() if k in domains} - if sites and fresh_music: - fresh_music = {k: v for k, v in fresh_music.items() if k in domains} if progress_callback: progress_callback( @@ -661,37 +631,10 @@ class TorrentsChain(ChainBase): data={"total": total_indexers, "finished": total_indexers}, ) - self._retain_cached_fresh(fresh_torrents, torrents_cache) - self._retain_cached_fresh(fresh_music, music_cache) - candidates = self._merge_torrent_caches( + return self._merge_torrent_caches( {domain: list(contexts) for domain, contexts in torrents_cache.items()}, music_cache, ) - fresh_candidates = self._merge_torrent_caches( - {domain: list(contexts) for domain, contexts in fresh_torrents.items()}, - fresh_music, - ) - return CandidateBatch.create( - source=stype, - candidates=candidates, - fresh_candidates=fresh_candidates, - sites=domains, - started_at=started_at, - ) - - @staticmethod - def _retain_cached_fresh( - fresh_candidates: Dict[str, List[Context]], - cached_candidates: Dict[str, List[Context]], - ) -> None: - """移除因缓存容量裁剪而未进入最终完整缓存的本轮候选。""" - for domain, contexts in list(fresh_candidates.items()): - cached_ids = {id(context) for context in cached_candidates.get(domain) or []} - retained = [context for context in contexts if id(context) in cached_ids] - if retained: - fresh_candidates[domain] = retained - else: - fresh_candidates.pop(domain, None) @staticmethod def _ensure_context_compatibility(torrents_cache: Dict[str, List[Context]], stype: Optional[str] = None): diff --git a/app/db/adapters/subscriptiondownload.py b/app/db/adapters/subscriptiondownload.py deleted file mode 100644 index 975e063a5..000000000 --- a/app/db/adapters/subscriptiondownload.py +++ /dev/null @@ -1,147 +0,0 @@ -"""订阅下载幂等账本的短事务适配器。""" - -from collections.abc import Callable -from typing import Optional, TypeVar - -from sqlalchemy.orm import Session - -from app.application.download.admission import ( - SubscriptionDownloadClaim, - SubscriptionDownloadRequest, - SubscriptionDownloadSnapshot, -) -from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission -from app.db.oper.subscriptiondownload import SubscriptionDownloadOper -from app.db.uow import SqlAlchemyUnitOfWork - -T = TypeVar("T") - - -def _snapshot(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot: - """在 Session 内投影不可变提交快照。""" - return SubscriptionDownloadSnapshot( - idempotency_key=record.idempotency_key, - subscription_id=record.subscription_id, - task_id=record.task_id, - state=record.state, - attempt_count=record.attempt_count, - attempt_token=record.attempt_token, - downloader=record.downloader, - download_hash=record.download_hash, - available_at=record.available_at, - last_error=record.last_error, - created_at=record.created_at, - updated_at=record.updated_at, - ) - - -class TransactionalSubscriptionDownloadRepository: - """为每次提交认领和状态变更创建独立短事务。""" - - def __init__(self, session_factory: Callable[[], Session]) -> None: - """保存由组合根提供的同步 Session 工厂。""" - self._session_factory = session_factory - - def _read(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T: - """在短 Session 中执行一次只读查询。""" - with self._session_factory() as session: - return operation(SubscriptionDownloadOper(session)) - - def _write(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T: - """在显式短事务中执行一次状态变更。""" - with self._session_factory() as session: - unit_of_work = SqlAlchemyUnitOfWork(session) - try: - result = operation(SubscriptionDownloadOper(session)) - unit_of_work.commit() - return result - except Exception: - unit_of_work.rollback() - raise - - def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim: - """认领唯一提交权并返回脱离 Session 的当前状态。""" - def operation(repository: SubscriptionDownloadOper) -> SubscriptionDownloadClaim: - """在同一事务内完成唯一插入或 fenced 重领。""" - record, acquired = repository.claim(request) - return SubscriptionDownloadClaim(acquired=acquired, snapshot=_snapshot(record)) - - return self._write(operation) - - def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]: - """读取一个已存在的提交快照。""" - return self._read( - lambda repository: ( - _snapshot(record) - if (record := repository.get(idempotency_key)) is not None - else None - ) - ) - - def mark_accepted( - self, - *, - idempotency_key: str, - attempt_token: str, - downloader: Optional[str], - download_hash: str, - ) -> bool: - """记录下载器已接受任务。""" - return self._write(lambda repository: repository.mark_accepted( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - downloader=downloader, - download_hash=download_hash, - )) - - def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool: - """记录 canonical 本地结算已完成。""" - return self._write(lambda repository: repository.mark_succeeded( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - )) - - def mark_retryable( - self, - *, - idempotency_key: str, - attempt_token: str, - available_at: str, - error: Optional[str], - ) -> bool: - """记录未产生外部副作用的延迟重试状态。""" - return self._write(lambda repository: repository.mark_retryable( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - available_at=available_at, - error=error, - )) - - def mark_reconcile_required( - self, - *, - idempotency_key: str, - attempt_token: str, - error: Optional[str], - downloader: Optional[str] = None, - download_hash: Optional[str] = None, - ) -> bool: - """冻结可能已被下载器接受的提交。""" - return self._write(lambda repository: repository.mark_reconcile_required( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - error=error, - downloader=downloader, - download_hash=download_hash, - )) - - def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool: - """在外部提交前收口取消。""" - return self._write(lambda repository: repository.mark_cancelled( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - )) - - def has_started_for_task(self, task_id: str) -> bool: - """查询搜索任务是否已越过可安全取消边界。""" - return self._read(lambda repository: repository.has_started_for_task(task_id)) diff --git a/app/db/adapters/subscriptionsearch.py b/app/db/adapters/subscriptionsearch.py index 8b060cc29..c18263dfd 100644 --- a/app/db/adapters/subscriptionsearch.py +++ b/app/db/adapters/subscriptionsearch.py @@ -1,6 +1,6 @@ """订阅搜索持久队列的 SQLAlchemy 适配器。""" -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import Optional, TypeVar from sqlalchemy.orm import Session @@ -32,6 +32,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot: finished_count=record.finished_count, failed_count=record.failed_count, cancelled_count=record.cancelled_count, + skipped_count=record.skipped_count, cancel_requested=bool(record.cancel_requested), created_at=record.created_at, updated_at=record.updated_at, @@ -95,7 +96,7 @@ class TransactionalSubscriptionSearchRepository: subscription_ids: tuple[int, ...], source: str, priority: int, - available_at: Optional[str] = None, + available_at_by_subscription: Optional[Mapping[int, str]] = None, ) -> SearchEnqueueResult: """创建批次并返回 single-flight 合并计数。""" def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult: @@ -104,7 +105,7 @@ class TransactionalSubscriptionSearchRepository: subscription_ids=subscription_ids, source=source, priority=priority, - available_at=available_at, + available_at_by_subscription=available_at_by_subscription, ) return SearchEnqueueResult( batch=_batch(record), @@ -231,7 +232,7 @@ class TransactionalSubscriptionSearchRepository: next_allowed_at: str, error: Optional[str] = None, ) -> bool: - """释放站点租约并持久化间隔或冷却。""" + """释放站点租约并持久化错误冷却或立即恢复。""" return self._write( lambda repository: repository.finish_site( site_id=site_id, diff --git a/app/db/adapters/subscriptionstatus.py b/app/db/adapters/subscriptionstatus.py index 6f927e836..cbfe8279b 100644 --- a/app/db/adapters/subscriptionstatus.py +++ b/app/db/adapters/subscriptionstatus.py @@ -5,9 +5,7 @@ from typing import Optional from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.application.download.admission import SubscriptionDownloadSnapshot from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot -from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission from app.db.models.subscriptionsearch import SubscriptionSearchBatch, SubscriptionSearchTask @@ -46,6 +44,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot: finished_count=record.finished_count, failed_count=record.failed_count, cancelled_count=record.cancelled_count, + skipped_count=record.skipped_count, cancel_requested=bool(record.cancel_requested), created_at=record.created_at, updated_at=record.updated_at, @@ -55,26 +54,8 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot: ) -def _download(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot: - """复制可脱离 AsyncSession 使用的下载提交快照。""" - return SubscriptionDownloadSnapshot( - idempotency_key=record.idempotency_key, - subscription_id=record.subscription_id, - task_id=record.task_id, - state=record.state, - attempt_count=record.attempt_count, - attempt_token=record.attempt_token, - downloader=record.downloader, - download_hash=record.download_hash, - available_at=record.available_at, - last_error=record.last_error, - created_at=record.created_at, - updated_at=record.updated_at, - ) - - class SessionSubscriptionExecutionStatusRepository: - """复用请求 AsyncSession 批量读取搜索和下载执行事实。""" + """复用请求 AsyncSession 批量读取搜索执行事实。""" def __init__(self, session: AsyncSession) -> None: """绑定请求持有的异步会话。""" @@ -98,24 +79,6 @@ class SessionSubscriptionExecutionStatusRepository: snapshots.setdefault(record.subscription_id, _task(record)) return snapshots - async def latest_download_submissions( - self, - subscription_ids: tuple[int, ...], - ) -> dict[int, SubscriptionDownloadSnapshot]: - """按更新时间倒序读取并在内存中保留每条订阅首项。""" - result = await self._session.execute( - select(SubscriptionDownloadSubmission) - .where(SubscriptionDownloadSubmission.subscription_id.in_(subscription_ids)) - .order_by( - SubscriptionDownloadSubmission.updated_at.desc(), - SubscriptionDownloadSubmission.id.desc(), - ) - ) - snapshots: dict[int, SubscriptionDownloadSnapshot] = {} - for record in result.scalars().all(): - snapshots.setdefault(record.subscription_id, _download(record)) - return snapshots - async def list_batches(self, *, limit: int) -> list[SearchBatchSnapshot]: """返回最近更新的批次，访问范围由应用服务依据任务校验。""" result = await self._session.execute( diff --git a/app/db/models/__init__.py b/app/db/models/__init__.py index d1a3c2b96..39e448ae5 100644 --- a/app/db/models/__init__.py +++ b/app/db/models/__init__.py @@ -46,10 +46,6 @@ _MODEL_EXPORTS = { "app.db.models.subscriptionsearch", "SubscriptionSiteBudget", ), - "SubscriptionDownloadSubmission": ( - "app.db.models.subscriptiondownload", - "SubscriptionDownloadSubmission", - ), "SystemConfig": ("app.db.models.systemconfig", "SystemConfig"), "TransferHistory": ("app.db.models.transferhistory", "TransferHistory"), "TransferExecutionStep": ( diff --git a/app/db/models/subscriptiondownload.py b/app/db/models/subscriptiondownload.py deleted file mode 100644 index d666d0ae4..000000000 --- a/app/db/models/subscriptiondownload.py +++ /dev/null @@ -1,53 +0,0 @@ -"""订阅下载提交幂等账本模型。""" - -from typing import Optional - -from sqlalchemy import Index, Integer, String, Text, UniqueConstraint -from sqlalchemy.orm import Mapped, mapped_column - -from app.db.base import Base, get_id_column - - -class SubscriptionDownloadSubmission(Base): - """记录订阅下载唯一提交权、下载器接受事实和待对账终态。""" - - id = get_id_column() - idempotency_key: Mapped[str] = mapped_column(String(64), nullable=False) - subscription_id: Mapped[int] = mapped_column(Integer, nullable=False) - task_id: Mapped[Optional[str]] = mapped_column(String(64)) - logical_identity: Mapped[str] = mapped_column(Text, nullable=False) - resource_key: Mapped[str] = mapped_column(Text, nullable=False) - coverage: Mapped[str] = mapped_column(Text, nullable=False) - mode: Mapped[str] = mapped_column(String(32), nullable=False) - delivery_scope: Mapped[str] = mapped_column(Text, nullable=False, default="legacy") - state: Mapped[str] = mapped_column(String(32), nullable=False) - attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - attempt_token: Mapped[Optional[str]] = mapped_column(String(64)) - downloader: Mapped[Optional[str]] = mapped_column(String(128)) - download_hash: Mapped[Optional[str]] = mapped_column(String(256)) - available_at: Mapped[Optional[str]] = mapped_column(String(40)) - last_error: Mapped[Optional[str]] = mapped_column(Text) - created_at: Mapped[str] = mapped_column(String(40), nullable=False) - updated_at: Mapped[str] = mapped_column(String(40), nullable=False) - started_at: Mapped[Optional[str]] = mapped_column(String(40)) - finished_at: Mapped[Optional[str]] = mapped_column(String(40)) - - __table_args__ = ( - UniqueConstraint( - "idempotency_key", - name="uq_subscriptiondownloadsubmission_idempotency_key", - ), - Index( - "ix_subscriptiondownloadsubmission_task_state", - "task_id", - "state", - "id", - ), - Index( - "ix_subscriptiondownloadsubmission_subscription_state", - "subscription_id", - "state", - "updated_at", - "id", - ), - ) diff --git a/app/db/models/subscriptionsearch.py b/app/db/models/subscriptionsearch.py index 9198cb296..c5355a30b 100644 --- a/app/db/models/subscriptionsearch.py +++ b/app/db/models/subscriptionsearch.py @@ -20,6 +20,7 @@ class SubscriptionSearchBatch(Base): finished_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + skipped_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) cancel_requested: Mapped[int] = mapped_column(Integer, nullable=False, default=0) created_at: Mapped[str] = mapped_column(String(40), nullable=False) updated_at: Mapped[str] = mapped_column(String(40), nullable=False) @@ -77,7 +78,7 @@ class SubscriptionSearchTask(Base): class SubscriptionSiteBudget(Base): - """记录兜底搜索对单个站点的唯一租约、间隔与错误冷却。""" + """记录兜底搜索对单个站点的唯一租约与错误冷却。""" id = get_id_column() site_id: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/app/db/oper/subscriptiondownload.py b/app/db/oper/subscriptiondownload.py deleted file mode 100644 index bbfd9efad..000000000 --- a/app/db/oper/subscriptiondownload.py +++ /dev/null @@ -1,242 +0,0 @@ -"""订阅下载提交账本的事务内状态转换。""" - -from datetime import datetime, timezone -from typing import Optional, cast -from uuid import uuid4 - -from sqlalchemy import and_, or_, select, update -from sqlalchemy.exc import IntegrityError -from sqlalchemy.orm import Session - -from app.application.download.admission import SubscriptionDownloadRequest -from app.db.base import DbOper, execute_dml -from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission - - -def utc_now_text() -> str: - """返回可按字符串稳定排序的 UTC ISO 时间。""" - return datetime.now(timezone.utc).isoformat(timespec="seconds") - - -class SubscriptionDownloadOper(DbOper): - """在调用方事务内维护唯一下载提交及其 fenced 终态。""" - - def claim(self, request: SubscriptionDownloadRequest) -> tuple[SubscriptionDownloadSubmission, bool]: - """创建或原子重领允许重试的提交记录。""" - if not isinstance(self._db, Session): - raise RuntimeError("订阅下载提交认领需要调用方提供同步 Session") - now = utc_now_text() - attempt_token = uuid4().hex - record = self._db.execute( - select(SubscriptionDownloadSubmission).where( - SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key - ) - ).scalars().first() - if record is None: - try: - with self._db.begin_nested(): - record = SubscriptionDownloadSubmission( - idempotency_key=request.idempotency_key, - subscription_id=request.subscription_id, - task_id=request.task_id, - logical_identity=request.logical_identity, - resource_key=request.resource_key, - coverage=request.coverage, - mode=request.mode, - delivery_scope=request.delivery_scope, - state="submitting", - attempt_count=1, - attempt_token=attempt_token, - created_at=now, - updated_at=now, - started_at=now, - ) - self._db.add(record) - self._db.flush() - return record, True - except IntegrityError: - self._db.expire_all() - record = self._db.execute( - select(SubscriptionDownloadSubmission).where( - SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key - ) - ).scalar_one() - - reclaimable = record.state in {"retryable", "cancelled"} and ( - record.available_at is None or record.available_at <= now - ) - if not reclaimable: - return record, False - updated = execute_dml( - self._db, - update(SubscriptionDownloadSubmission) - .where( - SubscriptionDownloadSubmission.id == record.id, - SubscriptionDownloadSubmission.state.in_(("retryable", "cancelled")), - or_( - SubscriptionDownloadSubmission.available_at.is_(None), - SubscriptionDownloadSubmission.available_at <= now, - ), - ) - .values( - state="submitting", - task_id=request.task_id, - attempt_count=SubscriptionDownloadSubmission.attempt_count + 1, - attempt_token=attempt_token, - downloader=None, - download_hash=None, - available_at=None, - last_error=None, - updated_at=now, - started_at=now, - finished_at=None, - ), - execution_options={"synchronize_session": False}, - ) - self._db.flush() - self._db.expire_all() - current = self._db.execute( - select(SubscriptionDownloadSubmission).where( - SubscriptionDownloadSubmission.id == record.id - ) - ).scalar_one() - return current, bool(updated and current.attempt_token == attempt_token) - - def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSubmission]: - """按稳定幂等键读取提交记录。""" - if not isinstance(self._db, Session): - raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session") - return cast(Optional[SubscriptionDownloadSubmission], self._db.execute( - select(SubscriptionDownloadSubmission).where( - SubscriptionDownloadSubmission.idempotency_key == idempotency_key - ) - ).scalars().first()) - - def mark_accepted( - self, - *, - idempotency_key: str, - attempt_token: str, - downloader: Optional[str], - download_hash: str, - ) -> bool: - """把当前 submitting 尝试推进为下载器已接受。""" - return self._transition( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - source_states=("submitting",), - values={ - "state": "accepted", - "downloader": downloader, - "download_hash": download_hash, - }, - ) - - def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool: - """把当前 accepted 尝试推进为本地结算成功。""" - return self._transition( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - source_states=("accepted",), - values={"state": "succeeded", "finished_at": utc_now_text()}, - ) - - def mark_retryable( - self, - *, - idempotency_key: str, - attempt_token: str, - available_at: str, - error: Optional[str], - ) -> bool: - """把下载器明确拒绝的当前尝试收口为延迟可重试。""" - return self._transition( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - source_states=("submitting",), - values={ - "state": "retryable", - "available_at": available_at, - "last_error": error, - "finished_at": utc_now_text(), - }, - ) - - def mark_reconcile_required( - self, - *, - idempotency_key: str, - attempt_token: str, - error: Optional[str], - downloader: Optional[str], - download_hash: Optional[str], - ) -> bool: - """冻结 submitting/accepted 尝试，禁止自动重新提交。""" - values: dict[str, object] = { - "state": "reconcile_required", - "last_error": error, - "finished_at": utc_now_text(), - } - if downloader is not None: - values["downloader"] = downloader - if download_hash is not None: - values["download_hash"] = download_hash - return self._transition( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - source_states=("submitting", "accepted"), - values=values, - ) - - def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool: - """在外部提交前把当前尝试收口为已取消。""" - return self._transition( - idempotency_key=idempotency_key, - attempt_token=attempt_token, - source_states=("submitting",), - values={"state": "cancelled", "finished_at": utc_now_text()}, - ) - - def has_started_for_task(self, task_id: str) -> bool: - """判断任务是否进入过外部副作用边界或已成功。""" - if not isinstance(self._db, Session): - raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session") - return bool(self._db.execute( - select(SubscriptionDownloadSubmission.id) - .where( - SubscriptionDownloadSubmission.task_id == task_id, - SubscriptionDownloadSubmission.state.in_(( - "submitting", - "accepted", - "succeeded", - "reconcile_required", - )), - ) - .limit(1) - ).scalar()) - - def _transition( - self, - *, - idempotency_key: str, - attempt_token: str, - source_states: tuple[str, ...], - values: dict[str, object], - ) -> bool: - """以当前尝试令牌原子推进状态，拒绝过期执行者写回。""" - if not isinstance(self._db, Session): - raise RuntimeError("订阅下载提交写入需要调用方提供同步 Session") - now = utc_now_text() - return bool(execute_dml( - self._db, - update(SubscriptionDownloadSubmission) - .where( - and_( - SubscriptionDownloadSubmission.idempotency_key == idempotency_key, - SubscriptionDownloadSubmission.attempt_token == attempt_token, - SubscriptionDownloadSubmission.state.in_(source_states), - ) - ) - .values(updated_at=now, **values), - execution_options={"synchronize_session": False}, - )) diff --git a/app/db/oper/subscriptionsearch.py b/app/db/oper/subscriptionsearch.py index 6e3ee9fa9..5ccc7b48f 100644 --- a/app/db/oper/subscriptionsearch.py +++ b/app/db/oper/subscriptionsearch.py @@ -1,7 +1,7 @@ """订阅搜索批次与任务的持久队列读写。""" from datetime import datetime, timedelta, timezone -from typing import Optional +from typing import Mapping, Optional from uuid import uuid4 from sqlalchemy import and_, case, func, or_, select, update @@ -30,7 +30,7 @@ class SubscriptionSearchOper(DbOper): subscription_ids: tuple[int, ...], source: str, priority: int, - available_at: Optional[str], + available_at_by_subscription: Optional[Mapping[int, str]], ) -> tuple[SubscriptionSearchBatch, int, int]: """创建批次，并以活动键合并同一订阅的重叠搜索入口。""" if not isinstance(self._db, Session): @@ -51,6 +51,11 @@ class SubscriptionSearchOper(DbOper): coalesced = 0 for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)): active_key = f"subscription:{subscription_id}" + available_at = ( + available_at_by_subscription.get(subscription_id, now) + if available_at_by_subscription + else now + ) task = SubscriptionSearchTask( task_id=uuid4().hex, batch_id=batch.batch_id, @@ -61,7 +66,7 @@ class SubscriptionSearchOper(DbOper): position=position, state="queued", phase="queued", - available_at=available_at or now, + available_at=available_at, created_at=now, updated_at=now, ) @@ -85,9 +90,9 @@ class SubscriptionSearchOper(DbOper): ( or_( SubscriptionSearchTask.available_at.is_(None), - SubscriptionSearchTask.available_at > (available_at or now), + SubscriptionSearchTask.available_at > available_at, ), - available_at or now, + available_at, ), else_=SubscriptionSearchTask.available_at, ), @@ -267,7 +272,7 @@ class SubscriptionSearchOper(DbOper): """以当前租约令牌收口任务，并重新计算批次终态。""" if not isinstance(self._db, Session): raise RuntimeError("订阅搜索收口需要调用方提供同步 Session") - if state not in {"completed", "failed", "cancelled"}: + if state not in {"completed", "failed", "cancelled", "skipped"}: raise ValueError(f"不支持的订阅搜索终态：{state}") task = self._db.execute( select(SubscriptionSearchTask).where( @@ -334,7 +339,7 @@ class SubscriptionSearchOper(DbOper): error=None, ) now = utc_now_text() - return bool(execute_dml( + updated = execute_dml( self._db, update(SubscriptionSearchTask) .where( @@ -352,7 +357,11 @@ class SubscriptionSearchOper(DbOper): updated_at=now, ), execution_options={"synchronize_session": False}, - )) + ) + if not updated: + return False + self._refresh_batch(task.batch_id, now=now, error=None) + return True def is_cancel_requested(self, task_id: str) -> bool: """读取任务和批次取消标记。""" @@ -437,7 +446,11 @@ class SubscriptionSearchOper(DbOper): and record.lease_expires_at and record.lease_expires_at > now ) - if lease_busy or record.next_allowed_at > now: + cooldown_active = bool( + record.last_outcome not in {None, "success", "skipped"} + and record.next_allowed_at > now + ) + if lease_busy or cooldown_active: return record, False lease_token = uuid4().hex lease_expires_at = ( @@ -453,12 +466,17 @@ class SubscriptionSearchOper(DbOper): SubscriptionSiteBudget.lease_expires_at.is_(None), SubscriptionSiteBudget.lease_expires_at <= now, ), - SubscriptionSiteBudget.next_allowed_at <= now, + or_( + SubscriptionSiteBudget.next_allowed_at <= now, + SubscriptionSiteBudget.last_outcome.is_(None), + SubscriptionSiteBudget.last_outcome.in_(("success", "skipped")), + ), ) .values( lease_owner=owner, lease_token=lease_token, lease_expires_at=lease_expires_at, + next_allowed_at=now, updated_at=now, ), execution_options={"synchronize_session": False}, @@ -586,7 +604,8 @@ class SubscriptionSearchOper(DbOper): completed = counts.get("completed", 0) failed = counts.get("failed", 0) cancelled = counts.get("cancelled", 0) - terminal = completed + failed + cancelled + skipped = counts.get("skipped", 0) + terminal = completed + failed + cancelled + skipped batch = self.get_batch(batch_id) if batch is None: return @@ -595,11 +614,18 @@ class SubscriptionSearchOper(DbOper): state = "failed" elif cancelled or batch.cancel_requested: state = "cancelled" + elif skipped: + state = "skipped" else: state = "completed" finished_at = now else: - state = "cancelling" if batch.cancel_requested else "running" + if batch.cancel_requested: + state = "cancelling" + elif counts.get("running", 0): + state = "running" + else: + state = "queued" finished_at = None execute_dml( self._db, @@ -610,6 +636,7 @@ class SubscriptionSearchOper(DbOper): finished_count=completed, failed_count=failed, cancelled_count=cancelled, + skipped_count=skipped, updated_at=now, finished_at=finished_at, last_error=error or batch.last_error, diff --git a/app/schemas/subscribe.py b/app/schemas/subscribe.py index 44c2e06ff..5684978f8 100644 --- a/app/schemas/subscribe.py +++ b/app/schemas/subscribe.py @@ -57,8 +57,6 @@ class SubscriptionExecutionStatus(BaseModel): # type: ignore[misc] current_site_id: Optional[int] = None error: Optional[str] = None can_cancel: bool = False - can_retry: bool = False - requires_reconciliation: bool = False model_config = ConfigDict(from_attributes=True) @@ -77,6 +75,7 @@ class SubscriptionBatchStatus(BaseModel): # type: ignore[misc] cancelled_count: int created_at: str updated_at: str + skipped_count: int = 0 current_subscription_id: Optional[int] = None current_site_id: Optional[int] = None error: Optional[str] = None diff --git a/app/startup/composition/chain.py b/app/startup/composition/chain.py index 4dcfd49b6..3be18bd01 100644 --- a/app/startup/composition/chain.py +++ b/app/startup/composition/chain.py @@ -36,7 +36,6 @@ from app.chain.transfer.filter import ( from app.db.adapters.chain import TransactionalChainDurableEventWriter from app.db.adapters.download import TransactionalDownloadFailureRepository from app.db.adapters.mediaserver import TransactionalMediaServerRepository -from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository from app.db.oper.message import MessageOper from app.db.oper.systemconfig import SystemConfigOper @@ -112,7 +111,6 @@ def build_chain_runtime_context( transfer_execution_repository=dependencies.transfer_execution, media_server_repository=TransactionalMediaServerRepository(SessionFactory), download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory), - subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory), user_repository=build_transactional_user_repository(), legacy_transfer_command=execute_legacy_transfer_command, configuration=configuration(), diff --git a/database/versions/b6c1d9e4a7f2_3_0_24.py b/database/versions/b6c1d9e4a7f2_3_0_24.py new file mode 100644 index 000000000..04fd8349d --- /dev/null +++ b/database/versions/b6c1d9e4a7f2_3_0_24.py @@ -0,0 +1,37 @@ +"""3.0.24 增加订阅搜索批次跳过计数。""" + +# Alembic 的 op 是运行期代理，静态分析无法看到实际操作方法。 +# pylint: disable=no-member + +import sqlalchemy as sa +from alembic import op + +revision = "b6c1d9e4a7f2" +down_revision = "a7d9e2c4f6b1" +branch_labels = None +depends_on = None + +_TABLE = "subscriptionsearchbatch" + + +def _column_names() -> set[str]: + """返回当前订阅搜索批次字段名集合。""" + inspector = sa.inspect(op.get_bind()) + if _TABLE not in set(inspector.get_table_names()): + return set() + return {column["name"] for column in inspector.get_columns(_TABLE)} + + +def upgrade() -> None: + """为存量批次增加跳过计数，保持旧数据库可重复升级。""" + if _TABLE in set(sa.inspect(op.get_bind()).get_table_names()) and "skipped_count" not in _column_names(): + op.add_column( + _TABLE, + sa.Column("skipped_count", sa.Integer(), nullable=False, server_default="0"), + ) + + +def downgrade() -> None: + """移除批次跳过计数，保留原搜索任务与批次记录。""" + if "skipped_count" in _column_names(): + op.drop_column(_TABLE, "skipped_count") diff --git a/database/versions/c8f2e6a1d4b9_3_0_25.py b/database/versions/c8f2e6a1d4b9_3_0_25.py new file mode 100644 index 000000000..9e0c46a6f --- /dev/null +++ b/database/versions/c8f2e6a1d4b9_3_0_25.py @@ -0,0 +1,73 @@ +"""3.0.25 移除订阅下载提交持久账本。 + +Revision ID: c8f2e6a1d4b9 +Revises: b6c1d9e4a7f2 +Create Date: 2026-09-03 +""" + +# Alembic 的 op 是运行期代理，静态分析无法看到实际操作方法。 +# pylint: disable=no-member + +import sqlalchemy as sa +from alembic import op + +revision = "c8f2e6a1d4b9" +down_revision = "b6c1d9e4a7f2" +branch_labels = None +depends_on = None + +_TABLE = "subscriptiondownloadsubmission" + + +def _table_names() -> set[str]: + """返回当前数据库表名集合。""" + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def upgrade() -> None: + """删除不再参与订阅执行的下载提交账本。""" + if _TABLE in _table_names(): + op.drop_table(_TABLE) + + +def downgrade() -> None: + """恢复 3.0.24 使用的下载提交账本结构。""" + if _TABLE in _table_names(): + return + op.create_table( + _TABLE, + sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("idempotency_key", sa.String(length=64), nullable=False), + sa.Column("subscription_id", sa.Integer(), nullable=False), + sa.Column("task_id", sa.String(length=64), nullable=True), + sa.Column("logical_identity", sa.Text(), nullable=False), + sa.Column("resource_key", sa.Text(), nullable=False), + sa.Column("coverage", sa.Text(), nullable=False), + sa.Column("mode", sa.String(length=32), nullable=False), + sa.Column("delivery_scope", sa.Text(), nullable=False, server_default="legacy"), + sa.Column("state", sa.String(length=32), nullable=False), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.Column("attempt_token", sa.String(length=64), nullable=True), + sa.Column("downloader", sa.String(length=128), nullable=True), + sa.Column("download_hash", sa.String(length=256), nullable=True), + sa.Column("available_at", sa.String(length=40), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("created_at", sa.String(length=40), nullable=False), + sa.Column("updated_at", sa.String(length=40), nullable=False), + sa.Column("started_at", sa.String(length=40), nullable=True), + sa.Column("finished_at", sa.String(length=40), nullable=True), + sa.UniqueConstraint( + "idempotency_key", + name="uq_subscriptiondownloadsubmission_idempotency_key", + ), + ) + op.create_index( + "ix_subscriptiondownloadsubmission_task_state", + _TABLE, + ["task_id", "state", "id"], + ) + op.create_index( + "ix_subscriptiondownloadsubmission_subscription_state", + _TABLE, + ["subscription_id", "state", "updated_at", "id"], + ) diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index 985901ad1..9f0dd9b06 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -754,8 +754,8 @@ flowchart LR | 指标 | 当前值 | |---|---:| -| Python 模块 | 944 | -| 内部导入边 | 7,874 | +| Python 模块 | 940 | +| 内部导入边 | 7,845 | | 非平凡 SCC | 1（精确 containment 的 TMDB 移植包环） | | Application / Chain 具体 Adapter 直连 | 0 / 0 | | Direct egress | 53（债务已清零，53 条精确 containment） | diff --git a/docs/architecture/optimization-checklist.md b/docs/architecture/optimization-checklist.md index 20864cf35..b676d614d 100644 --- a/docs/architecture/optimization-checklist.md +++ b/docs/architecture/optimization-checklist.md @@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁 | 指标 | 当前值 | 解释 | |---|---:|---| -| 宿主 Python 模块 / 内部依赖边 | 944 / 7,874 | `dependency-baseline.json` 当前快照 | +| 宿主 Python 模块 / 内部依赖边 | 940 / 7,845 | `dependency-baseline.json` 当前快照 | | 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 | | 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 | | Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 | @@ -102,7 +102,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁 | Event Contract | 53 | 均已有 payload model，但当前全部是 diagnostic enforcement | | Python 源码量 | 305,884 行 | 排除 `app/plugins/**`；61 个文件超过 1,000 行，11 个超过 2,000 行 | | 长方法 | 290 个超过 80 行 | AST 统计排除 `app/plugins/**`；65 个超过 150 行，21 个超过 250 行 | -| 全量 mypy 历史债务 | 9,581 / 516 文件 | Agent API 重构后的现状基线；canonical Facade 与 endpoint 类型边界已补齐，低水位只允许继续下降 | +| 全量 mypy 历史债务 | 9,580 / 516 文件 | Agent API 重构后的现状基线；canonical Facade 与 endpoint 类型边界已补齐，低水位只允许继续下降 | | Ruff 历史诊断 | 559 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` | | 覆盖率固定基线 | Application 80.00%，Domain 80.00% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 | diff --git a/docs/refactor/moviepilot-subscription-governance-handoff.md b/docs/refactor/moviepilot-subscription-governance-handoff.md index 7885ee184..acd51c97e 100644 --- a/docs/refactor/moviepilot-subscription-governance-handoff.md +++ b/docs/refactor/moviepilot-subscription-governance-handoff.md @@ -1,5 +1,9 @@ # MoviePilot 订阅执行治理上游 Handoff +> 历史交接：本文保留 2026-09-01 的问题证据，不再作为当前实施合同。 +> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required` +> 已由 2026-09-03 的简化方案废止；当前行为以代码、迁移和回归测试为准。 +> > 日期：2026-09-01 > 目标版本：MoviePilot V3 > 生产参考：MoviePilot V2 `2.15.5` 的一次只读样本 diff --git a/docs/refactor/moviepilot-subscription-governance-roadmap.md b/docs/refactor/moviepilot-subscription-governance-roadmap.md index 94ea4f874..bee7cb845 100644 --- a/docs/refactor/moviepilot-subscription-governance-roadmap.md +++ b/docs/refactor/moviepilot-subscription-governance-roadmap.md @@ -1,5 +1,9 @@ # MoviePilot 订阅执行治理 +> 历史方案：本文件记录 2026-09-01 的上游接管与验证证据，不再作为当前实施合同。 +> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required` +> 已由 2026-09-03 的简化方案废止；当前行为以代码、迁移和回归测试为准。 +> > 状态：`completed（2026-09-01）` > 当前叶：`none（全部叶已完成或按条件停止）` > 迁移来源：`V3-RDY-009A`、`V3-RDY-009A1`、`V3-RDY-009A1B`、 diff --git a/scripts/generate_agent_skill_docs.py b/scripts/generate_agent_skill_docs.py index efb089354..797c73d3d 100644 --- a/scripts/generate_agent_skill_docs.py +++ b/scripts/generate_agent_skill_docs.py @@ -116,11 +116,6 @@ DATABASE_TABLE_GUIDES: dict[str, tuple[str, str, str]] = { "Auditing historical subscriptions, media identity, completion criteria, and filter configuration.", "Generated by subscription completion and archival; restore or delete through its business API.", ), - "subscriptiondownloadsubmission": ( - "Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes.", - "Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership.", - "Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states.", - ), "subscriptionsearchbatch": ( "Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.", "Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.", diff --git a/scripts/validation/subscription_governance_scale.py b/scripts/validation/subscription_governance_scale.py index c6b69ba18..cdb416205 100644 --- a/scripts/validation/subscription_governance_scale.py +++ b/scripts/validation/subscription_governance_scale.py @@ -11,36 +11,51 @@ import platform import subprocess import sys import tempfile +import threading import time +from collections import Counter from dataclasses import asdict, dataclass from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any +from types import SimpleNamespace +from typing import Any, Optional +from unittest.mock import patch -from sqlalchemy import create_engine, update +from sqlalchemy import create_engine, select, update from sqlalchemy.orm import Session, sessionmaker PROJECT_ROOT = Path(__file__).resolve().parents[2] if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) -from app.application.download.admission import SubscriptionDownloadRequest # noqa: E402 +from app.application.site.observation import report_site_search_outcome # noqa: E402 from app.application.subscription.candidates import CandidateIndex # noqa: E402 from app.application.subscription.contract import SubscriptionSnapshot # noqa: E402 +from app.application.subscription.execution import SubscriptionExecutionAdmission # noqa: E402 from app.application.subscription.facts import FreshFactLease # noqa: E402 -from app.db.adapters.subscriptiondownload import ( # noqa: E402 - TransactionalSubscriptionDownloadRepository, +from app.application.subscription.sitebudget import ( # noqa: E402 + SubscriptionSiteBudget as SubscriptionSiteBudgetController, ) +from app.chain.download import batch as download_batch # noqa: E402 +from app.chain.download.facade import DownloadChain # noqa: E402 +from app.chain.search.facade import SearchChain # noqa: E402 +from app.chain.subscribe import match as subscribe_match # noqa: E402 +from app.chain.subscribe import policy as subscribe_policy # noqa: E402 +from app.chain.subscribe.facade import SubscribeChain # noqa: E402 from app.db.adapters.subscriptionsearch import ( # noqa: E402 TransactionalSubscriptionSearchRepository, ) from app.db.base import Base # noqa: E402 from app.db.models.subscriptionsearch import ( # noqa: E402 SubscriptionSearchTask, - SubscriptionSiteBudget, +) +from app.db.models.subscriptionsearch import ( # noqa: E402 + SubscriptionSiteBudget as SubscriptionSiteBudgetRecord, ) from app.domain.context import Context, MediaInfo, TorrentInfo # noqa: E402 from app.domain.metainfo import MetaInfo # noqa: E402 +from app.runtime.stop import ProcessStopState # noqa: E402 +from app.schemas.mediaserver import NotExistMediaInfo # noqa: E402 from app.schemas.types import MediaSource, MediaType # noqa: E402 @@ -78,10 +93,12 @@ def _git_revision() -> str: def _context(*, candidate_index: int, media_id: str, site_id: int) -> Context: """构造同时携带 canonical 媒体身份和稳定资源身份的候选。""" - title = f"Governance Show {media_id} S01E{candidate_index + 1:04d}" + title = f"Governance Show {media_id} S01E01" meta = MetaInfo(title) meta.type = MediaType.TV meta.begin_season = 1 + meta.begin_episode = 1 + meta.end_episode = None meta.media_source = MediaSource.TMDB meta.media_id = media_id return Context( @@ -116,6 +133,10 @@ def _subscription(subscription_id: int, media_id: str, site_count: int) -> Subsc media_source=MediaSource.TMDB, media_id=media_id, season=1, + total_episode=2, + start_episode=1, + lack_episode=1 if subscription_id % 2 == 0 else 2, + note=[], state="R", sites=list(range(1, site_count + 1)), best_version=0, @@ -146,8 +167,10 @@ def _baseline_route( return routed -def _run_match_case(case: ScaleCase) -> dict[str, Any]: - """比较完整扫描与无损索引，并记录单轮事实租约和本地等待分位。""" +def _build_match_inputs( + case: ScaleCase, +) -> tuple[dict[str, list[Context]], list[SubscriptionSnapshot]]: + """为索引基线和实际 Match 构造彼此独立的固定输入快照。""" candidates = { f"site-{site_id}.example": [] for site_id in range(1, case.site_count + 1) @@ -170,6 +193,509 @@ def _run_match_case(case: ScaleCase) -> dict[str, Any]: ) for index in range(case.subscription_count) ] + return candidates, subscribes + + +def _collection_digest(values: list[str]) -> str: + """返回保留重复次数的稳定摘要，便于审计完整集合而不膨胀输出。""" + payload = "\n".join(sorted(values)).encode("utf-8") + return hashlib.sha256(payload).hexdigest() + + +def _compare_collections(expected: set[str], actual: list[str]) -> dict[str, Any]: + """比较完整多重集合并保留计数、摘要、重复项和有限差异样本。""" + expected_values = sorted(expected) + actual_values = sorted(actual) + expected_counts = Counter(expected_values) + actual_counts = Counter(actual_values) + missing = expected_counts - actual_counts + unexpected = actual_counts - expected_counts + return { + "equal": actual_counts == expected_counts, + "expected_count": len(expected_values), + "actual_count": len(actual_values), + "actual_distinct_count": len(actual_counts), + "duplicate_count": sum(max(count - 1, 0) for count in actual_counts.values()), + "expected_sha256": _collection_digest(expected_values), + "actual_sha256": _collection_digest(actual_values), + "missing_sample": sorted(missing.elements())[:5], + "unexpected_sample": sorted(unexpected.elements())[:5], + } + + +class _ScaleSubscriptionRepository: + """为实际 Match 提供固定订阅列表和准入后的最新快照。""" + + def __init__(self, subscribes: list[SubscriptionSnapshot]) -> None: + self._subscribes = list(subscribes) + self._by_id = {subscribe.id: subscribe for subscribe in subscribes} + + def list(self, _state: str) -> list[SubscriptionSnapshot]: + """返回固定顺序的完整订阅快照。""" + return list(self._subscribes) + + def get(self, subscription_id: int) -> Optional[SubscriptionSnapshot]: + """返回准入和提交前复读使用的当前订阅快照。""" + return self._by_id.get(subscription_id) + + +class _ScaleSiteRepository: + """把受控站点 ID 映射为候选分组域名。""" + + @staticmethod + def get_domains_by_ids(site_ids: list[int]) -> list[str]: + """返回固定快照内与站点 ID 一一对应的域名。""" + return [f"site-{site_id}.example" for site_id in site_ids] + + +class _ScaleTorrentHelper: + """隔离实际 Match 的种子属性过滤外部依赖。""" + + @staticmethod + def filter_torrent(**_kwargs: Any) -> bool: + """固定样本中的候选均通过订阅属性过滤。""" + return True + + +_SITE_PRESSURE_SYNC_TIMEOUT = 5.0 + + +class _ScaleSiteRequestBoundary: + """记录确定性站点请求替身的在途数量和请求结果。""" + + def __init__(self, *, site_id: int) -> None: + self.site_id = site_id + self._lock = threading.Lock() + self._active = 0 + self._peak = 0 + self._calls = 0 + self._outcomes: list[str] = [] + self._outcome = "success" + self._error: Optional[str] = None + self._active_at_rejection = 0 + self.first_started = threading.Event() + self.round_observed = threading.Event() + self.release_first = threading.Event() + + @property + def active(self) -> int: + """返回请求边界当前真实在途数量。""" + with self._lock: + return self._active + + @property + def peak(self) -> int: + """返回请求边界观测到的最大在途数量。""" + with self._lock: + return self._peak + + @property + def calls(self) -> int: + """返回请求边界实际进入次数。""" + with self._lock: + return self._calls + + @property + def outcomes(self) -> tuple[str, ...]: + """返回请求边界按进入顺序记录的结果分类。""" + with self._lock: + return tuple(self._outcomes) + + @property + def active_at_rejection(self) -> int: + """返回预算拒绝发生时请求边界的在途数量。""" + with self._lock: + return self._active_at_rejection + + def set_outcome(self, outcome: str, error: Optional[str] = None) -> None: + """设置后续确定性请求要发布的站点观察结果。""" + with self._lock: + self._outcome = outcome + self._error = error + + def record_budget_rejection(self) -> None: + """记录 wrapper 拒绝时请求边界仍保持在途的事实。""" + with self._lock: + self._active_at_rejection = max(self._active_at_rejection, self._active) + self.round_observed.set() + + def request(self, *, owner: str) -> list[str]: + """在真实请求边界更新 active/peak 并返回固定站点结果。""" + with self._lock: + call_index = self._calls + self._calls += 1 + self._active += 1 + self._peak = max(self._peak, self._active) + outcome = self._outcome + error = self._error + self._outcomes.append(outcome) + first_request = call_index == 0 + if call_index == 1: + self.round_observed.set() + if first_request: + self.first_started.set() + self.release_first.wait(timeout=_SITE_PRESSURE_SYNC_TIMEOUT) + try: + report_site_search_outcome( + attempted=True, + outcome=outcome, + error=error, + ) + if outcome == "rate_limited": + return [] + return [f"site-{self.site_id}-{owner}-{call_index}"] + finally: + with self._lock: + self._active -= 1 + + +def _site_budget_record( + engine: Any, + site_id: int, +) -> Optional[SubscriptionSiteBudgetRecord]: + """读取站点预算最终记录，核对 observation 是否实际持久化。""" + with Session(engine) as session: + return session.execute( + select(SubscriptionSiteBudgetRecord).where( + SubscriptionSiteBudgetRecord.site_id == site_id + ) + ).scalars().first() + + +def _run_site_pressure_case( + case: ScaleCase, + search: TransactionalSubscriptionSearchRepository, + engine: Any, +) -> dict[str, Any]: + """让两个并发 SearchChain owner 通过 wrapper 验证每个站点的容量和冷却。""" + site_observations: list[dict[str, Any]] = [] + duplicate_site_claims_blocked = 0 + error_cooldown_claims_blocked = 0 + successful_site_claims_reused = 0 + + for site_id in range(1, case.site_count + 1): + boundary = _ScaleSiteRequestBoundary(site_id=site_id) + pressure_event = boundary.round_observed + start_barrier = threading.Barrier(2) + owners = ("scale-owner-a", "scale-owner-b") + chains: dict[str, SearchChain] = {} + owner_failures: dict[str, list[str]] = {owner: [] for owner in owners} + + for owner in owners: + controller = SubscriptionSiteBudgetController( + repository=search, + owner=owner, + cancelled=lambda: False, + stop_state=ProcessStopState(), + lease_seconds=900, + ) + chain = object.__new__(SearchChain) + chain.configure_subscription_site_budget(controller) + original_record_failure = chain.record_subscription_site_budget_failure + + def record_failure( + error: str, + *, + original=original_record_failure, + owner_name=owner, + ) -> None: + """保留 wrapper 聚合失败并同步压力轮次观测。""" + original(error) + owner_failures[owner_name].append(error) + if "冷却或已有在途搜索" in error: + boundary.record_budget_rejection() + pressure_event.set() + + chain.record_subscription_site_budget_failure = record_failure + + def search_site_torrents(*, _owner=owner, **_kwargs: Any) -> list[str]: + """将 SearchChain 的真实站点请求委托给固定边界。""" + return boundary.request(owner=_owner) + + chain.search_site_torrents = search_site_torrents + chains[owner] = chain + + owner_results: dict[str, list[str]] = {} + owner_errors: dict[str, str] = {} + owner_invocations: dict[str, int] = {owner: 0 for owner in owners} + result_lock = threading.Lock() + + def run_owner(owner: str) -> None: + """同步启动一个 owner，并保存 wrapper 的结果或异常。""" + try: + start_barrier.wait(timeout=_SITE_PRESSURE_SYNC_TIMEOUT) + with result_lock: + owner_invocations[owner] += 1 + result = chains[owner]._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": site_id, "name": f"Site {site_id}"}, + keyword="governance-pressure", + mtype=None, + page=0, + ) + with result_lock: + owner_results[owner] = result + except Exception as error: # noqa: BLE001 + with result_lock: + owner_errors[owner] = str(error) + + threads = [ + threading.Thread(target=run_owner, args=(owner,), daemon=True) + for owner in owners + ] + for thread in threads: + thread.start() + first_request_started = boundary.first_started.wait( + timeout=_SITE_PRESSURE_SYNC_TIMEOUT, + ) + contention_observed = pressure_event.wait(timeout=_SITE_PRESSURE_SYNC_TIMEOUT) + boundary.release_first.set() + for thread in threads: + thread.join(timeout=_SITE_PRESSURE_SYNC_TIMEOUT) + owners_finished = all(not thread.is_alive() for thread in threads) + + initial_failures = [ + error + for owner in owners + for error in owner_failures[owner] + if "冷却或已有在途搜索" in error + ] + successful_owners = [ + owner + for owner in owners + if owner in owner_results and owner_results[owner] + ] + blocked_owners = [ + owner + for owner in owners + if owner in owner_results + and not owner_results[owner] + and any( + "冷却或已有在途搜索" in error + for error in owner_failures[owner] + ) + ] + successful_owner = successful_owners[0] if len(successful_owners) == 1 else None + blocked_owner = blocked_owners[0] if len(blocked_owners) == 1 else None + initial_pressure_valid = bool( + first_request_started + and contention_observed + and boundary.active_at_rejection == 1 + and boundary.peak == 1 + and boundary.active == 0 + and len(initial_failures) == 1 + and len(successful_owners) == 1 + and len(blocked_owners) == 1 + and all(count == 1 for count in owner_invocations.values()) + and owners_finished + and set(owner_results) == set(owners) + and not owner_errors + ) + + success_reused = False + error_observed = False + error_cooldown_blocked = False + error_cooldown_persisted = False + reuse_calls = 0 + error_calls = 0 + cooldown_calls = 0 + if successful_owner and blocked_owner: + winner = chains[successful_owner] + loser = chains[blocked_owner] + winner.consume_subscription_site_budget_failures() + loser.consume_subscription_site_budget_failures() + + if site_id != case.site_count: + boundary.set_outcome("success") + calls_before_reuse = boundary.calls + reused = winner._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": site_id, "name": f"Site {site_id}"}, + keyword="governance-pressure-reuse", + mtype=None, + page=0, + ) + reuse_calls = boundary.calls - calls_before_reuse + reuse_record = _site_budget_record(engine, site_id) + success_reused = bool( + reused + and reuse_calls == 1 + and boundary.active == 0 + and reuse_record + and reuse_record.last_outcome == "success" + and reuse_record.lease_token is None + ) + winner.consume_subscription_site_budget_failures() + + if site_id == case.site_count: + boundary.set_outcome("rate_limited", error="HTTP 429") + calls_before_error = boundary.calls + error_result = winner._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": site_id, "name": f"Site {site_id}"}, + keyword="governance-pressure-error", + mtype=None, + page=0, + ) + error_calls = boundary.calls - calls_before_error + error_record = _site_budget_record(engine, site_id) + error_observed = bool( + not error_result + and error_calls == 1 + and boundary.active == 0 + and boundary.outcomes[-1:] == ("rate_limited",) + ) + winner.consume_subscription_site_budget_failures() + + calls_before_cooldown = boundary.calls + cooldown_result = loser._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": site_id, "name": f"Site {site_id}"}, + keyword="governance-pressure-cooldown", + mtype=None, + page=0, + ) + cooldown_calls = boundary.calls - calls_before_cooldown + cooldown_failures = loser.consume_subscription_site_budget_failures() + error_cooldown_blocked = bool( + not cooldown_result + and cooldown_calls == 0 + and cooldown_failures + ) + error_cooldown_persisted = bool( + error_record + and error_record.last_outcome == "rate_limited" + and error_record.lease_token is None + and error_record.next_allowed_at + > datetime.now(timezone.utc).isoformat(timespec="seconds") + ) + + site_pressure_valid = bool( + initial_pressure_valid + and ( + success_reused + if site_id != case.site_count + else error_observed + and error_cooldown_blocked + and error_cooldown_persisted + ) + ) + duplicate_site_claims_blocked += len(initial_failures) + successful_site_claims_reused += int(success_reused) + error_cooldown_claims_blocked += int(error_cooldown_blocked) + site_observations.append( + { + "site_id": site_id, + "owner_count": len(owners), + "owner_invocations": sum(owner_invocations.values()), + "owners_finished": owners_finished, + "request_active": boundary.active, + "request_peak": boundary.peak, + "request_calls": boundary.calls, + "request_active_at_rejection": boundary.active_at_rejection, + "budget_rejections": len(initial_failures), + "success_reused": success_reused, + "error_observed": error_observed, + "error_cooldown_blocked": error_cooldown_blocked, + "error_cooldown_persisted": error_cooldown_persisted, + "reuse_calls": reuse_calls, + "error_calls": error_calls, + "cooldown_calls": cooldown_calls, + "pressure_valid": site_pressure_valid, + } + ) + + peaks = [item["request_peak"] for item in site_observations] + return { + "site_peak_inflight_per_site": max(peaks, default=0), + "site_request_boundary_active": max( + (item["request_active"] for item in site_observations), + default=0, + ), + "site_request_boundary_peak": max(peaks, default=0), + "site_request_boundary_calls": sum( + item["request_calls"] for item in site_observations + ), + "site_pressure_owner_count": 2, + "site_pressure_concurrency_verified": all( + item["owner_count"] == 2 + and item["owner_invocations"] == 2 + and item["owners_finished"] + and item["request_active_at_rejection"] == 1 + for item in site_observations + ), + "site_pressure_success_release_reused": successful_site_claims_reused + == max(0, case.site_count - 1), + "site_pressure_error_observation_cooled": ( + error_cooldown_claims_blocked == 1 + and bool(site_observations) + and site_observations[-1]["error_observed"] + and site_observations[-1]["error_cooldown_persisted"] + ), + "site_pressure_valid": all( + item["pressure_valid"] for item in site_observations + ), + "site_observations": site_observations, + "duplicate_site_claims_blocked": duplicate_site_claims_blocked, + "error_cooldown_claims_blocked": error_cooldown_claims_blocked, + "successful_site_claims_reused": successful_site_claims_reused, + } + + +class _ScaleDownloadBoundary(DownloadChain): + """运行 canonical 批量选择，并在实际下载器边界返回确定性成功。""" + + def __init__(self, matched: list[str], downloaded: list[str]) -> None: + self._matched = matched + self._downloaded = downloaded + self._subscription_id = 0 + + @staticmethod + def _prepare_batch_download_contexts( + *, + contexts: list[Context], + **_kwargs: Any, + ) -> tuple[list[Context], dict[str, Any]]: + """固定快照保持候选顺序且不读取历史失败冷却。""" + return contexts, {} + + def batch_download( + self, + *, + contexts: list[Context], + no_exists: dict[Any, dict[int, NotExistMediaInfo]], + source: str, + governance: Any, + **_kwargs: Any, + ) -> tuple[list[Context], dict[Any, dict[int, NotExistMediaInfo]]]: + """记录 Match 提交集合后运行真实批量候选选择。""" + source_data = json.loads(source.removeprefix("Subscribe|")) + self._subscription_id = int(source_data["id"]) + for context in contexts: + self._matched.append(f"{self._subscription_id}|{_candidate_key(context)}") + return super().batch_download( + contexts=contexts, + no_exists=no_exists, + source=source, + governance=governance, + **_kwargs, + ) + + def download_single( + self, + context: Context, + *, + governance: Any, + **_kwargs: Any, + ) -> str: + """模拟下载器明确成功，并记录 canonical 选择出的资源。""" + if governance.cancelled and governance.cancelled(): + raise AssertionError("固定快照执行不应在下载前取消") + if governance.mark_started: + governance.mark_started() + self._downloaded.append(f"{self._subscription_id}|{_candidate_key(context)}") + return f"scale-download-{self._subscription_id}" + + +def _run_match_case(case: ScaleCase) -> dict[str, Any]: + """比较完整扫描与无损索引，并记录单轮事实租约和本地等待分位。""" + candidates, subscribes = _build_match_inputs(case) index = CandidateIndex(candidates) baseline_comparisons = case.subscription_count * case.candidate_count @@ -226,23 +752,199 @@ def _run_match_case(case: ScaleCase) -> dict[str, Any]: } -def _download_request(*, key: str, subscription_id: int, task_id: str) -> SubscriptionDownloadRequest: - """构造两个订阅记录共享的规范下载意图。""" - return SubscriptionDownloadRequest( - idempotency_key=key, - legacy_idempotency_key=None, - subscription_id=subscription_id, - task_id=task_id, - logical_identity='{"media_id":"77","media_type":"电视剧","season":1}', - resource_key="site-1.example:torrent-77", - coverage="episodes:E01-E03", - mode="normal", - delivery_scope='{"download_uri":"local:/downloads","downloader":"auto"}', +def _run_match_execution_case(case: ScaleCase) -> dict[str, Any]: + """运行实际 SubscribeChain Match 并对照四类完整业务集合。""" + candidates, subscribes = _build_match_inputs(case) + expected_matched: set[str] = set() + expected_downloaded: set[str] = set() + expected_missing = { + f"{subscribe.id}|2" + for subscribe in subscribes + if subscribe.id % 2 == 1 + } + expected_completed = { + str(subscribe.id) + for subscribe in subscribes + if subscribe.id % 2 == 0 + } + for subscribe in subscribes: + routed = _baseline_route(candidates, subscribe) + expected_matched.update( + f"{subscribe.id}|{candidate_key}" + for candidate_key in routed + ) + expected_downloaded.add(f"{subscribe.id}|{routed[0]}") + + actual_matched: list[str] = [] + actual_downloaded: list[str] = [] + actual_missing: list[str] = [] + actual_completed: list[str] = [] + settlement_ms: list[float] = [] + recognition_calls = 0 + progress_snapshots: list[dict[str, int]] = [] + started = time.perf_counter() + + class _ScaleMediaChain: + """用固定新鲜媒体事实替代目标规模下的真实 TMDB 请求。""" + + def recognize_media(self, **kwargs: Any) -> MediaInfo: + """按订阅身份返回两集电视剧的新鲜事实。""" + nonlocal recognition_calls + recognition_calls += 1 + media_id = str(kwargs["media_id"]) + return MediaInfo( + media_source=kwargs["media_source"], + media_id=media_id, + type=MediaType.TV, + title=f"Governance Show {media_id}", + year="2026", + season=1, + seasons={1: [1, 2]}, + ) + + @staticmethod + def recognize_by_meta(*_args: Any, **_kwargs: Any) -> MediaInfo: + """明确身份候选不应进入二次识别。""" + raise AssertionError("明确身份候选不应重新识别") + + @staticmethod + def supplement_media_info(mediainfo: MediaInfo) -> MediaInfo: + """固定样本无需访问其他元数据来源。""" + return mediainfo + + chain = object.__new__(SubscribeChain) + chain.subscription_repository = _ScaleSubscriptionRepository(subscribes) + chain.site_repository = _ScaleSiteRepository() + chain._match_lock = threading.Lock() + chain._search_queue_lock = threading.Lock() + chain._subscription_execution_admission = SubscriptionExecutionAdmission() + chain.get_sub_sites = lambda subscribe: list(subscribe.sites or []) + chain.get_params = lambda _subscribe: {} + chain.filter_torrents = lambda *, torrent_list, **_kwargs: torrent_list + + initial_missing = { + subscribe.id: [1] if subscribe.id % 2 == 0 else [1, 2] + for subscribe in subscribes + } + + def check_existing( + *, + subscribe: SubscriptionSnapshot, + mediakey: Any, + **_kwargs: Any, + ) -> tuple[bool, dict[Any, dict[int, NotExistMediaInfo]]]: + """返回每条订阅固定的初始缺集快照。""" + return False, { + mediakey: { + 1: NotExistMediaInfo( + season=1, + episodes=list(initial_missing[subscribe.id]), + total_episode=2, + start_episode=1, + ) + } + } + + def record_download_facts( + subscribe: SubscriptionSnapshot, + **_kwargs: Any, + ) -> dict[str, Any]: + """让完成判断消费下载边界结果，不引入持久化测试替身。""" + return { + "episodes": [1], + "fields": [], + "updated": False, + "subscribe": subscribe, + } + + def record_missing( + *, + no_exists: dict[Any, dict[int, NotExistMediaInfo]], + subscribe: SubscriptionSnapshot, + **_kwargs: Any, + ) -> None: + """记录实际完成判断收到的剩余缺集集合。""" + for seasons in no_exists.values(): + for missing in seasons.values(): + actual_missing.extend( + f"{subscribe.id}|{episode}" + for episode in missing.episodes or [] + ) + + chain.check_and_handle_existing_media = check_existing + chain._SubscribeChain__record_subscribe_download_facts = record_download_facts + chain._SubscribeChain__refresh_subscribe_progress_with_no_exists = record_missing + chain._SubscribeChain__finish_subscribe = ( + lambda *, subscribe, **_kwargs: actual_completed.append(str(subscribe.id)) ) + original_finish = chain.finish_subscribe_or_not + + def finish_and_record(**kwargs: Any) -> None: + """运行 canonical 完成判断并记录每条订阅收口时间。""" + original_finish(**kwargs) + settlement_ms.append((time.perf_counter() - started) * 1000) + + chain.finish_subscribe_or_not = finish_and_record + download_boundary = _ScaleDownloadBoundary(actual_matched, actual_downloaded) + + def record_progress(**kwargs: Any) -> None: + """保存 Match 对外发布的真实批次聚合状态。""" + if kwargs.get("data"): + progress_snapshots.append(dict(kwargs["data"])) + + with ( + patch.object(subscribe_match, "MediaChain", _ScaleMediaChain), + patch.object(subscribe_match, "TorrentHelper", _ScaleTorrentHelper), + patch.object( + subscribe_match, + "get_configured_system_config", + return_value=SimpleNamespace(get=lambda _key: []), + ), + patch.object( + subscribe_match, + "runtime_stop_state", + SimpleNamespace(is_system_stopped=False), + ), + patch.object( + download_batch, + "runtime_stop_state", + SimpleNamespace(is_system_stopped=False), + ), + patch.object( + subscribe_policy, + "DownloadChain", + return_value=download_boundary, + ), + ): + chain.match(candidates, progress_callback=record_progress) + + final_progress = progress_snapshots[-1] + return { + "name": case.name, + "subscription_count": case.subscription_count, + "site_count": case.site_count, + "candidate_count": case.candidate_count, + "matched_candidates": _compare_collections(expected_matched, actual_matched), + "downloaded_candidates": _compare_collections(expected_downloaded, actual_downloaded), + "remaining_missing_episodes": _compare_collections(expected_missing, actual_missing), + "completed_subscriptions": _compare_collections(expected_completed, actual_completed), + "fresh_fact_loads": recognition_calls, + "match_execution_completed": final_progress == { + "total": case.subscription_count, + "finished": case.subscription_count, + "completed": case.subscription_count, + "skipped": 0, + "failed": 0, + }, + "first_settlement_local_ms": round(settlement_ms[0], 3), + "subscription_settlement_local_p50_ms": _percentile(settlement_ms, 50), + "subscription_settlement_local_p95_ms": _percentile(settlement_ms, 95), + "batch_local_ms": round((time.perf_counter() - started) * 1000, 3), + } def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: - """验证持久队列、站点预算、恢复、取消与跨记录下载幂等。""" + """验证持久队列、站点预算、恢复、取消与订阅级准入。""" engine = create_engine( f"sqlite:///{workdir / (case.name + '.db')}", connect_args={"check_same_thread": False}, @@ -250,7 +952,6 @@ def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: Base.metadata.create_all(engine) factory = sessionmaker(bind=engine) search = TransactionalSubscriptionSearchRepository(factory) - download = TransactionalSubscriptionDownloadRepository(factory) started = time.perf_counter() enqueued = search.enqueue( @@ -327,42 +1028,7 @@ def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: ) cancelled_batch = search.get_batch(cancellation.batch.batch_id) - now = datetime.now(timezone.utc) - duplicate_site_claims_blocked = 0 - cooldown_claims_blocked = 0 - site_tokens: dict[int, str] = {} - for site_id in range(1, case.site_count + 1): - claim = search.claim_site( - site_id=site_id, - owner="site-worker-a", - lease_seconds=900, - ) - duplicate = search.claim_site( - site_id=site_id, - owner="site-worker-b", - lease_seconds=900, - ) - if not claim.acquired or not claim.lease_token: - raise AssertionError(f"站点 {site_id} 未取得初始预算") - site_tokens[site_id] = claim.lease_token - duplicate_site_claims_blocked += int(not duplicate.acquired) - for site_id, lease_token in site_tokens.items(): - outcome = "rate_limited" if site_id == case.site_count else "success" - delay = 900 if outcome == "rate_limited" else 60 - if not search.finish_site( - site_id=site_id, - lease_token=lease_token, - outcome=outcome, - next_allowed_at=(now + timedelta(seconds=delay)).isoformat(timespec="seconds"), - error="HTTP 429" if outcome == "rate_limited" else None, - ): - raise AssertionError(f"站点 {site_id} 预算无法收口") - cooldown = search.claim_site( - site_id=site_id, - owner="site-worker-c", - lease_seconds=900, - ) - cooldown_claims_blocked += int(not cooldown.acquired) + site_pressure = _run_site_pressure_case(case, search, engine) recovery_site = case.site_count + 1 site_first = search.claim_site( @@ -372,8 +1038,8 @@ def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: ) with Session(engine) as session: session.execute( - update(SubscriptionSiteBudget) - .where(SubscriptionSiteBudget.site_id == recovery_site) + update(SubscriptionSiteBudgetRecord) + .where(SubscriptionSiteBudgetRecord.site_id == recovery_site) .values(lease_expires_at=expired_at) ) session.commit() @@ -388,51 +1054,38 @@ def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: and site_recovered_claim.lease_token != site_first.lease_token ) - key = hashlib.sha256(f"{case.name}:shared-download".encode()).hexdigest() - first_download = download.claim( - _download_request(key=key, subscription_id=1, task_id="download-task-a") + admission = SubscriptionExecutionAdmission() + leases = [ + admission.try_acquire( + subscription_id=subscription_id, + operation="search", + ttl_seconds=60, + ) + for subscription_id in range(1, case.subscription_count + 1) + ] + acquired_leases = [lease for lease in leases if lease is not None] + same_subscription_conflicts_blocked = sum( + admission.try_acquire( + subscription_id=subscription_id, + operation="match", + ttl_seconds=60, + ) is None + for subscription_id in range(1, case.subscription_count + 1) ) - external_submissions = int(first_download.acquired) - first_token = first_download.snapshot.attempt_token or "" - if first_download.acquired: - download.mark_accepted( - idempotency_key=key, - attempt_token=first_token, - downloader="qb", - download_hash="hash-77", + initial_release_count = sum(admission.release(lease) for lease in acquired_leases) + reacquired_leases = [ + admission.try_acquire( + subscription_id=subscription_id, + operation="match", + ttl_seconds=60, ) - download.mark_succeeded( - idempotency_key=key, - attempt_token=first_token, - ) - duplicate_download = download.claim( - _download_request(key=key, subscription_id=2, task_id="download-task-b") - ) - external_submissions += int(duplicate_download.acquired) - - uncertain_key = hashlib.sha256(f"{case.name}:uncertain-download".encode()).hexdigest() - uncertain = download.claim( - _download_request( - key=uncertain_key, - subscription_id=3, - task_id="uncertain-task-a", - ) - ) - uncertain_token = uncertain.snapshot.attempt_token or "" - uncertain_frozen = bool( - uncertain.acquired - and download.mark_reconcile_required( - idempotency_key=uncertain_key, - attempt_token=uncertain_token, - error="controlled timeout", - ) - and not download.claim( - _download_request( - key=uncertain_key, - subscription_id=4, - task_id="uncertain-task-b", - ) - ).acquired + for subscription_id in range(1, case.subscription_count + 1) + ] + reacquired_count = sum(lease is not None for lease in reacquired_leases) + final_release_count = sum( + admission.release(lease) + for lease in reacquired_leases + if lease is not None ) return { @@ -452,18 +1105,13 @@ def _run_durable_governance(case: ScaleCase, workdir: Path) -> dict[str, Any]: "cancelled_batch_state": cancelled_batch.state if cancelled_batch else None, "cancelled_task_count": cancelled_batch.cancelled_count if cancelled_batch else 0, "site_count": case.site_count, - "site_peak_inflight_per_site": 1, - "duplicate_site_claims_blocked": duplicate_site_claims_blocked, - "cooldown_claims_blocked": cooldown_claims_blocked, + **site_pressure, "site_recovered_after_lease_expiry": site_recovered, - "external_download_submissions": external_submissions, - "duplicate_download_submissions": max(0, external_submissions - 1), - "duplicate_download_reused_success": bool( - not duplicate_download.acquired - and duplicate_download.snapshot.state == "succeeded" - and duplicate_download.snapshot.download_hash == "hash-77" - ), - "uncertain_download_frozen": uncertain_frozen, + "subscription_leases_acquired": len(acquired_leases), + "same_subscription_conflicts_blocked": same_subscription_conflicts_blocked, + "subscription_leases_released": initial_release_count, + "subscription_leases_reacquired": reacquired_count, + "reacquired_subscription_leases_released": final_release_count, } @@ -474,6 +1122,7 @@ def run_acceptance() -> dict[str, Any]: ScaleCase("large", 200, 20, 1200, 100), ) match_cases = [_run_match_case(case) for case in cases] + match_execution_cases = [_run_match_execution_case(case) for case in cases] with tempfile.TemporaryDirectory(prefix="subscription-governance-") as directory: durable_cases = [ _run_durable_governance(case, Path(directory)) @@ -490,6 +1139,30 @@ def run_acceptance() -> dict[str, Any]: case["fact_loads"] == case["media_count"] for case in match_cases ), + "match_execution_candidate_sets_equal": all( + case["matched_candidates"]["equal"] + for case in match_execution_cases + ), + "match_execution_download_sets_equal": all( + case["downloaded_candidates"]["equal"] + for case in match_execution_cases + ), + "match_execution_missing_sets_equal": all( + case["remaining_missing_episodes"]["equal"] + for case in match_execution_cases + ), + "match_execution_completion_sets_equal": all( + case["completed_subscriptions"]["equal"] + for case in match_execution_cases + ), + "match_execution_batches_completed": all( + case["match_execution_completed"] + for case in match_execution_cases + ), + "match_execution_fresh_facts_bounded": all( + case["fresh_fact_loads"] == match_case["media_count"] + for case, match_case in zip(match_execution_cases, match_cases) + ), "queue_completed": all( case["queue_state"] == "completed" and case["queue_finished_count"] == next( @@ -511,20 +1184,33 @@ def run_acceptance() -> dict[str, Any]: ), "site_pressure_bounded": all( case["site_peak_inflight_per_site"] == 1 + and case["site_request_boundary_peak"] == 1 + and case["site_request_boundary_active"] == 0 and case["duplicate_site_claims_blocked"] == case["site_count"] - and case["cooldown_claims_blocked"] == case["site_count"] + and case["error_cooldown_claims_blocked"] == 1 + and case["successful_site_claims_reused"] == case["site_count"] - 1 + and case["site_pressure_concurrency_verified"] + and case["site_pressure_success_release_reused"] + and case["site_pressure_error_observation_cooled"] + and case["site_pressure_valid"] for case in durable_cases ), - "download_idempotent": all( - case["external_download_submissions"] == 1 - and case["duplicate_download_submissions"] == 0 - and case["duplicate_download_reused_success"] - and case["uncertain_download_frozen"] + "subscription_admission_serializes": all( + case["subscription_leases_acquired"] + == next(item.subscription_count for item in cases if item.name == case["name"]) + and case["same_subscription_conflicts_blocked"] + == next(item.subscription_count for item in cases if item.name == case["name"]) + and case["subscription_leases_released"] + == next(item.subscription_count for item in cases if item.name == case["name"]) + and case["subscription_leases_reacquired"] + == next(item.subscription_count for item in cases if item.name == case["name"]) + and case["reacquired_subscription_leases_released"] + == next(item.subscription_count for item in cases if item.name == case["name"]) for case in durable_cases ), } return { - "schema_version": 1, + "schema_version": 4, "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "revision": _git_revision(), "environment": { @@ -534,11 +1220,33 @@ def run_acceptance() -> dict[str, Any]: "external_requests": 0, "external_downloads": 0, }, - "method": ( - "Deterministic local controlled sample. Latency fields measure only in-process " - "routing and isolated SQLite queue work; they are not production network SLOs." - ), + "method": { + "sample": "Deterministic local controlled snapshot", + "match_entrypoint": "SubscribeChain.match", + "download_selection": "DownloadChain.batch_download", + "fixed_external_boundaries": [ + "TMDB", + "media_server", + "downloader", + ], + "fixed_persistence_boundaries": [ + "subscription_repository", + "download_facts", + "subscription_progress", + "completion_side_effects", + ], + "fixed_policy_inputs": [ + "site_mapping", + "system_configuration", + "subscription_filters", + "torrent_attribute_filter", + "download_preparation", + ], + "latency_scope": "in-process Match and isolated SQLite queue work", + "production_network_slo": False, + }, "match_cases": match_cases, + "match_execution_cases": match_execution_cases, "durable_cases": durable_cases, "gates": gates, "passed": all(gates.values()), @@ -547,7 +1255,13 @@ def run_acceptance() -> dict[str, Any]: def main() -> int: """解析输出路径，运行门禁并以进程状态反映验收结果。""" - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + description=( + "运行 100/10/1000 与 200/20/1200 两档离线订阅治理验收；" + "结果仅代表固定输入下的本地 Match、下载选择和隔离 SQLite，" + "不代表生产网络 SLO。" + ) + ) parser.add_argument("--output", type=Path) args = parser.parse_args() result = run_acceptance() diff --git a/skills/database-operation/SKILL.md b/skills/database-operation/SKILL.md index 06f834d3c..7412f9236 100644 --- a/skills/database-operation/SKILL.md +++ b/skills/database-operation/SKILL.md @@ -236,17 +236,11 @@ python scripts/mp-db.py write "UPDATE subscribe SET state = 'S' WHERE id = 123" - Write boundary: Generated by subscription completion and archival; restore or delete through its business API. - Columns: `id`, `name`, `year`, `type`, `keyword`, `media_source`, `media_id`, `music_type`, `total_tracks`, `season`, `poster`, `backdrop`, `vote`, `description`, `filter`, `include`, `exclude`, `quality`, `resolution`, `effect`, `audio_quality`, `audio_format`, `min_bitrate`, `min_bit_depth`, `min_sample_rate`, `total_episode`, `start_episode`, `date`, `username`, `sites`, `best_version`, `best_version_full`, `current_priority`, `current_audio_format`, `current_bitrate`, `current_bit_depth`, `current_sample_rate`, `episode_priority`, `save_path`, `search_imdbid`, `custom_words`, `media_category`, `filter_groups`, `episode_group` -### `subscriptiondownloadsubmission` -- Purpose: Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes. -- Useful queries: Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership. -- Write boundary: Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states. -- Columns: `id`, `idempotency_key`, `subscription_id`, `task_id`, `logical_identity`, `resource_key`, `coverage`, `mode`, `delivery_scope`, `state`, `attempt_count`, `attempt_token`, `downloader`, `download_hash`, `available_at`, `last_error`, `created_at`, `updated_at`, `started_at`, `finished_at` - ### `subscriptionsearchbatch` - Purpose: Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests. - Useful queries: Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes. - Write boundary: Owned by subscription search orchestration; create and cancel batches through the subscription API. -- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error` +- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `skipped_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error` ### `subscriptionsearchtask` - Purpose: Stores one durable subscription search task per batch and subscription with leases and execution phases. diff --git a/skills/moviepilot-api/SKILL.md b/skills/moviepilot-api/SKILL.md index adadd6dbe..8825f24cd 100644 --- a/skills/moviepilot-api/SKILL.md +++ b/skills/moviepilot-api/SKILL.md @@ -1103,7 +1103,7 @@ Purpose: Read configured directory or storage settings. Purpose: Create one movie, TV, or music subscription. - `path_params`: none - `query`: none -- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title. +- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title. ### `subscription.delete` `DELETE /api/v1/subscribe/{subscribe_id}`; policy effect: `destructive_write`. @@ -1277,7 +1277,7 @@ Purpose: Set one accessible subscription to running, paused, or stopped state. Purpose: Update one existing movie, TV, or music subscription. - `path_params`: none - `query`: none -- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title. +- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title. ### `subscription.user.list` `GET /api/v1/subscribe/user/{username}`; policy effect: `safe_read`. @@ -1681,6 +1681,18 @@ This runtime model has no directly writable fields. MoviePilot media type. This runtime model has no directly writable fields. +#### `SubscriptionExecutionStatus` +Subscription refresh execution status and progress summary. +- `batch_id` (string|null): Stable subscription search batch identifier. +- `can_cancel` (boolean; default `False`): Whether the current subscription execution can be cancelled. +- `current_site_id` (integer|null): Configured site ID currently handling the subscription execution. +- `error` (string|null): Human-readable workflow, provider, or execution error message. +- `phase*` (string): Current phase of a subscription execution. +- `source` (string|null): Exact metadata or recommendation source selected by the operation. +- `state*` (string): Current site, subscription, marketplace, or transfer state filter. +- `task_id` (string|null): Stable durable transfer task ID returned by transfer.manual_reviews. +- `updated_at*` (string): Timestamp when the subscription execution status was last updated. + #### `TorrentInfo` One torrent candidate returned by MoviePilot search. - `category` (string|null): MoviePilot media category or filter-group category, depending on the operation. diff --git a/tests/conftest.py b/tests/conftest.py index 03f2bca31..6b4415d09 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -285,7 +285,6 @@ def configure_plugin_system_services(): from app.db.adapters.mediaserver import TransactionalMediaServerRepository from app.db.adapters.site import TransactionalSiteRepository from app.db.adapters.subscription import TransactionalSubscriptionRepository - from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository from app.db.adapters.transaction import TransactionalWriteRunner from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository from app.db.adapters.transfer.execution import ( @@ -413,7 +412,6 @@ def configure_plugin_system_services(): transfer_execution_repository=TransactionalTransferExecutionRepository(SessionFactory), media_server_repository=TransactionalMediaServerRepository(SessionFactory), download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory), - subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory), user_repository=user_repository(), configuration=build_chain_runtime_config(settings), ) diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index e70f5101f..2432b3bb5 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -1089,8 +1089,8 @@ "runtime_only": true } }, - "edge_count": 7874, - "edge_sha256": "b4ca47899f8af4366446ad76546b5b0cf3c5e87f38b603d593241acd52fba9f3", + "edge_count": 7845, + "edge_sha256": "aca69eac132e327d7a1dea770c185e7be5ab47bb41211749f6130c065a957919", "edges": [ "app -> app.foundation", "app -> app.foundation.environment", @@ -3527,8 +3527,6 @@ "app.application.subscription.sitebudget -> app.runtime", "app.application.subscription.sitebudget -> app.runtime.stop", "app.application.subscription.status -> app.application", - "app.application.subscription.status -> app.application.download", - "app.application.subscription.status -> app.application.download.admission", "app.application.subscription.status -> app.application.subscription", "app.application.subscription.status -> app.application.subscription.execution", "app.application.subscription.write -> app.application", @@ -3634,9 +3632,13 @@ "app.chain._messaging -> app.schemas.types", "app.chain._music -> app.application", "app.chain._music -> app.application.configuration", + "app.chain._music -> app.application.download", + "app.chain._music -> app.application.download.admission", "app.chain._music -> app.application.subscription", "app.chain._music -> app.application.subscription.contract", + "app.chain._music -> app.application.subscription.execution", "app.chain._music -> app.application.subscription.mutation", + "app.chain._music -> app.application.subscription.sitebudget", "app.chain._music -> app.application.torrent", "app.chain._music -> app.application.torrent.download", "app.chain._music -> app.chain", @@ -3725,18 +3727,6 @@ "app.chain.douban -> app.schemas", "app.chain.douban -> app.schemas.context", "app.chain.douban -> app.schemas.types", - "app.chain.download.admission -> app.application", - "app.chain.download.admission -> app.application.download", - "app.chain.download.admission -> app.application.download.admission", - "app.chain.download.admission -> app.chain", - "app.chain.download.admission -> app.chain.download", - "app.chain.download.admission -> app.chain.download.contract", - "app.chain.download.admission -> app.domain", - "app.chain.download.admission -> app.domain.context", - "app.chain.download.admission -> app.domain.episode", - "app.chain.download.admission -> app.schemas", - "app.chain.download.admission -> app.schemas.media", - "app.chain.download.admission -> app.schemas.types", "app.chain.download.batch -> app.application", "app.chain.download.batch -> app.application.download", "app.chain.download.batch -> app.application.download.admission", @@ -3773,7 +3763,6 @@ "app.chain.download.existence -> app.schemas.types", "app.chain.download.facade -> app.chain", "app.chain.download.facade -> app.chain.download", - "app.chain.download.facade -> app.chain.download.admission", "app.chain.download.facade -> app.chain.download.batch", "app.chain.download.facade -> app.chain.download.existence", "app.chain.download.facade -> app.chain.download.failure", @@ -4482,6 +4471,8 @@ "app.chain.subscribe.facade -> app.application", "app.chain.subscribe.facade -> app.application.messaging", "app.chain.subscribe.facade -> app.application.messaging.subscribe", + "app.chain.subscribe.facade -> app.application.subscription", + "app.chain.subscribe.facade -> app.application.subscription.execution", "app.chain.subscribe.facade -> app.chain", "app.chain.subscribe.facade -> app.chain._interaction", "app.chain.subscribe.facade -> app.chain._music", @@ -4530,7 +4521,9 @@ "app.chain.subscribe.match -> app.application.subscription", "app.chain.subscribe.match -> app.application.subscription.candidates", "app.chain.subscribe.match -> app.application.subscription.contract", + "app.chain.subscribe.match -> app.application.subscription.execution", "app.chain.subscribe.match -> app.application.subscription.facts", + "app.chain.subscribe.match -> app.application.subscription.sitebudget", "app.chain.subscribe.match -> app.application.torrent", "app.chain.subscribe.match -> app.application.torrent.download", "app.chain.subscribe.match -> app.chain", @@ -4588,6 +4581,7 @@ "app.chain.subscribe.policy -> app.application.download.admission", "app.chain.subscribe.policy -> app.application.subscription", "app.chain.subscribe.policy -> app.application.subscription.contract", + "app.chain.subscribe.policy -> app.application.subscription.execution", "app.chain.subscribe.policy -> app.application.subscription.priority", "app.chain.subscribe.policy -> app.chain", "app.chain.subscribe.policy -> app.chain.download", @@ -4637,7 +4631,6 @@ "app.chain.subscribe.reconcile -> app.schemas.types", "app.chain.subscribe.refresh -> app.application", "app.chain.subscribe.refresh -> app.application.subscription", - "app.chain.subscribe.refresh -> app.application.subscription.candidates", "app.chain.subscribe.refresh -> app.application.subscription.contract", "app.chain.subscribe.refresh -> app.application.subscription.facts", "app.chain.subscribe.refresh -> app.application.subscription.priority", @@ -5211,15 +5204,6 @@ "app.db.adapters.subscription -> app.schemas", "app.db.adapters.subscription -> app.schemas.common", "app.db.adapters.subscription -> app.schemas.types", - "app.db.adapters.subscriptiondownload -> app.application", - "app.db.adapters.subscriptiondownload -> app.application.download", - "app.db.adapters.subscriptiondownload -> app.application.download.admission", - "app.db.adapters.subscriptiondownload -> app.db", - "app.db.adapters.subscriptiondownload -> app.db.models", - "app.db.adapters.subscriptiondownload -> app.db.models.subscriptiondownload", - "app.db.adapters.subscriptiondownload -> app.db.oper", - "app.db.adapters.subscriptiondownload -> app.db.oper.subscriptiondownload", - "app.db.adapters.subscriptiondownload -> app.db.uow", "app.db.adapters.subscriptionsearch -> app.application", "app.db.adapters.subscriptionsearch -> app.application.subscription", "app.db.adapters.subscriptionsearch -> app.application.subscription.execution", @@ -5231,13 +5215,10 @@ "app.db.adapters.subscriptionsearch -> app.db.oper.subscriptionsearch", "app.db.adapters.subscriptionsearch -> app.db.uow", "app.db.adapters.subscriptionstatus -> app.application", - "app.db.adapters.subscriptionstatus -> app.application.download", - "app.db.adapters.subscriptionstatus -> app.application.download.admission", "app.db.adapters.subscriptionstatus -> app.application.subscription", "app.db.adapters.subscriptionstatus -> app.application.subscription.execution", "app.db.adapters.subscriptionstatus -> app.db", "app.db.adapters.subscriptionstatus -> app.db.models", - "app.db.adapters.subscriptionstatus -> app.db.models.subscriptiondownload", "app.db.adapters.subscriptionstatus -> app.db.models.subscriptionsearch", "app.db.adapters.transaction -> app.db", "app.db.adapters.transaction -> app.db.uow", @@ -5377,8 +5358,6 @@ "app.db.models.subscribehistory -> app.db.models._constraints", "app.db.models.subscribehistory -> app.schemas", "app.db.models.subscribehistory -> app.schemas.types", - "app.db.models.subscriptiondownload -> app.db", - "app.db.models.subscriptiondownload -> app.db.base", "app.db.models.subscriptionsearch -> app.db", "app.db.models.subscriptionsearch -> app.db.base", "app.db.models.systemconfig -> app.db", @@ -5487,13 +5466,6 @@ "app.db.oper.subscribehistory -> app.schemas.common", "app.db.oper.subscribehistory -> app.schemas.query", "app.db.oper.subscribehistory -> app.schemas.types", - "app.db.oper.subscriptiondownload -> app.application", - "app.db.oper.subscriptiondownload -> app.application.download", - "app.db.oper.subscriptiondownload -> app.application.download.admission", - "app.db.oper.subscriptiondownload -> app.db", - "app.db.oper.subscriptiondownload -> app.db.base", - "app.db.oper.subscriptiondownload -> app.db.models", - "app.db.oper.subscriptiondownload -> app.db.models.subscriptiondownload", "app.db.oper.subscriptionsearch -> app.db", "app.db.oper.subscriptionsearch -> app.db.base", "app.db.oper.subscriptionsearch -> app.db.models", @@ -8165,7 +8137,6 @@ "app.startup.composition.chain -> app.db.adapters.chain", "app.startup.composition.chain -> app.db.adapters.download", "app.startup.composition.chain -> app.db.adapters.mediaserver", - "app.startup.composition.chain -> app.db.adapters.subscriptiondownload", "app.startup.composition.chain -> app.db.adapters.transfer", "app.startup.composition.chain -> app.db.adapters.transfer.admission", "app.startup.composition.chain -> app.db.oper", @@ -8967,7 +8938,7 @@ "app.workflow.actions.transfer_file -> app.workflow", "app.workflow.actions.transfer_file -> app.workflow.actions" ], - "module_count": 944, + "module_count": 940, "modules": [ "app", "app.adapters", @@ -9294,7 +9265,6 @@ "app.chain.dashboard", "app.chain.douban", "app.chain.download", - "app.chain.download.admission", "app.chain.download.batch", "app.chain.download.contract", "app.chain.download.existence", @@ -9401,7 +9371,6 @@ "app.db.adapters.query", "app.db.adapters.site", "app.db.adapters.subscription", - "app.db.adapters.subscriptiondownload", "app.db.adapters.subscriptionsearch", "app.db.adapters.subscriptionstatus", "app.db.adapters.transaction", @@ -9437,7 +9406,6 @@ "app.db.models.siteuserdata", "app.db.models.subscribe", "app.db.models.subscribehistory", - "app.db.models.subscriptiondownload", "app.db.models.subscriptionsearch", "app.db.models.systemconfig", "app.db.models.transferexecutionstep", @@ -9461,7 +9429,6 @@ "app.db.oper.site", "app.db.oper.subscribe", "app.db.oper.subscribehistory", - "app.db.oper.subscriptiondownload", "app.db.oper.subscriptionsearch", "app.db.oper.systemconfig", "app.db.oper.transferexecutionstep", diff --git a/tests/fixtures/architecture/mypy-baseline.json b/tests/fixtures/architecture/mypy-baseline.json index 6d1382d28..37903351d 100644 --- a/tests/fixtures/architecture/mypy-baseline.json +++ b/tests/fixtures/architecture/mypy-baseline.json @@ -1006,7 +1006,7 @@ }, "app/chain/torrents.py": { "arg-type": 22, - "assignment": 3, + "assignment": 2, "no-untyped-call": 1, "no-untyped-def": 15, "type-arg": 5, diff --git a/tests/fixtures/architecture/startup-performance-baseline.json b/tests/fixtures/architecture/startup-performance-baseline.json index cdfcc788f..b2b0415df 100644 --- a/tests/fixtures/architecture/startup-performance-baseline.json +++ b/tests/fixtures/architecture/startup-performance-baseline.json @@ -6,7 +6,7 @@ "repeat": 3, "targets": { "app.startup.lifecycle": { - "loaded_app_module_count": 515, + "loaded_app_module_count": 511, "max_ms": 1115.407, "median_ms": 1106.606, "min_ms": 1098.166, @@ -17,7 +17,7 @@ ] }, "app.factory": { - "loaded_app_module_count": 527, + "loaded_app_module_count": 523, "max_ms": 1140.393, "median_ms": 1134.83, "min_ms": 1119.004, @@ -28,7 +28,7 @@ ] }, "app.main": { - "loaded_app_module_count": 529, + "loaded_app_module_count": 525, "max_ms": 1180.675, "median_ms": 1165.915, "min_ms": 1161.859, diff --git a/tests/test_download_architecture.py b/tests/test_download_architecture.py index 8b49dade2..30ba6fbc0 100644 --- a/tests/test_download_architecture.py +++ b/tests/test_download_architecture.py @@ -39,7 +39,6 @@ def test_download_monolith_is_retired_and_package_owners_are_complete() -> None: assert not (CHAIN_ROOT / "download.py").exists() assert {path.name for path in DOWNLOAD_PACKAGE.glob("*.py")} == { "__init__.py", - "admission.py", "batch.py", "contract.py", "existence.py", diff --git a/tests/test_music_subscribe.py b/tests/test_music_subscribe.py index b012013ef..f02670d81 100644 --- a/tests/test_music_subscribe.py +++ b/tests/test_music_subscribe.py @@ -11,8 +11,13 @@ from app.application.subscription.contract import ( SubscriptionSnapshot, build_subscribe_meta, ) +from app.application.subscription.execution import ( + SubscriptionExecutionAdmission, + SubscriptionExecutionContext, +) from app.application.subscription.mutation import SubscriptionMutation -from app.chain.subscribe import SubscribeChain +from app.application.subscription.sitebudget import SubscriptionSearchCancelled +from app.chain.subscribe.facade import SubscribeChain from app.domain.context import ( MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_ARTIST, @@ -120,6 +125,23 @@ def _configure_subscription_write(chain, repository) -> None: chain.sync_subscription_mutation_scope = mutation_scope +def _execution_context(*, cancelled=None) -> SubscriptionExecutionContext: + """构造音乐订阅使用的独立执行上下文。""" + admission = SubscriptionExecutionAdmission() + lease = admission.try_acquire( + subscription_id=7, + operation="search", + ttl_seconds=60, + ) + assert lease is not None + return SubscriptionExecutionContext( + lease=lease, + admission=admission, + task_id="music-task-7", + cancel_requested=cancelled, + ) + + def test_build_subscribe_meta_returns_music_meta(): """音乐订阅应构造 MetaMusic，而不是交给影视标题解析器。""" meta = build_subscribe_meta(_subscribe()) @@ -190,6 +212,85 @@ def test_music_subscribe_reuses_search_download_and_finish_flow(): chain.finish_subscribe_or_not.assert_called_once() +def test_music_search_honours_cancel_before_external_work(): + """音乐搜索在安全边界收到取消后不得继续访问站点。""" + subscribe = _subscribe() + chain = SubscribeChain() + execution_context = _execution_context(cancelled=lambda: True) + + with patch("app.chain._music.SearchChain") as search_chain, \ + pytest.raises(SubscriptionSearchCancelled): + chain._search_music_subscribe( + subscribe, + execution_context=execution_context, + ) + + search_chain.assert_not_called() + + +def test_music_download_marks_shared_execution_context_before_side_effect(): + """音乐下载必须把取消和副作用边界传入统一下载治理。""" + cancelled = [False] + subscribe = _subscribe() + target = _music_info() + downloaded = Context( + torrent_info=TorrentInfo( + title="周杰伦 - 晴天 FLAC", + category=MediaType.MUSIC.value, + ), + meta_info=MetaMusic.from_music_info(target), + media_info=target, + ) + execution_context = _execution_context(cancelled=lambda: cancelled[0]) + download_chain = Mock() + + def batch_download(**kwargs): + """模拟下载器边界内开始提交后才收到取消。""" + governance = kwargs["governance"] + assert governance.cancelled() is False + governance.mark_started() + cancelled[0] = True + return [downloaded], None + + download_chain.batch_download.side_effect = batch_download + repository = Mock() + repository.get.return_value = subscribe + chain = SubscribeChain() + chain.subscription_repository = repository + chain.finish_subscribe_or_not = Mock() + + with patch("app.chain._music.DownloadChain", return_value=download_chain): + chain._download_music_subscribe( + subscribe, + target, + [downloaded], + execution_context=execution_context, + ) + + assert execution_context.download_started is True + chain.finish_subscribe_or_not.assert_called_once() + + +def test_music_download_rechecks_paused_state_before_submission(): + """候选准备后暂停的音乐订阅不得进入下载器。""" + subscribe = _subscribe() + paused = _subscribe(state="S") + repository = Mock() + repository.get.return_value = paused + chain = SubscribeChain() + chain.subscription_repository = repository + + with patch("app.chain._music.DownloadChain") as download_chain: + chain._download_music_subscribe( + subscribe, + _music_info(), + [Context()], + execution_context=_execution_context(), + ) + + download_chain.assert_not_called() + + def test_music_subscribe_filters_declared_bitrate_and_format(): """音乐订阅应按规范化格式和最低码率过滤站点资源。""" subscribe = _subscribe(audio_format="MP3", min_bitrate=320000) @@ -274,6 +375,7 @@ def test_music_best_version_persists_downloaded_rule_priority(): download_chain.batch_download.return_value = ([downloaded], None) subscribe_oper = Mock() updated = _subscribe(best_version=1, current_priority=100) + subscribe_oper.get.return_value = subscribe subscribe_oper.update.return_value = updated chain = SubscribeChain() chain.finish_subscribe_or_not = Mock() diff --git a/tests/test_subscribe_chain.py b/tests/test_subscribe_chain.py index cff3fe1c5..e8a57bcac 100644 --- a/tests/test_subscribe_chain.py +++ b/tests/test_subscribe_chain.py @@ -701,16 +701,15 @@ class TestSubscribeChain: def test_match_title_fallback_calls_torrent_match_from_class(self): """确保标题兜底匹配不依赖 TorrentHelper 实例绑定。""" - - class _ReachedTitleMatch(Exception): - """标记测试已经进入标题匹配函数体。""" + reached = [] class _PlainTorrentHelper: """模拟需要按类调用的 TorrentHelper 形态。""" def match_torrent(mediainfo, torrent_meta, torrent): """标记类级调用已经正确进入匹配逻辑。""" - raise _ReachedTitleMatch + reached.append((mediainfo, torrent_meta, torrent)) + return False def filter_torrent(self, *args, **kwargs): """保持订阅匹配后续过滤流程可继续执行。""" @@ -753,9 +752,14 @@ class TestSubscribeChain: """返回当前测试构造的订阅列表。""" return [subscribe] + def get(self, subscribe_id): + """返回取得订阅准入后的最新快照。""" + return subscribe if subscribe_id == subscribe.id else None + chain = SubscribeChain() chain.subscription_repository = _SubscribeOper() chain.check_and_handle_existing_media = lambda **kwargs: (False, {}) + chain.finish_subscribe_or_not = lambda **kwargs: None with ( patch.object( @@ -764,10 +768,15 @@ class TestSubscribeChain: _PlainTorrentHelper, ), _patch_media_recognize(SUBSCRIBE_CHAIN_MODULE, mediainfo), - pytest.raises(_ReachedTitleMatch), ): chain.match({"test.example": [context]}) + assert len(reached) == 1 + actual_media, actual_meta, actual_torrent = reached[0] + assert actual_media.title_year == mediainfo.title_year + assert actual_meta is context.meta_info + assert actual_torrent is context.torrent_info + def test_match_accepts_special_season_zero_candidate(self): """S0 订阅应允许 S00 候选资源进入下载候选，不能按未指定季处理。""" @@ -862,6 +871,16 @@ class TestSubscribeChain: assert len(download_calls) == 1 assert download_calls[0]["contexts"][0].meta_info.begin_season == 0 + execution_context = download_calls[0]["execution_context"] + assert execution_context.lease.subscription_id == subscribe.id + assert execution_context.lease.operation == "match" + replacement = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="search", + ttl_seconds=60, + ) + assert replacement is not None + assert chain._subscription_execution_admission.release(replacement) is True def test_get_episode_priority_falls_back_to_current_priority(self): subscribe = self._build_subscribe(current_priority=80, episode_priority=None) diff --git a/tests/test_subscribe_search_state.py b/tests/test_subscribe_search_state.py index dd7a2eb5c..3e80f531c 100644 --- a/tests/test_subscribe_search_state.py +++ b/tests/test_subscribe_search_state.py @@ -1,3 +1,4 @@ +import threading from contextlib import contextmanager from dataclasses import replace from datetime import datetime, timedelta @@ -7,9 +8,10 @@ from unittest.mock import Mock, patch import pytest from app.application.subscription.contract import SubscriptionPatch, SubscriptionSnapshot +from app.application.subscription.execution import SubscriptionExecutionAdmission from app.application.subscription.mutation import SubscriptionMutation -from app.chain.subscribe import SubscribeChain from app.chain.subscribe import search as subscribe_search +from app.chain.subscribe.facade import SubscribeChain from app.schemas.types import MediaType @@ -150,14 +152,19 @@ def test_targeted_batch_searches_all_ids_without_state_scan(monkeypatch) -> None chain.subscription_repository = subscribe_oper chain.search(sids=(31, 32), state=None, manual=False) - assert [item.args for item in subscribe_oper.get.call_args_list] == [(31,), (32,)] + assert [item.args for item in subscribe_oper.get.call_args_list] == [ + (31,), + (32,), + (31,), + (32,), + ] subscribe_oper.list.assert_not_called() assert media_chain.recognize_media.call_count == 2 def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None: """订阅搜索锁超时后必须中止，不能在无锁状态下继续访问订阅。""" - monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock()) + monkeypatch.setattr(SubscribeChain, "_search_queue_lock", _TimedOutLock()) subscribe_oper = Mock() progress = Mock() @@ -176,7 +183,7 @@ def test_subscribe_search_releases_lock_when_repository_query_fails(monkeypatch) """取得搜索锁后即使订阅查询失败，也必须释放进程级互斥锁。""" lock = Mock() lock.acquire.return_value = True - monkeypatch.setattr(SubscribeChain, "_rlock", lock) + monkeypatch.setattr(SubscribeChain, "_search_queue_lock", lock) repository = Mock() repository.list.side_effect = RuntimeError("query failed") chain = object.__new__(SubscribeChain) @@ -228,9 +235,45 @@ def test_subscribe_search_progress_preserves_public_callback_payload() -> None: ] +def test_inline_search_conflict_does_not_report_false_completion(monkeypatch) -> None: + """兼容搜索遇到同订阅冲突时必须报告未执行，而不是成功完成。""" + subscribe = replace( + _new_subscribe(datetime.now() - timedelta(minutes=2)), + state="R", + ) + repository = Mock() + repository.get.return_value = subscribe + progress = Mock() + chain = object.__new__(SubscribeChain) + chain.subscription_repository = repository + chain._search_queue_lock = threading.Lock() + chain._match_lock = threading.Lock() + chain._subscription_execution_admission = SubscriptionExecutionAdmission() + match_lease = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="match", + ttl_seconds=60, + ) + assert match_lease is not None + + with patch.object(subscribe_search, "SearchChain", return_value=Mock()): + chain.search( + sid=subscribe.id, + state=None, + progress_callback=progress, + ) + + assert progress.call_args.kwargs == { + "value": 100, + "text": "订阅搜索结束，部分订阅本轮未执行或未完成", + "data": {"total": 1, "finished": 0}, + } + assert chain._subscription_execution_admission.release(match_lease) is True + + def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None: """订阅匹配锁超时后必须中止，不能绕过防重复下载边界。""" - monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock()) + monkeypatch.setattr(SubscribeChain, "_match_lock", _TimedOutLock()) progress = Mock() chain = object.__new__(SubscribeChain) diff --git a/tests/test_subscription_candidate_batch.py b/tests/test_subscription_candidate_batch.py index dee071249..75925c9f7 100644 --- a/tests/test_subscription_candidate_batch.py +++ b/tests/test_subscription_candidate_batch.py @@ -3,7 +3,7 @@ from types import SimpleNamespace from unittest.mock import Mock, patch -from app.application.subscription.candidates import CandidateBatch, CandidateIndex +from app.application.subscription.candidates import CandidateIndex from app.application.subscription.contract import SubscriptionSnapshot from app.application.subscription.facts import FreshFactLease from app.chain.torrents import TorrentsChain @@ -68,8 +68,13 @@ def _subscribe(**overrides) -> SubscriptionSnapshot: return SubscriptionSnapshot(**values) -def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta(): - """刷新批次必须保留完整缓存，同时只把本轮新增资源放入 fresh 集合。""" +def _candidate_count(groups: dict[str, list[Context]]) -> int: + """统计分站点候选总数。""" + return sum(len(contexts) for contexts in groups.values()) + + +def test_refresh_returns_complete_cache_with_new_candidates(): + """刷新结果必须包含既有缓存和本轮新增资源的完整候选。""" chain = TorrentsChain() existing = _context("既有剧集 S01E01", media_id="100") duplicate = TorrentInfo( @@ -115,18 +120,13 @@ def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta(): ), ), ): - batch = chain.refresh_batch(stype="rss", sites=[1]) + candidates = chain.refresh(stype="rss", sites=[1]) - assert isinstance(batch, CandidateBatch) - assert batch.source == "rss" - assert batch.finished_at is not None - assert [item.torrent_info.title for item in batch.candidates["example.com"]] == [ + assert [item.torrent_info.title for item in candidates["example.com"]] == [ existing.torrent_info.title, fresh.title, ] - assert [item.torrent_info.title for item in batch.fresh_candidates["example.com"]] == [fresh.title] - assert CandidateBatch.count(batch.candidates) == 2 - assert CandidateBatch.count(batch.fresh_candidates) == 1 + assert _candidate_count(candidates) == 2 def test_candidate_index_routes_all_canonical_fallback_classes_without_loss(): @@ -214,7 +214,7 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product(): for subscribe in subscribes: groups = candidate_index.route_for_match(subscribe) examined += candidate_index.last_examined_count - routed += CandidateBatch.count(groups) + routed += _candidate_count(groups) lease = FreshFactLease() fact_loads = [] @@ -231,9 +231,9 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product(): ) assert len(candidates) == site_count - assert CandidateBatch.count(candidates) == 1000 + assert _candidate_count(candidates) == 1000 assert examined == routed == 2000 - assert examined < subscription_count * CandidateBatch.count(candidates) // 50 + assert examined < subscription_count * _candidate_count(candidates) // 50 assert len(fact_loads) == media_count assert lease.loads == media_count assert lease.hits == subscription_count - media_count diff --git a/tests/test_subscription_cross_channel_admission.py b/tests/test_subscription_cross_channel_admission.py new file mode 100644 index 000000000..3066be8df --- /dev/null +++ b/tests/test_subscription_cross_channel_admission.py @@ -0,0 +1,129 @@ +"""订阅 Search/Match 公共入口的跨通道准入测试。""" + +import threading +from datetime import datetime, timedelta +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from app.application.subscription.contract import SubscriptionSnapshot +from app.application.subscription.execution import SubscriptionExecutionAdmission +from app.chain.subscribe.facade import SubscribeChain +from app.domain.context import Context +from app.schemas.types import MediaType + + +def _subscribe(subscribe_id: int) -> SubscriptionSnapshot: + """构造已越过新增保护期的活动电影订阅。""" + return SubscriptionSnapshot( + id=subscribe_id, + name=f"并发电影 {subscribe_id}", + year="2026", + type=MediaType.MOVIE.value, + media_source="themoviedb", + media_id=str(5000 + subscribe_id), + state="R", + date=(datetime.now() - timedelta(minutes=2)).strftime("%Y-%m-%d %H:%M:%S"), + ) + + +def _repository(*subscribes: SubscriptionSnapshot) -> SimpleNamespace: + """提供公共入口重新读取所需的稳定订阅仓储。""" + snapshots = {subscribe.id: subscribe for subscribe in subscribes} + return SimpleNamespace( + get=snapshots.get, + list=lambda _state: list(snapshots.values()), + ) + + +def _configure_channel_state(monkeypatch) -> None: + """为不同 SubscribeChain 实例安装进程级共享准入与独立通道锁。""" + monkeypatch.setattr(SubscribeChain, "_match_lock", threading.Lock()) + monkeypatch.setattr(SubscribeChain, "_search_queue_lock", threading.Lock()) + monkeypatch.setattr( + SubscribeChain, + "_subscription_execution_admission", + SubscriptionExecutionAdmission(), + ) + + +def test_public_search_and_match_skip_same_subscription_across_instances(monkeypatch) -> None: + """两个独立 Chain 实例处理同一订阅时只能有一个进入业务路径。""" + _configure_channel_state(monkeypatch) + subscribe = _subscribe(7) + repository = _repository(subscribe) + search_chain = object.__new__(SubscribeChain) + search_chain.subscription_repository = repository + match_chain = object.__new__(SubscribeChain) + match_chain.subscription_repository = repository + match_chain.get_states_for_search = lambda state: state + match_chain._prepare_match_torrents = lambda torrents: torrents + + search_started = threading.Event() + release_search = threading.Event() + + def process_search(item, _searchchain, *, execution_context): + """持有 Search 准入，直到 Match 完成冲突探测。""" + assert execution_context.lease.subscription_id == item.id + search_started.set() + assert release_search.wait(timeout=3) + return item + + match_subscription = Mock(side_effect=AssertionError("冲突 Match 不得进入订阅业务路径")) + search_chain._process_search_subscription = process_search + match_chain._match_subscription = match_subscription + worker = threading.Thread( + target=search_chain.search, + kwargs={"sid": subscribe.id, "state": None}, + ) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + worker.start() + assert search_started.wait(timeout=3) + match_chain.match({"example.org": [Context()]}) + release_search.set() + worker.join(timeout=3) + + assert worker.is_alive() is False + match_subscription.assert_not_called() + + +def test_public_search_and_match_allow_different_subscriptions_in_parallel(monkeypatch) -> None: + """Search 持有一条订阅时 Match 仍可处理另一条订阅。""" + _configure_channel_state(monkeypatch) + search_subscribe = _subscribe(8) + match_subscribe = _subscribe(9) + search_chain = object.__new__(SubscribeChain) + search_chain.subscription_repository = _repository(search_subscribe) + match_chain = object.__new__(SubscribeChain) + match_chain.subscription_repository = _repository(match_subscribe) + match_chain.get_states_for_search = lambda state: state + match_chain._prepare_match_torrents = lambda torrents: torrents + + search_started = threading.Event() + release_search = threading.Event() + + def process_search(item, _searchchain, *, execution_context): + """阻塞 Search 以证明 Match 不依赖 Search 通道锁。""" + assert execution_context.lease.subscription_id == item.id + search_started.set() + assert release_search.wait(timeout=3) + return item + + match_subscription = Mock() + search_chain._process_search_subscription = process_search + match_chain._match_subscription = match_subscription + worker = threading.Thread( + target=search_chain.search, + kwargs={"sid": search_subscribe.id, "state": None}, + ) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + worker.start() + assert search_started.wait(timeout=3) + match_chain.match({"example.org": [Context()]}) + release_search.set() + worker.join(timeout=3) + + assert worker.is_alive() is False + match_subscription.assert_called_once() + assert match_subscription.call_args.kwargs["subscribe"].id == match_subscribe.id diff --git a/tests/test_subscription_download_governance.py b/tests/test_subscription_download_governance.py index b92ff4763..7c1d385d9 100644 --- a/tests/test_subscription_download_governance.py +++ b/tests/test_subscription_download_governance.py @@ -1,9 +1,8 @@ -"""订阅下载跨入口幂等、不确定终态与取消补偿测试。""" +"""订阅下载取消边界、普通失败语义与账本移除迁移测试。""" from __future__ import annotations import importlib -from concurrent.futures import ThreadPoolExecutor from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock @@ -13,98 +12,22 @@ import sqlalchemy as sa from alembic.migration import MigrationContext from alembic.operations import Operations from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker import app.chain.download.submission as download_submission -from app.application.download.admission import ( - DownloadReconciliationRequired, - SubscriptionDownloadGovernance, - SubscriptionDownloadRequest, -) +from app.application.download.admission import SubscriptionDownloadGovernance from app.application.subscription.contract import SubscriptionSnapshot +from app.application.subscription.execution import ( + SubscriptionExecutionAdmission, + SubscriptionExecutionContext, +) from app.chain.download import DownloadChain from app.chain.subscribe import policy as subscribe_policy from app.chain.subscribe.facade import SubscribeChain -from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository -from app.db.base import Base -from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission from app.domain.context import Context, MediaInfo, TorrentInfo from app.domain.metainfo import MetaInfo from app.schemas.types import MediaSource, MediaType -def _request(key: str = "key-1", task_id: str | None = "task-1") -> SubscriptionDownloadRequest: - """构造固定身份的提交认领请求。""" - return SubscriptionDownloadRequest( - idempotency_key=key, - legacy_idempotency_key=None, - subscription_id=7, - task_id=task_id, - logical_identity='{"subscription_id":7}', - resource_key="example.com:id=42", - coverage="episodes:E01-E03", - mode="normal", - delivery_scope='{"download_uri":"local:/downloads","downloader":"auto"}', - ) - - -def _repository(tmp_path) -> tuple[TransactionalSubscriptionDownloadRepository, object]: - """创建独立 SQLite 幂等账本仓储与 Session 工厂。""" - engine = create_engine( - f"sqlite:///{tmp_path / 'subscription-download.db'}", - connect_args={"check_same_thread": False}, - ) - Base.metadata.create_all(engine) - factory = sessionmaker(bind=engine) - return TransactionalSubscriptionDownloadRepository(factory), factory - - -def test_repository_claims_same_submission_once_across_workers(tmp_path) -> None: - """并发入口对同一幂等键只能有一个取得下载器提交权。""" - repository, _factory = _repository(tmp_path) - - with ThreadPoolExecutor(max_workers=2) as executor: - claims = list(executor.map(lambda _index: repository.claim(_request()), range(2))) - - assert sum(claim.acquired for claim in claims) == 1 - assert {claim.snapshot.state for claim in claims} == {"submitting"} - assert {claim.snapshot.attempt_count for claim in claims} == {1} - - -def test_repository_fences_retry_and_preserves_uncertain_terminal(tmp_path) -> None: - """明确拒绝可延迟重试，过期令牌和待对账状态均不得重新提交。""" - repository, _factory = _repository(tmp_path) - first = repository.claim(_request()) - token = first.snapshot.attempt_token - assert token - assert repository.mark_retryable( - idempotency_key="key-1", - attempt_token=token, - available_at="1970-01-01T00:00:00+00:00", - error="downloader rejected", - ) - - second = repository.claim(_request(task_id="task-2")) - second_token = second.snapshot.attempt_token - assert second.acquired - assert second_token and second_token != token - assert not repository.mark_succeeded( - idempotency_key="key-1", - attempt_token=token, - ) - assert repository.mark_reconcile_required( - idempotency_key="key-1", - attempt_token=second_token, - error="transport timeout", - ) - - blocked = repository.claim(_request(task_id="task-3")) - assert not blocked.acquired - assert blocked.snapshot.state == "reconcile_required" - assert blocked.snapshot.attempt_count == 2 - assert repository.has_started_for_task("task-2") - - class _FakeTorrentHelper: """返回固定种子目录和文件清单，隔离真实 bencode 解析。""" @@ -114,10 +37,9 @@ class _FakeTorrentHelper: return "Demo.Show.S01", ["Demo.Show.S01E01.mkv"] -def _download_chain(repository) -> DownloadChain: +def _download_chain() -> DownloadChain: """构造只执行下载提交边界的 Chain 测试实例。""" chain = DownloadChain.__new__(DownloadChain) - chain.subscription_download_repository = repository chain.download_history_repository = MagicMock() chain.download_history_repository.get_by_media_identity.return_value = [] chain.download_failure_repository = MagicMock() @@ -174,12 +96,11 @@ def _submission_dependencies(monkeypatch): ) -def test_download_chain_reuses_success_without_second_downloader_call(tmp_path) -> None: - """重叠入口在首个提交成功后复用 hash，不再次调用下载器。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) +def test_download_chain_uses_normal_submission_for_each_execution() -> None: + """下载边界不保留本地提交账本，由下载器处理相同 torrent 的复用。""" + chain = _download_chain() chain.download = MagicMock(return_value=("qb", "hash-1", "Original", "accepted")) - governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal") + governance = SubscriptionDownloadGovernance() first = chain.download_single( context=_context(), @@ -197,306 +118,91 @@ def test_download_chain_reuses_success_without_second_downloader_call(tmp_path) ) assert first == second == "hash-1" - chain.download.assert_called_once() - chain._settle_download_success.assert_called_once() + assert chain.download.call_count == 2 + assert chain._settle_download_success.call_count == 2 -def test_download_chain_deduplicates_same_media_across_subscription_rows(tmp_path) -> None: - """同媒体同覆盖同交付目标的重复订阅只允许一个下载器提交。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock(return_value=("qb", "hash-cross-row", "Original", "accepted")) - - first = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"), - ) - second = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"), - ) - - assert first == second == "hash-cross-row" - chain.download.assert_called_once() - chain._settle_download_success.assert_called_once() - - -def test_download_chain_keeps_distinct_delivery_targets_separate(tmp_path) -> None: - """不同保存目标属于独立产品意图，不得跨记录误去重。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - chain._resolve_media_download_dir.side_effect = ( - lambda *, save_path, **_kwargs: ("local", Path(save_path), None) - ) +def test_download_chain_retries_after_downloader_exception() -> None: + """下载器异常不冻结后续执行，下一轮仍按普通下载合同重新提交。""" + chain = _download_chain() chain.download = MagicMock(side_effect=[ - ("qb", "hash-a", "Original", "accepted"), - ("qb", "hash-b", "Original", "accepted"), + TimeoutError("response timeout"), + ("qb", "hash-after-timeout", "Original", "accepted"), ]) - first = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads/a", - governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"), - ) - second = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads/b", - governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"), - ) - - assert (first, second) == ("hash-a", "hash-b") - assert chain.download.call_count == 2 - - -def test_download_chain_keeps_distinct_downloaders_separate(tmp_path) -> None: - """不同下载器属于独立交付策略，不得跨记录误去重。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock(side_effect=[ - ("qb-a", "hash-a", "Original", "accepted"), - ("qb-b", "hash-b", "Original", "accepted"), - ]) - - first = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - downloader="qb-a", - governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"), - ) - second = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - downloader="qb-b", - governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"), - ) - - assert (first, second) == ("hash-a", "hash-b") - assert chain.download.call_count == 2 - - -def test_download_chain_keeps_distinct_episode_coverage_separate(tmp_path) -> None: - """跨记录仅复用精确覆盖，新增目标集不得被已有子集吞掉。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock(side_effect=[ - ("qb", "hash-e1", "Original", "accepted"), - ("qb", "hash-e12", "Original", "accepted"), - ]) - - first = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"), - ) - second = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1, 2}, - save_path="/downloads", - governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"), - ) - - assert (first, second) == ("hash-e1", "hash-e12") - assert chain.download.call_count == 2 - - -def test_download_chain_reuses_pre_004_ledger_key(tmp_path) -> None: - """键升级后仍应读取 003A 同记录成功账本，避免版本升级造成重复下载。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal") - request = chain._build_subscription_download_request( - context=_context(), - episodes={1}, - governance=governance, - delivery_scope=chain._subscription_delivery_scope( - downloader=None, - download_uri="local:/downloads", - ), - ) - legacy = repository.claim( - SubscriptionDownloadRequest( - idempotency_key=request.legacy_idempotency_key or "", - legacy_idempotency_key=None, - subscription_id=7, - task_id=None, - logical_identity='{"subscription_id":7}', - resource_key=request.resource_key, - coverage=request.coverage, - mode=request.mode, - delivery_scope="legacy", + with pytest.raises(TimeoutError, match="response timeout"): + chain.download_single( + context=_context(), + torrent_content=b"torrent", + episodes={1}, + save_path="/downloads", + governance=SubscriptionDownloadGovernance(), ) - ) - token = legacy.snapshot.attempt_token or "" - assert repository.mark_accepted( - idempotency_key=legacy.snapshot.idempotency_key, - attempt_token=token, - downloader="qb", - download_hash="legacy-ledger-hash", - ) - assert repository.mark_succeeded( - idempotency_key=legacy.snapshot.idempotency_key, - attempt_token=token, - ) - chain.download = MagicMock() result = chain.download_single( context=_context(), torrent_content=b"torrent", episodes={1}, save_path="/downloads", - governance=governance, + governance=SubscriptionDownloadGovernance(), ) - assert result == "legacy-ledger-hash" - chain.download.assert_not_called() + assert result == "hash-after-timeout" + assert chain.download.call_count == 2 + chain._settle_download_success.assert_called_once() -def test_download_chain_freezes_when_local_settlement_fails(tmp_path) -> None: - """下载器接受而历史结算失败时进入待对账，后续入口不得盲重试。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) +def test_download_chain_retries_after_local_settlement_failure() -> None: + """本地结算异常直接失败，下一轮不依赖补偿状态即可重新执行。""" + chain = _download_chain() chain.download = MagicMock(return_value=("qb", "hash-2", "Original", "accepted")) - chain._settle_download_success.side_effect = RuntimeError("history unavailable") - governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal") + chain._settle_download_success.side_effect = [RuntimeError("history unavailable"), None] - with pytest.raises(DownloadReconciliationRequired): + with pytest.raises(RuntimeError, match="history unavailable"): chain.download_single( context=_context(), torrent_content=b"torrent", episodes={1}, save_path="/downloads", - governance=governance, + governance=SubscriptionDownloadGovernance(), ) - with pytest.raises(DownloadReconciliationRequired): - chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=governance, - ) - - chain.download.assert_called_once() - - -def test_download_chain_freezes_when_downloader_result_is_uncertain(tmp_path) -> None: - """下载器调用抛错可能已产生副作用，重启后的新实例仍不得自动重试。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock(side_effect=TimeoutError("response timeout")) - governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal") - - with pytest.raises(DownloadReconciliationRequired): - chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=governance, - ) - - restarted = _download_chain(repository) - restarted.download = MagicMock() - with pytest.raises(DownloadReconciliationRequired): - restarted.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=governance, - ) - restarted.download.assert_not_called() - - -def test_download_chain_delays_retry_after_explicit_rejection(tmp_path) -> None: - """下载器明确拒绝且无 hash 时进入冷却，不把不确定和已拒绝混为一谈。""" - repository, factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected")) - governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal") - - first = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=governance, - ) - second = chain.download_single( - context=_context(), - torrent_content=b"torrent", - episodes={1}, - save_path="/downloads", - governance=governance, - ) - - assert first is None and second is None - chain.download.assert_called_once() - with factory() as session: - record = session.query(SubscriptionDownloadSubmission).one() - assert record.state == "retryable" - assert record.available_at > record.updated_at - - -def test_download_chain_reads_legacy_history_before_new_ledger(tmp_path) -> None: - """迁移前同订阅同 torrent 同覆盖的成功历史仍能阻止重复提交。""" - repository, factory = _repository(tmp_path) - chain = _download_chain(repository) - chain.download = MagicMock() - chain.download_history_repository.get_by_media_identity.return_value = [ - SimpleNamespace( - download_hash="legacy-hash", - torrent_name="Demo Show S01E01 1080p", - torrent_site="TestSite", - episodes="E01", - seasons="S01", - episode_group=None, - note={"source": 'Subscribe|{"id": 7}'}, - ) - ] result = chain.download_single( context=_context(), torrent_content=b"torrent", episodes={1}, save_path="/downloads", - governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"), + governance=SubscriptionDownloadGovernance(), ) - assert result == "legacy-hash" - chain.download.assert_not_called() - with factory() as session: - assert session.query(SubscriptionDownloadSubmission).count() == 0 + assert result == "hash-2" + assert chain.download.call_count == 2 + assert chain._settle_download_success.call_count == 2 -def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None: +def test_download_chain_records_explicit_rejection_as_normal_failure() -> None: + """下载器明确拒绝仍写入既有资源失败冷却，不建立提交恢复状态。""" + chain = _download_chain() + chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected")) + + result = chain.download_single( + context=_context(), + torrent_content=b"torrent", + episodes={1}, + save_path="/downloads", + governance=SubscriptionDownloadGovernance(), + ) + + assert result is None + chain.download_failure_repository.record_failure.assert_called_once() + chain.post_message.assert_called_once() + + +def test_download_chain_cancels_before_external_side_effect() -> None: """取消在下载器边界前生效时不得创建下载任务或成功事实。""" - repository, _factory = _repository(tmp_path) - chain = _download_chain(repository) + chain = _download_chain() chain.download = MagicMock() - governance = SubscriptionDownloadGovernance( - subscription_id=7, - mode="normal", - task_id="cancel-task", - cancelled=lambda: True, - ) + governance = SubscriptionDownloadGovernance(cancelled=lambda: True) result = chain.download_single( context=_context(), @@ -508,7 +214,34 @@ def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None: assert result is None chain.download.assert_not_called() - assert not repository.has_started_for_task("cancel-task") + chain._settle_download_success.assert_not_called() + + +def test_download_chain_marks_side_effect_boundary_before_downloader_call() -> None: + """下载器调用前必须标记副作用起点，使晚到取消按真实执行结果收口。""" + chain = _download_chain() + order: list[str] = [] + + def download(**_kwargs): + """记录下载器调用顺序并返回成功结果。""" + order.append("download") + return "qb", "hash-3", "Original", "accepted" + + chain.download = download + governance = SubscriptionDownloadGovernance( + mark_started=lambda: order.append("started"), + ) + + result = chain.download_single( + context=_context(), + torrent_content=b"torrent", + episodes={1}, + save_path="/downloads", + governance=governance, + ) + + assert result == "hash-3" + assert order == ["started", "download"] def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -> None: @@ -554,6 +287,18 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) - lambda _subscribe, contexts: contexts, ) monkeypatch.setattr(subscribe_policy, "DownloadChain", _FakeDownloadChain) + admission = SubscriptionExecutionAdmission() + lease = admission.try_acquire( + subscription_id=current.id, + operation="search", + ttl_seconds=60, + ) + assert lease is not None + execution_context = SubscriptionExecutionContext( + lease=lease, + admission=admission, + task_id="search-task-7", + ) _downloads, lefts = chain._SubscribeChain__download_best_version_with_full_pack_first( contexts=[context], @@ -562,6 +307,7 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) - mediakey="themoviedb:77", save_path="/old", downloader="old", + execution_context=execution_context, ) assert lefts is fresh_missing @@ -574,8 +320,10 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) - assert captured["no_exists"] is fresh_missing assert captured["save_path"] == "/current" assert captured["downloader"] == "current" - assert captured["governance"].subscription_id == 7 - assert captured["governance"].mode == "normal" + assert captured["governance"].cancelled() is False + captured["governance"].mark_started() + assert execution_context.download_started is True + assert admission.release(lease) is True def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch) -> None: @@ -610,58 +358,50 @@ def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch batch_download.assert_not_called() -def test_subscription_download_migration_is_idempotent_and_reversible(tmp_path, monkeypatch) -> None: - """3.0.21 迁移可重复升级，并只移除自身新增的兼容表。""" - engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}") - migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21") - with engine.begin() as connection: - operations = Operations(MigrationContext.configure(connection)) - monkeypatch.setattr(migration, "op", operations) - migration.upgrade() - migration.upgrade() - inspector = sa.inspect(connection) - assert "subscriptiondownloadsubmission" in inspector.get_table_names() - indexes = {item["name"] for item in inspector.get_indexes("subscriptiondownloadsubmission")} - assert "ix_subscriptiondownloadsubmission_task_state" in indexes - migration.downgrade() - assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names() - - -def test_subscription_delivery_scope_migration_is_idempotent_and_reversible( +def test_subscription_download_ledger_removal_migration_is_reversible( tmp_path, monkeypatch, ) -> None: - """3.0.23 为存量提交填充兼容范围，并支持重复升级和完整回滚。""" - engine = create_engine(f"sqlite:///{tmp_path / 'delivery-migration.db'}") - base_migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21") - migration = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23") + """3.0.25 删除旧账本；降级只恢复结构，不伪造已删除的运行状态。""" + engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}") + create_ledger = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21") + add_delivery_scope = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23") + remove_ledger = importlib.import_module("database.versions.c8f2e6a1d4b9_3_0_25") + with engine.begin() as connection: operations = Operations(MigrationContext.configure(connection)) - monkeypatch.setattr(base_migration, "op", operations) - monkeypatch.setattr(migration, "op", operations) - base_migration.upgrade() + monkeypatch.setattr(create_ledger, "op", operations) + monkeypatch.setattr(add_delivery_scope, "op", operations) + monkeypatch.setattr(remove_ledger, "op", operations) + create_ledger.upgrade() + add_delivery_scope.upgrade() connection.execute(sa.text( "INSERT INTO subscriptiondownloadsubmission " "(idempotency_key, subscription_id, logical_identity, resource_key, coverage, mode, " - "state, attempt_count, created_at, updated_at) VALUES " - "('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'succeeded', 1, 'now', 'now')" + "delivery_scope, state, attempt_count, created_at, updated_at) VALUES " + "('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'legacy', " + "'reconcile_required', 1, 'now', 'now')" )) - migration.upgrade() - migration.upgrade() + remove_ledger.upgrade() + remove_ledger.upgrade() + assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names() - assert connection.execute(sa.text( - "SELECT delivery_scope FROM subscriptiondownloadsubmission WHERE idempotency_key='legacy-key'" - )).scalar_one() == "legacy" - migration.downgrade() - columns = { - column["name"] - for column in sa.inspect(connection).get_columns("subscriptiondownloadsubmission") + remove_ledger.downgrade() + remove_ledger.downgrade() + inspector = sa.inspect(connection) + assert "subscriptiondownloadsubmission" in inspector.get_table_names() + assert {column["name"] for column in inspector.get_columns( + "subscriptiondownloadsubmission" + )} >= {"idempotency_key", "delivery_scope", "state", "download_hash"} + indexes = { + item["name"] + for item in inspector.get_indexes("subscriptiondownloadsubmission") } - assert "delivery_scope" not in columns - - -def test_model_metadata_registers_submission_table() -> None: - """显式模型注册必须让 fresh create_all 包含订阅提交账本。""" - assert SubscriptionDownloadSubmission.__tablename__ == "subscriptiondownloadsubmission" - assert "subscriptiondownloadsubmission" in Base.metadata.tables + assert indexes == { + "ix_subscriptiondownloadsubmission_subscription_state", + "ix_subscriptiondownloadsubmission_task_state", + } + assert connection.execute(sa.text( + "SELECT COUNT(*) FROM subscriptiondownloadsubmission" + )).scalar_one() == 0 diff --git a/tests/test_subscription_execution_admission.py b/tests/test_subscription_execution_admission.py new file mode 100644 index 000000000..3f2d2f8a9 --- /dev/null +++ b/tests/test_subscription_execution_admission.py @@ -0,0 +1,138 @@ +"""订阅级进程内准入与显式执行上下文测试。""" + +from concurrent.futures import ThreadPoolExecutor + +from app.application.subscription.execution import ( + SubscriptionExecutionAdmission, + SubscriptionExecutionContext, +) + + +def test_same_subscription_is_mutually_exclusive_across_channels() -> None: + """同一订阅只能由 Search 或 Match 中的一条路径持有。""" + admission = SubscriptionExecutionAdmission() + + search = admission.try_acquire( + subscription_id=7, + operation="search", + ttl_seconds=60, + ) + match = admission.try_acquire( + subscription_id=7, + operation="match", + ttl_seconds=60, + ) + + assert search is not None + assert match is None + assert admission.release(search) is True + assert admission.try_acquire( + subscription_id=7, + operation="match", + ttl_seconds=60, + ) is not None + + +def test_concurrent_channels_admit_only_one_owner_for_same_subscription() -> None: + """Search 与 Match 同时申请同一订阅时只能有一个取得 owner。""" + admission = SubscriptionExecutionAdmission() + + def acquire(operation: str): + """并发申请固定订阅的通道所有权。""" + return admission.try_acquire( + subscription_id=11, + operation=operation, + ttl_seconds=60, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + leases = list(executor.map(acquire, ("search", "match"))) + + assert sum(lease is not None for lease in leases) == 1 + + +def test_different_subscriptions_can_run_across_channels() -> None: + """不同订阅可以分别占用 Search 与 Match 通道。""" + admission = SubscriptionExecutionAdmission() + + search = admission.try_acquire( + subscription_id=7, + operation="search", + ttl_seconds=60, + ) + match = admission.try_acquire( + subscription_id=8, + operation="match", + ttl_seconds=60, + ) + + assert search is not None + assert match is not None + assert search.owner_token != match.owner_token + + +def test_expired_owner_keeps_ownership_until_finally_release() -> None: + """TTL 只请求 owner 协作退出，不允许后到任务强制接管活跃执行。""" + now = [100.0] + admission = SubscriptionExecutionAdmission(clock=lambda: now[0]) + lease = admission.try_acquire( + subscription_id=9, + operation="search", + ttl_seconds=5, + ) + assert lease is not None + + now[0] = 106.0 + + assert admission.is_expired(lease) is True + assert admission.try_acquire( + subscription_id=9, + operation="match", + ttl_seconds=5, + ) is None + assert admission.release(lease) is True + + replacement = admission.try_acquire( + subscription_id=9, + operation="match", + ttl_seconds=5, + ) + assert replacement is not None + assert admission.release(lease) is False + + +def test_execution_context_combines_cancel_ttl_and_download_boundary() -> None: + """执行上下文独立承载取消、TTL、阶段与下载副作用状态。""" + now = [100.0] + cancelled = [False] + phases: list[tuple[str, int | None]] = [] + admission = SubscriptionExecutionAdmission(clock=lambda: now[0]) + lease = admission.try_acquire( + subscription_id=10, + operation="search", + ttl_seconds=5, + ) + assert lease is not None + context = SubscriptionExecutionContext( + lease=lease, + admission=admission, + task_id="task-10", + cancel_requested=lambda: cancelled[0], + phase_changed=lambda phase, site_id: phases.append((phase, site_id)), + ) + + assert context.should_stop() is False + context.report_phase("searching", 3) + context.mark_download_started() + + assert context.download_started is True + assert phases == [("searching", 3), ("submitting", None)] + + cancelled[0] = True + assert context.is_cancel_requested() is True + assert context.should_stop() is True + + cancelled[0] = False + now[0] = 106.0 + assert context.is_expired() is True + assert context.should_stop() is True diff --git a/tests/test_subscription_execution_status.py b/tests/test_subscription_execution_status.py index 6e340b551..66ad5ef1c 100644 --- a/tests/test_subscription_execution_status.py +++ b/tests/test_subscription_execution_status.py @@ -1,8 +1,8 @@ """订阅执行状态合并、批次权限和操作能力测试。""" import asyncio +from dataclasses import replace -from app.application.download.admission import SubscriptionDownloadSnapshot from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot from app.application.subscription.status import SubscriptionExecutionStatusService @@ -35,31 +35,12 @@ def _task( ) -def _download(subscription_id: int, state: str) -> SubscriptionDownloadSnapshot: - """构造一个比搜索任务更早的下载提交快照。""" - return SubscriptionDownloadSnapshot( - idempotency_key=f"key-{subscription_id}", - subscription_id=subscription_id, - task_id=f"task-{subscription_id}", - state=state, - attempt_count=1, - attempt_token="attempt", - downloader=None, - download_hash=None, - available_at=None, - last_error="downloader response uncertain" if state == "reconcile_required" else None, - created_at="2026-09-01T00:30:00+00:00", - updated_at="2026-09-01T00:59:00+00:00", - ) - - class _Repository: """保存测试快照的异步状态仓储。""" def __init__(self) -> None: """初始化可由测试覆盖的快照集合。""" self.tasks: dict[int, SearchTaskSnapshot] = {} - self.downloads: dict[int, SubscriptionDownloadSnapshot] = {} self.batch = SearchBatchSnapshot( batch_id="batch-1", source="manual", @@ -78,10 +59,6 @@ class _Repository: """返回请求范围内搜索任务。""" return {key: value for key, value in self.tasks.items() if key in subscription_ids} - async def latest_download_submissions(self, subscription_ids): - """返回请求范围内下载提交。""" - return {key: value for key, value in self.downloads.items() if key in subscription_ids} - async def list_batches(self, *, limit): """返回一个测试批次。""" return [self.batch][:limit] @@ -107,29 +84,14 @@ def test_execution_status_exposes_site_wait_and_cancel_capability(): assert statuses[1].can_cancel is True -def test_reconciliation_state_overrides_newer_search_terminal(): - """不确定下载副作用不得被稍晚写入的搜索失败掩盖。""" - repository = _Repository() - repository.tasks[2] = _task(2, state="failed", phase="failed") - repository.downloads[2] = _download(2, "reconcile_required") - - statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,))) - - assert statuses[2].state == "reconcile_required" - assert statuses[2].requires_reconciliation is True - assert statuses[2].can_retry is False - assert statuses[2].error == "downloader response uncertain" - - -def test_failed_search_exposes_safe_retryable_error(): - """搜索失败文本必须压平且仅声明安全重试能力。""" +def test_failed_search_exposes_safe_error(): + """搜索失败文本必须压平且不暴露内部错误细节。""" repository = _Repository() repository.tasks[3] = _task(3, state="failed", phase="failed") statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((3,))) assert statuses[3].state == "failed" - assert statuses[3].can_retry is True assert statuses[3].error == "provider timeout" @@ -149,6 +111,35 @@ def test_batch_requires_complete_subscription_access(): assert visible.can_cancel is True +def test_batch_projection_exposes_skipped_count_as_processed_without_success(): + """批次跳过应计入处理总数，同时保留独立的完成计数。""" + repository = _Repository() + repository.batch = replace( + repository.batch, + state="skipped", + total_count=2, + finished_count=1, + skipped_count=1, + ) + repository.tasks = { + 1: _task(1, state="completed", phase="completed"), + 2: _task(2, state="skipped", phase="skipped"), + } + + visible = asyncio.run( + SubscriptionExecutionStatusService(repository).get_batch( + "batch-1", + accessible_subscription_ids={1, 2}, + ) + ) + + assert visible is not None + assert visible.state == "skipped" + assert visible.finished_count == 1 + assert visible.skipped_count == 1 + assert visible.processed_count == 2 + + def test_request_cancel_uses_injected_execution_boundary(): """取消必须通过组合根注入的异步执行边界并返回真实结果。""" repository = _Repository() diff --git a/tests/test_subscription_governance_joint.py b/tests/test_subscription_governance_joint.py index e83463296..4fafc9efb 100644 --- a/tests/test_subscription_governance_joint.py +++ b/tests/test_subscription_governance_joint.py @@ -60,8 +60,6 @@ def test_cooled_site_does_not_block_independent_site_or_hide_batch_failure(): owner="fallback-task", cancelled=lambda: False, stop_state=ProcessStopState(), - max_wait_seconds=0, - random_uniform=lambda _low, _high: 60.0, ) chain = object.__new__(SearchChain) chain._runtime_config = SimpleNamespace(search_threadpool_size=2) diff --git a/tests/test_subscription_governance_replay.py b/tests/test_subscription_governance_replay.py index e2f8410fd..d01b6feab 100644 --- a/tests/test_subscription_governance_replay.py +++ b/tests/test_subscription_governance_replay.py @@ -8,7 +8,7 @@ from types import SimpleNamespace import pytest from app.application.subscription.contract import SubscriptionSnapshot -from app.chain.subscribe import SubscribeChain +from app.chain.subscribe.facade import SubscribeChain from app.domain.context import Context, MediaInfo, TorrentInfo from app.domain.metainfo import MetaInfo from app.schemas.mediaserver import NotExistMediaInfo @@ -42,11 +42,16 @@ class _ReplaySubscriptionListRepository: def __init__(self, subscribes: list[SubscriptionSnapshot]) -> None: self.subscribes = subscribes + self._by_id = {subscribe.id: subscribe for subscribe in subscribes} def list(self, _state: str = None) -> list[SubscriptionSnapshot]: """返回当前批次的全部订阅快照。""" return self.subscribes + def get(self, subscribe_id: int) -> SubscriptionSnapshot | None: + """返回取得订阅准入后的最新快照。""" + return self._by_id.get(subscribe_id) + class _ReplayTorrentHelper: """让无关候选稳定停在身份冲突边界。""" diff --git a/tests/test_subscription_governance_scale.py b/tests/test_subscription_governance_scale.py index c3b5af4dc..dd8c5699e 100644 --- a/tests/test_subscription_governance_scale.py +++ b/tests/test_subscription_governance_scale.py @@ -1,14 +1,298 @@ """订阅执行治理最终受控规模门禁。""" -from scripts.validation.subscription_governance_scale import run_acceptance +import threading + +import pytest + +from app.application.site.observation import capture_site_search_observation +from app.application.subscription.candidates import CandidateIndex +from app.application.subscription.sitebudget import SiteBudgetClaim +from app.chain.search.facade import SearchChain +from app.chain.subscribe.facade import SubscribeChain +from scripts.validation import subscription_governance_scale as scale +from scripts.validation.subscription_governance_scale import ( + ScaleCase, + _run_durable_governance, + _run_match_execution_case, + run_acceptance, +) def test_subscription_governance_controlled_scale_matrix() -> None: - """两档最终矩阵必须同时满足正确性、压力、恢复和幂等门禁。""" + """两档最终矩阵必须同时满足正确性、压力、恢复和订阅准入门禁。""" result = run_acceptance() + assert result["schema_version"] == 4 assert result["passed"] is True assert all(result["gates"].values()) + assert result["method"]["match_entrypoint"] == "SubscribeChain.match" + assert result["method"]["download_selection"] == "DownloadChain.batch_download" + assert result["method"]["fixed_external_boundaries"] == [ + "TMDB", + "media_server", + "downloader", + ] + assert result["method"]["fixed_persistence_boundaries"] == [ + "subscription_repository", + "download_facts", + "subscription_progress", + "completion_side_effects", + ] + assert result["method"]["fixed_policy_inputs"] == [ + "site_mapping", + "system_configuration", + "subscription_filters", + "torrent_attribute_filter", + "download_preparation", + ] + assert result["method"]["production_network_slo"] is False + assert result["gates"]["subscription_admission_serializes"] is True + assert result["gates"]["match_execution_candidate_sets_equal"] is True + assert result["gates"]["match_execution_download_sets_equal"] is True + assert result["gates"]["match_execution_missing_sets_equal"] is True + assert result["gates"]["match_execution_completion_sets_equal"] is True assert [case["subscription_count"] for case in result["match_cases"]] == [100, 200] assert [case["site_count"] for case in result["match_cases"]] == [10, 20] assert min(case["candidate_count"] for case in result["match_cases"]) >= 1000 + assert [ + case["candidate_check_reduction_percent"] + for case in result["match_cases"] + ] == [99.0, 99.0] + assert [ + case["matched_candidates"]["actual_count"] + for case in result["match_execution_cases"] + ] == [1000, 2400] + assert [ + case["downloaded_candidates"]["actual_count"] + for case in result["match_execution_cases"] + ] == [100, 200] + assert all( + case["downloaded_candidates"]["duplicate_count"] == 0 + and case["completed_subscriptions"]["duplicate_count"] == 0 + for case in result["match_execution_cases"] + ) + assert [ + case["site_peak_inflight_per_site"] + for case in result["durable_cases"] + ] == [1, 1] + assert all( + case["site_request_boundary_active"] == 0 + and case["site_request_boundary_peak"] == 1 + and case["site_pressure_owner_count"] == 2 + and case["site_pressure_concurrency_verified"] + and case["site_pressure_success_release_reused"] + and case["site_pressure_error_observation_cooled"] + and case["site_pressure_valid"] + for case in result["durable_cases"] + ) + assert all( + observation["request_active"] == 0 + and observation["request_peak"] == 1 + and observation["owners_finished"] is True + and observation["request_active_at_rejection"] == 1 + and observation["budget_rejections"] == 1 + for case in result["durable_cases"] + for observation in case["site_observations"] + ) + + +@pytest.mark.parametrize("mutation", ["bypass", "early_release", "allow_concurrent"]) +def test_scale_validator_rejects_fake_site_pressure(monkeypatch, tmp_path, mutation): + """绕过 wrapper、提前释放或并发放行时，请求边界峰值必须暴露门禁失效。""" + if mutation == "bypass": + def bypass_wrapper(self, *, site, keyword, mtype, page): + """模拟绕过预算 wrapper 的请求路径。""" + return self.search_site_torrents( + site=site, + keyword=keyword, + mtype=mtype, + page=page, + ) + + monkeypatch.setattr( + SearchChain, + "_search_site_torrents_with_budget", + bypass_wrapper, + ) + elif mutation == "early_release": + release_guard = threading.Lock() + + def early_release_wrapper(self, *, site, keyword, mtype, page): + """模拟请求边界前错误释放站点租约。""" + budget = self._subscription_site_budget + # 将错误释放本身串行化，避免第二个 owner 在首个释放动作完成前 + # 被正常拒绝，从而掩盖“请求期间已失去租约”的变异。 + with release_guard: + claim = budget.acquire(site["id"]) + with capture_site_search_observation() as observation: + budget.finish(claim, observation) + return self.search_site_torrents( + site=site, + keyword=keyword, + mtype=mtype, + page=page, + ) + + monkeypatch.setattr( + SearchChain, + "_search_site_torrents_with_budget", + early_release_wrapper, + ) + else: + original_claim = scale.TransactionalSubscriptionSearchRepository.claim_site + + def allow_concurrent_claim(self, *, site_id, owner, lease_seconds): + """模拟仓储错误地向第二 owner 发放同站点租约。""" + claim = original_claim( + self, + site_id=site_id, + owner=owner, + lease_seconds=lease_seconds, + ) + if claim.acquired: + return claim + return SiteBudgetClaim( + site_id=site_id, + acquired=True, + retry_at=claim.retry_at, + consecutive_failures=claim.consecutive_failures, + lease_token=f"mutant-{owner}", + ) + + monkeypatch.setattr( + scale.TransactionalSubscriptionSearchRepository, + "claim_site", + allow_concurrent_claim, + ) + + result = _run_durable_governance( + ScaleCase(f"mutant-{mutation}", 2, 1, 2, 1), + tmp_path, + ) + + assert result["site_pressure_valid"] is False + assert result["site_peak_inflight_per_site"] >= 2 + assert result["site_request_boundary_peak"] >= 2 + + +def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path): + """失败已登记但 wrapper 尚未返回时，压力门禁必须拒绝并暴露未收口 owner。""" + original_wrapper = SearchChain._search_site_torrents_with_budget + release_stalled = threading.Event() + threads_before = set(threading.enumerate()) + + def stall_after_rejection(self, *, site, keyword, mtype, page): + """模拟预算拒绝已记录、调用方却未取得返回值的挂起路径。""" + original_record_failure = self.record_subscription_site_budget_failure + + def record_failure_and_stall(error: str) -> None: + """在拒绝已登记后阻塞原始 wrapper 的返回。""" + original_record_failure(error) + if "冷却或已有在途搜索" in error: + release_stalled.wait() + + self.record_subscription_site_budget_failure = record_failure_and_stall + try: + return original_wrapper( + self, + site=site, + keyword=keyword, + mtype=mtype, + page=page, + ) + finally: + self.record_subscription_site_budget_failure = original_record_failure + + monkeypatch.setattr( + SearchChain, + "_search_site_torrents_with_budget", + stall_after_rejection, + ) + monkeypatch.setattr(scale, "_SITE_PRESSURE_SYNC_TIMEOUT", 0.05) + + try: + result = _run_durable_governance( + ScaleCase("mutant-unfinished", 2, 1, 2, 1), + tmp_path, + ) + + assert result["site_pressure_valid"] is False + assert result["site_pressure_concurrency_verified"] is False + assert result["site_observations"][0]["owners_finished"] is False + finally: + release_stalled.set() + for thread in set(threading.enumerate()) - threads_before: + thread.join(timeout=1) + + +def test_scale_validator_rejects_candidate_loss(monkeypatch) -> None: + """候选路由少返回任意资源时，完整匹配集合门禁必须失败。""" + original_route = CandidateIndex.route_for_match + + def route_with_loss(self, *args, **kwargs): + routed = original_route(self, *args, **kwargs) + for domain, contexts in routed.items(): + if contexts: + return {**routed, domain: contexts[1:]} + return routed + + monkeypatch.setattr(CandidateIndex, "route_for_match", route_with_loss) + + result = _run_match_execution_case(ScaleCase("candidate-loss", 4, 2, 8, 4)) + + assert result["matched_candidates"]["equal"] is False + assert result["matched_candidates"]["missing_sample"] + + +def test_scale_validator_rejects_duplicate_download_and_completion(monkeypatch) -> None: + """同订阅重复下载或错误完成时，多重集合门禁必须失败。""" + from scripts.validation import subscription_governance_scale as scale + + original_download = scale._ScaleDownloadBoundary.download_single + + def duplicate_download(self, context, *, governance, **kwargs): + result = original_download( + self, + context, + governance=governance, + **kwargs, + ) + original_download( + self, + context, + governance=governance, + **kwargs, + ) + return result + + def finish_every_subscription( + self, + *, + subscribe, + meta, + mediainfo, + **_kwargs, + ): + self._SubscribeChain__finish_subscribe( + subscribe=subscribe, + meta=meta, + mediainfo=mediainfo, + ) + + monkeypatch.setattr( + scale._ScaleDownloadBoundary, + "download_single", + duplicate_download, + ) + monkeypatch.setattr( + SubscribeChain, + "finish_subscribe_or_not", + finish_every_subscription, + ) + + result = _run_match_execution_case(ScaleCase("duplicate-effects", 4, 2, 8, 4)) + + assert result["downloaded_candidates"]["equal"] is False + assert result["downloaded_candidates"]["duplicate_count"] == 4 + assert result["completed_subscriptions"]["equal"] is False + assert result["completed_subscriptions"]["unexpected_sample"] diff --git a/tests/test_subscription_match_admission.py b/tests/test_subscription_match_admission.py new file mode 100644 index 000000000..099898a79 --- /dev/null +++ b/tests/test_subscription_match_admission.py @@ -0,0 +1,231 @@ +"""订阅 Match 通道的逐订阅准入与异常隔离测试。""" + +import threading +from dataclasses import replace +from types import SimpleNamespace +from unittest.mock import Mock + +from app.application.subscription.contract import SubscriptionSnapshot +from app.application.subscription.execution import SubscriptionExecutionAdmission +from app.chain.subscribe import match as subscribe_match +from app.chain.subscribe.facade import SubscribeChain +from app.schemas.types import MediaType + + +def _subscribe(subscribe_id: int, *, name: str | None = None) -> SubscriptionSnapshot: + """构造 Match 编排所需的活动电影订阅快照。""" + return SubscriptionSnapshot( + id=subscribe_id, + name=name or f"匹配电影 {subscribe_id}", + year="2026", + type=MediaType.MOVIE.value, + media_source="themoviedb", + media_id=str(2000 + subscribe_id), + state="R", + ) + + +def _chain( + listed: list[SubscriptionSnapshot], + current: dict[int, SubscriptionSnapshot] | None = None, +) -> SubscribeChain: + """构造只执行 Match 编排层的订阅链。""" + snapshots = current or {subscribe.id: subscribe for subscribe in listed} + chain = object.__new__(SubscribeChain) + chain.subscription_repository = SimpleNamespace( + list=lambda _state: list(listed), + get=snapshots.get, + ) + chain.get_states_for_search = lambda state: state + chain._match_lock = threading.Lock() + chain._search_queue_lock = threading.Lock() + chain._subscription_execution_admission = SubscriptionExecutionAdmission() + chain._prepare_match_torrents = lambda torrents: torrents + return chain + + +def _assert_channel_and_subscription_released( + chain: SubscribeChain, + subscription_ids: tuple[int, ...], +) -> None: + """验证 Match 通道锁和所有订阅 owner 均已释放。""" + assert chain._match_lock.acquire(blocking=False) is True + chain._match_lock.release() + for subscription_id in subscription_ids: + lease = chain._subscription_execution_admission.try_acquire( + subscription_id=subscription_id, + operation="search", + ttl_seconds=60, + ) + assert lease is not None + assert chain._subscription_execution_admission.release(lease) is True + + +def test_match_skips_subscription_owned_by_search() -> None: + """Search 已持有同一订阅时 Match 本轮直接跳过。""" + subscribe = _subscribe(1) + chain = _chain([subscribe]) + process = Mock(side_effect=AssertionError("冲突订阅不应进入 Match")) + chain._match_subscription = process + search_lease = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="search", + ttl_seconds=60, + ) + assert search_lease is not None + + chain.match({"example.org": []}) + + process.assert_not_called() + assert chain._match_lock.acquire(blocking=False) is True + chain._match_lock.release() + assert chain._subscription_execution_admission.release(search_lease) is True + + +def test_match_reloads_subscription_after_admission() -> None: + """Match 取得 owner 后使用最新订阅快照并传入独立执行上下文。""" + listed = _subscribe(2, name="旧快照") + current = replace(listed, name="最新快照") + chain = _chain([listed], {listed.id: current}) + captured = [] + + def process(**kwargs): + """记录 Match 单订阅执行入口收到的快照与上下文。""" + captured.append(kwargs) + + chain._match_subscription = process + + chain.match({"example.org": []}) + + assert len(captured) == 1 + assert captured[0]["subscribe"] is current + context = captured[0]["execution_context"] + assert context.lease.subscription_id == listed.id + assert context.lease.operation == "match" + _assert_channel_and_subscription_released(chain, (listed.id,)) + + +def test_match_isolates_one_subscription_failure_and_releases_all_owners() -> None: + """单条 Match 异常不阻止后续订阅，且所有 owner 最终释放。""" + first = _subscribe(3) + second = _subscribe(4) + chain = _chain([first, second]) + processed = [] + + def process(**kwargs): + """让首条订阅失败并记录后续订阅仍被执行。""" + subscribe = kwargs["subscribe"] + processed.append(subscribe.id) + if subscribe.id == first.id: + raise RuntimeError("candidate failure") + + chain._match_subscription = process + + chain.match({"example.org": []}) + + assert processed == [first.id, second.id] + _assert_channel_and_subscription_released(chain, (first.id, second.id)) + + +def test_match_continues_when_latest_subscription_read_fails() -> None: + """单条最新快照读取失败不能泄漏 owner 或中止后续 Match。""" + first = _subscribe(5) + second = _subscribe(6) + chain = _chain([first, second]) + processed = [] + + def get(subscription_id: int): + """模拟首条读取异常并返回第二条当前快照。""" + if subscription_id == first.id: + raise RuntimeError("repository failure") + return second + + chain.subscription_repository.get = get + chain._match_subscription = lambda **kwargs: processed.append(kwargs["subscribe"].id) + + chain.match({"example.org": []}) + + assert processed == [second.id] + _assert_channel_and_subscription_released(chain, (first.id, second.id)) + + +def test_match_skips_subscription_paused_after_admission() -> None: + """取得准入后订阅变为暂停态时，不应继续访问候选或下载边界。""" + listed = _subscribe(7) + paused = replace(listed, state="S") + chain = _chain([listed], {listed.id: paused}) + process = Mock(side_effect=AssertionError("暂停订阅不应进入 Match")) + chain._match_subscription = process + progress = Mock() + + chain.match({"example.org": []}, progress_callback=progress) + + process.assert_not_called() + final_data = progress.call_args.kwargs["data"] + assert final_data == { + "total": 1, + "finished": 1, + "completed": 0, + "skipped": 1, + "failed": 0, + } + assert progress.call_args.kwargs["text"] == "订阅资源匹配完成，部分订阅跳过" + _assert_channel_and_subscription_released(chain, (listed.id,)) + + +def test_match_progress_reports_completed_skipped_and_failed_counts() -> None: + """Match 最终进度必须区分正常执行、准入跳过和单订阅失败。""" + completed = _subscribe(8) + skipped = _subscribe(9) + failed = _subscribe(10) + chain = _chain([completed, skipped, failed]) + chain._match_subscription = Mock( + side_effect=["completed", RuntimeError("candidate failure")] + ) + skipped_lease = chain._subscription_execution_admission.try_acquire( + subscription_id=skipped.id, + operation="search", + ttl_seconds=60, + ) + assert skipped_lease is not None + progress = Mock() + + chain.match({"example.org": []}, progress_callback=progress) + + final_data = progress.call_args.kwargs["data"] + assert final_data == { + "total": 3, + "finished": 3, + "completed": 1, + "skipped": 1, + "failed": 1, + } + assert progress.call_args.kwargs["text"] == "订阅资源匹配完成，部分订阅失败" + assert chain._subscription_execution_admission.release(skipped_lease) is True + _assert_channel_and_subscription_released(chain, (completed.id, failed.id)) + + +def test_match_stop_does_not_report_unvisited_subscriptions_as_completed(monkeypatch) -> None: + """系统停止后未访问的订阅不得计入完成或跳过。""" + subscribes = [_subscribe(11), _subscribe(12)] + chain = _chain(subscribes) + progress = Mock() + monkeypatch.setattr( + subscribe_match, + "runtime_stop_state", + SimpleNamespace(is_system_stopped=True), + ) + + chain.match({"example.org": []}, progress_callback=progress) + + assert progress.call_args.kwargs == { + "value": 100, + "text": "订阅资源匹配已停止，部分订阅未执行", + "data": { + "total": 2, + "finished": 0, + "completed": 0, + "skipped": 0, + "failed": 0, + }, + } diff --git a/tests/test_subscription_search_governance.py b/tests/test_subscription_search_governance.py index 9d27c8779..52490840b 100644 --- a/tests/test_subscription_search_governance.py +++ b/tests/test_subscription_search_governance.py @@ -1,17 +1,24 @@ """订阅搜索队列接入、锁隔离和批次失败治理测试。""" +import threading from dataclasses import replace -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import Mock, patch from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from app.application.site.observation import report_site_search_outcome from app.application.subscription.contract import SubscriptionSnapshot +from app.application.subscription.execution import SubscriptionExecutionAdmission +from app.application.subscription.sitebudget import SubscriptionSearchCancelled +from app.chain.search.facade import SearchChain from app.chain.subscribe.facade import SubscribeChain +from app.chain.subscribe.search import _search_task_available_at from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository from app.db.base import Base +from app.modules.indexer import IndexerModule from app.schemas.types import MediaType @@ -66,27 +73,133 @@ def _chain(tmp_path, subscribes: list[SubscriptionSnapshot]): sessionmaker(bind=engine) ) chain.get_states_for_search = lambda state: state - chain._rlock = _ForbiddenLock() + chain._match_lock = _ForbiddenLock() + chain._search_queue_lock = threading.Lock() + chain._subscription_execution_admission = SubscriptionExecutionAdmission() return chain +def _make_tasks_ready(monkeypatch) -> None: + """让治理测试中的持久任务立即到期，避免依赖真实随机时钟。""" + ready_at = "1970-01-01T00:00:00+00:00" + monkeypatch.setattr( + "app.chain.subscribe.search._search_task_available_at", + lambda _source, subscription_ids: { + subscription_id: ready_at for subscription_id in subscription_ids + }, + ) + + +def test_fallback_task_schedule_staggers_each_subscription(monkeypatch): + """兜底批次首条抖动后，每条后续订阅都按独立随机间隔到期。""" + now = datetime(2026, 9, 3, 1, 2, 3, tzinfo=timezone.utc) + delays = iter((12, 60, 300)) + monkeypatch.setattr( + "app.chain.subscribe.search.random.randint", + lambda _low, _high: next(delays), + ) + + schedule = _search_task_available_at( + "fallback", + (1, 2, 3), + now=now, + ) + + available = [datetime.fromisoformat(schedule[subscribe_id]) for subscribe_id in (1, 2, 3)] + assert (available[0] - now).total_seconds() == 12 + assert (available[1] - available[0]).total_seconds() == 60 + assert (available[2] - available[1]).total_seconds() == 300 + + +def test_inline_fallback_search_preserves_site_pressure_stagger(tmp_path, monkeypatch): + """无持久队列的兼容宿主仍须在每条自动兜底订阅前错峰。""" + subscribes = [_subscribe(20), _subscribe(21)] + chain = _chain(tmp_path, subscribes) + del chain.subscription_search_repository + waits = [] + delays = iter((60, 300)) + monkeypatch.setattr( + "app.chain.subscribe.search.random.randint", + lambda _low, _high: next(delays), + ) + monkeypatch.setattr("app.chain.subscribe.search.time.sleep", waits.append) + monkeypatch.setattr( + chain, + "_process_search_subscription", + lambda item, _searchchain, **_kwargs: item, + ) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + chain.search(state="R") + + assert waits == [60, 300] + + +def test_successful_sites_remain_available_to_next_due_subscription(tmp_path, monkeypatch): + """正常站点请求不能让同批下一条到期订阅漏掉目标站点。""" + subscribes = [_subscribe(40), _subscribe(41)] + chain = _chain(tmp_path, subscribes) + _make_tasks_ready(monkeypatch) + searchchain = object.__new__(SearchChain) + searchchain.configure_subscription_site_budget(None) + current_subscription_id = 0 + calls = [] + + def search_site_torrents(*, site, **_kwargs): + """记录真实发出的站点请求并发布正常完成观察结果。""" + calls.append((current_subscription_id, site["id"])) + report_site_search_outcome(attempted=True, outcome="success") + return [f"torrent-{current_subscription_id}-{site['id']}"] + + def process(subscribe, current_searchchain, **_kwargs): + """让每条到期订阅访问同一组完整目标站点。""" + nonlocal current_subscription_id + current_subscription_id = subscribe.id + for site_id in (11, 12): + current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": site_id, "name": f"Site {site_id}"}, + keyword=subscribe.name, + mtype=MediaType.MOVIE, + page=0, + ) + return subscribe + + searchchain.search_site_torrents = search_site_torrents + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain): + batch_id = chain.search(state="R") + + assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)] + assert chain.get_search_batch(batch_id).state == "completed" + + def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch): """R/P 兜底搜索在持久队列中执行，不受日常 Match 长锁阻塞。""" subscribes = [_subscribe(1), _subscribe(2)] chain = _chain(tmp_path, subscribes) - monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00") + _make_tasks_ready(monkeypatch) processed = [] + + def process(subscribe, _searchchain, *, execution_context): + """记录每条任务独立的订阅执行上下文。""" + processed.append((subscribe.id, execution_context)) + return subscribe + monkeypatch.setattr( chain, "_process_search_subscription", - lambda subscribe, _searchchain: processed.append(subscribe.id) or subscribe, + process, ) with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): batch_id = chain.search(state="R") batch = chain.get_search_batch(batch_id) - assert processed == [1, 2] + assert [subscribe_id for subscribe_id, _context in processed] == [1, 2] + assert [context.lease.subscription_id for _subscribe_id, context in processed] == [1, 2] + assert len({context.task_id for _subscribe_id, context in processed}) == 2 + assert len({id(context) for _subscribe_id, context in processed}) == 2 assert batch.state == "completed" assert batch.finished_count == 2 assert batch.failed_count == 0 @@ -96,10 +209,10 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke """单订阅异常不得中止批次后续任务，聚合终态必须暴露失败。""" subscribes = [_subscribe(3), _subscribe(4)] chain = _chain(tmp_path, subscribes) - monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00") + _make_tasks_ready(monkeypatch) processed = [] - def process(subscribe, _searchchain): + def process(subscribe, _searchchain, **_kwargs): """让首条失败并保持第二条正常完成。""" processed.append(subscribe.id) if subscribe.id == 3: @@ -118,21 +231,218 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke assert batch.failed_count == 1 assert batch.last_error == "provider timeout" + for subscribe_id in (3, 4): + lease = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe_id, + operation="match", + ttl_seconds=60, + ) + assert lease is not None + assert chain._subscription_execution_admission.release(lease) is True -def test_late_cancel_completes_when_download_submission_already_started(tmp_path, monkeypatch): + +def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch): + """索引器吞错后仍须失败收口当前任务，并继续处理后续订阅。""" + subscribes = [_subscribe(13), _subscribe(14)] + chain = _chain(tmp_path, subscribes) + _make_tasks_ready(monkeypatch) + searchchain = object.__new__(SearchChain) + attempts = 0 + processed = [] + + def execute_search(_site, _request): + """首条站点请求模拟被 IndexerModule 吞掉的 HTTP 429。""" + nonlocal attempts + attempts += 1 + if attempts == 1: + raise RuntimeError("HTTP 429") + return False, [] + + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__search_check", + staticmethod(lambda _site, _keyword=None: True), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__execute_search", + staticmethod(execute_search), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__indexer_statistic", + staticmethod(lambda **_kwargs: None), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__parse_result", + staticmethod(lambda **_kwargs: []), + ) + searchchain.search_site_torrents = object.__new__(IndexerModule).search_torrents + + def process(subscribe, current_searchchain, **_kwargs): + """让每条任务经过同一预算包装，并把观察到的站点失败抛给队列。""" + processed.append(subscribe.id) + current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access + site={ + "id": 31 if subscribe.id == 13 else 32, + "name": "Flaky" if subscribe.id == 13 else "Healthy", + }, + keyword=subscribe.name, + mtype=MediaType.MOVIE, + page=0, + ) + failures = current_searchchain.consume_subscription_site_budget_failures() + if failures: + raise RuntimeError("；".join(failures)) + return subscribe + + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert processed == [13, 14] + assert batch.state == "failed" + assert batch.finished_count == 1 + assert batch.failed_count == 1 + assert "Flaky" in batch.last_error + assert "HTTP 429" in batch.last_error + + +def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch): + """Match 已持有同一订阅时，Search 本轮完成为跳过且不进入业务处理。""" + subscribe = _subscribe(6) + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + process = Mock(side_effect=AssertionError("冲突订阅不应进入 Search")) + monkeypatch.setattr(chain, "_process_search_subscription", process) + match_lease = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="match", + ttl_seconds=60, + ) + assert match_lease is not None + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + process.assert_not_called() + assert batch.state == "skipped" + assert batch.finished_count == 0 + assert batch.skipped_count == 1 + assert batch.last_error == "同一订阅正在由其他通道处理，本轮搜索已跳过" + assert chain._subscription_execution_admission.release(match_lease) is True + + +def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch): + """准入后重新读取到暂停订阅时不得开始搜索或伪造完成。""" + subscribe = _subscribe(9) + paused = replace(subscribe, state="S") + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + chain.subscription_repository.get = Mock(side_effect=(subscribe, paused)) + process = Mock(side_effect=AssertionError("暂停订阅不应进入 Search")) + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + process.assert_not_called() + assert batch.state == "skipped" + assert batch.finished_count == 0 + assert batch.skipped_count == 1 + assert batch.last_error == "订阅已暂停，本轮搜索已跳过" + + +def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch): + """站点预算和状态清理都失败时仍必须释放当前订阅 owner。""" + subscribe = replace(_subscribe(7), state="N") + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + monkeypatch.setattr( + chain, + "_process_search_subscription", + lambda item, _searchchain, **_kwargs: item, + ) + monkeypatch.setattr( + chain, + "_SubscribeChain__apply_subscribe_update", + Mock(side_effect=RuntimeError("state cleanup failed")), + ) + searchchain = Mock() + + def configure_site_budget(budget): + """仅让释放站点预算的清理步骤失败。""" + if budget is None: + raise RuntimeError("budget cleanup failed") + + searchchain.configure_subscription_site_budget.side_effect = configure_site_budget + + with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain): + batch_id = chain.search(state="N") + + assert chain.get_search_batch(batch_id).state == "completed" + replacement = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="match", + ttl_seconds=60, + ) + assert replacement is not None + assert chain._subscription_execution_admission.release(replacement) is True + + +def test_search_state_reset_stays_inside_subscription_admission(tmp_path, monkeypatch): + """N 到 R 的本地状态写回完成前不能让 Match 接管同一订阅。""" + subscribe = replace(_subscribe(8), state="N") + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + monkeypatch.setattr( + chain, + "_process_search_subscription", + lambda item, _searchchain, **_kwargs: item, + ) + competing_leases = [] + + def apply_update(*_args, **_kwargs): + """在状态写回时探测同一订阅仍由 Search 持有。""" + competing_leases.append( + chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="match", + ttl_seconds=60, + ) + ) + + monkeypatch.setattr(chain, "_SubscribeChain__apply_subscribe_update", apply_update) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + chain.search(state="N") + + assert competing_leases == [None] + replacement = chain._subscription_execution_admission.try_acquire( + subscription_id=subscribe.id, + operation="match", + ttl_seconds=60, + ) + assert replacement is not None + assert chain._subscription_execution_admission.release(replacement) is True + + +def test_late_cancel_completes_when_download_side_effect_already_started(tmp_path, monkeypatch): """取消晚于下载器副作用边界时按真实结果完成，不能伪装成未执行取消。""" subscribe = _subscribe(5) chain = _chain(tmp_path, [subscribe]) - monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00") - chain.subscription_download_repository = SimpleNamespace( - has_started_for_task=lambda _task_id: False, - ) + _make_tasks_ready(monkeypatch) queue = chain.subscription_search_repository cancel_checks = iter((False, True)) monkeypatch.setattr(queue, "is_cancel_requested", lambda _task_id: next(cancel_checks)) - def process(item, _searchchain): + def process(item, _searchchain, *, execution_context): """模拟当前任务复用或完成下载后才收到取消。""" - chain._mark_subscription_download_started() + execution_context.mark_download_started() return item monkeypatch.setattr(chain, "_process_search_subscription", process) @@ -144,3 +454,148 @@ def test_late_cancel_completes_when_download_submission_already_started(tmp_path assert batch.state == "completed" assert batch.finished_count == 1 assert batch.cancelled_count == 0 + + +def test_system_stop_requeues_task_after_search_returns(tmp_path, monkeypatch): + """系统停机应阻止后续副作用，并把未完成任务退回可恢复队列。""" + subscribe = _subscribe(10) + chain = _chain(tmp_path, [subscribe]) + stop_state = SimpleNamespace(is_system_stopped=False) + chain.stop_state = stop_state + _make_tasks_ready(monkeypatch) + + def process(item, _searchchain, *, execution_context): + """模拟站点搜索返回部分候选时进程进入停止阶段。""" + stop_state.is_system_stopped = True + assert execution_context.should_stop() is True + return item + + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert batch.state == "queued" + assert batch.finished_count == 0 + assert batch.failed_count == 0 + assert batch.cancelled_count == 0 + assert batch.skipped_count == 0 + + stop_state.is_system_stopped = False + recovered = chain.subscription_search_repository.claim_next(owner="worker-after-restart") + assert recovered is not None + assert recovered.subscription_id == subscribe.id + assert recovered.attempt_count == 2 + + +def test_system_stop_completes_when_download_side_effect_already_started(tmp_path, monkeypatch): + """停机晚于下载器副作用边界时必须终结任务，避免重启后重复提交。""" + subscribe = _subscribe(11) + chain = _chain(tmp_path, [subscribe]) + stop_state = SimpleNamespace(is_system_stopped=False) + chain.stop_state = stop_state + _make_tasks_ready(monkeypatch) + + def process(item, _searchchain, *, execution_context): + """模拟下载器已接收任务后进程进入停止阶段。""" + execution_context.mark_download_started() + stop_state.is_system_stopped = True + return item + + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert batch.state == "completed" + assert batch.finished_count == 1 + assert batch.failed_count == 0 + assert batch.cancelled_count == 0 + assert batch.skipped_count == 0 + assert chain.subscription_search_repository.claim_next(owner="worker-after-restart") is None + + +def test_ttl_expiry_after_normal_return_marks_task_and_batch_failed(tmp_path, monkeypatch): + """正常返回也必须检查执行 TTL，未提交下载时按失败收口。""" + subscribe = _subscribe(15) + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + monkeypatch.setattr( + chain._subscription_execution_admission, + "is_expired", + lambda _lease: True, + ) + monkeypatch.setattr( + chain, + "_process_search_subscription", + lambda item, _searchchain, **_kwargs: item, + ) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert batch.state == "failed" + assert batch.finished_count == 0 + assert batch.failed_count == 1 + assert batch.cancelled_count == 0 + assert batch.last_error == "订阅执行已超过协作截止时间" + + +def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch): + """TTL 晚于下载器副作用边界时按实际结果完成，避免下轮重复提交。""" + subscribe = _subscribe(16) + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + monkeypatch.setattr( + chain._subscription_execution_admission, + "is_expired", + lambda _lease: True, + ) + + def process(item, _searchchain, *, execution_context): + """模拟下载器已接收任务后搜索链正常返回。""" + execution_context.mark_download_started() + return item + + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert batch.state == "completed" + assert batch.finished_count == 1 + assert batch.failed_count == 0 + assert batch.cancelled_count == 0 + + +def test_site_budget_ttl_expiry_marks_task_and_batch_failed(tmp_path, monkeypatch): + """站点预算观察到执行 TTL 到期时必须失败收口，不能伪装成用户取消。""" + subscribe = _subscribe(12) + chain = _chain(tmp_path, [subscribe]) + _make_tasks_ready(monkeypatch) + monkeypatch.setattr( + chain._subscription_execution_admission, + "is_expired", + lambda _lease: True, + ) + + def process(_item, _searchchain, *, execution_context): + """模拟站点预算将协作超时传播为搜索取消异常。""" + assert execution_context.should_stop() is True + raise SubscriptionSearchCancelled("订阅搜索已取消") + + monkeypatch.setattr(chain, "_process_search_subscription", process) + + with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): + batch_id = chain.search(state="R") + + batch = chain.get_search_batch(batch_id) + assert batch.state == "failed" + assert batch.finished_count == 0 + assert batch.failed_count == 1 + assert batch.cancelled_count == 0 + assert batch.last_error == "订阅执行已超过协作截止时间" diff --git a/tests/test_subscription_search_queue.py b/tests/test_subscription_search_queue.py index f51907bc1..1dcf2d06d 100644 --- a/tests/test_subscription_search_queue.py +++ b/tests/test_subscription_search_queue.py @@ -46,6 +46,42 @@ def test_search_queue_coalesces_active_subscription_and_raises_priority(tmp_path assert first.task_id != second.task_id +def test_search_queue_claims_each_subscription_only_after_its_available_at(tmp_path): + """逐订阅到期时间必须持久化，未到期任务不能占用同步 worker。""" + repository, _engine = _repository(tmp_path) + now = datetime.now(timezone.utc) + ready_at = (now - timedelta(seconds=1)).isoformat(timespec="seconds") + later_at = (now + timedelta(minutes=5)).isoformat(timespec="seconds") + repository.enqueue( + subscription_ids=(20, 21), + source="fallback", + priority=10, + available_at_by_subscription={20: ready_at, 21: later_at}, + ) + + first = repository.claim_next(owner="worker-a") + + assert first.subscription_id == 20 + assert first.available_at == ready_at + assert repository.finish_task( + task_id=first.task_id, + lease_token=first.lease_token, + state="completed", + ) is True + assert repository.claim_next(owner="worker-a") is None + + repository.enqueue( + subscription_ids=(21,), + source="manual", + priority=100, + available_at_by_subscription={21: ready_at}, + ) + accelerated = repository.claim_next(owner="worker-b") + assert accelerated.subscription_id == 21 + assert accelerated.available_at == ready_at + assert accelerated.priority == 100 + + def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path): """进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。""" repository, engine = _repository(tmp_path) @@ -160,6 +196,38 @@ def test_search_queue_finishes_batch_with_aggregated_failure(tmp_path): assert batch.last_error == "site timeout" +def test_search_queue_aggregates_skipped_tasks_without_marking_success(tmp_path): + """跳过任务应单独计数并让批次暴露 skipped 聚合终态。""" + repository, _engine = _repository(tmp_path) + enqueued = repository.enqueue( + subscription_ids=(10, 11), + source="fallback", + priority=10, + ) + first = repository.claim_next(owner="worker-a") + assert repository.finish_task( + task_id=first.task_id, + lease_token=first.lease_token, + state="skipped", + error="同一订阅正在由其他通道处理，本轮搜索已跳过", + ) is True + second = repository.claim_next(owner="worker-a") + assert repository.finish_task( + task_id=second.task_id, + lease_token=second.lease_token, + state="completed", + ) is True + + batch = repository.get_batch(enqueued.batch.batch_id) + + assert batch.state == "skipped" + assert batch.finished_count == 1 + assert batch.failed_count == 0 + assert batch.cancelled_count == 0 + assert batch.skipped_count == 1 + assert batch.last_error == "同一订阅正在由其他通道处理，本轮搜索已跳过" + + def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path): """手工任务可优先，但等待超过公平窗口的兜底任务不得持续饥饿。""" repository, engine = _repository(tmp_path) diff --git a/tests/test_subscription_search_skipped_migration.py b/tests/test_subscription_search_skipped_migration.py new file mode 100644 index 000000000..8677698ee --- /dev/null +++ b/tests/test_subscription_search_skipped_migration.py @@ -0,0 +1,61 @@ +"""订阅搜索批次跳过计数的 Alembic 可逆迁移测试。""" + +import importlib + +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + +MIGRATION = "database.versions.b6c1d9e4a7f2_3_0_24" + + +def _bind_migration(monkeypatch, connection): + """把跳过计数迁移绑定到隔离 SQLite 连接。""" + migration = importlib.import_module(MIGRATION) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + return migration + + +def test_subscription_search_skipped_count_migration_is_reversible(monkeypatch) -> None: + """存量批次获得零默认值，迁移可重复执行并完整回滚。""" + engine = sa.create_engine("sqlite://") + metadata = sa.MetaData() + batch = sa.Table( + "subscriptionsearchbatch", + metadata, + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("batch_id", sa.String(64), nullable=False), + ) + with engine.begin() as connection: + metadata.create_all(connection) + connection.execute(batch.insert(), {"id": 1, "batch_id": "batch-1"}) + migration = _bind_migration(monkeypatch, connection) + + migration.upgrade() + migration.upgrade() + + columns = { + column["name"] + for column in sa.inspect(connection).get_columns("subscriptionsearchbatch") + } + assert "skipped_count" in columns + assert connection.execute( + sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1") + ).scalar_one() == 0 + + migration.downgrade() + downgraded = { + column["name"] + for column in sa.inspect(connection).get_columns("subscriptionsearchbatch") + } + assert "skipped_count" not in downgraded + + migration.upgrade() + assert connection.execute( + sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1") + ).scalar_one() == 0 + engine.dispose() diff --git a/tests/test_subscription_site_budget.py b/tests/test_subscription_site_budget.py index 3e76326f8..0cdb0c2a3 100644 --- a/tests/test_subscription_site_budget.py +++ b/tests/test_subscription_site_budget.py @@ -14,12 +14,13 @@ from app.application.subscription.sitebudget import ( SiteBudgetClaim, SubscriptionSearchCancelled, SubscriptionSiteBudget, + SubscriptionSiteBudgetUnavailable, ) from app.chain.search.facade import SearchChain from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository from app.db.base import Base from app.db.models.subscriptionsearch import SubscriptionSiteBudget as SiteBudgetRecord -from app.modules.indexer import _classify_search_failure +from app.modules.indexer import IndexerModule, _classify_search_failure from app.runtime.stop import ProcessStopState @@ -76,6 +77,8 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat next_allowed_at=(now + timedelta(minutes=5)).isoformat(timespec="seconds"), error="request timeout", ) is True + cooled = repository.claim_site(site_id=3, owner="worker-b", lease_seconds=900) + assert cooled.acquired is False with Session(engine) as session: failed = session.execute( select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3) @@ -103,6 +106,29 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat assert healthy.last_outcome == "success" +def test_site_budget_ignores_legacy_success_interval(tmp_path): + """升级前留下的正常成功间隔不能阻止新语义下的到期订阅。""" + repository, engine = _repository(tmp_path) + first = repository.claim_site(site_id=4, owner="worker-a", lease_seconds=900) + future = (datetime.now(timezone.utc) + timedelta(minutes=10)).isoformat(timespec="seconds") + assert repository.finish_site( + site_id=4, + lease_token=first.lease_token, + outcome="success", + next_allowed_at=future, + ) is True + + recovered = repository.claim_site(site_id=4, owner="worker-b", lease_seconds=900) + + assert recovered.acquired is True + assert recovered.lease_token is not None + with Session(engine) as session: + record = session.execute( + select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 4) + ).scalar_one() + assert record.next_allowed_at < future + + @pytest.mark.parametrize( ("error", "outcome"), [ @@ -135,40 +161,50 @@ class _WaitingRepository: raise AssertionError("未取得站点预算时不应收口") -def test_site_budget_wait_is_cancellable_without_business_lock(): - """批次取消应在一次短等待后终止预算获取。""" - cancelled = False - - def sleeper(_seconds: float) -> None: - """模拟等待一次后收到取消请求。""" - nonlocal cancelled - cancelled = True - +def test_unavailable_site_budget_does_not_sleep_in_worker(): + """错误冷却中的站点应立即留待下轮，不占用同步 worker 等待。""" budget = SubscriptionSiteBudget( repository=_WaitingRepository(), owner="worker-a", - cancelled=lambda: cancelled, + cancelled=lambda: False, + stop_state=ProcessStopState(), + ) + + with pytest.raises(SubscriptionSiteBudgetUnavailable): + budget.acquire(9) + + +def test_site_budget_checks_cancellation_before_repository_claim(): + """取消或停机必须在创建任何站点租约前终止当前任务。""" + class _ForbiddenRepository(_WaitingRepository): + """取消路径不允许触发持久认领。""" + + def claim_site(self, **_kwargs) -> SiteBudgetClaim: + """若取消检查失效则立即暴露。""" + raise AssertionError("取消任务不应认领站点预算") + + budget = SubscriptionSiteBudget( + repository=_ForbiddenRepository(), + owner="worker-a", + cancelled=lambda: True, stop_state=ProcessStopState(), - sleeper=sleeper, - max_wait_seconds=600, ) with pytest.raises(SubscriptionSearchCancelled): budget.acquire(9) -def test_site_budget_delay_keeps_manual_requests_under_same_budget(): - """来源优先级不参与站点冷却计算，手工任务不能获得旁路。""" +def test_site_budget_only_delays_external_failures(): + """正常完成立即恢复，外站错误仍按类别进入冷却。""" budget = SubscriptionSiteBudget( repository=_WaitingRepository(), owner="manual-worker", cancelled=lambda: False, stop_state=ProcessStopState(), - random_uniform=lambda _low, _high: 60.0, ) - assert budget._next_delay("success", consecutive_failures=0) == 60.0 # pylint: disable=protected-access - assert budget._next_delay("success", consecutive_failures=2) == 120.0 # pylint: disable=protected-access + assert budget._next_delay("success", consecutive_failures=0) == 0.0 # pylint: disable=protected-access + assert budget._next_delay("success", consecutive_failures=2) == 0.0 # pylint: disable=protected-access assert budget._next_delay("rate_limited", consecutive_failures=0) == 900.0 # pylint: disable=protected-access @@ -206,14 +242,13 @@ def test_skipped_search_releases_budget_without_external_interval(): def test_search_provider_reports_cooled_site_without_blocking_other_results(): - """超出短等待窗口的站点返回空页并记录聚合失败，而非阻塞整个 provider。""" + """错误冷却中的站点返回空页并记录聚合失败，而非阻塞 provider。""" repository = _WaitingRepository() budget = SubscriptionSiteBudget( repository=repository, owner="fallback-worker", cancelled=lambda: False, stop_state=ProcessStopState(), - max_wait_seconds=0, ) chain = object.__new__(SearchChain) chain.configure_subscription_site_budget(budget) @@ -233,7 +268,7 @@ def test_search_provider_reports_cooled_site_without_blocking_other_results(): def test_search_provider_releases_successful_site_budget(): - """真实站点页完成后必须释放租约并写入成功间隔。""" + """真实站点页正常完成后必须释放租约并立即允许下一任务。""" captured = {} class _Repository(_WaitingRepository): @@ -260,7 +295,6 @@ def test_search_provider_releases_successful_site_budget(): owner="manual-worker", cancelled=lambda: False, stop_state=ProcessStopState(), - random_uniform=lambda _low, _high: 60.0, ) chain = object.__new__(SearchChain) chain.configure_subscription_site_budget(budget) @@ -283,3 +317,78 @@ def test_search_provider_releases_successful_site_budget(): assert captured["site_id"] == 12 assert captured["outcome"] == "success" assert captured["lease_token"] == "lease-token" + + +def test_search_provider_aggregates_swallowed_indexer_failure(monkeypatch): + """索引器吞掉外站异常返回空页时，provider 仍须暴露站点失败。""" + captured = {} + + class _Repository(_WaitingRepository): + """提供立即可用租约并记录失败收口。""" + + def claim_site(self, *, site_id: int, owner: str, lease_seconds: int) -> SiteBudgetClaim: + """返回当前调用独占的站点租约。""" + del owner, lease_seconds + now = datetime.now(timezone.utc) + return SiteBudgetClaim( + site_id=site_id, + acquired=True, + retry_at=now.isoformat(timespec="seconds"), + consecutive_failures=0, + lease_token="lease-token", + ) + + def finish_site(self, **kwargs) -> bool: + """记录预算收口，确认错误冷却仍由 budget.finish 负责。""" + captured.update(kwargs) + return True + + def execute_search(_site, _request): + """模拟 IndexerModule 捕获的 HTTP 429。""" + raise RuntimeError("HTTP 429") + + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__search_check", + staticmethod(lambda _site, _keyword=None: True), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__execute_search", + staticmethod(execute_search), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__indexer_statistic", + staticmethod(lambda **_kwargs: None), + ) + monkeypatch.setattr( + IndexerModule, + "_IndexerModule__parse_result", + staticmethod(lambda **_kwargs: []), + ) + + budget = SubscriptionSiteBudget( + repository=_Repository(), + owner="fallback-worker", + cancelled=lambda: False, + stop_state=ProcessStopState(), + ) + chain = object.__new__(SearchChain) + chain.configure_subscription_site_budget(budget) + chain.search_site_torrents = object.__new__(IndexerModule).search_torrents + + result = chain._search_site_torrents_with_budget( # pylint: disable=protected-access + site={"id": 13, "name": "Flaky"}, + keyword="movie", + mtype=None, + page=0, + ) + + assert result == [] + assert captured["outcome"] == "rate_limited" + assert captured["error"] == "HTTP 429" + failures = chain.consume_subscription_site_budget_failures() + assert len(failures) == 1 + assert "Flaky" in failures[0] + assert "HTTP 429" in failures[0]