优化订阅执行治理与规模性能 (#6557)

* feat(subscription): govern concurrent search and match execution

* refactor(subscription): consume complete candidate snapshots

* refactor(subscription): remove download submission ledger

* feat(subscription): stagger fallback searches without site gaps

* refactor(subscription): remove unused retry capability

* test(subscription): verify complete scale execution

* fix(subscription): surface site search failures

* test(subscription): verify site pressure through search

* test(subscription): use static facade imports

* fix(subscription): preserve completed downloads on shutdown

* fix(subscription): classify execution expiry as failure

* fix(subscription): preserve fallback search staggering

* test(subscription): stabilize unfinished wrapper gate

* fix(subscription): settle expired search returns

* style(subscription): normalize governance imports

* fix(subscription): align governance type contracts

* docs(architecture): sync mypy debt metric

* docs(architecture): sync startup module counts
This commit is contained in:
InfinityPacer
2026-09-03 11:26:24 +08:00
committed by GitHub
parent 7e42ab215d
commit 68880dbbf1
66 changed files with 4262 additions and 2649 deletions
-2
View File
@@ -467,7 +467,6 @@ FIELD_DESCRIPTIONS.update(
"cat": "Exact site category identifier returned by site.category.",
"check_only": "Validate or preview the recommendation without applying search-result filtering.",
"can_cancel": "Whether the current subscription execution can be cancelled.",
"can_retry": "Whether the current subscription execution can be retried.",
"concurrency_key": "Workflow expression used to serialize actions sharing the same runtime key.",
"condition": "Workflow branch or flow condition expression evaluated at runtime.",
"context": "Persisted workflow execution context available to later actions.",
@@ -517,7 +516,6 @@ FIELD_DESCRIPTIONS.update(
"rating": "Numeric plugin rating accepted by the endpoint's declared bounds.",
"reason": "Human-readable justification recorded with a manual-review decision.",
"recursive": "Apply media-aware renaming recursively to child files when true.",
"requires_reconciliation": "Whether the execution requires reconciliation before it can continue.",
"release_year": "Release year matched by an automatic category rule.",
"result": "Persisted workflow action result value.",
"result_payload": "Structured external-operation result recorded with manual review.",
@@ -3532,12 +3532,6 @@
"title": "Can Cancel",
"type": "boolean"
},
"can_retry": {
"default": false,
"description": "Whether the current subscription execution can be retried.",
"title": "Can Retry",
"type": "boolean"
},
"current_site_id": {
"anyOf": [
{
@@ -3567,12 +3561,6 @@
"title": "Phase",
"type": "string"
},
"requires_reconciliation": {
"default": false,
"description": "Whether the execution requires reconciliation before it can continue.",
"title": "Requires Reconciliation",
"type": "boolean"
},
"source": {
"anyOf": [
{
-2
View File
@@ -12,7 +12,6 @@ from app.application.configuration import ChainRuntimeConfig
from app.runtime.stop import StopState, runtime_stop_state
if TYPE_CHECKING:
from app.application.download.admission import SubscriptionDownloadRepository
from app.application.download.failures import DownloadFailureRepository
from app.application.history import (
DownloadHistoryRepository,
@@ -77,7 +76,6 @@ class ChainRuntimeContext:
download_failure_repository: DownloadFailureRepository
user_repository: ChainUserRepository
subscription_search_repository: Optional[SubscriptionSearchRepository] = None
subscription_download_repository: Optional[SubscriptionDownloadRepository] = None
legacy_transfer_command: Optional[LegacyTransferCommand] = None
durable_event_writer: Optional[ChainDurableEventWriter] = None
configuration: ChainRuntimeConfig = field(
+3 -119
View File
@@ -1,130 +1,14 @@
"""订阅下载提交的持久幂等与不确定终态合同"""
"""订阅下载提交前的可取消执行边界"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Callable, Optional, Protocol
class DownloadReconciliationRequired(RuntimeError):
"""表示下载器可能已接受任务,必须先对账才能继续自动提交。"""
from typing import Callable, Optional
@dataclass(frozen=True, slots=True)
class SubscriptionDownloadGovernance:
"""订阅身份、入口任务与取消检查传入下载提交边界"""
"""取消检查与副作用起点回传给订阅执行上下文"""
subscription_id: int
mode: str
task_id: Optional[str] = None
cancelled: Optional[Callable[[], bool]] = None
mark_started: Optional[Callable[[], None]] = None
@dataclass(frozen=True, slots=True)
class SubscriptionDownloadRequest:
"""一次订阅下载提交认领所需的规范身份。"""
idempotency_key: str
legacy_idempotency_key: Optional[str]
subscription_id: int
task_id: Optional[str]
logical_identity: str
resource_key: str
coverage: str
mode: str
delivery_scope: str
@dataclass(frozen=True, slots=True)
class SubscriptionDownloadSnapshot:
"""脱离 Session 的订阅下载提交状态快照。"""
idempotency_key: str
subscription_id: int
task_id: Optional[str]
state: str
attempt_count: int
attempt_token: Optional[str]
downloader: Optional[str]
download_hash: Optional[str]
available_at: Optional[str]
last_error: Optional[str]
created_at: str
updated_at: str
@dataclass(frozen=True, slots=True)
class SubscriptionDownloadClaim:
"""返回本次是否取得唯一提交权以及当前持久状态。"""
acquired: bool
snapshot: SubscriptionDownloadSnapshot
class SubscriptionDownloadRepository(Protocol):
"""订阅下载幂等账本所需的最小持久化端口。"""
def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim:
"""按唯一键认领提交;仅到期 retryable/cancelled 状态允许重新认领。"""
...
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]:
"""按幂等键读取现有提交,供键版本兼容检查。"""
...
def mark_accepted(
self,
*,
idempotency_key: str,
attempt_token: str,
downloader: Optional[str],
download_hash: str,
) -> bool:
"""记录下载器已明确接受任务,后续任何失败都不得自动重试。"""
...
def mark_succeeded(
self,
*,
idempotency_key: str,
attempt_token: str,
) -> bool:
"""在 canonical 本地结算完成后写入成功终态。"""
...
def mark_retryable(
self,
*,
idempotency_key: str,
attempt_token: str,
available_at: str,
error: Optional[str],
) -> bool:
"""记录下载器明确拒绝、尚未产生外部副作用的可重试状态。"""
...
def mark_reconcile_required(
self,
*,
idempotency_key: str,
attempt_token: str,
error: Optional[str],
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
) -> bool:
"""冻结可能已产生外部副作用的提交,等待下载器对账。"""
...
def mark_cancelled(
self,
*,
idempotency_key: str,
attempt_token: str,
) -> bool:
"""仅在进入下载器副作用前收口取消。"""
...
def has_started_for_task(self, task_id: str) -> bool:
"""判断搜索任务是否已有不能按未执行处理的下载提交。"""
...
+1 -55
View File
@@ -1,10 +1,7 @@
"""订阅候选批次与无损路由合同。"""
"""订阅候选无损路由合同。"""
import copy
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Dict, List, Optional, cast
from uuid import uuid4
from app.application.subscription.contract import SubscriptionSnapshot
from app.domain.context import Context, MediaInfo
@@ -15,57 +12,6 @@ from app.schemas.types import MediaSource, MediaType
CandidateGroups = Dict[str, List[Context]]
@dataclass(slots=True)
class CandidateBatch:
"""一次资源获取产生的完整候选、增量候选与重试候选边界。"""
batch_id: str
source: str
candidates: CandidateGroups
fresh_candidates: CandidateGroups = field(default_factory=dict)
retry_candidates: CandidateGroups = field(default_factory=dict)
sites: tuple[str, ...] = ()
started_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
finished_at: Optional[datetime] = None
@classmethod
def create(
cls,
*,
source: str,
candidates: CandidateGroups,
fresh_candidates: Optional[CandidateGroups] = None,
retry_candidates: Optional[CandidateGroups] = None,
sites: Optional[List[str]] = None,
started_at: Optional[datetime] = None,
) -> "CandidateBatch":
"""构造已完成获取的候选批次。"""
return cls(
batch_id=uuid4().hex,
source=source,
candidates=candidates,
fresh_candidates=fresh_candidates or {},
retry_candidates=retry_candidates or {},
sites=tuple(sites or candidates.keys()),
started_at=started_at or datetime.now(timezone.utc),
finished_at=datetime.now(timezone.utc),
)
@classmethod
def from_legacy(cls, candidates: CandidateGroups, source: str = "legacy") -> "CandidateBatch":
"""把旧入口传入的完整候选包装为无增量声明的兼容批次。"""
return cls.create(source=source, candidates=candidates)
@staticmethod
def count(groups: CandidateGroups) -> int:
"""统计分站点候选总数。"""
return sum(len(contexts) for contexts in groups.values())
def build_index(self) -> "CandidateIndex":
"""基于完整候选构建一次性无损索引。"""
return CandidateIndex(self.candidates)
class CandidateIndex:
"""一次构建并保持原顺序的订阅候选身份索引。"""
+97 -6
View File
@@ -1,11 +1,101 @@
"""订阅搜索执行批次任务与持久队列端口。"""
"""订阅执行准入、搜索上下文、批次任务与持久队列端口。"""
import threading
import time
from dataclasses import dataclass
from typing import Optional, Protocol
from typing import Callable, Mapping, Optional, Protocol
from uuid import uuid4
from app.application.subscription.sitebudget import SiteBudgetClaim
@dataclass(frozen=True, slots=True)
class SubscriptionExecutionLease:
"""一次订阅执行的进程内所有权及协作截止时间。"""
subscription_id: int
operation: str
owner_token: str
expires_at: float
class SubscriptionExecutionAdmission:
"""按订阅 ID 控制 Search 与 Match 的进程内互斥。"""
def __init__(self, clock: Callable[[], float] = time.monotonic) -> None:
"""保存单调时钟和当前活跃 owner。"""
self._clock = clock
self._lock = threading.Lock()
self._owners: dict[int, SubscriptionExecutionLease] = {}
def try_acquire(
self,
*,
subscription_id: int,
operation: str,
ttl_seconds: float,
) -> Optional[SubscriptionExecutionLease]:
"""无等待取得订阅所有权;已有 owner 时直接返回空。"""
with self._lock:
if subscription_id in self._owners:
return None
lease = SubscriptionExecutionLease(
subscription_id=subscription_id,
operation=operation,
owner_token=uuid4().hex,
expires_at=self._clock() + max(1.0, ttl_seconds),
)
self._owners[subscription_id] = lease
return lease
def release(self, lease: SubscriptionExecutionLease) -> bool:
"""仅允许当前 owner 释放订阅所有权。"""
with self._lock:
current = self._owners.get(lease.subscription_id)
if current is None or current.owner_token != lease.owner_token:
return False
self._owners.pop(lease.subscription_id)
return True
def is_expired(self, lease: SubscriptionExecutionLease) -> bool:
"""判断 owner 是否已超过协作执行截止时间。"""
return self._clock() >= lease.expires_at
@dataclass(slots=True)
class SubscriptionExecutionContext:
"""一次订阅执行的显式取消、阶段和副作用边界。"""
lease: SubscriptionExecutionLease
admission: SubscriptionExecutionAdmission
task_id: Optional[str] = None
cancel_requested: Optional[Callable[[], bool]] = None
phase_changed: Optional[Callable[[str, Optional[int]], None]] = None
download_started: bool = False
def is_cancel_requested(self) -> bool:
"""判断调用入口是否请求在下一个安全边界退出。"""
return bool(self.cancel_requested and self.cancel_requested())
def is_expired(self) -> bool:
"""判断本次执行是否已经超过协作截止时间。"""
return self.admission.is_expired(self.lease)
def should_stop(self) -> bool:
"""在安全边界合并用户取消和执行 TTL。"""
return self.is_expired() or self.is_cancel_requested()
def report_phase(self, phase: str, current_site_id: Optional[int] = None) -> None:
"""向当前搜索任务报告业务阶段。"""
if self.phase_changed:
self.phase_changed(phase, current_site_id)
def mark_download_started(self) -> None:
"""标记执行已越过下载器副作用边界。"""
self.download_started = True
self.report_phase("submitting")
@dataclass(frozen=True, slots=True)
class SearchBatchSnapshot:
"""订阅搜索批次的持久业务状态快照。"""
@@ -24,6 +114,7 @@ class SearchBatchSnapshot:
started_at: Optional[str] = None
finished_at: Optional[str] = None
last_error: Optional[str] = None
skipped_count: int = 0
@dataclass(frozen=True, slots=True)
@@ -68,9 +159,9 @@ class SubscriptionSearchRepository(Protocol):
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str] = None,
available_at_by_subscription: Optional[Mapping[int, str]] = None,
) -> SearchEnqueueResult:
"""按订阅 ID 建立批次,在启动抖动后合并活动任务。"""
"""按订阅 ID 和各自到期时间建立或合并活动任务。"""
...
def claim_next(self, *, owner: str, lease_seconds: int = 900) -> Optional[SearchTaskSnapshot]:
@@ -85,7 +176,7 @@ class SubscriptionSearchRepository(Protocol):
state: str,
error: Optional[str] = None,
) -> bool:
"""以租约令牌收口任务,并推进所属批次聚合状态。"""
"""以租约令牌收口任务,并推进所属批次聚合状态(含 skipped"""
...
def update_task_phase(
@@ -140,5 +231,5 @@ class SubscriptionSearchRepository(Protocol):
next_allowed_at: str,
error: Optional[str] = None,
) -> bool:
"""释放站点预算并写入间隔、冷却和恢复状态。"""
"""释放站点预算并写入错误冷却和恢复状态。"""
...
+8 -28
View File
@@ -2,8 +2,6 @@
from __future__ import annotations
import random
import time
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Callable, Optional, Protocol
@@ -17,7 +15,7 @@ class SubscriptionSearchCancelled(RuntimeError):
class SubscriptionSiteBudgetUnavailable(RuntimeError):
"""表示站点预算超出本任务允许的短等待窗口"""
"""表示站点仍处于错误冷却或已有未释放租约"""
def __init__(self, *, site_id: int, retry_at: str) -> None:
"""保存站点和下一次可尝试时间,供批次聚合失败展示。"""
@@ -69,7 +67,7 @@ def _utc_now() -> datetime:
class SubscriptionSiteBudget:
"""在同步搜索 worker 内执行可取消的站点预算等待与反馈"""
"""以非阻塞认领保护站点租约,并持久化外站错误冷却"""
def __init__(
self,
@@ -78,31 +76,21 @@ class SubscriptionSiteBudget:
owner: str,
cancelled: Callable[[], bool],
stop_state: StopState,
interval_range: tuple[float, float] = (60.0, 300.0),
lease_seconds: int = 900,
max_wait_seconds: float = 5.0,
random_uniform: Callable[[float, float], float] = random.uniform,
sleeper: Callable[[float], None] = time.sleep,
clock: Callable[[], datetime] = _utc_now,
phase_changed: Optional[Callable[[str, Optional[int]], None]] = None,
) -> None:
"""保存持久化端口及可注入的时钟、随机数和等待实现"""
"""保存持久化端口及可注入的时钟和阶段回调"""
self._repository = repository
self._owner = owner
self._cancelled = cancelled
self._stop_state = stop_state
self._interval_range = interval_range
self._lease_seconds = max(1, lease_seconds)
self._max_wait_seconds = max(0.0, max_wait_seconds)
self._random_uniform = random_uniform
self._sleeper = sleeper
self._clock = clock
self._phase_changed = phase_changed
def acquire(self, site_id: int) -> SiteBudgetClaim:
"""循环认领指定站点,并在每秒边界检查取消与停机"""
deadline = time.monotonic() + self._max_wait_seconds
while True:
"""只认领一次指定站点,未就绪时留待下一次正常调度"""
self._raise_if_cancelled()
claim = self._repository.claim_site(
site_id=site_id,
@@ -113,14 +101,10 @@ class SubscriptionSiteBudget:
self._report_phase("searching", site_id)
return claim
self._report_phase("waiting_site_budget", site_id)
retry_at = datetime.fromisoformat(claim.retry_at)
remaining = max(0.0, (retry_at - self._clock()).total_seconds())
if remaining > max(0.0, deadline - time.monotonic()):
raise SubscriptionSiteBudgetUnavailable(
site_id=site_id,
retry_at=claim.retry_at,
)
self._sleeper(min(max(remaining, 0.05), 1.0))
def _report_phase(self, phase: str, site_id: Optional[int]) -> None:
"""向任务所有者报告不改变预算语义的业务阶段。"""
@@ -128,7 +112,7 @@ class SubscriptionSiteBudget:
self._phase_changed(phase, site_id)
def finish(self, claim: SiteBudgetClaim, observation: SiteSearchObservation) -> bool:
"""依据调用结果计算随机间隔或错误冷却并释放租约"""
"""正常结果立即释放站点,错误结果按类别写入冷却"""
if not claim.lease_token:
return False
outcome = observation.outcome if observation.attempted else "skipped"
@@ -143,13 +127,9 @@ class SubscriptionSiteBudget:
)
def _next_delay(self, outcome: str, consecutive_failures: int) -> float:
"""为成功渐进恢复错误退避和本地跳过计算下一次等待"""
if outcome == "skipped":
"""正常或本地跳过立即恢复错误按连续失败次数退避。"""
if outcome in {"success", "skipped"}:
return 0.0
if outcome == "success":
low, high = self._interval_range
recovery_factor = 1.0 + min(max(consecutive_failures, 0), 3) * 0.5
return self._random_uniform(low, high) * recovery_factor
exponent = min(max(consecutive_failures, 0), 5)
base, ceiling = {
"rate_limited": (900.0, 21600.0),
@@ -160,6 +140,6 @@ class SubscriptionSiteBudget:
return float(min(base * (2**exponent), ceiling))
def _raise_if_cancelled(self) -> None:
"""不持有业务锁的等待边界传播取消或停机。"""
"""创建站点租约前传播取消或停机。"""
if self._stop_state.is_system_stopped or self._cancelled():
raise SubscriptionSearchCancelled("订阅搜索已取消")
+12 -69
View File
@@ -4,13 +4,12 @@ from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Optional, Protocol
from app.application.download.admission import SubscriptionDownloadSnapshot
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
@dataclass(frozen=True, slots=True)
class SubscriptionExecutionStatus:
"""一个订阅跨搜索与下载账本合并后的用户可见状态。"""
"""一个订阅最近搜索任务的用户可见状态。"""
state: str
phase: str
@@ -21,8 +20,6 @@ class SubscriptionExecutionStatus:
current_site_id: Optional[int] = None
error: Optional[str] = None
can_cancel: bool = False
can_retry: bool = False
requires_reconciliation: bool = False
@dataclass(frozen=True, slots=True)
@@ -44,10 +41,11 @@ class SubscriptionBatchStatus:
current_site_id: Optional[int] = None
error: Optional[str] = None
can_cancel: bool = False
skipped_count: int = 0
class SubscriptionExecutionReadRepository(Protocol):
"""请求级读取搜索任务批次与下载提交事实的端口。"""
"""请求级读取搜索任务批次事实的端口。"""
async def latest_search_tasks(
self,
@@ -56,13 +54,6 @@ class SubscriptionExecutionReadRepository(Protocol):
"""返回每条订阅最近更新的搜索任务。"""
...
async def latest_download_submissions(
self,
subscription_ids: tuple[int, ...],
) -> dict[int, SubscriptionDownloadSnapshot]:
"""返回每条订阅最近更新的下载提交。"""
...
async def list_batches(
self,
*,
@@ -81,16 +72,8 @@ class SubscriptionExecutionReadRepository(Protocol):
class SubscriptionExecutionStatusService:
"""把搜索队列与下载幂等账本投影为稳定业务状态。"""
"""把搜索队列投影为稳定业务状态。"""
_DOWNLOAD_STATES = {
"submitting": "submitting",
"accepted": "accepted",
"succeeded": "completed",
"retryable": "retryable",
"reconcile_required": "reconcile_required",
"cancelled": "cancelled",
}
_ACTIVE_STATES = {
"queued",
"running",
@@ -99,16 +82,8 @@ class SubscriptionExecutionStatusService:
"waiting_site_budget",
"preparing",
"submitting",
"accepted",
"cancelling",
}
_DOWNLOAD_OVERRIDE_STATES = {
"submitting",
"accepted",
"retryable",
"reconcile_required",
}
def __init__(
self,
repository: SubscriptionExecutionReadRepository,
@@ -133,14 +108,10 @@ class SubscriptionExecutionStatusService:
if not ids:
return {}
tasks = await self._repository.latest_search_tasks(ids)
downloads = await self._repository.latest_download_submissions(ids)
result: dict[int, SubscriptionExecutionStatus] = {}
for subscription_id in ids:
task = tasks.get(subscription_id)
download = downloads.get(subscription_id)
if download and self._download_wins(task, download):
result[subscription_id] = self._from_download(download, task)
elif task:
if task:
result[subscription_id] = self._from_task(task)
return result
@@ -175,17 +146,6 @@ class SubscriptionExecutionStatusService:
return None
return self._from_batch(batch, tasks)
@classmethod
def _download_wins(
cls,
task: Optional[SearchTaskSnapshot],
download: SubscriptionDownloadSnapshot,
) -> bool:
"""下载风险状态优先,其余事实按更新时间选择。"""
if download.state in cls._DOWNLOAD_OVERRIDE_STATES:
return True
return task is None or download.updated_at >= task.updated_at
@classmethod
def _from_task(cls, task: SearchTaskSnapshot) -> SubscriptionExecutionStatus:
"""把搜索任务状态归一为稳定业务词汇。"""
@@ -205,29 +165,6 @@ class SubscriptionExecutionStatusService:
updated_at=task.updated_at,
error=cls._safe_error(task.last_error),
can_cancel=state in cls._ACTIVE_STATES,
can_retry=state == "failed",
)
@classmethod
def _from_download(
cls,
download: SubscriptionDownloadSnapshot,
task: Optional[SearchTaskSnapshot],
) -> SubscriptionExecutionStatus:
"""把下载提交账本状态投影为业务状态并保留搜索来源。"""
state = cls._DOWNLOAD_STATES.get(download.state, download.state)
return SubscriptionExecutionStatus(
state=state,
phase=state,
source=task.source if task else None,
batch_id=task.batch_id if task else None,
task_id=download.task_id or (task.task_id if task else None),
current_site_id=task.current_site_id if task else None,
updated_at=download.updated_at,
error=cls._safe_error(download.last_error),
can_cancel=state == "submitting",
can_retry=state == "retryable",
requires_reconciliation=state == "reconcile_required",
)
@classmethod
@@ -240,7 +177,12 @@ class SubscriptionExecutionStatusService:
current = next((task for task in tasks if task.state == "running"), None)
if current is None:
current = next((task for task in tasks if task.state == "queued"), None)
processed = batch.finished_count + batch.failed_count + batch.cancelled_count
processed = (
batch.finished_count
+ batch.failed_count
+ batch.cancelled_count
+ batch.skipped_count
)
phase = current.phase if current else batch.state
return SubscriptionBatchStatus(
batch_id=batch.batch_id,
@@ -252,6 +194,7 @@ class SubscriptionExecutionStatusService:
finished_count=batch.finished_count,
failed_count=batch.failed_count,
cancelled_count=batch.cancelled_count,
skipped_count=batch.skipped_count,
current_subscription_id=current.subscription_id if current else None,
current_site_id=current.current_site_id if current else None,
created_at=batch.created_at,
+8
View File
@@ -66,6 +66,14 @@ class MusicSubscribeMixinHost(Protocol):
def get_subscribe_source_keyword(self, subscribe: Any) -> str: ...
def _SubscribeChain__candidate_contract_changed(
self,
prepared: Any,
current: Any,
) -> bool:
"""判断准备候选期间订阅身份或过滤合同是否已经变化。"""
...
class InteractionMixinHost(Protocol):
"""交互委托 mixin 对业务 Chain 的最小要求。"""
+70 -5
View File
@@ -1,15 +1,18 @@
import copy
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
from app.application.configuration import get_configured_system_config
from app.application.download.admission import SubscriptionDownloadGovernance
from app.application.subscription.contract import (
SubscriptionRepository,
SubscriptionSnapshot,
build_subscribe_meta,
subscribe_media_key,
)
from app.application.subscription.execution import SubscriptionExecutionContext
from app.application.subscription.mutation import SubscriptionActor
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
from app.application.torrent.download import TorrentHelper
from app.chain._contracts import MusicSubscribeMixinHost
from app.chain.download import DownloadChain
@@ -44,6 +47,9 @@ class MusicSubscribeMixin:
__mixin_host_protocol__ = MusicSubscribeMixinHost
subscription_repository: SubscriptionRepository
sync_subscription_mutation_scope: "SyncSubscriptionMutationScope"
_SubscribeChain__candidate_contract_changed: Callable[
[SubscriptionSnapshot, SubscriptionSnapshot], bool
]
"""
音乐订阅功能域 mixin:单曲/专辑目标识别、实体快照同步、候选筛选、
择优下载与完成推进。
@@ -348,10 +354,32 @@ class MusicSubscribeMixin:
subscribe: SubscriptionSnapshot,
mediainfo: MusicInfo,
contexts: List[Context],
execution_context: Optional[SubscriptionExecutionContext] = None,
) -> None:
"""批量择优下载音乐候选,并按单曲或整专完成语义推进订阅。"""
if not contexts:
return
self._ensure_music_execution_active(execution_context)
repository = self.subscription_repository
current_subscribe = repository.get(subscribe.id)
if current_subscribe is None:
logger.info(f"音乐订阅 {subscribe.id} 已删除,放弃本轮下载提交")
return
if current_subscribe.state == "S":
logger.info(f"音乐订阅 {current_subscribe.name} 已暂停,放弃本轮下载提交")
return
if self._SubscribeChain__candidate_contract_changed(subscribe, current_subscribe):
logger.info(f"音乐订阅 {current_subscribe.name} 的筛选或媒体身份已变化,放弃旧候选并等待下一轮")
return
subscribe = current_subscribe
governance = None
if execution_context:
governance = SubscriptionDownloadGovernance(
cancelled=execution_context.should_stop,
mark_started=execution_context.mark_download_started,
)
execution_context.report_phase("preparing")
self._ensure_music_execution_active(execution_context)
downloads, _ = DownloadChain().batch_download(
contexts=contexts,
username=subscribe.username,
@@ -359,6 +387,7 @@ class MusicSubscribeMixin:
downloader=subscribe.downloader,
source=self.get_subscribe_source_keyword(subscribe),
custom_words=subscribe.custom_words,
governance=governance,
)
successful = [
context for context in downloads or []
@@ -370,7 +399,6 @@ class MusicSubscribeMixin:
context for context in successful
if context.confirmed_full_coverage
]
repository = self.subscription_repository
current_subscribe = None
if subscribe.best_version and quality_downloads:
best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order)
@@ -397,12 +425,18 @@ class MusicSubscribeMixin:
downloads=downloads,
)
def _search_music_subscribe(self, subscribe: SubscriptionSnapshot) -> None:
def _search_music_subscribe(
self,
subscribe: SubscriptionSnapshot,
execution_context: Optional[SubscriptionExecutionContext] = None,
) -> None:
"""复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。"""
self._ensure_music_execution_active(execution_context)
target = self._prepare_music_subscribe(subscribe)
if not target:
return
subscribe, mediainfo, _ = target
self._ensure_music_execution_active(execution_context)
sites = self.get_sub_sites(subscribe)
default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \
@@ -414,13 +448,17 @@ class MusicSubscribeMixin:
searchchain = SearchChain()
contexts: List[Context] = []
if execution_context:
execution_context.report_phase("searching")
for keyword in keywords:
self._ensure_music_execution_active(execution_context)
contexts = searchchain.search_by_title(
title=keyword,
sites=sites,
mtype=MediaType.MUSIC,
rule_groups=rule_groups,
)
self._ensure_music_execution_active(execution_context)
contexts = self._filter_music_subscribe_contexts(
subscribe=subscribe,
mediainfo=mediainfo,
@@ -433,18 +471,28 @@ class MusicSubscribeMixin:
logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源")
return
self._download_music_subscribe(subscribe, mediainfo, contexts)
self._download_music_subscribe(
subscribe,
mediainfo,
contexts,
execution_context=execution_context,
)
def _match_music_subscribe(
self,
subscribe: SubscriptionSnapshot,
contexts: List[Context],
execution_context: Optional[SubscriptionExecutionContext] = None,
) -> None:
"""直接匹配本轮 RSS 缓存中的音乐资源,避免再次调用站点搜索接口。"""
self._ensure_music_execution_active(execution_context)
target = self._prepare_music_subscribe(subscribe)
if not target:
return
subscribe, mediainfo, _ = target
if execution_context:
execution_context.report_phase("matching")
self._ensure_music_execution_active(execution_context)
matched = self._filter_music_subscribe_contexts(
subscribe=subscribe,
mediainfo=mediainfo,
@@ -453,4 +501,21 @@ class MusicSubscribeMixin:
if not matched:
logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源")
return
self._download_music_subscribe(subscribe, mediainfo, matched)
self._download_music_subscribe(
subscribe,
mediainfo,
matched,
execution_context=execution_context,
)
@staticmethod
def _ensure_music_execution_active(
execution_context: Optional[SubscriptionExecutionContext],
) -> None:
"""在音乐搜索、匹配和提交前的安全边界执行取消与 TTL 检查。"""
if execution_context is None:
return
if execution_context.is_cancel_requested():
raise SubscriptionSearchCancelled("订阅搜索已取消")
if execution_context.is_expired():
raise TimeoutError("订阅执行已超过协作截止时间")
-1
View File
@@ -71,7 +71,6 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
self.transfer_execution_repository = context.transfer_execution_repository
self.media_server_repository = context.media_server_repository
self.download_failure_repository = context.download_failure_repository
self.subscription_download_repository = context.subscription_download_repository
self.user_repository = context.user_repository
self.runtime_config = context.configuration
self.stop_state = context.stop_state
-258
View File
@@ -1,258 +0,0 @@
"""订阅下载提交幂等身份与状态转换 owner。"""
from __future__ import annotations
import hashlib
import json
from datetime import datetime, timedelta, timezone
from typing import Optional, Set, cast
from app.application.download.admission import (
DownloadReconciliationRequired,
SubscriptionDownloadClaim,
SubscriptionDownloadGovernance,
SubscriptionDownloadRepository,
SubscriptionDownloadRequest,
)
from app.chain.download.contract import _DownloadOwnerBase
from app.domain import episode as episode_rules
from app.domain.context import Context
from app.schemas.media import build_media_key, resolve_media_identity
from app.schemas.types import MediaType
class DownloadAdmissionOwner(_DownloadOwnerBase):
"""计算订阅提交唯一键并通过持久账本控制外部副作用。"""
def _build_subscription_download_request(
self,
*,
context: Context,
episodes: Optional[Set[int]],
governance: SubscriptionDownloadGovernance,
delivery_scope: str,
) -> SubscriptionDownloadRequest:
"""组合逻辑媒体、资源、覆盖、模式和交付目标生成规范幂等请求。"""
media = context.media_info
meta = context.meta_info
torrent = context.torrent_info
media_source, media_id = resolve_media_identity(media=media)
media_key = build_media_key(media_source, media_id) or (
f"{getattr(media, 'title', '')}:{getattr(media, 'year', '')}"
)
seasons = sorted(set(getattr(meta, "season_list", None) or []))
selected = sorted(set(episodes or getattr(meta, "episode_list", None) or []))
if selected:
coverage = f"episodes:{episode_rules.format_ranges(selected)}"
elif seasons:
coverage = "seasons:" + ",".join(str(season) for season in seasons) + ":full"
else:
coverage = "full"
media_type = getattr(media, "type", None)
media_type_value = getattr(media_type, "value", media_type)
media_season = getattr(media, "season", None)
meta_season = getattr(meta, "season", None)
logical_identity = json.dumps(
{
"media_key": str(media_key or ""),
"media_type": str(media_type_value or ""),
"season": media_season if media_season is not None else meta_season,
"episode_group": getattr(media, "episode_group", None),
"music_type": getattr(media, "music_type", None),
},
ensure_ascii=False,
sort_keys=True,
)
resource_key = self._torrent_resource_key(torrent)
canonical = json.dumps(
{
"logical_identity": logical_identity,
"resource_key": resource_key,
"coverage": coverage,
"mode": governance.mode,
"delivery_scope": delivery_scope,
},
ensure_ascii=False,
sort_keys=True,
)
legacy_logical_identity = json.dumps(
{
"subscription_id": governance.subscription_id,
"media_key": str(media_key or ""),
"media_type": str(media_type_value or ""),
"season": media_season if media_season is not None else meta_season,
"episode_group": getattr(media, "episode_group", None),
},
ensure_ascii=False,
sort_keys=True,
)
legacy_canonical = json.dumps(
{
"logical_identity": legacy_logical_identity,
"resource_key": resource_key,
"coverage": coverage,
"mode": governance.mode,
},
ensure_ascii=False,
sort_keys=True,
)
return SubscriptionDownloadRequest(
idempotency_key=hashlib.sha256(canonical.encode("utf-8")).hexdigest(),
legacy_idempotency_key=hashlib.sha256(
legacy_canonical.encode("utf-8")
).hexdigest(),
subscription_id=governance.subscription_id,
task_id=governance.task_id,
logical_identity=logical_identity,
resource_key=resource_key,
coverage=coverage,
mode=governance.mode,
delivery_scope=delivery_scope,
)
def _claim_subscription_download(
self,
*,
context: Context,
episodes: Optional[Set[int]],
governance: Optional[SubscriptionDownloadGovernance],
downloader: Optional[str],
download_uri: str,
) -> tuple[Optional[SubscriptionDownloadClaim], Optional[str]]:
"""在下载器调用前认领唯一提交权,并返回已成功提交的历史 hash。"""
if governance is None:
return None, None
legacy_hash = self._legacy_subscription_download_hash(
context=context,
episodes=episodes,
governance=governance,
)
if legacy_hash:
return None, legacy_hash
repository = getattr(self, "subscription_download_repository", None)
if repository is None:
raise RuntimeError("订阅下载幂等仓储尚未配置")
request = self._build_subscription_download_request(
context=context,
episodes=episodes,
governance=governance,
delivery_scope=self._subscription_delivery_scope(
downloader=downloader,
download_uri=download_uri,
),
)
if request.legacy_idempotency_key:
legacy = repository.get(request.legacy_idempotency_key)
if legacy and legacy.state == "succeeded" and legacy.download_hash:
return None, legacy.download_hash
if legacy and legacy.state in {"submitting", "accepted", "reconcile_required"}:
raise DownloadReconciliationRequired(
f"订阅下载提交 {legacy.idempotency_key} 当前为 {legacy.state},需要先对账下载器"
)
claim = repository.claim(request)
snapshot = claim.snapshot
if claim.acquired:
if not snapshot.attempt_token:
raise RuntimeError("订阅下载提交已认领但缺少尝试令牌")
return claim, None
if snapshot.state == "succeeded" and snapshot.download_hash:
return claim, snapshot.download_hash
if snapshot.state in {"submitting", "accepted", "reconcile_required"}:
raise DownloadReconciliationRequired(
f"订阅下载提交 {snapshot.idempotency_key} 当前为 {snapshot.state},需要先对账下载器"
)
return claim, None
@staticmethod
def _subscription_delivery_scope(
*,
downloader: Optional[str],
download_uri: str,
) -> str:
"""规范化实际下载器和保存目标,限定跨记录去重的产品边界。"""
return json.dumps(
{
"downloader": downloader or "auto",
"download_uri": download_uri,
},
ensure_ascii=False,
sort_keys=True,
)
def _legacy_subscription_download_hash(
self,
*,
context: Context,
episodes: Optional[Set[int]],
governance: SubscriptionDownloadGovernance,
) -> Optional[str]:
"""重读迁移前下载历史,兼容识别同订阅同 torrent 同覆盖的成功提交。"""
media = context.media_info
meta = context.meta_info
torrent = context.torrent_info
repository = getattr(self, "download_history_repository", None)
media_source, media_id = resolve_media_identity(media=media)
if repository is None or not media_source or not media_id or not torrent:
return None
histories = repository.get_by_media_identity(
media_source=media_source,
media_id=str(media_id),
music_type=getattr(media, "music_type", None),
)
expected_episodes = episode_rules.format_ranges(
sorted(set(episodes or getattr(meta, "episode_list", None) or []))
)
expected_season = str(getattr(meta, "season", None) or "")
for history in histories:
if not history.download_hash:
continue
if history.torrent_name != torrent.title or history.torrent_site != torrent.site_name:
continue
if not self._history_matches_subscription(history.note, governance.subscription_id):
continue
if getattr(history, "episode_group", None) != getattr(media, "episode_group", None):
continue
if expected_episodes:
if history.episodes == expected_episodes:
return cast(str, history.download_hash)
continue
if getattr(media, "type", None) == MediaType.TV and str(history.seasons or "") != expected_season:
continue
return cast(str, history.download_hash)
return None
@staticmethod
def _history_matches_subscription(note: object, subscription_id: int) -> bool:
"""从下载历史来源快照确认记录属于同一订阅,而非同媒体其他记录。"""
if not isinstance(note, dict):
return False
source = note.get("source")
if not isinstance(source, str) or not source.startswith("Subscribe|"):
return False
try:
payload = json.loads(source.split("|", 1)[1])
return int(payload.get("id")) == subscription_id
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
return False
@staticmethod
def _subscription_download_cancelled(
governance: Optional[SubscriptionDownloadGovernance],
) -> bool:
"""在可安全取消边界调用入口提供的取消检查。"""
return bool(governance and governance.cancelled and governance.cancelled())
@staticmethod
def _subscription_download_retry_at(error: Optional[str], ttl_seconds: int) -> str:
"""把明确拒绝的下载提交推迟到失败冷却到期后。"""
del error
return (
datetime.now(timezone.utc) + timedelta(seconds=max(1, ttl_seconds))
).isoformat(timespec="seconds")
def _subscription_download_repository(self) -> SubscriptionDownloadRepository:
"""返回已配置的订阅下载仓储,缺失时拒绝越过外部副作用边界。"""
repository = getattr(self, "subscription_download_repository", None)
if repository is None:
raise RuntimeError("订阅下载幂等仓储尚未配置")
return cast(SubscriptionDownloadRepository, repository)
+2 -2
View File
@@ -87,13 +87,13 @@ class DownloadBatchOwner(_DownloadOwnerBase):
:param userid: 用户ID
:param username: 调用下载的用户名/插件名
:param downloader: 下载器
:param custom_words: 下载来源自定义词governance: 订阅幂等、入口任务和取消边界
:param custom_words: 下载来源自定义词
:param governance: 订阅取消与下载器副作用边界
:return: 已下载资源列表及剩余缺集,键格式为 no_exists[source:id]
"""
no_exists_was_none = no_exists is None
if no_exists is None:
no_exists = {}
# 已下载的项目
downloaded_list: List[Context] = []
custom_word_list = custom_words.splitlines() if custom_words else None
-7
View File
@@ -9,7 +9,6 @@ if TYPE_CHECKING:
"""向类型检查器声明同一 Facade 上的跨 owner 方法。"""
_active_download_failure_fingerprints: Callable[..., Any]
_build_subscription_download_request: Callable[..., Any]
_append_no_exists: Callable[..., None]
_build_download_notification: Callable[..., Any]
_build_download_failure_fingerprint: Callable[..., Any]
@@ -23,20 +22,14 @@ if TYPE_CHECKING:
_is_job_active: Callable[..., bool]
_is_subscribe_source: Callable[..., bool]
_log_download_failure_cooldown: Callable[..., None]
_legacy_subscription_download_hash: Callable[..., Any]
_history_matches_subscription: Callable[..., bool]
_media_identity_keys: Callable[..., set[str]]
_matches_media_identity: Callable[..., bool]
_prepare_batch_download_contexts: Callable[..., Any]
_claim_subscription_download: Callable[..., Any]
_record_download_failure: Callable[..., Any]
_resolve_media_download_dir: Callable[..., Any]
_settle_download_success: Callable[..., None]
_submit_download_added_task: Callable[..., None]
_torrent_resource_key: Callable[..., str]
_subscription_download_cancelled: Callable[..., bool]
_subscription_download_repository: Callable[..., Any]
_subscription_download_retry_at: Callable[..., str]
_validate_music_album_resource: Callable[..., Any]
batch_download: Callable[..., Any]
download_single: Callable[..., Any]
-2
View File
@@ -3,7 +3,6 @@
from collections.abc import Callable
from typing import Any, TypeVar
from app.chain.download.admission import DownloadAdmissionOwner
from app.chain.download.batch import DownloadBatchOwner
from app.chain.download.existence import DownloadExistenceOwner
from app.chain.download.failure import DownloadFailureOwner
@@ -30,7 +29,6 @@ class DownloadChain(
DownloadSubtitleOwner,
DownloadSelectionOwner,
DownloadFailureOwner,
DownloadAdmissionOwner,
DownloadSubmissionOwner,
DownloadBatchOwner,
DownloadExistenceOwner,
+11 -103
View File
@@ -10,10 +10,7 @@ from urllib.parse import urlencode, urljoin, urlparse
from app.application.configuration import get_chain_runtime_config_snapshot
from app.application.directory import validate_download_save_path
from app.application.download.admission import (
DownloadReconciliationRequired,
SubscriptionDownloadGovernance,
)
from app.application.download.admission import SubscriptionDownloadGovernance
from app.application.torrent.download import TorrentHelper
from app.chain.download.contract import _DownloadOwnerBase
from app.chain.download.ports import (
@@ -284,6 +281,13 @@ class _DownloadResourceOwner(_DownloadOwnerBase):
class DownloadSubmissionOwner(_DownloadResourceOwner):
"""单任务下载准备、提交与结算 owner。"""
@staticmethod
def _subscription_download_cancelled(
governance: Optional[SubscriptionDownloadGovernance],
) -> bool:
"""在下载器副作用前读取订阅执行上下文的停止信号。"""
return bool(governance and governance.cancelled and governance.cancelled())
def download_single(self, context: Context,
torrent_file: Optional[Path] = None,
torrent_content: Optional[Union[str, bytes]] = None,
@@ -478,37 +482,11 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
custom_words: Optional[str],
governance: Optional[SubscriptionDownloadGovernance],
) -> tuple[Optional[str], Optional[str]]:
"""认领幂等提交权,调用下载器并分派成功或拒绝结算。"""
"""在可取消边界后调用下载器并分派成功或拒绝结算。"""
if self._subscription_download_cancelled(governance):
return None, "订阅下载在提交前已取消"
admission, duplicate_hash = self._claim_subscription_download(
context=context,
episodes=episodes,
governance=governance,
downloader=downloader or prepared.site_downloader,
download_uri=prepared.download_uri,
)
if duplicate_hash:
if governance and governance.mark_started:
governance.mark_started()
logger.info(f"{prepared.torrent.title} 已由重叠订阅入口提交,复用任务 {duplicate_hash}")
return duplicate_hash, "下载任务已由重叠入口提交"
if admission is not None and not admission.acquired:
return None, (
f"订阅下载提交当前为 {admission.snapshot.state}"
f"最早可重试:{admission.snapshot.available_at or '待下一轮'}"
)
attempt_token = admission.snapshot.attempt_token if admission is not None else None
admission_key = admission.snapshot.idempotency_key if admission is not None else None
if admission is not None and self._subscription_download_cancelled(governance):
self._subscription_download_repository().mark_cancelled(
idempotency_key=admission.snapshot.idempotency_key,
attempt_token=admission.snapshot.attempt_token or "",
)
return None, "订阅下载在下载器调用前已取消"
if governance and governance.mark_started:
governance.mark_started()
try:
result = self.download(
content=prepared.torrent_content,
cookie=prepared.torrent.site_cookie,
@@ -518,17 +496,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
label=label,
downloader=downloader or prepared.site_downloader,
)
except Exception as err:
if admission_key and attempt_token:
self._subscription_download_repository().mark_reconcile_required(
idempotency_key=admission_key,
attempt_token=attempt_token,
error=f"下载器调用异常:{str(err)}",
)
raise DownloadReconciliationRequired(
f"{prepared.torrent.title} 下载器结果不确定,已冻结自动重试"
) from err
raise
actual_downloader, download_hash, layout, error_msg = (
result if result else (None, None, None, "未找到下载器")
)
@@ -545,8 +512,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
actual_downloader=actual_downloader,
download_hash=download_hash,
layout=layout,
admission_key=admission_key,
attempt_token=attempt_token,
)
else:
self._record_rejected_download(
@@ -559,8 +524,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
userid=userid,
actual_downloader=actual_downloader,
error_msg=error_msg,
admission_key=admission_key,
attempt_token=attempt_token,
)
return download_hash, error_msg
@@ -578,22 +541,8 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
actual_downloader: Optional[str],
download_hash: str,
layout: Optional[str],
admission_key: Optional[str],
attempt_token: Optional[str],
) -> None:
"""持久化下载器接受事实,执行本地结算并确认成功终态"""
if admission_key and attempt_token:
accepted = self._subscription_download_repository().mark_accepted(
idempotency_key=admission_key,
attempt_token=attempt_token,
downloader=actual_downloader,
download_hash=download_hash,
)
if not accepted:
raise DownloadReconciliationRequired(
f"{prepared.torrent.title} 已被下载器接受,但本地接受状态写入失败"
)
try:
"""按普通下载合同结算下载器明确返回的成功结果"""
self._settle_download_success(
context=context,
media=prepared.media,
@@ -614,35 +563,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
torrent_content=prepared.torrent_content,
custom_words=custom_words,
)
except Exception as err:
if admission_key and attempt_token:
self._subscription_download_repository().mark_reconcile_required(
idempotency_key=admission_key,
attempt_token=attempt_token,
error=f"下载器已接受但本地结算失败:{str(err)}",
downloader=actual_downloader,
download_hash=download_hash,
)
raise DownloadReconciliationRequired(
f"{prepared.torrent.title} 已被下载器接受但本地结算失败,已转待对账"
) from err
raise
if admission_key and attempt_token:
succeeded = self._subscription_download_repository().mark_succeeded(
idempotency_key=admission_key,
attempt_token=attempt_token,
)
if not succeeded:
self._subscription_download_repository().mark_reconcile_required(
idempotency_key=admission_key,
attempt_token=attempt_token,
error="本地结算完成但幂等成功终态写入失败",
downloader=actual_downloader,
download_hash=download_hash,
)
raise DownloadReconciliationRequired(
f"{prepared.torrent.title} 本地结算完成但提交终态未确认,已转待对账"
)
def _record_rejected_download(
self,
@@ -656,20 +576,8 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
userid: Union[str, int, None],
actual_downloader: Optional[str],
error_msg: str,
admission_key: Optional[str],
attempt_token: Optional[str],
) -> None:
"""记录无外部副作用的明确拒绝并通知原调用渠道。"""
if admission_key and attempt_token:
self._subscription_download_repository().mark_retryable(
idempotency_key=admission_key,
attempt_token=attempt_token,
available_at=self._subscription_download_retry_at(
error_msg,
self._download_failure_ttl(error_msg),
),
error=error_msg,
)
"""按普通失败合同记录下载器明确拒绝并通知原调用渠道。"""
logger.error(
f"{prepared.media.title_year} 添加下载任务失败:"
f"{prepared.torrent.title} - {prepared.torrent.enclosure}{error_msg}"
+1 -1
View File
@@ -52,7 +52,7 @@ class SearchChain(ChainBase):
self._subscription_site_budget_failure_lock = threading.Lock()
def record_subscription_site_budget_failure(self, error: str) -> None:
"""线程安全地记录一个未执行站点,供订阅任务暴露聚合失败。"""
"""线程安全地记录一个站点执行失败,供订阅任务暴露聚合失败。"""
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
if lock is None:
return
+8 -1
View File
@@ -220,7 +220,7 @@ class _SearchProviderSyncOwner(_SearchOwnerBase):
return []
with capture_site_search_observation() as observation:
try:
return _search_site_page(
result = _search_site_page(
self,
site=site,
keyword=keyword,
@@ -234,6 +234,13 @@ class _SearchProviderSyncOwner(_SearchOwnerBase):
error=str(error),
)
raise
else:
if observation.attempted and observation.outcome not in {"success", "skipped"}:
failure = observation.error or observation.outcome
self.record_subscription_site_budget_failure(
f"站点 {site.get('name') or site_id} 搜索失败:{failure}"
)
return result
finally:
try:
budget.finish(claim, observation)
+4 -3
View File
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
class _SubscribeOwnerHost:
"""声明 SubscribeChain 组合后向各 owner 提供的属性和兄弟职责。"""
_LOCK_TIMOUT: int
_rlock: Any
_SUBSCRIPTION_EXECUTION_TTL: int
_match_lock: Any
_search_queue_lock: Any
_subscription_execution_admission: Any
download_history_repository: Any
eventmanager: Any
messagehelper: Any
@@ -98,7 +100,6 @@ if TYPE_CHECKING:
get_subscribed_sites: Callable[..., Any]
has_music_subscribe: Callable[..., Any]
match: Callable[..., Any]
match_batch: Callable[..., Any]
media_exists: Callable[..., Any]
media_files: Callable[..., Any]
obtain_images: Callable[..., Any]
+4 -2
View File
@@ -4,6 +4,7 @@ import threading
from typing import Any
from app.application.messaging.subscribe import SubscribeInteractionHandler
from app.application.subscription.execution import SubscriptionExecutionAdmission
from app.chain._interaction import InteractionChainMixin
from app.chain._music import MusicSubscribeMixin
from app.chain.base import ChainBase
@@ -48,9 +49,10 @@ class SubscribeChain(
"""
_interaction_handler_type = SubscribeInteractionHandler
_rlock = threading.RLock()
_match_lock = threading.Lock()
_search_queue_lock = threading.Lock()
_LOCK_TIMOUT = 3600 * 2
_subscription_execution_admission = SubscriptionExecutionAdmission()
_SUBSCRIPTION_EXECUTION_TTL = 3600 * 2
@classmethod
def _music_media_chain(cls) -> MediaChain:
+333 -157
View File
@@ -1,13 +1,20 @@
"""订阅候选准备、身份复核与资源匹配"""
import copy
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional
from typing import Any, Callable, Dict, List, Literal, Optional
from app.application.configuration import get_configured_system_config
from app.application.subscription.candidates import CandidateBatch, CandidateIndex
from app.application.subscription.contract import build_subscribe_meta, subscribe_media_key
from app.application.subscription.candidates import CandidateIndex
from app.application.subscription.contract import (
SubscriptionSnapshot,
build_subscribe_meta,
subscribe_media_key,
)
from app.application.subscription.execution import SubscriptionExecutionContext
from app.application.subscription.facts import FreshFactLease
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
from app.application.torrent.download import TorrentHelper
from app.chain.media import MediaChain
from app.chain.subscribe.contract import _SubscribeOwnerBase
@@ -28,10 +35,176 @@ from app.schemas.types import (
SystemConfigKey,
)
_MatchOutcome = Literal["completed", "skipped", "failed"]
@dataclass(slots=True)
class _MatchExecutionSummary:
"""保存一次 Match 批次的实际订阅执行结果。"""
total: int
completed: int = 0
skipped: int = 0
failed: int = 0
@property
def finished(self) -> int:
"""返回已经收口的订阅数量。"""
return self.completed + self.skipped + self.failed
def record(self, outcome: Optional[str]) -> None:
"""将单订阅结果归入稳定的完成、跳过或失败计数。"""
if outcome == "completed":
self.completed += 1
elif outcome == "skipped":
self.skipped += 1
else:
self.failed += 1
def as_data(self, current: Optional[int] = None) -> dict[str, int]:
"""构造供进度消费者读取的实际计数。"""
data = {
"total": self.total,
"finished": self.finished,
"completed": self.completed,
"skipped": self.skipped,
"failed": self.failed,
}
if current is not None:
data["current"] = current
return data
def _report_match_progress(
progress_callback: Optional[Callable[..., None]],
summary: _MatchExecutionSummary,
*,
value: float,
text: str,
current: Optional[int] = None,
) -> None:
"""发布 Match 进度,并始终携带真实的完成、跳过和失败计数。"""
if progress_callback:
progress_callback(
value=value,
text=text,
data=summary.as_data(current=current),
)
def _is_title_match_allowed(
mediainfo: MediaInfo,
torrent_meta: MetaBase,
torrent_info: TorrentInfo,
) -> bool:
"""判断资源是否可以通过标题兜底匹配订阅目标。"""
if not TorrentHelper.match_torrent(
mediainfo=mediainfo,
torrent_meta=torrent_meta,
torrent=torrent_info,
):
return False
if not TorrentHelper.requires_identity_disambiguation(
mediainfo=mediainfo,
torrent_meta=torrent_meta,
):
return True
logger.info(
f"{torrent_info.site_name} - {torrent_info.title} 仅通过无年份别名命中且候选媒体身份无法确认,已跳过"
)
return False
def _get_media_id_match_source(mediainfo: Optional[MediaInfo]) -> str:
"""返回候选自身识别命中的明确媒体 ID 类型。"""
media_source, media_id = resolve_media_identity(media=mediainfo)
if media_source and media_id:
return str(media_source)
return "unknown"
def _reconcile_candidate_media(
target_mediainfo: MediaInfo,
candidate_mediainfo: MediaInfo,
torrent_meta: MetaBase,
torrent_info: TorrentInfo,
context: Context,
) -> Optional[MediaInfo]:
"""
在推断媒体 ID 冲突时以订阅目标对候选标题做严格复核
标题携带明确媒体身份或目标缺少同来源 ID 时保持严格拒绝只有普通
标题推断产生冲突且标题别名类型和年份复核通过时才回填订阅目标
"""
candidate_source, candidate_id = resolve_media_identity(media=candidate_mediainfo)
target_source, target_id = resolve_media_identity(media=target_mediainfo)
conflicts = []
if candidate_source and candidate_id and (candidate_source != target_source or candidate_id != target_id):
conflicts.append(
(
str(candidate_source),
candidate_id,
f"{target_source}:{target_id}" if target_source and target_id else None,
)
)
if not conflicts:
return candidate_mediainfo
conflict_text = "".join(
f"{source} {candidate_id} != {target_id if target_id is not None else '缺失'}"
for source, candidate_id, target_id in conflicts
)
if any(target_id is None for _, _, target_id in conflicts):
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID与订阅目标不可比较:{conflict_text}"
)
return None
explicit_source, explicit_id = resolve_media_identity(media=torrent_meta)
if explicit_source and explicit_id:
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 标题含明确媒体ID "
f"{explicit_source}:{explicit_id},保持严格校验:{conflict_text}"
)
return None
if not TorrentHelper.match_torrent(
mediainfo=target_mediainfo,
torrent_meta=torrent_meta,
torrent=torrent_info,
):
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且标题复核失败:{conflict_text}"
)
return None
evidence_matched, evidence = TorrentHelper.match_same_work_evidence(
target_mediainfo=target_mediainfo,
candidate_mediainfo=candidate_mediainfo,
torrent_meta=torrent_meta,
)
if not evidence_matched:
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且缺少同作品证据:"
f"{evidence}{conflict_text}"
)
return None
context.media_info = target_mediainfo
context.match_source = "title"
context.candidate_recognized = False
context.media_info_is_target = True
logger.debug(
f"{target_mediainfo.title_year} 候选媒体ID冲突({conflict_text}),"
f"经标题或别名及{evidence}复核匹配到订阅目标:"
f"{torrent_info.site_name} - {torrent_info.title}"
)
return target_mediainfo
def _prepare_subscription_match(
owner: Any,
subscribe: Any,
subscribe: SubscriptionSnapshot,
candidate_index: CandidateIndex,
fresh_fact_lease: FreshFactLease,
) -> Optional[
@@ -109,7 +282,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
if re_mediainfo:
re_mediainfo.clear()
context.media_info = re_mediainfo
context.match_source = self._SubscribeChain__get_media_id_match_source(re_mediainfo)
context.match_source = _get_media_id_match_source(re_mediainfo)
context.candidate_recognized = bool(resolve_media_identity(media=re_mediainfo)[1])
context.media_info_is_target = False
context.media_recognize_fail_count = 0
@@ -142,23 +315,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
return
if progress_callback:
progress_callback(value=0, text="正在预处理订阅资源 ...")
return self._execute_match(
return self._run_match(
torrents=torrents,
progress_callback=progress_callback,
)
def match_batch(
self,
batch: CandidateBatch,
progress_callback: Optional[Callable[..., None]] = None,
) -> None:
"""消费包含完整缓存与本轮增量边界的订阅候选批次。"""
return self._execute_match(
torrents=batch.candidates,
progress_callback=progress_callback,
)
def _execute_match(
def _run_match(
self,
torrents: Dict[str, List[Context]],
progress_callback: Optional[Callable[..., None]] = None,
@@ -190,37 +352,151 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
# 所有订阅
subscribes = self.subscription_repository.list(self.get_states_for_search("R"))
total_num = len(subscribes)
summary = _MatchExecutionSummary(total=total_num)
if progress_callback:
progress_callback(
_report_match_progress(
progress_callback,
summary,
value=20,
text=f"资源预处理完成,开始匹配 {total_num} 个订阅 ...",
data={"total": total_num, "finished": 0},
)
try:
for index, subscribe in enumerate(subscribes, start=1):
for index, listed_subscribe in enumerate(subscribes, start=1):
if runtime_stop_state.is_system_stopped:
break
if progress_callback:
progress_callback(
_report_match_progress(
progress_callback,
summary,
value=20 + ((index - 1) / total_num * 80 if total_num else 80),
text=(f"正在匹配订阅({index}/{total_num}{subscribe.name} ..."),
data={
"total": total_num,
"finished": index - 1,
"current": subscribe.id,
},
text=(f"正在匹配订阅({index}/{total_num}{listed_subscribe.name} ..."),
current=listed_subscribe.id,
)
outcome: Optional[str] = "skipped"
lease = self._subscription_execution_admission.try_acquire(
subscription_id=listed_subscribe.id,
operation="match",
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
)
if lease is None:
logger.info(f"订阅 {listed_subscribe.name} 正在由其他通道处理,本轮匹配已跳过")
else:
current_subscribe = None
execution_context = SubscriptionExecutionContext(
lease=lease,
admission=self._subscription_execution_admission,
cancel_requested=lambda: runtime_stop_state.is_system_stopped,
)
try:
current_subscribe = self.subscription_repository.get(listed_subscribe.id)
if current_subscribe is None:
logger.info(f"订阅 {listed_subscribe.id} 已不存在,本轮匹配跳过")
elif current_subscribe.state not in {"R", "P"}:
logger.info(
f"订阅 {current_subscribe.name} 当前状态为 {current_subscribe.state},本轮匹配跳过"
)
elif execution_context.should_stop():
logger.info(f"订阅 {current_subscribe.name} 已取消或超时,本轮匹配跳过")
else:
outcome = self._match_subscription(
subscribe=current_subscribe,
processed_torrents=processed_torrents,
candidate_index=candidate_index,
fresh_fact_lease=fresh_fact_lease,
execution_context=execution_context,
)
except SubscriptionSearchCancelled as err:
outcome = "skipped"
logger.info(f"订阅 {listed_subscribe.name} 匹配已取消:{str(err)}")
except Exception as err:
outcome = "failed"
subscribe_name = (
current_subscribe.name if current_subscribe is not None else listed_subscribe.name
)
logger.error(f"订阅 {subscribe_name} 匹配失败:{str(err)}", exc_info=True)
finally:
self._subscription_execution_admission.release(lease)
summary.record(outcome)
_report_match_progress(
progress_callback,
summary,
value=20 + (summary.finished / total_num * 80 if total_num else 80),
text=(f"订阅匹配({index}/{total_num})处理完成"),
)
finally:
processed_torrents.clear()
del processed_torrents
subscribes.clear()
del subscribes
if progress_callback:
final_text = "订阅资源匹配完成"
if summary.finished < summary.total:
final_text = "订阅资源匹配已停止,部分订阅未执行"
elif summary.failed:
final_text = "订阅资源匹配完成,部分订阅失败"
elif summary.skipped:
final_text = "订阅资源匹配完成,部分订阅跳过"
_report_match_progress(
progress_callback,
summary,
value=100,
text=final_text,
)
finally:
if lock_acquired:
self._match_lock.release()
logger.debug(f"match Lock released at {datetime.now()}")
def _match_subscription(
self,
*,
subscribe: SubscriptionSnapshot,
processed_torrents: Dict[str, List[Context]],
candidate_index: CandidateIndex,
fresh_fact_lease: FreshFactLease,
execution_context: SubscriptionExecutionContext,
) -> _MatchOutcome:
"""在订阅准入边界调用单订阅 Match 执行。"""
return self._execute_match(
subscribe=subscribe,
processed_torrents=processed_torrents,
candidate_index=candidate_index,
fresh_fact_lease=fresh_fact_lease,
execution_context=execution_context,
)
def _execute_match(
self,
*,
subscribe: SubscriptionSnapshot,
processed_torrents: Dict[str, List[Context]],
candidate_index: CandidateIndex,
fresh_fact_lease: FreshFactLease,
execution_context: SubscriptionExecutionContext,
) -> _MatchOutcome:
"""在单条订阅准入期间完成匹配、下载和完成判定。"""
if execution_context.should_stop():
return "skipped"
logger.info(f"开始匹配订阅,标题:{subscribe.name} ...")
if subscribe.type == MediaType.MUSIC.value:
music_contexts = [context for contexts in processed_torrents.values() for context in contexts]
self._match_music_subscribe(subscribe, music_contexts)
continue
execution_context.report_phase("matching")
if not execution_context.should_stop():
self._match_music_subscribe(
subscribe,
music_contexts,
execution_context=execution_context,
)
return "completed"
return "skipped"
mediakey = subscribe_media_key(subscribe)
prepared = _prepare_subscription_match(
self, subscribe, candidate_index, fresh_fact_lease
)
if prepared is None:
continue
return "completed"
if execution_context.should_stop():
return "skipped"
meta, mediainfo, routed_torrents, domains, sub_sites = prepared
# 如果媒体已存在或已下载完毕,跳过当前订阅处理
@@ -228,7 +504,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
subscribe=subscribe, meta=meta, mediainfo=mediainfo, mediakey=mediakey
)
if exist_flag:
continue
return "completed"
# 匹配前聚合多来源别名;清理大字段时保留匹配所需的标题合集。
mediainfo = MediaChain().supplement_media_info(mediainfo) or mediainfo
@@ -247,14 +523,15 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
torrenthelper = TorrentHelper()
systemconfig = get_configured_system_config()
wordsmatcher = WordsMatcher()
execution_context.report_phase("matching")
for domain, contexts in routed_torrents.items():
if runtime_stop_state.is_system_stopped:
if runtime_stop_state.is_system_stopped or execution_context.should_stop():
break
if domains and domain not in domains:
continue
logger.debug(f"开始匹配站点:{domain},共缓存了 {len(contexts)} 个种子...")
for context in contexts:
if runtime_stop_state.is_system_stopped:
if runtime_stop_state.is_system_stopped or execution_context.should_stop():
break
# 提取信息
_context = copy.copy(context)
@@ -296,7 +573,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
torrent_mediainfo.clear()
# 更新种子缓存
_context.media_info = torrent_mediainfo
_context.match_source = self._SubscribeChain__get_media_id_match_source(
_context.match_source = _get_media_id_match_source(
torrent_mediainfo
)
_context.candidate_recognized = bool(
@@ -309,7 +586,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 重新识别失败,尝试通过标题匹配..."
)
if not self._SubscribeChain__is_title_match_allowed(
if not _is_title_match_allowed(
mediainfo=mediainfo,
torrent_meta=torrent_meta,
torrent_info=torrent_info,
@@ -346,7 +623,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
_context.media_info_is_target = True
match_label = "标题复核"
elif match_source == "unknown":
_context.match_source = self._SubscribeChain__get_media_id_match_source(
_context.match_source = _get_media_id_match_source(
torrent_mediainfo
)
_context.candidate_recognized = True
@@ -470,13 +747,19 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
torrent_mediainfo.episode_group = subscribe.episode_group
_match_context.append(_context)
if execution_context.should_stop():
return "skipped"
if not _match_context:
# 未匹配到资源
logger.info(f"{mediainfo.title_year} 未匹配到符合条件的资源")
self.finish_subscribe_or_not(
subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists
)
continue
return "completed"
if execution_context.should_stop():
return "skipped"
# 开始批量择优下载
logger.info(f"{mediainfo.title_year} 匹配完成,共匹配到{len(_match_context)}个资源")
@@ -489,6 +772,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
save_path=subscribe.save_path,
downloader=subscribe.downloader,
source=self.get_subscribe_source_keyword(subscribe),
execution_context=execution_context,
)
# 同步外部修改,更新订阅信息
@@ -503,53 +787,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
downloads=downloads,
lefts=lefts,
)
finally:
processed_torrents.clear()
del processed_torrents
subscribes.clear()
del subscribes
if progress_callback:
progress_callback(
value=100,
text="订阅资源匹配完成",
data={"total": total_num, "finished": total_num},
)
finally:
if lock_acquired:
self._rlock.release()
logger.debug(f"match Lock released at {datetime.now()}")
@staticmethod
def _SubscribeChain__get_media_id_match_source(mediainfo: Optional[MediaInfo]) -> str:
"""
返回候选自身识别命中的明确媒体 ID 类型
"""
media_source, media_id = resolve_media_identity(media=mediainfo)
if media_source and media_id:
return str(media_source)
return "unknown"
@staticmethod
def _SubscribeChain__is_title_match_allowed(
mediainfo: MediaInfo,
torrent_meta: MetaBase,
torrent_info: TorrentInfo,
) -> bool:
"""判断资源是否可以通过标题兜底匹配订阅目标。"""
if not TorrentHelper.match_torrent(
mediainfo=mediainfo,
torrent_meta=torrent_meta,
torrent=torrent_info,
):
return False
if not TorrentHelper.requires_identity_disambiguation(
mediainfo=mediainfo,
torrent_meta=torrent_meta,
):
return True
logger.info(
f"{torrent_info.site_name} - {torrent_info.title} 仅通过无年份别名命中且候选媒体身份无法确认,已跳过"
)
return False
return "completed"
@staticmethod
def _SubscribeChain__reconcile_candidate_media(
@@ -559,80 +797,18 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
torrent_info: TorrentInfo,
context: Context,
) -> Optional[MediaInfo]:
"""
在推断媒体 ID 冲突时以订阅目标对候选标题做严格复核
标题携带明确媒体身份或目标缺少同来源 ID 时保持严格拒绝只有普通
标题推断产生冲突且标题别名类型和年份复核通过时才回填订阅目标
:param target_mediainfo: 订阅目标媒体信息
:param candidate_mediainfo: RSS 或首页候选的识别结果
:param torrent_meta: 候选标题解析信息
:param torrent_info: 候选种子信息
:param context: 当前订阅使用的候选上下文副本
:return: 可继续匹配的媒体信息拒绝时返回 None
"""
candidate_source, candidate_id = resolve_media_identity(media=candidate_mediainfo)
target_source, target_id = resolve_media_identity(media=target_mediainfo)
conflicts = []
if candidate_source and candidate_id and (candidate_source != target_source or candidate_id != target_id):
conflicts.append(
(
str(candidate_source),
candidate_id,
f"{target_source}:{target_id}" if target_source and target_id else None,
)
)
if not conflicts:
return candidate_mediainfo
conflict_text = "".join(
f"{source} {candidate_id} != {target_id if target_id is not None else '缺失'}"
for source, candidate_id, target_id in conflicts
)
if any(target_id is None for _, _, target_id in conflicts):
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID与订阅目标不可比较:{conflict_text}"
)
return None
explicit_source, explicit_id = resolve_media_identity(media=torrent_meta)
if explicit_source and explicit_id:
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 标题含明确媒体ID "
f"{explicit_source}:{explicit_id},保持严格校验:{conflict_text}"
)
return None
if not TorrentHelper.match_torrent(
mediainfo=target_mediainfo,
torrent_meta=torrent_meta,
torrent=torrent_info,
):
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且标题复核失败:{conflict_text}"
)
return None
evidence_matched, evidence = TorrentHelper.match_same_work_evidence(
"""保留组合宿主的稳定私有调用入口。"""
return _reconcile_candidate_media(
target_mediainfo=target_mediainfo,
candidate_mediainfo=candidate_mediainfo,
torrent_meta=torrent_meta,
torrent_info=torrent_info,
context=context,
)
if not evidence_matched:
logger.debug(
f"{torrent_info.site_name} - {torrent_info.title} 候选媒体ID冲突且缺少同作品证据:"
f"{evidence}{conflict_text}"
)
return None
context.media_info = target_mediainfo
context.match_source = "title"
context.candidate_recognized = False
context.media_info_is_target = True
logger.debug(
f"{target_mediainfo.title_year} 候选媒体ID冲突({conflict_text}),"
f"经标题或别名及{evidence}复核匹配到订阅目标:"
f"{torrent_info.site_name} - {torrent_info.title}"
)
return target_mediainfo
@staticmethod
def _SubscribeChain__get_media_id_match_source(
mediainfo: Optional[MediaInfo],
) -> str:
"""保留组合宿主解析媒体 ID 匹配来源的稳定入口。"""
return _get_media_id_match_source(mediainfo)
+4 -22
View File
@@ -8,6 +8,7 @@ from app.application.subscription import priority as _priority
from app.application.subscription.contract import (
SubscriptionSnapshot,
)
from app.application.subscription.execution import SubscriptionExecutionContext
from app.chain.download import DownloadChain
from app.chain.subscribe.contract import _SubscribeOwnerBase
from app.domain.context import (
@@ -281,6 +282,7 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
save_path: Optional[str] = None,
downloader: Optional[str] = None,
source: Optional[str] = None,
execution_context: Optional[SubscriptionExecutionContext] = None,
) -> Tuple[List[Context], Dict[Union[int, str], Dict[int, _SchemaNotExistMediaInfo]]]:
"""
TV 分集洗版先尝试覆盖目标范围的全集资源失败后回退到按集下载
@@ -327,17 +329,8 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
downloader = current.downloader
source = self.get_subscribe_source_keyword(current)
governance = SubscriptionDownloadGovernance(
subscription_id=current.id,
mode=self._SubscribeChain__download_governance_mode(current),
task_id=cast(Optional[str], getattr(self, "_subscription_download_task_id", None)),
cancelled=cast(
Optional[Callable[[], bool]],
getattr(self, "_subscription_download_cancelled", None),
),
mark_started=cast(
Optional[Callable[[], None]],
getattr(self, "_subscription_download_mark_started", None),
),
cancelled=execution_context.should_stop if execution_context else None,
mark_started=execution_context.mark_download_started if execution_context else None,
)
full_pack_no_exists = self._SubscribeChain__build_full_pack_first_no_exists(
subscribe=subscribe, mediakey=mediakey
@@ -465,17 +458,6 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
accepted.append(context)
return accepted
@staticmethod
def _SubscribeChain__download_governance_mode(subscribe: SubscriptionSnapshot) -> str:
"""把订阅模式归一为稳定幂等键组成部分。"""
if not subscribe.best_version:
return "normal"
if subscribe.type == MediaType.TV.value and subscribe.best_version_full:
return "best_version_full"
if subscribe.type == MediaType.TV.value:
return "best_version_episode"
return "best_version"
@classmethod
def _is_episode_range_covered(cls, meta: MetaBase, subscribe: SubscriptionSnapshot) -> bool:
"""
+3 -11
View File
@@ -5,7 +5,6 @@ from datetime import datetime
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
from app.application.subscription import priority as _priority
from app.application.subscription.candidates import CandidateBatch
from app.application.subscription.contract import (
SubscriptionSnapshot,
build_subscribe_meta,
@@ -78,21 +77,14 @@ class SubscribeRefreshOwner(SubscribeMetadataOwner):
)
torrents_chain = TorrentsChain()
candidate_batch = torrents_chain.refresh_batch(
candidates = torrents_chain.refresh(
sites=sites,
progress_callback=_update_refresh_progress if progress_callback else None,
# 存在音乐订阅时额外抓取站点音乐专用入口,音乐不一定在默认种子首页
include_music=self.has_music_subscribe(),
)
if not isinstance(candidate_batch, CandidateBatch):
legacy_candidates = torrents_chain.refresh(
sites=sites,
progress_callback=_update_refresh_progress if progress_callback else None,
include_music=self.has_music_subscribe(),
)
candidate_batch = CandidateBatch.from_legacy(legacy_candidates or {}, source="refresh")
self.match_batch(
candidate_batch,
self.match(
candidates,
progress_callback=_update_match_progress if progress_callback else None,
)
if progress_callback:
+424 -168
View File
@@ -14,7 +14,12 @@ from app.application.subscription.contract import (
build_subscribe_meta,
subscribe_media_key,
)
from app.application.subscription.execution import SearchBatchSnapshot, SubscriptionSearchRepository
from app.application.subscription.execution import (
SearchBatchSnapshot,
SearchTaskSnapshot,
SubscriptionExecutionContext,
SubscriptionSearchRepository,
)
from app.application.subscription.query import SubscriptionQueryService
from app.application.subscription.sitebudget import (
SubscriptionSearchCancelled,
@@ -36,6 +41,198 @@ from app.schemas.types import (
)
def _ensure_execution_active(
execution_context: Optional[SubscriptionExecutionContext],
) -> None:
"""在可安全停止的边界区分用户取消与执行超时。"""
if execution_context is None:
return
if execution_context.is_cancel_requested():
raise SubscriptionSearchCancelled("订阅搜索已取消")
if execution_context.is_expired():
raise TimeoutError("订阅执行已超过协作截止时间")
def _update_search_task_phase(
queue: SubscriptionSearchRepository,
task_id: str,
lease_token: str,
phase: str,
current_site_id: Optional[int] = None,
) -> None:
"""以当前任务租约持久化业务阶段,过期执行者不得覆盖新状态。"""
queue.update_task_phase(
task_id=task_id,
lease_token=lease_token,
phase=phase,
current_site_id=current_site_id,
)
def _search_source_and_priority(
*,
sid: Optional[int],
sids: Optional[tuple[int, ...]],
state: Optional[str],
manual: Optional[bool],
) -> tuple[str, int]:
"""把兼容入口归一为持久来源和公平队列优先级。"""
if manual:
return "manual", 100
if sid or sids is not None:
return "targeted", 80
if state in {"R", "P"}:
return "fallback", 10
return "new", 50
def _search_task_available_at(
source: str,
subscription_ids: tuple[int, ...],
*,
now: Optional[datetime] = None,
) -> dict[int, str]:
"""把兜底搜索的随机节奏持久化为逐订阅到期时间。"""
ordered_ids = tuple(dict.fromkeys(subscription_ids))
if not ordered_ids:
return {}
cursor = now or datetime.now(timezone.utc)
if source == "fallback":
cursor += timedelta(seconds=random.randint(0, 60))
available_at: dict[int, str] = {}
for position, subscription_id in enumerate(ordered_ids):
if source == "fallback" and position:
cursor += timedelta(seconds=random.randint(60, 300))
available_at[subscription_id] = cursor.isoformat(timespec="seconds")
return available_at
def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
"""把批次聚合终态转为兼容进度文案。"""
if batch is None:
return "订阅搜索任务已提交"
if batch.state == "failed":
return "订阅搜索完成,部分任务失败"
if batch.state == "cancelled":
return "订阅搜索已取消"
if batch.state == "skipped":
return "订阅搜索完成,部分任务本轮已跳过"
if batch.state in {"queued", "running", "cancelling"}:
return "订阅搜索任务已排队"
if batch.skipped_count:
return "订阅搜索完成,部分任务本轮已跳过"
return "订阅搜索完成"
def _inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]:
"""返回兼容搜索的真实终态文案与计数。"""
text = (
"订阅搜索完成"
if finished == total
else "订阅搜索结束,部分订阅本轮未执行或未完成"
)
return text, {"total": total, "finished": finished}
def _batch_finished_count(
batch: Optional[SearchBatchSnapshot],
fallback: int,
) -> int:
"""返回批次所有终态任务数;批次暂不可读时使用本轮实际完成数。"""
if batch is None:
return fallback
return (
batch.finished_count
+ batch.failed_count
+ batch.cancelled_count
+ batch.skipped_count
)
def _skip_search_task(
queue: SubscriptionSearchRepository,
task: SearchTaskSnapshot,
reason: str,
) -> bool:
"""以 skipped 终态收口未执行任务,并保留可见原因。"""
if task.lease_token is None:
return False
return queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
state="skipped",
error=reason,
)
def _release_cancelled_or_stopped_search_task(
queue: SubscriptionSearchRepository,
task_id: str,
lease_token: str,
system_stopped: bool,
) -> bool:
"""用户取消落终态,系统停机仅释放任务以供重启恢复。"""
return queue.release_task(
task_id=task_id,
lease_token=lease_token,
cancelled=not system_stopped,
)
def _finish_returned_search_task(
*,
queue: SubscriptionSearchRepository,
task_id: str,
lease_token: str,
subscription_id: int,
execution_context: SubscriptionExecutionContext,
system_stopped: bool,
cancel_requested: bool,
) -> Optional[int]:
"""按 TTL、停机和取消边界收口正常返回的搜索任务。"""
download_started = execution_context.download_started
if execution_context.is_expired() and not (system_stopped or cancel_requested):
if download_started:
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="执行截止时间晚于下载提交边界,已按实际结果完成",
)
return subscription_id
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="failed",
error="订阅执行已超过协作截止时间",
)
return None
if system_stopped:
if download_started:
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="停机请求晚于下载提交边界,已按实际结果完成",
)
return subscription_id
queue.release_task(task_id=task_id, lease_token=lease_token)
return None
if cancel_requested:
if download_started:
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="取消请求晚于下载提交边界,已按实际结果完成",
)
return subscription_id
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
return None
queue.finish_task(task_id=task_id, lease_token=lease_token, state="completed")
return subscription_id
class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
"""订阅主动搜索入口与持久队列消费 owner。"""
@@ -64,8 +261,9 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
operation: str,
progress_callback: Optional[Callable[..., None]],
) -> bool:
"""获取订阅任务锁,超时时统一记录并结束本轮进度"""
if self._rlock.acquire(blocking=True, timeout=self._LOCK_TIMOUT):
"""获取 Search 或 Match 通道锁,保持各通道内部串行"""
lock = self._match_lock if operation == "match" else self._search_queue_lock
if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL):
logger.debug(f"{operation} lock acquired at {datetime.now()}")
return True
operation_label = {"search": "搜索", "match": "匹配"}[operation]
@@ -148,7 +346,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
manual: Optional[bool],
progress_callback: Optional[Callable[..., None]],
) -> None:
"""保留未注入持久队列宿主的旧串行锁语义"""
"""在独立 Search 通道内按订阅准入执行兼容搜索"""
lock_acquired = self._acquire_run_lock("search", progress_callback)
if not lock_acquired:
return
@@ -167,28 +365,70 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
for index, subscribe in enumerate(subscribes, start=1):
if runtime_stop_state.is_system_stopped:
break
processed.append(subscribe)
self._report_search_progress(progress_callback, subscribe, index, total)
if self._defer_recent_subscription(subscribe):
continue
self._wait_before_scheduled_search(sid, sids, state, progress_callback)
current = subscribe
lease = self._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="search",
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
)
if lease is None:
logger.info(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过")
continue
execution_context = SubscriptionExecutionContext(
lease=lease,
admission=self._subscription_execution_admission,
cancel_requested=lambda: runtime_stop_state.is_system_stopped,
)
current = None
try:
current = self._process_search_subscription(subscribe, searchchain)
current = self.subscription_repository.get(subscribe.id)
if current is None or current.state == "S":
if current and current.state == "S":
logger.info(f"订阅 {current.name} 已暂停,本轮搜索已跳过")
continue
processed_result = self._process_search_subscription(
current,
searchchain,
execution_context=execution_context,
)
processed.append(processed_result or current)
current = processed_result
except SubscriptionSearchCancelled:
logger.info(f"订阅 {subscribe.name} 搜索已在安全边界取消")
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
finally:
try:
if current and current.state == "N":
self._SubscribeChain__apply_subscribe_update(
current,
{"state": "R"},
scene="search_reset",
)
self._report_search_progress(progress_callback, subscribe, index, total, finished=True)
except Exception as err:
logger.error(
f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}",
exc_info=True,
)
finally:
self._subscription_execution_admission.release(lease)
self._report_search_progress(
progress_callback,
subscribe,
index,
total,
finished=True,
)
self._notify_manual_search(manual, sid, sids, subscribes, processed)
if progress_callback:
progress_callback(value=100, text="订阅搜索完成")
text, data = _inline_search_result(total, len(processed))
progress_callback(value=100, text=text, data=data)
finally:
subscribes.clear()
self._rlock.release()
self._search_queue_lock.release()
logger.debug(f"search Lock released at {datetime.now()}")
def _execute_queued_search(
@@ -203,17 +443,21 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
) -> str:
"""将搜索转为持久任务并在无 Match 长锁的短租约中串行消费。"""
subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state)
source, priority = self._search_source_and_priority(
source, priority = _search_source_and_priority(
sid=sid,
sids=sids,
state=state,
manual=manual,
)
subscription_ids = tuple(subscribe.id for subscribe in subscribes)
enqueued = queue.enqueue(
subscription_ids=tuple(subscribe.id for subscribe in subscribes),
subscription_ids=subscription_ids,
source=source,
priority=priority,
available_at=self._search_batch_available_at(source),
available_at_by_subscription=_search_task_available_at(
source,
subscription_ids,
),
)
total = len(subscribes)
if progress_callback:
@@ -238,11 +482,11 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
batch = queue.get_batch(enqueued.batch.batch_id)
progress_callback(
value=100,
text=self._batch_progress_text(batch),
text=_batch_progress_text(batch),
data={
"batch_id": enqueued.batch.batch_id,
"total": total,
"finished": len(processed_subscribes),
"finished": _batch_finished_count(batch, len(processed_subscribes)),
"coalesced": enqueued.coalesced_count,
},
)
@@ -256,8 +500,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
progress_callback: Optional[Callable[..., None]],
) -> set[int]:
"""有界消费可恢复任务;单任务失败不得阻止后续订阅。"""
queue_lock = getattr(self, "_search_queue_lock", None)
if queue_lock is not None and not queue_lock.acquire(blocking=False):
if not self._search_queue_lock.acquire(blocking=False):
logger.debug("订阅搜索队列已有消费者,本轮仅保留持久任务")
return set()
owner = f"subscribe-search:{uuid4().hex}"
@@ -270,146 +513,182 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
task = queue.claim_next(owner=owner)
if task is None:
break
if not task.lease_token:
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行")
continue
task_id = str(task.task_id)
cancelled = partial(queue.is_cancel_requested, task_id)
if queue.is_cancel_requested(task.task_id):
queue.release_task(
task_id=task.task_id,
lease_token=task.lease_token,
cancelled=True,
subscription_id = self._execute_search_task(
queue=queue,
task=task,
owner=owner,
searchchain=searchchain,
index=index,
limit=limit,
progress_callback=progress_callback,
)
continue
if subscription_id is not None:
processed.add(subscription_id)
finally:
self._search_queue_lock.release()
return processed
def _execute_search_task(
self,
*,
queue: SubscriptionSearchRepository,
task: SearchTaskSnapshot,
owner: str,
searchchain: SearchChain,
index: int,
limit: int,
progress_callback: Optional[Callable[..., None]],
) -> Optional[int]:
"""执行一条已认领任务,返回实际进入搜索处理的订阅 ID。"""
if task.lease_token is None:
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行")
return None
task_id = str(task.task_id)
lease_token = task.lease_token
cancelled = partial(queue.is_cancel_requested, task_id)
stop_state = getattr(self, "stop_state", runtime_stop_state)
if cancelled():
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
return None
subscribe = self.subscription_repository.get(task.subscription_id)
if subscribe is None:
queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
task_id=task_id,
lease_token=lease_token,
state="cancelled",
error="订阅已不存在",
)
continue
return None
self._report_search_progress(progress_callback, subscribe, index, limit)
if self._defer_recent_subscription(subscribe):
queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
state="completed",
_skip_search_task(queue, task, "订阅仍在新增保护期,本轮搜索已跳过")
return None
lease = self._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="search",
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
)
if lease is None:
logger.info(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过")
_skip_search_task(queue, task, "同一订阅正在由其他通道处理,本轮搜索已跳过")
return None
phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token)
execution_context = SubscriptionExecutionContext(
lease=lease,
admission=self._subscription_execution_admission,
task_id=task_id,
cancel_requested=lambda: cancelled() or stop_state.is_system_stopped,
phase_changed=phase_changed,
)
continue
current = subscribe
phase_changed = partial(
self._update_search_task_phase,
queue,
task_id,
task.lease_token,
try:
current = self.subscription_repository.get(task.subscription_id)
if current is None:
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="cancelled",
error="订阅已不存在",
)
return None
if current.state == "S":
_skip_search_task(queue, task, "订阅已暂停,本轮搜索已跳过")
return None
searchchain.configure_subscription_site_budget(
SubscriptionSiteBudget(
repository=queue,
owner=f"{owner}:{task_id}",
cancelled=cancelled,
stop_state=getattr(self, "stop_state", runtime_stop_state),
cancelled=execution_context.should_stop,
stop_state=stop_state,
phase_changed=phase_changed,
)
)
self._subscription_download_task_id = task_id
self._subscription_download_cancelled = cancelled
self._subscription_download_crossed_boundary = False
self._subscription_download_mark_started = self._mark_subscription_download_started
self._subscription_execution_phase = phase_changed
try:
current = self._process_search_subscription(subscribe, searchchain)
if queue.is_cancel_requested(task.task_id):
if self._subscription_download_started_for_task(task.task_id):
queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
state="completed",
error="取消请求晚于下载提交边界,已按实际结果完成",
current = self._process_search_subscription(
current,
searchchain,
execution_context=execution_context,
)
else:
queue.release_task(
task_id=task.task_id,
lease_token=task.lease_token,
cancelled=True,
system_stopped = stop_state.is_system_stopped
cancel_requested = False if system_stopped else cancelled()
return _finish_returned_search_task(
queue=queue,
task_id=task_id,
lease_token=lease_token,
subscription_id=task.subscription_id,
execution_context=execution_context,
system_stopped=system_stopped,
cancel_requested=cancel_requested,
)
else:
queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
state="completed",
)
processed.add(subscribe.id)
except SubscriptionSearchCancelled:
queue.release_task(
task_id=task.task_id,
lease_token=task.lease_token,
cancelled=True,
if execution_context.is_expired() and not execution_context.is_cancel_requested():
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="failed",
error="订阅执行已超过协作截止时间",
)
else:
_release_cancelled_or_stopped_search_task(
queue,
task_id,
lease_token,
stop_state.is_system_stopped,
)
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
queue.finish_task(
task_id=task.task_id,
lease_token=task.lease_token,
task_id=task_id,
lease_token=lease_token,
state="failed",
error=str(err),
)
finally:
delattr(self, "_subscription_download_task_id")
delattr(self, "_subscription_download_cancelled")
delattr(self, "_subscription_download_mark_started")
delattr(self, "_subscription_execution_phase")
self._subscription_download_crossed_boundary = False
searchchain.configure_subscription_site_budget(None)
if current and current.state == "N":
self._cleanup_search_task(
queue=queue,
searchchain=searchchain,
subscribe=subscribe,
current=current,
lease=lease,
progress_callback=progress_callback,
index=index,
limit=limit,
)
return None
def _cleanup_search_task(
self,
*,
queue: SubscriptionSearchRepository,
searchchain: SearchChain,
subscribe: SubscriptionSnapshot,
current: Optional[SubscriptionSnapshot],
lease: Any,
progress_callback: Optional[Callable[..., None]],
index: int,
limit: int,
) -> None:
"""清理站点预算和订阅状态,并在所有异常路径释放 owner。"""
try:
searchchain.configure_subscription_site_budget(None)
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索站点预算清理失败:{str(err)}", exc_info=True)
try:
if current and current.state == "N":
self._SubscribeChain__apply_subscribe_update(
current,
{"state": "R"},
scene="search_reset",
)
except Exception as err:
logger.error(
f"订阅 {current.name} 搜索后状态重置失败:{str(err)}",
exc_info=True,
)
self._report_search_progress(progress_callback, subscribe, index, limit, finished=True)
logger.error(f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}", exc_info=True)
finally:
if queue_lock is not None:
queue_lock.release()
return processed
def _subscription_download_started_for_task(self, task_id: str) -> bool:
"""判断取消是否已晚于下载器副作用边界。"""
if bool(getattr(self, "_subscription_download_crossed_boundary", False)):
return True
repository = getattr(self, "subscription_download_repository", None)
return bool(repository and repository.has_started_for_task(task_id))
def _mark_subscription_download_started(self) -> None:
"""记录当前搜索任务已提交或复用了真实下载结果。"""
self._subscription_download_crossed_boundary = True
phase_changed = getattr(self, "_subscription_execution_phase", None)
if phase_changed:
phase_changed("submitting", None)
@staticmethod
def _update_search_task_phase(
queue: SubscriptionSearchRepository,
task_id: str,
lease_token: str,
phase: str,
current_site_id: Optional[int] = None,
) -> None:
"""以当前任务租约持久化业务阶段,过期执行者不得覆盖新状态。"""
queue.update_task_phase(
task_id=task_id,
lease_token=lease_token,
phase=phase,
current_site_id=current_site_id,
self._subscription_execution_admission.release(lease)
self._report_search_progress(
progress_callback,
subscribe,
index,
limit,
finished=True,
)
def resume_search_queue(
@@ -429,44 +708,6 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
progress_callback=progress_callback,
)
@staticmethod
def _search_source_and_priority(
*,
sid: Optional[int],
sids: Optional[tuple[int, ...]],
state: Optional[str],
manual: Optional[bool],
) -> tuple[str, int]:
"""把兼容入口归一为持久来源和公平队列优先级。"""
if manual:
return "manual", 100
if sid or sids is not None:
return "targeted", 80
if state in {"R", "P"}:
return "fallback", 10
return "new", 50
@staticmethod
def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
"""把批次聚合终态转为兼容进度文案。"""
if batch is None:
return "订阅搜索任务已提交"
if batch.state == "failed":
return "订阅搜索完成,部分任务失败"
if batch.state == "cancelled":
return "订阅搜索已取消"
if batch.state in {"queued", "running", "cancelling"}:
return "订阅搜索任务已排队"
return "订阅搜索完成"
@staticmethod
def _search_batch_available_at(source: str) -> str:
"""为自动兜底批次持久化一次全局启动抖动。"""
delay = random.randint(0, 60) if source == "fallback" else 0
return (
datetime.now(timezone.utc) + timedelta(seconds=delay)
).isoformat(timespec="seconds")
def cancel_search_batch(self, batch_id: str) -> bool:
"""请求取消持久搜索批次;未注入队列时返回失败。"""
queue: Optional[SubscriptionSearchRepository] = getattr(
@@ -518,8 +759,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
state: Optional[str],
progress_callback: Optional[Callable[..., None]],
) -> None:
"""为自动搜索增加随机间隔,手动和定向批次不等待"""
if sid or sids is not None or state not in ["R", "P"]:
"""未使用持久队列时为自动兜底搜索保留逐订阅随机间隔"""
if sid or sids is not None or state not in {"R", "P"}:
return
sleep_time = random.randint(60, 300)
logger.info(f"订阅搜索随机休眠 {sleep_time} 秒 ...")
@@ -531,11 +772,13 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
self,
subscribe: SubscriptionSnapshot,
searchchain: SearchChain,
execution_context: Optional[SubscriptionExecutionContext] = None,
) -> Optional[SubscriptionSnapshot]:
"""处理单个订阅,并返回下载后重新读取的状态快照。"""
_ensure_execution_active(execution_context)
logger.info(f"开始搜索订阅,标题:{subscribe.name} ...")
if subscribe.type == MediaType.MUSIC.value:
self._search_music_subscribe(subscribe)
self._search_music_subscribe(subscribe, execution_context=execution_context)
return subscribe
try:
meta = build_subscribe_meta(subscribe)
@@ -555,6 +798,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
f"媒体来源:{subscribe.media_source},媒体 ID{subscribe.media_id}"
)
return subscribe
_ensure_execution_active(execution_context)
mediakey = subscribe_media_key(subscribe)
exists, no_exists = self.check_and_handle_existing_media(
subscribe=subscribe,
@@ -569,9 +813,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
if subscribe.best_version
else SystemConfigKey.SubscribeFilterRuleGroups
)
phase_changed = getattr(self, "_subscription_execution_phase", None)
if phase_changed:
phase_changed("searching", None)
if execution_context:
execution_context.report_phase("searching")
contexts = searchchain.process(
mediainfo=mediainfo,
keyword=subscribe.keyword,
@@ -583,6 +826,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
filter_params=self.get_params(subscribe),
)
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
_ensure_execution_active(execution_context)
if not contexts:
logger.warning(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
self.finish_subscribe_or_not(
@@ -599,8 +843,9 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
self._raise_site_budget_failures(site_budget_failures)
return subscribe
if phase_changed:
phase_changed("preparing", None)
if execution_context:
execution_context.report_phase("preparing")
_ensure_execution_active(execution_context)
downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first(
contexts=matched,
no_exists=no_exists,
@@ -610,6 +855,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
save_path=subscribe.save_path,
downloader=subscribe.downloader,
source=self.get_subscribe_source_keyword(subscribe),
execution_context=execution_context,
)
current = self.subscription_repository.get(subscribe.id)
if current:
@@ -625,7 +871,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
@staticmethod
def _raise_site_budget_failures(failures: tuple[str, ...]) -> None:
"""在成功站点结果完成处理后暴露未执行站点的聚合失败。"""
"""在成功站点结果完成处理后暴露其余站点的聚合失败。"""
if failures:
raise RuntimeError("".join(failures))
@@ -713,9 +959,19 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
if not subscribes:
self.messagehelper.put("没有找到订阅!", title="订阅搜索", role="system")
elif sid:
self.messagehelper.put(f"{subscribes[0].name} 搜索完成!", title="订阅搜索", role="system")
message = (
f"{subscribes[0].name} 搜索完成!"
if processed
else f"{subscribes[0].name} 本轮未执行或未完成,将等待下一次正常调度。"
)
self.messagehelper.put(message, title="订阅搜索", role="system")
elif sids is not None:
for subscribe in processed:
self.messagehelper.put(f"{subscribe.name} 搜索完成!", title="订阅搜索", role="system")
else:
self.messagehelper.put("所有订阅搜索完成!", title="订阅搜索", role="system")
message = (
"所有订阅搜索完成!"
if len(processed) == len(subscribes)
else "订阅搜索完成,部分订阅本轮未执行或未完成。"
)
self.messagehelper.put(message, title="订阅搜索", role="system")
+3 -60
View File
@@ -1,12 +1,11 @@
import re
import traceback
from datetime import datetime, timezone
from typing import Callable, Dict, List, Optional, Union
from app.application.configuration import get_configured_system_config
from app.application.rss import RssHelper
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
from app.application.subscription.candidates import CandidateBatch, CandidateIndex
from app.application.subscription.candidates import CandidateIndex
from app.application.torrent.download import TorrentHelper
from app.chain.base import ChainBase
from app.chain.media import MediaChain
@@ -422,8 +421,6 @@ class TorrentsChain(ChainBase):
include_music: bool,
torrents_cache: Dict[str, List[Context]],
music_cache: Dict[str, List[Context]],
fresh_torrents: Dict[str, List[Context]],
fresh_music: Dict[str, List[Context]],
) -> str:
"""抓取并写入单个站点的影视、音乐资源缓存。"""
domain = site_rules.extract_domain(indexer.get("domain"))
@@ -483,9 +480,7 @@ class TorrentsChain(ChainBase):
continue
context = self._build_refresh_context(torrent, stype)
target_cache = music_cache if torrent.category == MediaType.MUSIC.value else torrents_cache
target_fresh = fresh_music if torrent.category == MediaType.MUSIC.value else fresh_torrents
target_cache.setdefault(domain, []).append(context)
target_fresh.setdefault(domain, []).append(context)
if len(target_cache[domain]) > self.runtime_config.torrent_cache_size:
target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:]
return domain
@@ -548,31 +543,14 @@ class TorrentsChain(ChainBase):
progress_callback: Optional[Callable[..., None]] = None,
include_music: bool = False,
) -> Dict[str, List[Context]]:
"""兼容旧调用返回完整候选字典,批次语义由 ``refresh_batch`` 提供。"""
return self.refresh_batch(
stype=stype,
sites=sites,
progress_callback=progress_callback,
include_music=include_music,
).candidates
def refresh_batch(
self,
stype: Optional[str] = None,
sites: List[int] = None,
progress_callback: Optional[Callable[..., None]] = None,
include_music: bool = False,
) -> CandidateBatch:
"""
刷新站点最新资源并返回完整缓存与本轮新增候选
刷新站点最新资源并返回本轮可匹配的完整候选缓存
:param stype: 强制指定缓存类型spider:爬虫缓存rss:rss缓存
:param sites: 强制指定站点ID列表为空则读取设置的订阅站点
:param progress_callback: 资源刷新进度更新回调
:param include_music: 是否额外抓取站点的音乐专用浏览入口服务音乐订阅
"""
started_at = datetime.now(timezone.utc)
# 刷新类型
if not stype:
stype = self.runtime_config.subscribe_mode
@@ -590,8 +568,6 @@ class TorrentsChain(ChainBase):
music_cache = self.load_cache(self._music_rss_file) or {}
self._ensure_context_compatibility(torrents_cache, stype=stype)
self._ensure_context_compatibility(music_cache, stype=stype)
fresh_torrents: Dict[str, List[Context]] = {}
fresh_music: Dict[str, List[Context]] = {}
# 缓存过滤掉无效种子(影视与音乐缓存分别处理)
for _cache in (torrents_cache, music_cache):
@@ -632,8 +608,6 @@ class TorrentsChain(ChainBase):
include_music=include_music,
torrents_cache=torrents_cache,
music_cache=music_cache,
fresh_torrents=fresh_torrents,
fresh_music=fresh_music,
))
# 保存缓存到本地,影视与音乐分别存储
@@ -649,10 +623,6 @@ class TorrentsChain(ChainBase):
torrents_cache = {k: v for k, v in torrents_cache.items() if k in domains}
if sites and music_cache:
music_cache = {k: v for k, v in music_cache.items() if k in domains}
if sites and fresh_torrents:
fresh_torrents = {k: v for k, v in fresh_torrents.items() if k in domains}
if sites and fresh_music:
fresh_music = {k: v for k, v in fresh_music.items() if k in domains}
if progress_callback:
progress_callback(
@@ -661,37 +631,10 @@ class TorrentsChain(ChainBase):
data={"total": total_indexers, "finished": total_indexers},
)
self._retain_cached_fresh(fresh_torrents, torrents_cache)
self._retain_cached_fresh(fresh_music, music_cache)
candidates = self._merge_torrent_caches(
return self._merge_torrent_caches(
{domain: list(contexts) for domain, contexts in torrents_cache.items()},
music_cache,
)
fresh_candidates = self._merge_torrent_caches(
{domain: list(contexts) for domain, contexts in fresh_torrents.items()},
fresh_music,
)
return CandidateBatch.create(
source=stype,
candidates=candidates,
fresh_candidates=fresh_candidates,
sites=domains,
started_at=started_at,
)
@staticmethod
def _retain_cached_fresh(
fresh_candidates: Dict[str, List[Context]],
cached_candidates: Dict[str, List[Context]],
) -> None:
"""移除因缓存容量裁剪而未进入最终完整缓存的本轮候选。"""
for domain, contexts in list(fresh_candidates.items()):
cached_ids = {id(context) for context in cached_candidates.get(domain) or []}
retained = [context for context in contexts if id(context) in cached_ids]
if retained:
fresh_candidates[domain] = retained
else:
fresh_candidates.pop(domain, None)
@staticmethod
def _ensure_context_compatibility(torrents_cache: Dict[str, List[Context]], stype: Optional[str] = None):
-147
View File
@@ -1,147 +0,0 @@
"""订阅下载幂等账本的短事务适配器。"""
from collections.abc import Callable
from typing import Optional, TypeVar
from sqlalchemy.orm import Session
from app.application.download.admission import (
SubscriptionDownloadClaim,
SubscriptionDownloadRequest,
SubscriptionDownloadSnapshot,
)
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
from app.db.oper.subscriptiondownload import SubscriptionDownloadOper
from app.db.uow import SqlAlchemyUnitOfWork
T = TypeVar("T")
def _snapshot(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot:
"""在 Session 内投影不可变提交快照。"""
return SubscriptionDownloadSnapshot(
idempotency_key=record.idempotency_key,
subscription_id=record.subscription_id,
task_id=record.task_id,
state=record.state,
attempt_count=record.attempt_count,
attempt_token=record.attempt_token,
downloader=record.downloader,
download_hash=record.download_hash,
available_at=record.available_at,
last_error=record.last_error,
created_at=record.created_at,
updated_at=record.updated_at,
)
class TransactionalSubscriptionDownloadRepository:
"""为每次提交认领和状态变更创建独立短事务。"""
def __init__(self, session_factory: Callable[[], Session]) -> None:
"""保存由组合根提供的同步 Session 工厂。"""
self._session_factory = session_factory
def _read(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T:
"""在短 Session 中执行一次只读查询。"""
with self._session_factory() as session:
return operation(SubscriptionDownloadOper(session))
def _write(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T:
"""在显式短事务中执行一次状态变更。"""
with self._session_factory() as session:
unit_of_work = SqlAlchemyUnitOfWork(session)
try:
result = operation(SubscriptionDownloadOper(session))
unit_of_work.commit()
return result
except Exception:
unit_of_work.rollback()
raise
def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim:
"""认领唯一提交权并返回脱离 Session 的当前状态。"""
def operation(repository: SubscriptionDownloadOper) -> SubscriptionDownloadClaim:
"""在同一事务内完成唯一插入或 fenced 重领。"""
record, acquired = repository.claim(request)
return SubscriptionDownloadClaim(acquired=acquired, snapshot=_snapshot(record))
return self._write(operation)
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]:
"""读取一个已存在的提交快照。"""
return self._read(
lambda repository: (
_snapshot(record)
if (record := repository.get(idempotency_key)) is not None
else None
)
)
def mark_accepted(
self,
*,
idempotency_key: str,
attempt_token: str,
downloader: Optional[str],
download_hash: str,
) -> bool:
"""记录下载器已接受任务。"""
return self._write(lambda repository: repository.mark_accepted(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
downloader=downloader,
download_hash=download_hash,
))
def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool:
"""记录 canonical 本地结算已完成。"""
return self._write(lambda repository: repository.mark_succeeded(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
))
def mark_retryable(
self,
*,
idempotency_key: str,
attempt_token: str,
available_at: str,
error: Optional[str],
) -> bool:
"""记录未产生外部副作用的延迟重试状态。"""
return self._write(lambda repository: repository.mark_retryable(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
available_at=available_at,
error=error,
))
def mark_reconcile_required(
self,
*,
idempotency_key: str,
attempt_token: str,
error: Optional[str],
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
) -> bool:
"""冻结可能已被下载器接受的提交。"""
return self._write(lambda repository: repository.mark_reconcile_required(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
error=error,
downloader=downloader,
download_hash=download_hash,
))
def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool:
"""在外部提交前收口取消。"""
return self._write(lambda repository: repository.mark_cancelled(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
))
def has_started_for_task(self, task_id: str) -> bool:
"""查询搜索任务是否已越过可安全取消边界。"""
return self._read(lambda repository: repository.has_started_for_task(task_id))
+5 -4
View File
@@ -1,6 +1,6 @@
"""订阅搜索持久队列的 SQLAlchemy 适配器。"""
from collections.abc import Callable
from collections.abc import Callable, Mapping
from typing import Optional, TypeVar
from sqlalchemy.orm import Session
@@ -32,6 +32,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
finished_count=record.finished_count,
failed_count=record.failed_count,
cancelled_count=record.cancelled_count,
skipped_count=record.skipped_count,
cancel_requested=bool(record.cancel_requested),
created_at=record.created_at,
updated_at=record.updated_at,
@@ -95,7 +96,7 @@ class TransactionalSubscriptionSearchRepository:
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str] = None,
available_at_by_subscription: Optional[Mapping[int, str]] = None,
) -> SearchEnqueueResult:
"""创建批次并返回 single-flight 合并计数。"""
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
@@ -104,7 +105,7 @@ class TransactionalSubscriptionSearchRepository:
subscription_ids=subscription_ids,
source=source,
priority=priority,
available_at=available_at,
available_at_by_subscription=available_at_by_subscription,
)
return SearchEnqueueResult(
batch=_batch(record),
@@ -231,7 +232,7 @@ class TransactionalSubscriptionSearchRepository:
next_allowed_at: str,
error: Optional[str] = None,
) -> bool:
"""释放站点租约并持久化间隔或冷却"""
"""释放站点租约并持久化错误冷却或立即恢复"""
return self._write(
lambda repository: repository.finish_site(
site_id=site_id,
+2 -39
View File
@@ -5,9 +5,7 @@ from typing import Optional
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.application.download.admission import SubscriptionDownloadSnapshot
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
from app.db.models.subscriptionsearch import SubscriptionSearchBatch, SubscriptionSearchTask
@@ -46,6 +44,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
finished_count=record.finished_count,
failed_count=record.failed_count,
cancelled_count=record.cancelled_count,
skipped_count=record.skipped_count,
cancel_requested=bool(record.cancel_requested),
created_at=record.created_at,
updated_at=record.updated_at,
@@ -55,26 +54,8 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
)
def _download(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot:
"""复制可脱离 AsyncSession 使用的下载提交快照。"""
return SubscriptionDownloadSnapshot(
idempotency_key=record.idempotency_key,
subscription_id=record.subscription_id,
task_id=record.task_id,
state=record.state,
attempt_count=record.attempt_count,
attempt_token=record.attempt_token,
downloader=record.downloader,
download_hash=record.download_hash,
available_at=record.available_at,
last_error=record.last_error,
created_at=record.created_at,
updated_at=record.updated_at,
)
class SessionSubscriptionExecutionStatusRepository:
"""复用请求 AsyncSession 批量读取搜索和下载执行事实。"""
"""复用请求 AsyncSession 批量读取搜索执行事实。"""
def __init__(self, session: AsyncSession) -> None:
"""绑定请求持有的异步会话。"""
@@ -98,24 +79,6 @@ class SessionSubscriptionExecutionStatusRepository:
snapshots.setdefault(record.subscription_id, _task(record))
return snapshots
async def latest_download_submissions(
self,
subscription_ids: tuple[int, ...],
) -> dict[int, SubscriptionDownloadSnapshot]:
"""按更新时间倒序读取并在内存中保留每条订阅首项。"""
result = await self._session.execute(
select(SubscriptionDownloadSubmission)
.where(SubscriptionDownloadSubmission.subscription_id.in_(subscription_ids))
.order_by(
SubscriptionDownloadSubmission.updated_at.desc(),
SubscriptionDownloadSubmission.id.desc(),
)
)
snapshots: dict[int, SubscriptionDownloadSnapshot] = {}
for record in result.scalars().all():
snapshots.setdefault(record.subscription_id, _download(record))
return snapshots
async def list_batches(self, *, limit: int) -> list[SearchBatchSnapshot]:
"""返回最近更新的批次,访问范围由应用服务依据任务校验。"""
result = await self._session.execute(
-4
View File
@@ -46,10 +46,6 @@ _MODEL_EXPORTS = {
"app.db.models.subscriptionsearch",
"SubscriptionSiteBudget",
),
"SubscriptionDownloadSubmission": (
"app.db.models.subscriptiondownload",
"SubscriptionDownloadSubmission",
),
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
"TransferExecutionStep": (
-53
View File
@@ -1,53 +0,0 @@
"""订阅下载提交幂等账本模型。"""
from typing import Optional
from sqlalchemy import Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from app.db.base import Base, get_id_column
class SubscriptionDownloadSubmission(Base):
"""记录订阅下载唯一提交权、下载器接受事实和待对账终态。"""
id = get_id_column()
idempotency_key: Mapped[str] = mapped_column(String(64), nullable=False)
subscription_id: Mapped[int] = mapped_column(Integer, nullable=False)
task_id: Mapped[Optional[str]] = mapped_column(String(64))
logical_identity: Mapped[str] = mapped_column(Text, nullable=False)
resource_key: Mapped[str] = mapped_column(Text, nullable=False)
coverage: Mapped[str] = mapped_column(Text, nullable=False)
mode: Mapped[str] = mapped_column(String(32), nullable=False)
delivery_scope: Mapped[str] = mapped_column(Text, nullable=False, default="legacy")
state: Mapped[str] = mapped_column(String(32), nullable=False)
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
attempt_token: Mapped[Optional[str]] = mapped_column(String(64))
downloader: Mapped[Optional[str]] = mapped_column(String(128))
download_hash: Mapped[Optional[str]] = mapped_column(String(256))
available_at: Mapped[Optional[str]] = mapped_column(String(40))
last_error: Mapped[Optional[str]] = mapped_column(Text)
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
started_at: Mapped[Optional[str]] = mapped_column(String(40))
finished_at: Mapped[Optional[str]] = mapped_column(String(40))
__table_args__ = (
UniqueConstraint(
"idempotency_key",
name="uq_subscriptiondownloadsubmission_idempotency_key",
),
Index(
"ix_subscriptiondownloadsubmission_task_state",
"task_id",
"state",
"id",
),
Index(
"ix_subscriptiondownloadsubmission_subscription_state",
"subscription_id",
"state",
"updated_at",
"id",
),
)
+2 -1
View File
@@ -20,6 +20,7 @@ class SubscriptionSearchBatch(Base):
finished_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
skipped_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
cancel_requested: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
@@ -77,7 +78,7 @@ class SubscriptionSearchTask(Base):
class SubscriptionSiteBudget(Base):
"""记录兜底搜索对单个站点的唯一租约、间隔与错误冷却。"""
"""记录兜底搜索对单个站点的唯一租约与错误冷却。"""
id = get_id_column()
site_id: Mapped[int] = mapped_column(Integer, nullable=False)
-242
View File
@@ -1,242 +0,0 @@
"""订阅下载提交账本的事务内状态转换。"""
from datetime import datetime, timezone
from typing import Optional, cast
from uuid import uuid4
from sqlalchemy import and_, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.application.download.admission import SubscriptionDownloadRequest
from app.db.base import DbOper, execute_dml
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
def utc_now_text() -> str:
"""返回可按字符串稳定排序的 UTC ISO 时间。"""
return datetime.now(timezone.utc).isoformat(timespec="seconds")
class SubscriptionDownloadOper(DbOper):
"""在调用方事务内维护唯一下载提交及其 fenced 终态。"""
def claim(self, request: SubscriptionDownloadRequest) -> tuple[SubscriptionDownloadSubmission, bool]:
"""创建或原子重领允许重试的提交记录。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅下载提交认领需要调用方提供同步 Session")
now = utc_now_text()
attempt_token = uuid4().hex
record = self._db.execute(
select(SubscriptionDownloadSubmission).where(
SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key
)
).scalars().first()
if record is None:
try:
with self._db.begin_nested():
record = SubscriptionDownloadSubmission(
idempotency_key=request.idempotency_key,
subscription_id=request.subscription_id,
task_id=request.task_id,
logical_identity=request.logical_identity,
resource_key=request.resource_key,
coverage=request.coverage,
mode=request.mode,
delivery_scope=request.delivery_scope,
state="submitting",
attempt_count=1,
attempt_token=attempt_token,
created_at=now,
updated_at=now,
started_at=now,
)
self._db.add(record)
self._db.flush()
return record, True
except IntegrityError:
self._db.expire_all()
record = self._db.execute(
select(SubscriptionDownloadSubmission).where(
SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key
)
).scalar_one()
reclaimable = record.state in {"retryable", "cancelled"} and (
record.available_at is None or record.available_at <= now
)
if not reclaimable:
return record, False
updated = execute_dml(
self._db,
update(SubscriptionDownloadSubmission)
.where(
SubscriptionDownloadSubmission.id == record.id,
SubscriptionDownloadSubmission.state.in_(("retryable", "cancelled")),
or_(
SubscriptionDownloadSubmission.available_at.is_(None),
SubscriptionDownloadSubmission.available_at <= now,
),
)
.values(
state="submitting",
task_id=request.task_id,
attempt_count=SubscriptionDownloadSubmission.attempt_count + 1,
attempt_token=attempt_token,
downloader=None,
download_hash=None,
available_at=None,
last_error=None,
updated_at=now,
started_at=now,
finished_at=None,
),
execution_options={"synchronize_session": False},
)
self._db.flush()
self._db.expire_all()
current = self._db.execute(
select(SubscriptionDownloadSubmission).where(
SubscriptionDownloadSubmission.id == record.id
)
).scalar_one()
return current, bool(updated and current.attempt_token == attempt_token)
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSubmission]:
"""按稳定幂等键读取提交记录。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session")
return cast(Optional[SubscriptionDownloadSubmission], self._db.execute(
select(SubscriptionDownloadSubmission).where(
SubscriptionDownloadSubmission.idempotency_key == idempotency_key
)
).scalars().first())
def mark_accepted(
self,
*,
idempotency_key: str,
attempt_token: str,
downloader: Optional[str],
download_hash: str,
) -> bool:
"""把当前 submitting 尝试推进为下载器已接受。"""
return self._transition(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
source_states=("submitting",),
values={
"state": "accepted",
"downloader": downloader,
"download_hash": download_hash,
},
)
def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool:
"""把当前 accepted 尝试推进为本地结算成功。"""
return self._transition(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
source_states=("accepted",),
values={"state": "succeeded", "finished_at": utc_now_text()},
)
def mark_retryable(
self,
*,
idempotency_key: str,
attempt_token: str,
available_at: str,
error: Optional[str],
) -> bool:
"""把下载器明确拒绝的当前尝试收口为延迟可重试。"""
return self._transition(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
source_states=("submitting",),
values={
"state": "retryable",
"available_at": available_at,
"last_error": error,
"finished_at": utc_now_text(),
},
)
def mark_reconcile_required(
self,
*,
idempotency_key: str,
attempt_token: str,
error: Optional[str],
downloader: Optional[str],
download_hash: Optional[str],
) -> bool:
"""冻结 submitting/accepted 尝试,禁止自动重新提交。"""
values: dict[str, object] = {
"state": "reconcile_required",
"last_error": error,
"finished_at": utc_now_text(),
}
if downloader is not None:
values["downloader"] = downloader
if download_hash is not None:
values["download_hash"] = download_hash
return self._transition(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
source_states=("submitting", "accepted"),
values=values,
)
def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool:
"""在外部提交前把当前尝试收口为已取消。"""
return self._transition(
idempotency_key=idempotency_key,
attempt_token=attempt_token,
source_states=("submitting",),
values={"state": "cancelled", "finished_at": utc_now_text()},
)
def has_started_for_task(self, task_id: str) -> bool:
"""判断任务是否进入过外部副作用边界或已成功。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session")
return bool(self._db.execute(
select(SubscriptionDownloadSubmission.id)
.where(
SubscriptionDownloadSubmission.task_id == task_id,
SubscriptionDownloadSubmission.state.in_((
"submitting",
"accepted",
"succeeded",
"reconcile_required",
)),
)
.limit(1)
).scalar())
def _transition(
self,
*,
idempotency_key: str,
attempt_token: str,
source_states: tuple[str, ...],
values: dict[str, object],
) -> bool:
"""以当前尝试令牌原子推进状态,拒绝过期执行者写回。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅下载提交写入需要调用方提供同步 Session")
now = utc_now_text()
return bool(execute_dml(
self._db,
update(SubscriptionDownloadSubmission)
.where(
and_(
SubscriptionDownloadSubmission.idempotency_key == idempotency_key,
SubscriptionDownloadSubmission.attempt_token == attempt_token,
SubscriptionDownloadSubmission.state.in_(source_states),
)
)
.values(updated_at=now, **values),
execution_options={"synchronize_session": False},
))
+38 -11
View File
@@ -1,7 +1,7 @@
"""订阅搜索批次与任务的持久队列读写。"""
from datetime import datetime, timedelta, timezone
from typing import Optional
from typing import Mapping, Optional
from uuid import uuid4
from sqlalchemy import and_, case, func, or_, select, update
@@ -30,7 +30,7 @@ class SubscriptionSearchOper(DbOper):
subscription_ids: tuple[int, ...],
source: str,
priority: int,
available_at: Optional[str],
available_at_by_subscription: Optional[Mapping[int, str]],
) -> tuple[SubscriptionSearchBatch, int, int]:
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
if not isinstance(self._db, Session):
@@ -51,6 +51,11 @@ class SubscriptionSearchOper(DbOper):
coalesced = 0
for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)):
active_key = f"subscription:{subscription_id}"
available_at = (
available_at_by_subscription.get(subscription_id, now)
if available_at_by_subscription
else now
)
task = SubscriptionSearchTask(
task_id=uuid4().hex,
batch_id=batch.batch_id,
@@ -61,7 +66,7 @@ class SubscriptionSearchOper(DbOper):
position=position,
state="queued",
phase="queued",
available_at=available_at or now,
available_at=available_at,
created_at=now,
updated_at=now,
)
@@ -85,9 +90,9 @@ class SubscriptionSearchOper(DbOper):
(
or_(
SubscriptionSearchTask.available_at.is_(None),
SubscriptionSearchTask.available_at > (available_at or now),
SubscriptionSearchTask.available_at > available_at,
),
available_at or now,
available_at,
),
else_=SubscriptionSearchTask.available_at,
),
@@ -267,7 +272,7 @@ class SubscriptionSearchOper(DbOper):
"""以当前租约令牌收口任务,并重新计算批次终态。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅搜索收口需要调用方提供同步 Session")
if state not in {"completed", "failed", "cancelled"}:
if state not in {"completed", "failed", "cancelled", "skipped"}:
raise ValueError(f"不支持的订阅搜索终态:{state}")
task = self._db.execute(
select(SubscriptionSearchTask).where(
@@ -334,7 +339,7 @@ class SubscriptionSearchOper(DbOper):
error=None,
)
now = utc_now_text()
return bool(execute_dml(
updated = execute_dml(
self._db,
update(SubscriptionSearchTask)
.where(
@@ -352,7 +357,11 @@ class SubscriptionSearchOper(DbOper):
updated_at=now,
),
execution_options={"synchronize_session": False},
))
)
if not updated:
return False
self._refresh_batch(task.batch_id, now=now, error=None)
return True
def is_cancel_requested(self, task_id: str) -> bool:
"""读取任务和批次取消标记。"""
@@ -437,7 +446,11 @@ class SubscriptionSearchOper(DbOper):
and record.lease_expires_at
and record.lease_expires_at > now
)
if lease_busy or record.next_allowed_at > now:
cooldown_active = bool(
record.last_outcome not in {None, "success", "skipped"}
and record.next_allowed_at > now
)
if lease_busy or cooldown_active:
return record, False
lease_token = uuid4().hex
lease_expires_at = (
@@ -453,12 +466,17 @@ class SubscriptionSearchOper(DbOper):
SubscriptionSiteBudget.lease_expires_at.is_(None),
SubscriptionSiteBudget.lease_expires_at <= now,
),
or_(
SubscriptionSiteBudget.next_allowed_at <= now,
SubscriptionSiteBudget.last_outcome.is_(None),
SubscriptionSiteBudget.last_outcome.in_(("success", "skipped")),
),
)
.values(
lease_owner=owner,
lease_token=lease_token,
lease_expires_at=lease_expires_at,
next_allowed_at=now,
updated_at=now,
),
execution_options={"synchronize_session": False},
@@ -586,7 +604,8 @@ class SubscriptionSearchOper(DbOper):
completed = counts.get("completed", 0)
failed = counts.get("failed", 0)
cancelled = counts.get("cancelled", 0)
terminal = completed + failed + cancelled
skipped = counts.get("skipped", 0)
terminal = completed + failed + cancelled + skipped
batch = self.get_batch(batch_id)
if batch is None:
return
@@ -595,11 +614,18 @@ class SubscriptionSearchOper(DbOper):
state = "failed"
elif cancelled or batch.cancel_requested:
state = "cancelled"
elif skipped:
state = "skipped"
else:
state = "completed"
finished_at = now
else:
state = "cancelling" if batch.cancel_requested else "running"
if batch.cancel_requested:
state = "cancelling"
elif counts.get("running", 0):
state = "running"
else:
state = "queued"
finished_at = None
execute_dml(
self._db,
@@ -610,6 +636,7 @@ class SubscriptionSearchOper(DbOper):
finished_count=completed,
failed_count=failed,
cancelled_count=cancelled,
skipped_count=skipped,
updated_at=now,
finished_at=finished_at,
last_error=error or batch.last_error,
+1 -2
View File
@@ -57,8 +57,6 @@ class SubscriptionExecutionStatus(BaseModel): # type: ignore[misc]
current_site_id: Optional[int] = None
error: Optional[str] = None
can_cancel: bool = False
can_retry: bool = False
requires_reconciliation: bool = False
model_config = ConfigDict(from_attributes=True)
@@ -77,6 +75,7 @@ class SubscriptionBatchStatus(BaseModel): # type: ignore[misc]
cancelled_count: int
created_at: str
updated_at: str
skipped_count: int = 0
current_subscription_id: Optional[int] = None
current_site_id: Optional[int] = None
error: Optional[str] = None
-2
View File
@@ -36,7 +36,6 @@ from app.chain.transfer.filter import (
from app.db.adapters.chain import TransactionalChainDurableEventWriter
from app.db.adapters.download import TransactionalDownloadFailureRepository
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.oper.message import MessageOper
from app.db.oper.systemconfig import SystemConfigOper
@@ -112,7 +111,6 @@ def build_chain_runtime_context(
transfer_execution_repository=dependencies.transfer_execution,
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory),
user_repository=build_transactional_user_repository(),
legacy_transfer_command=execute_legacy_transfer_command,
configuration=configuration(),
+37
View File
@@ -0,0 +1,37 @@
"""3.0.24 增加订阅搜索批次跳过计数。"""
# Alembic 的 op 是运行期代理,静态分析无法看到实际操作方法。
# pylint: disable=no-member
import sqlalchemy as sa
from alembic import op
revision = "b6c1d9e4a7f2"
down_revision = "a7d9e2c4f6b1"
branch_labels = None
depends_on = None
_TABLE = "subscriptionsearchbatch"
def _column_names() -> set[str]:
"""返回当前订阅搜索批次字段名集合。"""
inspector = sa.inspect(op.get_bind())
if _TABLE not in set(inspector.get_table_names()):
return set()
return {column["name"] for column in inspector.get_columns(_TABLE)}
def upgrade() -> None:
"""为存量批次增加跳过计数,保持旧数据库可重复升级。"""
if _TABLE in set(sa.inspect(op.get_bind()).get_table_names()) and "skipped_count" not in _column_names():
op.add_column(
_TABLE,
sa.Column("skipped_count", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
"""移除批次跳过计数,保留原搜索任务与批次记录。"""
if "skipped_count" in _column_names():
op.drop_column(_TABLE, "skipped_count")
+73
View File
@@ -0,0 +1,73 @@
"""3.0.25 移除订阅下载提交持久账本。
Revision ID: c8f2e6a1d4b9
Revises: b6c1d9e4a7f2
Create Date: 2026-09-03
"""
# Alembic 的 op 是运行期代理,静态分析无法看到实际操作方法。
# pylint: disable=no-member
import sqlalchemy as sa
from alembic import op
revision = "c8f2e6a1d4b9"
down_revision = "b6c1d9e4a7f2"
branch_labels = None
depends_on = None
_TABLE = "subscriptiondownloadsubmission"
def _table_names() -> set[str]:
"""返回当前数据库表名集合。"""
return set(sa.inspect(op.get_bind()).get_table_names())
def upgrade() -> None:
"""删除不再参与订阅执行的下载提交账本。"""
if _TABLE in _table_names():
op.drop_table(_TABLE)
def downgrade() -> None:
"""恢复 3.0.24 使用的下载提交账本结构。"""
if _TABLE in _table_names():
return
op.create_table(
_TABLE,
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("idempotency_key", sa.String(length=64), nullable=False),
sa.Column("subscription_id", sa.Integer(), nullable=False),
sa.Column("task_id", sa.String(length=64), nullable=True),
sa.Column("logical_identity", sa.Text(), nullable=False),
sa.Column("resource_key", sa.Text(), nullable=False),
sa.Column("coverage", sa.Text(), nullable=False),
sa.Column("mode", sa.String(length=32), nullable=False),
sa.Column("delivery_scope", sa.Text(), nullable=False, server_default="legacy"),
sa.Column("state", sa.String(length=32), nullable=False),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.Column("attempt_token", sa.String(length=64), nullable=True),
sa.Column("downloader", sa.String(length=128), nullable=True),
sa.Column("download_hash", sa.String(length=256), nullable=True),
sa.Column("available_at", sa.String(length=40), nullable=True),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("created_at", sa.String(length=40), nullable=False),
sa.Column("updated_at", sa.String(length=40), nullable=False),
sa.Column("started_at", sa.String(length=40), nullable=True),
sa.Column("finished_at", sa.String(length=40), nullable=True),
sa.UniqueConstraint(
"idempotency_key",
name="uq_subscriptiondownloadsubmission_idempotency_key",
),
)
op.create_index(
"ix_subscriptiondownloadsubmission_task_state",
_TABLE,
["task_id", "state", "id"],
)
op.create_index(
"ix_subscriptiondownloadsubmission_subscription_state",
_TABLE,
["subscription_id", "state", "updated_at", "id"],
)
+2 -2
View File
@@ -754,8 +754,8 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 944 |
| 内部导入边 | 7,874 |
| Python 模块 | 940 |
| 内部导入边 | 7,845 |
| 非平凡 SCC | 1(精确 containment 的 TMDB 移植包环) |
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
| Direct egress | 53(债务已清零,53 条精确 containment |
+2 -2
View File
@@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
| 指标 | 当前值 | 解释 |
|---|---:|---|
| 宿主 Python 模块 / 内部依赖边 | 944 / 7,874 | `dependency-baseline.json` 当前快照 |
| 宿主 Python 模块 / 内部依赖边 | 940 / 7,845 | `dependency-baseline.json` 当前快照 |
| 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 |
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
@@ -102,7 +102,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
| Event Contract | 53 | 均已有 payload model,但当前全部是 diagnostic enforcement |
| Python 源码量 | 305,884 行 | 排除 `app/plugins/**`61 个文件超过 1,000 行,11 个超过 2,000 行 |
| 长方法 | 290 个超过 80 行 | AST 统计排除 `app/plugins/**`;65 个超过 150 行,21 个超过 250 行 |
| 全量 mypy 历史债务 | 9,581 / 516 文件 | Agent API 重构后的现状基线;canonical Facade 与 endpoint 类型边界已补齐,低水位只允许继续下降 |
| 全量 mypy 历史债务 | 9,580 / 516 文件 | Agent API 重构后的现状基线;canonical Facade 与 endpoint 类型边界已补齐,低水位只允许继续下降 |
| Ruff 历史诊断 | 559 | 低水位门禁通过,但规则集只覆盖 `E4/E7/E9/F/I` |
| 覆盖率固定基线 | Application 80.00%Domain 80.00% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
@@ -1,5 +1,9 @@
# MoviePilot 订阅执行治理上游 Handoff
> 历史交接:本文保留 2026-09-01 的问题证据,不再作为当前实施合同。
> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required`
> 已由 2026-09-03 的简化方案废止;当前行为以代码、迁移和回归测试为准。
>
> 日期:2026-09-01
> 目标版本:MoviePilot V3
> 生产参考:MoviePilot V2 `2.15.5` 的一次只读样本
@@ -1,5 +1,9 @@
# MoviePilot 订阅执行治理
> 历史方案:本文件记录 2026-09-01 的上游接管与验证证据,不再作为当前实施合同。
> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required`
> 已由 2026-09-03 的简化方案废止;当前行为以代码、迁移和回归测试为准。
>
> 状态:`completed2026-09-01`
> 当前叶:`none(全部叶已完成或按条件停止)`
> 迁移来源:`V3-RDY-009A``V3-RDY-009A1``V3-RDY-009A1B`
-5
View File
@@ -116,11 +116,6 @@ DATABASE_TABLE_GUIDES: dict[str, tuple[str, str, str]] = {
"Auditing historical subscriptions, media identity, completion criteria, and filter configuration.",
"Generated by subscription completion and archival; restore or delete through its business API.",
),
"subscriptiondownloadsubmission": (
"Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes.",
"Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership.",
"Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states.",
),
"subscriptionsearchbatch": (
"Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.",
"Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.",
File diff suppressed because it is too large Load Diff
+1 -7
View File
@@ -236,17 +236,11 @@ python scripts/mp-db.py write "UPDATE subscribe SET state = 'S' WHERE id = 123"
- Write boundary: Generated by subscription completion and archival; restore or delete through its business API.
- Columns: `id`, `name`, `year`, `type`, `keyword`, `media_source`, `media_id`, `music_type`, `total_tracks`, `season`, `poster`, `backdrop`, `vote`, `description`, `filter`, `include`, `exclude`, `quality`, `resolution`, `effect`, `audio_quality`, `audio_format`, `min_bitrate`, `min_bit_depth`, `min_sample_rate`, `total_episode`, `start_episode`, `date`, `username`, `sites`, `best_version`, `best_version_full`, `current_priority`, `current_audio_format`, `current_bitrate`, `current_bit_depth`, `current_sample_rate`, `episode_priority`, `save_path`, `search_imdbid`, `custom_words`, `media_category`, `filter_groups`, `episode_group`
### `subscriptiondownloadsubmission`
- Purpose: Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes.
- Useful queries: Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership.
- Write boundary: Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states.
- Columns: `id`, `idempotency_key`, `subscription_id`, `task_id`, `logical_identity`, `resource_key`, `coverage`, `mode`, `delivery_scope`, `state`, `attempt_count`, `attempt_token`, `downloader`, `download_hash`, `available_at`, `last_error`, `created_at`, `updated_at`, `started_at`, `finished_at`
### `subscriptionsearchbatch`
- Purpose: Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.
- Useful queries: Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.
- Write boundary: Owned by subscription search orchestration; create and cancel batches through the subscription API.
- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error`
- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `skipped_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error`
### `subscriptionsearchtask`
- Purpose: Stores one durable subscription search task per batch and subscription with leases and execution phases.
+14 -2
View File
@@ -1103,7 +1103,7 @@ Purpose: Read configured directory or storage settings.
Purpose: Create one movie, TV, or music subscription.
- `path_params`: none
- `query`: none
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
### `subscription.delete`
`DELETE /api/v1/subscribe/{subscribe_id}`; policy effect: `destructive_write`.
@@ -1277,7 +1277,7 @@ Purpose: Set one accessible subscription to running, paused, or stopped state.
Purpose: Update one existing movie, TV, or music subscription.
- `path_params`: none
- `query`: none
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
### `subscription.user.list`
`GET /api/v1/subscribe/user/{username}`; policy effect: `safe_read`.
@@ -1681,6 +1681,18 @@ This runtime model has no directly writable fields.
MoviePilot media type.
This runtime model has no directly writable fields.
#### `SubscriptionExecutionStatus`
Subscription refresh execution status and progress summary.
- `batch_id` (string|null): Stable subscription search batch identifier.
- `can_cancel` (boolean; default `False`): Whether the current subscription execution can be cancelled.
- `current_site_id` (integer|null): Configured site ID currently handling the subscription execution.
- `error` (string|null): Human-readable workflow, provider, or execution error message.
- `phase*` (string): Current phase of a subscription execution.
- `source` (string|null): Exact metadata or recommendation source selected by the operation.
- `state*` (string): Current site, subscription, marketplace, or transfer state filter.
- `task_id` (string|null): Stable durable transfer task ID returned by transfer.manual_reviews.
- `updated_at*` (string): Timestamp when the subscription execution status was last updated.
#### `TorrentInfo`
One torrent candidate returned by MoviePilot search.
- `category` (string|null): MoviePilot media category or filter-group category, depending on the operation.
-2
View File
@@ -285,7 +285,6 @@ def configure_plugin_system_services():
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
from app.db.adapters.site import TransactionalSiteRepository
from app.db.adapters.subscription import TransactionalSubscriptionRepository
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
from app.db.adapters.transaction import TransactionalWriteRunner
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.adapters.transfer.execution import (
@@ -413,7 +412,6 @@ def configure_plugin_system_services():
transfer_execution_repository=TransactionalTransferExecutionRepository(SessionFactory),
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory),
user_repository=user_repository(),
configuration=build_chain_runtime_config(settings),
)
+12 -45
View File
@@ -1089,8 +1089,8 @@
"runtime_only": true
}
},
"edge_count": 7874,
"edge_sha256": "b4ca47899f8af4366446ad76546b5b0cf3c5e87f38b603d593241acd52fba9f3",
"edge_count": 7845,
"edge_sha256": "aca69eac132e327d7a1dea770c185e7be5ab47bb41211749f6130c065a957919",
"edges": [
"app -> app.foundation",
"app -> app.foundation.environment",
@@ -3527,8 +3527,6 @@
"app.application.subscription.sitebudget -> app.runtime",
"app.application.subscription.sitebudget -> app.runtime.stop",
"app.application.subscription.status -> app.application",
"app.application.subscription.status -> app.application.download",
"app.application.subscription.status -> app.application.download.admission",
"app.application.subscription.status -> app.application.subscription",
"app.application.subscription.status -> app.application.subscription.execution",
"app.application.subscription.write -> app.application",
@@ -3634,9 +3632,13 @@
"app.chain._messaging -> app.schemas.types",
"app.chain._music -> app.application",
"app.chain._music -> app.application.configuration",
"app.chain._music -> app.application.download",
"app.chain._music -> app.application.download.admission",
"app.chain._music -> app.application.subscription",
"app.chain._music -> app.application.subscription.contract",
"app.chain._music -> app.application.subscription.execution",
"app.chain._music -> app.application.subscription.mutation",
"app.chain._music -> app.application.subscription.sitebudget",
"app.chain._music -> app.application.torrent",
"app.chain._music -> app.application.torrent.download",
"app.chain._music -> app.chain",
@@ -3725,18 +3727,6 @@
"app.chain.douban -> app.schemas",
"app.chain.douban -> app.schemas.context",
"app.chain.douban -> app.schemas.types",
"app.chain.download.admission -> app.application",
"app.chain.download.admission -> app.application.download",
"app.chain.download.admission -> app.application.download.admission",
"app.chain.download.admission -> app.chain",
"app.chain.download.admission -> app.chain.download",
"app.chain.download.admission -> app.chain.download.contract",
"app.chain.download.admission -> app.domain",
"app.chain.download.admission -> app.domain.context",
"app.chain.download.admission -> app.domain.episode",
"app.chain.download.admission -> app.schemas",
"app.chain.download.admission -> app.schemas.media",
"app.chain.download.admission -> app.schemas.types",
"app.chain.download.batch -> app.application",
"app.chain.download.batch -> app.application.download",
"app.chain.download.batch -> app.application.download.admission",
@@ -3773,7 +3763,6 @@
"app.chain.download.existence -> app.schemas.types",
"app.chain.download.facade -> app.chain",
"app.chain.download.facade -> app.chain.download",
"app.chain.download.facade -> app.chain.download.admission",
"app.chain.download.facade -> app.chain.download.batch",
"app.chain.download.facade -> app.chain.download.existence",
"app.chain.download.facade -> app.chain.download.failure",
@@ -4482,6 +4471,8 @@
"app.chain.subscribe.facade -> app.application",
"app.chain.subscribe.facade -> app.application.messaging",
"app.chain.subscribe.facade -> app.application.messaging.subscribe",
"app.chain.subscribe.facade -> app.application.subscription",
"app.chain.subscribe.facade -> app.application.subscription.execution",
"app.chain.subscribe.facade -> app.chain",
"app.chain.subscribe.facade -> app.chain._interaction",
"app.chain.subscribe.facade -> app.chain._music",
@@ -4530,7 +4521,9 @@
"app.chain.subscribe.match -> app.application.subscription",
"app.chain.subscribe.match -> app.application.subscription.candidates",
"app.chain.subscribe.match -> app.application.subscription.contract",
"app.chain.subscribe.match -> app.application.subscription.execution",
"app.chain.subscribe.match -> app.application.subscription.facts",
"app.chain.subscribe.match -> app.application.subscription.sitebudget",
"app.chain.subscribe.match -> app.application.torrent",
"app.chain.subscribe.match -> app.application.torrent.download",
"app.chain.subscribe.match -> app.chain",
@@ -4588,6 +4581,7 @@
"app.chain.subscribe.policy -> app.application.download.admission",
"app.chain.subscribe.policy -> app.application.subscription",
"app.chain.subscribe.policy -> app.application.subscription.contract",
"app.chain.subscribe.policy -> app.application.subscription.execution",
"app.chain.subscribe.policy -> app.application.subscription.priority",
"app.chain.subscribe.policy -> app.chain",
"app.chain.subscribe.policy -> app.chain.download",
@@ -4637,7 +4631,6 @@
"app.chain.subscribe.reconcile -> app.schemas.types",
"app.chain.subscribe.refresh -> app.application",
"app.chain.subscribe.refresh -> app.application.subscription",
"app.chain.subscribe.refresh -> app.application.subscription.candidates",
"app.chain.subscribe.refresh -> app.application.subscription.contract",
"app.chain.subscribe.refresh -> app.application.subscription.facts",
"app.chain.subscribe.refresh -> app.application.subscription.priority",
@@ -5211,15 +5204,6 @@
"app.db.adapters.subscription -> app.schemas",
"app.db.adapters.subscription -> app.schemas.common",
"app.db.adapters.subscription -> app.schemas.types",
"app.db.adapters.subscriptiondownload -> app.application",
"app.db.adapters.subscriptiondownload -> app.application.download",
"app.db.adapters.subscriptiondownload -> app.application.download.admission",
"app.db.adapters.subscriptiondownload -> app.db",
"app.db.adapters.subscriptiondownload -> app.db.models",
"app.db.adapters.subscriptiondownload -> app.db.models.subscriptiondownload",
"app.db.adapters.subscriptiondownload -> app.db.oper",
"app.db.adapters.subscriptiondownload -> app.db.oper.subscriptiondownload",
"app.db.adapters.subscriptiondownload -> app.db.uow",
"app.db.adapters.subscriptionsearch -> app.application",
"app.db.adapters.subscriptionsearch -> app.application.subscription",
"app.db.adapters.subscriptionsearch -> app.application.subscription.execution",
@@ -5231,13 +5215,10 @@
"app.db.adapters.subscriptionsearch -> app.db.oper.subscriptionsearch",
"app.db.adapters.subscriptionsearch -> app.db.uow",
"app.db.adapters.subscriptionstatus -> app.application",
"app.db.adapters.subscriptionstatus -> app.application.download",
"app.db.adapters.subscriptionstatus -> app.application.download.admission",
"app.db.adapters.subscriptionstatus -> app.application.subscription",
"app.db.adapters.subscriptionstatus -> app.application.subscription.execution",
"app.db.adapters.subscriptionstatus -> app.db",
"app.db.adapters.subscriptionstatus -> app.db.models",
"app.db.adapters.subscriptionstatus -> app.db.models.subscriptiondownload",
"app.db.adapters.subscriptionstatus -> app.db.models.subscriptionsearch",
"app.db.adapters.transaction -> app.db",
"app.db.adapters.transaction -> app.db.uow",
@@ -5377,8 +5358,6 @@
"app.db.models.subscribehistory -> app.db.models._constraints",
"app.db.models.subscribehistory -> app.schemas",
"app.db.models.subscribehistory -> app.schemas.types",
"app.db.models.subscriptiondownload -> app.db",
"app.db.models.subscriptiondownload -> app.db.base",
"app.db.models.subscriptionsearch -> app.db",
"app.db.models.subscriptionsearch -> app.db.base",
"app.db.models.systemconfig -> app.db",
@@ -5487,13 +5466,6 @@
"app.db.oper.subscribehistory -> app.schemas.common",
"app.db.oper.subscribehistory -> app.schemas.query",
"app.db.oper.subscribehistory -> app.schemas.types",
"app.db.oper.subscriptiondownload -> app.application",
"app.db.oper.subscriptiondownload -> app.application.download",
"app.db.oper.subscriptiondownload -> app.application.download.admission",
"app.db.oper.subscriptiondownload -> app.db",
"app.db.oper.subscriptiondownload -> app.db.base",
"app.db.oper.subscriptiondownload -> app.db.models",
"app.db.oper.subscriptiondownload -> app.db.models.subscriptiondownload",
"app.db.oper.subscriptionsearch -> app.db",
"app.db.oper.subscriptionsearch -> app.db.base",
"app.db.oper.subscriptionsearch -> app.db.models",
@@ -8165,7 +8137,6 @@
"app.startup.composition.chain -> app.db.adapters.chain",
"app.startup.composition.chain -> app.db.adapters.download",
"app.startup.composition.chain -> app.db.adapters.mediaserver",
"app.startup.composition.chain -> app.db.adapters.subscriptiondownload",
"app.startup.composition.chain -> app.db.adapters.transfer",
"app.startup.composition.chain -> app.db.adapters.transfer.admission",
"app.startup.composition.chain -> app.db.oper",
@@ -8967,7 +8938,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 944,
"module_count": 940,
"modules": [
"app",
"app.adapters",
@@ -9294,7 +9265,6 @@
"app.chain.dashboard",
"app.chain.douban",
"app.chain.download",
"app.chain.download.admission",
"app.chain.download.batch",
"app.chain.download.contract",
"app.chain.download.existence",
@@ -9401,7 +9371,6 @@
"app.db.adapters.query",
"app.db.adapters.site",
"app.db.adapters.subscription",
"app.db.adapters.subscriptiondownload",
"app.db.adapters.subscriptionsearch",
"app.db.adapters.subscriptionstatus",
"app.db.adapters.transaction",
@@ -9437,7 +9406,6 @@
"app.db.models.siteuserdata",
"app.db.models.subscribe",
"app.db.models.subscribehistory",
"app.db.models.subscriptiondownload",
"app.db.models.subscriptionsearch",
"app.db.models.systemconfig",
"app.db.models.transferexecutionstep",
@@ -9461,7 +9429,6 @@
"app.db.oper.site",
"app.db.oper.subscribe",
"app.db.oper.subscribehistory",
"app.db.oper.subscriptiondownload",
"app.db.oper.subscriptionsearch",
"app.db.oper.systemconfig",
"app.db.oper.transferexecutionstep",
+1 -1
View File
@@ -1006,7 +1006,7 @@
},
"app/chain/torrents.py": {
"arg-type": 22,
"assignment": 3,
"assignment": 2,
"no-untyped-call": 1,
"no-untyped-def": 15,
"type-arg": 5,
@@ -6,7 +6,7 @@
"repeat": 3,
"targets": {
"app.startup.lifecycle": {
"loaded_app_module_count": 515,
"loaded_app_module_count": 511,
"max_ms": 1115.407,
"median_ms": 1106.606,
"min_ms": 1098.166,
@@ -17,7 +17,7 @@
]
},
"app.factory": {
"loaded_app_module_count": 527,
"loaded_app_module_count": 523,
"max_ms": 1140.393,
"median_ms": 1134.83,
"min_ms": 1119.004,
@@ -28,7 +28,7 @@
]
},
"app.main": {
"loaded_app_module_count": 529,
"loaded_app_module_count": 525,
"max_ms": 1180.675,
"median_ms": 1165.915,
"min_ms": 1161.859,
-1
View File
@@ -39,7 +39,6 @@ def test_download_monolith_is_retired_and_package_owners_are_complete() -> None:
assert not (CHAIN_ROOT / "download.py").exists()
assert {path.name for path in DOWNLOAD_PACKAGE.glob("*.py")} == {
"__init__.py",
"admission.py",
"batch.py",
"contract.py",
"existence.py",
+103 -1
View File
@@ -11,8 +11,13 @@ from app.application.subscription.contract import (
SubscriptionSnapshot,
build_subscribe_meta,
)
from app.application.subscription.execution import (
SubscriptionExecutionAdmission,
SubscriptionExecutionContext,
)
from app.application.subscription.mutation import SubscriptionMutation
from app.chain.subscribe import SubscribeChain
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
from app.chain.subscribe.facade import SubscribeChain
from app.domain.context import (
MUSIC_ENTITY_ALBUM,
MUSIC_ENTITY_ARTIST,
@@ -120,6 +125,23 @@ def _configure_subscription_write(chain, repository) -> None:
chain.sync_subscription_mutation_scope = mutation_scope
def _execution_context(*, cancelled=None) -> SubscriptionExecutionContext:
"""构造音乐订阅使用的独立执行上下文。"""
admission = SubscriptionExecutionAdmission()
lease = admission.try_acquire(
subscription_id=7,
operation="search",
ttl_seconds=60,
)
assert lease is not None
return SubscriptionExecutionContext(
lease=lease,
admission=admission,
task_id="music-task-7",
cancel_requested=cancelled,
)
def test_build_subscribe_meta_returns_music_meta():
"""音乐订阅应构造 MetaMusic,而不是交给影视标题解析器。"""
meta = build_subscribe_meta(_subscribe())
@@ -190,6 +212,85 @@ def test_music_subscribe_reuses_search_download_and_finish_flow():
chain.finish_subscribe_or_not.assert_called_once()
def test_music_search_honours_cancel_before_external_work():
"""音乐搜索在安全边界收到取消后不得继续访问站点。"""
subscribe = _subscribe()
chain = SubscribeChain()
execution_context = _execution_context(cancelled=lambda: True)
with patch("app.chain._music.SearchChain") as search_chain, \
pytest.raises(SubscriptionSearchCancelled):
chain._search_music_subscribe(
subscribe,
execution_context=execution_context,
)
search_chain.assert_not_called()
def test_music_download_marks_shared_execution_context_before_side_effect():
"""音乐下载必须把取消和副作用边界传入统一下载治理。"""
cancelled = [False]
subscribe = _subscribe()
target = _music_info()
downloaded = Context(
torrent_info=TorrentInfo(
title="周杰伦 - 晴天 FLAC",
category=MediaType.MUSIC.value,
),
meta_info=MetaMusic.from_music_info(target),
media_info=target,
)
execution_context = _execution_context(cancelled=lambda: cancelled[0])
download_chain = Mock()
def batch_download(**kwargs):
"""模拟下载器边界内开始提交后才收到取消。"""
governance = kwargs["governance"]
assert governance.cancelled() is False
governance.mark_started()
cancelled[0] = True
return [downloaded], None
download_chain.batch_download.side_effect = batch_download
repository = Mock()
repository.get.return_value = subscribe
chain = SubscribeChain()
chain.subscription_repository = repository
chain.finish_subscribe_or_not = Mock()
with patch("app.chain._music.DownloadChain", return_value=download_chain):
chain._download_music_subscribe(
subscribe,
target,
[downloaded],
execution_context=execution_context,
)
assert execution_context.download_started is True
chain.finish_subscribe_or_not.assert_called_once()
def test_music_download_rechecks_paused_state_before_submission():
"""候选准备后暂停的音乐订阅不得进入下载器。"""
subscribe = _subscribe()
paused = _subscribe(state="S")
repository = Mock()
repository.get.return_value = paused
chain = SubscribeChain()
chain.subscription_repository = repository
with patch("app.chain._music.DownloadChain") as download_chain:
chain._download_music_subscribe(
subscribe,
_music_info(),
[Context()],
execution_context=_execution_context(),
)
download_chain.assert_not_called()
def test_music_subscribe_filters_declared_bitrate_and_format():
"""音乐订阅应按规范化格式和最低码率过滤站点资源。"""
subscribe = _subscribe(audio_format="MP3", min_bitrate=320000)
@@ -274,6 +375,7 @@ def test_music_best_version_persists_downloaded_rule_priority():
download_chain.batch_download.return_value = ([downloaded], None)
subscribe_oper = Mock()
updated = _subscribe(best_version=1, current_priority=100)
subscribe_oper.get.return_value = subscribe
subscribe_oper.update.return_value = updated
chain = SubscribeChain()
chain.finish_subscribe_or_not = Mock()
+24 -5
View File
@@ -701,16 +701,15 @@ class TestSubscribeChain:
def test_match_title_fallback_calls_torrent_match_from_class(self):
"""确保标题兜底匹配不依赖 TorrentHelper 实例绑定。"""
class _ReachedTitleMatch(Exception):
"""标记测试已经进入标题匹配函数体。"""
reached = []
class _PlainTorrentHelper:
"""模拟需要按类调用的 TorrentHelper 形态。"""
def match_torrent(mediainfo, torrent_meta, torrent):
"""标记类级调用已经正确进入匹配逻辑。"""
raise _ReachedTitleMatch
reached.append((mediainfo, torrent_meta, torrent))
return False
def filter_torrent(self, *args, **kwargs):
"""保持订阅匹配后续过滤流程可继续执行。"""
@@ -753,9 +752,14 @@ class TestSubscribeChain:
"""返回当前测试构造的订阅列表。"""
return [subscribe]
def get(self, subscribe_id):
"""返回取得订阅准入后的最新快照。"""
return subscribe if subscribe_id == subscribe.id else None
chain = SubscribeChain()
chain.subscription_repository = _SubscribeOper()
chain.check_and_handle_existing_media = lambda **kwargs: (False, {})
chain.finish_subscribe_or_not = lambda **kwargs: None
with (
patch.object(
@@ -764,10 +768,15 @@ class TestSubscribeChain:
_PlainTorrentHelper,
),
_patch_media_recognize(SUBSCRIBE_CHAIN_MODULE, mediainfo),
pytest.raises(_ReachedTitleMatch),
):
chain.match({"test.example": [context]})
assert len(reached) == 1
actual_media, actual_meta, actual_torrent = reached[0]
assert actual_media.title_year == mediainfo.title_year
assert actual_meta is context.meta_info
assert actual_torrent is context.torrent_info
def test_match_accepts_special_season_zero_candidate(self):
"""S0 订阅应允许 S00 候选资源进入下载候选,不能按未指定季处理。"""
@@ -862,6 +871,16 @@ class TestSubscribeChain:
assert len(download_calls) == 1
assert download_calls[0]["contexts"][0].meta_info.begin_season == 0
execution_context = download_calls[0]["execution_context"]
assert execution_context.lease.subscription_id == subscribe.id
assert execution_context.lease.operation == "match"
replacement = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="search",
ttl_seconds=60,
)
assert replacement is not None
assert chain._subscription_execution_admission.release(replacement) is True
def test_get_episode_priority_falls_back_to_current_priority(self):
subscribe = self._build_subscribe(current_priority=80, episode_priority=None)
+48 -5
View File
@@ -1,3 +1,4 @@
import threading
from contextlib import contextmanager
from dataclasses import replace
from datetime import datetime, timedelta
@@ -7,9 +8,10 @@ from unittest.mock import Mock, patch
import pytest
from app.application.subscription.contract import SubscriptionPatch, SubscriptionSnapshot
from app.application.subscription.execution import SubscriptionExecutionAdmission
from app.application.subscription.mutation import SubscriptionMutation
from app.chain.subscribe import SubscribeChain
from app.chain.subscribe import search as subscribe_search
from app.chain.subscribe.facade import SubscribeChain
from app.schemas.types import MediaType
@@ -150,14 +152,19 @@ def test_targeted_batch_searches_all_ids_without_state_scan(monkeypatch) -> None
chain.subscription_repository = subscribe_oper
chain.search(sids=(31, 32), state=None, manual=False)
assert [item.args for item in subscribe_oper.get.call_args_list] == [(31,), (32,)]
assert [item.args for item in subscribe_oper.get.call_args_list] == [
(31,),
(32,),
(31,),
(32,),
]
subscribe_oper.list.assert_not_called()
assert media_chain.recognize_media.call_count == 2
def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None:
"""订阅搜索锁超时后必须中止,不能在无锁状态下继续访问订阅。"""
monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock())
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", _TimedOutLock())
subscribe_oper = Mock()
progress = Mock()
@@ -176,7 +183,7 @@ def test_subscribe_search_releases_lock_when_repository_query_fails(monkeypatch)
"""取得搜索锁后即使订阅查询失败,也必须释放进程级互斥锁。"""
lock = Mock()
lock.acquire.return_value = True
monkeypatch.setattr(SubscribeChain, "_rlock", lock)
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", lock)
repository = Mock()
repository.list.side_effect = RuntimeError("query failed")
chain = object.__new__(SubscribeChain)
@@ -228,9 +235,45 @@ def test_subscribe_search_progress_preserves_public_callback_payload() -> None:
]
def test_inline_search_conflict_does_not_report_false_completion(monkeypatch) -> None:
"""兼容搜索遇到同订阅冲突时必须报告未执行,而不是成功完成。"""
subscribe = replace(
_new_subscribe(datetime.now() - timedelta(minutes=2)),
state="R",
)
repository = Mock()
repository.get.return_value = subscribe
progress = Mock()
chain = object.__new__(SubscribeChain)
chain.subscription_repository = repository
chain._search_queue_lock = threading.Lock()
chain._match_lock = threading.Lock()
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
match_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert match_lease is not None
with patch.object(subscribe_search, "SearchChain", return_value=Mock()):
chain.search(
sid=subscribe.id,
state=None,
progress_callback=progress,
)
assert progress.call_args.kwargs == {
"value": 100,
"text": "订阅搜索结束,部分订阅本轮未执行或未完成",
"data": {"total": 1, "finished": 0},
}
assert chain._subscription_execution_admission.release(match_lease) is True
def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None:
"""订阅匹配锁超时后必须中止,不能绕过防重复下载边界。"""
monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock())
monkeypatch.setattr(SubscribeChain, "_match_lock", _TimedOutLock())
progress = Mock()
chain = object.__new__(SubscribeChain)
+14 -14
View File
@@ -3,7 +3,7 @@
from types import SimpleNamespace
from unittest.mock import Mock, patch
from app.application.subscription.candidates import CandidateBatch, CandidateIndex
from app.application.subscription.candidates import CandidateIndex
from app.application.subscription.contract import SubscriptionSnapshot
from app.application.subscription.facts import FreshFactLease
from app.chain.torrents import TorrentsChain
@@ -68,8 +68,13 @@ def _subscribe(**overrides) -> SubscriptionSnapshot:
return SubscriptionSnapshot(**values)
def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta():
"""刷新批次必须保留完整缓存,同时只把本轮新增资源放入 fresh 集合"""
def _candidate_count(groups: dict[str, list[Context]]) -> int:
"""统计分站点候选总数"""
return sum(len(contexts) for contexts in groups.values())
def test_refresh_returns_complete_cache_with_new_candidates():
"""刷新结果必须包含既有缓存和本轮新增资源的完整候选。"""
chain = TorrentsChain()
existing = _context("既有剧集 S01E01", media_id="100")
duplicate = TorrentInfo(
@@ -115,18 +120,13 @@ def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta():
),
),
):
batch = chain.refresh_batch(stype="rss", sites=[1])
candidates = chain.refresh(stype="rss", sites=[1])
assert isinstance(batch, CandidateBatch)
assert batch.source == "rss"
assert batch.finished_at is not None
assert [item.torrent_info.title for item in batch.candidates["example.com"]] == [
assert [item.torrent_info.title for item in candidates["example.com"]] == [
existing.torrent_info.title,
fresh.title,
]
assert [item.torrent_info.title for item in batch.fresh_candidates["example.com"]] == [fresh.title]
assert CandidateBatch.count(batch.candidates) == 2
assert CandidateBatch.count(batch.fresh_candidates) == 1
assert _candidate_count(candidates) == 2
def test_candidate_index_routes_all_canonical_fallback_classes_without_loss():
@@ -214,7 +214,7 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product():
for subscribe in subscribes:
groups = candidate_index.route_for_match(subscribe)
examined += candidate_index.last_examined_count
routed += CandidateBatch.count(groups)
routed += _candidate_count(groups)
lease = FreshFactLease()
fact_loads = []
@@ -231,9 +231,9 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product():
)
assert len(candidates) == site_count
assert CandidateBatch.count(candidates) == 1000
assert _candidate_count(candidates) == 1000
assert examined == routed == 2000
assert examined < subscription_count * CandidateBatch.count(candidates) // 50
assert examined < subscription_count * _candidate_count(candidates) // 50
assert len(fact_loads) == media_count
assert lease.loads == media_count
assert lease.hits == subscription_count - media_count
@@ -0,0 +1,129 @@
"""订阅 Search/Match 公共入口的跨通道准入测试。"""
import threading
from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import Mock, patch
from app.application.subscription.contract import SubscriptionSnapshot
from app.application.subscription.execution import SubscriptionExecutionAdmission
from app.chain.subscribe.facade import SubscribeChain
from app.domain.context import Context
from app.schemas.types import MediaType
def _subscribe(subscribe_id: int) -> SubscriptionSnapshot:
"""构造已越过新增保护期的活动电影订阅。"""
return SubscriptionSnapshot(
id=subscribe_id,
name=f"并发电影 {subscribe_id}",
year="2026",
type=MediaType.MOVIE.value,
media_source="themoviedb",
media_id=str(5000 + subscribe_id),
state="R",
date=(datetime.now() - timedelta(minutes=2)).strftime("%Y-%m-%d %H:%M:%S"),
)
def _repository(*subscribes: SubscriptionSnapshot) -> SimpleNamespace:
"""提供公共入口重新读取所需的稳定订阅仓储。"""
snapshots = {subscribe.id: subscribe for subscribe in subscribes}
return SimpleNamespace(
get=snapshots.get,
list=lambda _state: list(snapshots.values()),
)
def _configure_channel_state(monkeypatch) -> None:
"""为不同 SubscribeChain 实例安装进程级共享准入与独立通道锁。"""
monkeypatch.setattr(SubscribeChain, "_match_lock", threading.Lock())
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", threading.Lock())
monkeypatch.setattr(
SubscribeChain,
"_subscription_execution_admission",
SubscriptionExecutionAdmission(),
)
def test_public_search_and_match_skip_same_subscription_across_instances(monkeypatch) -> None:
"""两个独立 Chain 实例处理同一订阅时只能有一个进入业务路径。"""
_configure_channel_state(monkeypatch)
subscribe = _subscribe(7)
repository = _repository(subscribe)
search_chain = object.__new__(SubscribeChain)
search_chain.subscription_repository = repository
match_chain = object.__new__(SubscribeChain)
match_chain.subscription_repository = repository
match_chain.get_states_for_search = lambda state: state
match_chain._prepare_match_torrents = lambda torrents: torrents
search_started = threading.Event()
release_search = threading.Event()
def process_search(item, _searchchain, *, execution_context):
"""持有 Search 准入,直到 Match 完成冲突探测。"""
assert execution_context.lease.subscription_id == item.id
search_started.set()
assert release_search.wait(timeout=3)
return item
match_subscription = Mock(side_effect=AssertionError("冲突 Match 不得进入订阅业务路径"))
search_chain._process_search_subscription = process_search
match_chain._match_subscription = match_subscription
worker = threading.Thread(
target=search_chain.search,
kwargs={"sid": subscribe.id, "state": None},
)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
worker.start()
assert search_started.wait(timeout=3)
match_chain.match({"example.org": [Context()]})
release_search.set()
worker.join(timeout=3)
assert worker.is_alive() is False
match_subscription.assert_not_called()
def test_public_search_and_match_allow_different_subscriptions_in_parallel(monkeypatch) -> None:
"""Search 持有一条订阅时 Match 仍可处理另一条订阅。"""
_configure_channel_state(monkeypatch)
search_subscribe = _subscribe(8)
match_subscribe = _subscribe(9)
search_chain = object.__new__(SubscribeChain)
search_chain.subscription_repository = _repository(search_subscribe)
match_chain = object.__new__(SubscribeChain)
match_chain.subscription_repository = _repository(match_subscribe)
match_chain.get_states_for_search = lambda state: state
match_chain._prepare_match_torrents = lambda torrents: torrents
search_started = threading.Event()
release_search = threading.Event()
def process_search(item, _searchchain, *, execution_context):
"""阻塞 Search 以证明 Match 不依赖 Search 通道锁。"""
assert execution_context.lease.subscription_id == item.id
search_started.set()
assert release_search.wait(timeout=3)
return item
match_subscription = Mock()
search_chain._process_search_subscription = process_search
match_chain._match_subscription = match_subscription
worker = threading.Thread(
target=search_chain.search,
kwargs={"sid": search_subscribe.id, "state": None},
)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
worker.start()
assert search_started.wait(timeout=3)
match_chain.match({"example.org": [Context()]})
release_search.set()
worker.join(timeout=3)
assert worker.is_alive() is False
match_subscription.assert_called_once()
assert match_subscription.call_args.kwargs["subscribe"].id == match_subscribe.id
+136 -396
View File
@@ -1,9 +1,8 @@
"""订阅下载跨入口幂等、不确定终态与取消补偿测试。"""
"""订阅下载取消边界、普通失败语义与账本移除迁移测试。"""
from __future__ import annotations
import importlib
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
@@ -13,98 +12,22 @@ import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import app.chain.download.submission as download_submission
from app.application.download.admission import (
DownloadReconciliationRequired,
SubscriptionDownloadGovernance,
SubscriptionDownloadRequest,
)
from app.application.download.admission import SubscriptionDownloadGovernance
from app.application.subscription.contract import SubscriptionSnapshot
from app.application.subscription.execution import (
SubscriptionExecutionAdmission,
SubscriptionExecutionContext,
)
from app.chain.download import DownloadChain
from app.chain.subscribe import policy as subscribe_policy
from app.chain.subscribe.facade import SubscribeChain
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
from app.db.base import Base
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
from app.domain.context import Context, MediaInfo, TorrentInfo
from app.domain.metainfo import MetaInfo
from app.schemas.types import MediaSource, MediaType
def _request(key: str = "key-1", task_id: str | None = "task-1") -> SubscriptionDownloadRequest:
"""构造固定身份的提交认领请求。"""
return SubscriptionDownloadRequest(
idempotency_key=key,
legacy_idempotency_key=None,
subscription_id=7,
task_id=task_id,
logical_identity='{"subscription_id":7}',
resource_key="example.com:id=42",
coverage="episodes:E01-E03",
mode="normal",
delivery_scope='{"download_uri":"local:/downloads","downloader":"auto"}',
)
def _repository(tmp_path) -> tuple[TransactionalSubscriptionDownloadRepository, object]:
"""创建独立 SQLite 幂等账本仓储与 Session 工厂。"""
engine = create_engine(
f"sqlite:///{tmp_path / 'subscription-download.db'}",
connect_args={"check_same_thread": False},
)
Base.metadata.create_all(engine)
factory = sessionmaker(bind=engine)
return TransactionalSubscriptionDownloadRepository(factory), factory
def test_repository_claims_same_submission_once_across_workers(tmp_path) -> None:
"""并发入口对同一幂等键只能有一个取得下载器提交权。"""
repository, _factory = _repository(tmp_path)
with ThreadPoolExecutor(max_workers=2) as executor:
claims = list(executor.map(lambda _index: repository.claim(_request()), range(2)))
assert sum(claim.acquired for claim in claims) == 1
assert {claim.snapshot.state for claim in claims} == {"submitting"}
assert {claim.snapshot.attempt_count for claim in claims} == {1}
def test_repository_fences_retry_and_preserves_uncertain_terminal(tmp_path) -> None:
"""明确拒绝可延迟重试,过期令牌和待对账状态均不得重新提交。"""
repository, _factory = _repository(tmp_path)
first = repository.claim(_request())
token = first.snapshot.attempt_token
assert token
assert repository.mark_retryable(
idempotency_key="key-1",
attempt_token=token,
available_at="1970-01-01T00:00:00+00:00",
error="downloader rejected",
)
second = repository.claim(_request(task_id="task-2"))
second_token = second.snapshot.attempt_token
assert second.acquired
assert second_token and second_token != token
assert not repository.mark_succeeded(
idempotency_key="key-1",
attempt_token=token,
)
assert repository.mark_reconcile_required(
idempotency_key="key-1",
attempt_token=second_token,
error="transport timeout",
)
blocked = repository.claim(_request(task_id="task-3"))
assert not blocked.acquired
assert blocked.snapshot.state == "reconcile_required"
assert blocked.snapshot.attempt_count == 2
assert repository.has_started_for_task("task-2")
class _FakeTorrentHelper:
"""返回固定种子目录和文件清单,隔离真实 bencode 解析。"""
@@ -114,10 +37,9 @@ class _FakeTorrentHelper:
return "Demo.Show.S01", ["Demo.Show.S01E01.mkv"]
def _download_chain(repository) -> DownloadChain:
def _download_chain() -> DownloadChain:
"""构造只执行下载提交边界的 Chain 测试实例。"""
chain = DownloadChain.__new__(DownloadChain)
chain.subscription_download_repository = repository
chain.download_history_repository = MagicMock()
chain.download_history_repository.get_by_media_identity.return_value = []
chain.download_failure_repository = MagicMock()
@@ -174,12 +96,11 @@ def _submission_dependencies(monkeypatch):
)
def test_download_chain_reuses_success_without_second_downloader_call(tmp_path) -> None:
"""重叠入口在首个提交成功后复用 hash,不再次调用下载器"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
def test_download_chain_uses_normal_submission_for_each_execution() -> None:
"""下载边界不保留本地提交账本,由下载器处理相同 torrent 的复用"""
chain = _download_chain()
chain.download = MagicMock(return_value=("qb", "hash-1", "Original", "accepted"))
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
governance = SubscriptionDownloadGovernance()
first = chain.download_single(
context=_context(),
@@ -197,306 +118,91 @@ def test_download_chain_reuses_success_without_second_downloader_call(tmp_path)
)
assert first == second == "hash-1"
chain.download.assert_called_once()
chain._settle_download_success.assert_called_once()
assert chain.download.call_count == 2
assert chain._settle_download_success.call_count == 2
def test_download_chain_deduplicates_same_media_across_subscription_rows(tmp_path) -> None:
"""同媒体同覆盖同交付目标的重复订阅只允许一个下载器提交。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock(return_value=("qb", "hash-cross-row", "Original", "accepted"))
first = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
)
second = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
)
assert first == second == "hash-cross-row"
chain.download.assert_called_once()
chain._settle_download_success.assert_called_once()
def test_download_chain_keeps_distinct_delivery_targets_separate(tmp_path) -> None:
"""不同保存目标属于独立产品意图,不得跨记录误去重。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain._resolve_media_download_dir.side_effect = (
lambda *, save_path, **_kwargs: ("local", Path(save_path), None)
)
def test_download_chain_retries_after_downloader_exception() -> None:
"""下载器异常不冻结后续执行,下一轮仍按普通下载合同重新提交。"""
chain = _download_chain()
chain.download = MagicMock(side_effect=[
("qb", "hash-a", "Original", "accepted"),
("qb", "hash-b", "Original", "accepted"),
TimeoutError("response timeout"),
("qb", "hash-after-timeout", "Original", "accepted"),
])
first = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads/a",
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
)
second = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads/b",
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
)
assert (first, second) == ("hash-a", "hash-b")
assert chain.download.call_count == 2
def test_download_chain_keeps_distinct_downloaders_separate(tmp_path) -> None:
"""不同下载器属于独立交付策略,不得跨记录误去重。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock(side_effect=[
("qb-a", "hash-a", "Original", "accepted"),
("qb-b", "hash-b", "Original", "accepted"),
])
first = chain.download_single(
with pytest.raises(TimeoutError, match="response timeout"):
chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
downloader="qb-a",
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
governance=SubscriptionDownloadGovernance(),
)
second = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
downloader="qb-b",
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
)
assert (first, second) == ("hash-a", "hash-b")
assert chain.download.call_count == 2
def test_download_chain_keeps_distinct_episode_coverage_separate(tmp_path) -> None:
"""跨记录仅复用精确覆盖,新增目标集不得被已有子集吞掉。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock(side_effect=[
("qb", "hash-e1", "Original", "accepted"),
("qb", "hash-e12", "Original", "accepted"),
])
first = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
)
second = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1, 2},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
)
assert (first, second) == ("hash-e1", "hash-e12")
assert chain.download.call_count == 2
def test_download_chain_reuses_pre_004_ledger_key(tmp_path) -> None:
"""键升级后仍应读取 003A 同记录成功账本,避免版本升级造成重复下载。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
request = chain._build_subscription_download_request(
context=_context(),
episodes={1},
governance=governance,
delivery_scope=chain._subscription_delivery_scope(
downloader=None,
download_uri="local:/downloads",
),
)
legacy = repository.claim(
SubscriptionDownloadRequest(
idempotency_key=request.legacy_idempotency_key or "",
legacy_idempotency_key=None,
subscription_id=7,
task_id=None,
logical_identity='{"subscription_id":7}',
resource_key=request.resource_key,
coverage=request.coverage,
mode=request.mode,
delivery_scope="legacy",
)
)
token = legacy.snapshot.attempt_token or ""
assert repository.mark_accepted(
idempotency_key=legacy.snapshot.idempotency_key,
attempt_token=token,
downloader="qb",
download_hash="legacy-ledger-hash",
)
assert repository.mark_succeeded(
idempotency_key=legacy.snapshot.idempotency_key,
attempt_token=token,
)
chain.download = MagicMock()
result = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
governance=SubscriptionDownloadGovernance(),
)
assert result == "legacy-ledger-hash"
chain.download.assert_not_called()
assert result == "hash-after-timeout"
assert chain.download.call_count == 2
chain._settle_download_success.assert_called_once()
def test_download_chain_freezes_when_local_settlement_fails(tmp_path) -> None:
"""下载器接受而历史结算失败时进入待对账,后续入口不得盲重试"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
def test_download_chain_retries_after_local_settlement_failure() -> None:
"""本地结算异常直接失败,下一轮不依赖补偿状态即可重新执行"""
chain = _download_chain()
chain.download = MagicMock(return_value=("qb", "hash-2", "Original", "accepted"))
chain._settle_download_success.side_effect = RuntimeError("history unavailable")
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
chain._settle_download_success.side_effect = [RuntimeError("history unavailable"), None]
with pytest.raises(DownloadReconciliationRequired):
with pytest.raises(RuntimeError, match="history unavailable"):
chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
governance=SubscriptionDownloadGovernance(),
)
with pytest.raises(DownloadReconciliationRequired):
chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
chain.download.assert_called_once()
def test_download_chain_freezes_when_downloader_result_is_uncertain(tmp_path) -> None:
"""下载器调用抛错可能已产生副作用,重启后的新实例仍不得自动重试。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock(side_effect=TimeoutError("response timeout"))
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
with pytest.raises(DownloadReconciliationRequired):
chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
restarted = _download_chain(repository)
restarted.download = MagicMock()
with pytest.raises(DownloadReconciliationRequired):
restarted.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
restarted.download.assert_not_called()
def test_download_chain_delays_retry_after_explicit_rejection(tmp_path) -> None:
"""下载器明确拒绝且无 hash 时进入冷却,不把不确定和已拒绝混为一谈。"""
repository, factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected"))
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
first = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
second = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
assert first is None and second is None
chain.download.assert_called_once()
with factory() as session:
record = session.query(SubscriptionDownloadSubmission).one()
assert record.state == "retryable"
assert record.available_at > record.updated_at
def test_download_chain_reads_legacy_history_before_new_ledger(tmp_path) -> None:
"""迁移前同订阅同 torrent 同覆盖的成功历史仍能阻止重复提交。"""
repository, factory = _repository(tmp_path)
chain = _download_chain(repository)
chain.download = MagicMock()
chain.download_history_repository.get_by_media_identity.return_value = [
SimpleNamespace(
download_hash="legacy-hash",
torrent_name="Demo Show S01E01 1080p",
torrent_site="TestSite",
episodes="E01",
seasons="S01",
episode_group=None,
note={"source": 'Subscribe|{"id": 7}'},
)
]
result = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
governance=SubscriptionDownloadGovernance(),
)
assert result == "legacy-hash"
chain.download.assert_not_called()
with factory() as session:
assert session.query(SubscriptionDownloadSubmission).count() == 0
assert result == "hash-2"
assert chain.download.call_count == 2
assert chain._settle_download_success.call_count == 2
def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None:
def test_download_chain_records_explicit_rejection_as_normal_failure() -> None:
"""下载器明确拒绝仍写入既有资源失败冷却,不建立提交恢复状态。"""
chain = _download_chain()
chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected"))
result = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=SubscriptionDownloadGovernance(),
)
assert result is None
chain.download_failure_repository.record_failure.assert_called_once()
chain.post_message.assert_called_once()
def test_download_chain_cancels_before_external_side_effect() -> None:
"""取消在下载器边界前生效时不得创建下载任务或成功事实。"""
repository, _factory = _repository(tmp_path)
chain = _download_chain(repository)
chain = _download_chain()
chain.download = MagicMock()
governance = SubscriptionDownloadGovernance(
subscription_id=7,
mode="normal",
task_id="cancel-task",
cancelled=lambda: True,
)
governance = SubscriptionDownloadGovernance(cancelled=lambda: True)
result = chain.download_single(
context=_context(),
@@ -508,7 +214,34 @@ def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None:
assert result is None
chain.download.assert_not_called()
assert not repository.has_started_for_task("cancel-task")
chain._settle_download_success.assert_not_called()
def test_download_chain_marks_side_effect_boundary_before_downloader_call() -> None:
"""下载器调用前必须标记副作用起点,使晚到取消按真实执行结果收口。"""
chain = _download_chain()
order: list[str] = []
def download(**_kwargs):
"""记录下载器调用顺序并返回成功结果。"""
order.append("download")
return "qb", "hash-3", "Original", "accepted"
chain.download = download
governance = SubscriptionDownloadGovernance(
mark_started=lambda: order.append("started"),
)
result = chain.download_single(
context=_context(),
torrent_content=b"torrent",
episodes={1},
save_path="/downloads",
governance=governance,
)
assert result == "hash-3"
assert order == ["started", "download"]
def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -> None:
@@ -554,6 +287,18 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
lambda _subscribe, contexts: contexts,
)
monkeypatch.setattr(subscribe_policy, "DownloadChain", _FakeDownloadChain)
admission = SubscriptionExecutionAdmission()
lease = admission.try_acquire(
subscription_id=current.id,
operation="search",
ttl_seconds=60,
)
assert lease is not None
execution_context = SubscriptionExecutionContext(
lease=lease,
admission=admission,
task_id="search-task-7",
)
_downloads, lefts = chain._SubscribeChain__download_best_version_with_full_pack_first(
contexts=[context],
@@ -562,6 +307,7 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
mediakey="themoviedb:77",
save_path="/old",
downloader="old",
execution_context=execution_context,
)
assert lefts is fresh_missing
@@ -574,8 +320,10 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
assert captured["no_exists"] is fresh_missing
assert captured["save_path"] == "/current"
assert captured["downloader"] == "current"
assert captured["governance"].subscription_id == 7
assert captured["governance"].mode == "normal"
assert captured["governance"].cancelled() is False
captured["governance"].mark_started()
assert execution_context.download_started is True
assert admission.release(lease) is True
def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch) -> None:
@@ -610,58 +358,50 @@ def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch
batch_download.assert_not_called()
def test_subscription_download_migration_is_idempotent_and_reversible(tmp_path, monkeypatch) -> None:
"""3.0.21 迁移可重复升级,并只移除自身新增的兼容表。"""
engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}")
migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
with engine.begin() as connection:
operations = Operations(MigrationContext.configure(connection))
monkeypatch.setattr(migration, "op", operations)
migration.upgrade()
migration.upgrade()
inspector = sa.inspect(connection)
assert "subscriptiondownloadsubmission" in inspector.get_table_names()
indexes = {item["name"] for item in inspector.get_indexes("subscriptiondownloadsubmission")}
assert "ix_subscriptiondownloadsubmission_task_state" in indexes
migration.downgrade()
assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names()
def test_subscription_delivery_scope_migration_is_idempotent_and_reversible(
def test_subscription_download_ledger_removal_migration_is_reversible(
tmp_path,
monkeypatch,
) -> None:
"""3.0.23 为存量提交填充兼容范围,并支持重复升级和完整回滚"""
engine = create_engine(f"sqlite:///{tmp_path / 'delivery-migration.db'}")
base_migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
migration = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23")
"""3.0.25 删除旧账本;降级只恢复结构,不伪造已删除的运行状态"""
engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}")
create_ledger = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
add_delivery_scope = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23")
remove_ledger = importlib.import_module("database.versions.c8f2e6a1d4b9_3_0_25")
with engine.begin() as connection:
operations = Operations(MigrationContext.configure(connection))
monkeypatch.setattr(base_migration, "op", operations)
monkeypatch.setattr(migration, "op", operations)
base_migration.upgrade()
monkeypatch.setattr(create_ledger, "op", operations)
monkeypatch.setattr(add_delivery_scope, "op", operations)
monkeypatch.setattr(remove_ledger, "op", operations)
create_ledger.upgrade()
add_delivery_scope.upgrade()
connection.execute(sa.text(
"INSERT INTO subscriptiondownloadsubmission "
"(idempotency_key, subscription_id, logical_identity, resource_key, coverage, mode, "
"state, attempt_count, created_at, updated_at) VALUES "
"('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'succeeded', 1, 'now', 'now')"
"delivery_scope, state, attempt_count, created_at, updated_at) VALUES "
"('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'legacy', "
"'reconcile_required', 1, 'now', 'now')"
))
migration.upgrade()
migration.upgrade()
remove_ledger.upgrade()
remove_ledger.upgrade()
assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names()
assert connection.execute(sa.text(
"SELECT delivery_scope FROM subscriptiondownloadsubmission WHERE idempotency_key='legacy-key'"
)).scalar_one() == "legacy"
migration.downgrade()
columns = {
column["name"]
for column in sa.inspect(connection).get_columns("subscriptiondownloadsubmission")
remove_ledger.downgrade()
remove_ledger.downgrade()
inspector = sa.inspect(connection)
assert "subscriptiondownloadsubmission" in inspector.get_table_names()
assert {column["name"] for column in inspector.get_columns(
"subscriptiondownloadsubmission"
)} >= {"idempotency_key", "delivery_scope", "state", "download_hash"}
indexes = {
item["name"]
for item in inspector.get_indexes("subscriptiondownloadsubmission")
}
assert "delivery_scope" not in columns
def test_model_metadata_registers_submission_table() -> None:
"""显式模型注册必须让 fresh create_all 包含订阅提交账本。"""
assert SubscriptionDownloadSubmission.__tablename__ == "subscriptiondownloadsubmission"
assert "subscriptiondownloadsubmission" in Base.metadata.tables
assert indexes == {
"ix_subscriptiondownloadsubmission_subscription_state",
"ix_subscriptiondownloadsubmission_task_state",
}
assert connection.execute(sa.text(
"SELECT COUNT(*) FROM subscriptiondownloadsubmission"
)).scalar_one() == 0
@@ -0,0 +1,138 @@
"""订阅级进程内准入与显式执行上下文测试。"""
from concurrent.futures import ThreadPoolExecutor
from app.application.subscription.execution import (
SubscriptionExecutionAdmission,
SubscriptionExecutionContext,
)
def test_same_subscription_is_mutually_exclusive_across_channels() -> None:
"""同一订阅只能由 Search 或 Match 中的一条路径持有。"""
admission = SubscriptionExecutionAdmission()
search = admission.try_acquire(
subscription_id=7,
operation="search",
ttl_seconds=60,
)
match = admission.try_acquire(
subscription_id=7,
operation="match",
ttl_seconds=60,
)
assert search is not None
assert match is None
assert admission.release(search) is True
assert admission.try_acquire(
subscription_id=7,
operation="match",
ttl_seconds=60,
) is not None
def test_concurrent_channels_admit_only_one_owner_for_same_subscription() -> None:
"""Search 与 Match 同时申请同一订阅时只能有一个取得 owner。"""
admission = SubscriptionExecutionAdmission()
def acquire(operation: str):
"""并发申请固定订阅的通道所有权。"""
return admission.try_acquire(
subscription_id=11,
operation=operation,
ttl_seconds=60,
)
with ThreadPoolExecutor(max_workers=2) as executor:
leases = list(executor.map(acquire, ("search", "match")))
assert sum(lease is not None for lease in leases) == 1
def test_different_subscriptions_can_run_across_channels() -> None:
"""不同订阅可以分别占用 Search 与 Match 通道。"""
admission = SubscriptionExecutionAdmission()
search = admission.try_acquire(
subscription_id=7,
operation="search",
ttl_seconds=60,
)
match = admission.try_acquire(
subscription_id=8,
operation="match",
ttl_seconds=60,
)
assert search is not None
assert match is not None
assert search.owner_token != match.owner_token
def test_expired_owner_keeps_ownership_until_finally_release() -> None:
"""TTL 只请求 owner 协作退出,不允许后到任务强制接管活跃执行。"""
now = [100.0]
admission = SubscriptionExecutionAdmission(clock=lambda: now[0])
lease = admission.try_acquire(
subscription_id=9,
operation="search",
ttl_seconds=5,
)
assert lease is not None
now[0] = 106.0
assert admission.is_expired(lease) is True
assert admission.try_acquire(
subscription_id=9,
operation="match",
ttl_seconds=5,
) is None
assert admission.release(lease) is True
replacement = admission.try_acquire(
subscription_id=9,
operation="match",
ttl_seconds=5,
)
assert replacement is not None
assert admission.release(lease) is False
def test_execution_context_combines_cancel_ttl_and_download_boundary() -> None:
"""执行上下文独立承载取消、TTL、阶段与下载副作用状态。"""
now = [100.0]
cancelled = [False]
phases: list[tuple[str, int | None]] = []
admission = SubscriptionExecutionAdmission(clock=lambda: now[0])
lease = admission.try_acquire(
subscription_id=10,
operation="search",
ttl_seconds=5,
)
assert lease is not None
context = SubscriptionExecutionContext(
lease=lease,
admission=admission,
task_id="task-10",
cancel_requested=lambda: cancelled[0],
phase_changed=lambda phase, site_id: phases.append((phase, site_id)),
)
assert context.should_stop() is False
context.report_phase("searching", 3)
context.mark_download_started()
assert context.download_started is True
assert phases == [("searching", 3), ("submitting", None)]
cancelled[0] = True
assert context.is_cancel_requested() is True
assert context.should_stop() is True
cancelled[0] = False
now[0] = 106.0
assert context.is_expired() is True
assert context.should_stop() is True
+32 -41
View File
@@ -1,8 +1,8 @@
"""订阅执行状态合并、批次权限和操作能力测试。"""
import asyncio
from dataclasses import replace
from app.application.download.admission import SubscriptionDownloadSnapshot
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
from app.application.subscription.status import SubscriptionExecutionStatusService
@@ -35,31 +35,12 @@ def _task(
)
def _download(subscription_id: int, state: str) -> SubscriptionDownloadSnapshot:
"""构造一个比搜索任务更早的下载提交快照。"""
return SubscriptionDownloadSnapshot(
idempotency_key=f"key-{subscription_id}",
subscription_id=subscription_id,
task_id=f"task-{subscription_id}",
state=state,
attempt_count=1,
attempt_token="attempt",
downloader=None,
download_hash=None,
available_at=None,
last_error="downloader response uncertain" if state == "reconcile_required" else None,
created_at="2026-09-01T00:30:00+00:00",
updated_at="2026-09-01T00:59:00+00:00",
)
class _Repository:
"""保存测试快照的异步状态仓储。"""
def __init__(self) -> None:
"""初始化可由测试覆盖的快照集合。"""
self.tasks: dict[int, SearchTaskSnapshot] = {}
self.downloads: dict[int, SubscriptionDownloadSnapshot] = {}
self.batch = SearchBatchSnapshot(
batch_id="batch-1",
source="manual",
@@ -78,10 +59,6 @@ class _Repository:
"""返回请求范围内搜索任务。"""
return {key: value for key, value in self.tasks.items() if key in subscription_ids}
async def latest_download_submissions(self, subscription_ids):
"""返回请求范围内下载提交。"""
return {key: value for key, value in self.downloads.items() if key in subscription_ids}
async def list_batches(self, *, limit):
"""返回一个测试批次。"""
return [self.batch][:limit]
@@ -107,29 +84,14 @@ def test_execution_status_exposes_site_wait_and_cancel_capability():
assert statuses[1].can_cancel is True
def test_reconciliation_state_overrides_newer_search_terminal():
"""不确定下载副作用不得被稍晚写入的搜索失败掩盖"""
repository = _Repository()
repository.tasks[2] = _task(2, state="failed", phase="failed")
repository.downloads[2] = _download(2, "reconcile_required")
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,)))
assert statuses[2].state == "reconcile_required"
assert statuses[2].requires_reconciliation is True
assert statuses[2].can_retry is False
assert statuses[2].error == "downloader response uncertain"
def test_failed_search_exposes_safe_retryable_error():
"""搜索失败文本必须压平且仅声明安全重试能力。"""
def test_failed_search_exposes_safe_error():
"""搜索失败文本必须压平且不暴露内部错误细节"""
repository = _Repository()
repository.tasks[3] = _task(3, state="failed", phase="failed")
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((3,)))
assert statuses[3].state == "failed"
assert statuses[3].can_retry is True
assert statuses[3].error == "provider timeout"
@@ -149,6 +111,35 @@ def test_batch_requires_complete_subscription_access():
assert visible.can_cancel is True
def test_batch_projection_exposes_skipped_count_as_processed_without_success():
"""批次跳过应计入处理总数,同时保留独立的完成计数。"""
repository = _Repository()
repository.batch = replace(
repository.batch,
state="skipped",
total_count=2,
finished_count=1,
skipped_count=1,
)
repository.tasks = {
1: _task(1, state="completed", phase="completed"),
2: _task(2, state="skipped", phase="skipped"),
}
visible = asyncio.run(
SubscriptionExecutionStatusService(repository).get_batch(
"batch-1",
accessible_subscription_ids={1, 2},
)
)
assert visible is not None
assert visible.state == "skipped"
assert visible.finished_count == 1
assert visible.skipped_count == 1
assert visible.processed_count == 2
def test_request_cancel_uses_injected_execution_boundary():
"""取消必须通过组合根注入的异步执行边界并返回真实结果。"""
repository = _Repository()
@@ -60,8 +60,6 @@ def test_cooled_site_does_not_block_independent_site_or_hide_batch_failure():
owner="fallback-task",
cancelled=lambda: False,
stop_state=ProcessStopState(),
max_wait_seconds=0,
random_uniform=lambda _low, _high: 60.0,
)
chain = object.__new__(SearchChain)
chain._runtime_config = SimpleNamespace(search_threadpool_size=2)
+6 -1
View File
@@ -8,7 +8,7 @@ from types import SimpleNamespace
import pytest
from app.application.subscription.contract import SubscriptionSnapshot
from app.chain.subscribe import SubscribeChain
from app.chain.subscribe.facade import SubscribeChain
from app.domain.context import Context, MediaInfo, TorrentInfo
from app.domain.metainfo import MetaInfo
from app.schemas.mediaserver import NotExistMediaInfo
@@ -42,11 +42,16 @@ class _ReplaySubscriptionListRepository:
def __init__(self, subscribes: list[SubscriptionSnapshot]) -> None:
self.subscribes = subscribes
self._by_id = {subscribe.id: subscribe for subscribe in subscribes}
def list(self, _state: str = None) -> list[SubscriptionSnapshot]:
"""返回当前批次的全部订阅快照。"""
return self.subscribes
def get(self, subscribe_id: int) -> SubscriptionSnapshot | None:
"""返回取得订阅准入后的最新快照。"""
return self._by_id.get(subscribe_id)
class _ReplayTorrentHelper:
"""让无关候选稳定停在身份冲突边界。"""
+286 -2
View File
@@ -1,14 +1,298 @@
"""订阅执行治理最终受控规模门禁。"""
from scripts.validation.subscription_governance_scale import run_acceptance
import threading
import pytest
from app.application.site.observation import capture_site_search_observation
from app.application.subscription.candidates import CandidateIndex
from app.application.subscription.sitebudget import SiteBudgetClaim
from app.chain.search.facade import SearchChain
from app.chain.subscribe.facade import SubscribeChain
from scripts.validation import subscription_governance_scale as scale
from scripts.validation.subscription_governance_scale import (
ScaleCase,
_run_durable_governance,
_run_match_execution_case,
run_acceptance,
)
def test_subscription_governance_controlled_scale_matrix() -> None:
"""两档最终矩阵必须同时满足正确性、压力、恢复和幂等门禁。"""
"""两档最终矩阵必须同时满足正确性、压力、恢复和订阅准入门禁。"""
result = run_acceptance()
assert result["schema_version"] == 4
assert result["passed"] is True
assert all(result["gates"].values())
assert result["method"]["match_entrypoint"] == "SubscribeChain.match"
assert result["method"]["download_selection"] == "DownloadChain.batch_download"
assert result["method"]["fixed_external_boundaries"] == [
"TMDB",
"media_server",
"downloader",
]
assert result["method"]["fixed_persistence_boundaries"] == [
"subscription_repository",
"download_facts",
"subscription_progress",
"completion_side_effects",
]
assert result["method"]["fixed_policy_inputs"] == [
"site_mapping",
"system_configuration",
"subscription_filters",
"torrent_attribute_filter",
"download_preparation",
]
assert result["method"]["production_network_slo"] is False
assert result["gates"]["subscription_admission_serializes"] is True
assert result["gates"]["match_execution_candidate_sets_equal"] is True
assert result["gates"]["match_execution_download_sets_equal"] is True
assert result["gates"]["match_execution_missing_sets_equal"] is True
assert result["gates"]["match_execution_completion_sets_equal"] is True
assert [case["subscription_count"] for case in result["match_cases"]] == [100, 200]
assert [case["site_count"] for case in result["match_cases"]] == [10, 20]
assert min(case["candidate_count"] for case in result["match_cases"]) >= 1000
assert [
case["candidate_check_reduction_percent"]
for case in result["match_cases"]
] == [99.0, 99.0]
assert [
case["matched_candidates"]["actual_count"]
for case in result["match_execution_cases"]
] == [1000, 2400]
assert [
case["downloaded_candidates"]["actual_count"]
for case in result["match_execution_cases"]
] == [100, 200]
assert all(
case["downloaded_candidates"]["duplicate_count"] == 0
and case["completed_subscriptions"]["duplicate_count"] == 0
for case in result["match_execution_cases"]
)
assert [
case["site_peak_inflight_per_site"]
for case in result["durable_cases"]
] == [1, 1]
assert all(
case["site_request_boundary_active"] == 0
and case["site_request_boundary_peak"] == 1
and case["site_pressure_owner_count"] == 2
and case["site_pressure_concurrency_verified"]
and case["site_pressure_success_release_reused"]
and case["site_pressure_error_observation_cooled"]
and case["site_pressure_valid"]
for case in result["durable_cases"]
)
assert all(
observation["request_active"] == 0
and observation["request_peak"] == 1
and observation["owners_finished"] is True
and observation["request_active_at_rejection"] == 1
and observation["budget_rejections"] == 1
for case in result["durable_cases"]
for observation in case["site_observations"]
)
@pytest.mark.parametrize("mutation", ["bypass", "early_release", "allow_concurrent"])
def test_scale_validator_rejects_fake_site_pressure(monkeypatch, tmp_path, mutation):
"""绕过 wrapper、提前释放或并发放行时,请求边界峰值必须暴露门禁失效。"""
if mutation == "bypass":
def bypass_wrapper(self, *, site, keyword, mtype, page):
"""模拟绕过预算 wrapper 的请求路径。"""
return self.search_site_torrents(
site=site,
keyword=keyword,
mtype=mtype,
page=page,
)
monkeypatch.setattr(
SearchChain,
"_search_site_torrents_with_budget",
bypass_wrapper,
)
elif mutation == "early_release":
release_guard = threading.Lock()
def early_release_wrapper(self, *, site, keyword, mtype, page):
"""模拟请求边界前错误释放站点租约。"""
budget = self._subscription_site_budget
# 将错误释放本身串行化,避免第二个 owner 在首个释放动作完成前
# 被正常拒绝,从而掩盖“请求期间已失去租约”的变异。
with release_guard:
claim = budget.acquire(site["id"])
with capture_site_search_observation() as observation:
budget.finish(claim, observation)
return self.search_site_torrents(
site=site,
keyword=keyword,
mtype=mtype,
page=page,
)
monkeypatch.setattr(
SearchChain,
"_search_site_torrents_with_budget",
early_release_wrapper,
)
else:
original_claim = scale.TransactionalSubscriptionSearchRepository.claim_site
def allow_concurrent_claim(self, *, site_id, owner, lease_seconds):
"""模拟仓储错误地向第二 owner 发放同站点租约。"""
claim = original_claim(
self,
site_id=site_id,
owner=owner,
lease_seconds=lease_seconds,
)
if claim.acquired:
return claim
return SiteBudgetClaim(
site_id=site_id,
acquired=True,
retry_at=claim.retry_at,
consecutive_failures=claim.consecutive_failures,
lease_token=f"mutant-{owner}",
)
monkeypatch.setattr(
scale.TransactionalSubscriptionSearchRepository,
"claim_site",
allow_concurrent_claim,
)
result = _run_durable_governance(
ScaleCase(f"mutant-{mutation}", 2, 1, 2, 1),
tmp_path,
)
assert result["site_pressure_valid"] is False
assert result["site_peak_inflight_per_site"] >= 2
assert result["site_request_boundary_peak"] >= 2
def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path):
"""失败已登记但 wrapper 尚未返回时,压力门禁必须拒绝并暴露未收口 owner。"""
original_wrapper = SearchChain._search_site_torrents_with_budget
release_stalled = threading.Event()
threads_before = set(threading.enumerate())
def stall_after_rejection(self, *, site, keyword, mtype, page):
"""模拟预算拒绝已记录、调用方却未取得返回值的挂起路径。"""
original_record_failure = self.record_subscription_site_budget_failure
def record_failure_and_stall(error: str) -> None:
"""在拒绝已登记后阻塞原始 wrapper 的返回。"""
original_record_failure(error)
if "冷却或已有在途搜索" in error:
release_stalled.wait()
self.record_subscription_site_budget_failure = record_failure_and_stall
try:
return original_wrapper(
self,
site=site,
keyword=keyword,
mtype=mtype,
page=page,
)
finally:
self.record_subscription_site_budget_failure = original_record_failure
monkeypatch.setattr(
SearchChain,
"_search_site_torrents_with_budget",
stall_after_rejection,
)
monkeypatch.setattr(scale, "_SITE_PRESSURE_SYNC_TIMEOUT", 0.05)
try:
result = _run_durable_governance(
ScaleCase("mutant-unfinished", 2, 1, 2, 1),
tmp_path,
)
assert result["site_pressure_valid"] is False
assert result["site_pressure_concurrency_verified"] is False
assert result["site_observations"][0]["owners_finished"] is False
finally:
release_stalled.set()
for thread in set(threading.enumerate()) - threads_before:
thread.join(timeout=1)
def test_scale_validator_rejects_candidate_loss(monkeypatch) -> None:
"""候选路由少返回任意资源时,完整匹配集合门禁必须失败。"""
original_route = CandidateIndex.route_for_match
def route_with_loss(self, *args, **kwargs):
routed = original_route(self, *args, **kwargs)
for domain, contexts in routed.items():
if contexts:
return {**routed, domain: contexts[1:]}
return routed
monkeypatch.setattr(CandidateIndex, "route_for_match", route_with_loss)
result = _run_match_execution_case(ScaleCase("candidate-loss", 4, 2, 8, 4))
assert result["matched_candidates"]["equal"] is False
assert result["matched_candidates"]["missing_sample"]
def test_scale_validator_rejects_duplicate_download_and_completion(monkeypatch) -> None:
"""同订阅重复下载或错误完成时,多重集合门禁必须失败。"""
from scripts.validation import subscription_governance_scale as scale
original_download = scale._ScaleDownloadBoundary.download_single
def duplicate_download(self, context, *, governance, **kwargs):
result = original_download(
self,
context,
governance=governance,
**kwargs,
)
original_download(
self,
context,
governance=governance,
**kwargs,
)
return result
def finish_every_subscription(
self,
*,
subscribe,
meta,
mediainfo,
**_kwargs,
):
self._SubscribeChain__finish_subscribe(
subscribe=subscribe,
meta=meta,
mediainfo=mediainfo,
)
monkeypatch.setattr(
scale._ScaleDownloadBoundary,
"download_single",
duplicate_download,
)
monkeypatch.setattr(
SubscribeChain,
"finish_subscribe_or_not",
finish_every_subscription,
)
result = _run_match_execution_case(ScaleCase("duplicate-effects", 4, 2, 8, 4))
assert result["downloaded_candidates"]["equal"] is False
assert result["downloaded_candidates"]["duplicate_count"] == 4
assert result["completed_subscriptions"]["equal"] is False
assert result["completed_subscriptions"]["unexpected_sample"]
+231
View File
@@ -0,0 +1,231 @@
"""订阅 Match 通道的逐订阅准入与异常隔离测试。"""
import threading
from dataclasses import replace
from types import SimpleNamespace
from unittest.mock import Mock
from app.application.subscription.contract import SubscriptionSnapshot
from app.application.subscription.execution import SubscriptionExecutionAdmission
from app.chain.subscribe import match as subscribe_match
from app.chain.subscribe.facade import SubscribeChain
from app.schemas.types import MediaType
def _subscribe(subscribe_id: int, *, name: str | None = None) -> SubscriptionSnapshot:
"""构造 Match 编排所需的活动电影订阅快照。"""
return SubscriptionSnapshot(
id=subscribe_id,
name=name or f"匹配电影 {subscribe_id}",
year="2026",
type=MediaType.MOVIE.value,
media_source="themoviedb",
media_id=str(2000 + subscribe_id),
state="R",
)
def _chain(
listed: list[SubscriptionSnapshot],
current: dict[int, SubscriptionSnapshot] | None = None,
) -> SubscribeChain:
"""构造只执行 Match 编排层的订阅链。"""
snapshots = current or {subscribe.id: subscribe for subscribe in listed}
chain = object.__new__(SubscribeChain)
chain.subscription_repository = SimpleNamespace(
list=lambda _state: list(listed),
get=snapshots.get,
)
chain.get_states_for_search = lambda state: state
chain._match_lock = threading.Lock()
chain._search_queue_lock = threading.Lock()
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
chain._prepare_match_torrents = lambda torrents: torrents
return chain
def _assert_channel_and_subscription_released(
chain: SubscribeChain,
subscription_ids: tuple[int, ...],
) -> None:
"""验证 Match 通道锁和所有订阅 owner 均已释放。"""
assert chain._match_lock.acquire(blocking=False) is True
chain._match_lock.release()
for subscription_id in subscription_ids:
lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscription_id,
operation="search",
ttl_seconds=60,
)
assert lease is not None
assert chain._subscription_execution_admission.release(lease) is True
def test_match_skips_subscription_owned_by_search() -> None:
"""Search 已持有同一订阅时 Match 本轮直接跳过。"""
subscribe = _subscribe(1)
chain = _chain([subscribe])
process = Mock(side_effect=AssertionError("冲突订阅不应进入 Match"))
chain._match_subscription = process
search_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="search",
ttl_seconds=60,
)
assert search_lease is not None
chain.match({"example.org": []})
process.assert_not_called()
assert chain._match_lock.acquire(blocking=False) is True
chain._match_lock.release()
assert chain._subscription_execution_admission.release(search_lease) is True
def test_match_reloads_subscription_after_admission() -> None:
"""Match 取得 owner 后使用最新订阅快照并传入独立执行上下文。"""
listed = _subscribe(2, name="旧快照")
current = replace(listed, name="最新快照")
chain = _chain([listed], {listed.id: current})
captured = []
def process(**kwargs):
"""记录 Match 单订阅执行入口收到的快照与上下文。"""
captured.append(kwargs)
chain._match_subscription = process
chain.match({"example.org": []})
assert len(captured) == 1
assert captured[0]["subscribe"] is current
context = captured[0]["execution_context"]
assert context.lease.subscription_id == listed.id
assert context.lease.operation == "match"
_assert_channel_and_subscription_released(chain, (listed.id,))
def test_match_isolates_one_subscription_failure_and_releases_all_owners() -> None:
"""单条 Match 异常不阻止后续订阅,且所有 owner 最终释放。"""
first = _subscribe(3)
second = _subscribe(4)
chain = _chain([first, second])
processed = []
def process(**kwargs):
"""让首条订阅失败并记录后续订阅仍被执行。"""
subscribe = kwargs["subscribe"]
processed.append(subscribe.id)
if subscribe.id == first.id:
raise RuntimeError("candidate failure")
chain._match_subscription = process
chain.match({"example.org": []})
assert processed == [first.id, second.id]
_assert_channel_and_subscription_released(chain, (first.id, second.id))
def test_match_continues_when_latest_subscription_read_fails() -> None:
"""单条最新快照读取失败不能泄漏 owner 或中止后续 Match。"""
first = _subscribe(5)
second = _subscribe(6)
chain = _chain([first, second])
processed = []
def get(subscription_id: int):
"""模拟首条读取异常并返回第二条当前快照。"""
if subscription_id == first.id:
raise RuntimeError("repository failure")
return second
chain.subscription_repository.get = get
chain._match_subscription = lambda **kwargs: processed.append(kwargs["subscribe"].id)
chain.match({"example.org": []})
assert processed == [second.id]
_assert_channel_and_subscription_released(chain, (first.id, second.id))
def test_match_skips_subscription_paused_after_admission() -> None:
"""取得准入后订阅变为暂停态时,不应继续访问候选或下载边界。"""
listed = _subscribe(7)
paused = replace(listed, state="S")
chain = _chain([listed], {listed.id: paused})
process = Mock(side_effect=AssertionError("暂停订阅不应进入 Match"))
chain._match_subscription = process
progress = Mock()
chain.match({"example.org": []}, progress_callback=progress)
process.assert_not_called()
final_data = progress.call_args.kwargs["data"]
assert final_data == {
"total": 1,
"finished": 1,
"completed": 0,
"skipped": 1,
"failed": 0,
}
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅跳过"
_assert_channel_and_subscription_released(chain, (listed.id,))
def test_match_progress_reports_completed_skipped_and_failed_counts() -> None:
"""Match 最终进度必须区分正常执行、准入跳过和单订阅失败。"""
completed = _subscribe(8)
skipped = _subscribe(9)
failed = _subscribe(10)
chain = _chain([completed, skipped, failed])
chain._match_subscription = Mock(
side_effect=["completed", RuntimeError("candidate failure")]
)
skipped_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=skipped.id,
operation="search",
ttl_seconds=60,
)
assert skipped_lease is not None
progress = Mock()
chain.match({"example.org": []}, progress_callback=progress)
final_data = progress.call_args.kwargs["data"]
assert final_data == {
"total": 3,
"finished": 3,
"completed": 1,
"skipped": 1,
"failed": 1,
}
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅失败"
assert chain._subscription_execution_admission.release(skipped_lease) is True
_assert_channel_and_subscription_released(chain, (completed.id, failed.id))
def test_match_stop_does_not_report_unvisited_subscriptions_as_completed(monkeypatch) -> None:
"""系统停止后未访问的订阅不得计入完成或跳过。"""
subscribes = [_subscribe(11), _subscribe(12)]
chain = _chain(subscribes)
progress = Mock()
monkeypatch.setattr(
subscribe_match,
"runtime_stop_state",
SimpleNamespace(is_system_stopped=True),
)
chain.match({"example.org": []}, progress_callback=progress)
assert progress.call_args.kwargs == {
"value": 100,
"text": "订阅资源匹配已停止,部分订阅未执行",
"data": {
"total": 2,
"finished": 0,
"completed": 0,
"skipped": 0,
"failed": 0,
},
}
+469 -14
View File
@@ -1,17 +1,24 @@
"""订阅搜索队列接入、锁隔离和批次失败治理测试。"""
import threading
from dataclasses import replace
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
from unittest.mock import Mock, patch
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from app.application.site.observation import report_site_search_outcome
from app.application.subscription.contract import SubscriptionSnapshot
from app.application.subscription.execution import SubscriptionExecutionAdmission
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
from app.chain.search.facade import SearchChain
from app.chain.subscribe.facade import SubscribeChain
from app.chain.subscribe.search import _search_task_available_at
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
from app.db.base import Base
from app.modules.indexer import IndexerModule
from app.schemas.types import MediaType
@@ -66,27 +73,133 @@ def _chain(tmp_path, subscribes: list[SubscriptionSnapshot]):
sessionmaker(bind=engine)
)
chain.get_states_for_search = lambda state: state
chain._rlock = _ForbiddenLock()
chain._match_lock = _ForbiddenLock()
chain._search_queue_lock = threading.Lock()
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
return chain
def _make_tasks_ready(monkeypatch) -> None:
"""让治理测试中的持久任务立即到期,避免依赖真实随机时钟。"""
ready_at = "1970-01-01T00:00:00+00:00"
monkeypatch.setattr(
"app.chain.subscribe.search._search_task_available_at",
lambda _source, subscription_ids: {
subscription_id: ready_at for subscription_id in subscription_ids
},
)
def test_fallback_task_schedule_staggers_each_subscription(monkeypatch):
"""兜底批次首条抖动后,每条后续订阅都按独立随机间隔到期。"""
now = datetime(2026, 9, 3, 1, 2, 3, tzinfo=timezone.utc)
delays = iter((12, 60, 300))
monkeypatch.setattr(
"app.chain.subscribe.search.random.randint",
lambda _low, _high: next(delays),
)
schedule = _search_task_available_at(
"fallback",
(1, 2, 3),
now=now,
)
available = [datetime.fromisoformat(schedule[subscribe_id]) for subscribe_id in (1, 2, 3)]
assert (available[0] - now).total_seconds() == 12
assert (available[1] - available[0]).total_seconds() == 60
assert (available[2] - available[1]).total_seconds() == 300
def test_inline_fallback_search_preserves_site_pressure_stagger(tmp_path, monkeypatch):
"""无持久队列的兼容宿主仍须在每条自动兜底订阅前错峰。"""
subscribes = [_subscribe(20), _subscribe(21)]
chain = _chain(tmp_path, subscribes)
del chain.subscription_search_repository
waits = []
delays = iter((60, 300))
monkeypatch.setattr(
"app.chain.subscribe.search.random.randint",
lambda _low, _high: next(delays),
)
monkeypatch.setattr("app.chain.subscribe.search.time.sleep", waits.append)
monkeypatch.setattr(
chain,
"_process_search_subscription",
lambda item, _searchchain, **_kwargs: item,
)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.search(state="R")
assert waits == [60, 300]
def test_successful_sites_remain_available_to_next_due_subscription(tmp_path, monkeypatch):
"""正常站点请求不能让同批下一条到期订阅漏掉目标站点。"""
subscribes = [_subscribe(40), _subscribe(41)]
chain = _chain(tmp_path, subscribes)
_make_tasks_ready(monkeypatch)
searchchain = object.__new__(SearchChain)
searchchain.configure_subscription_site_budget(None)
current_subscription_id = 0
calls = []
def search_site_torrents(*, site, **_kwargs):
"""记录真实发出的站点请求并发布正常完成观察结果。"""
calls.append((current_subscription_id, site["id"]))
report_site_search_outcome(attempted=True, outcome="success")
return [f"torrent-{current_subscription_id}-{site['id']}"]
def process(subscribe, current_searchchain, **_kwargs):
"""让每条到期订阅访问同一组完整目标站点。"""
nonlocal current_subscription_id
current_subscription_id = subscribe.id
for site_id in (11, 12):
current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access
site={"id": site_id, "name": f"Site {site_id}"},
keyword=subscribe.name,
mtype=MediaType.MOVIE,
page=0,
)
return subscribe
searchchain.search_site_torrents = search_site_torrents
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
batch_id = chain.search(state="R")
assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)]
assert chain.get_search_batch(batch_id).state == "completed"
def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch):
"""R/P 兜底搜索在持久队列中执行,不受日常 Match 长锁阻塞。"""
subscribes = [_subscribe(1), _subscribe(2)]
chain = _chain(tmp_path, subscribes)
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
_make_tasks_ready(monkeypatch)
processed = []
def process(subscribe, _searchchain, *, execution_context):
"""记录每条任务独立的订阅执行上下文。"""
processed.append((subscribe.id, execution_context))
return subscribe
monkeypatch.setattr(
chain,
"_process_search_subscription",
lambda subscribe, _searchchain: processed.append(subscribe.id) or subscribe,
process,
)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert processed == [1, 2]
assert [subscribe_id for subscribe_id, _context in processed] == [1, 2]
assert [context.lease.subscription_id for _subscribe_id, context in processed] == [1, 2]
assert len({context.task_id for _subscribe_id, context in processed}) == 2
assert len({id(context) for _subscribe_id, context in processed}) == 2
assert batch.state == "completed"
assert batch.finished_count == 2
assert batch.failed_count == 0
@@ -96,10 +209,10 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
"""单订阅异常不得中止批次后续任务,聚合终态必须暴露失败。"""
subscribes = [_subscribe(3), _subscribe(4)]
chain = _chain(tmp_path, subscribes)
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
_make_tasks_ready(monkeypatch)
processed = []
def process(subscribe, _searchchain):
def process(subscribe, _searchchain, **_kwargs):
"""让首条失败并保持第二条正常完成。"""
processed.append(subscribe.id)
if subscribe.id == 3:
@@ -118,21 +231,218 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
assert batch.failed_count == 1
assert batch.last_error == "provider timeout"
for subscribe_id in (3, 4):
lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe_id,
operation="match",
ttl_seconds=60,
)
assert lease is not None
assert chain._subscription_execution_admission.release(lease) is True
def test_late_cancel_completes_when_download_submission_already_started(tmp_path, monkeypatch):
def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch):
"""索引器吞错后仍须失败收口当前任务,并继续处理后续订阅。"""
subscribes = [_subscribe(13), _subscribe(14)]
chain = _chain(tmp_path, subscribes)
_make_tasks_ready(monkeypatch)
searchchain = object.__new__(SearchChain)
attempts = 0
processed = []
def execute_search(_site, _request):
"""首条站点请求模拟被 IndexerModule 吞掉的 HTTP 429。"""
nonlocal attempts
attempts += 1
if attempts == 1:
raise RuntimeError("HTTP 429")
return False, []
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__search_check",
staticmethod(lambda _site, _keyword=None: True),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__execute_search",
staticmethod(execute_search),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__indexer_statistic",
staticmethod(lambda **_kwargs: None),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__parse_result",
staticmethod(lambda **_kwargs: []),
)
searchchain.search_site_torrents = object.__new__(IndexerModule).search_torrents
def process(subscribe, current_searchchain, **_kwargs):
"""让每条任务经过同一预算包装,并把观察到的站点失败抛给队列。"""
processed.append(subscribe.id)
current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access
site={
"id": 31 if subscribe.id == 13 else 32,
"name": "Flaky" if subscribe.id == 13 else "Healthy",
},
keyword=subscribe.name,
mtype=MediaType.MOVIE,
page=0,
)
failures = current_searchchain.consume_subscription_site_budget_failures()
if failures:
raise RuntimeError("".join(failures))
return subscribe
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert processed == [13, 14]
assert batch.state == "failed"
assert batch.finished_count == 1
assert batch.failed_count == 1
assert "Flaky" in batch.last_error
assert "HTTP 429" in batch.last_error
def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch):
"""Match 已持有同一订阅时,Search 本轮完成为跳过且不进入业务处理。"""
subscribe = _subscribe(6)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(side_effect=AssertionError("冲突订阅不应进入 Search"))
monkeypatch.setattr(chain, "_process_search_subscription", process)
match_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert match_lease is not None
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
process.assert_not_called()
assert batch.state == "skipped"
assert batch.finished_count == 0
assert batch.skipped_count == 1
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
assert chain._subscription_execution_admission.release(match_lease) is True
def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch):
"""准入后重新读取到暂停订阅时不得开始搜索或伪造完成。"""
subscribe = _subscribe(9)
paused = replace(subscribe, state="S")
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
chain.subscription_repository.get = Mock(side_effect=(subscribe, paused))
process = Mock(side_effect=AssertionError("暂停订阅不应进入 Search"))
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
process.assert_not_called()
assert batch.state == "skipped"
assert batch.finished_count == 0
assert batch.skipped_count == 1
assert batch.last_error == "订阅已暂停,本轮搜索已跳过"
def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch):
"""站点预算和状态清理都失败时仍必须释放当前订阅 owner。"""
subscribe = replace(_subscribe(7), state="N")
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
monkeypatch.setattr(
chain,
"_process_search_subscription",
lambda item, _searchchain, **_kwargs: item,
)
monkeypatch.setattr(
chain,
"_SubscribeChain__apply_subscribe_update",
Mock(side_effect=RuntimeError("state cleanup failed")),
)
searchchain = Mock()
def configure_site_budget(budget):
"""仅让释放站点预算的清理步骤失败。"""
if budget is None:
raise RuntimeError("budget cleanup failed")
searchchain.configure_subscription_site_budget.side_effect = configure_site_budget
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
batch_id = chain.search(state="N")
assert chain.get_search_batch(batch_id).state == "completed"
replacement = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert replacement is not None
assert chain._subscription_execution_admission.release(replacement) is True
def test_search_state_reset_stays_inside_subscription_admission(tmp_path, monkeypatch):
"""N 到 R 的本地状态写回完成前不能让 Match 接管同一订阅。"""
subscribe = replace(_subscribe(8), state="N")
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
monkeypatch.setattr(
chain,
"_process_search_subscription",
lambda item, _searchchain, **_kwargs: item,
)
competing_leases = []
def apply_update(*_args, **_kwargs):
"""在状态写回时探测同一订阅仍由 Search 持有。"""
competing_leases.append(
chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
)
monkeypatch.setattr(chain, "_SubscribeChain__apply_subscribe_update", apply_update)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.search(state="N")
assert competing_leases == [None]
replacement = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert replacement is not None
assert chain._subscription_execution_admission.release(replacement) is True
def test_late_cancel_completes_when_download_side_effect_already_started(tmp_path, monkeypatch):
"""取消晚于下载器副作用边界时按真实结果完成,不能伪装成未执行取消。"""
subscribe = _subscribe(5)
chain = _chain(tmp_path, [subscribe])
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
chain.subscription_download_repository = SimpleNamespace(
has_started_for_task=lambda _task_id: False,
)
_make_tasks_ready(monkeypatch)
queue = chain.subscription_search_repository
cancel_checks = iter((False, True))
monkeypatch.setattr(queue, "is_cancel_requested", lambda _task_id: next(cancel_checks))
def process(item, _searchchain):
def process(item, _searchchain, *, execution_context):
"""模拟当前任务复用或完成下载后才收到取消。"""
chain._mark_subscription_download_started()
execution_context.mark_download_started()
return item
monkeypatch.setattr(chain, "_process_search_subscription", process)
@@ -144,3 +454,148 @@ def test_late_cancel_completes_when_download_submission_already_started(tmp_path
assert batch.state == "completed"
assert batch.finished_count == 1
assert batch.cancelled_count == 0
def test_system_stop_requeues_task_after_search_returns(tmp_path, monkeypatch):
"""系统停机应阻止后续副作用,并把未完成任务退回可恢复队列。"""
subscribe = _subscribe(10)
chain = _chain(tmp_path, [subscribe])
stop_state = SimpleNamespace(is_system_stopped=False)
chain.stop_state = stop_state
_make_tasks_ready(monkeypatch)
def process(item, _searchchain, *, execution_context):
"""模拟站点搜索返回部分候选时进程进入停止阶段。"""
stop_state.is_system_stopped = True
assert execution_context.should_stop() is True
return item
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert batch.state == "queued"
assert batch.finished_count == 0
assert batch.failed_count == 0
assert batch.cancelled_count == 0
assert batch.skipped_count == 0
stop_state.is_system_stopped = False
recovered = chain.subscription_search_repository.claim_next(owner="worker-after-restart")
assert recovered is not None
assert recovered.subscription_id == subscribe.id
assert recovered.attempt_count == 2
def test_system_stop_completes_when_download_side_effect_already_started(tmp_path, monkeypatch):
"""停机晚于下载器副作用边界时必须终结任务,避免重启后重复提交。"""
subscribe = _subscribe(11)
chain = _chain(tmp_path, [subscribe])
stop_state = SimpleNamespace(is_system_stopped=False)
chain.stop_state = stop_state
_make_tasks_ready(monkeypatch)
def process(item, _searchchain, *, execution_context):
"""模拟下载器已接收任务后进程进入停止阶段。"""
execution_context.mark_download_started()
stop_state.is_system_stopped = True
return item
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert batch.state == "completed"
assert batch.finished_count == 1
assert batch.failed_count == 0
assert batch.cancelled_count == 0
assert batch.skipped_count == 0
assert chain.subscription_search_repository.claim_next(owner="worker-after-restart") is None
def test_ttl_expiry_after_normal_return_marks_task_and_batch_failed(tmp_path, monkeypatch):
"""正常返回也必须检查执行 TTL,未提交下载时按失败收口。"""
subscribe = _subscribe(15)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
monkeypatch.setattr(
chain._subscription_execution_admission,
"is_expired",
lambda _lease: True,
)
monkeypatch.setattr(
chain,
"_process_search_subscription",
lambda item, _searchchain, **_kwargs: item,
)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert batch.state == "failed"
assert batch.finished_count == 0
assert batch.failed_count == 1
assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间"
def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch):
"""TTL 晚于下载器副作用边界时按实际结果完成,避免下轮重复提交。"""
subscribe = _subscribe(16)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
monkeypatch.setattr(
chain._subscription_execution_admission,
"is_expired",
lambda _lease: True,
)
def process(item, _searchchain, *, execution_context):
"""模拟下载器已接收任务后搜索链正常返回。"""
execution_context.mark_download_started()
return item
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert batch.state == "completed"
assert batch.finished_count == 1
assert batch.failed_count == 0
assert batch.cancelled_count == 0
def test_site_budget_ttl_expiry_marks_task_and_batch_failed(tmp_path, monkeypatch):
"""站点预算观察到执行 TTL 到期时必须失败收口,不能伪装成用户取消。"""
subscribe = _subscribe(12)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
monkeypatch.setattr(
chain._subscription_execution_admission,
"is_expired",
lambda _lease: True,
)
def process(_item, _searchchain, *, execution_context):
"""模拟站点预算将协作超时传播为搜索取消异常。"""
assert execution_context.should_stop() is True
raise SubscriptionSearchCancelled("订阅搜索已取消")
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="R")
batch = chain.get_search_batch(batch_id)
assert batch.state == "failed"
assert batch.finished_count == 0
assert batch.failed_count == 1
assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间"
+68
View File
@@ -46,6 +46,42 @@ def test_search_queue_coalesces_active_subscription_and_raises_priority(tmp_path
assert first.task_id != second.task_id
def test_search_queue_claims_each_subscription_only_after_its_available_at(tmp_path):
"""逐订阅到期时间必须持久化,未到期任务不能占用同步 worker。"""
repository, _engine = _repository(tmp_path)
now = datetime.now(timezone.utc)
ready_at = (now - timedelta(seconds=1)).isoformat(timespec="seconds")
later_at = (now + timedelta(minutes=5)).isoformat(timespec="seconds")
repository.enqueue(
subscription_ids=(20, 21),
source="fallback",
priority=10,
available_at_by_subscription={20: ready_at, 21: later_at},
)
first = repository.claim_next(owner="worker-a")
assert first.subscription_id == 20
assert first.available_at == ready_at
assert repository.finish_task(
task_id=first.task_id,
lease_token=first.lease_token,
state="completed",
) is True
assert repository.claim_next(owner="worker-a") is None
repository.enqueue(
subscription_ids=(21,),
source="manual",
priority=100,
available_at_by_subscription={21: ready_at},
)
accelerated = repository.claim_next(owner="worker-b")
assert accelerated.subscription_id == 21
assert accelerated.available_at == ready_at
assert accelerated.priority == 100
def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path):
"""进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。"""
repository, engine = _repository(tmp_path)
@@ -160,6 +196,38 @@ def test_search_queue_finishes_batch_with_aggregated_failure(tmp_path):
assert batch.last_error == "site timeout"
def test_search_queue_aggregates_skipped_tasks_without_marking_success(tmp_path):
"""跳过任务应单独计数并让批次暴露 skipped 聚合终态。"""
repository, _engine = _repository(tmp_path)
enqueued = repository.enqueue(
subscription_ids=(10, 11),
source="fallback",
priority=10,
)
first = repository.claim_next(owner="worker-a")
assert repository.finish_task(
task_id=first.task_id,
lease_token=first.lease_token,
state="skipped",
error="同一订阅正在由其他通道处理,本轮搜索已跳过",
) is True
second = repository.claim_next(owner="worker-a")
assert repository.finish_task(
task_id=second.task_id,
lease_token=second.lease_token,
state="completed",
) is True
batch = repository.get_batch(enqueued.batch.batch_id)
assert batch.state == "skipped"
assert batch.finished_count == 1
assert batch.failed_count == 0
assert batch.cancelled_count == 0
assert batch.skipped_count == 1
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
"""手工任务可优先,但等待超过公平窗口的兜底任务不得持续饥饿。"""
repository, engine = _repository(tmp_path)
@@ -0,0 +1,61 @@
"""订阅搜索批次跳过计数的 Alembic 可逆迁移测试。"""
import importlib
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
MIGRATION = "database.versions.b6c1d9e4a7f2_3_0_24"
def _bind_migration(monkeypatch, connection):
"""把跳过计数迁移绑定到隔离 SQLite 连接。"""
migration = importlib.import_module(MIGRATION)
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
return migration
def test_subscription_search_skipped_count_migration_is_reversible(monkeypatch) -> None:
"""存量批次获得零默认值,迁移可重复执行并完整回滚。"""
engine = sa.create_engine("sqlite://")
metadata = sa.MetaData()
batch = sa.Table(
"subscriptionsearchbatch",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("batch_id", sa.String(64), nullable=False),
)
with engine.begin() as connection:
metadata.create_all(connection)
connection.execute(batch.insert(), {"id": 1, "batch_id": "batch-1"})
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
migration.upgrade()
columns = {
column["name"]
for column in sa.inspect(connection).get_columns("subscriptionsearchbatch")
}
assert "skipped_count" in columns
assert connection.execute(
sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1")
).scalar_one() == 0
migration.downgrade()
downgraded = {
column["name"]
for column in sa.inspect(connection).get_columns("subscriptionsearchbatch")
}
assert "skipped_count" not in downgraded
migration.upgrade()
assert connection.execute(
sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1")
).scalar_one() == 0
engine.dispose()
+131 -22
View File
@@ -14,12 +14,13 @@ from app.application.subscription.sitebudget import (
SiteBudgetClaim,
SubscriptionSearchCancelled,
SubscriptionSiteBudget,
SubscriptionSiteBudgetUnavailable,
)
from app.chain.search.facade import SearchChain
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
from app.db.base import Base
from app.db.models.subscriptionsearch import SubscriptionSiteBudget as SiteBudgetRecord
from app.modules.indexer import _classify_search_failure
from app.modules.indexer import IndexerModule, _classify_search_failure
from app.runtime.stop import ProcessStopState
@@ -76,6 +77,8 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat
next_allowed_at=(now + timedelta(minutes=5)).isoformat(timespec="seconds"),
error="request timeout",
) is True
cooled = repository.claim_site(site_id=3, owner="worker-b", lease_seconds=900)
assert cooled.acquired is False
with Session(engine) as session:
failed = session.execute(
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3)
@@ -103,6 +106,29 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat
assert healthy.last_outcome == "success"
def test_site_budget_ignores_legacy_success_interval(tmp_path):
"""升级前留下的正常成功间隔不能阻止新语义下的到期订阅。"""
repository, engine = _repository(tmp_path)
first = repository.claim_site(site_id=4, owner="worker-a", lease_seconds=900)
future = (datetime.now(timezone.utc) + timedelta(minutes=10)).isoformat(timespec="seconds")
assert repository.finish_site(
site_id=4,
lease_token=first.lease_token,
outcome="success",
next_allowed_at=future,
) is True
recovered = repository.claim_site(site_id=4, owner="worker-b", lease_seconds=900)
assert recovered.acquired is True
assert recovered.lease_token is not None
with Session(engine) as session:
record = session.execute(
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 4)
).scalar_one()
assert record.next_allowed_at < future
@pytest.mark.parametrize(
("error", "outcome"),
[
@@ -135,40 +161,50 @@ class _WaitingRepository:
raise AssertionError("未取得站点预算时不应收口")
def test_site_budget_wait_is_cancellable_without_business_lock():
"""批次取消应在一次短等待后终止预算获取"""
cancelled = False
def sleeper(_seconds: float) -> None:
"""模拟等待一次后收到取消请求。"""
nonlocal cancelled
cancelled = True
def test_unavailable_site_budget_does_not_sleep_in_worker():
"""错误冷却中的站点应立即留待下轮,不占用同步 worker 等待"""
budget = SubscriptionSiteBudget(
repository=_WaitingRepository(),
owner="worker-a",
cancelled=lambda: cancelled,
cancelled=lambda: False,
stop_state=ProcessStopState(),
)
with pytest.raises(SubscriptionSiteBudgetUnavailable):
budget.acquire(9)
def test_site_budget_checks_cancellation_before_repository_claim():
"""取消或停机必须在创建任何站点租约前终止当前任务。"""
class _ForbiddenRepository(_WaitingRepository):
"""取消路径不允许触发持久认领。"""
def claim_site(self, **_kwargs) -> SiteBudgetClaim:
"""若取消检查失效则立即暴露。"""
raise AssertionError("取消任务不应认领站点预算")
budget = SubscriptionSiteBudget(
repository=_ForbiddenRepository(),
owner="worker-a",
cancelled=lambda: True,
stop_state=ProcessStopState(),
sleeper=sleeper,
max_wait_seconds=600,
)
with pytest.raises(SubscriptionSearchCancelled):
budget.acquire(9)
def test_site_budget_delay_keeps_manual_requests_under_same_budget():
"""来源优先级不参与站点冷却计算,手工任务不能获得旁路"""
def test_site_budget_only_delays_external_failures():
"""正常完成立即恢复,外站错误仍按类别进入冷却"""
budget = SubscriptionSiteBudget(
repository=_WaitingRepository(),
owner="manual-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
random_uniform=lambda _low, _high: 60.0,
)
assert budget._next_delay("success", consecutive_failures=0) == 60.0 # pylint: disable=protected-access
assert budget._next_delay("success", consecutive_failures=2) == 120.0 # pylint: disable=protected-access
assert budget._next_delay("success", consecutive_failures=0) == 0.0 # pylint: disable=protected-access
assert budget._next_delay("success", consecutive_failures=2) == 0.0 # pylint: disable=protected-access
assert budget._next_delay("rate_limited", consecutive_failures=0) == 900.0 # pylint: disable=protected-access
@@ -206,14 +242,13 @@ def test_skipped_search_releases_budget_without_external_interval():
def test_search_provider_reports_cooled_site_without_blocking_other_results():
"""超出短等待窗口的站点返回空页并记录聚合失败,而非阻塞整个 provider。"""
"""错误冷却中的站点返回空页并记录聚合失败,而非阻塞 provider。"""
repository = _WaitingRepository()
budget = SubscriptionSiteBudget(
repository=repository,
owner="fallback-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
max_wait_seconds=0,
)
chain = object.__new__(SearchChain)
chain.configure_subscription_site_budget(budget)
@@ -233,7 +268,7 @@ def test_search_provider_reports_cooled_site_without_blocking_other_results():
def test_search_provider_releases_successful_site_budget():
"""真实站点页完成后必须释放租约并写入成功间隔"""
"""真实站点页正常完成后必须释放租约并立即允许下一任务"""
captured = {}
class _Repository(_WaitingRepository):
@@ -260,7 +295,6 @@ def test_search_provider_releases_successful_site_budget():
owner="manual-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
random_uniform=lambda _low, _high: 60.0,
)
chain = object.__new__(SearchChain)
chain.configure_subscription_site_budget(budget)
@@ -283,3 +317,78 @@ def test_search_provider_releases_successful_site_budget():
assert captured["site_id"] == 12
assert captured["outcome"] == "success"
assert captured["lease_token"] == "lease-token"
def test_search_provider_aggregates_swallowed_indexer_failure(monkeypatch):
"""索引器吞掉外站异常返回空页时,provider 仍须暴露站点失败。"""
captured = {}
class _Repository(_WaitingRepository):
"""提供立即可用租约并记录失败收口。"""
def claim_site(self, *, site_id: int, owner: str, lease_seconds: int) -> SiteBudgetClaim:
"""返回当前调用独占的站点租约。"""
del owner, lease_seconds
now = datetime.now(timezone.utc)
return SiteBudgetClaim(
site_id=site_id,
acquired=True,
retry_at=now.isoformat(timespec="seconds"),
consecutive_failures=0,
lease_token="lease-token",
)
def finish_site(self, **kwargs) -> bool:
"""记录预算收口,确认错误冷却仍由 budget.finish 负责。"""
captured.update(kwargs)
return True
def execute_search(_site, _request):
"""模拟 IndexerModule 捕获的 HTTP 429。"""
raise RuntimeError("HTTP 429")
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__search_check",
staticmethod(lambda _site, _keyword=None: True),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__execute_search",
staticmethod(execute_search),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__indexer_statistic",
staticmethod(lambda **_kwargs: None),
)
monkeypatch.setattr(
IndexerModule,
"_IndexerModule__parse_result",
staticmethod(lambda **_kwargs: []),
)
budget = SubscriptionSiteBudget(
repository=_Repository(),
owner="fallback-worker",
cancelled=lambda: False,
stop_state=ProcessStopState(),
)
chain = object.__new__(SearchChain)
chain.configure_subscription_site_budget(budget)
chain.search_site_torrents = object.__new__(IndexerModule).search_torrents
result = chain._search_site_torrents_with_budget( # pylint: disable=protected-access
site={"id": 13, "name": "Flaky"},
keyword="movie",
mtype=None,
page=0,
)
assert result == []
assert captured["outcome"] == "rate_limited"
assert captured["error"] == "HTTP 429"
failures = chain.consume_subscription_site_budget_failures()
assert len(failures) == 1
assert "Flaky" in failures[0]
assert "HTTP 429" in failures[0]