mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-10 09:56:48 +08:00
优化订阅执行治理与规模性能 (#6557)
* feat(subscription): govern concurrent search and match execution * refactor(subscription): consume complete candidate snapshots * refactor(subscription): remove download submission ledger * feat(subscription): stagger fallback searches without site gaps * refactor(subscription): remove unused retry capability * test(subscription): verify complete scale execution * fix(subscription): surface site search failures * test(subscription): verify site pressure through search * test(subscription): use static facade imports * fix(subscription): preserve completed downloads on shutdown * fix(subscription): classify execution expiry as failure * fix(subscription): preserve fallback search staggering * test(subscription): stabilize unfinished wrapper gate * fix(subscription): settle expired search returns * style(subscription): normalize governance imports * fix(subscription): align governance type contracts * docs(architecture): sync mypy debt metric * docs(architecture): sync startup module counts
This commit is contained in:
@@ -467,7 +467,6 @@ FIELD_DESCRIPTIONS.update(
|
|||||||
"cat": "Exact site category identifier returned by site.category.",
|
"cat": "Exact site category identifier returned by site.category.",
|
||||||
"check_only": "Validate or preview the recommendation without applying search-result filtering.",
|
"check_only": "Validate or preview the recommendation without applying search-result filtering.",
|
||||||
"can_cancel": "Whether the current subscription execution can be cancelled.",
|
"can_cancel": "Whether the current subscription execution can be cancelled.",
|
||||||
"can_retry": "Whether the current subscription execution can be retried.",
|
|
||||||
"concurrency_key": "Workflow expression used to serialize actions sharing the same runtime key.",
|
"concurrency_key": "Workflow expression used to serialize actions sharing the same runtime key.",
|
||||||
"condition": "Workflow branch or flow condition expression evaluated at runtime.",
|
"condition": "Workflow branch or flow condition expression evaluated at runtime.",
|
||||||
"context": "Persisted workflow execution context available to later actions.",
|
"context": "Persisted workflow execution context available to later actions.",
|
||||||
@@ -517,7 +516,6 @@ FIELD_DESCRIPTIONS.update(
|
|||||||
"rating": "Numeric plugin rating accepted by the endpoint's declared bounds.",
|
"rating": "Numeric plugin rating accepted by the endpoint's declared bounds.",
|
||||||
"reason": "Human-readable justification recorded with a manual-review decision.",
|
"reason": "Human-readable justification recorded with a manual-review decision.",
|
||||||
"recursive": "Apply media-aware renaming recursively to child files when true.",
|
"recursive": "Apply media-aware renaming recursively to child files when true.",
|
||||||
"requires_reconciliation": "Whether the execution requires reconciliation before it can continue.",
|
|
||||||
"release_year": "Release year matched by an automatic category rule.",
|
"release_year": "Release year matched by an automatic category rule.",
|
||||||
"result": "Persisted workflow action result value.",
|
"result": "Persisted workflow action result value.",
|
||||||
"result_payload": "Structured external-operation result recorded with manual review.",
|
"result_payload": "Structured external-operation result recorded with manual review.",
|
||||||
|
|||||||
@@ -3532,12 +3532,6 @@
|
|||||||
"title": "Can Cancel",
|
"title": "Can Cancel",
|
||||||
"type": "boolean"
|
"type": "boolean"
|
||||||
},
|
},
|
||||||
"can_retry": {
|
|
||||||
"default": false,
|
|
||||||
"description": "Whether the current subscription execution can be retried.",
|
|
||||||
"title": "Can Retry",
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"current_site_id": {
|
"current_site_id": {
|
||||||
"anyOf": [
|
"anyOf": [
|
||||||
{
|
{
|
||||||
@@ -3567,12 +3561,6 @@
|
|||||||
"title": "Phase",
|
"title": "Phase",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"requires_reconciliation": {
|
|
||||||
"default": false,
|
|
||||||
"description": "Whether the execution requires reconciliation before it can continue.",
|
|
||||||
"title": "Requires Reconciliation",
|
|
||||||
"type": "boolean"
|
|
||||||
},
|
|
||||||
"source": {
|
"source": {
|
||||||
"anyOf": [
|
"anyOf": [
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -12,7 +12,6 @@ from app.application.configuration import ChainRuntimeConfig
|
|||||||
from app.runtime.stop import StopState, runtime_stop_state
|
from app.runtime.stop import StopState, runtime_stop_state
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from app.application.download.admission import SubscriptionDownloadRepository
|
|
||||||
from app.application.download.failures import DownloadFailureRepository
|
from app.application.download.failures import DownloadFailureRepository
|
||||||
from app.application.history import (
|
from app.application.history import (
|
||||||
DownloadHistoryRepository,
|
DownloadHistoryRepository,
|
||||||
@@ -77,7 +76,6 @@ class ChainRuntimeContext:
|
|||||||
download_failure_repository: DownloadFailureRepository
|
download_failure_repository: DownloadFailureRepository
|
||||||
user_repository: ChainUserRepository
|
user_repository: ChainUserRepository
|
||||||
subscription_search_repository: Optional[SubscriptionSearchRepository] = None
|
subscription_search_repository: Optional[SubscriptionSearchRepository] = None
|
||||||
subscription_download_repository: Optional[SubscriptionDownloadRepository] = None
|
|
||||||
legacy_transfer_command: Optional[LegacyTransferCommand] = None
|
legacy_transfer_command: Optional[LegacyTransferCommand] = None
|
||||||
durable_event_writer: Optional[ChainDurableEventWriter] = None
|
durable_event_writer: Optional[ChainDurableEventWriter] = None
|
||||||
configuration: ChainRuntimeConfig = field(
|
configuration: ChainRuntimeConfig = field(
|
||||||
|
|||||||
@@ -1,130 +1,14 @@
|
|||||||
"""订阅下载提交的持久幂等与不确定终态合同。"""
|
"""订阅下载提交前的可取消执行边界。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Callable, Optional, Protocol
|
from typing import Callable, Optional
|
||||||
|
|
||||||
|
|
||||||
class DownloadReconciliationRequired(RuntimeError):
|
|
||||||
"""表示下载器可能已接受任务,必须先对账才能继续自动提交。"""
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class SubscriptionDownloadGovernance:
|
class SubscriptionDownloadGovernance:
|
||||||
"""把订阅身份、入口任务与取消检查传入下载提交边界。"""
|
"""把取消检查与副作用起点回传给订阅执行上下文。"""
|
||||||
|
|
||||||
subscription_id: int
|
|
||||||
mode: str
|
|
||||||
task_id: Optional[str] = None
|
|
||||||
cancelled: Optional[Callable[[], bool]] = None
|
cancelled: Optional[Callable[[], bool]] = None
|
||||||
mark_started: Optional[Callable[[], None]] = None
|
mark_started: Optional[Callable[[], None]] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class SubscriptionDownloadRequest:
|
|
||||||
"""一次订阅下载提交认领所需的规范身份。"""
|
|
||||||
|
|
||||||
idempotency_key: str
|
|
||||||
legacy_idempotency_key: Optional[str]
|
|
||||||
subscription_id: int
|
|
||||||
task_id: Optional[str]
|
|
||||||
logical_identity: str
|
|
||||||
resource_key: str
|
|
||||||
coverage: str
|
|
||||||
mode: str
|
|
||||||
delivery_scope: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class SubscriptionDownloadSnapshot:
|
|
||||||
"""脱离 Session 的订阅下载提交状态快照。"""
|
|
||||||
|
|
||||||
idempotency_key: str
|
|
||||||
subscription_id: int
|
|
||||||
task_id: Optional[str]
|
|
||||||
state: str
|
|
||||||
attempt_count: int
|
|
||||||
attempt_token: Optional[str]
|
|
||||||
downloader: Optional[str]
|
|
||||||
download_hash: Optional[str]
|
|
||||||
available_at: Optional[str]
|
|
||||||
last_error: Optional[str]
|
|
||||||
created_at: str
|
|
||||||
updated_at: str
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class SubscriptionDownloadClaim:
|
|
||||||
"""返回本次是否取得唯一提交权以及当前持久状态。"""
|
|
||||||
|
|
||||||
acquired: bool
|
|
||||||
snapshot: SubscriptionDownloadSnapshot
|
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionDownloadRepository(Protocol):
|
|
||||||
"""订阅下载幂等账本所需的最小持久化端口。"""
|
|
||||||
|
|
||||||
def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim:
|
|
||||||
"""按唯一键认领提交;仅到期 retryable/cancelled 状态允许重新认领。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]:
|
|
||||||
"""按幂等键读取现有提交,供键版本兼容检查。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def mark_accepted(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_hash: str,
|
|
||||||
) -> bool:
|
|
||||||
"""记录下载器已明确接受任务,后续任何失败都不得自动重试。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def mark_succeeded(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
) -> bool:
|
|
||||||
"""在 canonical 本地结算完成后写入成功终态。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def mark_retryable(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
available_at: str,
|
|
||||||
error: Optional[str],
|
|
||||||
) -> bool:
|
|
||||||
"""记录下载器明确拒绝、尚未产生外部副作用的可重试状态。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def mark_reconcile_required(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
error: Optional[str],
|
|
||||||
downloader: Optional[str] = None,
|
|
||||||
download_hash: Optional[str] = None,
|
|
||||||
) -> bool:
|
|
||||||
"""冻结可能已产生外部副作用的提交,等待下载器对账。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def mark_cancelled(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
) -> bool:
|
|
||||||
"""仅在进入下载器副作用前收口取消。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
def has_started_for_task(self, task_id: str) -> bool:
|
|
||||||
"""判断搜索任务是否已有不能按未执行处理的下载提交。"""
|
|
||||||
...
|
|
||||||
|
|||||||
@@ -1,10 +1,7 @@
|
|||||||
"""订阅候选批次与无损路由合同。"""
|
"""订阅候选的无损路由合同。"""
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Any, Dict, List, Optional, cast
|
from typing import Any, Dict, List, Optional, cast
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
from app.domain.context import Context, MediaInfo
|
from app.domain.context import Context, MediaInfo
|
||||||
@@ -15,57 +12,6 @@ from app.schemas.types import MediaSource, MediaType
|
|||||||
CandidateGroups = Dict[str, List[Context]]
|
CandidateGroups = Dict[str, List[Context]]
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class CandidateBatch:
|
|
||||||
"""一次资源获取产生的完整候选、增量候选与重试候选边界。"""
|
|
||||||
|
|
||||||
batch_id: str
|
|
||||||
source: str
|
|
||||||
candidates: CandidateGroups
|
|
||||||
fresh_candidates: CandidateGroups = field(default_factory=dict)
|
|
||||||
retry_candidates: CandidateGroups = field(default_factory=dict)
|
|
||||||
sites: tuple[str, ...] = ()
|
|
||||||
started_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
|
|
||||||
finished_at: Optional[datetime] = None
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def create(
|
|
||||||
cls,
|
|
||||||
*,
|
|
||||||
source: str,
|
|
||||||
candidates: CandidateGroups,
|
|
||||||
fresh_candidates: Optional[CandidateGroups] = None,
|
|
||||||
retry_candidates: Optional[CandidateGroups] = None,
|
|
||||||
sites: Optional[List[str]] = None,
|
|
||||||
started_at: Optional[datetime] = None,
|
|
||||||
) -> "CandidateBatch":
|
|
||||||
"""构造已完成获取的候选批次。"""
|
|
||||||
return cls(
|
|
||||||
batch_id=uuid4().hex,
|
|
||||||
source=source,
|
|
||||||
candidates=candidates,
|
|
||||||
fresh_candidates=fresh_candidates or {},
|
|
||||||
retry_candidates=retry_candidates or {},
|
|
||||||
sites=tuple(sites or candidates.keys()),
|
|
||||||
started_at=started_at or datetime.now(timezone.utc),
|
|
||||||
finished_at=datetime.now(timezone.utc),
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_legacy(cls, candidates: CandidateGroups, source: str = "legacy") -> "CandidateBatch":
|
|
||||||
"""把旧入口传入的完整候选包装为无增量声明的兼容批次。"""
|
|
||||||
return cls.create(source=source, candidates=candidates)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def count(groups: CandidateGroups) -> int:
|
|
||||||
"""统计分站点候选总数。"""
|
|
||||||
return sum(len(contexts) for contexts in groups.values())
|
|
||||||
|
|
||||||
def build_index(self) -> "CandidateIndex":
|
|
||||||
"""基于完整候选构建一次性无损索引。"""
|
|
||||||
return CandidateIndex(self.candidates)
|
|
||||||
|
|
||||||
|
|
||||||
class CandidateIndex:
|
class CandidateIndex:
|
||||||
"""一次构建并保持原顺序的订阅候选身份索引。"""
|
"""一次构建并保持原顺序的订阅候选身份索引。"""
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,101 @@
|
|||||||
"""订阅搜索执行批次、任务与持久队列端口。"""
|
"""订阅执行准入、搜索上下文、批次任务与持久队列端口。"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional, Protocol
|
from typing import Callable, Mapping, Optional, Protocol
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
from app.application.subscription.sitebudget import SiteBudgetClaim
|
from app.application.subscription.sitebudget import SiteBudgetClaim
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SubscriptionExecutionLease:
|
||||||
|
"""一次订阅执行的进程内所有权及协作截止时间。"""
|
||||||
|
|
||||||
|
subscription_id: int
|
||||||
|
operation: str
|
||||||
|
owner_token: str
|
||||||
|
expires_at: float
|
||||||
|
|
||||||
|
|
||||||
|
class SubscriptionExecutionAdmission:
|
||||||
|
"""按订阅 ID 控制 Search 与 Match 的进程内互斥。"""
|
||||||
|
|
||||||
|
def __init__(self, clock: Callable[[], float] = time.monotonic) -> None:
|
||||||
|
"""保存单调时钟和当前活跃 owner。"""
|
||||||
|
self._clock = clock
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
self._owners: dict[int, SubscriptionExecutionLease] = {}
|
||||||
|
|
||||||
|
def try_acquire(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
subscription_id: int,
|
||||||
|
operation: str,
|
||||||
|
ttl_seconds: float,
|
||||||
|
) -> Optional[SubscriptionExecutionLease]:
|
||||||
|
"""无等待取得订阅所有权;已有 owner 时直接返回空。"""
|
||||||
|
with self._lock:
|
||||||
|
if subscription_id in self._owners:
|
||||||
|
return None
|
||||||
|
lease = SubscriptionExecutionLease(
|
||||||
|
subscription_id=subscription_id,
|
||||||
|
operation=operation,
|
||||||
|
owner_token=uuid4().hex,
|
||||||
|
expires_at=self._clock() + max(1.0, ttl_seconds),
|
||||||
|
)
|
||||||
|
self._owners[subscription_id] = lease
|
||||||
|
return lease
|
||||||
|
|
||||||
|
def release(self, lease: SubscriptionExecutionLease) -> bool:
|
||||||
|
"""仅允许当前 owner 释放订阅所有权。"""
|
||||||
|
with self._lock:
|
||||||
|
current = self._owners.get(lease.subscription_id)
|
||||||
|
if current is None or current.owner_token != lease.owner_token:
|
||||||
|
return False
|
||||||
|
self._owners.pop(lease.subscription_id)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def is_expired(self, lease: SubscriptionExecutionLease) -> bool:
|
||||||
|
"""判断 owner 是否已超过协作执行截止时间。"""
|
||||||
|
return self._clock() >= lease.expires_at
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class SubscriptionExecutionContext:
|
||||||
|
"""一次订阅执行的显式取消、阶段和副作用边界。"""
|
||||||
|
|
||||||
|
lease: SubscriptionExecutionLease
|
||||||
|
admission: SubscriptionExecutionAdmission
|
||||||
|
task_id: Optional[str] = None
|
||||||
|
cancel_requested: Optional[Callable[[], bool]] = None
|
||||||
|
phase_changed: Optional[Callable[[str, Optional[int]], None]] = None
|
||||||
|
download_started: bool = False
|
||||||
|
|
||||||
|
def is_cancel_requested(self) -> bool:
|
||||||
|
"""判断调用入口是否请求在下一个安全边界退出。"""
|
||||||
|
return bool(self.cancel_requested and self.cancel_requested())
|
||||||
|
|
||||||
|
def is_expired(self) -> bool:
|
||||||
|
"""判断本次执行是否已经超过协作截止时间。"""
|
||||||
|
return self.admission.is_expired(self.lease)
|
||||||
|
|
||||||
|
def should_stop(self) -> bool:
|
||||||
|
"""在安全边界合并用户取消和执行 TTL。"""
|
||||||
|
return self.is_expired() or self.is_cancel_requested()
|
||||||
|
|
||||||
|
def report_phase(self, phase: str, current_site_id: Optional[int] = None) -> None:
|
||||||
|
"""向当前搜索任务报告业务阶段。"""
|
||||||
|
if self.phase_changed:
|
||||||
|
self.phase_changed(phase, current_site_id)
|
||||||
|
|
||||||
|
def mark_download_started(self) -> None:
|
||||||
|
"""标记执行已越过下载器副作用边界。"""
|
||||||
|
self.download_started = True
|
||||||
|
self.report_phase("submitting")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class SearchBatchSnapshot:
|
class SearchBatchSnapshot:
|
||||||
"""订阅搜索批次的持久业务状态快照。"""
|
"""订阅搜索批次的持久业务状态快照。"""
|
||||||
@@ -24,6 +114,7 @@ class SearchBatchSnapshot:
|
|||||||
started_at: Optional[str] = None
|
started_at: Optional[str] = None
|
||||||
finished_at: Optional[str] = None
|
finished_at: Optional[str] = None
|
||||||
last_error: Optional[str] = None
|
last_error: Optional[str] = None
|
||||||
|
skipped_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -68,9 +159,9 @@ class SubscriptionSearchRepository(Protocol):
|
|||||||
subscription_ids: tuple[int, ...],
|
subscription_ids: tuple[int, ...],
|
||||||
source: str,
|
source: str,
|
||||||
priority: int,
|
priority: int,
|
||||||
available_at: Optional[str] = None,
|
available_at_by_subscription: Optional[Mapping[int, str]] = None,
|
||||||
) -> SearchEnqueueResult:
|
) -> SearchEnqueueResult:
|
||||||
"""按订阅 ID 建立批次,在启动抖动后合并活动任务。"""
|
"""按订阅 ID 和各自到期时间建立或合并活动任务。"""
|
||||||
...
|
...
|
||||||
|
|
||||||
def claim_next(self, *, owner: str, lease_seconds: int = 900) -> Optional[SearchTaskSnapshot]:
|
def claim_next(self, *, owner: str, lease_seconds: int = 900) -> Optional[SearchTaskSnapshot]:
|
||||||
@@ -85,7 +176,7 @@ class SubscriptionSearchRepository(Protocol):
|
|||||||
state: str,
|
state: str,
|
||||||
error: Optional[str] = None,
|
error: Optional[str] = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""以租约令牌收口任务,并推进所属批次聚合状态。"""
|
"""以租约令牌收口任务,并推进所属批次聚合状态(含 skipped)。"""
|
||||||
...
|
...
|
||||||
|
|
||||||
def update_task_phase(
|
def update_task_phase(
|
||||||
@@ -140,5 +231,5 @@ class SubscriptionSearchRepository(Protocol):
|
|||||||
next_allowed_at: str,
|
next_allowed_at: str,
|
||||||
error: Optional[str] = None,
|
error: Optional[str] = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""释放站点预算并写入间隔、冷却和恢复状态。"""
|
"""释放站点预算并写入错误冷却和恢复状态。"""
|
||||||
...
|
...
|
||||||
|
|||||||
@@ -2,8 +2,6 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import random
|
|
||||||
import time
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Callable, Optional, Protocol
|
from typing import Callable, Optional, Protocol
|
||||||
@@ -17,7 +15,7 @@ class SubscriptionSearchCancelled(RuntimeError):
|
|||||||
|
|
||||||
|
|
||||||
class SubscriptionSiteBudgetUnavailable(RuntimeError):
|
class SubscriptionSiteBudgetUnavailable(RuntimeError):
|
||||||
"""表示站点预算超出本任务允许的短等待窗口。"""
|
"""表示站点仍处于错误冷却或已有未释放租约。"""
|
||||||
|
|
||||||
def __init__(self, *, site_id: int, retry_at: str) -> None:
|
def __init__(self, *, site_id: int, retry_at: str) -> None:
|
||||||
"""保存站点和下一次可尝试时间,供批次聚合失败展示。"""
|
"""保存站点和下一次可尝试时间,供批次聚合失败展示。"""
|
||||||
@@ -69,7 +67,7 @@ def _utc_now() -> datetime:
|
|||||||
|
|
||||||
|
|
||||||
class SubscriptionSiteBudget:
|
class SubscriptionSiteBudget:
|
||||||
"""在同步搜索 worker 内执行可取消的站点预算等待与反馈。"""
|
"""以非阻塞认领保护站点租约,并持久化外站错误冷却。"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -78,49 +76,35 @@ class SubscriptionSiteBudget:
|
|||||||
owner: str,
|
owner: str,
|
||||||
cancelled: Callable[[], bool],
|
cancelled: Callable[[], bool],
|
||||||
stop_state: StopState,
|
stop_state: StopState,
|
||||||
interval_range: tuple[float, float] = (60.0, 300.0),
|
|
||||||
lease_seconds: int = 900,
|
lease_seconds: int = 900,
|
||||||
max_wait_seconds: float = 5.0,
|
|
||||||
random_uniform: Callable[[float, float], float] = random.uniform,
|
|
||||||
sleeper: Callable[[float], None] = time.sleep,
|
|
||||||
clock: Callable[[], datetime] = _utc_now,
|
clock: Callable[[], datetime] = _utc_now,
|
||||||
phase_changed: Optional[Callable[[str, Optional[int]], None]] = None,
|
phase_changed: Optional[Callable[[str, Optional[int]], None]] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""保存持久化端口及可注入的时钟、随机数和等待实现。"""
|
"""保存持久化端口及可注入的时钟和阶段回调。"""
|
||||||
self._repository = repository
|
self._repository = repository
|
||||||
self._owner = owner
|
self._owner = owner
|
||||||
self._cancelled = cancelled
|
self._cancelled = cancelled
|
||||||
self._stop_state = stop_state
|
self._stop_state = stop_state
|
||||||
self._interval_range = interval_range
|
|
||||||
self._lease_seconds = max(1, lease_seconds)
|
self._lease_seconds = max(1, lease_seconds)
|
||||||
self._max_wait_seconds = max(0.0, max_wait_seconds)
|
|
||||||
self._random_uniform = random_uniform
|
|
||||||
self._sleeper = sleeper
|
|
||||||
self._clock = clock
|
self._clock = clock
|
||||||
self._phase_changed = phase_changed
|
self._phase_changed = phase_changed
|
||||||
|
|
||||||
def acquire(self, site_id: int) -> SiteBudgetClaim:
|
def acquire(self, site_id: int) -> SiteBudgetClaim:
|
||||||
"""循环认领指定站点,并在每秒边界检查取消与停机。"""
|
"""只认领一次指定站点,未就绪时留待下一次正常调度。"""
|
||||||
deadline = time.monotonic() + self._max_wait_seconds
|
self._raise_if_cancelled()
|
||||||
while True:
|
claim = self._repository.claim_site(
|
||||||
self._raise_if_cancelled()
|
site_id=site_id,
|
||||||
claim = self._repository.claim_site(
|
owner=self._owner,
|
||||||
site_id=site_id,
|
lease_seconds=self._lease_seconds,
|
||||||
owner=self._owner,
|
)
|
||||||
lease_seconds=self._lease_seconds,
|
if claim.acquired:
|
||||||
)
|
self._report_phase("searching", site_id)
|
||||||
if claim.acquired:
|
return claim
|
||||||
self._report_phase("searching", site_id)
|
self._report_phase("waiting_site_budget", site_id)
|
||||||
return claim
|
raise SubscriptionSiteBudgetUnavailable(
|
||||||
self._report_phase("waiting_site_budget", site_id)
|
site_id=site_id,
|
||||||
retry_at = datetime.fromisoformat(claim.retry_at)
|
retry_at=claim.retry_at,
|
||||||
remaining = max(0.0, (retry_at - self._clock()).total_seconds())
|
)
|
||||||
if remaining > max(0.0, deadline - time.monotonic()):
|
|
||||||
raise SubscriptionSiteBudgetUnavailable(
|
|
||||||
site_id=site_id,
|
|
||||||
retry_at=claim.retry_at,
|
|
||||||
)
|
|
||||||
self._sleeper(min(max(remaining, 0.05), 1.0))
|
|
||||||
|
|
||||||
def _report_phase(self, phase: str, site_id: Optional[int]) -> None:
|
def _report_phase(self, phase: str, site_id: Optional[int]) -> None:
|
||||||
"""向任务所有者报告不改变预算语义的业务阶段。"""
|
"""向任务所有者报告不改变预算语义的业务阶段。"""
|
||||||
@@ -128,7 +112,7 @@ class SubscriptionSiteBudget:
|
|||||||
self._phase_changed(phase, site_id)
|
self._phase_changed(phase, site_id)
|
||||||
|
|
||||||
def finish(self, claim: SiteBudgetClaim, observation: SiteSearchObservation) -> bool:
|
def finish(self, claim: SiteBudgetClaim, observation: SiteSearchObservation) -> bool:
|
||||||
"""依据调用结果计算随机间隔或错误冷却并释放租约。"""
|
"""正常结果立即释放站点,错误结果按类别写入冷却。"""
|
||||||
if not claim.lease_token:
|
if not claim.lease_token:
|
||||||
return False
|
return False
|
||||||
outcome = observation.outcome if observation.attempted else "skipped"
|
outcome = observation.outcome if observation.attempted else "skipped"
|
||||||
@@ -143,13 +127,9 @@ class SubscriptionSiteBudget:
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _next_delay(self, outcome: str, consecutive_failures: int) -> float:
|
def _next_delay(self, outcome: str, consecutive_failures: int) -> float:
|
||||||
"""为成功渐进恢复、错误退避和本地跳过计算下一次等待。"""
|
"""正常或本地跳过立即恢复,错误按连续失败次数退避。"""
|
||||||
if outcome == "skipped":
|
if outcome in {"success", "skipped"}:
|
||||||
return 0.0
|
return 0.0
|
||||||
if outcome == "success":
|
|
||||||
low, high = self._interval_range
|
|
||||||
recovery_factor = 1.0 + min(max(consecutive_failures, 0), 3) * 0.5
|
|
||||||
return self._random_uniform(low, high) * recovery_factor
|
|
||||||
exponent = min(max(consecutive_failures, 0), 5)
|
exponent = min(max(consecutive_failures, 0), 5)
|
||||||
base, ceiling = {
|
base, ceiling = {
|
||||||
"rate_limited": (900.0, 21600.0),
|
"rate_limited": (900.0, 21600.0),
|
||||||
@@ -160,6 +140,6 @@ class SubscriptionSiteBudget:
|
|||||||
return float(min(base * (2**exponent), ceiling))
|
return float(min(base * (2**exponent), ceiling))
|
||||||
|
|
||||||
def _raise_if_cancelled(self) -> None:
|
def _raise_if_cancelled(self) -> None:
|
||||||
"""在不持有业务锁的等待边界传播取消或停机。"""
|
"""在创建站点租约前传播取消或停机。"""
|
||||||
if self._stop_state.is_system_stopped or self._cancelled():
|
if self._stop_state.is_system_stopped or self._cancelled():
|
||||||
raise SubscriptionSearchCancelled("订阅搜索已取消")
|
raise SubscriptionSearchCancelled("订阅搜索已取消")
|
||||||
|
|||||||
@@ -4,13 +4,12 @@ from collections.abc import Awaitable, Callable
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Optional, Protocol
|
from typing import Optional, Protocol
|
||||||
|
|
||||||
from app.application.download.admission import SubscriptionDownloadSnapshot
|
|
||||||
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
class SubscriptionExecutionStatus:
|
class SubscriptionExecutionStatus:
|
||||||
"""一个订阅跨搜索与下载账本合并后的用户可见状态。"""
|
"""一个订阅最近搜索任务的用户可见状态。"""
|
||||||
|
|
||||||
state: str
|
state: str
|
||||||
phase: str
|
phase: str
|
||||||
@@ -21,8 +20,6 @@ class SubscriptionExecutionStatus:
|
|||||||
current_site_id: Optional[int] = None
|
current_site_id: Optional[int] = None
|
||||||
error: Optional[str] = None
|
error: Optional[str] = None
|
||||||
can_cancel: bool = False
|
can_cancel: bool = False
|
||||||
can_retry: bool = False
|
|
||||||
requires_reconciliation: bool = False
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
@dataclass(frozen=True, slots=True)
|
||||||
@@ -44,10 +41,11 @@ class SubscriptionBatchStatus:
|
|||||||
current_site_id: Optional[int] = None
|
current_site_id: Optional[int] = None
|
||||||
error: Optional[str] = None
|
error: Optional[str] = None
|
||||||
can_cancel: bool = False
|
can_cancel: bool = False
|
||||||
|
skipped_count: int = 0
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionExecutionReadRepository(Protocol):
|
class SubscriptionExecutionReadRepository(Protocol):
|
||||||
"""请求级读取搜索任务、批次与下载提交事实的端口。"""
|
"""请求级读取搜索任务与批次事实的端口。"""
|
||||||
|
|
||||||
async def latest_search_tasks(
|
async def latest_search_tasks(
|
||||||
self,
|
self,
|
||||||
@@ -56,13 +54,6 @@ class SubscriptionExecutionReadRepository(Protocol):
|
|||||||
"""返回每条订阅最近更新的搜索任务。"""
|
"""返回每条订阅最近更新的搜索任务。"""
|
||||||
...
|
...
|
||||||
|
|
||||||
async def latest_download_submissions(
|
|
||||||
self,
|
|
||||||
subscription_ids: tuple[int, ...],
|
|
||||||
) -> dict[int, SubscriptionDownloadSnapshot]:
|
|
||||||
"""返回每条订阅最近更新的下载提交。"""
|
|
||||||
...
|
|
||||||
|
|
||||||
async def list_batches(
|
async def list_batches(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -81,16 +72,8 @@ class SubscriptionExecutionReadRepository(Protocol):
|
|||||||
|
|
||||||
|
|
||||||
class SubscriptionExecutionStatusService:
|
class SubscriptionExecutionStatusService:
|
||||||
"""把搜索队列与下载幂等账本投影为稳定业务状态。"""
|
"""把搜索队列投影为稳定业务状态。"""
|
||||||
|
|
||||||
_DOWNLOAD_STATES = {
|
|
||||||
"submitting": "submitting",
|
|
||||||
"accepted": "accepted",
|
|
||||||
"succeeded": "completed",
|
|
||||||
"retryable": "retryable",
|
|
||||||
"reconcile_required": "reconcile_required",
|
|
||||||
"cancelled": "cancelled",
|
|
||||||
}
|
|
||||||
_ACTIVE_STATES = {
|
_ACTIVE_STATES = {
|
||||||
"queued",
|
"queued",
|
||||||
"running",
|
"running",
|
||||||
@@ -99,16 +82,8 @@ class SubscriptionExecutionStatusService:
|
|||||||
"waiting_site_budget",
|
"waiting_site_budget",
|
||||||
"preparing",
|
"preparing",
|
||||||
"submitting",
|
"submitting",
|
||||||
"accepted",
|
|
||||||
"cancelling",
|
"cancelling",
|
||||||
}
|
}
|
||||||
_DOWNLOAD_OVERRIDE_STATES = {
|
|
||||||
"submitting",
|
|
||||||
"accepted",
|
|
||||||
"retryable",
|
|
||||||
"reconcile_required",
|
|
||||||
}
|
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
repository: SubscriptionExecutionReadRepository,
|
repository: SubscriptionExecutionReadRepository,
|
||||||
@@ -133,14 +108,10 @@ class SubscriptionExecutionStatusService:
|
|||||||
if not ids:
|
if not ids:
|
||||||
return {}
|
return {}
|
||||||
tasks = await self._repository.latest_search_tasks(ids)
|
tasks = await self._repository.latest_search_tasks(ids)
|
||||||
downloads = await self._repository.latest_download_submissions(ids)
|
|
||||||
result: dict[int, SubscriptionExecutionStatus] = {}
|
result: dict[int, SubscriptionExecutionStatus] = {}
|
||||||
for subscription_id in ids:
|
for subscription_id in ids:
|
||||||
task = tasks.get(subscription_id)
|
task = tasks.get(subscription_id)
|
||||||
download = downloads.get(subscription_id)
|
if task:
|
||||||
if download and self._download_wins(task, download):
|
|
||||||
result[subscription_id] = self._from_download(download, task)
|
|
||||||
elif task:
|
|
||||||
result[subscription_id] = self._from_task(task)
|
result[subscription_id] = self._from_task(task)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
@@ -175,17 +146,6 @@ class SubscriptionExecutionStatusService:
|
|||||||
return None
|
return None
|
||||||
return self._from_batch(batch, tasks)
|
return self._from_batch(batch, tasks)
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _download_wins(
|
|
||||||
cls,
|
|
||||||
task: Optional[SearchTaskSnapshot],
|
|
||||||
download: SubscriptionDownloadSnapshot,
|
|
||||||
) -> bool:
|
|
||||||
"""下载风险状态优先,其余事实按更新时间选择。"""
|
|
||||||
if download.state in cls._DOWNLOAD_OVERRIDE_STATES:
|
|
||||||
return True
|
|
||||||
return task is None or download.updated_at >= task.updated_at
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _from_task(cls, task: SearchTaskSnapshot) -> SubscriptionExecutionStatus:
|
def _from_task(cls, task: SearchTaskSnapshot) -> SubscriptionExecutionStatus:
|
||||||
"""把搜索任务状态归一为稳定业务词汇。"""
|
"""把搜索任务状态归一为稳定业务词汇。"""
|
||||||
@@ -205,29 +165,6 @@ class SubscriptionExecutionStatusService:
|
|||||||
updated_at=task.updated_at,
|
updated_at=task.updated_at,
|
||||||
error=cls._safe_error(task.last_error),
|
error=cls._safe_error(task.last_error),
|
||||||
can_cancel=state in cls._ACTIVE_STATES,
|
can_cancel=state in cls._ACTIVE_STATES,
|
||||||
can_retry=state == "failed",
|
|
||||||
)
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def _from_download(
|
|
||||||
cls,
|
|
||||||
download: SubscriptionDownloadSnapshot,
|
|
||||||
task: Optional[SearchTaskSnapshot],
|
|
||||||
) -> SubscriptionExecutionStatus:
|
|
||||||
"""把下载提交账本状态投影为业务状态并保留搜索来源。"""
|
|
||||||
state = cls._DOWNLOAD_STATES.get(download.state, download.state)
|
|
||||||
return SubscriptionExecutionStatus(
|
|
||||||
state=state,
|
|
||||||
phase=state,
|
|
||||||
source=task.source if task else None,
|
|
||||||
batch_id=task.batch_id if task else None,
|
|
||||||
task_id=download.task_id or (task.task_id if task else None),
|
|
||||||
current_site_id=task.current_site_id if task else None,
|
|
||||||
updated_at=download.updated_at,
|
|
||||||
error=cls._safe_error(download.last_error),
|
|
||||||
can_cancel=state == "submitting",
|
|
||||||
can_retry=state == "retryable",
|
|
||||||
requires_reconciliation=state == "reconcile_required",
|
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -240,7 +177,12 @@ class SubscriptionExecutionStatusService:
|
|||||||
current = next((task for task in tasks if task.state == "running"), None)
|
current = next((task for task in tasks if task.state == "running"), None)
|
||||||
if current is None:
|
if current is None:
|
||||||
current = next((task for task in tasks if task.state == "queued"), None)
|
current = next((task for task in tasks if task.state == "queued"), None)
|
||||||
processed = batch.finished_count + batch.failed_count + batch.cancelled_count
|
processed = (
|
||||||
|
batch.finished_count
|
||||||
|
+ batch.failed_count
|
||||||
|
+ batch.cancelled_count
|
||||||
|
+ batch.skipped_count
|
||||||
|
)
|
||||||
phase = current.phase if current else batch.state
|
phase = current.phase if current else batch.state
|
||||||
return SubscriptionBatchStatus(
|
return SubscriptionBatchStatus(
|
||||||
batch_id=batch.batch_id,
|
batch_id=batch.batch_id,
|
||||||
@@ -252,6 +194,7 @@ class SubscriptionExecutionStatusService:
|
|||||||
finished_count=batch.finished_count,
|
finished_count=batch.finished_count,
|
||||||
failed_count=batch.failed_count,
|
failed_count=batch.failed_count,
|
||||||
cancelled_count=batch.cancelled_count,
|
cancelled_count=batch.cancelled_count,
|
||||||
|
skipped_count=batch.skipped_count,
|
||||||
current_subscription_id=current.subscription_id if current else None,
|
current_subscription_id=current.subscription_id if current else None,
|
||||||
current_site_id=current.current_site_id if current else None,
|
current_site_id=current.current_site_id if current else None,
|
||||||
created_at=batch.created_at,
|
created_at=batch.created_at,
|
||||||
|
|||||||
@@ -66,6 +66,14 @@ class MusicSubscribeMixinHost(Protocol):
|
|||||||
|
|
||||||
def get_subscribe_source_keyword(self, subscribe: Any) -> str: ...
|
def get_subscribe_source_keyword(self, subscribe: Any) -> str: ...
|
||||||
|
|
||||||
|
def _SubscribeChain__candidate_contract_changed(
|
||||||
|
self,
|
||||||
|
prepared: Any,
|
||||||
|
current: Any,
|
||||||
|
) -> bool:
|
||||||
|
"""判断准备候选期间订阅身份或过滤合同是否已经变化。"""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class InteractionMixinHost(Protocol):
|
class InteractionMixinHost(Protocol):
|
||||||
"""交互委托 mixin 对业务 Chain 的最小要求。"""
|
"""交互委托 mixin 对业务 Chain 的最小要求。"""
|
||||||
|
|||||||
+70
-5
@@ -1,15 +1,18 @@
|
|||||||
import copy
|
import copy
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from typing import TYPE_CHECKING, Any, List, Optional, Tuple
|
from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple
|
||||||
|
|
||||||
from app.application.configuration import get_configured_system_config
|
from app.application.configuration import get_configured_system_config
|
||||||
|
from app.application.download.admission import SubscriptionDownloadGovernance
|
||||||
from app.application.subscription.contract import (
|
from app.application.subscription.contract import (
|
||||||
SubscriptionRepository,
|
SubscriptionRepository,
|
||||||
SubscriptionSnapshot,
|
SubscriptionSnapshot,
|
||||||
build_subscribe_meta,
|
build_subscribe_meta,
|
||||||
subscribe_media_key,
|
subscribe_media_key,
|
||||||
)
|
)
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionContext
|
||||||
from app.application.subscription.mutation import SubscriptionActor
|
from app.application.subscription.mutation import SubscriptionActor
|
||||||
|
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
|
||||||
from app.application.torrent.download import TorrentHelper
|
from app.application.torrent.download import TorrentHelper
|
||||||
from app.chain._contracts import MusicSubscribeMixinHost
|
from app.chain._contracts import MusicSubscribeMixinHost
|
||||||
from app.chain.download import DownloadChain
|
from app.chain.download import DownloadChain
|
||||||
@@ -44,6 +47,9 @@ class MusicSubscribeMixin:
|
|||||||
__mixin_host_protocol__ = MusicSubscribeMixinHost
|
__mixin_host_protocol__ = MusicSubscribeMixinHost
|
||||||
subscription_repository: SubscriptionRepository
|
subscription_repository: SubscriptionRepository
|
||||||
sync_subscription_mutation_scope: "SyncSubscriptionMutationScope"
|
sync_subscription_mutation_scope: "SyncSubscriptionMutationScope"
|
||||||
|
_SubscribeChain__candidate_contract_changed: Callable[
|
||||||
|
[SubscriptionSnapshot, SubscriptionSnapshot], bool
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
音乐订阅功能域 mixin:单曲/专辑目标识别、实体快照同步、候选筛选、
|
音乐订阅功能域 mixin:单曲/专辑目标识别、实体快照同步、候选筛选、
|
||||||
择优下载与完成推进。
|
择优下载与完成推进。
|
||||||
@@ -348,10 +354,32 @@ class MusicSubscribeMixin:
|
|||||||
subscribe: SubscriptionSnapshot,
|
subscribe: SubscriptionSnapshot,
|
||||||
mediainfo: MusicInfo,
|
mediainfo: MusicInfo,
|
||||||
contexts: List[Context],
|
contexts: List[Context],
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""批量择优下载音乐候选,并按单曲或整专完成语义推进订阅。"""
|
"""批量择优下载音乐候选,并按单曲或整专完成语义推进订阅。"""
|
||||||
if not contexts:
|
if not contexts:
|
||||||
return
|
return
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
|
repository = self.subscription_repository
|
||||||
|
current_subscribe = repository.get(subscribe.id)
|
||||||
|
if current_subscribe is None:
|
||||||
|
logger.info(f"音乐订阅 {subscribe.id} 已删除,放弃本轮下载提交")
|
||||||
|
return
|
||||||
|
if current_subscribe.state == "S":
|
||||||
|
logger.info(f"音乐订阅 {current_subscribe.name} 已暂停,放弃本轮下载提交")
|
||||||
|
return
|
||||||
|
if self._SubscribeChain__candidate_contract_changed(subscribe, current_subscribe):
|
||||||
|
logger.info(f"音乐订阅 {current_subscribe.name} 的筛选或媒体身份已变化,放弃旧候选并等待下一轮")
|
||||||
|
return
|
||||||
|
subscribe = current_subscribe
|
||||||
|
governance = None
|
||||||
|
if execution_context:
|
||||||
|
governance = SubscriptionDownloadGovernance(
|
||||||
|
cancelled=execution_context.should_stop,
|
||||||
|
mark_started=execution_context.mark_download_started,
|
||||||
|
)
|
||||||
|
execution_context.report_phase("preparing")
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
downloads, _ = DownloadChain().batch_download(
|
downloads, _ = DownloadChain().batch_download(
|
||||||
contexts=contexts,
|
contexts=contexts,
|
||||||
username=subscribe.username,
|
username=subscribe.username,
|
||||||
@@ -359,6 +387,7 @@ class MusicSubscribeMixin:
|
|||||||
downloader=subscribe.downloader,
|
downloader=subscribe.downloader,
|
||||||
source=self.get_subscribe_source_keyword(subscribe),
|
source=self.get_subscribe_source_keyword(subscribe),
|
||||||
custom_words=subscribe.custom_words,
|
custom_words=subscribe.custom_words,
|
||||||
|
governance=governance,
|
||||||
)
|
)
|
||||||
successful = [
|
successful = [
|
||||||
context for context in downloads or []
|
context for context in downloads or []
|
||||||
@@ -370,7 +399,6 @@ class MusicSubscribeMixin:
|
|||||||
context for context in successful
|
context for context in successful
|
||||||
if context.confirmed_full_coverage
|
if context.confirmed_full_coverage
|
||||||
]
|
]
|
||||||
repository = self.subscription_repository
|
|
||||||
current_subscribe = None
|
current_subscribe = None
|
||||||
if subscribe.best_version and quality_downloads:
|
if subscribe.best_version and quality_downloads:
|
||||||
best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order)
|
best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order)
|
||||||
@@ -397,12 +425,18 @@ class MusicSubscribeMixin:
|
|||||||
downloads=downloads,
|
downloads=downloads,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _search_music_subscribe(self, subscribe: SubscriptionSnapshot) -> None:
|
def _search_music_subscribe(
|
||||||
|
self,
|
||||||
|
subscribe: SubscriptionSnapshot,
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext] = None,
|
||||||
|
) -> None:
|
||||||
"""复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。"""
|
"""复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。"""
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
target = self._prepare_music_subscribe(subscribe)
|
target = self._prepare_music_subscribe(subscribe)
|
||||||
if not target:
|
if not target:
|
||||||
return
|
return
|
||||||
subscribe, mediainfo, _ = target
|
subscribe, mediainfo, _ = target
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
|
|
||||||
sites = self.get_sub_sites(subscribe)
|
sites = self.get_sub_sites(subscribe)
|
||||||
default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \
|
default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \
|
||||||
@@ -414,13 +448,17 @@ class MusicSubscribeMixin:
|
|||||||
|
|
||||||
searchchain = SearchChain()
|
searchchain = SearchChain()
|
||||||
contexts: List[Context] = []
|
contexts: List[Context] = []
|
||||||
|
if execution_context:
|
||||||
|
execution_context.report_phase("searching")
|
||||||
for keyword in keywords:
|
for keyword in keywords:
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
contexts = searchchain.search_by_title(
|
contexts = searchchain.search_by_title(
|
||||||
title=keyword,
|
title=keyword,
|
||||||
sites=sites,
|
sites=sites,
|
||||||
mtype=MediaType.MUSIC,
|
mtype=MediaType.MUSIC,
|
||||||
rule_groups=rule_groups,
|
rule_groups=rule_groups,
|
||||||
)
|
)
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
contexts = self._filter_music_subscribe_contexts(
|
contexts = self._filter_music_subscribe_contexts(
|
||||||
subscribe=subscribe,
|
subscribe=subscribe,
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
@@ -433,18 +471,28 @@ class MusicSubscribeMixin:
|
|||||||
logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源")
|
logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源")
|
||||||
return
|
return
|
||||||
|
|
||||||
self._download_music_subscribe(subscribe, mediainfo, contexts)
|
self._download_music_subscribe(
|
||||||
|
subscribe,
|
||||||
|
mediainfo,
|
||||||
|
contexts,
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
|
||||||
def _match_music_subscribe(
|
def _match_music_subscribe(
|
||||||
self,
|
self,
|
||||||
subscribe: SubscriptionSnapshot,
|
subscribe: SubscriptionSnapshot,
|
||||||
contexts: List[Context],
|
contexts: List[Context],
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""直接匹配本轮 RSS 缓存中的音乐资源,避免再次调用站点搜索接口。"""
|
"""直接匹配本轮 RSS 缓存中的音乐资源,避免再次调用站点搜索接口。"""
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
target = self._prepare_music_subscribe(subscribe)
|
target = self._prepare_music_subscribe(subscribe)
|
||||||
if not target:
|
if not target:
|
||||||
return
|
return
|
||||||
subscribe, mediainfo, _ = target
|
subscribe, mediainfo, _ = target
|
||||||
|
if execution_context:
|
||||||
|
execution_context.report_phase("matching")
|
||||||
|
self._ensure_music_execution_active(execution_context)
|
||||||
matched = self._filter_music_subscribe_contexts(
|
matched = self._filter_music_subscribe_contexts(
|
||||||
subscribe=subscribe,
|
subscribe=subscribe,
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
@@ -453,4 +501,21 @@ class MusicSubscribeMixin:
|
|||||||
if not matched:
|
if not matched:
|
||||||
logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源")
|
logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源")
|
||||||
return
|
return
|
||||||
self._download_music_subscribe(subscribe, mediainfo, matched)
|
self._download_music_subscribe(
|
||||||
|
subscribe,
|
||||||
|
mediainfo,
|
||||||
|
matched,
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ensure_music_execution_active(
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext],
|
||||||
|
) -> None:
|
||||||
|
"""在音乐搜索、匹配和提交前的安全边界执行取消与 TTL 检查。"""
|
||||||
|
if execution_context is None:
|
||||||
|
return
|
||||||
|
if execution_context.is_cancel_requested():
|
||||||
|
raise SubscriptionSearchCancelled("订阅搜索已取消")
|
||||||
|
if execution_context.is_expired():
|
||||||
|
raise TimeoutError("订阅执行已超过协作截止时间")
|
||||||
|
|||||||
@@ -71,7 +71,6 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
|
|||||||
self.transfer_execution_repository = context.transfer_execution_repository
|
self.transfer_execution_repository = context.transfer_execution_repository
|
||||||
self.media_server_repository = context.media_server_repository
|
self.media_server_repository = context.media_server_repository
|
||||||
self.download_failure_repository = context.download_failure_repository
|
self.download_failure_repository = context.download_failure_repository
|
||||||
self.subscription_download_repository = context.subscription_download_repository
|
|
||||||
self.user_repository = context.user_repository
|
self.user_repository = context.user_repository
|
||||||
self.runtime_config = context.configuration
|
self.runtime_config = context.configuration
|
||||||
self.stop_state = context.stop_state
|
self.stop_state = context.stop_state
|
||||||
|
|||||||
@@ -1,258 +0,0 @@
|
|||||||
"""订阅下载提交幂等身份与状态转换 owner。"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
from typing import Optional, Set, cast
|
|
||||||
|
|
||||||
from app.application.download.admission import (
|
|
||||||
DownloadReconciliationRequired,
|
|
||||||
SubscriptionDownloadClaim,
|
|
||||||
SubscriptionDownloadGovernance,
|
|
||||||
SubscriptionDownloadRepository,
|
|
||||||
SubscriptionDownloadRequest,
|
|
||||||
)
|
|
||||||
from app.chain.download.contract import _DownloadOwnerBase
|
|
||||||
from app.domain import episode as episode_rules
|
|
||||||
from app.domain.context import Context
|
|
||||||
from app.schemas.media import build_media_key, resolve_media_identity
|
|
||||||
from app.schemas.types import MediaType
|
|
||||||
|
|
||||||
|
|
||||||
class DownloadAdmissionOwner(_DownloadOwnerBase):
|
|
||||||
"""计算订阅提交唯一键并通过持久账本控制外部副作用。"""
|
|
||||||
|
|
||||||
def _build_subscription_download_request(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
context: Context,
|
|
||||||
episodes: Optional[Set[int]],
|
|
||||||
governance: SubscriptionDownloadGovernance,
|
|
||||||
delivery_scope: str,
|
|
||||||
) -> SubscriptionDownloadRequest:
|
|
||||||
"""组合逻辑媒体、资源、覆盖、模式和交付目标生成规范幂等请求。"""
|
|
||||||
media = context.media_info
|
|
||||||
meta = context.meta_info
|
|
||||||
torrent = context.torrent_info
|
|
||||||
media_source, media_id = resolve_media_identity(media=media)
|
|
||||||
media_key = build_media_key(media_source, media_id) or (
|
|
||||||
f"{getattr(media, 'title', '')}:{getattr(media, 'year', '')}"
|
|
||||||
)
|
|
||||||
seasons = sorted(set(getattr(meta, "season_list", None) or []))
|
|
||||||
selected = sorted(set(episodes or getattr(meta, "episode_list", None) or []))
|
|
||||||
if selected:
|
|
||||||
coverage = f"episodes:{episode_rules.format_ranges(selected)}"
|
|
||||||
elif seasons:
|
|
||||||
coverage = "seasons:" + ",".join(str(season) for season in seasons) + ":full"
|
|
||||||
else:
|
|
||||||
coverage = "full"
|
|
||||||
media_type = getattr(media, "type", None)
|
|
||||||
media_type_value = getattr(media_type, "value", media_type)
|
|
||||||
media_season = getattr(media, "season", None)
|
|
||||||
meta_season = getattr(meta, "season", None)
|
|
||||||
logical_identity = json.dumps(
|
|
||||||
{
|
|
||||||
"media_key": str(media_key or ""),
|
|
||||||
"media_type": str(media_type_value or ""),
|
|
||||||
"season": media_season if media_season is not None else meta_season,
|
|
||||||
"episode_group": getattr(media, "episode_group", None),
|
|
||||||
"music_type": getattr(media, "music_type", None),
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
resource_key = self._torrent_resource_key(torrent)
|
|
||||||
canonical = json.dumps(
|
|
||||||
{
|
|
||||||
"logical_identity": logical_identity,
|
|
||||||
"resource_key": resource_key,
|
|
||||||
"coverage": coverage,
|
|
||||||
"mode": governance.mode,
|
|
||||||
"delivery_scope": delivery_scope,
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
legacy_logical_identity = json.dumps(
|
|
||||||
{
|
|
||||||
"subscription_id": governance.subscription_id,
|
|
||||||
"media_key": str(media_key or ""),
|
|
||||||
"media_type": str(media_type_value or ""),
|
|
||||||
"season": media_season if media_season is not None else meta_season,
|
|
||||||
"episode_group": getattr(media, "episode_group", None),
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
legacy_canonical = json.dumps(
|
|
||||||
{
|
|
||||||
"logical_identity": legacy_logical_identity,
|
|
||||||
"resource_key": resource_key,
|
|
||||||
"coverage": coverage,
|
|
||||||
"mode": governance.mode,
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
return SubscriptionDownloadRequest(
|
|
||||||
idempotency_key=hashlib.sha256(canonical.encode("utf-8")).hexdigest(),
|
|
||||||
legacy_idempotency_key=hashlib.sha256(
|
|
||||||
legacy_canonical.encode("utf-8")
|
|
||||||
).hexdigest(),
|
|
||||||
subscription_id=governance.subscription_id,
|
|
||||||
task_id=governance.task_id,
|
|
||||||
logical_identity=logical_identity,
|
|
||||||
resource_key=resource_key,
|
|
||||||
coverage=coverage,
|
|
||||||
mode=governance.mode,
|
|
||||||
delivery_scope=delivery_scope,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _claim_subscription_download(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
context: Context,
|
|
||||||
episodes: Optional[Set[int]],
|
|
||||||
governance: Optional[SubscriptionDownloadGovernance],
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_uri: str,
|
|
||||||
) -> tuple[Optional[SubscriptionDownloadClaim], Optional[str]]:
|
|
||||||
"""在下载器调用前认领唯一提交权,并返回已成功提交的历史 hash。"""
|
|
||||||
if governance is None:
|
|
||||||
return None, None
|
|
||||||
legacy_hash = self._legacy_subscription_download_hash(
|
|
||||||
context=context,
|
|
||||||
episodes=episodes,
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
if legacy_hash:
|
|
||||||
return None, legacy_hash
|
|
||||||
repository = getattr(self, "subscription_download_repository", None)
|
|
||||||
if repository is None:
|
|
||||||
raise RuntimeError("订阅下载幂等仓储尚未配置")
|
|
||||||
request = self._build_subscription_download_request(
|
|
||||||
context=context,
|
|
||||||
episodes=episodes,
|
|
||||||
governance=governance,
|
|
||||||
delivery_scope=self._subscription_delivery_scope(
|
|
||||||
downloader=downloader,
|
|
||||||
download_uri=download_uri,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if request.legacy_idempotency_key:
|
|
||||||
legacy = repository.get(request.legacy_idempotency_key)
|
|
||||||
if legacy and legacy.state == "succeeded" and legacy.download_hash:
|
|
||||||
return None, legacy.download_hash
|
|
||||||
if legacy and legacy.state in {"submitting", "accepted", "reconcile_required"}:
|
|
||||||
raise DownloadReconciliationRequired(
|
|
||||||
f"订阅下载提交 {legacy.idempotency_key} 当前为 {legacy.state},需要先对账下载器"
|
|
||||||
)
|
|
||||||
claim = repository.claim(request)
|
|
||||||
snapshot = claim.snapshot
|
|
||||||
if claim.acquired:
|
|
||||||
if not snapshot.attempt_token:
|
|
||||||
raise RuntimeError("订阅下载提交已认领但缺少尝试令牌")
|
|
||||||
return claim, None
|
|
||||||
if snapshot.state == "succeeded" and snapshot.download_hash:
|
|
||||||
return claim, snapshot.download_hash
|
|
||||||
if snapshot.state in {"submitting", "accepted", "reconcile_required"}:
|
|
||||||
raise DownloadReconciliationRequired(
|
|
||||||
f"订阅下载提交 {snapshot.idempotency_key} 当前为 {snapshot.state},需要先对账下载器"
|
|
||||||
)
|
|
||||||
return claim, None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _subscription_delivery_scope(
|
|
||||||
*,
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_uri: str,
|
|
||||||
) -> str:
|
|
||||||
"""规范化实际下载器和保存目标,限定跨记录去重的产品边界。"""
|
|
||||||
return json.dumps(
|
|
||||||
{
|
|
||||||
"downloader": downloader or "auto",
|
|
||||||
"download_uri": download_uri,
|
|
||||||
},
|
|
||||||
ensure_ascii=False,
|
|
||||||
sort_keys=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
def _legacy_subscription_download_hash(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
context: Context,
|
|
||||||
episodes: Optional[Set[int]],
|
|
||||||
governance: SubscriptionDownloadGovernance,
|
|
||||||
) -> Optional[str]:
|
|
||||||
"""重读迁移前下载历史,兼容识别同订阅同 torrent 同覆盖的成功提交。"""
|
|
||||||
media = context.media_info
|
|
||||||
meta = context.meta_info
|
|
||||||
torrent = context.torrent_info
|
|
||||||
repository = getattr(self, "download_history_repository", None)
|
|
||||||
media_source, media_id = resolve_media_identity(media=media)
|
|
||||||
if repository is None or not media_source or not media_id or not torrent:
|
|
||||||
return None
|
|
||||||
histories = repository.get_by_media_identity(
|
|
||||||
media_source=media_source,
|
|
||||||
media_id=str(media_id),
|
|
||||||
music_type=getattr(media, "music_type", None),
|
|
||||||
)
|
|
||||||
expected_episodes = episode_rules.format_ranges(
|
|
||||||
sorted(set(episodes or getattr(meta, "episode_list", None) or []))
|
|
||||||
)
|
|
||||||
expected_season = str(getattr(meta, "season", None) or "")
|
|
||||||
for history in histories:
|
|
||||||
if not history.download_hash:
|
|
||||||
continue
|
|
||||||
if history.torrent_name != torrent.title or history.torrent_site != torrent.site_name:
|
|
||||||
continue
|
|
||||||
if not self._history_matches_subscription(history.note, governance.subscription_id):
|
|
||||||
continue
|
|
||||||
if getattr(history, "episode_group", None) != getattr(media, "episode_group", None):
|
|
||||||
continue
|
|
||||||
if expected_episodes:
|
|
||||||
if history.episodes == expected_episodes:
|
|
||||||
return cast(str, history.download_hash)
|
|
||||||
continue
|
|
||||||
if getattr(media, "type", None) == MediaType.TV and str(history.seasons or "") != expected_season:
|
|
||||||
continue
|
|
||||||
return cast(str, history.download_hash)
|
|
||||||
return None
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _history_matches_subscription(note: object, subscription_id: int) -> bool:
|
|
||||||
"""从下载历史来源快照确认记录属于同一订阅,而非同媒体其他记录。"""
|
|
||||||
if not isinstance(note, dict):
|
|
||||||
return False
|
|
||||||
source = note.get("source")
|
|
||||||
if not isinstance(source, str) or not source.startswith("Subscribe|"):
|
|
||||||
return False
|
|
||||||
try:
|
|
||||||
payload = json.loads(source.split("|", 1)[1])
|
|
||||||
return int(payload.get("id")) == subscription_id
|
|
||||||
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _subscription_download_cancelled(
|
|
||||||
governance: Optional[SubscriptionDownloadGovernance],
|
|
||||||
) -> bool:
|
|
||||||
"""在可安全取消边界调用入口提供的取消检查。"""
|
|
||||||
return bool(governance and governance.cancelled and governance.cancelled())
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _subscription_download_retry_at(error: Optional[str], ttl_seconds: int) -> str:
|
|
||||||
"""把明确拒绝的下载提交推迟到失败冷却到期后。"""
|
|
||||||
del error
|
|
||||||
return (
|
|
||||||
datetime.now(timezone.utc) + timedelta(seconds=max(1, ttl_seconds))
|
|
||||||
).isoformat(timespec="seconds")
|
|
||||||
|
|
||||||
def _subscription_download_repository(self) -> SubscriptionDownloadRepository:
|
|
||||||
"""返回已配置的订阅下载仓储,缺失时拒绝越过外部副作用边界。"""
|
|
||||||
repository = getattr(self, "subscription_download_repository", None)
|
|
||||||
if repository is None:
|
|
||||||
raise RuntimeError("订阅下载幂等仓储尚未配置")
|
|
||||||
return cast(SubscriptionDownloadRepository, repository)
|
|
||||||
@@ -87,13 +87,13 @@ class DownloadBatchOwner(_DownloadOwnerBase):
|
|||||||
:param userid: 用户ID
|
:param userid: 用户ID
|
||||||
:param username: 调用下载的用户名/插件名
|
:param username: 调用下载的用户名/插件名
|
||||||
:param downloader: 下载器
|
:param downloader: 下载器
|
||||||
:param custom_words: 下载来源自定义词;governance: 订阅幂等、入口任务和取消边界
|
:param custom_words: 下载来源自定义词
|
||||||
|
:param governance: 订阅取消与下载器副作用边界
|
||||||
:return: 已下载资源列表及剩余缺集,键格式为 no_exists[source:id]
|
:return: 已下载资源列表及剩余缺集,键格式为 no_exists[source:id]
|
||||||
"""
|
"""
|
||||||
no_exists_was_none = no_exists is None
|
no_exists_was_none = no_exists is None
|
||||||
if no_exists is None:
|
if no_exists is None:
|
||||||
no_exists = {}
|
no_exists = {}
|
||||||
|
|
||||||
# 已下载的项目
|
# 已下载的项目
|
||||||
downloaded_list: List[Context] = []
|
downloaded_list: List[Context] = []
|
||||||
custom_word_list = custom_words.splitlines() if custom_words else None
|
custom_word_list = custom_words.splitlines() if custom_words else None
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ if TYPE_CHECKING:
|
|||||||
"""向类型检查器声明同一 Facade 上的跨 owner 方法。"""
|
"""向类型检查器声明同一 Facade 上的跨 owner 方法。"""
|
||||||
|
|
||||||
_active_download_failure_fingerprints: Callable[..., Any]
|
_active_download_failure_fingerprints: Callable[..., Any]
|
||||||
_build_subscription_download_request: Callable[..., Any]
|
|
||||||
_append_no_exists: Callable[..., None]
|
_append_no_exists: Callable[..., None]
|
||||||
_build_download_notification: Callable[..., Any]
|
_build_download_notification: Callable[..., Any]
|
||||||
_build_download_failure_fingerprint: Callable[..., Any]
|
_build_download_failure_fingerprint: Callable[..., Any]
|
||||||
@@ -23,20 +22,14 @@ if TYPE_CHECKING:
|
|||||||
_is_job_active: Callable[..., bool]
|
_is_job_active: Callable[..., bool]
|
||||||
_is_subscribe_source: Callable[..., bool]
|
_is_subscribe_source: Callable[..., bool]
|
||||||
_log_download_failure_cooldown: Callable[..., None]
|
_log_download_failure_cooldown: Callable[..., None]
|
||||||
_legacy_subscription_download_hash: Callable[..., Any]
|
|
||||||
_history_matches_subscription: Callable[..., bool]
|
|
||||||
_media_identity_keys: Callable[..., set[str]]
|
_media_identity_keys: Callable[..., set[str]]
|
||||||
_matches_media_identity: Callable[..., bool]
|
_matches_media_identity: Callable[..., bool]
|
||||||
_prepare_batch_download_contexts: Callable[..., Any]
|
_prepare_batch_download_contexts: Callable[..., Any]
|
||||||
_claim_subscription_download: Callable[..., Any]
|
|
||||||
_record_download_failure: Callable[..., Any]
|
_record_download_failure: Callable[..., Any]
|
||||||
_resolve_media_download_dir: Callable[..., Any]
|
_resolve_media_download_dir: Callable[..., Any]
|
||||||
_settle_download_success: Callable[..., None]
|
_settle_download_success: Callable[..., None]
|
||||||
_submit_download_added_task: Callable[..., None]
|
_submit_download_added_task: Callable[..., None]
|
||||||
_torrent_resource_key: Callable[..., str]
|
|
||||||
_subscription_download_cancelled: Callable[..., bool]
|
_subscription_download_cancelled: Callable[..., bool]
|
||||||
_subscription_download_repository: Callable[..., Any]
|
|
||||||
_subscription_download_retry_at: Callable[..., str]
|
|
||||||
_validate_music_album_resource: Callable[..., Any]
|
_validate_music_album_resource: Callable[..., Any]
|
||||||
batch_download: Callable[..., Any]
|
batch_download: Callable[..., Any]
|
||||||
download_single: Callable[..., Any]
|
download_single: Callable[..., Any]
|
||||||
|
|||||||
@@ -3,7 +3,6 @@
|
|||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from typing import Any, TypeVar
|
from typing import Any, TypeVar
|
||||||
|
|
||||||
from app.chain.download.admission import DownloadAdmissionOwner
|
|
||||||
from app.chain.download.batch import DownloadBatchOwner
|
from app.chain.download.batch import DownloadBatchOwner
|
||||||
from app.chain.download.existence import DownloadExistenceOwner
|
from app.chain.download.existence import DownloadExistenceOwner
|
||||||
from app.chain.download.failure import DownloadFailureOwner
|
from app.chain.download.failure import DownloadFailureOwner
|
||||||
@@ -30,7 +29,6 @@ class DownloadChain(
|
|||||||
DownloadSubtitleOwner,
|
DownloadSubtitleOwner,
|
||||||
DownloadSelectionOwner,
|
DownloadSelectionOwner,
|
||||||
DownloadFailureOwner,
|
DownloadFailureOwner,
|
||||||
DownloadAdmissionOwner,
|
|
||||||
DownloadSubmissionOwner,
|
DownloadSubmissionOwner,
|
||||||
DownloadBatchOwner,
|
DownloadBatchOwner,
|
||||||
DownloadExistenceOwner,
|
DownloadExistenceOwner,
|
||||||
|
|||||||
@@ -10,10 +10,7 @@ from urllib.parse import urlencode, urljoin, urlparse
|
|||||||
|
|
||||||
from app.application.configuration import get_chain_runtime_config_snapshot
|
from app.application.configuration import get_chain_runtime_config_snapshot
|
||||||
from app.application.directory import validate_download_save_path
|
from app.application.directory import validate_download_save_path
|
||||||
from app.application.download.admission import (
|
from app.application.download.admission import SubscriptionDownloadGovernance
|
||||||
DownloadReconciliationRequired,
|
|
||||||
SubscriptionDownloadGovernance,
|
|
||||||
)
|
|
||||||
from app.application.torrent.download import TorrentHelper
|
from app.application.torrent.download import TorrentHelper
|
||||||
from app.chain.download.contract import _DownloadOwnerBase
|
from app.chain.download.contract import _DownloadOwnerBase
|
||||||
from app.chain.download.ports import (
|
from app.chain.download.ports import (
|
||||||
@@ -284,6 +281,13 @@ class _DownloadResourceOwner(_DownloadOwnerBase):
|
|||||||
class DownloadSubmissionOwner(_DownloadResourceOwner):
|
class DownloadSubmissionOwner(_DownloadResourceOwner):
|
||||||
"""单任务下载准备、提交与结算 owner。"""
|
"""单任务下载准备、提交与结算 owner。"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _subscription_download_cancelled(
|
||||||
|
governance: Optional[SubscriptionDownloadGovernance],
|
||||||
|
) -> bool:
|
||||||
|
"""在下载器副作用前读取订阅执行上下文的停止信号。"""
|
||||||
|
return bool(governance and governance.cancelled and governance.cancelled())
|
||||||
|
|
||||||
def download_single(self, context: Context,
|
def download_single(self, context: Context,
|
||||||
torrent_file: Optional[Path] = None,
|
torrent_file: Optional[Path] = None,
|
||||||
torrent_content: Optional[Union[str, bytes]] = None,
|
torrent_content: Optional[Union[str, bytes]] = None,
|
||||||
@@ -478,57 +482,20 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
|
|||||||
custom_words: Optional[str],
|
custom_words: Optional[str],
|
||||||
governance: Optional[SubscriptionDownloadGovernance],
|
governance: Optional[SubscriptionDownloadGovernance],
|
||||||
) -> tuple[Optional[str], Optional[str]]:
|
) -> tuple[Optional[str], Optional[str]]:
|
||||||
"""认领幂等提交权,调用下载器并分派成功或拒绝结算。"""
|
"""在可取消边界后调用下载器,并分派成功或拒绝结算。"""
|
||||||
if self._subscription_download_cancelled(governance):
|
if self._subscription_download_cancelled(governance):
|
||||||
return None, "订阅下载在提交前已取消"
|
return None, "订阅下载在提交前已取消"
|
||||||
admission, duplicate_hash = self._claim_subscription_download(
|
|
||||||
context=context,
|
|
||||||
episodes=episodes,
|
|
||||||
governance=governance,
|
|
||||||
downloader=downloader or prepared.site_downloader,
|
|
||||||
download_uri=prepared.download_uri,
|
|
||||||
)
|
|
||||||
if duplicate_hash:
|
|
||||||
if governance and governance.mark_started:
|
|
||||||
governance.mark_started()
|
|
||||||
logger.info(f"{prepared.torrent.title} 已由重叠订阅入口提交,复用任务 {duplicate_hash}")
|
|
||||||
return duplicate_hash, "下载任务已由重叠入口提交"
|
|
||||||
if admission is not None and not admission.acquired:
|
|
||||||
return None, (
|
|
||||||
f"订阅下载提交当前为 {admission.snapshot.state},"
|
|
||||||
f"最早可重试:{admission.snapshot.available_at or '待下一轮'}"
|
|
||||||
)
|
|
||||||
attempt_token = admission.snapshot.attempt_token if admission is not None else None
|
|
||||||
admission_key = admission.snapshot.idempotency_key if admission is not None else None
|
|
||||||
if admission is not None and self._subscription_download_cancelled(governance):
|
|
||||||
self._subscription_download_repository().mark_cancelled(
|
|
||||||
idempotency_key=admission.snapshot.idempotency_key,
|
|
||||||
attempt_token=admission.snapshot.attempt_token or "",
|
|
||||||
)
|
|
||||||
return None, "订阅下载在下载器调用前已取消"
|
|
||||||
if governance and governance.mark_started:
|
if governance and governance.mark_started:
|
||||||
governance.mark_started()
|
governance.mark_started()
|
||||||
try:
|
result = self.download(
|
||||||
result = self.download(
|
content=prepared.torrent_content,
|
||||||
content=prepared.torrent_content,
|
cookie=prepared.torrent.site_cookie,
|
||||||
cookie=prepared.torrent.site_cookie,
|
episodes=cast(Set[int], episodes),
|
||||||
episodes=cast(Set[int], episodes),
|
download_dir=prepared.download_dir,
|
||||||
download_dir=prepared.download_dir,
|
category=prepared.media.category,
|
||||||
category=prepared.media.category,
|
label=label,
|
||||||
label=label,
|
downloader=downloader or prepared.site_downloader,
|
||||||
downloader=downloader or prepared.site_downloader,
|
)
|
||||||
)
|
|
||||||
except Exception as err:
|
|
||||||
if admission_key and attempt_token:
|
|
||||||
self._subscription_download_repository().mark_reconcile_required(
|
|
||||||
idempotency_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
error=f"下载器调用异常:{str(err)}",
|
|
||||||
)
|
|
||||||
raise DownloadReconciliationRequired(
|
|
||||||
f"{prepared.torrent.title} 下载器结果不确定,已冻结自动重试"
|
|
||||||
) from err
|
|
||||||
raise
|
|
||||||
actual_downloader, download_hash, layout, error_msg = (
|
actual_downloader, download_hash, layout, error_msg = (
|
||||||
result if result else (None, None, None, "未找到下载器")
|
result if result else (None, None, None, "未找到下载器")
|
||||||
)
|
)
|
||||||
@@ -545,8 +512,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
|
|||||||
actual_downloader=actual_downloader,
|
actual_downloader=actual_downloader,
|
||||||
download_hash=download_hash,
|
download_hash=download_hash,
|
||||||
layout=layout,
|
layout=layout,
|
||||||
admission_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
self._record_rejected_download(
|
self._record_rejected_download(
|
||||||
@@ -559,8 +524,6 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
|
|||||||
userid=userid,
|
userid=userid,
|
||||||
actual_downloader=actual_downloader,
|
actual_downloader=actual_downloader,
|
||||||
error_msg=error_msg,
|
error_msg=error_msg,
|
||||||
admission_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
)
|
)
|
||||||
return download_hash, error_msg
|
return download_hash, error_msg
|
||||||
|
|
||||||
@@ -578,71 +541,28 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
|
|||||||
actual_downloader: Optional[str],
|
actual_downloader: Optional[str],
|
||||||
download_hash: str,
|
download_hash: str,
|
||||||
layout: Optional[str],
|
layout: Optional[str],
|
||||||
admission_key: Optional[str],
|
|
||||||
attempt_token: Optional[str],
|
|
||||||
) -> None:
|
) -> None:
|
||||||
"""持久化下载器接受事实,执行本地结算并确认成功终态。"""
|
"""按普通下载合同结算下载器明确返回的成功结果。"""
|
||||||
if admission_key and attempt_token:
|
self._settle_download_success(
|
||||||
accepted = self._subscription_download_repository().mark_accepted(
|
context=context,
|
||||||
idempotency_key=admission_key,
|
media=prepared.media,
|
||||||
attempt_token=attempt_token,
|
meta=prepared.meta,
|
||||||
downloader=actual_downloader,
|
torrent=prepared.torrent,
|
||||||
download_hash=download_hash,
|
folder_name=prepared.folder_name,
|
||||||
)
|
file_list=prepared.file_list,
|
||||||
if not accepted:
|
download_dir=prepared.download_dir,
|
||||||
raise DownloadReconciliationRequired(
|
layout=layout,
|
||||||
f"{prepared.torrent.title} 已被下载器接受,但本地接受状态写入失败"
|
downloader=actual_downloader,
|
||||||
)
|
download_hash=download_hash,
|
||||||
try:
|
download_episodes=prepared.download_episodes,
|
||||||
self._settle_download_success(
|
episodes=episodes,
|
||||||
context=context,
|
channel=channel,
|
||||||
media=prepared.media,
|
source=source,
|
||||||
meta=prepared.meta,
|
userid=userid,
|
||||||
torrent=prepared.torrent,
|
username=username,
|
||||||
folder_name=prepared.folder_name,
|
torrent_content=prepared.torrent_content,
|
||||||
file_list=prepared.file_list,
|
custom_words=custom_words,
|
||||||
download_dir=prepared.download_dir,
|
)
|
||||||
layout=layout,
|
|
||||||
downloader=actual_downloader,
|
|
||||||
download_hash=download_hash,
|
|
||||||
download_episodes=prepared.download_episodes,
|
|
||||||
episodes=episodes,
|
|
||||||
channel=channel,
|
|
||||||
source=source,
|
|
||||||
userid=userid,
|
|
||||||
username=username,
|
|
||||||
torrent_content=prepared.torrent_content,
|
|
||||||
custom_words=custom_words,
|
|
||||||
)
|
|
||||||
except Exception as err:
|
|
||||||
if admission_key and attempt_token:
|
|
||||||
self._subscription_download_repository().mark_reconcile_required(
|
|
||||||
idempotency_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
error=f"下载器已接受但本地结算失败:{str(err)}",
|
|
||||||
downloader=actual_downloader,
|
|
||||||
download_hash=download_hash,
|
|
||||||
)
|
|
||||||
raise DownloadReconciliationRequired(
|
|
||||||
f"{prepared.torrent.title} 已被下载器接受但本地结算失败,已转待对账"
|
|
||||||
) from err
|
|
||||||
raise
|
|
||||||
if admission_key and attempt_token:
|
|
||||||
succeeded = self._subscription_download_repository().mark_succeeded(
|
|
||||||
idempotency_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
)
|
|
||||||
if not succeeded:
|
|
||||||
self._subscription_download_repository().mark_reconcile_required(
|
|
||||||
idempotency_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
error="本地结算完成但幂等成功终态写入失败",
|
|
||||||
downloader=actual_downloader,
|
|
||||||
download_hash=download_hash,
|
|
||||||
)
|
|
||||||
raise DownloadReconciliationRequired(
|
|
||||||
f"{prepared.torrent.title} 本地结算完成但提交终态未确认,已转待对账"
|
|
||||||
)
|
|
||||||
|
|
||||||
def _record_rejected_download(
|
def _record_rejected_download(
|
||||||
self,
|
self,
|
||||||
@@ -656,20 +576,8 @@ class DownloadSubmissionOwner(_DownloadResourceOwner):
|
|||||||
userid: Union[str, int, None],
|
userid: Union[str, int, None],
|
||||||
actual_downloader: Optional[str],
|
actual_downloader: Optional[str],
|
||||||
error_msg: str,
|
error_msg: str,
|
||||||
admission_key: Optional[str],
|
|
||||||
attempt_token: Optional[str],
|
|
||||||
) -> None:
|
) -> None:
|
||||||
"""记录无外部副作用的明确拒绝,并通知原调用渠道。"""
|
"""按普通失败合同记录下载器明确拒绝并通知原调用渠道。"""
|
||||||
if admission_key and attempt_token:
|
|
||||||
self._subscription_download_repository().mark_retryable(
|
|
||||||
idempotency_key=admission_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
available_at=self._subscription_download_retry_at(
|
|
||||||
error_msg,
|
|
||||||
self._download_failure_ttl(error_msg),
|
|
||||||
),
|
|
||||||
error=error_msg,
|
|
||||||
)
|
|
||||||
logger.error(
|
logger.error(
|
||||||
f"{prepared.media.title_year} 添加下载任务失败:"
|
f"{prepared.media.title_year} 添加下载任务失败:"
|
||||||
f"{prepared.torrent.title} - {prepared.torrent.enclosure},{error_msg}"
|
f"{prepared.torrent.title} - {prepared.torrent.enclosure},{error_msg}"
|
||||||
|
|||||||
@@ -52,7 +52,7 @@ class SearchChain(ChainBase):
|
|||||||
self._subscription_site_budget_failure_lock = threading.Lock()
|
self._subscription_site_budget_failure_lock = threading.Lock()
|
||||||
|
|
||||||
def record_subscription_site_budget_failure(self, error: str) -> None:
|
def record_subscription_site_budget_failure(self, error: str) -> None:
|
||||||
"""线程安全地记录一个未执行站点,供订阅任务暴露聚合失败。"""
|
"""线程安全地记录一个站点执行失败,供订阅任务暴露聚合失败。"""
|
||||||
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
|
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
|
||||||
if lock is None:
|
if lock is None:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -220,7 +220,7 @@ class _SearchProviderSyncOwner(_SearchOwnerBase):
|
|||||||
return []
|
return []
|
||||||
with capture_site_search_observation() as observation:
|
with capture_site_search_observation() as observation:
|
||||||
try:
|
try:
|
||||||
return _search_site_page(
|
result = _search_site_page(
|
||||||
self,
|
self,
|
||||||
site=site,
|
site=site,
|
||||||
keyword=keyword,
|
keyword=keyword,
|
||||||
@@ -234,6 +234,13 @@ class _SearchProviderSyncOwner(_SearchOwnerBase):
|
|||||||
error=str(error),
|
error=str(error),
|
||||||
)
|
)
|
||||||
raise
|
raise
|
||||||
|
else:
|
||||||
|
if observation.attempted and observation.outcome not in {"success", "skipped"}:
|
||||||
|
failure = observation.error or observation.outcome
|
||||||
|
self.record_subscription_site_budget_failure(
|
||||||
|
f"站点 {site.get('name') or site_id} 搜索失败:{failure}"
|
||||||
|
)
|
||||||
|
return result
|
||||||
finally:
|
finally:
|
||||||
try:
|
try:
|
||||||
budget.finish(claim, observation)
|
budget.finish(claim, observation)
|
||||||
|
|||||||
@@ -14,8 +14,10 @@ if TYPE_CHECKING:
|
|||||||
class _SubscribeOwnerHost:
|
class _SubscribeOwnerHost:
|
||||||
"""声明 SubscribeChain 组合后向各 owner 提供的属性和兄弟职责。"""
|
"""声明 SubscribeChain 组合后向各 owner 提供的属性和兄弟职责。"""
|
||||||
|
|
||||||
_LOCK_TIMOUT: int
|
_SUBSCRIPTION_EXECUTION_TTL: int
|
||||||
_rlock: Any
|
_match_lock: Any
|
||||||
|
_search_queue_lock: Any
|
||||||
|
_subscription_execution_admission: Any
|
||||||
download_history_repository: Any
|
download_history_repository: Any
|
||||||
eventmanager: Any
|
eventmanager: Any
|
||||||
messagehelper: Any
|
messagehelper: Any
|
||||||
@@ -98,7 +100,6 @@ if TYPE_CHECKING:
|
|||||||
get_subscribed_sites: Callable[..., Any]
|
get_subscribed_sites: Callable[..., Any]
|
||||||
has_music_subscribe: Callable[..., Any]
|
has_music_subscribe: Callable[..., Any]
|
||||||
match: Callable[..., Any]
|
match: Callable[..., Any]
|
||||||
match_batch: Callable[..., Any]
|
|
||||||
media_exists: Callable[..., Any]
|
media_exists: Callable[..., Any]
|
||||||
media_files: Callable[..., Any]
|
media_files: Callable[..., Any]
|
||||||
obtain_images: Callable[..., Any]
|
obtain_images: Callable[..., Any]
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import threading
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from app.application.messaging.subscribe import SubscribeInteractionHandler
|
from app.application.messaging.subscribe import SubscribeInteractionHandler
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
from app.chain._interaction import InteractionChainMixin
|
from app.chain._interaction import InteractionChainMixin
|
||||||
from app.chain._music import MusicSubscribeMixin
|
from app.chain._music import MusicSubscribeMixin
|
||||||
from app.chain.base import ChainBase
|
from app.chain.base import ChainBase
|
||||||
@@ -48,9 +49,10 @@ class SubscribeChain(
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
_interaction_handler_type = SubscribeInteractionHandler
|
_interaction_handler_type = SubscribeInteractionHandler
|
||||||
_rlock = threading.RLock()
|
_match_lock = threading.Lock()
|
||||||
_search_queue_lock = threading.Lock()
|
_search_queue_lock = threading.Lock()
|
||||||
_LOCK_TIMOUT = 3600 * 2
|
_subscription_execution_admission = SubscriptionExecutionAdmission()
|
||||||
|
_SUBSCRIPTION_EXECUTION_TTL = 3600 * 2
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _music_media_chain(cls) -> MediaChain:
|
def _music_media_chain(cls) -> MediaChain:
|
||||||
|
|||||||
+595
-419
File diff suppressed because it is too large
Load Diff
@@ -8,6 +8,7 @@ from app.application.subscription import priority as _priority
|
|||||||
from app.application.subscription.contract import (
|
from app.application.subscription.contract import (
|
||||||
SubscriptionSnapshot,
|
SubscriptionSnapshot,
|
||||||
)
|
)
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionContext
|
||||||
from app.chain.download import DownloadChain
|
from app.chain.download import DownloadChain
|
||||||
from app.chain.subscribe.contract import _SubscribeOwnerBase
|
from app.chain.subscribe.contract import _SubscribeOwnerBase
|
||||||
from app.domain.context import (
|
from app.domain.context import (
|
||||||
@@ -281,6 +282,7 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
|
|||||||
save_path: Optional[str] = None,
|
save_path: Optional[str] = None,
|
||||||
downloader: Optional[str] = None,
|
downloader: Optional[str] = None,
|
||||||
source: Optional[str] = None,
|
source: Optional[str] = None,
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext] = None,
|
||||||
) -> Tuple[List[Context], Dict[Union[int, str], Dict[int, _SchemaNotExistMediaInfo]]]:
|
) -> Tuple[List[Context], Dict[Union[int, str], Dict[int, _SchemaNotExistMediaInfo]]]:
|
||||||
"""
|
"""
|
||||||
TV 分集洗版先尝试覆盖目标范围的全集资源,失败后回退到按集下载。
|
TV 分集洗版先尝试覆盖目标范围的全集资源,失败后回退到按集下载。
|
||||||
@@ -327,17 +329,8 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
|
|||||||
downloader = current.downloader
|
downloader = current.downloader
|
||||||
source = self.get_subscribe_source_keyword(current)
|
source = self.get_subscribe_source_keyword(current)
|
||||||
governance = SubscriptionDownloadGovernance(
|
governance = SubscriptionDownloadGovernance(
|
||||||
subscription_id=current.id,
|
cancelled=execution_context.should_stop if execution_context else None,
|
||||||
mode=self._SubscribeChain__download_governance_mode(current),
|
mark_started=execution_context.mark_download_started if execution_context else None,
|
||||||
task_id=cast(Optional[str], getattr(self, "_subscription_download_task_id", None)),
|
|
||||||
cancelled=cast(
|
|
||||||
Optional[Callable[[], bool]],
|
|
||||||
getattr(self, "_subscription_download_cancelled", None),
|
|
||||||
),
|
|
||||||
mark_started=cast(
|
|
||||||
Optional[Callable[[], None]],
|
|
||||||
getattr(self, "_subscription_download_mark_started", None),
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
full_pack_no_exists = self._SubscribeChain__build_full_pack_first_no_exists(
|
full_pack_no_exists = self._SubscribeChain__build_full_pack_first_no_exists(
|
||||||
subscribe=subscribe, mediakey=mediakey
|
subscribe=subscribe, mediakey=mediakey
|
||||||
@@ -465,17 +458,6 @@ class SubscribePolicyOwner(_SubscribePriorityPolicyOwner):
|
|||||||
accepted.append(context)
|
accepted.append(context)
|
||||||
return accepted
|
return accepted
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _SubscribeChain__download_governance_mode(subscribe: SubscriptionSnapshot) -> str:
|
|
||||||
"""把订阅模式归一为稳定幂等键组成部分。"""
|
|
||||||
if not subscribe.best_version:
|
|
||||||
return "normal"
|
|
||||||
if subscribe.type == MediaType.TV.value and subscribe.best_version_full:
|
|
||||||
return "best_version_full"
|
|
||||||
if subscribe.type == MediaType.TV.value:
|
|
||||||
return "best_version_episode"
|
|
||||||
return "best_version"
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _is_episode_range_covered(cls, meta: MetaBase, subscribe: SubscriptionSnapshot) -> bool:
|
def _is_episode_range_covered(cls, meta: MetaBase, subscribe: SubscriptionSnapshot) -> bool:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ from datetime import datetime
|
|||||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
||||||
|
|
||||||
from app.application.subscription import priority as _priority
|
from app.application.subscription import priority as _priority
|
||||||
from app.application.subscription.candidates import CandidateBatch
|
|
||||||
from app.application.subscription.contract import (
|
from app.application.subscription.contract import (
|
||||||
SubscriptionSnapshot,
|
SubscriptionSnapshot,
|
||||||
build_subscribe_meta,
|
build_subscribe_meta,
|
||||||
@@ -78,21 +77,14 @@ class SubscribeRefreshOwner(SubscribeMetadataOwner):
|
|||||||
)
|
)
|
||||||
|
|
||||||
torrents_chain = TorrentsChain()
|
torrents_chain = TorrentsChain()
|
||||||
candidate_batch = torrents_chain.refresh_batch(
|
candidates = torrents_chain.refresh(
|
||||||
sites=sites,
|
sites=sites,
|
||||||
progress_callback=_update_refresh_progress if progress_callback else None,
|
progress_callback=_update_refresh_progress if progress_callback else None,
|
||||||
# 存在音乐订阅时额外抓取站点音乐专用入口,音乐不一定在默认种子首页
|
# 存在音乐订阅时额外抓取站点音乐专用入口,音乐不一定在默认种子首页
|
||||||
include_music=self.has_music_subscribe(),
|
include_music=self.has_music_subscribe(),
|
||||||
)
|
)
|
||||||
if not isinstance(candidate_batch, CandidateBatch):
|
self.match(
|
||||||
legacy_candidates = torrents_chain.refresh(
|
candidates,
|
||||||
sites=sites,
|
|
||||||
progress_callback=_update_refresh_progress if progress_callback else None,
|
|
||||||
include_music=self.has_music_subscribe(),
|
|
||||||
)
|
|
||||||
candidate_batch = CandidateBatch.from_legacy(legacy_candidates or {}, source="refresh")
|
|
||||||
self.match_batch(
|
|
||||||
candidate_batch,
|
|
||||||
progress_callback=_update_match_progress if progress_callback else None,
|
progress_callback=_update_match_progress if progress_callback else None,
|
||||||
)
|
)
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
|
|||||||
+462
-206
@@ -14,7 +14,12 @@ from app.application.subscription.contract import (
|
|||||||
build_subscribe_meta,
|
build_subscribe_meta,
|
||||||
subscribe_media_key,
|
subscribe_media_key,
|
||||||
)
|
)
|
||||||
from app.application.subscription.execution import SearchBatchSnapshot, SubscriptionSearchRepository
|
from app.application.subscription.execution import (
|
||||||
|
SearchBatchSnapshot,
|
||||||
|
SearchTaskSnapshot,
|
||||||
|
SubscriptionExecutionContext,
|
||||||
|
SubscriptionSearchRepository,
|
||||||
|
)
|
||||||
from app.application.subscription.query import SubscriptionQueryService
|
from app.application.subscription.query import SubscriptionQueryService
|
||||||
from app.application.subscription.sitebudget import (
|
from app.application.subscription.sitebudget import (
|
||||||
SubscriptionSearchCancelled,
|
SubscriptionSearchCancelled,
|
||||||
@@ -36,6 +41,198 @@ from app.schemas.types import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _ensure_execution_active(
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext],
|
||||||
|
) -> None:
|
||||||
|
"""在可安全停止的边界区分用户取消与执行超时。"""
|
||||||
|
if execution_context is None:
|
||||||
|
return
|
||||||
|
if execution_context.is_cancel_requested():
|
||||||
|
raise SubscriptionSearchCancelled("订阅搜索已取消")
|
||||||
|
if execution_context.is_expired():
|
||||||
|
raise TimeoutError("订阅执行已超过协作截止时间")
|
||||||
|
|
||||||
|
|
||||||
|
def _update_search_task_phase(
|
||||||
|
queue: SubscriptionSearchRepository,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
phase: str,
|
||||||
|
current_site_id: Optional[int] = None,
|
||||||
|
) -> None:
|
||||||
|
"""以当前任务租约持久化业务阶段,过期执行者不得覆盖新状态。"""
|
||||||
|
queue.update_task_phase(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
phase=phase,
|
||||||
|
current_site_id=current_site_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _search_source_and_priority(
|
||||||
|
*,
|
||||||
|
sid: Optional[int],
|
||||||
|
sids: Optional[tuple[int, ...]],
|
||||||
|
state: Optional[str],
|
||||||
|
manual: Optional[bool],
|
||||||
|
) -> tuple[str, int]:
|
||||||
|
"""把兼容入口归一为持久来源和公平队列优先级。"""
|
||||||
|
if manual:
|
||||||
|
return "manual", 100
|
||||||
|
if sid or sids is not None:
|
||||||
|
return "targeted", 80
|
||||||
|
if state in {"R", "P"}:
|
||||||
|
return "fallback", 10
|
||||||
|
return "new", 50
|
||||||
|
|
||||||
|
|
||||||
|
def _search_task_available_at(
|
||||||
|
source: str,
|
||||||
|
subscription_ids: tuple[int, ...],
|
||||||
|
*,
|
||||||
|
now: Optional[datetime] = None,
|
||||||
|
) -> dict[int, str]:
|
||||||
|
"""把兜底搜索的随机节奏持久化为逐订阅到期时间。"""
|
||||||
|
ordered_ids = tuple(dict.fromkeys(subscription_ids))
|
||||||
|
if not ordered_ids:
|
||||||
|
return {}
|
||||||
|
cursor = now or datetime.now(timezone.utc)
|
||||||
|
if source == "fallback":
|
||||||
|
cursor += timedelta(seconds=random.randint(0, 60))
|
||||||
|
available_at: dict[int, str] = {}
|
||||||
|
for position, subscription_id in enumerate(ordered_ids):
|
||||||
|
if source == "fallback" and position:
|
||||||
|
cursor += timedelta(seconds=random.randint(60, 300))
|
||||||
|
available_at[subscription_id] = cursor.isoformat(timespec="seconds")
|
||||||
|
return available_at
|
||||||
|
|
||||||
|
|
||||||
|
def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
|
||||||
|
"""把批次聚合终态转为兼容进度文案。"""
|
||||||
|
if batch is None:
|
||||||
|
return "订阅搜索任务已提交"
|
||||||
|
if batch.state == "failed":
|
||||||
|
return "订阅搜索完成,部分任务失败"
|
||||||
|
if batch.state == "cancelled":
|
||||||
|
return "订阅搜索已取消"
|
||||||
|
if batch.state == "skipped":
|
||||||
|
return "订阅搜索完成,部分任务本轮已跳过"
|
||||||
|
if batch.state in {"queued", "running", "cancelling"}:
|
||||||
|
return "订阅搜索任务已排队"
|
||||||
|
if batch.skipped_count:
|
||||||
|
return "订阅搜索完成,部分任务本轮已跳过"
|
||||||
|
return "订阅搜索完成"
|
||||||
|
|
||||||
|
|
||||||
|
def _inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]:
|
||||||
|
"""返回兼容搜索的真实终态文案与计数。"""
|
||||||
|
text = (
|
||||||
|
"订阅搜索完成"
|
||||||
|
if finished == total
|
||||||
|
else "订阅搜索结束,部分订阅本轮未执行或未完成"
|
||||||
|
)
|
||||||
|
return text, {"total": total, "finished": finished}
|
||||||
|
|
||||||
|
|
||||||
|
def _batch_finished_count(
|
||||||
|
batch: Optional[SearchBatchSnapshot],
|
||||||
|
fallback: int,
|
||||||
|
) -> int:
|
||||||
|
"""返回批次所有终态任务数;批次暂不可读时使用本轮实际完成数。"""
|
||||||
|
if batch is None:
|
||||||
|
return fallback
|
||||||
|
return (
|
||||||
|
batch.finished_count
|
||||||
|
+ batch.failed_count
|
||||||
|
+ batch.cancelled_count
|
||||||
|
+ batch.skipped_count
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _skip_search_task(
|
||||||
|
queue: SubscriptionSearchRepository,
|
||||||
|
task: SearchTaskSnapshot,
|
||||||
|
reason: str,
|
||||||
|
) -> bool:
|
||||||
|
"""以 skipped 终态收口未执行任务,并保留可见原因。"""
|
||||||
|
if task.lease_token is None:
|
||||||
|
return False
|
||||||
|
return queue.finish_task(
|
||||||
|
task_id=task.task_id,
|
||||||
|
lease_token=task.lease_token,
|
||||||
|
state="skipped",
|
||||||
|
error=reason,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _release_cancelled_or_stopped_search_task(
|
||||||
|
queue: SubscriptionSearchRepository,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
system_stopped: bool,
|
||||||
|
) -> bool:
|
||||||
|
"""用户取消落终态,系统停机仅释放任务以供重启恢复。"""
|
||||||
|
return queue.release_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
cancelled=not system_stopped,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _finish_returned_search_task(
|
||||||
|
*,
|
||||||
|
queue: SubscriptionSearchRepository,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
subscription_id: int,
|
||||||
|
execution_context: SubscriptionExecutionContext,
|
||||||
|
system_stopped: bool,
|
||||||
|
cancel_requested: bool,
|
||||||
|
) -> Optional[int]:
|
||||||
|
"""按 TTL、停机和取消边界收口正常返回的搜索任务。"""
|
||||||
|
download_started = execution_context.download_started
|
||||||
|
if execution_context.is_expired() and not (system_stopped or cancel_requested):
|
||||||
|
if download_started:
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="completed",
|
||||||
|
error="执行截止时间晚于下载提交边界,已按实际结果完成",
|
||||||
|
)
|
||||||
|
return subscription_id
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="failed",
|
||||||
|
error="订阅执行已超过协作截止时间",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
if system_stopped:
|
||||||
|
if download_started:
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="completed",
|
||||||
|
error="停机请求晚于下载提交边界,已按实际结果完成",
|
||||||
|
)
|
||||||
|
return subscription_id
|
||||||
|
queue.release_task(task_id=task_id, lease_token=lease_token)
|
||||||
|
return None
|
||||||
|
if cancel_requested:
|
||||||
|
if download_started:
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="completed",
|
||||||
|
error="取消请求晚于下载提交边界,已按实际结果完成",
|
||||||
|
)
|
||||||
|
return subscription_id
|
||||||
|
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
|
||||||
|
return None
|
||||||
|
queue.finish_task(task_id=task_id, lease_token=lease_token, state="completed")
|
||||||
|
return subscription_id
|
||||||
|
|
||||||
|
|
||||||
class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
||||||
"""订阅主动搜索入口与持久队列消费 owner。"""
|
"""订阅主动搜索入口与持久队列消费 owner。"""
|
||||||
|
|
||||||
@@ -64,8 +261,9 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
operation: str,
|
operation: str,
|
||||||
progress_callback: Optional[Callable[..., None]],
|
progress_callback: Optional[Callable[..., None]],
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""获取订阅任务锁,超时时统一记录并结束本轮进度。"""
|
"""获取 Search 或 Match 通道锁,保持各通道内部串行。"""
|
||||||
if self._rlock.acquire(blocking=True, timeout=self._LOCK_TIMOUT):
|
lock = self._match_lock if operation == "match" else self._search_queue_lock
|
||||||
|
if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL):
|
||||||
logger.debug(f"{operation} lock acquired at {datetime.now()}")
|
logger.debug(f"{operation} lock acquired at {datetime.now()}")
|
||||||
return True
|
return True
|
||||||
operation_label = {"search": "搜索", "match": "匹配"}[operation]
|
operation_label = {"search": "搜索", "match": "匹配"}[operation]
|
||||||
@@ -148,7 +346,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
manual: Optional[bool],
|
manual: Optional[bool],
|
||||||
progress_callback: Optional[Callable[..., None]],
|
progress_callback: Optional[Callable[..., None]],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""保留未注入持久队列宿主的旧串行锁语义。"""
|
"""在独立 Search 通道内按订阅准入执行兼容搜索。"""
|
||||||
lock_acquired = self._acquire_run_lock("search", progress_callback)
|
lock_acquired = self._acquire_run_lock("search", progress_callback)
|
||||||
if not lock_acquired:
|
if not lock_acquired:
|
||||||
return
|
return
|
||||||
@@ -167,28 +365,70 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
for index, subscribe in enumerate(subscribes, start=1):
|
for index, subscribe in enumerate(subscribes, start=1):
|
||||||
if runtime_stop_state.is_system_stopped:
|
if runtime_stop_state.is_system_stopped:
|
||||||
break
|
break
|
||||||
processed.append(subscribe)
|
|
||||||
self._report_search_progress(progress_callback, subscribe, index, total)
|
self._report_search_progress(progress_callback, subscribe, index, total)
|
||||||
if self._defer_recent_subscription(subscribe):
|
if self._defer_recent_subscription(subscribe):
|
||||||
continue
|
continue
|
||||||
self._wait_before_scheduled_search(sid, sids, state, progress_callback)
|
self._wait_before_scheduled_search(sid, sids, state, progress_callback)
|
||||||
current = subscribe
|
lease = self._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
|
||||||
|
)
|
||||||
|
if lease is None:
|
||||||
|
logger.info(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过")
|
||||||
|
continue
|
||||||
|
execution_context = SubscriptionExecutionContext(
|
||||||
|
lease=lease,
|
||||||
|
admission=self._subscription_execution_admission,
|
||||||
|
cancel_requested=lambda: runtime_stop_state.is_system_stopped,
|
||||||
|
)
|
||||||
|
current = None
|
||||||
try:
|
try:
|
||||||
current = self._process_search_subscription(subscribe, searchchain)
|
current = self.subscription_repository.get(subscribe.id)
|
||||||
|
if current is None or current.state == "S":
|
||||||
|
if current and current.state == "S":
|
||||||
|
logger.info(f"订阅 {current.name} 已暂停,本轮搜索已跳过")
|
||||||
|
continue
|
||||||
|
processed_result = self._process_search_subscription(
|
||||||
|
current,
|
||||||
|
searchchain,
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
processed.append(processed_result or current)
|
||||||
|
current = processed_result
|
||||||
|
except SubscriptionSearchCancelled:
|
||||||
|
logger.info(f"订阅 {subscribe.name} 搜索已在安全边界取消")
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
|
||||||
finally:
|
finally:
|
||||||
if current and current.state == "N":
|
try:
|
||||||
self._SubscribeChain__apply_subscribe_update(
|
if current and current.state == "N":
|
||||||
current,
|
self._SubscribeChain__apply_subscribe_update(
|
||||||
{"state": "R"},
|
current,
|
||||||
scene="search_reset",
|
{"state": "R"},
|
||||||
|
scene="search_reset",
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(
|
||||||
|
f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}",
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self._subscription_execution_admission.release(lease)
|
||||||
|
self._report_search_progress(
|
||||||
|
progress_callback,
|
||||||
|
subscribe,
|
||||||
|
index,
|
||||||
|
total,
|
||||||
|
finished=True,
|
||||||
)
|
)
|
||||||
self._report_search_progress(progress_callback, subscribe, index, total, finished=True)
|
|
||||||
self._notify_manual_search(manual, sid, sids, subscribes, processed)
|
self._notify_manual_search(manual, sid, sids, subscribes, processed)
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
progress_callback(value=100, text="订阅搜索完成")
|
text, data = _inline_search_result(total, len(processed))
|
||||||
|
progress_callback(value=100, text=text, data=data)
|
||||||
finally:
|
finally:
|
||||||
subscribes.clear()
|
subscribes.clear()
|
||||||
self._rlock.release()
|
self._search_queue_lock.release()
|
||||||
logger.debug(f"search Lock released at {datetime.now()}")
|
logger.debug(f"search Lock released at {datetime.now()}")
|
||||||
|
|
||||||
def _execute_queued_search(
|
def _execute_queued_search(
|
||||||
@@ -203,17 +443,21 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
) -> str:
|
) -> str:
|
||||||
"""将搜索转为持久任务并在无 Match 长锁的短租约中串行消费。"""
|
"""将搜索转为持久任务并在无 Match 长锁的短租约中串行消费。"""
|
||||||
subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state)
|
subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state)
|
||||||
source, priority = self._search_source_and_priority(
|
source, priority = _search_source_and_priority(
|
||||||
sid=sid,
|
sid=sid,
|
||||||
sids=sids,
|
sids=sids,
|
||||||
state=state,
|
state=state,
|
||||||
manual=manual,
|
manual=manual,
|
||||||
)
|
)
|
||||||
|
subscription_ids = tuple(subscribe.id for subscribe in subscribes)
|
||||||
enqueued = queue.enqueue(
|
enqueued = queue.enqueue(
|
||||||
subscription_ids=tuple(subscribe.id for subscribe in subscribes),
|
subscription_ids=subscription_ids,
|
||||||
source=source,
|
source=source,
|
||||||
priority=priority,
|
priority=priority,
|
||||||
available_at=self._search_batch_available_at(source),
|
available_at_by_subscription=_search_task_available_at(
|
||||||
|
source,
|
||||||
|
subscription_ids,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
total = len(subscribes)
|
total = len(subscribes)
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
@@ -238,11 +482,11 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
batch = queue.get_batch(enqueued.batch.batch_id)
|
batch = queue.get_batch(enqueued.batch.batch_id)
|
||||||
progress_callback(
|
progress_callback(
|
||||||
value=100,
|
value=100,
|
||||||
text=self._batch_progress_text(batch),
|
text=_batch_progress_text(batch),
|
||||||
data={
|
data={
|
||||||
"batch_id": enqueued.batch.batch_id,
|
"batch_id": enqueued.batch.batch_id,
|
||||||
"total": total,
|
"total": total,
|
||||||
"finished": len(processed_subscribes),
|
"finished": _batch_finished_count(batch, len(processed_subscribes)),
|
||||||
"coalesced": enqueued.coalesced_count,
|
"coalesced": enqueued.coalesced_count,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -256,8 +500,7 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
progress_callback: Optional[Callable[..., None]],
|
progress_callback: Optional[Callable[..., None]],
|
||||||
) -> set[int]:
|
) -> set[int]:
|
||||||
"""有界消费可恢复任务;单任务失败不得阻止后续订阅。"""
|
"""有界消费可恢复任务;单任务失败不得阻止后续订阅。"""
|
||||||
queue_lock = getattr(self, "_search_queue_lock", None)
|
if not self._search_queue_lock.acquire(blocking=False):
|
||||||
if queue_lock is not None and not queue_lock.acquire(blocking=False):
|
|
||||||
logger.debug("订阅搜索队列已有消费者,本轮仅保留持久任务")
|
logger.debug("订阅搜索队列已有消费者,本轮仅保留持久任务")
|
||||||
return set()
|
return set()
|
||||||
owner = f"subscribe-search:{uuid4().hex}"
|
owner = f"subscribe-search:{uuid4().hex}"
|
||||||
@@ -270,147 +513,183 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
task = queue.claim_next(owner=owner)
|
task = queue.claim_next(owner=owner)
|
||||||
if task is None:
|
if task is None:
|
||||||
break
|
break
|
||||||
if not task.lease_token:
|
subscription_id = self._execute_search_task(
|
||||||
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行")
|
queue=queue,
|
||||||
continue
|
task=task,
|
||||||
task_id = str(task.task_id)
|
owner=owner,
|
||||||
cancelled = partial(queue.is_cancel_requested, task_id)
|
searchchain=searchchain,
|
||||||
if queue.is_cancel_requested(task.task_id):
|
index=index,
|
||||||
queue.release_task(
|
limit=limit,
|
||||||
task_id=task.task_id,
|
progress_callback=progress_callback,
|
||||||
lease_token=task.lease_token,
|
|
||||||
cancelled=True,
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
subscribe = self.subscription_repository.get(task.subscription_id)
|
|
||||||
if subscribe is None:
|
|
||||||
queue.finish_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
state="cancelled",
|
|
||||||
error="订阅已不存在",
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
self._report_search_progress(progress_callback, subscribe, index, limit)
|
|
||||||
if self._defer_recent_subscription(subscribe):
|
|
||||||
queue.finish_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
state="completed",
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
current = subscribe
|
|
||||||
phase_changed = partial(
|
|
||||||
self._update_search_task_phase,
|
|
||||||
queue,
|
|
||||||
task_id,
|
|
||||||
task.lease_token,
|
|
||||||
)
|
)
|
||||||
searchchain.configure_subscription_site_budget(
|
if subscription_id is not None:
|
||||||
SubscriptionSiteBudget(
|
processed.add(subscription_id)
|
||||||
repository=queue,
|
|
||||||
owner=f"{owner}:{task_id}",
|
|
||||||
cancelled=cancelled,
|
|
||||||
stop_state=getattr(self, "stop_state", runtime_stop_state),
|
|
||||||
phase_changed=phase_changed,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
self._subscription_download_task_id = task_id
|
|
||||||
self._subscription_download_cancelled = cancelled
|
|
||||||
self._subscription_download_crossed_boundary = False
|
|
||||||
self._subscription_download_mark_started = self._mark_subscription_download_started
|
|
||||||
self._subscription_execution_phase = phase_changed
|
|
||||||
try:
|
|
||||||
current = self._process_search_subscription(subscribe, searchchain)
|
|
||||||
if queue.is_cancel_requested(task.task_id):
|
|
||||||
if self._subscription_download_started_for_task(task.task_id):
|
|
||||||
queue.finish_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
state="completed",
|
|
||||||
error="取消请求晚于下载提交边界,已按实际结果完成",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
queue.release_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
cancelled=True,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
queue.finish_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
state="completed",
|
|
||||||
)
|
|
||||||
processed.add(subscribe.id)
|
|
||||||
except SubscriptionSearchCancelled:
|
|
||||||
queue.release_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
cancelled=True,
|
|
||||||
)
|
|
||||||
except Exception as err:
|
|
||||||
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
|
|
||||||
queue.finish_task(
|
|
||||||
task_id=task.task_id,
|
|
||||||
lease_token=task.lease_token,
|
|
||||||
state="failed",
|
|
||||||
error=str(err),
|
|
||||||
)
|
|
||||||
finally:
|
|
||||||
delattr(self, "_subscription_download_task_id")
|
|
||||||
delattr(self, "_subscription_download_cancelled")
|
|
||||||
delattr(self, "_subscription_download_mark_started")
|
|
||||||
delattr(self, "_subscription_execution_phase")
|
|
||||||
self._subscription_download_crossed_boundary = False
|
|
||||||
searchchain.configure_subscription_site_budget(None)
|
|
||||||
if current and current.state == "N":
|
|
||||||
try:
|
|
||||||
self._SubscribeChain__apply_subscribe_update(
|
|
||||||
current,
|
|
||||||
{"state": "R"},
|
|
||||||
scene="search_reset",
|
|
||||||
)
|
|
||||||
except Exception as err:
|
|
||||||
logger.error(
|
|
||||||
f"订阅 {current.name} 搜索后状态重置失败:{str(err)}",
|
|
||||||
exc_info=True,
|
|
||||||
)
|
|
||||||
self._report_search_progress(progress_callback, subscribe, index, limit, finished=True)
|
|
||||||
finally:
|
finally:
|
||||||
if queue_lock is not None:
|
self._search_queue_lock.release()
|
||||||
queue_lock.release()
|
|
||||||
return processed
|
return processed
|
||||||
|
|
||||||
def _subscription_download_started_for_task(self, task_id: str) -> bool:
|
def _execute_search_task(
|
||||||
"""判断取消是否已晚于下载器副作用边界。"""
|
self,
|
||||||
if bool(getattr(self, "_subscription_download_crossed_boundary", False)):
|
*,
|
||||||
return True
|
|
||||||
repository = getattr(self, "subscription_download_repository", None)
|
|
||||||
return bool(repository and repository.has_started_for_task(task_id))
|
|
||||||
|
|
||||||
def _mark_subscription_download_started(self) -> None:
|
|
||||||
"""记录当前搜索任务已提交或复用了真实下载结果。"""
|
|
||||||
self._subscription_download_crossed_boundary = True
|
|
||||||
phase_changed = getattr(self, "_subscription_execution_phase", None)
|
|
||||||
if phase_changed:
|
|
||||||
phase_changed("submitting", None)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _update_search_task_phase(
|
|
||||||
queue: SubscriptionSearchRepository,
|
queue: SubscriptionSearchRepository,
|
||||||
task_id: str,
|
task: SearchTaskSnapshot,
|
||||||
lease_token: str,
|
owner: str,
|
||||||
phase: str,
|
searchchain: SearchChain,
|
||||||
current_site_id: Optional[int] = None,
|
index: int,
|
||||||
) -> None:
|
limit: int,
|
||||||
"""以当前任务租约持久化业务阶段,过期执行者不得覆盖新状态。"""
|
progress_callback: Optional[Callable[..., None]],
|
||||||
queue.update_task_phase(
|
) -> Optional[int]:
|
||||||
task_id=task_id,
|
"""执行一条已认领任务,返回实际进入搜索处理的订阅 ID。"""
|
||||||
lease_token=lease_token,
|
if task.lease_token is None:
|
||||||
phase=phase,
|
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行")
|
||||||
current_site_id=current_site_id,
|
return None
|
||||||
|
task_id = str(task.task_id)
|
||||||
|
lease_token = task.lease_token
|
||||||
|
cancelled = partial(queue.is_cancel_requested, task_id)
|
||||||
|
stop_state = getattr(self, "stop_state", runtime_stop_state)
|
||||||
|
if cancelled():
|
||||||
|
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
|
||||||
|
return None
|
||||||
|
subscribe = self.subscription_repository.get(task.subscription_id)
|
||||||
|
if subscribe is None:
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="cancelled",
|
||||||
|
error="订阅已不存在",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
self._report_search_progress(progress_callback, subscribe, index, limit)
|
||||||
|
if self._defer_recent_subscription(subscribe):
|
||||||
|
_skip_search_task(queue, task, "订阅仍在新增保护期,本轮搜索已跳过")
|
||||||
|
return None
|
||||||
|
lease = self._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
|
||||||
)
|
)
|
||||||
|
if lease is None:
|
||||||
|
logger.info(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过")
|
||||||
|
_skip_search_task(queue, task, "同一订阅正在由其他通道处理,本轮搜索已跳过")
|
||||||
|
return None
|
||||||
|
phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token)
|
||||||
|
execution_context = SubscriptionExecutionContext(
|
||||||
|
lease=lease,
|
||||||
|
admission=self._subscription_execution_admission,
|
||||||
|
task_id=task_id,
|
||||||
|
cancel_requested=lambda: cancelled() or stop_state.is_system_stopped,
|
||||||
|
phase_changed=phase_changed,
|
||||||
|
)
|
||||||
|
current = subscribe
|
||||||
|
try:
|
||||||
|
current = self.subscription_repository.get(task.subscription_id)
|
||||||
|
if current is None:
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="cancelled",
|
||||||
|
error="订阅已不存在",
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
if current.state == "S":
|
||||||
|
_skip_search_task(queue, task, "订阅已暂停,本轮搜索已跳过")
|
||||||
|
return None
|
||||||
|
searchchain.configure_subscription_site_budget(
|
||||||
|
SubscriptionSiteBudget(
|
||||||
|
repository=queue,
|
||||||
|
owner=f"{owner}:{task_id}",
|
||||||
|
cancelled=execution_context.should_stop,
|
||||||
|
stop_state=stop_state,
|
||||||
|
phase_changed=phase_changed,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
current = self._process_search_subscription(
|
||||||
|
current,
|
||||||
|
searchchain,
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
system_stopped = stop_state.is_system_stopped
|
||||||
|
cancel_requested = False if system_stopped else cancelled()
|
||||||
|
return _finish_returned_search_task(
|
||||||
|
queue=queue,
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
subscription_id=task.subscription_id,
|
||||||
|
execution_context=execution_context,
|
||||||
|
system_stopped=system_stopped,
|
||||||
|
cancel_requested=cancel_requested,
|
||||||
|
)
|
||||||
|
except SubscriptionSearchCancelled:
|
||||||
|
if execution_context.is_expired() and not execution_context.is_cancel_requested():
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="failed",
|
||||||
|
error="订阅执行已超过协作截止时间",
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
_release_cancelled_or_stopped_search_task(
|
||||||
|
queue,
|
||||||
|
task_id,
|
||||||
|
lease_token,
|
||||||
|
stop_state.is_system_stopped,
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
|
||||||
|
queue.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="failed",
|
||||||
|
error=str(err),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self._cleanup_search_task(
|
||||||
|
queue=queue,
|
||||||
|
searchchain=searchchain,
|
||||||
|
subscribe=subscribe,
|
||||||
|
current=current,
|
||||||
|
lease=lease,
|
||||||
|
progress_callback=progress_callback,
|
||||||
|
index=index,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _cleanup_search_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
queue: SubscriptionSearchRepository,
|
||||||
|
searchchain: SearchChain,
|
||||||
|
subscribe: SubscriptionSnapshot,
|
||||||
|
current: Optional[SubscriptionSnapshot],
|
||||||
|
lease: Any,
|
||||||
|
progress_callback: Optional[Callable[..., None]],
|
||||||
|
index: int,
|
||||||
|
limit: int,
|
||||||
|
) -> None:
|
||||||
|
"""清理站点预算和订阅状态,并在所有异常路径释放 owner。"""
|
||||||
|
try:
|
||||||
|
searchchain.configure_subscription_site_budget(None)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"订阅 {subscribe.name} 搜索站点预算清理失败:{str(err)}", exc_info=True)
|
||||||
|
try:
|
||||||
|
if current and current.state == "N":
|
||||||
|
self._SubscribeChain__apply_subscribe_update(
|
||||||
|
current,
|
||||||
|
{"state": "R"},
|
||||||
|
scene="search_reset",
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}", exc_info=True)
|
||||||
|
finally:
|
||||||
|
self._subscription_execution_admission.release(lease)
|
||||||
|
self._report_search_progress(
|
||||||
|
progress_callback,
|
||||||
|
subscribe,
|
||||||
|
index,
|
||||||
|
limit,
|
||||||
|
finished=True,
|
||||||
|
)
|
||||||
|
|
||||||
def resume_search_queue(
|
def resume_search_queue(
|
||||||
self,
|
self,
|
||||||
@@ -429,44 +708,6 @@ class _SubscribeSearchQueueOwner(_SubscribeOwnerBase):
|
|||||||
progress_callback=progress_callback,
|
progress_callback=progress_callback,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _search_source_and_priority(
|
|
||||||
*,
|
|
||||||
sid: Optional[int],
|
|
||||||
sids: Optional[tuple[int, ...]],
|
|
||||||
state: Optional[str],
|
|
||||||
manual: Optional[bool],
|
|
||||||
) -> tuple[str, int]:
|
|
||||||
"""把兼容入口归一为持久来源和公平队列优先级。"""
|
|
||||||
if manual:
|
|
||||||
return "manual", 100
|
|
||||||
if sid or sids is not None:
|
|
||||||
return "targeted", 80
|
|
||||||
if state in {"R", "P"}:
|
|
||||||
return "fallback", 10
|
|
||||||
return "new", 50
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
|
|
||||||
"""把批次聚合终态转为兼容进度文案。"""
|
|
||||||
if batch is None:
|
|
||||||
return "订阅搜索任务已提交"
|
|
||||||
if batch.state == "failed":
|
|
||||||
return "订阅搜索完成,部分任务失败"
|
|
||||||
if batch.state == "cancelled":
|
|
||||||
return "订阅搜索已取消"
|
|
||||||
if batch.state in {"queued", "running", "cancelling"}:
|
|
||||||
return "订阅搜索任务已排队"
|
|
||||||
return "订阅搜索完成"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _search_batch_available_at(source: str) -> str:
|
|
||||||
"""为自动兜底批次持久化一次全局启动抖动。"""
|
|
||||||
delay = random.randint(0, 60) if source == "fallback" else 0
|
|
||||||
return (
|
|
||||||
datetime.now(timezone.utc) + timedelta(seconds=delay)
|
|
||||||
).isoformat(timespec="seconds")
|
|
||||||
|
|
||||||
def cancel_search_batch(self, batch_id: str) -> bool:
|
def cancel_search_batch(self, batch_id: str) -> bool:
|
||||||
"""请求取消持久搜索批次;未注入队列时返回失败。"""
|
"""请求取消持久搜索批次;未注入队列时返回失败。"""
|
||||||
queue: Optional[SubscriptionSearchRepository] = getattr(
|
queue: Optional[SubscriptionSearchRepository] = getattr(
|
||||||
@@ -518,8 +759,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
state: Optional[str],
|
state: Optional[str],
|
||||||
progress_callback: Optional[Callable[..., None]],
|
progress_callback: Optional[Callable[..., None]],
|
||||||
) -> None:
|
) -> None:
|
||||||
"""为自动搜索增加随机间隔,手动和定向批次不等待。"""
|
"""未使用持久队列时为自动兜底搜索保留逐订阅随机间隔。"""
|
||||||
if sid or sids is not None or state not in ["R", "P"]:
|
if sid or sids is not None or state not in {"R", "P"}:
|
||||||
return
|
return
|
||||||
sleep_time = random.randint(60, 300)
|
sleep_time = random.randint(60, 300)
|
||||||
logger.info(f"订阅搜索随机休眠 {sleep_time} 秒 ...")
|
logger.info(f"订阅搜索随机休眠 {sleep_time} 秒 ...")
|
||||||
@@ -531,11 +772,13 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
self,
|
self,
|
||||||
subscribe: SubscriptionSnapshot,
|
subscribe: SubscriptionSnapshot,
|
||||||
searchchain: SearchChain,
|
searchchain: SearchChain,
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext] = None,
|
||||||
) -> Optional[SubscriptionSnapshot]:
|
) -> Optional[SubscriptionSnapshot]:
|
||||||
"""处理单个订阅,并返回下载后重新读取的状态快照。"""
|
"""处理单个订阅,并返回下载后重新读取的状态快照。"""
|
||||||
|
_ensure_execution_active(execution_context)
|
||||||
logger.info(f"开始搜索订阅,标题:{subscribe.name} ...")
|
logger.info(f"开始搜索订阅,标题:{subscribe.name} ...")
|
||||||
if subscribe.type == MediaType.MUSIC.value:
|
if subscribe.type == MediaType.MUSIC.value:
|
||||||
self._search_music_subscribe(subscribe)
|
self._search_music_subscribe(subscribe, execution_context=execution_context)
|
||||||
return subscribe
|
return subscribe
|
||||||
try:
|
try:
|
||||||
meta = build_subscribe_meta(subscribe)
|
meta = build_subscribe_meta(subscribe)
|
||||||
@@ -555,6 +798,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
f"媒体来源:{subscribe.media_source},媒体 ID:{subscribe.media_id}"
|
f"媒体来源:{subscribe.media_source},媒体 ID:{subscribe.media_id}"
|
||||||
)
|
)
|
||||||
return subscribe
|
return subscribe
|
||||||
|
_ensure_execution_active(execution_context)
|
||||||
mediakey = subscribe_media_key(subscribe)
|
mediakey = subscribe_media_key(subscribe)
|
||||||
exists, no_exists = self.check_and_handle_existing_media(
|
exists, no_exists = self.check_and_handle_existing_media(
|
||||||
subscribe=subscribe,
|
subscribe=subscribe,
|
||||||
@@ -569,9 +813,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
if subscribe.best_version
|
if subscribe.best_version
|
||||||
else SystemConfigKey.SubscribeFilterRuleGroups
|
else SystemConfigKey.SubscribeFilterRuleGroups
|
||||||
)
|
)
|
||||||
phase_changed = getattr(self, "_subscription_execution_phase", None)
|
if execution_context:
|
||||||
if phase_changed:
|
execution_context.report_phase("searching")
|
||||||
phase_changed("searching", None)
|
|
||||||
contexts = searchchain.process(
|
contexts = searchchain.process(
|
||||||
mediainfo=mediainfo,
|
mediainfo=mediainfo,
|
||||||
keyword=subscribe.keyword,
|
keyword=subscribe.keyword,
|
||||||
@@ -583,6 +826,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
filter_params=self.get_params(subscribe),
|
filter_params=self.get_params(subscribe),
|
||||||
)
|
)
|
||||||
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
|
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
|
||||||
|
_ensure_execution_active(execution_context)
|
||||||
if not contexts:
|
if not contexts:
|
||||||
logger.warning(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
|
logger.warning(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
|
||||||
self.finish_subscribe_or_not(
|
self.finish_subscribe_or_not(
|
||||||
@@ -599,8 +843,9 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
|
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
|
||||||
self._raise_site_budget_failures(site_budget_failures)
|
self._raise_site_budget_failures(site_budget_failures)
|
||||||
return subscribe
|
return subscribe
|
||||||
if phase_changed:
|
if execution_context:
|
||||||
phase_changed("preparing", None)
|
execution_context.report_phase("preparing")
|
||||||
|
_ensure_execution_active(execution_context)
|
||||||
downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first(
|
downloads, lefts = self._SubscribeChain__download_best_version_with_full_pack_first(
|
||||||
contexts=matched,
|
contexts=matched,
|
||||||
no_exists=no_exists,
|
no_exists=no_exists,
|
||||||
@@ -610,6 +855,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
save_path=subscribe.save_path,
|
save_path=subscribe.save_path,
|
||||||
downloader=subscribe.downloader,
|
downloader=subscribe.downloader,
|
||||||
source=self.get_subscribe_source_keyword(subscribe),
|
source=self.get_subscribe_source_keyword(subscribe),
|
||||||
|
execution_context=execution_context,
|
||||||
)
|
)
|
||||||
current = self.subscription_repository.get(subscribe.id)
|
current = self.subscription_repository.get(subscribe.id)
|
||||||
if current:
|
if current:
|
||||||
@@ -625,7 +871,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _raise_site_budget_failures(failures: tuple[str, ...]) -> None:
|
def _raise_site_budget_failures(failures: tuple[str, ...]) -> None:
|
||||||
"""在成功站点结果完成处理后暴露未执行站点的聚合失败。"""
|
"""在成功站点结果完成处理后暴露其余站点的聚合失败。"""
|
||||||
if failures:
|
if failures:
|
||||||
raise RuntimeError(";".join(failures))
|
raise RuntimeError(";".join(failures))
|
||||||
|
|
||||||
@@ -713,9 +959,19 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
if not subscribes:
|
if not subscribes:
|
||||||
self.messagehelper.put("没有找到订阅!", title="订阅搜索", role="system")
|
self.messagehelper.put("没有找到订阅!", title="订阅搜索", role="system")
|
||||||
elif sid:
|
elif sid:
|
||||||
self.messagehelper.put(f"{subscribes[0].name} 搜索完成!", title="订阅搜索", role="system")
|
message = (
|
||||||
|
f"{subscribes[0].name} 搜索完成!"
|
||||||
|
if processed
|
||||||
|
else f"{subscribes[0].name} 本轮未执行或未完成,将等待下一次正常调度。"
|
||||||
|
)
|
||||||
|
self.messagehelper.put(message, title="订阅搜索", role="system")
|
||||||
elif sids is not None:
|
elif sids is not None:
|
||||||
for subscribe in processed:
|
for subscribe in processed:
|
||||||
self.messagehelper.put(f"{subscribe.name} 搜索完成!", title="订阅搜索", role="system")
|
self.messagehelper.put(f"{subscribe.name} 搜索完成!", title="订阅搜索", role="system")
|
||||||
else:
|
else:
|
||||||
self.messagehelper.put("所有订阅搜索完成!", title="订阅搜索", role="system")
|
message = (
|
||||||
|
"所有订阅搜索完成!"
|
||||||
|
if len(processed) == len(subscribes)
|
||||||
|
else "订阅搜索完成,部分订阅本轮未执行或未完成。"
|
||||||
|
)
|
||||||
|
self.messagehelper.put(message, title="订阅搜索", role="system")
|
||||||
|
|||||||
+3
-60
@@ -1,12 +1,11 @@
|
|||||||
import re
|
import re
|
||||||
import traceback
|
import traceback
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Callable, Dict, List, Optional, Union
|
from typing import Callable, Dict, List, Optional, Union
|
||||||
|
|
||||||
from app.application.configuration import get_configured_system_config
|
from app.application.configuration import get_configured_system_config
|
||||||
from app.application.rss import RssHelper
|
from app.application.rss import RssHelper
|
||||||
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
|
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
|
||||||
from app.application.subscription.candidates import CandidateBatch, CandidateIndex
|
from app.application.subscription.candidates import CandidateIndex
|
||||||
from app.application.torrent.download import TorrentHelper
|
from app.application.torrent.download import TorrentHelper
|
||||||
from app.chain.base import ChainBase
|
from app.chain.base import ChainBase
|
||||||
from app.chain.media import MediaChain
|
from app.chain.media import MediaChain
|
||||||
@@ -422,8 +421,6 @@ class TorrentsChain(ChainBase):
|
|||||||
include_music: bool,
|
include_music: bool,
|
||||||
torrents_cache: Dict[str, List[Context]],
|
torrents_cache: Dict[str, List[Context]],
|
||||||
music_cache: Dict[str, List[Context]],
|
music_cache: Dict[str, List[Context]],
|
||||||
fresh_torrents: Dict[str, List[Context]],
|
|
||||||
fresh_music: Dict[str, List[Context]],
|
|
||||||
) -> str:
|
) -> str:
|
||||||
"""抓取并写入单个站点的影视、音乐资源缓存。"""
|
"""抓取并写入单个站点的影视、音乐资源缓存。"""
|
||||||
domain = site_rules.extract_domain(indexer.get("domain"))
|
domain = site_rules.extract_domain(indexer.get("domain"))
|
||||||
@@ -483,9 +480,7 @@ class TorrentsChain(ChainBase):
|
|||||||
continue
|
continue
|
||||||
context = self._build_refresh_context(torrent, stype)
|
context = self._build_refresh_context(torrent, stype)
|
||||||
target_cache = music_cache if torrent.category == MediaType.MUSIC.value else torrents_cache
|
target_cache = music_cache if torrent.category == MediaType.MUSIC.value else torrents_cache
|
||||||
target_fresh = fresh_music if torrent.category == MediaType.MUSIC.value else fresh_torrents
|
|
||||||
target_cache.setdefault(domain, []).append(context)
|
target_cache.setdefault(domain, []).append(context)
|
||||||
target_fresh.setdefault(domain, []).append(context)
|
|
||||||
if len(target_cache[domain]) > self.runtime_config.torrent_cache_size:
|
if len(target_cache[domain]) > self.runtime_config.torrent_cache_size:
|
||||||
target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:]
|
target_cache[domain] = target_cache[domain][-self.runtime_config.torrent_cache_size:]
|
||||||
return domain
|
return domain
|
||||||
@@ -548,31 +543,14 @@ class TorrentsChain(ChainBase):
|
|||||||
progress_callback: Optional[Callable[..., None]] = None,
|
progress_callback: Optional[Callable[..., None]] = None,
|
||||||
include_music: bool = False,
|
include_music: bool = False,
|
||||||
) -> Dict[str, List[Context]]:
|
) -> Dict[str, List[Context]]:
|
||||||
"""兼容旧调用返回完整候选字典,批次语义由 ``refresh_batch`` 提供。"""
|
|
||||||
return self.refresh_batch(
|
|
||||||
stype=stype,
|
|
||||||
sites=sites,
|
|
||||||
progress_callback=progress_callback,
|
|
||||||
include_music=include_music,
|
|
||||||
).candidates
|
|
||||||
|
|
||||||
def refresh_batch(
|
|
||||||
self,
|
|
||||||
stype: Optional[str] = None,
|
|
||||||
sites: List[int] = None,
|
|
||||||
progress_callback: Optional[Callable[..., None]] = None,
|
|
||||||
include_music: bool = False,
|
|
||||||
) -> CandidateBatch:
|
|
||||||
"""
|
"""
|
||||||
刷新站点最新资源并返回完整缓存与本轮新增候选。
|
刷新站点最新资源并返回本轮可匹配的完整候选缓存。
|
||||||
|
|
||||||
:param stype: 强制指定缓存类型,spider:爬虫缓存,rss:rss缓存
|
:param stype: 强制指定缓存类型,spider:爬虫缓存,rss:rss缓存
|
||||||
:param sites: 强制指定站点ID列表,为空则读取设置的订阅站点
|
:param sites: 强制指定站点ID列表,为空则读取设置的订阅站点
|
||||||
:param progress_callback: 资源刷新进度更新回调
|
:param progress_callback: 资源刷新进度更新回调
|
||||||
:param include_music: 是否额外抓取站点的音乐专用浏览入口,服务音乐订阅
|
:param include_music: 是否额外抓取站点的音乐专用浏览入口,服务音乐订阅
|
||||||
"""
|
"""
|
||||||
started_at = datetime.now(timezone.utc)
|
|
||||||
|
|
||||||
# 刷新类型
|
# 刷新类型
|
||||||
if not stype:
|
if not stype:
|
||||||
stype = self.runtime_config.subscribe_mode
|
stype = self.runtime_config.subscribe_mode
|
||||||
@@ -590,8 +568,6 @@ class TorrentsChain(ChainBase):
|
|||||||
music_cache = self.load_cache(self._music_rss_file) or {}
|
music_cache = self.load_cache(self._music_rss_file) or {}
|
||||||
self._ensure_context_compatibility(torrents_cache, stype=stype)
|
self._ensure_context_compatibility(torrents_cache, stype=stype)
|
||||||
self._ensure_context_compatibility(music_cache, stype=stype)
|
self._ensure_context_compatibility(music_cache, stype=stype)
|
||||||
fresh_torrents: Dict[str, List[Context]] = {}
|
|
||||||
fresh_music: Dict[str, List[Context]] = {}
|
|
||||||
|
|
||||||
# 缓存过滤掉无效种子(影视与音乐缓存分别处理)
|
# 缓存过滤掉无效种子(影视与音乐缓存分别处理)
|
||||||
for _cache in (torrents_cache, music_cache):
|
for _cache in (torrents_cache, music_cache):
|
||||||
@@ -632,8 +608,6 @@ class TorrentsChain(ChainBase):
|
|||||||
include_music=include_music,
|
include_music=include_music,
|
||||||
torrents_cache=torrents_cache,
|
torrents_cache=torrents_cache,
|
||||||
music_cache=music_cache,
|
music_cache=music_cache,
|
||||||
fresh_torrents=fresh_torrents,
|
|
||||||
fresh_music=fresh_music,
|
|
||||||
))
|
))
|
||||||
|
|
||||||
# 保存缓存到本地,影视与音乐分别存储
|
# 保存缓存到本地,影视与音乐分别存储
|
||||||
@@ -649,10 +623,6 @@ class TorrentsChain(ChainBase):
|
|||||||
torrents_cache = {k: v for k, v in torrents_cache.items() if k in domains}
|
torrents_cache = {k: v for k, v in torrents_cache.items() if k in domains}
|
||||||
if sites and music_cache:
|
if sites and music_cache:
|
||||||
music_cache = {k: v for k, v in music_cache.items() if k in domains}
|
music_cache = {k: v for k, v in music_cache.items() if k in domains}
|
||||||
if sites and fresh_torrents:
|
|
||||||
fresh_torrents = {k: v for k, v in fresh_torrents.items() if k in domains}
|
|
||||||
if sites and fresh_music:
|
|
||||||
fresh_music = {k: v for k, v in fresh_music.items() if k in domains}
|
|
||||||
|
|
||||||
if progress_callback:
|
if progress_callback:
|
||||||
progress_callback(
|
progress_callback(
|
||||||
@@ -661,37 +631,10 @@ class TorrentsChain(ChainBase):
|
|||||||
data={"total": total_indexers, "finished": total_indexers},
|
data={"total": total_indexers, "finished": total_indexers},
|
||||||
)
|
)
|
||||||
|
|
||||||
self._retain_cached_fresh(fresh_torrents, torrents_cache)
|
return self._merge_torrent_caches(
|
||||||
self._retain_cached_fresh(fresh_music, music_cache)
|
|
||||||
candidates = self._merge_torrent_caches(
|
|
||||||
{domain: list(contexts) for domain, contexts in torrents_cache.items()},
|
{domain: list(contexts) for domain, contexts in torrents_cache.items()},
|
||||||
music_cache,
|
music_cache,
|
||||||
)
|
)
|
||||||
fresh_candidates = self._merge_torrent_caches(
|
|
||||||
{domain: list(contexts) for domain, contexts in fresh_torrents.items()},
|
|
||||||
fresh_music,
|
|
||||||
)
|
|
||||||
return CandidateBatch.create(
|
|
||||||
source=stype,
|
|
||||||
candidates=candidates,
|
|
||||||
fresh_candidates=fresh_candidates,
|
|
||||||
sites=domains,
|
|
||||||
started_at=started_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _retain_cached_fresh(
|
|
||||||
fresh_candidates: Dict[str, List[Context]],
|
|
||||||
cached_candidates: Dict[str, List[Context]],
|
|
||||||
) -> None:
|
|
||||||
"""移除因缓存容量裁剪而未进入最终完整缓存的本轮候选。"""
|
|
||||||
for domain, contexts in list(fresh_candidates.items()):
|
|
||||||
cached_ids = {id(context) for context in cached_candidates.get(domain) or []}
|
|
||||||
retained = [context for context in contexts if id(context) in cached_ids]
|
|
||||||
if retained:
|
|
||||||
fresh_candidates[domain] = retained
|
|
||||||
else:
|
|
||||||
fresh_candidates.pop(domain, None)
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _ensure_context_compatibility(torrents_cache: Dict[str, List[Context]], stype: Optional[str] = None):
|
def _ensure_context_compatibility(torrents_cache: Dict[str, List[Context]], stype: Optional[str] = None):
|
||||||
|
|||||||
@@ -1,147 +0,0 @@
|
|||||||
"""订阅下载幂等账本的短事务适配器。"""
|
|
||||||
|
|
||||||
from collections.abc import Callable
|
|
||||||
from typing import Optional, TypeVar
|
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.application.download.admission import (
|
|
||||||
SubscriptionDownloadClaim,
|
|
||||||
SubscriptionDownloadRequest,
|
|
||||||
SubscriptionDownloadSnapshot,
|
|
||||||
)
|
|
||||||
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
|
|
||||||
from app.db.oper.subscriptiondownload import SubscriptionDownloadOper
|
|
||||||
from app.db.uow import SqlAlchemyUnitOfWork
|
|
||||||
|
|
||||||
T = TypeVar("T")
|
|
||||||
|
|
||||||
|
|
||||||
def _snapshot(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot:
|
|
||||||
"""在 Session 内投影不可变提交快照。"""
|
|
||||||
return SubscriptionDownloadSnapshot(
|
|
||||||
idempotency_key=record.idempotency_key,
|
|
||||||
subscription_id=record.subscription_id,
|
|
||||||
task_id=record.task_id,
|
|
||||||
state=record.state,
|
|
||||||
attempt_count=record.attempt_count,
|
|
||||||
attempt_token=record.attempt_token,
|
|
||||||
downloader=record.downloader,
|
|
||||||
download_hash=record.download_hash,
|
|
||||||
available_at=record.available_at,
|
|
||||||
last_error=record.last_error,
|
|
||||||
created_at=record.created_at,
|
|
||||||
updated_at=record.updated_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class TransactionalSubscriptionDownloadRepository:
|
|
||||||
"""为每次提交认领和状态变更创建独立短事务。"""
|
|
||||||
|
|
||||||
def __init__(self, session_factory: Callable[[], Session]) -> None:
|
|
||||||
"""保存由组合根提供的同步 Session 工厂。"""
|
|
||||||
self._session_factory = session_factory
|
|
||||||
|
|
||||||
def _read(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T:
|
|
||||||
"""在短 Session 中执行一次只读查询。"""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
return operation(SubscriptionDownloadOper(session))
|
|
||||||
|
|
||||||
def _write(self, operation: Callable[[SubscriptionDownloadOper], T]) -> T:
|
|
||||||
"""在显式短事务中执行一次状态变更。"""
|
|
||||||
with self._session_factory() as session:
|
|
||||||
unit_of_work = SqlAlchemyUnitOfWork(session)
|
|
||||||
try:
|
|
||||||
result = operation(SubscriptionDownloadOper(session))
|
|
||||||
unit_of_work.commit()
|
|
||||||
return result
|
|
||||||
except Exception:
|
|
||||||
unit_of_work.rollback()
|
|
||||||
raise
|
|
||||||
|
|
||||||
def claim(self, request: SubscriptionDownloadRequest) -> SubscriptionDownloadClaim:
|
|
||||||
"""认领唯一提交权并返回脱离 Session 的当前状态。"""
|
|
||||||
def operation(repository: SubscriptionDownloadOper) -> SubscriptionDownloadClaim:
|
|
||||||
"""在同一事务内完成唯一插入或 fenced 重领。"""
|
|
||||||
record, acquired = repository.claim(request)
|
|
||||||
return SubscriptionDownloadClaim(acquired=acquired, snapshot=_snapshot(record))
|
|
||||||
|
|
||||||
return self._write(operation)
|
|
||||||
|
|
||||||
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSnapshot]:
|
|
||||||
"""读取一个已存在的提交快照。"""
|
|
||||||
return self._read(
|
|
||||||
lambda repository: (
|
|
||||||
_snapshot(record)
|
|
||||||
if (record := repository.get(idempotency_key)) is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
def mark_accepted(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_hash: str,
|
|
||||||
) -> bool:
|
|
||||||
"""记录下载器已接受任务。"""
|
|
||||||
return self._write(lambda repository: repository.mark_accepted(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
downloader=downloader,
|
|
||||||
download_hash=download_hash,
|
|
||||||
))
|
|
||||||
|
|
||||||
def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool:
|
|
||||||
"""记录 canonical 本地结算已完成。"""
|
|
||||||
return self._write(lambda repository: repository.mark_succeeded(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
))
|
|
||||||
|
|
||||||
def mark_retryable(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
available_at: str,
|
|
||||||
error: Optional[str],
|
|
||||||
) -> bool:
|
|
||||||
"""记录未产生外部副作用的延迟重试状态。"""
|
|
||||||
return self._write(lambda repository: repository.mark_retryable(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
available_at=available_at,
|
|
||||||
error=error,
|
|
||||||
))
|
|
||||||
|
|
||||||
def mark_reconcile_required(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
error: Optional[str],
|
|
||||||
downloader: Optional[str] = None,
|
|
||||||
download_hash: Optional[str] = None,
|
|
||||||
) -> bool:
|
|
||||||
"""冻结可能已被下载器接受的提交。"""
|
|
||||||
return self._write(lambda repository: repository.mark_reconcile_required(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
error=error,
|
|
||||||
downloader=downloader,
|
|
||||||
download_hash=download_hash,
|
|
||||||
))
|
|
||||||
|
|
||||||
def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool:
|
|
||||||
"""在外部提交前收口取消。"""
|
|
||||||
return self._write(lambda repository: repository.mark_cancelled(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
))
|
|
||||||
|
|
||||||
def has_started_for_task(self, task_id: str) -> bool:
|
|
||||||
"""查询搜索任务是否已越过可安全取消边界。"""
|
|
||||||
return self._read(lambda repository: repository.has_started_for_task(task_id))
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
"""订阅搜索持久队列的 SQLAlchemy 适配器。"""
|
"""订阅搜索持久队列的 SQLAlchemy 适配器。"""
|
||||||
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable, Mapping
|
||||||
from typing import Optional, TypeVar
|
from typing import Optional, TypeVar
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -32,6 +32,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
|
|||||||
finished_count=record.finished_count,
|
finished_count=record.finished_count,
|
||||||
failed_count=record.failed_count,
|
failed_count=record.failed_count,
|
||||||
cancelled_count=record.cancelled_count,
|
cancelled_count=record.cancelled_count,
|
||||||
|
skipped_count=record.skipped_count,
|
||||||
cancel_requested=bool(record.cancel_requested),
|
cancel_requested=bool(record.cancel_requested),
|
||||||
created_at=record.created_at,
|
created_at=record.created_at,
|
||||||
updated_at=record.updated_at,
|
updated_at=record.updated_at,
|
||||||
@@ -95,7 +96,7 @@ class TransactionalSubscriptionSearchRepository:
|
|||||||
subscription_ids: tuple[int, ...],
|
subscription_ids: tuple[int, ...],
|
||||||
source: str,
|
source: str,
|
||||||
priority: int,
|
priority: int,
|
||||||
available_at: Optional[str] = None,
|
available_at_by_subscription: Optional[Mapping[int, str]] = None,
|
||||||
) -> SearchEnqueueResult:
|
) -> SearchEnqueueResult:
|
||||||
"""创建批次并返回 single-flight 合并计数。"""
|
"""创建批次并返回 single-flight 合并计数。"""
|
||||||
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
|
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
|
||||||
@@ -104,7 +105,7 @@ class TransactionalSubscriptionSearchRepository:
|
|||||||
subscription_ids=subscription_ids,
|
subscription_ids=subscription_ids,
|
||||||
source=source,
|
source=source,
|
||||||
priority=priority,
|
priority=priority,
|
||||||
available_at=available_at,
|
available_at_by_subscription=available_at_by_subscription,
|
||||||
)
|
)
|
||||||
return SearchEnqueueResult(
|
return SearchEnqueueResult(
|
||||||
batch=_batch(record),
|
batch=_batch(record),
|
||||||
@@ -231,7 +232,7 @@ class TransactionalSubscriptionSearchRepository:
|
|||||||
next_allowed_at: str,
|
next_allowed_at: str,
|
||||||
error: Optional[str] = None,
|
error: Optional[str] = None,
|
||||||
) -> bool:
|
) -> bool:
|
||||||
"""释放站点租约并持久化间隔或冷却。"""
|
"""释放站点租约并持久化错误冷却或立即恢复。"""
|
||||||
return self._write(
|
return self._write(
|
||||||
lambda repository: repository.finish_site(
|
lambda repository: repository.finish_site(
|
||||||
site_id=site_id,
|
site_id=site_id,
|
||||||
|
|||||||
@@ -5,9 +5,7 @@ from typing import Optional
|
|||||||
from sqlalchemy import select
|
from sqlalchemy import select
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.application.download.admission import SubscriptionDownloadSnapshot
|
|
||||||
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
||||||
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
|
|
||||||
from app.db.models.subscriptionsearch import SubscriptionSearchBatch, SubscriptionSearchTask
|
from app.db.models.subscriptionsearch import SubscriptionSearchBatch, SubscriptionSearchTask
|
||||||
|
|
||||||
|
|
||||||
@@ -46,6 +44,7 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
|
|||||||
finished_count=record.finished_count,
|
finished_count=record.finished_count,
|
||||||
failed_count=record.failed_count,
|
failed_count=record.failed_count,
|
||||||
cancelled_count=record.cancelled_count,
|
cancelled_count=record.cancelled_count,
|
||||||
|
skipped_count=record.skipped_count,
|
||||||
cancel_requested=bool(record.cancel_requested),
|
cancel_requested=bool(record.cancel_requested),
|
||||||
created_at=record.created_at,
|
created_at=record.created_at,
|
||||||
updated_at=record.updated_at,
|
updated_at=record.updated_at,
|
||||||
@@ -55,26 +54,8 @@ def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _download(record: SubscriptionDownloadSubmission) -> SubscriptionDownloadSnapshot:
|
|
||||||
"""复制可脱离 AsyncSession 使用的下载提交快照。"""
|
|
||||||
return SubscriptionDownloadSnapshot(
|
|
||||||
idempotency_key=record.idempotency_key,
|
|
||||||
subscription_id=record.subscription_id,
|
|
||||||
task_id=record.task_id,
|
|
||||||
state=record.state,
|
|
||||||
attempt_count=record.attempt_count,
|
|
||||||
attempt_token=record.attempt_token,
|
|
||||||
downloader=record.downloader,
|
|
||||||
download_hash=record.download_hash,
|
|
||||||
available_at=record.available_at,
|
|
||||||
last_error=record.last_error,
|
|
||||||
created_at=record.created_at,
|
|
||||||
updated_at=record.updated_at,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class SessionSubscriptionExecutionStatusRepository:
|
class SessionSubscriptionExecutionStatusRepository:
|
||||||
"""复用请求 AsyncSession 批量读取搜索和下载执行事实。"""
|
"""复用请求 AsyncSession 批量读取搜索执行事实。"""
|
||||||
|
|
||||||
def __init__(self, session: AsyncSession) -> None:
|
def __init__(self, session: AsyncSession) -> None:
|
||||||
"""绑定请求持有的异步会话。"""
|
"""绑定请求持有的异步会话。"""
|
||||||
@@ -98,24 +79,6 @@ class SessionSubscriptionExecutionStatusRepository:
|
|||||||
snapshots.setdefault(record.subscription_id, _task(record))
|
snapshots.setdefault(record.subscription_id, _task(record))
|
||||||
return snapshots
|
return snapshots
|
||||||
|
|
||||||
async def latest_download_submissions(
|
|
||||||
self,
|
|
||||||
subscription_ids: tuple[int, ...],
|
|
||||||
) -> dict[int, SubscriptionDownloadSnapshot]:
|
|
||||||
"""按更新时间倒序读取并在内存中保留每条订阅首项。"""
|
|
||||||
result = await self._session.execute(
|
|
||||||
select(SubscriptionDownloadSubmission)
|
|
||||||
.where(SubscriptionDownloadSubmission.subscription_id.in_(subscription_ids))
|
|
||||||
.order_by(
|
|
||||||
SubscriptionDownloadSubmission.updated_at.desc(),
|
|
||||||
SubscriptionDownloadSubmission.id.desc(),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
snapshots: dict[int, SubscriptionDownloadSnapshot] = {}
|
|
||||||
for record in result.scalars().all():
|
|
||||||
snapshots.setdefault(record.subscription_id, _download(record))
|
|
||||||
return snapshots
|
|
||||||
|
|
||||||
async def list_batches(self, *, limit: int) -> list[SearchBatchSnapshot]:
|
async def list_batches(self, *, limit: int) -> list[SearchBatchSnapshot]:
|
||||||
"""返回最近更新的批次,访问范围由应用服务依据任务校验。"""
|
"""返回最近更新的批次,访问范围由应用服务依据任务校验。"""
|
||||||
result = await self._session.execute(
|
result = await self._session.execute(
|
||||||
|
|||||||
@@ -46,10 +46,6 @@ _MODEL_EXPORTS = {
|
|||||||
"app.db.models.subscriptionsearch",
|
"app.db.models.subscriptionsearch",
|
||||||
"SubscriptionSiteBudget",
|
"SubscriptionSiteBudget",
|
||||||
),
|
),
|
||||||
"SubscriptionDownloadSubmission": (
|
|
||||||
"app.db.models.subscriptiondownload",
|
|
||||||
"SubscriptionDownloadSubmission",
|
|
||||||
),
|
|
||||||
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
|
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
|
||||||
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
|
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
|
||||||
"TransferExecutionStep": (
|
"TransferExecutionStep": (
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
"""订阅下载提交幂等账本模型。"""
|
|
||||||
|
|
||||||
from typing import Optional
|
|
||||||
|
|
||||||
from sqlalchemy import Index, Integer, String, Text, UniqueConstraint
|
|
||||||
from sqlalchemy.orm import Mapped, mapped_column
|
|
||||||
|
|
||||||
from app.db.base import Base, get_id_column
|
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionDownloadSubmission(Base):
|
|
||||||
"""记录订阅下载唯一提交权、下载器接受事实和待对账终态。"""
|
|
||||||
|
|
||||||
id = get_id_column()
|
|
||||||
idempotency_key: Mapped[str] = mapped_column(String(64), nullable=False)
|
|
||||||
subscription_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
|
||||||
task_id: Mapped[Optional[str]] = mapped_column(String(64))
|
|
||||||
logical_identity: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
resource_key: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
coverage: Mapped[str] = mapped_column(Text, nullable=False)
|
|
||||||
mode: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
delivery_scope: Mapped[str] = mapped_column(Text, nullable=False, default="legacy")
|
|
||||||
state: Mapped[str] = mapped_column(String(32), nullable=False)
|
|
||||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
|
||||||
attempt_token: Mapped[Optional[str]] = mapped_column(String(64))
|
|
||||||
downloader: Mapped[Optional[str]] = mapped_column(String(128))
|
|
||||||
download_hash: Mapped[Optional[str]] = mapped_column(String(256))
|
|
||||||
available_at: Mapped[Optional[str]] = mapped_column(String(40))
|
|
||||||
last_error: Mapped[Optional[str]] = mapped_column(Text)
|
|
||||||
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
|
||||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
|
||||||
started_at: Mapped[Optional[str]] = mapped_column(String(40))
|
|
||||||
finished_at: Mapped[Optional[str]] = mapped_column(String(40))
|
|
||||||
|
|
||||||
__table_args__ = (
|
|
||||||
UniqueConstraint(
|
|
||||||
"idempotency_key",
|
|
||||||
name="uq_subscriptiondownloadsubmission_idempotency_key",
|
|
||||||
),
|
|
||||||
Index(
|
|
||||||
"ix_subscriptiondownloadsubmission_task_state",
|
|
||||||
"task_id",
|
|
||||||
"state",
|
|
||||||
"id",
|
|
||||||
),
|
|
||||||
Index(
|
|
||||||
"ix_subscriptiondownloadsubmission_subscription_state",
|
|
||||||
"subscription_id",
|
|
||||||
"state",
|
|
||||||
"updated_at",
|
|
||||||
"id",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
@@ -20,6 +20,7 @@ class SubscriptionSearchBatch(Base):
|
|||||||
finished_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
finished_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
failed_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
cancelled_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
|
skipped_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
cancel_requested: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
cancel_requested: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||||
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||||
@@ -77,7 +78,7 @@ class SubscriptionSearchTask(Base):
|
|||||||
|
|
||||||
|
|
||||||
class SubscriptionSiteBudget(Base):
|
class SubscriptionSiteBudget(Base):
|
||||||
"""记录兜底搜索对单个站点的唯一租约、间隔与错误冷却。"""
|
"""记录兜底搜索对单个站点的唯一租约与错误冷却。"""
|
||||||
|
|
||||||
id = get_id_column()
|
id = get_id_column()
|
||||||
site_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
site_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
|||||||
@@ -1,242 +0,0 @@
|
|||||||
"""订阅下载提交账本的事务内状态转换。"""
|
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
|
||||||
from typing import Optional, cast
|
|
||||||
from uuid import uuid4
|
|
||||||
|
|
||||||
from sqlalchemy import and_, or_, select, update
|
|
||||||
from sqlalchemy.exc import IntegrityError
|
|
||||||
from sqlalchemy.orm import Session
|
|
||||||
|
|
||||||
from app.application.download.admission import SubscriptionDownloadRequest
|
|
||||||
from app.db.base import DbOper, execute_dml
|
|
||||||
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
|
|
||||||
|
|
||||||
|
|
||||||
def utc_now_text() -> str:
|
|
||||||
"""返回可按字符串稳定排序的 UTC ISO 时间。"""
|
|
||||||
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionDownloadOper(DbOper):
|
|
||||||
"""在调用方事务内维护唯一下载提交及其 fenced 终态。"""
|
|
||||||
|
|
||||||
def claim(self, request: SubscriptionDownloadRequest) -> tuple[SubscriptionDownloadSubmission, bool]:
|
|
||||||
"""创建或原子重领允许重试的提交记录。"""
|
|
||||||
if not isinstance(self._db, Session):
|
|
||||||
raise RuntimeError("订阅下载提交认领需要调用方提供同步 Session")
|
|
||||||
now = utc_now_text()
|
|
||||||
attempt_token = uuid4().hex
|
|
||||||
record = self._db.execute(
|
|
||||||
select(SubscriptionDownloadSubmission).where(
|
|
||||||
SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key
|
|
||||||
)
|
|
||||||
).scalars().first()
|
|
||||||
if record is None:
|
|
||||||
try:
|
|
||||||
with self._db.begin_nested():
|
|
||||||
record = SubscriptionDownloadSubmission(
|
|
||||||
idempotency_key=request.idempotency_key,
|
|
||||||
subscription_id=request.subscription_id,
|
|
||||||
task_id=request.task_id,
|
|
||||||
logical_identity=request.logical_identity,
|
|
||||||
resource_key=request.resource_key,
|
|
||||||
coverage=request.coverage,
|
|
||||||
mode=request.mode,
|
|
||||||
delivery_scope=request.delivery_scope,
|
|
||||||
state="submitting",
|
|
||||||
attempt_count=1,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
created_at=now,
|
|
||||||
updated_at=now,
|
|
||||||
started_at=now,
|
|
||||||
)
|
|
||||||
self._db.add(record)
|
|
||||||
self._db.flush()
|
|
||||||
return record, True
|
|
||||||
except IntegrityError:
|
|
||||||
self._db.expire_all()
|
|
||||||
record = self._db.execute(
|
|
||||||
select(SubscriptionDownloadSubmission).where(
|
|
||||||
SubscriptionDownloadSubmission.idempotency_key == request.idempotency_key
|
|
||||||
)
|
|
||||||
).scalar_one()
|
|
||||||
|
|
||||||
reclaimable = record.state in {"retryable", "cancelled"} and (
|
|
||||||
record.available_at is None or record.available_at <= now
|
|
||||||
)
|
|
||||||
if not reclaimable:
|
|
||||||
return record, False
|
|
||||||
updated = execute_dml(
|
|
||||||
self._db,
|
|
||||||
update(SubscriptionDownloadSubmission)
|
|
||||||
.where(
|
|
||||||
SubscriptionDownloadSubmission.id == record.id,
|
|
||||||
SubscriptionDownloadSubmission.state.in_(("retryable", "cancelled")),
|
|
||||||
or_(
|
|
||||||
SubscriptionDownloadSubmission.available_at.is_(None),
|
|
||||||
SubscriptionDownloadSubmission.available_at <= now,
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.values(
|
|
||||||
state="submitting",
|
|
||||||
task_id=request.task_id,
|
|
||||||
attempt_count=SubscriptionDownloadSubmission.attempt_count + 1,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
downloader=None,
|
|
||||||
download_hash=None,
|
|
||||||
available_at=None,
|
|
||||||
last_error=None,
|
|
||||||
updated_at=now,
|
|
||||||
started_at=now,
|
|
||||||
finished_at=None,
|
|
||||||
),
|
|
||||||
execution_options={"synchronize_session": False},
|
|
||||||
)
|
|
||||||
self._db.flush()
|
|
||||||
self._db.expire_all()
|
|
||||||
current = self._db.execute(
|
|
||||||
select(SubscriptionDownloadSubmission).where(
|
|
||||||
SubscriptionDownloadSubmission.id == record.id
|
|
||||||
)
|
|
||||||
).scalar_one()
|
|
||||||
return current, bool(updated and current.attempt_token == attempt_token)
|
|
||||||
|
|
||||||
def get(self, idempotency_key: str) -> Optional[SubscriptionDownloadSubmission]:
|
|
||||||
"""按稳定幂等键读取提交记录。"""
|
|
||||||
if not isinstance(self._db, Session):
|
|
||||||
raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session")
|
|
||||||
return cast(Optional[SubscriptionDownloadSubmission], self._db.execute(
|
|
||||||
select(SubscriptionDownloadSubmission).where(
|
|
||||||
SubscriptionDownloadSubmission.idempotency_key == idempotency_key
|
|
||||||
)
|
|
||||||
).scalars().first())
|
|
||||||
|
|
||||||
def mark_accepted(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_hash: str,
|
|
||||||
) -> bool:
|
|
||||||
"""把当前 submitting 尝试推进为下载器已接受。"""
|
|
||||||
return self._transition(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
source_states=("submitting",),
|
|
||||||
values={
|
|
||||||
"state": "accepted",
|
|
||||||
"downloader": downloader,
|
|
||||||
"download_hash": download_hash,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def mark_succeeded(self, *, idempotency_key: str, attempt_token: str) -> bool:
|
|
||||||
"""把当前 accepted 尝试推进为本地结算成功。"""
|
|
||||||
return self._transition(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
source_states=("accepted",),
|
|
||||||
values={"state": "succeeded", "finished_at": utc_now_text()},
|
|
||||||
)
|
|
||||||
|
|
||||||
def mark_retryable(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
available_at: str,
|
|
||||||
error: Optional[str],
|
|
||||||
) -> bool:
|
|
||||||
"""把下载器明确拒绝的当前尝试收口为延迟可重试。"""
|
|
||||||
return self._transition(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
source_states=("submitting",),
|
|
||||||
values={
|
|
||||||
"state": "retryable",
|
|
||||||
"available_at": available_at,
|
|
||||||
"last_error": error,
|
|
||||||
"finished_at": utc_now_text(),
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def mark_reconcile_required(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
error: Optional[str],
|
|
||||||
downloader: Optional[str],
|
|
||||||
download_hash: Optional[str],
|
|
||||||
) -> bool:
|
|
||||||
"""冻结 submitting/accepted 尝试,禁止自动重新提交。"""
|
|
||||||
values: dict[str, object] = {
|
|
||||||
"state": "reconcile_required",
|
|
||||||
"last_error": error,
|
|
||||||
"finished_at": utc_now_text(),
|
|
||||||
}
|
|
||||||
if downloader is not None:
|
|
||||||
values["downloader"] = downloader
|
|
||||||
if download_hash is not None:
|
|
||||||
values["download_hash"] = download_hash
|
|
||||||
return self._transition(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
source_states=("submitting", "accepted"),
|
|
||||||
values=values,
|
|
||||||
)
|
|
||||||
|
|
||||||
def mark_cancelled(self, *, idempotency_key: str, attempt_token: str) -> bool:
|
|
||||||
"""在外部提交前把当前尝试收口为已取消。"""
|
|
||||||
return self._transition(
|
|
||||||
idempotency_key=idempotency_key,
|
|
||||||
attempt_token=attempt_token,
|
|
||||||
source_states=("submitting",),
|
|
||||||
values={"state": "cancelled", "finished_at": utc_now_text()},
|
|
||||||
)
|
|
||||||
|
|
||||||
def has_started_for_task(self, task_id: str) -> bool:
|
|
||||||
"""判断任务是否进入过外部副作用边界或已成功。"""
|
|
||||||
if not isinstance(self._db, Session):
|
|
||||||
raise RuntimeError("订阅下载提交查询需要调用方提供同步 Session")
|
|
||||||
return bool(self._db.execute(
|
|
||||||
select(SubscriptionDownloadSubmission.id)
|
|
||||||
.where(
|
|
||||||
SubscriptionDownloadSubmission.task_id == task_id,
|
|
||||||
SubscriptionDownloadSubmission.state.in_((
|
|
||||||
"submitting",
|
|
||||||
"accepted",
|
|
||||||
"succeeded",
|
|
||||||
"reconcile_required",
|
|
||||||
)),
|
|
||||||
)
|
|
||||||
.limit(1)
|
|
||||||
).scalar())
|
|
||||||
|
|
||||||
def _transition(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
idempotency_key: str,
|
|
||||||
attempt_token: str,
|
|
||||||
source_states: tuple[str, ...],
|
|
||||||
values: dict[str, object],
|
|
||||||
) -> bool:
|
|
||||||
"""以当前尝试令牌原子推进状态,拒绝过期执行者写回。"""
|
|
||||||
if not isinstance(self._db, Session):
|
|
||||||
raise RuntimeError("订阅下载提交写入需要调用方提供同步 Session")
|
|
||||||
now = utc_now_text()
|
|
||||||
return bool(execute_dml(
|
|
||||||
self._db,
|
|
||||||
update(SubscriptionDownloadSubmission)
|
|
||||||
.where(
|
|
||||||
and_(
|
|
||||||
SubscriptionDownloadSubmission.idempotency_key == idempotency_key,
|
|
||||||
SubscriptionDownloadSubmission.attempt_token == attempt_token,
|
|
||||||
SubscriptionDownloadSubmission.state.in_(source_states),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.values(updated_at=now, **values),
|
|
||||||
execution_options={"synchronize_session": False},
|
|
||||||
))
|
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
"""订阅搜索批次与任务的持久队列读写。"""
|
"""订阅搜索批次与任务的持久队列读写。"""
|
||||||
|
|
||||||
from datetime import datetime, timedelta, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Optional
|
from typing import Mapping, Optional
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from sqlalchemy import and_, case, func, or_, select, update
|
from sqlalchemy import and_, case, func, or_, select, update
|
||||||
@@ -30,7 +30,7 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
subscription_ids: tuple[int, ...],
|
subscription_ids: tuple[int, ...],
|
||||||
source: str,
|
source: str,
|
||||||
priority: int,
|
priority: int,
|
||||||
available_at: Optional[str],
|
available_at_by_subscription: Optional[Mapping[int, str]],
|
||||||
) -> tuple[SubscriptionSearchBatch, int, int]:
|
) -> tuple[SubscriptionSearchBatch, int, int]:
|
||||||
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
|
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
|
||||||
if not isinstance(self._db, Session):
|
if not isinstance(self._db, Session):
|
||||||
@@ -51,6 +51,11 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
coalesced = 0
|
coalesced = 0
|
||||||
for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)):
|
for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)):
|
||||||
active_key = f"subscription:{subscription_id}"
|
active_key = f"subscription:{subscription_id}"
|
||||||
|
available_at = (
|
||||||
|
available_at_by_subscription.get(subscription_id, now)
|
||||||
|
if available_at_by_subscription
|
||||||
|
else now
|
||||||
|
)
|
||||||
task = SubscriptionSearchTask(
|
task = SubscriptionSearchTask(
|
||||||
task_id=uuid4().hex,
|
task_id=uuid4().hex,
|
||||||
batch_id=batch.batch_id,
|
batch_id=batch.batch_id,
|
||||||
@@ -61,7 +66,7 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
position=position,
|
position=position,
|
||||||
state="queued",
|
state="queued",
|
||||||
phase="queued",
|
phase="queued",
|
||||||
available_at=available_at or now,
|
available_at=available_at,
|
||||||
created_at=now,
|
created_at=now,
|
||||||
updated_at=now,
|
updated_at=now,
|
||||||
)
|
)
|
||||||
@@ -85,9 +90,9 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
(
|
(
|
||||||
or_(
|
or_(
|
||||||
SubscriptionSearchTask.available_at.is_(None),
|
SubscriptionSearchTask.available_at.is_(None),
|
||||||
SubscriptionSearchTask.available_at > (available_at or now),
|
SubscriptionSearchTask.available_at > available_at,
|
||||||
),
|
),
|
||||||
available_at or now,
|
available_at,
|
||||||
),
|
),
|
||||||
else_=SubscriptionSearchTask.available_at,
|
else_=SubscriptionSearchTask.available_at,
|
||||||
),
|
),
|
||||||
@@ -267,7 +272,7 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
"""以当前租约令牌收口任务,并重新计算批次终态。"""
|
"""以当前租约令牌收口任务,并重新计算批次终态。"""
|
||||||
if not isinstance(self._db, Session):
|
if not isinstance(self._db, Session):
|
||||||
raise RuntimeError("订阅搜索收口需要调用方提供同步 Session")
|
raise RuntimeError("订阅搜索收口需要调用方提供同步 Session")
|
||||||
if state not in {"completed", "failed", "cancelled"}:
|
if state not in {"completed", "failed", "cancelled", "skipped"}:
|
||||||
raise ValueError(f"不支持的订阅搜索终态:{state}")
|
raise ValueError(f"不支持的订阅搜索终态:{state}")
|
||||||
task = self._db.execute(
|
task = self._db.execute(
|
||||||
select(SubscriptionSearchTask).where(
|
select(SubscriptionSearchTask).where(
|
||||||
@@ -334,7 +339,7 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
error=None,
|
error=None,
|
||||||
)
|
)
|
||||||
now = utc_now_text()
|
now = utc_now_text()
|
||||||
return bool(execute_dml(
|
updated = execute_dml(
|
||||||
self._db,
|
self._db,
|
||||||
update(SubscriptionSearchTask)
|
update(SubscriptionSearchTask)
|
||||||
.where(
|
.where(
|
||||||
@@ -352,7 +357,11 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
updated_at=now,
|
updated_at=now,
|
||||||
),
|
),
|
||||||
execution_options={"synchronize_session": False},
|
execution_options={"synchronize_session": False},
|
||||||
))
|
)
|
||||||
|
if not updated:
|
||||||
|
return False
|
||||||
|
self._refresh_batch(task.batch_id, now=now, error=None)
|
||||||
|
return True
|
||||||
|
|
||||||
def is_cancel_requested(self, task_id: str) -> bool:
|
def is_cancel_requested(self, task_id: str) -> bool:
|
||||||
"""读取任务和批次取消标记。"""
|
"""读取任务和批次取消标记。"""
|
||||||
@@ -437,7 +446,11 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
and record.lease_expires_at
|
and record.lease_expires_at
|
||||||
and record.lease_expires_at > now
|
and record.lease_expires_at > now
|
||||||
)
|
)
|
||||||
if lease_busy or record.next_allowed_at > now:
|
cooldown_active = bool(
|
||||||
|
record.last_outcome not in {None, "success", "skipped"}
|
||||||
|
and record.next_allowed_at > now
|
||||||
|
)
|
||||||
|
if lease_busy or cooldown_active:
|
||||||
return record, False
|
return record, False
|
||||||
lease_token = uuid4().hex
|
lease_token = uuid4().hex
|
||||||
lease_expires_at = (
|
lease_expires_at = (
|
||||||
@@ -453,12 +466,17 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
SubscriptionSiteBudget.lease_expires_at.is_(None),
|
SubscriptionSiteBudget.lease_expires_at.is_(None),
|
||||||
SubscriptionSiteBudget.lease_expires_at <= now,
|
SubscriptionSiteBudget.lease_expires_at <= now,
|
||||||
),
|
),
|
||||||
SubscriptionSiteBudget.next_allowed_at <= now,
|
or_(
|
||||||
|
SubscriptionSiteBudget.next_allowed_at <= now,
|
||||||
|
SubscriptionSiteBudget.last_outcome.is_(None),
|
||||||
|
SubscriptionSiteBudget.last_outcome.in_(("success", "skipped")),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
.values(
|
.values(
|
||||||
lease_owner=owner,
|
lease_owner=owner,
|
||||||
lease_token=lease_token,
|
lease_token=lease_token,
|
||||||
lease_expires_at=lease_expires_at,
|
lease_expires_at=lease_expires_at,
|
||||||
|
next_allowed_at=now,
|
||||||
updated_at=now,
|
updated_at=now,
|
||||||
),
|
),
|
||||||
execution_options={"synchronize_session": False},
|
execution_options={"synchronize_session": False},
|
||||||
@@ -586,7 +604,8 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
completed = counts.get("completed", 0)
|
completed = counts.get("completed", 0)
|
||||||
failed = counts.get("failed", 0)
|
failed = counts.get("failed", 0)
|
||||||
cancelled = counts.get("cancelled", 0)
|
cancelled = counts.get("cancelled", 0)
|
||||||
terminal = completed + failed + cancelled
|
skipped = counts.get("skipped", 0)
|
||||||
|
terminal = completed + failed + cancelled + skipped
|
||||||
batch = self.get_batch(batch_id)
|
batch = self.get_batch(batch_id)
|
||||||
if batch is None:
|
if batch is None:
|
||||||
return
|
return
|
||||||
@@ -595,11 +614,18 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
state = "failed"
|
state = "failed"
|
||||||
elif cancelled or batch.cancel_requested:
|
elif cancelled or batch.cancel_requested:
|
||||||
state = "cancelled"
|
state = "cancelled"
|
||||||
|
elif skipped:
|
||||||
|
state = "skipped"
|
||||||
else:
|
else:
|
||||||
state = "completed"
|
state = "completed"
|
||||||
finished_at = now
|
finished_at = now
|
||||||
else:
|
else:
|
||||||
state = "cancelling" if batch.cancel_requested else "running"
|
if batch.cancel_requested:
|
||||||
|
state = "cancelling"
|
||||||
|
elif counts.get("running", 0):
|
||||||
|
state = "running"
|
||||||
|
else:
|
||||||
|
state = "queued"
|
||||||
finished_at = None
|
finished_at = None
|
||||||
execute_dml(
|
execute_dml(
|
||||||
self._db,
|
self._db,
|
||||||
@@ -610,6 +636,7 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
finished_count=completed,
|
finished_count=completed,
|
||||||
failed_count=failed,
|
failed_count=failed,
|
||||||
cancelled_count=cancelled,
|
cancelled_count=cancelled,
|
||||||
|
skipped_count=skipped,
|
||||||
updated_at=now,
|
updated_at=now,
|
||||||
finished_at=finished_at,
|
finished_at=finished_at,
|
||||||
last_error=error or batch.last_error,
|
last_error=error or batch.last_error,
|
||||||
|
|||||||
@@ -57,8 +57,6 @@ class SubscriptionExecutionStatus(BaseModel): # type: ignore[misc]
|
|||||||
current_site_id: Optional[int] = None
|
current_site_id: Optional[int] = None
|
||||||
error: Optional[str] = None
|
error: Optional[str] = None
|
||||||
can_cancel: bool = False
|
can_cancel: bool = False
|
||||||
can_retry: bool = False
|
|
||||||
requires_reconciliation: bool = False
|
|
||||||
|
|
||||||
model_config = ConfigDict(from_attributes=True)
|
model_config = ConfigDict(from_attributes=True)
|
||||||
|
|
||||||
@@ -77,6 +75,7 @@ class SubscriptionBatchStatus(BaseModel): # type: ignore[misc]
|
|||||||
cancelled_count: int
|
cancelled_count: int
|
||||||
created_at: str
|
created_at: str
|
||||||
updated_at: str
|
updated_at: str
|
||||||
|
skipped_count: int = 0
|
||||||
current_subscription_id: Optional[int] = None
|
current_subscription_id: Optional[int] = None
|
||||||
current_site_id: Optional[int] = None
|
current_site_id: Optional[int] = None
|
||||||
error: Optional[str] = None
|
error: Optional[str] = None
|
||||||
|
|||||||
@@ -36,7 +36,6 @@ from app.chain.transfer.filter import (
|
|||||||
from app.db.adapters.chain import TransactionalChainDurableEventWriter
|
from app.db.adapters.chain import TransactionalChainDurableEventWriter
|
||||||
from app.db.adapters.download import TransactionalDownloadFailureRepository
|
from app.db.adapters.download import TransactionalDownloadFailureRepository
|
||||||
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
|
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
|
||||||
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
|
|
||||||
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
|
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
|
||||||
from app.db.oper.message import MessageOper
|
from app.db.oper.message import MessageOper
|
||||||
from app.db.oper.systemconfig import SystemConfigOper
|
from app.db.oper.systemconfig import SystemConfigOper
|
||||||
@@ -112,7 +111,6 @@ def build_chain_runtime_context(
|
|||||||
transfer_execution_repository=dependencies.transfer_execution,
|
transfer_execution_repository=dependencies.transfer_execution,
|
||||||
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
|
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
|
||||||
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
|
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
|
||||||
subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory),
|
|
||||||
user_repository=build_transactional_user_repository(),
|
user_repository=build_transactional_user_repository(),
|
||||||
legacy_transfer_command=execute_legacy_transfer_command,
|
legacy_transfer_command=execute_legacy_transfer_command,
|
||||||
configuration=configuration(),
|
configuration=configuration(),
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
"""3.0.24 增加订阅搜索批次跳过计数。"""
|
||||||
|
|
||||||
|
# Alembic 的 op 是运行期代理,静态分析无法看到实际操作方法。
|
||||||
|
# pylint: disable=no-member
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "b6c1d9e4a7f2"
|
||||||
|
down_revision = "a7d9e2c4f6b1"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
_TABLE = "subscriptionsearchbatch"
|
||||||
|
|
||||||
|
|
||||||
|
def _column_names() -> set[str]:
|
||||||
|
"""返回当前订阅搜索批次字段名集合。"""
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if _TABLE not in set(inspector.get_table_names()):
|
||||||
|
return set()
|
||||||
|
return {column["name"] for column in inspector.get_columns(_TABLE)}
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""为存量批次增加跳过计数,保持旧数据库可重复升级。"""
|
||||||
|
if _TABLE in set(sa.inspect(op.get_bind()).get_table_names()) and "skipped_count" not in _column_names():
|
||||||
|
op.add_column(
|
||||||
|
_TABLE,
|
||||||
|
sa.Column("skipped_count", sa.Integer(), nullable=False, server_default="0"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""移除批次跳过计数,保留原搜索任务与批次记录。"""
|
||||||
|
if "skipped_count" in _column_names():
|
||||||
|
op.drop_column(_TABLE, "skipped_count")
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
"""3.0.25 移除订阅下载提交持久账本。
|
||||||
|
|
||||||
|
Revision ID: c8f2e6a1d4b9
|
||||||
|
Revises: b6c1d9e4a7f2
|
||||||
|
Create Date: 2026-09-03
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Alembic 的 op 是运行期代理,静态分析无法看到实际操作方法。
|
||||||
|
# pylint: disable=no-member
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "c8f2e6a1d4b9"
|
||||||
|
down_revision = "b6c1d9e4a7f2"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
_TABLE = "subscriptiondownloadsubmission"
|
||||||
|
|
||||||
|
|
||||||
|
def _table_names() -> set[str]:
|
||||||
|
"""返回当前数据库表名集合。"""
|
||||||
|
return set(sa.inspect(op.get_bind()).get_table_names())
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""删除不再参与订阅执行的下载提交账本。"""
|
||||||
|
if _TABLE in _table_names():
|
||||||
|
op.drop_table(_TABLE)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""恢复 3.0.24 使用的下载提交账本结构。"""
|
||||||
|
if _TABLE in _table_names():
|
||||||
|
return
|
||||||
|
op.create_table(
|
||||||
|
_TABLE,
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||||
|
sa.Column("idempotency_key", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("subscription_id", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("task_id", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("logical_identity", sa.Text(), nullable=False),
|
||||||
|
sa.Column("resource_key", sa.Text(), nullable=False),
|
||||||
|
sa.Column("coverage", sa.Text(), nullable=False),
|
||||||
|
sa.Column("mode", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("delivery_scope", sa.Text(), nullable=False, server_default="legacy"),
|
||||||
|
sa.Column("state", sa.String(length=32), nullable=False),
|
||||||
|
sa.Column("attempt_count", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("attempt_token", sa.String(length=64), nullable=True),
|
||||||
|
sa.Column("downloader", sa.String(length=128), nullable=True),
|
||||||
|
sa.Column("download_hash", sa.String(length=256), nullable=True),
|
||||||
|
sa.Column("available_at", sa.String(length=40), nullable=True),
|
||||||
|
sa.Column("last_error", sa.Text(), nullable=True),
|
||||||
|
sa.Column("created_at", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.String(length=40), nullable=False),
|
||||||
|
sa.Column("started_at", sa.String(length=40), nullable=True),
|
||||||
|
sa.Column("finished_at", sa.String(length=40), nullable=True),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"idempotency_key",
|
||||||
|
name="uq_subscriptiondownloadsubmission_idempotency_key",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_subscriptiondownloadsubmission_task_state",
|
||||||
|
_TABLE,
|
||||||
|
["task_id", "state", "id"],
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_subscriptiondownloadsubmission_subscription_state",
|
||||||
|
_TABLE,
|
||||||
|
["subscription_id", "state", "updated_at", "id"],
|
||||||
|
)
|
||||||
@@ -754,8 +754,8 @@ flowchart LR
|
|||||||
|
|
||||||
| 指标 | 当前值 |
|
| 指标 | 当前值 |
|
||||||
|---|---:|
|
|---|---:|
|
||||||
| Python 模块 | 944 |
|
| Python 模块 | 940 |
|
||||||
| 内部导入边 | 7,874 |
|
| 内部导入边 | 7,845 |
|
||||||
| 非平凡 SCC | 1(精确 containment 的 TMDB 移植包环) |
|
| 非平凡 SCC | 1(精确 containment 的 TMDB 移植包环) |
|
||||||
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
|
| Application / Chain 具体 Adapter 直连 | 0 / 0 |
|
||||||
| Direct egress | 53(债务已清零,53 条精确 containment) |
|
| Direct egress | 53(债务已清零,53 条精确 containment) |
|
||||||
|
|||||||
@@ -94,7 +94,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
|
|||||||
|
|
||||||
| 指标 | 当前值 | 解释 |
|
| 指标 | 当前值 | 解释 |
|
||||||
|---|---:|---|
|
|---|---:|---|
|
||||||
| 宿主 Python 模块 / 内部依赖边 | 944 / 7,874 | `dependency-baseline.json` 当前快照 |
|
| 宿主 Python 模块 / 内部依赖边 | 940 / 7,845 | `dependency-baseline.json` 当前快照 |
|
||||||
| 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 |
|
| 非平凡 SCC | 1 | 仅保留精确 containment 的 29 模块 TMDB 移植包环 |
|
||||||
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
|
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
|
||||||
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
|
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
|
||||||
@@ -102,7 +102,7 @@ ARCH-201 至 ARCH-204 均达到实现、验证、提交、推送和远端门禁
|
|||||||
| Event Contract | 53 | 均已有 payload model,但当前全部是 diagnostic enforcement |
|
| Event Contract | 53 | 均已有 payload model,但当前全部是 diagnostic enforcement |
|
||||||
| Python 源码量 | 305,884 行 | 排除 `app/plugins/**`;61 个文件超过 1,000 行,11 个超过 2,000 行 |
|
| Python 源码量 | 305,884 行 | 排除 `app/plugins/**`;61 个文件超过 1,000 行,11 个超过 2,000 行 |
|
||||||
| 长方法 | 290 个超过 80 行 | AST 统计排除 `app/plugins/**`;65 个超过 150 行,21 个超过 250 行 |
|
| 长方法 | 290 个超过 80 行 | AST 统计排除 `app/plugins/**`;65 个超过 150 行,21 个超过 250 行 |
|
||||||
| 全量 mypy 历史债务 | 9,581 / 516 文件 | Agent API 重构后的现状基线;canonical Facade 与 endpoint 类型边界已补齐,低水位只允许继续下降 |
|
| 全量 mypy 历史债务 | 9,580 / 516 文件 | Agent API 重构后的现状基线;canonical Facade 与 endpoint 类型边界已补齐,低水位只允许继续下降 |
|
||||||
| Ruff 历史诊断 | 559 | 低水位门禁通过,但规则集只覆盖 `E4/E7/E9/F/I` |
|
| Ruff 历史诊断 | 559 | 低水位门禁通过,但规则集只覆盖 `E4/E7/E9/F/I` |
|
||||||
| 覆盖率固定基线 | Application 80.00%,Domain 80.00% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
|
| 覆盖率固定基线 | Application 80.00%,Domain 80.00% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
# MoviePilot 订阅执行治理上游 Handoff
|
# MoviePilot 订阅执行治理上游 Handoff
|
||||||
|
|
||||||
|
> 历史交接:本文保留 2026-09-01 的问题证据,不再作为当前实施合同。
|
||||||
|
> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required`
|
||||||
|
> 已由 2026-09-03 的简化方案废止;当前行为以代码、迁移和回归测试为准。
|
||||||
|
>
|
||||||
> 日期:2026-09-01
|
> 日期:2026-09-01
|
||||||
> 目标版本:MoviePilot V3
|
> 目标版本:MoviePilot V3
|
||||||
> 生产参考:MoviePilot V2 `2.15.5` 的一次只读样本
|
> 生产参考:MoviePilot V2 `2.15.5` 的一次只读样本
|
||||||
|
|||||||
@@ -1,5 +1,9 @@
|
|||||||
# MoviePilot 订阅执行治理
|
# MoviePilot 订阅执行治理
|
||||||
|
|
||||||
|
> 历史方案:本文件记录 2026-09-01 的上游接管与验证证据,不再作为当前实施合同。
|
||||||
|
> 其中 `CandidateBatch`、候选恢复账本、订阅下载提交持久账本和 `reconcile_required`
|
||||||
|
> 已由 2026-09-03 的简化方案废止;当前行为以代码、迁移和回归测试为准。
|
||||||
|
>
|
||||||
> 状态:`completed(2026-09-01)`
|
> 状态:`completed(2026-09-01)`
|
||||||
> 当前叶:`none(全部叶已完成或按条件停止)`
|
> 当前叶:`none(全部叶已完成或按条件停止)`
|
||||||
> 迁移来源:`V3-RDY-009A`、`V3-RDY-009A1`、`V3-RDY-009A1B`、
|
> 迁移来源:`V3-RDY-009A`、`V3-RDY-009A1`、`V3-RDY-009A1B`、
|
||||||
|
|||||||
@@ -116,11 +116,6 @@ DATABASE_TABLE_GUIDES: dict[str, tuple[str, str, str]] = {
|
|||||||
"Auditing historical subscriptions, media identity, completion criteria, and filter configuration.",
|
"Auditing historical subscriptions, media identity, completion criteria, and filter configuration.",
|
||||||
"Generated by subscription completion and archival; restore or delete through its business API.",
|
"Generated by subscription completion and archival; restore or delete through its business API.",
|
||||||
),
|
),
|
||||||
"subscriptiondownloadsubmission": (
|
|
||||||
"Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes.",
|
|
||||||
"Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership.",
|
|
||||||
"Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states.",
|
|
||||||
),
|
|
||||||
"subscriptionsearchbatch": (
|
"subscriptionsearchbatch": (
|
||||||
"Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.",
|
"Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.",
|
||||||
"Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.",
|
"Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.",
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -236,17 +236,11 @@ python scripts/mp-db.py write "UPDATE subscribe SET state = 'S' WHERE id = 123"
|
|||||||
- Write boundary: Generated by subscription completion and archival; restore or delete through its business API.
|
- Write boundary: Generated by subscription completion and archival; restore or delete through its business API.
|
||||||
- Columns: `id`, `name`, `year`, `type`, `keyword`, `media_source`, `media_id`, `music_type`, `total_tracks`, `season`, `poster`, `backdrop`, `vote`, `description`, `filter`, `include`, `exclude`, `quality`, `resolution`, `effect`, `audio_quality`, `audio_format`, `min_bitrate`, `min_bit_depth`, `min_sample_rate`, `total_episode`, `start_episode`, `date`, `username`, `sites`, `best_version`, `best_version_full`, `current_priority`, `current_audio_format`, `current_bitrate`, `current_bit_depth`, `current_sample_rate`, `episode_priority`, `save_path`, `search_imdbid`, `custom_words`, `media_category`, `filter_groups`, `episode_group`
|
- Columns: `id`, `name`, `year`, `type`, `keyword`, `media_source`, `media_id`, `music_type`, `total_tracks`, `season`, `poster`, `backdrop`, `vote`, `description`, `filter`, `include`, `exclude`, `quality`, `resolution`, `effect`, `audio_quality`, `audio_format`, `min_bitrate`, `min_bit_depth`, `min_sample_rate`, `total_episode`, `start_episode`, `date`, `username`, `sites`, `best_version`, `best_version_full`, `current_priority`, `current_audio_format`, `current_bitrate`, `current_bit_depth`, `current_sample_rate`, `episode_priority`, `save_path`, `search_imdbid`, `custom_words`, `media_category`, `filter_groups`, `episode_group`
|
||||||
|
|
||||||
### `subscriptiondownloadsubmission`
|
|
||||||
- Purpose: Stores subscription download idempotency claims, downloader acceptance facts, retries, and reconciliation freezes.
|
|
||||||
- Useful queries: Diagnosing duplicate suppression, uncertain downloader outcomes, retry timing, and task ownership.
|
|
||||||
- Write boundary: Owned by the subscription download submission state machine; never force accepted, succeeded, or retryable states.
|
|
||||||
- Columns: `id`, `idempotency_key`, `subscription_id`, `task_id`, `logical_identity`, `resource_key`, `coverage`, `mode`, `delivery_scope`, `state`, `attempt_count`, `attempt_token`, `downloader`, `download_hash`, `available_at`, `last_error`, `created_at`, `updated_at`, `started_at`, `finished_at`
|
|
||||||
|
|
||||||
### `subscriptionsearchbatch`
|
### `subscriptionsearchbatch`
|
||||||
- Purpose: Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.
|
- Purpose: Stores durable subscription search batches, source, aggregate state, counts, and cancellation requests.
|
||||||
- Useful queries: Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.
|
- Useful queries: Inspecting user-visible search progress, recovery state, cancellation, and terminal outcomes.
|
||||||
- Write boundary: Owned by subscription search orchestration; create and cancel batches through the subscription API.
|
- Write boundary: Owned by subscription search orchestration; create and cancel batches through the subscription API.
|
||||||
- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error`
|
- Columns: `id`, `batch_id`, `source`, `state`, `priority`, `total_count`, `finished_count`, `failed_count`, `cancelled_count`, `skipped_count`, `cancel_requested`, `created_at`, `updated_at`, `started_at`, `finished_at`, `last_error`
|
||||||
|
|
||||||
### `subscriptionsearchtask`
|
### `subscriptionsearchtask`
|
||||||
- Purpose: Stores one durable subscription search task per batch and subscription with leases and execution phases.
|
- Purpose: Stores one durable subscription search task per batch and subscription with leases and execution phases.
|
||||||
|
|||||||
@@ -1103,7 +1103,7 @@ Purpose: Read configured directory or storage settings.
|
|||||||
Purpose: Create one movie, TV, or music subscription.
|
Purpose: Create one movie, TV, or music subscription.
|
||||||
- `path_params`: none
|
- `path_params`: none
|
||||||
- `query`: none
|
- `query`: none
|
||||||
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
|
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
|
||||||
|
|
||||||
### `subscription.delete`
|
### `subscription.delete`
|
||||||
`DELETE /api/v1/subscribe/{subscribe_id}`; policy effect: `destructive_write`.
|
`DELETE /api/v1/subscribe/{subscribe_id}`; policy effect: `destructive_write`.
|
||||||
@@ -1277,7 +1277,7 @@ Purpose: Set one accessible subscription to running, paused, or stopped state.
|
|||||||
Purpose: Update one existing movie, TV, or music subscription.
|
Purpose: Update one existing movie, TV, or music subscription.
|
||||||
- `path_params`: none
|
- `path_params`: none
|
||||||
- `query`: none
|
- `query`: none
|
||||||
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
|
- `body`: `audio_format` (string|null): Requested or recorded audio container or codec, such as FLAC or MP3.; `audio_quality` (string|null): Subscription audio-quality rule, such as hires, lossless, or lossy.; `backdrop` (string|null): Backdrop image URL stored with the media or subscription.; `best_version` (integer|null): Enable normal best-version upgrading when set to 1.; `best_version_full` (integer|null): Enable full best-version upgrading when set to 1.; `completed_episode` (integer|null): Highest episode number already completed for the subscription.; `current_audio_format` (string|null): Audio format of the best version currently held.; `current_bit_depth` (integer|null): Bit depth of the best version currently held.; `current_bitrate` (integer|null): Bitrate of the best version currently held.; `current_priority` (integer|null): Calculated priority of the best version currently held.; `current_sample_rate` (integer|null): Sample rate of the best version currently held.; `custom_words` (string|null): Custom recognition or rename words applied to this media workflow.; `date` (string|null): Record creation or completion timestamp used by the history item.; `description` (string|null): Human-readable media, torrent, or subscription description.; `downloader` (string|null): Configured downloader instance name.; `effect` (string|null): Video or release-effect filter expression used by the subscription.; `episode_group` (string|null): TMDB episode-group identifier used for alternate episode ordering.; `episode_priority` (object|null): Per-episode best-version priority state.; `exclude` (string|null): Regular expression or filter expression that rejects matching releases.; `execution_status` (SubscriptionExecutionStatus|null): Current subscription execution status returned with the subscription snapshot.; `filter` (string|null): Named filter rule or rule expression applied to this site or subscription.; `filter_groups` (array<string>|null): Ordered filter-rule group names applied to the subscription.; `id` (integer|null): Persistent database identifier of the supplied record.; `include` (string|null): Regular expression or filter expression that a release must match.; `keyword` (string|null): Case-insensitive substring used to discover settings or filter storage entries.; `lack_episode` (integer|null; default `0`): Number of episodes still missing from the subscription.; `last_update` (string|null): Timestamp of the subscription's most recent update.; `media_category` (string|null): MoviePilot library category assigned to the media.; `media_id` (string|null): Source-native media ID. Always pair it with the exact media_source returned by search.; `media_source` (MediaSource|null): Metadata source identifier. Preserve the exact value returned with media_id.; `min_bit_depth` (integer|null): Minimum acceptable audio bit depth in bits.; `min_bitrate` (integer|null): Minimum acceptable audio bitrate in bits per second.; `min_sample_rate` (integer|null): Minimum acceptable audio sample rate in hertz.; `music_type` (string|null): Music identity level: recording, album, or artist where supported.; `name` (string|null): Human-readable name of the site, storage item, subscription, or rule group.; `note` (array<integer>|null): Structured auxiliary metadata stored with the record.; `poster` (string|null): Poster image URL stored with the media or subscription.; `quality` (string|null): Video or release quality filter expression.; `resolution` (string|null): Video resolution filter expression, such as 1080p or 2160p.; `save_path` (string|null): Configured downloader-side save path for the download or subscription.; `search_imdbid` (integer|null; default `0`): Use IMDb identity during subscription search when set to 1.; `season` (integer|null): Season number used by the media, search, subscription, or transfer operation.; `sites` (array<integer>|null): Exact site IDs included in the search or subscription scope.; `start_episode` (integer|null; default `0`): First episode number requested by the subscription.; `state` (string|null): Current site, subscription, marketplace, or transfer state filter.; `total_episode` (integer|null; default `0`): Expected total episode count for the subscription.; `total_tracks` (integer|null): Expected or recorded track count for a music item.; `type` (string|null): MoviePilot media or storage item type required by the selected operation.; `username` (string|null): MoviePilot or site username required by the selected operation.; `vote` (number|null; default `0.0`): Media vote average stored with the subscription.; `year` (string|null): Release or premiere year used to disambiguate the media title.
|
||||||
|
|
||||||
### `subscription.user.list`
|
### `subscription.user.list`
|
||||||
`GET /api/v1/subscribe/user/{username}`; policy effect: `safe_read`.
|
`GET /api/v1/subscribe/user/{username}`; policy effect: `safe_read`.
|
||||||
@@ -1681,6 +1681,18 @@ This runtime model has no directly writable fields.
|
|||||||
MoviePilot media type.
|
MoviePilot media type.
|
||||||
This runtime model has no directly writable fields.
|
This runtime model has no directly writable fields.
|
||||||
|
|
||||||
|
#### `SubscriptionExecutionStatus`
|
||||||
|
Subscription refresh execution status and progress summary.
|
||||||
|
- `batch_id` (string|null): Stable subscription search batch identifier.
|
||||||
|
- `can_cancel` (boolean; default `False`): Whether the current subscription execution can be cancelled.
|
||||||
|
- `current_site_id` (integer|null): Configured site ID currently handling the subscription execution.
|
||||||
|
- `error` (string|null): Human-readable workflow, provider, or execution error message.
|
||||||
|
- `phase*` (string): Current phase of a subscription execution.
|
||||||
|
- `source` (string|null): Exact metadata or recommendation source selected by the operation.
|
||||||
|
- `state*` (string): Current site, subscription, marketplace, or transfer state filter.
|
||||||
|
- `task_id` (string|null): Stable durable transfer task ID returned by transfer.manual_reviews.
|
||||||
|
- `updated_at*` (string): Timestamp when the subscription execution status was last updated.
|
||||||
|
|
||||||
#### `TorrentInfo`
|
#### `TorrentInfo`
|
||||||
One torrent candidate returned by MoviePilot search.
|
One torrent candidate returned by MoviePilot search.
|
||||||
- `category` (string|null): MoviePilot media category or filter-group category, depending on the operation.
|
- `category` (string|null): MoviePilot media category or filter-group category, depending on the operation.
|
||||||
|
|||||||
@@ -285,7 +285,6 @@ def configure_plugin_system_services():
|
|||||||
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
|
from app.db.adapters.mediaserver import TransactionalMediaServerRepository
|
||||||
from app.db.adapters.site import TransactionalSiteRepository
|
from app.db.adapters.site import TransactionalSiteRepository
|
||||||
from app.db.adapters.subscription import TransactionalSubscriptionRepository
|
from app.db.adapters.subscription import TransactionalSubscriptionRepository
|
||||||
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
|
|
||||||
from app.db.adapters.transaction import TransactionalWriteRunner
|
from app.db.adapters.transaction import TransactionalWriteRunner
|
||||||
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
|
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
|
||||||
from app.db.adapters.transfer.execution import (
|
from app.db.adapters.transfer.execution import (
|
||||||
@@ -413,7 +412,6 @@ def configure_plugin_system_services():
|
|||||||
transfer_execution_repository=TransactionalTransferExecutionRepository(SessionFactory),
|
transfer_execution_repository=TransactionalTransferExecutionRepository(SessionFactory),
|
||||||
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
|
media_server_repository=TransactionalMediaServerRepository(SessionFactory),
|
||||||
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
|
download_failure_repository=TransactionalDownloadFailureRepository(SessionFactory),
|
||||||
subscription_download_repository=TransactionalSubscriptionDownloadRepository(SessionFactory),
|
|
||||||
user_repository=user_repository(),
|
user_repository=user_repository(),
|
||||||
configuration=build_chain_runtime_config(settings),
|
configuration=build_chain_runtime_config(settings),
|
||||||
)
|
)
|
||||||
|
|||||||
+12
-45
@@ -1089,8 +1089,8 @@
|
|||||||
"runtime_only": true
|
"runtime_only": true
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"edge_count": 7874,
|
"edge_count": 7845,
|
||||||
"edge_sha256": "b4ca47899f8af4366446ad76546b5b0cf3c5e87f38b603d593241acd52fba9f3",
|
"edge_sha256": "aca69eac132e327d7a1dea770c185e7be5ab47bb41211749f6130c065a957919",
|
||||||
"edges": [
|
"edges": [
|
||||||
"app -> app.foundation",
|
"app -> app.foundation",
|
||||||
"app -> app.foundation.environment",
|
"app -> app.foundation.environment",
|
||||||
@@ -3527,8 +3527,6 @@
|
|||||||
"app.application.subscription.sitebudget -> app.runtime",
|
"app.application.subscription.sitebudget -> app.runtime",
|
||||||
"app.application.subscription.sitebudget -> app.runtime.stop",
|
"app.application.subscription.sitebudget -> app.runtime.stop",
|
||||||
"app.application.subscription.status -> app.application",
|
"app.application.subscription.status -> app.application",
|
||||||
"app.application.subscription.status -> app.application.download",
|
|
||||||
"app.application.subscription.status -> app.application.download.admission",
|
|
||||||
"app.application.subscription.status -> app.application.subscription",
|
"app.application.subscription.status -> app.application.subscription",
|
||||||
"app.application.subscription.status -> app.application.subscription.execution",
|
"app.application.subscription.status -> app.application.subscription.execution",
|
||||||
"app.application.subscription.write -> app.application",
|
"app.application.subscription.write -> app.application",
|
||||||
@@ -3634,9 +3632,13 @@
|
|||||||
"app.chain._messaging -> app.schemas.types",
|
"app.chain._messaging -> app.schemas.types",
|
||||||
"app.chain._music -> app.application",
|
"app.chain._music -> app.application",
|
||||||
"app.chain._music -> app.application.configuration",
|
"app.chain._music -> app.application.configuration",
|
||||||
|
"app.chain._music -> app.application.download",
|
||||||
|
"app.chain._music -> app.application.download.admission",
|
||||||
"app.chain._music -> app.application.subscription",
|
"app.chain._music -> app.application.subscription",
|
||||||
"app.chain._music -> app.application.subscription.contract",
|
"app.chain._music -> app.application.subscription.contract",
|
||||||
|
"app.chain._music -> app.application.subscription.execution",
|
||||||
"app.chain._music -> app.application.subscription.mutation",
|
"app.chain._music -> app.application.subscription.mutation",
|
||||||
|
"app.chain._music -> app.application.subscription.sitebudget",
|
||||||
"app.chain._music -> app.application.torrent",
|
"app.chain._music -> app.application.torrent",
|
||||||
"app.chain._music -> app.application.torrent.download",
|
"app.chain._music -> app.application.torrent.download",
|
||||||
"app.chain._music -> app.chain",
|
"app.chain._music -> app.chain",
|
||||||
@@ -3725,18 +3727,6 @@
|
|||||||
"app.chain.douban -> app.schemas",
|
"app.chain.douban -> app.schemas",
|
||||||
"app.chain.douban -> app.schemas.context",
|
"app.chain.douban -> app.schemas.context",
|
||||||
"app.chain.douban -> app.schemas.types",
|
"app.chain.douban -> app.schemas.types",
|
||||||
"app.chain.download.admission -> app.application",
|
|
||||||
"app.chain.download.admission -> app.application.download",
|
|
||||||
"app.chain.download.admission -> app.application.download.admission",
|
|
||||||
"app.chain.download.admission -> app.chain",
|
|
||||||
"app.chain.download.admission -> app.chain.download",
|
|
||||||
"app.chain.download.admission -> app.chain.download.contract",
|
|
||||||
"app.chain.download.admission -> app.domain",
|
|
||||||
"app.chain.download.admission -> app.domain.context",
|
|
||||||
"app.chain.download.admission -> app.domain.episode",
|
|
||||||
"app.chain.download.admission -> app.schemas",
|
|
||||||
"app.chain.download.admission -> app.schemas.media",
|
|
||||||
"app.chain.download.admission -> app.schemas.types",
|
|
||||||
"app.chain.download.batch -> app.application",
|
"app.chain.download.batch -> app.application",
|
||||||
"app.chain.download.batch -> app.application.download",
|
"app.chain.download.batch -> app.application.download",
|
||||||
"app.chain.download.batch -> app.application.download.admission",
|
"app.chain.download.batch -> app.application.download.admission",
|
||||||
@@ -3773,7 +3763,6 @@
|
|||||||
"app.chain.download.existence -> app.schemas.types",
|
"app.chain.download.existence -> app.schemas.types",
|
||||||
"app.chain.download.facade -> app.chain",
|
"app.chain.download.facade -> app.chain",
|
||||||
"app.chain.download.facade -> app.chain.download",
|
"app.chain.download.facade -> app.chain.download",
|
||||||
"app.chain.download.facade -> app.chain.download.admission",
|
|
||||||
"app.chain.download.facade -> app.chain.download.batch",
|
"app.chain.download.facade -> app.chain.download.batch",
|
||||||
"app.chain.download.facade -> app.chain.download.existence",
|
"app.chain.download.facade -> app.chain.download.existence",
|
||||||
"app.chain.download.facade -> app.chain.download.failure",
|
"app.chain.download.facade -> app.chain.download.failure",
|
||||||
@@ -4482,6 +4471,8 @@
|
|||||||
"app.chain.subscribe.facade -> app.application",
|
"app.chain.subscribe.facade -> app.application",
|
||||||
"app.chain.subscribe.facade -> app.application.messaging",
|
"app.chain.subscribe.facade -> app.application.messaging",
|
||||||
"app.chain.subscribe.facade -> app.application.messaging.subscribe",
|
"app.chain.subscribe.facade -> app.application.messaging.subscribe",
|
||||||
|
"app.chain.subscribe.facade -> app.application.subscription",
|
||||||
|
"app.chain.subscribe.facade -> app.application.subscription.execution",
|
||||||
"app.chain.subscribe.facade -> app.chain",
|
"app.chain.subscribe.facade -> app.chain",
|
||||||
"app.chain.subscribe.facade -> app.chain._interaction",
|
"app.chain.subscribe.facade -> app.chain._interaction",
|
||||||
"app.chain.subscribe.facade -> app.chain._music",
|
"app.chain.subscribe.facade -> app.chain._music",
|
||||||
@@ -4530,7 +4521,9 @@
|
|||||||
"app.chain.subscribe.match -> app.application.subscription",
|
"app.chain.subscribe.match -> app.application.subscription",
|
||||||
"app.chain.subscribe.match -> app.application.subscription.candidates",
|
"app.chain.subscribe.match -> app.application.subscription.candidates",
|
||||||
"app.chain.subscribe.match -> app.application.subscription.contract",
|
"app.chain.subscribe.match -> app.application.subscription.contract",
|
||||||
|
"app.chain.subscribe.match -> app.application.subscription.execution",
|
||||||
"app.chain.subscribe.match -> app.application.subscription.facts",
|
"app.chain.subscribe.match -> app.application.subscription.facts",
|
||||||
|
"app.chain.subscribe.match -> app.application.subscription.sitebudget",
|
||||||
"app.chain.subscribe.match -> app.application.torrent",
|
"app.chain.subscribe.match -> app.application.torrent",
|
||||||
"app.chain.subscribe.match -> app.application.torrent.download",
|
"app.chain.subscribe.match -> app.application.torrent.download",
|
||||||
"app.chain.subscribe.match -> app.chain",
|
"app.chain.subscribe.match -> app.chain",
|
||||||
@@ -4588,6 +4581,7 @@
|
|||||||
"app.chain.subscribe.policy -> app.application.download.admission",
|
"app.chain.subscribe.policy -> app.application.download.admission",
|
||||||
"app.chain.subscribe.policy -> app.application.subscription",
|
"app.chain.subscribe.policy -> app.application.subscription",
|
||||||
"app.chain.subscribe.policy -> app.application.subscription.contract",
|
"app.chain.subscribe.policy -> app.application.subscription.contract",
|
||||||
|
"app.chain.subscribe.policy -> app.application.subscription.execution",
|
||||||
"app.chain.subscribe.policy -> app.application.subscription.priority",
|
"app.chain.subscribe.policy -> app.application.subscription.priority",
|
||||||
"app.chain.subscribe.policy -> app.chain",
|
"app.chain.subscribe.policy -> app.chain",
|
||||||
"app.chain.subscribe.policy -> app.chain.download",
|
"app.chain.subscribe.policy -> app.chain.download",
|
||||||
@@ -4637,7 +4631,6 @@
|
|||||||
"app.chain.subscribe.reconcile -> app.schemas.types",
|
"app.chain.subscribe.reconcile -> app.schemas.types",
|
||||||
"app.chain.subscribe.refresh -> app.application",
|
"app.chain.subscribe.refresh -> app.application",
|
||||||
"app.chain.subscribe.refresh -> app.application.subscription",
|
"app.chain.subscribe.refresh -> app.application.subscription",
|
||||||
"app.chain.subscribe.refresh -> app.application.subscription.candidates",
|
|
||||||
"app.chain.subscribe.refresh -> app.application.subscription.contract",
|
"app.chain.subscribe.refresh -> app.application.subscription.contract",
|
||||||
"app.chain.subscribe.refresh -> app.application.subscription.facts",
|
"app.chain.subscribe.refresh -> app.application.subscription.facts",
|
||||||
"app.chain.subscribe.refresh -> app.application.subscription.priority",
|
"app.chain.subscribe.refresh -> app.application.subscription.priority",
|
||||||
@@ -5211,15 +5204,6 @@
|
|||||||
"app.db.adapters.subscription -> app.schemas",
|
"app.db.adapters.subscription -> app.schemas",
|
||||||
"app.db.adapters.subscription -> app.schemas.common",
|
"app.db.adapters.subscription -> app.schemas.common",
|
||||||
"app.db.adapters.subscription -> app.schemas.types",
|
"app.db.adapters.subscription -> app.schemas.types",
|
||||||
"app.db.adapters.subscriptiondownload -> app.application",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.application.download",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.application.download.admission",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db.models",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db.models.subscriptiondownload",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db.oper",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db.oper.subscriptiondownload",
|
|
||||||
"app.db.adapters.subscriptiondownload -> app.db.uow",
|
|
||||||
"app.db.adapters.subscriptionsearch -> app.application",
|
"app.db.adapters.subscriptionsearch -> app.application",
|
||||||
"app.db.adapters.subscriptionsearch -> app.application.subscription",
|
"app.db.adapters.subscriptionsearch -> app.application.subscription",
|
||||||
"app.db.adapters.subscriptionsearch -> app.application.subscription.execution",
|
"app.db.adapters.subscriptionsearch -> app.application.subscription.execution",
|
||||||
@@ -5231,13 +5215,10 @@
|
|||||||
"app.db.adapters.subscriptionsearch -> app.db.oper.subscriptionsearch",
|
"app.db.adapters.subscriptionsearch -> app.db.oper.subscriptionsearch",
|
||||||
"app.db.adapters.subscriptionsearch -> app.db.uow",
|
"app.db.adapters.subscriptionsearch -> app.db.uow",
|
||||||
"app.db.adapters.subscriptionstatus -> app.application",
|
"app.db.adapters.subscriptionstatus -> app.application",
|
||||||
"app.db.adapters.subscriptionstatus -> app.application.download",
|
|
||||||
"app.db.adapters.subscriptionstatus -> app.application.download.admission",
|
|
||||||
"app.db.adapters.subscriptionstatus -> app.application.subscription",
|
"app.db.adapters.subscriptionstatus -> app.application.subscription",
|
||||||
"app.db.adapters.subscriptionstatus -> app.application.subscription.execution",
|
"app.db.adapters.subscriptionstatus -> app.application.subscription.execution",
|
||||||
"app.db.adapters.subscriptionstatus -> app.db",
|
"app.db.adapters.subscriptionstatus -> app.db",
|
||||||
"app.db.adapters.subscriptionstatus -> app.db.models",
|
"app.db.adapters.subscriptionstatus -> app.db.models",
|
||||||
"app.db.adapters.subscriptionstatus -> app.db.models.subscriptiondownload",
|
|
||||||
"app.db.adapters.subscriptionstatus -> app.db.models.subscriptionsearch",
|
"app.db.adapters.subscriptionstatus -> app.db.models.subscriptionsearch",
|
||||||
"app.db.adapters.transaction -> app.db",
|
"app.db.adapters.transaction -> app.db",
|
||||||
"app.db.adapters.transaction -> app.db.uow",
|
"app.db.adapters.transaction -> app.db.uow",
|
||||||
@@ -5377,8 +5358,6 @@
|
|||||||
"app.db.models.subscribehistory -> app.db.models._constraints",
|
"app.db.models.subscribehistory -> app.db.models._constraints",
|
||||||
"app.db.models.subscribehistory -> app.schemas",
|
"app.db.models.subscribehistory -> app.schemas",
|
||||||
"app.db.models.subscribehistory -> app.schemas.types",
|
"app.db.models.subscribehistory -> app.schemas.types",
|
||||||
"app.db.models.subscriptiondownload -> app.db",
|
|
||||||
"app.db.models.subscriptiondownload -> app.db.base",
|
|
||||||
"app.db.models.subscriptionsearch -> app.db",
|
"app.db.models.subscriptionsearch -> app.db",
|
||||||
"app.db.models.subscriptionsearch -> app.db.base",
|
"app.db.models.subscriptionsearch -> app.db.base",
|
||||||
"app.db.models.systemconfig -> app.db",
|
"app.db.models.systemconfig -> app.db",
|
||||||
@@ -5487,13 +5466,6 @@
|
|||||||
"app.db.oper.subscribehistory -> app.schemas.common",
|
"app.db.oper.subscribehistory -> app.schemas.common",
|
||||||
"app.db.oper.subscribehistory -> app.schemas.query",
|
"app.db.oper.subscribehistory -> app.schemas.query",
|
||||||
"app.db.oper.subscribehistory -> app.schemas.types",
|
"app.db.oper.subscribehistory -> app.schemas.types",
|
||||||
"app.db.oper.subscriptiondownload -> app.application",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.application.download",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.application.download.admission",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.db",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.db.base",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.db.models",
|
|
||||||
"app.db.oper.subscriptiondownload -> app.db.models.subscriptiondownload",
|
|
||||||
"app.db.oper.subscriptionsearch -> app.db",
|
"app.db.oper.subscriptionsearch -> app.db",
|
||||||
"app.db.oper.subscriptionsearch -> app.db.base",
|
"app.db.oper.subscriptionsearch -> app.db.base",
|
||||||
"app.db.oper.subscriptionsearch -> app.db.models",
|
"app.db.oper.subscriptionsearch -> app.db.models",
|
||||||
@@ -8165,7 +8137,6 @@
|
|||||||
"app.startup.composition.chain -> app.db.adapters.chain",
|
"app.startup.composition.chain -> app.db.adapters.chain",
|
||||||
"app.startup.composition.chain -> app.db.adapters.download",
|
"app.startup.composition.chain -> app.db.adapters.download",
|
||||||
"app.startup.composition.chain -> app.db.adapters.mediaserver",
|
"app.startup.composition.chain -> app.db.adapters.mediaserver",
|
||||||
"app.startup.composition.chain -> app.db.adapters.subscriptiondownload",
|
|
||||||
"app.startup.composition.chain -> app.db.adapters.transfer",
|
"app.startup.composition.chain -> app.db.adapters.transfer",
|
||||||
"app.startup.composition.chain -> app.db.adapters.transfer.admission",
|
"app.startup.composition.chain -> app.db.adapters.transfer.admission",
|
||||||
"app.startup.composition.chain -> app.db.oper",
|
"app.startup.composition.chain -> app.db.oper",
|
||||||
@@ -8967,7 +8938,7 @@
|
|||||||
"app.workflow.actions.transfer_file -> app.workflow",
|
"app.workflow.actions.transfer_file -> app.workflow",
|
||||||
"app.workflow.actions.transfer_file -> app.workflow.actions"
|
"app.workflow.actions.transfer_file -> app.workflow.actions"
|
||||||
],
|
],
|
||||||
"module_count": 944,
|
"module_count": 940,
|
||||||
"modules": [
|
"modules": [
|
||||||
"app",
|
"app",
|
||||||
"app.adapters",
|
"app.adapters",
|
||||||
@@ -9294,7 +9265,6 @@
|
|||||||
"app.chain.dashboard",
|
"app.chain.dashboard",
|
||||||
"app.chain.douban",
|
"app.chain.douban",
|
||||||
"app.chain.download",
|
"app.chain.download",
|
||||||
"app.chain.download.admission",
|
|
||||||
"app.chain.download.batch",
|
"app.chain.download.batch",
|
||||||
"app.chain.download.contract",
|
"app.chain.download.contract",
|
||||||
"app.chain.download.existence",
|
"app.chain.download.existence",
|
||||||
@@ -9401,7 +9371,6 @@
|
|||||||
"app.db.adapters.query",
|
"app.db.adapters.query",
|
||||||
"app.db.adapters.site",
|
"app.db.adapters.site",
|
||||||
"app.db.adapters.subscription",
|
"app.db.adapters.subscription",
|
||||||
"app.db.adapters.subscriptiondownload",
|
|
||||||
"app.db.adapters.subscriptionsearch",
|
"app.db.adapters.subscriptionsearch",
|
||||||
"app.db.adapters.subscriptionstatus",
|
"app.db.adapters.subscriptionstatus",
|
||||||
"app.db.adapters.transaction",
|
"app.db.adapters.transaction",
|
||||||
@@ -9437,7 +9406,6 @@
|
|||||||
"app.db.models.siteuserdata",
|
"app.db.models.siteuserdata",
|
||||||
"app.db.models.subscribe",
|
"app.db.models.subscribe",
|
||||||
"app.db.models.subscribehistory",
|
"app.db.models.subscribehistory",
|
||||||
"app.db.models.subscriptiondownload",
|
|
||||||
"app.db.models.subscriptionsearch",
|
"app.db.models.subscriptionsearch",
|
||||||
"app.db.models.systemconfig",
|
"app.db.models.systemconfig",
|
||||||
"app.db.models.transferexecutionstep",
|
"app.db.models.transferexecutionstep",
|
||||||
@@ -9461,7 +9429,6 @@
|
|||||||
"app.db.oper.site",
|
"app.db.oper.site",
|
||||||
"app.db.oper.subscribe",
|
"app.db.oper.subscribe",
|
||||||
"app.db.oper.subscribehistory",
|
"app.db.oper.subscribehistory",
|
||||||
"app.db.oper.subscriptiondownload",
|
|
||||||
"app.db.oper.subscriptionsearch",
|
"app.db.oper.subscriptionsearch",
|
||||||
"app.db.oper.systemconfig",
|
"app.db.oper.systemconfig",
|
||||||
"app.db.oper.transferexecutionstep",
|
"app.db.oper.transferexecutionstep",
|
||||||
|
|||||||
+1
-1
@@ -1006,7 +1006,7 @@
|
|||||||
},
|
},
|
||||||
"app/chain/torrents.py": {
|
"app/chain/torrents.py": {
|
||||||
"arg-type": 22,
|
"arg-type": 22,
|
||||||
"assignment": 3,
|
"assignment": 2,
|
||||||
"no-untyped-call": 1,
|
"no-untyped-call": 1,
|
||||||
"no-untyped-def": 15,
|
"no-untyped-def": 15,
|
||||||
"type-arg": 5,
|
"type-arg": 5,
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
"repeat": 3,
|
"repeat": 3,
|
||||||
"targets": {
|
"targets": {
|
||||||
"app.startup.lifecycle": {
|
"app.startup.lifecycle": {
|
||||||
"loaded_app_module_count": 515,
|
"loaded_app_module_count": 511,
|
||||||
"max_ms": 1115.407,
|
"max_ms": 1115.407,
|
||||||
"median_ms": 1106.606,
|
"median_ms": 1106.606,
|
||||||
"min_ms": 1098.166,
|
"min_ms": 1098.166,
|
||||||
@@ -17,7 +17,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"app.factory": {
|
"app.factory": {
|
||||||
"loaded_app_module_count": 527,
|
"loaded_app_module_count": 523,
|
||||||
"max_ms": 1140.393,
|
"max_ms": 1140.393,
|
||||||
"median_ms": 1134.83,
|
"median_ms": 1134.83,
|
||||||
"min_ms": 1119.004,
|
"min_ms": 1119.004,
|
||||||
@@ -28,7 +28,7 @@
|
|||||||
]
|
]
|
||||||
},
|
},
|
||||||
"app.main": {
|
"app.main": {
|
||||||
"loaded_app_module_count": 529,
|
"loaded_app_module_count": 525,
|
||||||
"max_ms": 1180.675,
|
"max_ms": 1180.675,
|
||||||
"median_ms": 1165.915,
|
"median_ms": 1165.915,
|
||||||
"min_ms": 1161.859,
|
"min_ms": 1161.859,
|
||||||
|
|||||||
@@ -39,7 +39,6 @@ def test_download_monolith_is_retired_and_package_owners_are_complete() -> None:
|
|||||||
assert not (CHAIN_ROOT / "download.py").exists()
|
assert not (CHAIN_ROOT / "download.py").exists()
|
||||||
assert {path.name for path in DOWNLOAD_PACKAGE.glob("*.py")} == {
|
assert {path.name for path in DOWNLOAD_PACKAGE.glob("*.py")} == {
|
||||||
"__init__.py",
|
"__init__.py",
|
||||||
"admission.py",
|
|
||||||
"batch.py",
|
"batch.py",
|
||||||
"contract.py",
|
"contract.py",
|
||||||
"existence.py",
|
"existence.py",
|
||||||
|
|||||||
@@ -11,8 +11,13 @@ from app.application.subscription.contract import (
|
|||||||
SubscriptionSnapshot,
|
SubscriptionSnapshot,
|
||||||
build_subscribe_meta,
|
build_subscribe_meta,
|
||||||
)
|
)
|
||||||
|
from app.application.subscription.execution import (
|
||||||
|
SubscriptionExecutionAdmission,
|
||||||
|
SubscriptionExecutionContext,
|
||||||
|
)
|
||||||
from app.application.subscription.mutation import SubscriptionMutation
|
from app.application.subscription.mutation import SubscriptionMutation
|
||||||
from app.chain.subscribe import SubscribeChain
|
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
|
||||||
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
from app.domain.context import (
|
from app.domain.context import (
|
||||||
MUSIC_ENTITY_ALBUM,
|
MUSIC_ENTITY_ALBUM,
|
||||||
MUSIC_ENTITY_ARTIST,
|
MUSIC_ENTITY_ARTIST,
|
||||||
@@ -120,6 +125,23 @@ def _configure_subscription_write(chain, repository) -> None:
|
|||||||
chain.sync_subscription_mutation_scope = mutation_scope
|
chain.sync_subscription_mutation_scope = mutation_scope
|
||||||
|
|
||||||
|
|
||||||
|
def _execution_context(*, cancelled=None) -> SubscriptionExecutionContext:
|
||||||
|
"""构造音乐订阅使用的独立执行上下文。"""
|
||||||
|
admission = SubscriptionExecutionAdmission()
|
||||||
|
lease = admission.try_acquire(
|
||||||
|
subscription_id=7,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
return SubscriptionExecutionContext(
|
||||||
|
lease=lease,
|
||||||
|
admission=admission,
|
||||||
|
task_id="music-task-7",
|
||||||
|
cancel_requested=cancelled,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_build_subscribe_meta_returns_music_meta():
|
def test_build_subscribe_meta_returns_music_meta():
|
||||||
"""音乐订阅应构造 MetaMusic,而不是交给影视标题解析器。"""
|
"""音乐订阅应构造 MetaMusic,而不是交给影视标题解析器。"""
|
||||||
meta = build_subscribe_meta(_subscribe())
|
meta = build_subscribe_meta(_subscribe())
|
||||||
@@ -190,6 +212,85 @@ def test_music_subscribe_reuses_search_download_and_finish_flow():
|
|||||||
chain.finish_subscribe_or_not.assert_called_once()
|
chain.finish_subscribe_or_not.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_music_search_honours_cancel_before_external_work():
|
||||||
|
"""音乐搜索在安全边界收到取消后不得继续访问站点。"""
|
||||||
|
subscribe = _subscribe()
|
||||||
|
chain = SubscribeChain()
|
||||||
|
execution_context = _execution_context(cancelled=lambda: True)
|
||||||
|
|
||||||
|
with patch("app.chain._music.SearchChain") as search_chain, \
|
||||||
|
pytest.raises(SubscriptionSearchCancelled):
|
||||||
|
chain._search_music_subscribe(
|
||||||
|
subscribe,
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
search_chain.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_music_download_marks_shared_execution_context_before_side_effect():
|
||||||
|
"""音乐下载必须把取消和副作用边界传入统一下载治理。"""
|
||||||
|
cancelled = [False]
|
||||||
|
subscribe = _subscribe()
|
||||||
|
target = _music_info()
|
||||||
|
downloaded = Context(
|
||||||
|
torrent_info=TorrentInfo(
|
||||||
|
title="周杰伦 - 晴天 FLAC",
|
||||||
|
category=MediaType.MUSIC.value,
|
||||||
|
),
|
||||||
|
meta_info=MetaMusic.from_music_info(target),
|
||||||
|
media_info=target,
|
||||||
|
)
|
||||||
|
execution_context = _execution_context(cancelled=lambda: cancelled[0])
|
||||||
|
download_chain = Mock()
|
||||||
|
|
||||||
|
def batch_download(**kwargs):
|
||||||
|
"""模拟下载器边界内开始提交后才收到取消。"""
|
||||||
|
governance = kwargs["governance"]
|
||||||
|
assert governance.cancelled() is False
|
||||||
|
governance.mark_started()
|
||||||
|
cancelled[0] = True
|
||||||
|
return [downloaded], None
|
||||||
|
|
||||||
|
download_chain.batch_download.side_effect = batch_download
|
||||||
|
repository = Mock()
|
||||||
|
repository.get.return_value = subscribe
|
||||||
|
chain = SubscribeChain()
|
||||||
|
chain.subscription_repository = repository
|
||||||
|
chain.finish_subscribe_or_not = Mock()
|
||||||
|
|
||||||
|
with patch("app.chain._music.DownloadChain", return_value=download_chain):
|
||||||
|
chain._download_music_subscribe(
|
||||||
|
subscribe,
|
||||||
|
target,
|
||||||
|
[downloaded],
|
||||||
|
execution_context=execution_context,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert execution_context.download_started is True
|
||||||
|
chain.finish_subscribe_or_not.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_music_download_rechecks_paused_state_before_submission():
|
||||||
|
"""候选准备后暂停的音乐订阅不得进入下载器。"""
|
||||||
|
subscribe = _subscribe()
|
||||||
|
paused = _subscribe(state="S")
|
||||||
|
repository = Mock()
|
||||||
|
repository.get.return_value = paused
|
||||||
|
chain = SubscribeChain()
|
||||||
|
chain.subscription_repository = repository
|
||||||
|
|
||||||
|
with patch("app.chain._music.DownloadChain") as download_chain:
|
||||||
|
chain._download_music_subscribe(
|
||||||
|
subscribe,
|
||||||
|
_music_info(),
|
||||||
|
[Context()],
|
||||||
|
execution_context=_execution_context(),
|
||||||
|
)
|
||||||
|
|
||||||
|
download_chain.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_music_subscribe_filters_declared_bitrate_and_format():
|
def test_music_subscribe_filters_declared_bitrate_and_format():
|
||||||
"""音乐订阅应按规范化格式和最低码率过滤站点资源。"""
|
"""音乐订阅应按规范化格式和最低码率过滤站点资源。"""
|
||||||
subscribe = _subscribe(audio_format="MP3", min_bitrate=320000)
|
subscribe = _subscribe(audio_format="MP3", min_bitrate=320000)
|
||||||
@@ -274,6 +375,7 @@ def test_music_best_version_persists_downloaded_rule_priority():
|
|||||||
download_chain.batch_download.return_value = ([downloaded], None)
|
download_chain.batch_download.return_value = ([downloaded], None)
|
||||||
subscribe_oper = Mock()
|
subscribe_oper = Mock()
|
||||||
updated = _subscribe(best_version=1, current_priority=100)
|
updated = _subscribe(best_version=1, current_priority=100)
|
||||||
|
subscribe_oper.get.return_value = subscribe
|
||||||
subscribe_oper.update.return_value = updated
|
subscribe_oper.update.return_value = updated
|
||||||
chain = SubscribeChain()
|
chain = SubscribeChain()
|
||||||
chain.finish_subscribe_or_not = Mock()
|
chain.finish_subscribe_or_not = Mock()
|
||||||
|
|||||||
@@ -701,16 +701,15 @@ class TestSubscribeChain:
|
|||||||
|
|
||||||
def test_match_title_fallback_calls_torrent_match_from_class(self):
|
def test_match_title_fallback_calls_torrent_match_from_class(self):
|
||||||
"""确保标题兜底匹配不依赖 TorrentHelper 实例绑定。"""
|
"""确保标题兜底匹配不依赖 TorrentHelper 实例绑定。"""
|
||||||
|
reached = []
|
||||||
class _ReachedTitleMatch(Exception):
|
|
||||||
"""标记测试已经进入标题匹配函数体。"""
|
|
||||||
|
|
||||||
class _PlainTorrentHelper:
|
class _PlainTorrentHelper:
|
||||||
"""模拟需要按类调用的 TorrentHelper 形态。"""
|
"""模拟需要按类调用的 TorrentHelper 形态。"""
|
||||||
|
|
||||||
def match_torrent(mediainfo, torrent_meta, torrent):
|
def match_torrent(mediainfo, torrent_meta, torrent):
|
||||||
"""标记类级调用已经正确进入匹配逻辑。"""
|
"""标记类级调用已经正确进入匹配逻辑。"""
|
||||||
raise _ReachedTitleMatch
|
reached.append((mediainfo, torrent_meta, torrent))
|
||||||
|
return False
|
||||||
|
|
||||||
def filter_torrent(self, *args, **kwargs):
|
def filter_torrent(self, *args, **kwargs):
|
||||||
"""保持订阅匹配后续过滤流程可继续执行。"""
|
"""保持订阅匹配后续过滤流程可继续执行。"""
|
||||||
@@ -753,9 +752,14 @@ class TestSubscribeChain:
|
|||||||
"""返回当前测试构造的订阅列表。"""
|
"""返回当前测试构造的订阅列表。"""
|
||||||
return [subscribe]
|
return [subscribe]
|
||||||
|
|
||||||
|
def get(self, subscribe_id):
|
||||||
|
"""返回取得订阅准入后的最新快照。"""
|
||||||
|
return subscribe if subscribe_id == subscribe.id else None
|
||||||
|
|
||||||
chain = SubscribeChain()
|
chain = SubscribeChain()
|
||||||
chain.subscription_repository = _SubscribeOper()
|
chain.subscription_repository = _SubscribeOper()
|
||||||
chain.check_and_handle_existing_media = lambda **kwargs: (False, {})
|
chain.check_and_handle_existing_media = lambda **kwargs: (False, {})
|
||||||
|
chain.finish_subscribe_or_not = lambda **kwargs: None
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch.object(
|
patch.object(
|
||||||
@@ -764,10 +768,15 @@ class TestSubscribeChain:
|
|||||||
_PlainTorrentHelper,
|
_PlainTorrentHelper,
|
||||||
),
|
),
|
||||||
_patch_media_recognize(SUBSCRIBE_CHAIN_MODULE, mediainfo),
|
_patch_media_recognize(SUBSCRIBE_CHAIN_MODULE, mediainfo),
|
||||||
pytest.raises(_ReachedTitleMatch),
|
|
||||||
):
|
):
|
||||||
chain.match({"test.example": [context]})
|
chain.match({"test.example": [context]})
|
||||||
|
|
||||||
|
assert len(reached) == 1
|
||||||
|
actual_media, actual_meta, actual_torrent = reached[0]
|
||||||
|
assert actual_media.title_year == mediainfo.title_year
|
||||||
|
assert actual_meta is context.meta_info
|
||||||
|
assert actual_torrent is context.torrent_info
|
||||||
|
|
||||||
def test_match_accepts_special_season_zero_candidate(self):
|
def test_match_accepts_special_season_zero_candidate(self):
|
||||||
"""S0 订阅应允许 S00 候选资源进入下载候选,不能按未指定季处理。"""
|
"""S0 订阅应允许 S00 候选资源进入下载候选,不能按未指定季处理。"""
|
||||||
|
|
||||||
@@ -862,6 +871,16 @@ class TestSubscribeChain:
|
|||||||
|
|
||||||
assert len(download_calls) == 1
|
assert len(download_calls) == 1
|
||||||
assert download_calls[0]["contexts"][0].meta_info.begin_season == 0
|
assert download_calls[0]["contexts"][0].meta_info.begin_season == 0
|
||||||
|
execution_context = download_calls[0]["execution_context"]
|
||||||
|
assert execution_context.lease.subscription_id == subscribe.id
|
||||||
|
assert execution_context.lease.operation == "match"
|
||||||
|
replacement = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert chain._subscription_execution_admission.release(replacement) is True
|
||||||
|
|
||||||
def test_get_episode_priority_falls_back_to_current_priority(self):
|
def test_get_episode_priority_falls_back_to_current_priority(self):
|
||||||
subscribe = self._build_subscribe(current_priority=80, episode_priority=None)
|
subscribe = self._build_subscribe(current_priority=80, episode_priority=None)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import threading
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta
|
||||||
@@ -7,9 +8,10 @@ from unittest.mock import Mock, patch
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.application.subscription.contract import SubscriptionPatch, SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionPatch, SubscriptionSnapshot
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
from app.application.subscription.mutation import SubscriptionMutation
|
from app.application.subscription.mutation import SubscriptionMutation
|
||||||
from app.chain.subscribe import SubscribeChain
|
|
||||||
from app.chain.subscribe import search as subscribe_search
|
from app.chain.subscribe import search as subscribe_search
|
||||||
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
from app.schemas.types import MediaType
|
from app.schemas.types import MediaType
|
||||||
|
|
||||||
|
|
||||||
@@ -150,14 +152,19 @@ def test_targeted_batch_searches_all_ids_without_state_scan(monkeypatch) -> None
|
|||||||
chain.subscription_repository = subscribe_oper
|
chain.subscription_repository = subscribe_oper
|
||||||
chain.search(sids=(31, 32), state=None, manual=False)
|
chain.search(sids=(31, 32), state=None, manual=False)
|
||||||
|
|
||||||
assert [item.args for item in subscribe_oper.get.call_args_list] == [(31,), (32,)]
|
assert [item.args for item in subscribe_oper.get.call_args_list] == [
|
||||||
|
(31,),
|
||||||
|
(32,),
|
||||||
|
(31,),
|
||||||
|
(32,),
|
||||||
|
]
|
||||||
subscribe_oper.list.assert_not_called()
|
subscribe_oper.list.assert_not_called()
|
||||||
assert media_chain.recognize_media.call_count == 2
|
assert media_chain.recognize_media.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None:
|
def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None:
|
||||||
"""订阅搜索锁超时后必须中止,不能在无锁状态下继续访问订阅。"""
|
"""订阅搜索锁超时后必须中止,不能在无锁状态下继续访问订阅。"""
|
||||||
monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock())
|
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", _TimedOutLock())
|
||||||
subscribe_oper = Mock()
|
subscribe_oper = Mock()
|
||||||
progress = Mock()
|
progress = Mock()
|
||||||
|
|
||||||
@@ -176,7 +183,7 @@ def test_subscribe_search_releases_lock_when_repository_query_fails(monkeypatch)
|
|||||||
"""取得搜索锁后即使订阅查询失败,也必须释放进程级互斥锁。"""
|
"""取得搜索锁后即使订阅查询失败,也必须释放进程级互斥锁。"""
|
||||||
lock = Mock()
|
lock = Mock()
|
||||||
lock.acquire.return_value = True
|
lock.acquire.return_value = True
|
||||||
monkeypatch.setattr(SubscribeChain, "_rlock", lock)
|
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", lock)
|
||||||
repository = Mock()
|
repository = Mock()
|
||||||
repository.list.side_effect = RuntimeError("query failed")
|
repository.list.side_effect = RuntimeError("query failed")
|
||||||
chain = object.__new__(SubscribeChain)
|
chain = object.__new__(SubscribeChain)
|
||||||
@@ -228,9 +235,45 @@ def test_subscribe_search_progress_preserves_public_callback_payload() -> None:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_search_conflict_does_not_report_false_completion(monkeypatch) -> None:
|
||||||
|
"""兼容搜索遇到同订阅冲突时必须报告未执行,而不是成功完成。"""
|
||||||
|
subscribe = replace(
|
||||||
|
_new_subscribe(datetime.now() - timedelta(minutes=2)),
|
||||||
|
state="R",
|
||||||
|
)
|
||||||
|
repository = Mock()
|
||||||
|
repository.get.return_value = subscribe
|
||||||
|
progress = Mock()
|
||||||
|
chain = object.__new__(SubscribeChain)
|
||||||
|
chain.subscription_repository = repository
|
||||||
|
chain._search_queue_lock = threading.Lock()
|
||||||
|
chain._match_lock = threading.Lock()
|
||||||
|
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
|
||||||
|
match_lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert match_lease is not None
|
||||||
|
|
||||||
|
with patch.object(subscribe_search, "SearchChain", return_value=Mock()):
|
||||||
|
chain.search(
|
||||||
|
sid=subscribe.id,
|
||||||
|
state=None,
|
||||||
|
progress_callback=progress,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert progress.call_args.kwargs == {
|
||||||
|
"value": 100,
|
||||||
|
"text": "订阅搜索结束,部分订阅本轮未执行或未完成",
|
||||||
|
"data": {"total": 1, "finished": 0},
|
||||||
|
}
|
||||||
|
assert chain._subscription_execution_admission.release(match_lease) is True
|
||||||
|
|
||||||
|
|
||||||
def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None:
|
def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None:
|
||||||
"""订阅匹配锁超时后必须中止,不能绕过防重复下载边界。"""
|
"""订阅匹配锁超时后必须中止,不能绕过防重复下载边界。"""
|
||||||
monkeypatch.setattr(SubscribeChain, "_rlock", _TimedOutLock())
|
monkeypatch.setattr(SubscribeChain, "_match_lock", _TimedOutLock())
|
||||||
progress = Mock()
|
progress = Mock()
|
||||||
|
|
||||||
chain = object.__new__(SubscribeChain)
|
chain = object.__new__(SubscribeChain)
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from app.application.subscription.candidates import CandidateBatch, CandidateIndex
|
from app.application.subscription.candidates import CandidateIndex
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
from app.application.subscription.facts import FreshFactLease
|
from app.application.subscription.facts import FreshFactLease
|
||||||
from app.chain.torrents import TorrentsChain
|
from app.chain.torrents import TorrentsChain
|
||||||
@@ -68,8 +68,13 @@ def _subscribe(**overrides) -> SubscriptionSnapshot:
|
|||||||
return SubscriptionSnapshot(**values)
|
return SubscriptionSnapshot(**values)
|
||||||
|
|
||||||
|
|
||||||
def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta():
|
def _candidate_count(groups: dict[str, list[Context]]) -> int:
|
||||||
"""刷新批次必须保留完整缓存,同时只把本轮新增资源放入 fresh 集合。"""
|
"""统计分站点候选总数。"""
|
||||||
|
return sum(len(contexts) for contexts in groups.values())
|
||||||
|
|
||||||
|
|
||||||
|
def test_refresh_returns_complete_cache_with_new_candidates():
|
||||||
|
"""刷新结果必须包含既有缓存和本轮新增资源的完整候选。"""
|
||||||
chain = TorrentsChain()
|
chain = TorrentsChain()
|
||||||
existing = _context("既有剧集 S01E01", media_id="100")
|
existing = _context("既有剧集 S01E01", media_id="100")
|
||||||
duplicate = TorrentInfo(
|
duplicate = TorrentInfo(
|
||||||
@@ -115,18 +120,13 @@ def test_refresh_batch_distinguishes_complete_cache_from_fresh_delta():
|
|||||||
),
|
),
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
batch = chain.refresh_batch(stype="rss", sites=[1])
|
candidates = chain.refresh(stype="rss", sites=[1])
|
||||||
|
|
||||||
assert isinstance(batch, CandidateBatch)
|
assert [item.torrent_info.title for item in candidates["example.com"]] == [
|
||||||
assert batch.source == "rss"
|
|
||||||
assert batch.finished_at is not None
|
|
||||||
assert [item.torrent_info.title for item in batch.candidates["example.com"]] == [
|
|
||||||
existing.torrent_info.title,
|
existing.torrent_info.title,
|
||||||
fresh.title,
|
fresh.title,
|
||||||
]
|
]
|
||||||
assert [item.torrent_info.title for item in batch.fresh_candidates["example.com"]] == [fresh.title]
|
assert _candidate_count(candidates) == 2
|
||||||
assert CandidateBatch.count(batch.candidates) == 2
|
|
||||||
assert CandidateBatch.count(batch.fresh_candidates) == 1
|
|
||||||
|
|
||||||
|
|
||||||
def test_candidate_index_routes_all_canonical_fallback_classes_without_loss():
|
def test_candidate_index_routes_all_canonical_fallback_classes_without_loss():
|
||||||
@@ -214,7 +214,7 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product():
|
|||||||
for subscribe in subscribes:
|
for subscribe in subscribes:
|
||||||
groups = candidate_index.route_for_match(subscribe)
|
groups = candidate_index.route_for_match(subscribe)
|
||||||
examined += candidate_index.last_examined_count
|
examined += candidate_index.last_examined_count
|
||||||
routed += CandidateBatch.count(groups)
|
routed += _candidate_count(groups)
|
||||||
|
|
||||||
lease = FreshFactLease()
|
lease = FreshFactLease()
|
||||||
fact_loads = []
|
fact_loads = []
|
||||||
@@ -231,9 +231,9 @@ def test_candidate_index_target_scale_avoids_subscription_candidate_product():
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert len(candidates) == site_count
|
assert len(candidates) == site_count
|
||||||
assert CandidateBatch.count(candidates) == 1000
|
assert _candidate_count(candidates) == 1000
|
||||||
assert examined == routed == 2000
|
assert examined == routed == 2000
|
||||||
assert examined < subscription_count * CandidateBatch.count(candidates) // 50
|
assert examined < subscription_count * _candidate_count(candidates) // 50
|
||||||
assert len(fact_loads) == media_count
|
assert len(fact_loads) == media_count
|
||||||
assert lease.loads == media_count
|
assert lease.loads == media_count
|
||||||
assert lease.hits == subscription_count - media_count
|
assert lease.hits == subscription_count - media_count
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
"""订阅 Search/Match 公共入口的跨通道准入测试。"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
|
from app.domain.context import Context
|
||||||
|
from app.schemas.types import MediaType
|
||||||
|
|
||||||
|
|
||||||
|
def _subscribe(subscribe_id: int) -> SubscriptionSnapshot:
|
||||||
|
"""构造已越过新增保护期的活动电影订阅。"""
|
||||||
|
return SubscriptionSnapshot(
|
||||||
|
id=subscribe_id,
|
||||||
|
name=f"并发电影 {subscribe_id}",
|
||||||
|
year="2026",
|
||||||
|
type=MediaType.MOVIE.value,
|
||||||
|
media_source="themoviedb",
|
||||||
|
media_id=str(5000 + subscribe_id),
|
||||||
|
state="R",
|
||||||
|
date=(datetime.now() - timedelta(minutes=2)).strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _repository(*subscribes: SubscriptionSnapshot) -> SimpleNamespace:
|
||||||
|
"""提供公共入口重新读取所需的稳定订阅仓储。"""
|
||||||
|
snapshots = {subscribe.id: subscribe for subscribe in subscribes}
|
||||||
|
return SimpleNamespace(
|
||||||
|
get=snapshots.get,
|
||||||
|
list=lambda _state: list(snapshots.values()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _configure_channel_state(monkeypatch) -> None:
|
||||||
|
"""为不同 SubscribeChain 实例安装进程级共享准入与独立通道锁。"""
|
||||||
|
monkeypatch.setattr(SubscribeChain, "_match_lock", threading.Lock())
|
||||||
|
monkeypatch.setattr(SubscribeChain, "_search_queue_lock", threading.Lock())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SubscribeChain,
|
||||||
|
"_subscription_execution_admission",
|
||||||
|
SubscriptionExecutionAdmission(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_search_and_match_skip_same_subscription_across_instances(monkeypatch) -> None:
|
||||||
|
"""两个独立 Chain 实例处理同一订阅时只能有一个进入业务路径。"""
|
||||||
|
_configure_channel_state(monkeypatch)
|
||||||
|
subscribe = _subscribe(7)
|
||||||
|
repository = _repository(subscribe)
|
||||||
|
search_chain = object.__new__(SubscribeChain)
|
||||||
|
search_chain.subscription_repository = repository
|
||||||
|
match_chain = object.__new__(SubscribeChain)
|
||||||
|
match_chain.subscription_repository = repository
|
||||||
|
match_chain.get_states_for_search = lambda state: state
|
||||||
|
match_chain._prepare_match_torrents = lambda torrents: torrents
|
||||||
|
|
||||||
|
search_started = threading.Event()
|
||||||
|
release_search = threading.Event()
|
||||||
|
|
||||||
|
def process_search(item, _searchchain, *, execution_context):
|
||||||
|
"""持有 Search 准入,直到 Match 完成冲突探测。"""
|
||||||
|
assert execution_context.lease.subscription_id == item.id
|
||||||
|
search_started.set()
|
||||||
|
assert release_search.wait(timeout=3)
|
||||||
|
return item
|
||||||
|
|
||||||
|
match_subscription = Mock(side_effect=AssertionError("冲突 Match 不得进入订阅业务路径"))
|
||||||
|
search_chain._process_search_subscription = process_search
|
||||||
|
match_chain._match_subscription = match_subscription
|
||||||
|
worker = threading.Thread(
|
||||||
|
target=search_chain.search,
|
||||||
|
kwargs={"sid": subscribe.id, "state": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
worker.start()
|
||||||
|
assert search_started.wait(timeout=3)
|
||||||
|
match_chain.match({"example.org": [Context()]})
|
||||||
|
release_search.set()
|
||||||
|
worker.join(timeout=3)
|
||||||
|
|
||||||
|
assert worker.is_alive() is False
|
||||||
|
match_subscription.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_search_and_match_allow_different_subscriptions_in_parallel(monkeypatch) -> None:
|
||||||
|
"""Search 持有一条订阅时 Match 仍可处理另一条订阅。"""
|
||||||
|
_configure_channel_state(monkeypatch)
|
||||||
|
search_subscribe = _subscribe(8)
|
||||||
|
match_subscribe = _subscribe(9)
|
||||||
|
search_chain = object.__new__(SubscribeChain)
|
||||||
|
search_chain.subscription_repository = _repository(search_subscribe)
|
||||||
|
match_chain = object.__new__(SubscribeChain)
|
||||||
|
match_chain.subscription_repository = _repository(match_subscribe)
|
||||||
|
match_chain.get_states_for_search = lambda state: state
|
||||||
|
match_chain._prepare_match_torrents = lambda torrents: torrents
|
||||||
|
|
||||||
|
search_started = threading.Event()
|
||||||
|
release_search = threading.Event()
|
||||||
|
|
||||||
|
def process_search(item, _searchchain, *, execution_context):
|
||||||
|
"""阻塞 Search 以证明 Match 不依赖 Search 通道锁。"""
|
||||||
|
assert execution_context.lease.subscription_id == item.id
|
||||||
|
search_started.set()
|
||||||
|
assert release_search.wait(timeout=3)
|
||||||
|
return item
|
||||||
|
|
||||||
|
match_subscription = Mock()
|
||||||
|
search_chain._process_search_subscription = process_search
|
||||||
|
match_chain._match_subscription = match_subscription
|
||||||
|
worker = threading.Thread(
|
||||||
|
target=search_chain.search,
|
||||||
|
kwargs={"sid": search_subscribe.id, "state": None},
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
worker.start()
|
||||||
|
assert search_started.wait(timeout=3)
|
||||||
|
match_chain.match({"example.org": [Context()]})
|
||||||
|
release_search.set()
|
||||||
|
worker.join(timeout=3)
|
||||||
|
|
||||||
|
assert worker.is_alive() is False
|
||||||
|
match_subscription.assert_called_once()
|
||||||
|
assert match_subscription.call_args.kwargs["subscribe"].id == match_subscribe.id
|
||||||
@@ -1,9 +1,8 @@
|
|||||||
"""订阅下载跨入口幂等、不确定终态与取消补偿测试。"""
|
"""订阅下载取消边界、普通失败语义与账本移除迁移测试。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib
|
import importlib
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
@@ -13,98 +12,22 @@ import sqlalchemy as sa
|
|||||||
from alembic.migration import MigrationContext
|
from alembic.migration import MigrationContext
|
||||||
from alembic.operations import Operations
|
from alembic.operations import Operations
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.orm import sessionmaker
|
|
||||||
|
|
||||||
import app.chain.download.submission as download_submission
|
import app.chain.download.submission as download_submission
|
||||||
from app.application.download.admission import (
|
from app.application.download.admission import SubscriptionDownloadGovernance
|
||||||
DownloadReconciliationRequired,
|
|
||||||
SubscriptionDownloadGovernance,
|
|
||||||
SubscriptionDownloadRequest,
|
|
||||||
)
|
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
|
from app.application.subscription.execution import (
|
||||||
|
SubscriptionExecutionAdmission,
|
||||||
|
SubscriptionExecutionContext,
|
||||||
|
)
|
||||||
from app.chain.download import DownloadChain
|
from app.chain.download import DownloadChain
|
||||||
from app.chain.subscribe import policy as subscribe_policy
|
from app.chain.subscribe import policy as subscribe_policy
|
||||||
from app.chain.subscribe.facade import SubscribeChain
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
from app.db.adapters.subscriptiondownload import TransactionalSubscriptionDownloadRepository
|
|
||||||
from app.db.base import Base
|
|
||||||
from app.db.models.subscriptiondownload import SubscriptionDownloadSubmission
|
|
||||||
from app.domain.context import Context, MediaInfo, TorrentInfo
|
from app.domain.context import Context, MediaInfo, TorrentInfo
|
||||||
from app.domain.metainfo import MetaInfo
|
from app.domain.metainfo import MetaInfo
|
||||||
from app.schemas.types import MediaSource, MediaType
|
from app.schemas.types import MediaSource, MediaType
|
||||||
|
|
||||||
|
|
||||||
def _request(key: str = "key-1", task_id: str | None = "task-1") -> SubscriptionDownloadRequest:
|
|
||||||
"""构造固定身份的提交认领请求。"""
|
|
||||||
return SubscriptionDownloadRequest(
|
|
||||||
idempotency_key=key,
|
|
||||||
legacy_idempotency_key=None,
|
|
||||||
subscription_id=7,
|
|
||||||
task_id=task_id,
|
|
||||||
logical_identity='{"subscription_id":7}',
|
|
||||||
resource_key="example.com:id=42",
|
|
||||||
coverage="episodes:E01-E03",
|
|
||||||
mode="normal",
|
|
||||||
delivery_scope='{"download_uri":"local:/downloads","downloader":"auto"}',
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _repository(tmp_path) -> tuple[TransactionalSubscriptionDownloadRepository, object]:
|
|
||||||
"""创建独立 SQLite 幂等账本仓储与 Session 工厂。"""
|
|
||||||
engine = create_engine(
|
|
||||||
f"sqlite:///{tmp_path / 'subscription-download.db'}",
|
|
||||||
connect_args={"check_same_thread": False},
|
|
||||||
)
|
|
||||||
Base.metadata.create_all(engine)
|
|
||||||
factory = sessionmaker(bind=engine)
|
|
||||||
return TransactionalSubscriptionDownloadRepository(factory), factory
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_claims_same_submission_once_across_workers(tmp_path) -> None:
|
|
||||||
"""并发入口对同一幂等键只能有一个取得下载器提交权。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
||||||
claims = list(executor.map(lambda _index: repository.claim(_request()), range(2)))
|
|
||||||
|
|
||||||
assert sum(claim.acquired for claim in claims) == 1
|
|
||||||
assert {claim.snapshot.state for claim in claims} == {"submitting"}
|
|
||||||
assert {claim.snapshot.attempt_count for claim in claims} == {1}
|
|
||||||
|
|
||||||
|
|
||||||
def test_repository_fences_retry_and_preserves_uncertain_terminal(tmp_path) -> None:
|
|
||||||
"""明确拒绝可延迟重试,过期令牌和待对账状态均不得重新提交。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
first = repository.claim(_request())
|
|
||||||
token = first.snapshot.attempt_token
|
|
||||||
assert token
|
|
||||||
assert repository.mark_retryable(
|
|
||||||
idempotency_key="key-1",
|
|
||||||
attempt_token=token,
|
|
||||||
available_at="1970-01-01T00:00:00+00:00",
|
|
||||||
error="downloader rejected",
|
|
||||||
)
|
|
||||||
|
|
||||||
second = repository.claim(_request(task_id="task-2"))
|
|
||||||
second_token = second.snapshot.attempt_token
|
|
||||||
assert second.acquired
|
|
||||||
assert second_token and second_token != token
|
|
||||||
assert not repository.mark_succeeded(
|
|
||||||
idempotency_key="key-1",
|
|
||||||
attempt_token=token,
|
|
||||||
)
|
|
||||||
assert repository.mark_reconcile_required(
|
|
||||||
idempotency_key="key-1",
|
|
||||||
attempt_token=second_token,
|
|
||||||
error="transport timeout",
|
|
||||||
)
|
|
||||||
|
|
||||||
blocked = repository.claim(_request(task_id="task-3"))
|
|
||||||
assert not blocked.acquired
|
|
||||||
assert blocked.snapshot.state == "reconcile_required"
|
|
||||||
assert blocked.snapshot.attempt_count == 2
|
|
||||||
assert repository.has_started_for_task("task-2")
|
|
||||||
|
|
||||||
|
|
||||||
class _FakeTorrentHelper:
|
class _FakeTorrentHelper:
|
||||||
"""返回固定种子目录和文件清单,隔离真实 bencode 解析。"""
|
"""返回固定种子目录和文件清单,隔离真实 bencode 解析。"""
|
||||||
|
|
||||||
@@ -114,10 +37,9 @@ class _FakeTorrentHelper:
|
|||||||
return "Demo.Show.S01", ["Demo.Show.S01E01.mkv"]
|
return "Demo.Show.S01", ["Demo.Show.S01E01.mkv"]
|
||||||
|
|
||||||
|
|
||||||
def _download_chain(repository) -> DownloadChain:
|
def _download_chain() -> DownloadChain:
|
||||||
"""构造只执行下载提交边界的 Chain 测试实例。"""
|
"""构造只执行下载提交边界的 Chain 测试实例。"""
|
||||||
chain = DownloadChain.__new__(DownloadChain)
|
chain = DownloadChain.__new__(DownloadChain)
|
||||||
chain.subscription_download_repository = repository
|
|
||||||
chain.download_history_repository = MagicMock()
|
chain.download_history_repository = MagicMock()
|
||||||
chain.download_history_repository.get_by_media_identity.return_value = []
|
chain.download_history_repository.get_by_media_identity.return_value = []
|
||||||
chain.download_failure_repository = MagicMock()
|
chain.download_failure_repository = MagicMock()
|
||||||
@@ -174,12 +96,11 @@ def _submission_dependencies(monkeypatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_reuses_success_without_second_downloader_call(tmp_path) -> None:
|
def test_download_chain_uses_normal_submission_for_each_execution() -> None:
|
||||||
"""重叠入口在首个提交成功后复用 hash,不再次调用下载器。"""
|
"""下载边界不保留本地提交账本,由下载器处理相同 torrent 的复用。"""
|
||||||
repository, _factory = _repository(tmp_path)
|
chain = _download_chain()
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(return_value=("qb", "hash-1", "Original", "accepted"))
|
chain.download = MagicMock(return_value=("qb", "hash-1", "Original", "accepted"))
|
||||||
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
|
governance = SubscriptionDownloadGovernance()
|
||||||
|
|
||||||
first = chain.download_single(
|
first = chain.download_single(
|
||||||
context=_context(),
|
context=_context(),
|
||||||
@@ -197,306 +118,91 @@ def test_download_chain_reuses_success_without_second_downloader_call(tmp_path)
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert first == second == "hash-1"
|
assert first == second == "hash-1"
|
||||||
chain.download.assert_called_once()
|
assert chain.download.call_count == 2
|
||||||
chain._settle_download_success.assert_called_once()
|
assert chain._settle_download_success.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_deduplicates_same_media_across_subscription_rows(tmp_path) -> None:
|
def test_download_chain_retries_after_downloader_exception() -> None:
|
||||||
"""同媒体同覆盖同交付目标的重复订阅只允许一个下载器提交。"""
|
"""下载器异常不冻结后续执行,下一轮仍按普通下载合同重新提交。"""
|
||||||
repository, _factory = _repository(tmp_path)
|
chain = _download_chain()
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(return_value=("qb", "hash-cross-row", "Original", "accepted"))
|
|
||||||
|
|
||||||
first = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
|
|
||||||
)
|
|
||||||
second = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert first == second == "hash-cross-row"
|
|
||||||
chain.download.assert_called_once()
|
|
||||||
chain._settle_download_success.assert_called_once()
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_keeps_distinct_delivery_targets_separate(tmp_path) -> None:
|
|
||||||
"""不同保存目标属于独立产品意图,不得跨记录误去重。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain._resolve_media_download_dir.side_effect = (
|
|
||||||
lambda *, save_path, **_kwargs: ("local", Path(save_path), None)
|
|
||||||
)
|
|
||||||
chain.download = MagicMock(side_effect=[
|
chain.download = MagicMock(side_effect=[
|
||||||
("qb", "hash-a", "Original", "accepted"),
|
TimeoutError("response timeout"),
|
||||||
("qb", "hash-b", "Original", "accepted"),
|
("qb", "hash-after-timeout", "Original", "accepted"),
|
||||||
])
|
])
|
||||||
|
|
||||||
first = chain.download_single(
|
with pytest.raises(TimeoutError, match="response timeout"):
|
||||||
context=_context(),
|
chain.download_single(
|
||||||
torrent_content=b"torrent",
|
context=_context(),
|
||||||
episodes={1},
|
torrent_content=b"torrent",
|
||||||
save_path="/downloads/a",
|
episodes={1},
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
|
save_path="/downloads",
|
||||||
)
|
governance=SubscriptionDownloadGovernance(),
|
||||||
second = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads/b",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (first, second) == ("hash-a", "hash-b")
|
|
||||||
assert chain.download.call_count == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_keeps_distinct_downloaders_separate(tmp_path) -> None:
|
|
||||||
"""不同下载器属于独立交付策略,不得跨记录误去重。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(side_effect=[
|
|
||||||
("qb-a", "hash-a", "Original", "accepted"),
|
|
||||||
("qb-b", "hash-b", "Original", "accepted"),
|
|
||||||
])
|
|
||||||
|
|
||||||
first = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
downloader="qb-a",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
|
|
||||||
)
|
|
||||||
second = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
downloader="qb-b",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (first, second) == ("hash-a", "hash-b")
|
|
||||||
assert chain.download.call_count == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_keeps_distinct_episode_coverage_separate(tmp_path) -> None:
|
|
||||||
"""跨记录仅复用精确覆盖,新增目标集不得被已有子集吞掉。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(side_effect=[
|
|
||||||
("qb", "hash-e1", "Original", "accepted"),
|
|
||||||
("qb", "hash-e12", "Original", "accepted"),
|
|
||||||
])
|
|
||||||
|
|
||||||
first = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
|
|
||||||
)
|
|
||||||
second = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1, 2},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=8, mode="normal"),
|
|
||||||
)
|
|
||||||
|
|
||||||
assert (first, second) == ("hash-e1", "hash-e12")
|
|
||||||
assert chain.download.call_count == 2
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_reuses_pre_004_ledger_key(tmp_path) -> None:
|
|
||||||
"""键升级后仍应读取 003A 同记录成功账本,避免版本升级造成重复下载。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
|
|
||||||
request = chain._build_subscription_download_request(
|
|
||||||
context=_context(),
|
|
||||||
episodes={1},
|
|
||||||
governance=governance,
|
|
||||||
delivery_scope=chain._subscription_delivery_scope(
|
|
||||||
downloader=None,
|
|
||||||
download_uri="local:/downloads",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
legacy = repository.claim(
|
|
||||||
SubscriptionDownloadRequest(
|
|
||||||
idempotency_key=request.legacy_idempotency_key or "",
|
|
||||||
legacy_idempotency_key=None,
|
|
||||||
subscription_id=7,
|
|
||||||
task_id=None,
|
|
||||||
logical_identity='{"subscription_id":7}',
|
|
||||||
resource_key=request.resource_key,
|
|
||||||
coverage=request.coverage,
|
|
||||||
mode=request.mode,
|
|
||||||
delivery_scope="legacy",
|
|
||||||
)
|
)
|
||||||
)
|
|
||||||
token = legacy.snapshot.attempt_token or ""
|
|
||||||
assert repository.mark_accepted(
|
|
||||||
idempotency_key=legacy.snapshot.idempotency_key,
|
|
||||||
attempt_token=token,
|
|
||||||
downloader="qb",
|
|
||||||
download_hash="legacy-ledger-hash",
|
|
||||||
)
|
|
||||||
assert repository.mark_succeeded(
|
|
||||||
idempotency_key=legacy.snapshot.idempotency_key,
|
|
||||||
attempt_token=token,
|
|
||||||
)
|
|
||||||
chain.download = MagicMock()
|
|
||||||
|
|
||||||
result = chain.download_single(
|
result = chain.download_single(
|
||||||
context=_context(),
|
context=_context(),
|
||||||
torrent_content=b"torrent",
|
torrent_content=b"torrent",
|
||||||
episodes={1},
|
episodes={1},
|
||||||
save_path="/downloads",
|
save_path="/downloads",
|
||||||
governance=governance,
|
governance=SubscriptionDownloadGovernance(),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result == "legacy-ledger-hash"
|
assert result == "hash-after-timeout"
|
||||||
chain.download.assert_not_called()
|
assert chain.download.call_count == 2
|
||||||
|
chain._settle_download_success.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_freezes_when_local_settlement_fails(tmp_path) -> None:
|
def test_download_chain_retries_after_local_settlement_failure() -> None:
|
||||||
"""下载器接受而历史结算失败时进入待对账,后续入口不得盲重试。"""
|
"""本地结算异常直接失败,下一轮不依赖补偿状态即可重新执行。"""
|
||||||
repository, _factory = _repository(tmp_path)
|
chain = _download_chain()
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(return_value=("qb", "hash-2", "Original", "accepted"))
|
chain.download = MagicMock(return_value=("qb", "hash-2", "Original", "accepted"))
|
||||||
chain._settle_download_success.side_effect = RuntimeError("history unavailable")
|
chain._settle_download_success.side_effect = [RuntimeError("history unavailable"), None]
|
||||||
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
|
|
||||||
|
|
||||||
with pytest.raises(DownloadReconciliationRequired):
|
with pytest.raises(RuntimeError, match="history unavailable"):
|
||||||
chain.download_single(
|
chain.download_single(
|
||||||
context=_context(),
|
context=_context(),
|
||||||
torrent_content=b"torrent",
|
torrent_content=b"torrent",
|
||||||
episodes={1},
|
episodes={1},
|
||||||
save_path="/downloads",
|
save_path="/downloads",
|
||||||
governance=governance,
|
governance=SubscriptionDownloadGovernance(),
|
||||||
)
|
)
|
||||||
with pytest.raises(DownloadReconciliationRequired):
|
|
||||||
chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
|
|
||||||
chain.download.assert_called_once()
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_freezes_when_downloader_result_is_uncertain(tmp_path) -> None:
|
|
||||||
"""下载器调用抛错可能已产生副作用,重启后的新实例仍不得自动重试。"""
|
|
||||||
repository, _factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(side_effect=TimeoutError("response timeout"))
|
|
||||||
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
|
|
||||||
|
|
||||||
with pytest.raises(DownloadReconciliationRequired):
|
|
||||||
chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
|
|
||||||
restarted = _download_chain(repository)
|
|
||||||
restarted.download = MagicMock()
|
|
||||||
with pytest.raises(DownloadReconciliationRequired):
|
|
||||||
restarted.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
restarted.download.assert_not_called()
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_delays_retry_after_explicit_rejection(tmp_path) -> None:
|
|
||||||
"""下载器明确拒绝且无 hash 时进入冷却,不把不确定和已拒绝混为一谈。"""
|
|
||||||
repository, factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected"))
|
|
||||||
governance = SubscriptionDownloadGovernance(subscription_id=7, mode="normal")
|
|
||||||
|
|
||||||
first = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
second = chain.download_single(
|
|
||||||
context=_context(),
|
|
||||||
torrent_content=b"torrent",
|
|
||||||
episodes={1},
|
|
||||||
save_path="/downloads",
|
|
||||||
governance=governance,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert first is None and second is None
|
|
||||||
chain.download.assert_called_once()
|
|
||||||
with factory() as session:
|
|
||||||
record = session.query(SubscriptionDownloadSubmission).one()
|
|
||||||
assert record.state == "retryable"
|
|
||||||
assert record.available_at > record.updated_at
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_reads_legacy_history_before_new_ledger(tmp_path) -> None:
|
|
||||||
"""迁移前同订阅同 torrent 同覆盖的成功历史仍能阻止重复提交。"""
|
|
||||||
repository, factory = _repository(tmp_path)
|
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock()
|
|
||||||
chain.download_history_repository.get_by_media_identity.return_value = [
|
|
||||||
SimpleNamespace(
|
|
||||||
download_hash="legacy-hash",
|
|
||||||
torrent_name="Demo Show S01E01 1080p",
|
|
||||||
torrent_site="TestSite",
|
|
||||||
episodes="E01",
|
|
||||||
seasons="S01",
|
|
||||||
episode_group=None,
|
|
||||||
note={"source": 'Subscribe|{"id": 7}'},
|
|
||||||
)
|
|
||||||
]
|
|
||||||
|
|
||||||
result = chain.download_single(
|
result = chain.download_single(
|
||||||
context=_context(),
|
context=_context(),
|
||||||
torrent_content=b"torrent",
|
torrent_content=b"torrent",
|
||||||
episodes={1},
|
episodes={1},
|
||||||
save_path="/downloads",
|
save_path="/downloads",
|
||||||
governance=SubscriptionDownloadGovernance(subscription_id=7, mode="normal"),
|
governance=SubscriptionDownloadGovernance(),
|
||||||
)
|
)
|
||||||
|
|
||||||
assert result == "legacy-hash"
|
assert result == "hash-2"
|
||||||
chain.download.assert_not_called()
|
assert chain.download.call_count == 2
|
||||||
with factory() as session:
|
assert chain._settle_download_success.call_count == 2
|
||||||
assert session.query(SubscriptionDownloadSubmission).count() == 0
|
|
||||||
|
|
||||||
|
|
||||||
def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None:
|
def test_download_chain_records_explicit_rejection_as_normal_failure() -> None:
|
||||||
|
"""下载器明确拒绝仍写入既有资源失败冷却,不建立提交恢复状态。"""
|
||||||
|
chain = _download_chain()
|
||||||
|
chain.download = MagicMock(return_value=("qb", None, "Original", "downloader rejected"))
|
||||||
|
|
||||||
|
result = chain.download_single(
|
||||||
|
context=_context(),
|
||||||
|
torrent_content=b"torrent",
|
||||||
|
episodes={1},
|
||||||
|
save_path="/downloads",
|
||||||
|
governance=SubscriptionDownloadGovernance(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
chain.download_failure_repository.record_failure.assert_called_once()
|
||||||
|
chain.post_message.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_chain_cancels_before_external_side_effect() -> None:
|
||||||
"""取消在下载器边界前生效时不得创建下载任务或成功事实。"""
|
"""取消在下载器边界前生效时不得创建下载任务或成功事实。"""
|
||||||
repository, _factory = _repository(tmp_path)
|
chain = _download_chain()
|
||||||
chain = _download_chain(repository)
|
|
||||||
chain.download = MagicMock()
|
chain.download = MagicMock()
|
||||||
governance = SubscriptionDownloadGovernance(
|
governance = SubscriptionDownloadGovernance(cancelled=lambda: True)
|
||||||
subscription_id=7,
|
|
||||||
mode="normal",
|
|
||||||
task_id="cancel-task",
|
|
||||||
cancelled=lambda: True,
|
|
||||||
)
|
|
||||||
|
|
||||||
result = chain.download_single(
|
result = chain.download_single(
|
||||||
context=_context(),
|
context=_context(),
|
||||||
@@ -508,7 +214,34 @@ def test_download_chain_cancels_before_external_side_effect(tmp_path) -> None:
|
|||||||
|
|
||||||
assert result is None
|
assert result is None
|
||||||
chain.download.assert_not_called()
|
chain.download.assert_not_called()
|
||||||
assert not repository.has_started_for_task("cancel-task")
|
chain._settle_download_success.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_chain_marks_side_effect_boundary_before_downloader_call() -> None:
|
||||||
|
"""下载器调用前必须标记副作用起点,使晚到取消按真实执行结果收口。"""
|
||||||
|
chain = _download_chain()
|
||||||
|
order: list[str] = []
|
||||||
|
|
||||||
|
def download(**_kwargs):
|
||||||
|
"""记录下载器调用顺序并返回成功结果。"""
|
||||||
|
order.append("download")
|
||||||
|
return "qb", "hash-3", "Original", "accepted"
|
||||||
|
|
||||||
|
chain.download = download
|
||||||
|
governance = SubscriptionDownloadGovernance(
|
||||||
|
mark_started=lambda: order.append("started"),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = chain.download_single(
|
||||||
|
context=_context(),
|
||||||
|
torrent_content=b"torrent",
|
||||||
|
episodes={1},
|
||||||
|
save_path="/downloads",
|
||||||
|
governance=governance,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == "hash-3"
|
||||||
|
assert order == ["started", "download"]
|
||||||
|
|
||||||
|
|
||||||
def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -> None:
|
def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -> None:
|
||||||
@@ -554,6 +287,18 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
|
|||||||
lambda _subscribe, contexts: contexts,
|
lambda _subscribe, contexts: contexts,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(subscribe_policy, "DownloadChain", _FakeDownloadChain)
|
monkeypatch.setattr(subscribe_policy, "DownloadChain", _FakeDownloadChain)
|
||||||
|
admission = SubscriptionExecutionAdmission()
|
||||||
|
lease = admission.try_acquire(
|
||||||
|
subscription_id=current.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
execution_context = SubscriptionExecutionContext(
|
||||||
|
lease=lease,
|
||||||
|
admission=admission,
|
||||||
|
task_id="search-task-7",
|
||||||
|
)
|
||||||
|
|
||||||
_downloads, lefts = chain._SubscribeChain__download_best_version_with_full_pack_first(
|
_downloads, lefts = chain._SubscribeChain__download_best_version_with_full_pack_first(
|
||||||
contexts=[context],
|
contexts=[context],
|
||||||
@@ -562,6 +307,7 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
|
|||||||
mediakey="themoviedb:77",
|
mediakey="themoviedb:77",
|
||||||
save_path="/old",
|
save_path="/old",
|
||||||
downloader="old",
|
downloader="old",
|
||||||
|
execution_context=execution_context,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert lefts is fresh_missing
|
assert lefts is fresh_missing
|
||||||
@@ -574,8 +320,10 @@ def test_subscription_policy_reloads_facts_and_threads_governance(monkeypatch) -
|
|||||||
assert captured["no_exists"] is fresh_missing
|
assert captured["no_exists"] is fresh_missing
|
||||||
assert captured["save_path"] == "/current"
|
assert captured["save_path"] == "/current"
|
||||||
assert captured["downloader"] == "current"
|
assert captured["downloader"] == "current"
|
||||||
assert captured["governance"].subscription_id == 7
|
assert captured["governance"].cancelled() is False
|
||||||
assert captured["governance"].mode == "normal"
|
captured["governance"].mark_started()
|
||||||
|
assert execution_context.download_started is True
|
||||||
|
assert admission.release(lease) is True
|
||||||
|
|
||||||
|
|
||||||
def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch) -> None:
|
def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch) -> None:
|
||||||
@@ -610,58 +358,50 @@ def test_subscription_policy_discards_candidates_after_filter_change(monkeypatch
|
|||||||
batch_download.assert_not_called()
|
batch_download.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
def test_subscription_download_migration_is_idempotent_and_reversible(tmp_path, monkeypatch) -> None:
|
def test_subscription_download_ledger_removal_migration_is_reversible(
|
||||||
"""3.0.21 迁移可重复升级,并只移除自身新增的兼容表。"""
|
|
||||||
engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}")
|
|
||||||
migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
|
|
||||||
with engine.begin() as connection:
|
|
||||||
operations = Operations(MigrationContext.configure(connection))
|
|
||||||
monkeypatch.setattr(migration, "op", operations)
|
|
||||||
migration.upgrade()
|
|
||||||
migration.upgrade()
|
|
||||||
inspector = sa.inspect(connection)
|
|
||||||
assert "subscriptiondownloadsubmission" in inspector.get_table_names()
|
|
||||||
indexes = {item["name"] for item in inspector.get_indexes("subscriptiondownloadsubmission")}
|
|
||||||
assert "ix_subscriptiondownloadsubmission_task_state" in indexes
|
|
||||||
migration.downgrade()
|
|
||||||
assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names()
|
|
||||||
|
|
||||||
|
|
||||||
def test_subscription_delivery_scope_migration_is_idempotent_and_reversible(
|
|
||||||
tmp_path,
|
tmp_path,
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""3.0.23 为存量提交填充兼容范围,并支持重复升级和完整回滚。"""
|
"""3.0.25 删除旧账本;降级只恢复结构,不伪造已删除的运行状态。"""
|
||||||
engine = create_engine(f"sqlite:///{tmp_path / 'delivery-migration.db'}")
|
engine = create_engine(f"sqlite:///{tmp_path / 'migration.db'}")
|
||||||
base_migration = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
|
create_ledger = importlib.import_module("database.versions.e1b6d4f8a2c7_3_0_21")
|
||||||
migration = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23")
|
add_delivery_scope = importlib.import_module("database.versions.a7d9e2c4f6b1_3_0_23")
|
||||||
|
remove_ledger = importlib.import_module("database.versions.c8f2e6a1d4b9_3_0_25")
|
||||||
|
|
||||||
with engine.begin() as connection:
|
with engine.begin() as connection:
|
||||||
operations = Operations(MigrationContext.configure(connection))
|
operations = Operations(MigrationContext.configure(connection))
|
||||||
monkeypatch.setattr(base_migration, "op", operations)
|
monkeypatch.setattr(create_ledger, "op", operations)
|
||||||
monkeypatch.setattr(migration, "op", operations)
|
monkeypatch.setattr(add_delivery_scope, "op", operations)
|
||||||
base_migration.upgrade()
|
monkeypatch.setattr(remove_ledger, "op", operations)
|
||||||
|
create_ledger.upgrade()
|
||||||
|
add_delivery_scope.upgrade()
|
||||||
connection.execute(sa.text(
|
connection.execute(sa.text(
|
||||||
"INSERT INTO subscriptiondownloadsubmission "
|
"INSERT INTO subscriptiondownloadsubmission "
|
||||||
"(idempotency_key, subscription_id, logical_identity, resource_key, coverage, mode, "
|
"(idempotency_key, subscription_id, logical_identity, resource_key, coverage, mode, "
|
||||||
"state, attempt_count, created_at, updated_at) VALUES "
|
"delivery_scope, state, attempt_count, created_at, updated_at) VALUES "
|
||||||
"('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'succeeded', 1, 'now', 'now')"
|
"('legacy-key', 7, '{}', 'resource', 'full', 'normal', 'legacy', "
|
||||||
|
"'reconcile_required', 1, 'now', 'now')"
|
||||||
))
|
))
|
||||||
|
|
||||||
migration.upgrade()
|
remove_ledger.upgrade()
|
||||||
migration.upgrade()
|
remove_ledger.upgrade()
|
||||||
|
assert "subscriptiondownloadsubmission" not in sa.inspect(connection).get_table_names()
|
||||||
|
|
||||||
assert connection.execute(sa.text(
|
remove_ledger.downgrade()
|
||||||
"SELECT delivery_scope FROM subscriptiondownloadsubmission WHERE idempotency_key='legacy-key'"
|
remove_ledger.downgrade()
|
||||||
)).scalar_one() == "legacy"
|
inspector = sa.inspect(connection)
|
||||||
migration.downgrade()
|
assert "subscriptiondownloadsubmission" in inspector.get_table_names()
|
||||||
columns = {
|
assert {column["name"] for column in inspector.get_columns(
|
||||||
column["name"]
|
"subscriptiondownloadsubmission"
|
||||||
for column in sa.inspect(connection).get_columns("subscriptiondownloadsubmission")
|
)} >= {"idempotency_key", "delivery_scope", "state", "download_hash"}
|
||||||
|
indexes = {
|
||||||
|
item["name"]
|
||||||
|
for item in inspector.get_indexes("subscriptiondownloadsubmission")
|
||||||
}
|
}
|
||||||
assert "delivery_scope" not in columns
|
assert indexes == {
|
||||||
|
"ix_subscriptiondownloadsubmission_subscription_state",
|
||||||
|
"ix_subscriptiondownloadsubmission_task_state",
|
||||||
def test_model_metadata_registers_submission_table() -> None:
|
}
|
||||||
"""显式模型注册必须让 fresh create_all 包含订阅提交账本。"""
|
assert connection.execute(sa.text(
|
||||||
assert SubscriptionDownloadSubmission.__tablename__ == "subscriptiondownloadsubmission"
|
"SELECT COUNT(*) FROM subscriptiondownloadsubmission"
|
||||||
assert "subscriptiondownloadsubmission" in Base.metadata.tables
|
)).scalar_one() == 0
|
||||||
|
|||||||
@@ -0,0 +1,138 @@
|
|||||||
|
"""订阅级进程内准入与显式执行上下文测试。"""
|
||||||
|
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
|
||||||
|
from app.application.subscription.execution import (
|
||||||
|
SubscriptionExecutionAdmission,
|
||||||
|
SubscriptionExecutionContext,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_subscription_is_mutually_exclusive_across_channels() -> None:
|
||||||
|
"""同一订阅只能由 Search 或 Match 中的一条路径持有。"""
|
||||||
|
admission = SubscriptionExecutionAdmission()
|
||||||
|
|
||||||
|
search = admission.try_acquire(
|
||||||
|
subscription_id=7,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
match = admission.try_acquire(
|
||||||
|
subscription_id=7,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert search is not None
|
||||||
|
assert match is None
|
||||||
|
assert admission.release(search) is True
|
||||||
|
assert admission.try_acquire(
|
||||||
|
subscription_id=7,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
) is not None
|
||||||
|
|
||||||
|
|
||||||
|
def test_concurrent_channels_admit_only_one_owner_for_same_subscription() -> None:
|
||||||
|
"""Search 与 Match 同时申请同一订阅时只能有一个取得 owner。"""
|
||||||
|
admission = SubscriptionExecutionAdmission()
|
||||||
|
|
||||||
|
def acquire(operation: str):
|
||||||
|
"""并发申请固定订阅的通道所有权。"""
|
||||||
|
return admission.try_acquire(
|
||||||
|
subscription_id=11,
|
||||||
|
operation=operation,
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||||
|
leases = list(executor.map(acquire, ("search", "match")))
|
||||||
|
|
||||||
|
assert sum(lease is not None for lease in leases) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_different_subscriptions_can_run_across_channels() -> None:
|
||||||
|
"""不同订阅可以分别占用 Search 与 Match 通道。"""
|
||||||
|
admission = SubscriptionExecutionAdmission()
|
||||||
|
|
||||||
|
search = admission.try_acquire(
|
||||||
|
subscription_id=7,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
match = admission.try_acquire(
|
||||||
|
subscription_id=8,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert search is not None
|
||||||
|
assert match is not None
|
||||||
|
assert search.owner_token != match.owner_token
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_owner_keeps_ownership_until_finally_release() -> None:
|
||||||
|
"""TTL 只请求 owner 协作退出,不允许后到任务强制接管活跃执行。"""
|
||||||
|
now = [100.0]
|
||||||
|
admission = SubscriptionExecutionAdmission(clock=lambda: now[0])
|
||||||
|
lease = admission.try_acquire(
|
||||||
|
subscription_id=9,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=5,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
|
||||||
|
now[0] = 106.0
|
||||||
|
|
||||||
|
assert admission.is_expired(lease) is True
|
||||||
|
assert admission.try_acquire(
|
||||||
|
subscription_id=9,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=5,
|
||||||
|
) is None
|
||||||
|
assert admission.release(lease) is True
|
||||||
|
|
||||||
|
replacement = admission.try_acquire(
|
||||||
|
subscription_id=9,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=5,
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert admission.release(lease) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_context_combines_cancel_ttl_and_download_boundary() -> None:
|
||||||
|
"""执行上下文独立承载取消、TTL、阶段与下载副作用状态。"""
|
||||||
|
now = [100.0]
|
||||||
|
cancelled = [False]
|
||||||
|
phases: list[tuple[str, int | None]] = []
|
||||||
|
admission = SubscriptionExecutionAdmission(clock=lambda: now[0])
|
||||||
|
lease = admission.try_acquire(
|
||||||
|
subscription_id=10,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=5,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
context = SubscriptionExecutionContext(
|
||||||
|
lease=lease,
|
||||||
|
admission=admission,
|
||||||
|
task_id="task-10",
|
||||||
|
cancel_requested=lambda: cancelled[0],
|
||||||
|
phase_changed=lambda phase, site_id: phases.append((phase, site_id)),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert context.should_stop() is False
|
||||||
|
context.report_phase("searching", 3)
|
||||||
|
context.mark_download_started()
|
||||||
|
|
||||||
|
assert context.download_started is True
|
||||||
|
assert phases == [("searching", 3), ("submitting", None)]
|
||||||
|
|
||||||
|
cancelled[0] = True
|
||||||
|
assert context.is_cancel_requested() is True
|
||||||
|
assert context.should_stop() is True
|
||||||
|
|
||||||
|
cancelled[0] = False
|
||||||
|
now[0] = 106.0
|
||||||
|
assert context.is_expired() is True
|
||||||
|
assert context.should_stop() is True
|
||||||
@@ -1,8 +1,8 @@
|
|||||||
"""订阅执行状态合并、批次权限和操作能力测试。"""
|
"""订阅执行状态合并、批次权限和操作能力测试。"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
from dataclasses import replace
|
||||||
|
|
||||||
from app.application.download.admission import SubscriptionDownloadSnapshot
|
|
||||||
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
from app.application.subscription.execution import SearchBatchSnapshot, SearchTaskSnapshot
|
||||||
from app.application.subscription.status import SubscriptionExecutionStatusService
|
from app.application.subscription.status import SubscriptionExecutionStatusService
|
||||||
|
|
||||||
@@ -35,31 +35,12 @@ def _task(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _download(subscription_id: int, state: str) -> SubscriptionDownloadSnapshot:
|
|
||||||
"""构造一个比搜索任务更早的下载提交快照。"""
|
|
||||||
return SubscriptionDownloadSnapshot(
|
|
||||||
idempotency_key=f"key-{subscription_id}",
|
|
||||||
subscription_id=subscription_id,
|
|
||||||
task_id=f"task-{subscription_id}",
|
|
||||||
state=state,
|
|
||||||
attempt_count=1,
|
|
||||||
attempt_token="attempt",
|
|
||||||
downloader=None,
|
|
||||||
download_hash=None,
|
|
||||||
available_at=None,
|
|
||||||
last_error="downloader response uncertain" if state == "reconcile_required" else None,
|
|
||||||
created_at="2026-09-01T00:30:00+00:00",
|
|
||||||
updated_at="2026-09-01T00:59:00+00:00",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class _Repository:
|
class _Repository:
|
||||||
"""保存测试快照的异步状态仓储。"""
|
"""保存测试快照的异步状态仓储。"""
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
"""初始化可由测试覆盖的快照集合。"""
|
"""初始化可由测试覆盖的快照集合。"""
|
||||||
self.tasks: dict[int, SearchTaskSnapshot] = {}
|
self.tasks: dict[int, SearchTaskSnapshot] = {}
|
||||||
self.downloads: dict[int, SubscriptionDownloadSnapshot] = {}
|
|
||||||
self.batch = SearchBatchSnapshot(
|
self.batch = SearchBatchSnapshot(
|
||||||
batch_id="batch-1",
|
batch_id="batch-1",
|
||||||
source="manual",
|
source="manual",
|
||||||
@@ -78,10 +59,6 @@ class _Repository:
|
|||||||
"""返回请求范围内搜索任务。"""
|
"""返回请求范围内搜索任务。"""
|
||||||
return {key: value for key, value in self.tasks.items() if key in subscription_ids}
|
return {key: value for key, value in self.tasks.items() if key in subscription_ids}
|
||||||
|
|
||||||
async def latest_download_submissions(self, subscription_ids):
|
|
||||||
"""返回请求范围内下载提交。"""
|
|
||||||
return {key: value for key, value in self.downloads.items() if key in subscription_ids}
|
|
||||||
|
|
||||||
async def list_batches(self, *, limit):
|
async def list_batches(self, *, limit):
|
||||||
"""返回一个测试批次。"""
|
"""返回一个测试批次。"""
|
||||||
return [self.batch][:limit]
|
return [self.batch][:limit]
|
||||||
@@ -107,29 +84,14 @@ def test_execution_status_exposes_site_wait_and_cancel_capability():
|
|||||||
assert statuses[1].can_cancel is True
|
assert statuses[1].can_cancel is True
|
||||||
|
|
||||||
|
|
||||||
def test_reconciliation_state_overrides_newer_search_terminal():
|
def test_failed_search_exposes_safe_error():
|
||||||
"""不确定下载副作用不得被稍晚写入的搜索失败掩盖。"""
|
"""搜索失败文本必须压平且不暴露内部错误细节。"""
|
||||||
repository = _Repository()
|
|
||||||
repository.tasks[2] = _task(2, state="failed", phase="failed")
|
|
||||||
repository.downloads[2] = _download(2, "reconcile_required")
|
|
||||||
|
|
||||||
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,)))
|
|
||||||
|
|
||||||
assert statuses[2].state == "reconcile_required"
|
|
||||||
assert statuses[2].requires_reconciliation is True
|
|
||||||
assert statuses[2].can_retry is False
|
|
||||||
assert statuses[2].error == "downloader response uncertain"
|
|
||||||
|
|
||||||
|
|
||||||
def test_failed_search_exposes_safe_retryable_error():
|
|
||||||
"""搜索失败文本必须压平且仅声明安全重试能力。"""
|
|
||||||
repository = _Repository()
|
repository = _Repository()
|
||||||
repository.tasks[3] = _task(3, state="failed", phase="failed")
|
repository.tasks[3] = _task(3, state="failed", phase="failed")
|
||||||
|
|
||||||
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((3,)))
|
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((3,)))
|
||||||
|
|
||||||
assert statuses[3].state == "failed"
|
assert statuses[3].state == "failed"
|
||||||
assert statuses[3].can_retry is True
|
|
||||||
assert statuses[3].error == "provider timeout"
|
assert statuses[3].error == "provider timeout"
|
||||||
|
|
||||||
|
|
||||||
@@ -149,6 +111,35 @@ def test_batch_requires_complete_subscription_access():
|
|||||||
assert visible.can_cancel is True
|
assert visible.can_cancel is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_batch_projection_exposes_skipped_count_as_processed_without_success():
|
||||||
|
"""批次跳过应计入处理总数,同时保留独立的完成计数。"""
|
||||||
|
repository = _Repository()
|
||||||
|
repository.batch = replace(
|
||||||
|
repository.batch,
|
||||||
|
state="skipped",
|
||||||
|
total_count=2,
|
||||||
|
finished_count=1,
|
||||||
|
skipped_count=1,
|
||||||
|
)
|
||||||
|
repository.tasks = {
|
||||||
|
1: _task(1, state="completed", phase="completed"),
|
||||||
|
2: _task(2, state="skipped", phase="skipped"),
|
||||||
|
}
|
||||||
|
|
||||||
|
visible = asyncio.run(
|
||||||
|
SubscriptionExecutionStatusService(repository).get_batch(
|
||||||
|
"batch-1",
|
||||||
|
accessible_subscription_ids={1, 2},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert visible is not None
|
||||||
|
assert visible.state == "skipped"
|
||||||
|
assert visible.finished_count == 1
|
||||||
|
assert visible.skipped_count == 1
|
||||||
|
assert visible.processed_count == 2
|
||||||
|
|
||||||
|
|
||||||
def test_request_cancel_uses_injected_execution_boundary():
|
def test_request_cancel_uses_injected_execution_boundary():
|
||||||
"""取消必须通过组合根注入的异步执行边界并返回真实结果。"""
|
"""取消必须通过组合根注入的异步执行边界并返回真实结果。"""
|
||||||
repository = _Repository()
|
repository = _Repository()
|
||||||
|
|||||||
@@ -60,8 +60,6 @@ def test_cooled_site_does_not_block_independent_site_or_hide_batch_failure():
|
|||||||
owner="fallback-task",
|
owner="fallback-task",
|
||||||
cancelled=lambda: False,
|
cancelled=lambda: False,
|
||||||
stop_state=ProcessStopState(),
|
stop_state=ProcessStopState(),
|
||||||
max_wait_seconds=0,
|
|
||||||
random_uniform=lambda _low, _high: 60.0,
|
|
||||||
)
|
)
|
||||||
chain = object.__new__(SearchChain)
|
chain = object.__new__(SearchChain)
|
||||||
chain._runtime_config = SimpleNamespace(search_threadpool_size=2)
|
chain._runtime_config = SimpleNamespace(search_threadpool_size=2)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from types import SimpleNamespace
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
from app.chain.subscribe import SubscribeChain
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
from app.domain.context import Context, MediaInfo, TorrentInfo
|
from app.domain.context import Context, MediaInfo, TorrentInfo
|
||||||
from app.domain.metainfo import MetaInfo
|
from app.domain.metainfo import MetaInfo
|
||||||
from app.schemas.mediaserver import NotExistMediaInfo
|
from app.schemas.mediaserver import NotExistMediaInfo
|
||||||
@@ -42,11 +42,16 @@ class _ReplaySubscriptionListRepository:
|
|||||||
|
|
||||||
def __init__(self, subscribes: list[SubscriptionSnapshot]) -> None:
|
def __init__(self, subscribes: list[SubscriptionSnapshot]) -> None:
|
||||||
self.subscribes = subscribes
|
self.subscribes = subscribes
|
||||||
|
self._by_id = {subscribe.id: subscribe for subscribe in subscribes}
|
||||||
|
|
||||||
def list(self, _state: str = None) -> list[SubscriptionSnapshot]:
|
def list(self, _state: str = None) -> list[SubscriptionSnapshot]:
|
||||||
"""返回当前批次的全部订阅快照。"""
|
"""返回当前批次的全部订阅快照。"""
|
||||||
return self.subscribes
|
return self.subscribes
|
||||||
|
|
||||||
|
def get(self, subscribe_id: int) -> SubscriptionSnapshot | None:
|
||||||
|
"""返回取得订阅准入后的最新快照。"""
|
||||||
|
return self._by_id.get(subscribe_id)
|
||||||
|
|
||||||
|
|
||||||
class _ReplayTorrentHelper:
|
class _ReplayTorrentHelper:
|
||||||
"""让无关候选稳定停在身份冲突边界。"""
|
"""让无关候选稳定停在身份冲突边界。"""
|
||||||
|
|||||||
@@ -1,14 +1,298 @@
|
|||||||
"""订阅执行治理最终受控规模门禁。"""
|
"""订阅执行治理最终受控规模门禁。"""
|
||||||
|
|
||||||
from scripts.validation.subscription_governance_scale import run_acceptance
|
import threading
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.application.site.observation import capture_site_search_observation
|
||||||
|
from app.application.subscription.candidates import CandidateIndex
|
||||||
|
from app.application.subscription.sitebudget import SiteBudgetClaim
|
||||||
|
from app.chain.search.facade import SearchChain
|
||||||
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
|
from scripts.validation import subscription_governance_scale as scale
|
||||||
|
from scripts.validation.subscription_governance_scale import (
|
||||||
|
ScaleCase,
|
||||||
|
_run_durable_governance,
|
||||||
|
_run_match_execution_case,
|
||||||
|
run_acceptance,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_subscription_governance_controlled_scale_matrix() -> None:
|
def test_subscription_governance_controlled_scale_matrix() -> None:
|
||||||
"""两档最终矩阵必须同时满足正确性、压力、恢复和幂等门禁。"""
|
"""两档最终矩阵必须同时满足正确性、压力、恢复和订阅准入门禁。"""
|
||||||
result = run_acceptance()
|
result = run_acceptance()
|
||||||
|
|
||||||
|
assert result["schema_version"] == 4
|
||||||
assert result["passed"] is True
|
assert result["passed"] is True
|
||||||
assert all(result["gates"].values())
|
assert all(result["gates"].values())
|
||||||
|
assert result["method"]["match_entrypoint"] == "SubscribeChain.match"
|
||||||
|
assert result["method"]["download_selection"] == "DownloadChain.batch_download"
|
||||||
|
assert result["method"]["fixed_external_boundaries"] == [
|
||||||
|
"TMDB",
|
||||||
|
"media_server",
|
||||||
|
"downloader",
|
||||||
|
]
|
||||||
|
assert result["method"]["fixed_persistence_boundaries"] == [
|
||||||
|
"subscription_repository",
|
||||||
|
"download_facts",
|
||||||
|
"subscription_progress",
|
||||||
|
"completion_side_effects",
|
||||||
|
]
|
||||||
|
assert result["method"]["fixed_policy_inputs"] == [
|
||||||
|
"site_mapping",
|
||||||
|
"system_configuration",
|
||||||
|
"subscription_filters",
|
||||||
|
"torrent_attribute_filter",
|
||||||
|
"download_preparation",
|
||||||
|
]
|
||||||
|
assert result["method"]["production_network_slo"] is False
|
||||||
|
assert result["gates"]["subscription_admission_serializes"] is True
|
||||||
|
assert result["gates"]["match_execution_candidate_sets_equal"] is True
|
||||||
|
assert result["gates"]["match_execution_download_sets_equal"] is True
|
||||||
|
assert result["gates"]["match_execution_missing_sets_equal"] is True
|
||||||
|
assert result["gates"]["match_execution_completion_sets_equal"] is True
|
||||||
assert [case["subscription_count"] for case in result["match_cases"]] == [100, 200]
|
assert [case["subscription_count"] for case in result["match_cases"]] == [100, 200]
|
||||||
assert [case["site_count"] for case in result["match_cases"]] == [10, 20]
|
assert [case["site_count"] for case in result["match_cases"]] == [10, 20]
|
||||||
assert min(case["candidate_count"] for case in result["match_cases"]) >= 1000
|
assert min(case["candidate_count"] for case in result["match_cases"]) >= 1000
|
||||||
|
assert [
|
||||||
|
case["candidate_check_reduction_percent"]
|
||||||
|
for case in result["match_cases"]
|
||||||
|
] == [99.0, 99.0]
|
||||||
|
assert [
|
||||||
|
case["matched_candidates"]["actual_count"]
|
||||||
|
for case in result["match_execution_cases"]
|
||||||
|
] == [1000, 2400]
|
||||||
|
assert [
|
||||||
|
case["downloaded_candidates"]["actual_count"]
|
||||||
|
for case in result["match_execution_cases"]
|
||||||
|
] == [100, 200]
|
||||||
|
assert all(
|
||||||
|
case["downloaded_candidates"]["duplicate_count"] == 0
|
||||||
|
and case["completed_subscriptions"]["duplicate_count"] == 0
|
||||||
|
for case in result["match_execution_cases"]
|
||||||
|
)
|
||||||
|
assert [
|
||||||
|
case["site_peak_inflight_per_site"]
|
||||||
|
for case in result["durable_cases"]
|
||||||
|
] == [1, 1]
|
||||||
|
assert all(
|
||||||
|
case["site_request_boundary_active"] == 0
|
||||||
|
and case["site_request_boundary_peak"] == 1
|
||||||
|
and case["site_pressure_owner_count"] == 2
|
||||||
|
and case["site_pressure_concurrency_verified"]
|
||||||
|
and case["site_pressure_success_release_reused"]
|
||||||
|
and case["site_pressure_error_observation_cooled"]
|
||||||
|
and case["site_pressure_valid"]
|
||||||
|
for case in result["durable_cases"]
|
||||||
|
)
|
||||||
|
assert all(
|
||||||
|
observation["request_active"] == 0
|
||||||
|
and observation["request_peak"] == 1
|
||||||
|
and observation["owners_finished"] is True
|
||||||
|
and observation["request_active_at_rejection"] == 1
|
||||||
|
and observation["budget_rejections"] == 1
|
||||||
|
for case in result["durable_cases"]
|
||||||
|
for observation in case["site_observations"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("mutation", ["bypass", "early_release", "allow_concurrent"])
|
||||||
|
def test_scale_validator_rejects_fake_site_pressure(monkeypatch, tmp_path, mutation):
|
||||||
|
"""绕过 wrapper、提前释放或并发放行时,请求边界峰值必须暴露门禁失效。"""
|
||||||
|
if mutation == "bypass":
|
||||||
|
def bypass_wrapper(self, *, site, keyword, mtype, page):
|
||||||
|
"""模拟绕过预算 wrapper 的请求路径。"""
|
||||||
|
return self.search_site_torrents(
|
||||||
|
site=site,
|
||||||
|
keyword=keyword,
|
||||||
|
mtype=mtype,
|
||||||
|
page=page,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SearchChain,
|
||||||
|
"_search_site_torrents_with_budget",
|
||||||
|
bypass_wrapper,
|
||||||
|
)
|
||||||
|
elif mutation == "early_release":
|
||||||
|
release_guard = threading.Lock()
|
||||||
|
|
||||||
|
def early_release_wrapper(self, *, site, keyword, mtype, page):
|
||||||
|
"""模拟请求边界前错误释放站点租约。"""
|
||||||
|
budget = self._subscription_site_budget
|
||||||
|
# 将错误释放本身串行化,避免第二个 owner 在首个释放动作完成前
|
||||||
|
# 被正常拒绝,从而掩盖“请求期间已失去租约”的变异。
|
||||||
|
with release_guard:
|
||||||
|
claim = budget.acquire(site["id"])
|
||||||
|
with capture_site_search_observation() as observation:
|
||||||
|
budget.finish(claim, observation)
|
||||||
|
return self.search_site_torrents(
|
||||||
|
site=site,
|
||||||
|
keyword=keyword,
|
||||||
|
mtype=mtype,
|
||||||
|
page=page,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SearchChain,
|
||||||
|
"_search_site_torrents_with_budget",
|
||||||
|
early_release_wrapper,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
original_claim = scale.TransactionalSubscriptionSearchRepository.claim_site
|
||||||
|
|
||||||
|
def allow_concurrent_claim(self, *, site_id, owner, lease_seconds):
|
||||||
|
"""模拟仓储错误地向第二 owner 发放同站点租约。"""
|
||||||
|
claim = original_claim(
|
||||||
|
self,
|
||||||
|
site_id=site_id,
|
||||||
|
owner=owner,
|
||||||
|
lease_seconds=lease_seconds,
|
||||||
|
)
|
||||||
|
if claim.acquired:
|
||||||
|
return claim
|
||||||
|
return SiteBudgetClaim(
|
||||||
|
site_id=site_id,
|
||||||
|
acquired=True,
|
||||||
|
retry_at=claim.retry_at,
|
||||||
|
consecutive_failures=claim.consecutive_failures,
|
||||||
|
lease_token=f"mutant-{owner}",
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scale.TransactionalSubscriptionSearchRepository,
|
||||||
|
"claim_site",
|
||||||
|
allow_concurrent_claim,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = _run_durable_governance(
|
||||||
|
ScaleCase(f"mutant-{mutation}", 2, 1, 2, 1),
|
||||||
|
tmp_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["site_pressure_valid"] is False
|
||||||
|
assert result["site_peak_inflight_per_site"] >= 2
|
||||||
|
assert result["site_request_boundary_peak"] >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path):
|
||||||
|
"""失败已登记但 wrapper 尚未返回时,压力门禁必须拒绝并暴露未收口 owner。"""
|
||||||
|
original_wrapper = SearchChain._search_site_torrents_with_budget
|
||||||
|
release_stalled = threading.Event()
|
||||||
|
threads_before = set(threading.enumerate())
|
||||||
|
|
||||||
|
def stall_after_rejection(self, *, site, keyword, mtype, page):
|
||||||
|
"""模拟预算拒绝已记录、调用方却未取得返回值的挂起路径。"""
|
||||||
|
original_record_failure = self.record_subscription_site_budget_failure
|
||||||
|
|
||||||
|
def record_failure_and_stall(error: str) -> None:
|
||||||
|
"""在拒绝已登记后阻塞原始 wrapper 的返回。"""
|
||||||
|
original_record_failure(error)
|
||||||
|
if "冷却或已有在途搜索" in error:
|
||||||
|
release_stalled.wait()
|
||||||
|
|
||||||
|
self.record_subscription_site_budget_failure = record_failure_and_stall
|
||||||
|
try:
|
||||||
|
return original_wrapper(
|
||||||
|
self,
|
||||||
|
site=site,
|
||||||
|
keyword=keyword,
|
||||||
|
mtype=mtype,
|
||||||
|
page=page,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
self.record_subscription_site_budget_failure = original_record_failure
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SearchChain,
|
||||||
|
"_search_site_torrents_with_budget",
|
||||||
|
stall_after_rejection,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(scale, "_SITE_PRESSURE_SYNC_TIMEOUT", 0.05)
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = _run_durable_governance(
|
||||||
|
ScaleCase("mutant-unfinished", 2, 1, 2, 1),
|
||||||
|
tmp_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["site_pressure_valid"] is False
|
||||||
|
assert result["site_pressure_concurrency_verified"] is False
|
||||||
|
assert result["site_observations"][0]["owners_finished"] is False
|
||||||
|
finally:
|
||||||
|
release_stalled.set()
|
||||||
|
for thread in set(threading.enumerate()) - threads_before:
|
||||||
|
thread.join(timeout=1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scale_validator_rejects_candidate_loss(monkeypatch) -> None:
|
||||||
|
"""候选路由少返回任意资源时,完整匹配集合门禁必须失败。"""
|
||||||
|
original_route = CandidateIndex.route_for_match
|
||||||
|
|
||||||
|
def route_with_loss(self, *args, **kwargs):
|
||||||
|
routed = original_route(self, *args, **kwargs)
|
||||||
|
for domain, contexts in routed.items():
|
||||||
|
if contexts:
|
||||||
|
return {**routed, domain: contexts[1:]}
|
||||||
|
return routed
|
||||||
|
|
||||||
|
monkeypatch.setattr(CandidateIndex, "route_for_match", route_with_loss)
|
||||||
|
|
||||||
|
result = _run_match_execution_case(ScaleCase("candidate-loss", 4, 2, 8, 4))
|
||||||
|
|
||||||
|
assert result["matched_candidates"]["equal"] is False
|
||||||
|
assert result["matched_candidates"]["missing_sample"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_scale_validator_rejects_duplicate_download_and_completion(monkeypatch) -> None:
|
||||||
|
"""同订阅重复下载或错误完成时,多重集合门禁必须失败。"""
|
||||||
|
from scripts.validation import subscription_governance_scale as scale
|
||||||
|
|
||||||
|
original_download = scale._ScaleDownloadBoundary.download_single
|
||||||
|
|
||||||
|
def duplicate_download(self, context, *, governance, **kwargs):
|
||||||
|
result = original_download(
|
||||||
|
self,
|
||||||
|
context,
|
||||||
|
governance=governance,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
original_download(
|
||||||
|
self,
|
||||||
|
context,
|
||||||
|
governance=governance,
|
||||||
|
**kwargs,
|
||||||
|
)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def finish_every_subscription(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
subscribe,
|
||||||
|
meta,
|
||||||
|
mediainfo,
|
||||||
|
**_kwargs,
|
||||||
|
):
|
||||||
|
self._SubscribeChain__finish_subscribe(
|
||||||
|
subscribe=subscribe,
|
||||||
|
meta=meta,
|
||||||
|
mediainfo=mediainfo,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
scale._ScaleDownloadBoundary,
|
||||||
|
"download_single",
|
||||||
|
duplicate_download,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SubscribeChain,
|
||||||
|
"finish_subscribe_or_not",
|
||||||
|
finish_every_subscription,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = _run_match_execution_case(ScaleCase("duplicate-effects", 4, 2, 8, 4))
|
||||||
|
|
||||||
|
assert result["downloaded_candidates"]["equal"] is False
|
||||||
|
assert result["downloaded_candidates"]["duplicate_count"] == 4
|
||||||
|
assert result["completed_subscriptions"]["equal"] is False
|
||||||
|
assert result["completed_subscriptions"]["unexpected_sample"]
|
||||||
|
|||||||
@@ -0,0 +1,231 @@
|
|||||||
|
"""订阅 Match 通道的逐订阅准入与异常隔离测试。"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
from dataclasses import replace
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
|
from app.chain.subscribe import match as subscribe_match
|
||||||
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
|
from app.schemas.types import MediaType
|
||||||
|
|
||||||
|
|
||||||
|
def _subscribe(subscribe_id: int, *, name: str | None = None) -> SubscriptionSnapshot:
|
||||||
|
"""构造 Match 编排所需的活动电影订阅快照。"""
|
||||||
|
return SubscriptionSnapshot(
|
||||||
|
id=subscribe_id,
|
||||||
|
name=name or f"匹配电影 {subscribe_id}",
|
||||||
|
year="2026",
|
||||||
|
type=MediaType.MOVIE.value,
|
||||||
|
media_source="themoviedb",
|
||||||
|
media_id=str(2000 + subscribe_id),
|
||||||
|
state="R",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _chain(
|
||||||
|
listed: list[SubscriptionSnapshot],
|
||||||
|
current: dict[int, SubscriptionSnapshot] | None = None,
|
||||||
|
) -> SubscribeChain:
|
||||||
|
"""构造只执行 Match 编排层的订阅链。"""
|
||||||
|
snapshots = current or {subscribe.id: subscribe for subscribe in listed}
|
||||||
|
chain = object.__new__(SubscribeChain)
|
||||||
|
chain.subscription_repository = SimpleNamespace(
|
||||||
|
list=lambda _state: list(listed),
|
||||||
|
get=snapshots.get,
|
||||||
|
)
|
||||||
|
chain.get_states_for_search = lambda state: state
|
||||||
|
chain._match_lock = threading.Lock()
|
||||||
|
chain._search_queue_lock = threading.Lock()
|
||||||
|
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
|
||||||
|
chain._prepare_match_torrents = lambda torrents: torrents
|
||||||
|
return chain
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_channel_and_subscription_released(
|
||||||
|
chain: SubscribeChain,
|
||||||
|
subscription_ids: tuple[int, ...],
|
||||||
|
) -> None:
|
||||||
|
"""验证 Match 通道锁和所有订阅 owner 均已释放。"""
|
||||||
|
assert chain._match_lock.acquire(blocking=False) is True
|
||||||
|
chain._match_lock.release()
|
||||||
|
for subscription_id in subscription_ids:
|
||||||
|
lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscription_id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
assert chain._subscription_execution_admission.release(lease) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_skips_subscription_owned_by_search() -> None:
|
||||||
|
"""Search 已持有同一订阅时 Match 本轮直接跳过。"""
|
||||||
|
subscribe = _subscribe(1)
|
||||||
|
chain = _chain([subscribe])
|
||||||
|
process = Mock(side_effect=AssertionError("冲突订阅不应进入 Match"))
|
||||||
|
chain._match_subscription = process
|
||||||
|
search_lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert search_lease is not None
|
||||||
|
|
||||||
|
chain.match({"example.org": []})
|
||||||
|
|
||||||
|
process.assert_not_called()
|
||||||
|
assert chain._match_lock.acquire(blocking=False) is True
|
||||||
|
chain._match_lock.release()
|
||||||
|
assert chain._subscription_execution_admission.release(search_lease) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_reloads_subscription_after_admission() -> None:
|
||||||
|
"""Match 取得 owner 后使用最新订阅快照并传入独立执行上下文。"""
|
||||||
|
listed = _subscribe(2, name="旧快照")
|
||||||
|
current = replace(listed, name="最新快照")
|
||||||
|
chain = _chain([listed], {listed.id: current})
|
||||||
|
captured = []
|
||||||
|
|
||||||
|
def process(**kwargs):
|
||||||
|
"""记录 Match 单订阅执行入口收到的快照与上下文。"""
|
||||||
|
captured.append(kwargs)
|
||||||
|
|
||||||
|
chain._match_subscription = process
|
||||||
|
|
||||||
|
chain.match({"example.org": []})
|
||||||
|
|
||||||
|
assert len(captured) == 1
|
||||||
|
assert captured[0]["subscribe"] is current
|
||||||
|
context = captured[0]["execution_context"]
|
||||||
|
assert context.lease.subscription_id == listed.id
|
||||||
|
assert context.lease.operation == "match"
|
||||||
|
_assert_channel_and_subscription_released(chain, (listed.id,))
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_isolates_one_subscription_failure_and_releases_all_owners() -> None:
|
||||||
|
"""单条 Match 异常不阻止后续订阅,且所有 owner 最终释放。"""
|
||||||
|
first = _subscribe(3)
|
||||||
|
second = _subscribe(4)
|
||||||
|
chain = _chain([first, second])
|
||||||
|
processed = []
|
||||||
|
|
||||||
|
def process(**kwargs):
|
||||||
|
"""让首条订阅失败并记录后续订阅仍被执行。"""
|
||||||
|
subscribe = kwargs["subscribe"]
|
||||||
|
processed.append(subscribe.id)
|
||||||
|
if subscribe.id == first.id:
|
||||||
|
raise RuntimeError("candidate failure")
|
||||||
|
|
||||||
|
chain._match_subscription = process
|
||||||
|
|
||||||
|
chain.match({"example.org": []})
|
||||||
|
|
||||||
|
assert processed == [first.id, second.id]
|
||||||
|
_assert_channel_and_subscription_released(chain, (first.id, second.id))
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_continues_when_latest_subscription_read_fails() -> None:
|
||||||
|
"""单条最新快照读取失败不能泄漏 owner 或中止后续 Match。"""
|
||||||
|
first = _subscribe(5)
|
||||||
|
second = _subscribe(6)
|
||||||
|
chain = _chain([first, second])
|
||||||
|
processed = []
|
||||||
|
|
||||||
|
def get(subscription_id: int):
|
||||||
|
"""模拟首条读取异常并返回第二条当前快照。"""
|
||||||
|
if subscription_id == first.id:
|
||||||
|
raise RuntimeError("repository failure")
|
||||||
|
return second
|
||||||
|
|
||||||
|
chain.subscription_repository.get = get
|
||||||
|
chain._match_subscription = lambda **kwargs: processed.append(kwargs["subscribe"].id)
|
||||||
|
|
||||||
|
chain.match({"example.org": []})
|
||||||
|
|
||||||
|
assert processed == [second.id]
|
||||||
|
_assert_channel_and_subscription_released(chain, (first.id, second.id))
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_skips_subscription_paused_after_admission() -> None:
|
||||||
|
"""取得准入后订阅变为暂停态时,不应继续访问候选或下载边界。"""
|
||||||
|
listed = _subscribe(7)
|
||||||
|
paused = replace(listed, state="S")
|
||||||
|
chain = _chain([listed], {listed.id: paused})
|
||||||
|
process = Mock(side_effect=AssertionError("暂停订阅不应进入 Match"))
|
||||||
|
chain._match_subscription = process
|
||||||
|
progress = Mock()
|
||||||
|
|
||||||
|
chain.match({"example.org": []}, progress_callback=progress)
|
||||||
|
|
||||||
|
process.assert_not_called()
|
||||||
|
final_data = progress.call_args.kwargs["data"]
|
||||||
|
assert final_data == {
|
||||||
|
"total": 1,
|
||||||
|
"finished": 1,
|
||||||
|
"completed": 0,
|
||||||
|
"skipped": 1,
|
||||||
|
"failed": 0,
|
||||||
|
}
|
||||||
|
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅跳过"
|
||||||
|
_assert_channel_and_subscription_released(chain, (listed.id,))
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_progress_reports_completed_skipped_and_failed_counts() -> None:
|
||||||
|
"""Match 最终进度必须区分正常执行、准入跳过和单订阅失败。"""
|
||||||
|
completed = _subscribe(8)
|
||||||
|
skipped = _subscribe(9)
|
||||||
|
failed = _subscribe(10)
|
||||||
|
chain = _chain([completed, skipped, failed])
|
||||||
|
chain._match_subscription = Mock(
|
||||||
|
side_effect=["completed", RuntimeError("candidate failure")]
|
||||||
|
)
|
||||||
|
skipped_lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=skipped.id,
|
||||||
|
operation="search",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert skipped_lease is not None
|
||||||
|
progress = Mock()
|
||||||
|
|
||||||
|
chain.match({"example.org": []}, progress_callback=progress)
|
||||||
|
|
||||||
|
final_data = progress.call_args.kwargs["data"]
|
||||||
|
assert final_data == {
|
||||||
|
"total": 3,
|
||||||
|
"finished": 3,
|
||||||
|
"completed": 1,
|
||||||
|
"skipped": 1,
|
||||||
|
"failed": 1,
|
||||||
|
}
|
||||||
|
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅失败"
|
||||||
|
assert chain._subscription_execution_admission.release(skipped_lease) is True
|
||||||
|
_assert_channel_and_subscription_released(chain, (completed.id, failed.id))
|
||||||
|
|
||||||
|
|
||||||
|
def test_match_stop_does_not_report_unvisited_subscriptions_as_completed(monkeypatch) -> None:
|
||||||
|
"""系统停止后未访问的订阅不得计入完成或跳过。"""
|
||||||
|
subscribes = [_subscribe(11), _subscribe(12)]
|
||||||
|
chain = _chain(subscribes)
|
||||||
|
progress = Mock()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
subscribe_match,
|
||||||
|
"runtime_stop_state",
|
||||||
|
SimpleNamespace(is_system_stopped=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
chain.match({"example.org": []}, progress_callback=progress)
|
||||||
|
|
||||||
|
assert progress.call_args.kwargs == {
|
||||||
|
"value": 100,
|
||||||
|
"text": "订阅资源匹配已停止,部分订阅未执行",
|
||||||
|
"data": {
|
||||||
|
"total": 2,
|
||||||
|
"finished": 0,
|
||||||
|
"completed": 0,
|
||||||
|
"skipped": 0,
|
||||||
|
"failed": 0,
|
||||||
|
},
|
||||||
|
}
|
||||||
@@ -1,17 +1,24 @@
|
|||||||
"""订阅搜索队列接入、锁隔离和批次失败治理测试。"""
|
"""订阅搜索队列接入、锁隔离和批次失败治理测试。"""
|
||||||
|
|
||||||
|
import threading
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from datetime import datetime, timedelta
|
from datetime import datetime, timedelta, timezone
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.orm import sessionmaker
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from app.application.site.observation import report_site_search_outcome
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
|
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
|
||||||
|
from app.chain.search.facade import SearchChain
|
||||||
from app.chain.subscribe.facade import SubscribeChain
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
|
from app.chain.subscribe.search import _search_task_available_at
|
||||||
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
||||||
from app.db.base import Base
|
from app.db.base import Base
|
||||||
|
from app.modules.indexer import IndexerModule
|
||||||
from app.schemas.types import MediaType
|
from app.schemas.types import MediaType
|
||||||
|
|
||||||
|
|
||||||
@@ -66,27 +73,133 @@ def _chain(tmp_path, subscribes: list[SubscriptionSnapshot]):
|
|||||||
sessionmaker(bind=engine)
|
sessionmaker(bind=engine)
|
||||||
)
|
)
|
||||||
chain.get_states_for_search = lambda state: state
|
chain.get_states_for_search = lambda state: state
|
||||||
chain._rlock = _ForbiddenLock()
|
chain._match_lock = _ForbiddenLock()
|
||||||
|
chain._search_queue_lock = threading.Lock()
|
||||||
|
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
|
||||||
return chain
|
return chain
|
||||||
|
|
||||||
|
|
||||||
|
def _make_tasks_ready(monkeypatch) -> None:
|
||||||
|
"""让治理测试中的持久任务立即到期,避免依赖真实随机时钟。"""
|
||||||
|
ready_at = "1970-01-01T00:00:00+00:00"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.chain.subscribe.search._search_task_available_at",
|
||||||
|
lambda _source, subscription_ids: {
|
||||||
|
subscription_id: ready_at for subscription_id in subscription_ids
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_fallback_task_schedule_staggers_each_subscription(monkeypatch):
|
||||||
|
"""兜底批次首条抖动后,每条后续订阅都按独立随机间隔到期。"""
|
||||||
|
now = datetime(2026, 9, 3, 1, 2, 3, tzinfo=timezone.utc)
|
||||||
|
delays = iter((12, 60, 300))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.chain.subscribe.search.random.randint",
|
||||||
|
lambda _low, _high: next(delays),
|
||||||
|
)
|
||||||
|
|
||||||
|
schedule = _search_task_available_at(
|
||||||
|
"fallback",
|
||||||
|
(1, 2, 3),
|
||||||
|
now=now,
|
||||||
|
)
|
||||||
|
|
||||||
|
available = [datetime.fromisoformat(schedule[subscribe_id]) for subscribe_id in (1, 2, 3)]
|
||||||
|
assert (available[0] - now).total_seconds() == 12
|
||||||
|
assert (available[1] - available[0]).total_seconds() == 60
|
||||||
|
assert (available[2] - available[1]).total_seconds() == 300
|
||||||
|
|
||||||
|
|
||||||
|
def test_inline_fallback_search_preserves_site_pressure_stagger(tmp_path, monkeypatch):
|
||||||
|
"""无持久队列的兼容宿主仍须在每条自动兜底订阅前错峰。"""
|
||||||
|
subscribes = [_subscribe(20), _subscribe(21)]
|
||||||
|
chain = _chain(tmp_path, subscribes)
|
||||||
|
del chain.subscription_search_repository
|
||||||
|
waits = []
|
||||||
|
delays = iter((60, 300))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.chain.subscribe.search.random.randint",
|
||||||
|
lambda _low, _high: next(delays),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("app.chain.subscribe.search.time.sleep", waits.append)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_process_search_subscription",
|
||||||
|
lambda item, _searchchain, **_kwargs: item,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
chain.search(state="R")
|
||||||
|
|
||||||
|
assert waits == [60, 300]
|
||||||
|
|
||||||
|
|
||||||
|
def test_successful_sites_remain_available_to_next_due_subscription(tmp_path, monkeypatch):
|
||||||
|
"""正常站点请求不能让同批下一条到期订阅漏掉目标站点。"""
|
||||||
|
subscribes = [_subscribe(40), _subscribe(41)]
|
||||||
|
chain = _chain(tmp_path, subscribes)
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
searchchain = object.__new__(SearchChain)
|
||||||
|
searchchain.configure_subscription_site_budget(None)
|
||||||
|
current_subscription_id = 0
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def search_site_torrents(*, site, **_kwargs):
|
||||||
|
"""记录真实发出的站点请求并发布正常完成观察结果。"""
|
||||||
|
calls.append((current_subscription_id, site["id"]))
|
||||||
|
report_site_search_outcome(attempted=True, outcome="success")
|
||||||
|
return [f"torrent-{current_subscription_id}-{site['id']}"]
|
||||||
|
|
||||||
|
def process(subscribe, current_searchchain, **_kwargs):
|
||||||
|
"""让每条到期订阅访问同一组完整目标站点。"""
|
||||||
|
nonlocal current_subscription_id
|
||||||
|
current_subscription_id = subscribe.id
|
||||||
|
for site_id in (11, 12):
|
||||||
|
current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access
|
||||||
|
site={"id": site_id, "name": f"Site {site_id}"},
|
||||||
|
keyword=subscribe.name,
|
||||||
|
mtype=MediaType.MOVIE,
|
||||||
|
page=0,
|
||||||
|
)
|
||||||
|
return subscribe
|
||||||
|
|
||||||
|
searchchain.search_site_torrents = search_site_torrents
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)]
|
||||||
|
assert chain.get_search_batch(batch_id).state == "completed"
|
||||||
|
|
||||||
|
|
||||||
def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch):
|
def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch):
|
||||||
"""R/P 兜底搜索在持久队列中执行,不受日常 Match 长锁阻塞。"""
|
"""R/P 兜底搜索在持久队列中执行,不受日常 Match 长锁阻塞。"""
|
||||||
subscribes = [_subscribe(1), _subscribe(2)]
|
subscribes = [_subscribe(1), _subscribe(2)]
|
||||||
chain = _chain(tmp_path, subscribes)
|
chain = _chain(tmp_path, subscribes)
|
||||||
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
|
_make_tasks_ready(monkeypatch)
|
||||||
processed = []
|
processed = []
|
||||||
|
|
||||||
|
def process(subscribe, _searchchain, *, execution_context):
|
||||||
|
"""记录每条任务独立的订阅执行上下文。"""
|
||||||
|
processed.append((subscribe.id, execution_context))
|
||||||
|
return subscribe
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
chain,
|
chain,
|
||||||
"_process_search_subscription",
|
"_process_search_subscription",
|
||||||
lambda subscribe, _searchchain: processed.append(subscribe.id) or subscribe,
|
process,
|
||||||
)
|
)
|
||||||
|
|
||||||
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
batch_id = chain.search(state="R")
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
batch = chain.get_search_batch(batch_id)
|
batch = chain.get_search_batch(batch_id)
|
||||||
assert processed == [1, 2]
|
assert [subscribe_id for subscribe_id, _context in processed] == [1, 2]
|
||||||
|
assert [context.lease.subscription_id for _subscribe_id, context in processed] == [1, 2]
|
||||||
|
assert len({context.task_id for _subscribe_id, context in processed}) == 2
|
||||||
|
assert len({id(context) for _subscribe_id, context in processed}) == 2
|
||||||
assert batch.state == "completed"
|
assert batch.state == "completed"
|
||||||
assert batch.finished_count == 2
|
assert batch.finished_count == 2
|
||||||
assert batch.failed_count == 0
|
assert batch.failed_count == 0
|
||||||
@@ -96,10 +209,10 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
|
|||||||
"""单订阅异常不得中止批次后续任务,聚合终态必须暴露失败。"""
|
"""单订阅异常不得中止批次后续任务,聚合终态必须暴露失败。"""
|
||||||
subscribes = [_subscribe(3), _subscribe(4)]
|
subscribes = [_subscribe(3), _subscribe(4)]
|
||||||
chain = _chain(tmp_path, subscribes)
|
chain = _chain(tmp_path, subscribes)
|
||||||
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
|
_make_tasks_ready(monkeypatch)
|
||||||
processed = []
|
processed = []
|
||||||
|
|
||||||
def process(subscribe, _searchchain):
|
def process(subscribe, _searchchain, **_kwargs):
|
||||||
"""让首条失败并保持第二条正常完成。"""
|
"""让首条失败并保持第二条正常完成。"""
|
||||||
processed.append(subscribe.id)
|
processed.append(subscribe.id)
|
||||||
if subscribe.id == 3:
|
if subscribe.id == 3:
|
||||||
@@ -118,21 +231,218 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
|
|||||||
assert batch.failed_count == 1
|
assert batch.failed_count == 1
|
||||||
assert batch.last_error == "provider timeout"
|
assert batch.last_error == "provider timeout"
|
||||||
|
|
||||||
|
for subscribe_id in (3, 4):
|
||||||
|
lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe_id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert lease is not None
|
||||||
|
assert chain._subscription_execution_admission.release(lease) is True
|
||||||
|
|
||||||
def test_late_cancel_completes_when_download_submission_already_started(tmp_path, monkeypatch):
|
|
||||||
|
def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch):
|
||||||
|
"""索引器吞错后仍须失败收口当前任务,并继续处理后续订阅。"""
|
||||||
|
subscribes = [_subscribe(13), _subscribe(14)]
|
||||||
|
chain = _chain(tmp_path, subscribes)
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
searchchain = object.__new__(SearchChain)
|
||||||
|
attempts = 0
|
||||||
|
processed = []
|
||||||
|
|
||||||
|
def execute_search(_site, _request):
|
||||||
|
"""首条站点请求模拟被 IndexerModule 吞掉的 HTTP 429。"""
|
||||||
|
nonlocal attempts
|
||||||
|
attempts += 1
|
||||||
|
if attempts == 1:
|
||||||
|
raise RuntimeError("HTTP 429")
|
||||||
|
return False, []
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__search_check",
|
||||||
|
staticmethod(lambda _site, _keyword=None: True),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__execute_search",
|
||||||
|
staticmethod(execute_search),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__indexer_statistic",
|
||||||
|
staticmethod(lambda **_kwargs: None),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__parse_result",
|
||||||
|
staticmethod(lambda **_kwargs: []),
|
||||||
|
)
|
||||||
|
searchchain.search_site_torrents = object.__new__(IndexerModule).search_torrents
|
||||||
|
|
||||||
|
def process(subscribe, current_searchchain, **_kwargs):
|
||||||
|
"""让每条任务经过同一预算包装,并把观察到的站点失败抛给队列。"""
|
||||||
|
processed.append(subscribe.id)
|
||||||
|
current_searchchain._search_site_torrents_with_budget( # pylint: disable=protected-access
|
||||||
|
site={
|
||||||
|
"id": 31 if subscribe.id == 13 else 32,
|
||||||
|
"name": "Flaky" if subscribe.id == 13 else "Healthy",
|
||||||
|
},
|
||||||
|
keyword=subscribe.name,
|
||||||
|
mtype=MediaType.MOVIE,
|
||||||
|
page=0,
|
||||||
|
)
|
||||||
|
failures = current_searchchain.consume_subscription_site_budget_failures()
|
||||||
|
if failures:
|
||||||
|
raise RuntimeError(";".join(failures))
|
||||||
|
return subscribe
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert processed == [13, 14]
|
||||||
|
assert batch.state == "failed"
|
||||||
|
assert batch.finished_count == 1
|
||||||
|
assert batch.failed_count == 1
|
||||||
|
assert "Flaky" in batch.last_error
|
||||||
|
assert "HTTP 429" in batch.last_error
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch):
|
||||||
|
"""Match 已持有同一订阅时,Search 本轮完成为跳过且不进入业务处理。"""
|
||||||
|
subscribe = _subscribe(6)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
process = Mock(side_effect=AssertionError("冲突订阅不应进入 Search"))
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
match_lease = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert match_lease is not None
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
process.assert_not_called()
|
||||||
|
assert batch.state == "skipped"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.skipped_count == 1
|
||||||
|
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
|
||||||
|
assert chain._subscription_execution_admission.release(match_lease) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch):
|
||||||
|
"""准入后重新读取到暂停订阅时不得开始搜索或伪造完成。"""
|
||||||
|
subscribe = _subscribe(9)
|
||||||
|
paused = replace(subscribe, state="S")
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
chain.subscription_repository.get = Mock(side_effect=(subscribe, paused))
|
||||||
|
process = Mock(side_effect=AssertionError("暂停订阅不应进入 Search"))
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
process.assert_not_called()
|
||||||
|
assert batch.state == "skipped"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.skipped_count == 1
|
||||||
|
assert batch.last_error == "订阅已暂停,本轮搜索已跳过"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch):
|
||||||
|
"""站点预算和状态清理都失败时仍必须释放当前订阅 owner。"""
|
||||||
|
subscribe = replace(_subscribe(7), state="N")
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_process_search_subscription",
|
||||||
|
lambda item, _searchchain, **_kwargs: item,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_SubscribeChain__apply_subscribe_update",
|
||||||
|
Mock(side_effect=RuntimeError("state cleanup failed")),
|
||||||
|
)
|
||||||
|
searchchain = Mock()
|
||||||
|
|
||||||
|
def configure_site_budget(budget):
|
||||||
|
"""仅让释放站点预算的清理步骤失败。"""
|
||||||
|
if budget is None:
|
||||||
|
raise RuntimeError("budget cleanup failed")
|
||||||
|
|
||||||
|
searchchain.configure_subscription_site_budget.side_effect = configure_site_budget
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=searchchain):
|
||||||
|
batch_id = chain.search(state="N")
|
||||||
|
|
||||||
|
assert chain.get_search_batch(batch_id).state == "completed"
|
||||||
|
replacement = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert chain._subscription_execution_admission.release(replacement) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_state_reset_stays_inside_subscription_admission(tmp_path, monkeypatch):
|
||||||
|
"""N 到 R 的本地状态写回完成前不能让 Match 接管同一订阅。"""
|
||||||
|
subscribe = replace(_subscribe(8), state="N")
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_process_search_subscription",
|
||||||
|
lambda item, _searchchain, **_kwargs: item,
|
||||||
|
)
|
||||||
|
competing_leases = []
|
||||||
|
|
||||||
|
def apply_update(*_args, **_kwargs):
|
||||||
|
"""在状态写回时探测同一订阅仍由 Search 持有。"""
|
||||||
|
competing_leases.append(
|
||||||
|
chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_SubscribeChain__apply_subscribe_update", apply_update)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
chain.search(state="N")
|
||||||
|
|
||||||
|
assert competing_leases == [None]
|
||||||
|
replacement = chain._subscription_execution_admission.try_acquire(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
operation="match",
|
||||||
|
ttl_seconds=60,
|
||||||
|
)
|
||||||
|
assert replacement is not None
|
||||||
|
assert chain._subscription_execution_admission.release(replacement) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_late_cancel_completes_when_download_side_effect_already_started(tmp_path, monkeypatch):
|
||||||
"""取消晚于下载器副作用边界时按真实结果完成,不能伪装成未执行取消。"""
|
"""取消晚于下载器副作用边界时按真实结果完成,不能伪装成未执行取消。"""
|
||||||
subscribe = _subscribe(5)
|
subscribe = _subscribe(5)
|
||||||
chain = _chain(tmp_path, [subscribe])
|
chain = _chain(tmp_path, [subscribe])
|
||||||
monkeypatch.setattr(chain, "_search_batch_available_at", lambda _source: "1970-01-01T00:00:00+00:00")
|
_make_tasks_ready(monkeypatch)
|
||||||
chain.subscription_download_repository = SimpleNamespace(
|
|
||||||
has_started_for_task=lambda _task_id: False,
|
|
||||||
)
|
|
||||||
queue = chain.subscription_search_repository
|
queue = chain.subscription_search_repository
|
||||||
cancel_checks = iter((False, True))
|
cancel_checks = iter((False, True))
|
||||||
monkeypatch.setattr(queue, "is_cancel_requested", lambda _task_id: next(cancel_checks))
|
monkeypatch.setattr(queue, "is_cancel_requested", lambda _task_id: next(cancel_checks))
|
||||||
def process(item, _searchchain):
|
def process(item, _searchchain, *, execution_context):
|
||||||
"""模拟当前任务复用或完成下载后才收到取消。"""
|
"""模拟当前任务复用或完成下载后才收到取消。"""
|
||||||
chain._mark_subscription_download_started()
|
execution_context.mark_download_started()
|
||||||
return item
|
return item
|
||||||
|
|
||||||
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
@@ -144,3 +454,148 @@ def test_late_cancel_completes_when_download_submission_already_started(tmp_path
|
|||||||
assert batch.state == "completed"
|
assert batch.state == "completed"
|
||||||
assert batch.finished_count == 1
|
assert batch.finished_count == 1
|
||||||
assert batch.cancelled_count == 0
|
assert batch.cancelled_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_stop_requeues_task_after_search_returns(tmp_path, monkeypatch):
|
||||||
|
"""系统停机应阻止后续副作用,并把未完成任务退回可恢复队列。"""
|
||||||
|
subscribe = _subscribe(10)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
stop_state = SimpleNamespace(is_system_stopped=False)
|
||||||
|
chain.stop_state = stop_state
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
|
||||||
|
def process(item, _searchchain, *, execution_context):
|
||||||
|
"""模拟站点搜索返回部分候选时进程进入停止阶段。"""
|
||||||
|
stop_state.is_system_stopped = True
|
||||||
|
assert execution_context.should_stop() is True
|
||||||
|
return item
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "queued"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
assert batch.skipped_count == 0
|
||||||
|
|
||||||
|
stop_state.is_system_stopped = False
|
||||||
|
recovered = chain.subscription_search_repository.claim_next(owner="worker-after-restart")
|
||||||
|
assert recovered is not None
|
||||||
|
assert recovered.subscription_id == subscribe.id
|
||||||
|
assert recovered.attempt_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_system_stop_completes_when_download_side_effect_already_started(tmp_path, monkeypatch):
|
||||||
|
"""停机晚于下载器副作用边界时必须终结任务,避免重启后重复提交。"""
|
||||||
|
subscribe = _subscribe(11)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
stop_state = SimpleNamespace(is_system_stopped=False)
|
||||||
|
chain.stop_state = stop_state
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
|
||||||
|
def process(item, _searchchain, *, execution_context):
|
||||||
|
"""模拟下载器已接收任务后进程进入停止阶段。"""
|
||||||
|
execution_context.mark_download_started()
|
||||||
|
stop_state.is_system_stopped = True
|
||||||
|
return item
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "completed"
|
||||||
|
assert batch.finished_count == 1
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
assert batch.skipped_count == 0
|
||||||
|
assert chain.subscription_search_repository.claim_next(owner="worker-after-restart") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_ttl_expiry_after_normal_return_marks_task_and_batch_failed(tmp_path, monkeypatch):
|
||||||
|
"""正常返回也必须检查执行 TTL,未提交下载时按失败收口。"""
|
||||||
|
subscribe = _subscribe(15)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain._subscription_execution_admission,
|
||||||
|
"is_expired",
|
||||||
|
lambda _lease: True,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_process_search_subscription",
|
||||||
|
lambda item, _searchchain, **_kwargs: item,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "failed"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.failed_count == 1
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
assert batch.last_error == "订阅执行已超过协作截止时间"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch):
|
||||||
|
"""TTL 晚于下载器副作用边界时按实际结果完成,避免下轮重复提交。"""
|
||||||
|
subscribe = _subscribe(16)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain._subscription_execution_admission,
|
||||||
|
"is_expired",
|
||||||
|
lambda _lease: True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def process(item, _searchchain, *, execution_context):
|
||||||
|
"""模拟下载器已接收任务后搜索链正常返回。"""
|
||||||
|
execution_context.mark_download_started()
|
||||||
|
return item
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "completed"
|
||||||
|
assert batch.finished_count == 1
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_site_budget_ttl_expiry_marks_task_and_batch_failed(tmp_path, monkeypatch):
|
||||||
|
"""站点预算观察到执行 TTL 到期时必须失败收口,不能伪装成用户取消。"""
|
||||||
|
subscribe = _subscribe(12)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain._subscription_execution_admission,
|
||||||
|
"is_expired",
|
||||||
|
lambda _lease: True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def process(_item, _searchchain, *, execution_context):
|
||||||
|
"""模拟站点预算将协作超时传播为搜索取消异常。"""
|
||||||
|
assert execution_context.should_stop() is True
|
||||||
|
raise SubscriptionSearchCancelled("订阅搜索已取消")
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "_process_search_subscription", process)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "failed"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.failed_count == 1
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
assert batch.last_error == "订阅执行已超过协作截止时间"
|
||||||
|
|||||||
@@ -46,6 +46,42 @@ def test_search_queue_coalesces_active_subscription_and_raises_priority(tmp_path
|
|||||||
assert first.task_id != second.task_id
|
assert first.task_id != second.task_id
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_queue_claims_each_subscription_only_after_its_available_at(tmp_path):
|
||||||
|
"""逐订阅到期时间必须持久化,未到期任务不能占用同步 worker。"""
|
||||||
|
repository, _engine = _repository(tmp_path)
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
ready_at = (now - timedelta(seconds=1)).isoformat(timespec="seconds")
|
||||||
|
later_at = (now + timedelta(minutes=5)).isoformat(timespec="seconds")
|
||||||
|
repository.enqueue(
|
||||||
|
subscription_ids=(20, 21),
|
||||||
|
source="fallback",
|
||||||
|
priority=10,
|
||||||
|
available_at_by_subscription={20: ready_at, 21: later_at},
|
||||||
|
)
|
||||||
|
|
||||||
|
first = repository.claim_next(owner="worker-a")
|
||||||
|
|
||||||
|
assert first.subscription_id == 20
|
||||||
|
assert first.available_at == ready_at
|
||||||
|
assert repository.finish_task(
|
||||||
|
task_id=first.task_id,
|
||||||
|
lease_token=first.lease_token,
|
||||||
|
state="completed",
|
||||||
|
) is True
|
||||||
|
assert repository.claim_next(owner="worker-a") is None
|
||||||
|
|
||||||
|
repository.enqueue(
|
||||||
|
subscription_ids=(21,),
|
||||||
|
source="manual",
|
||||||
|
priority=100,
|
||||||
|
available_at_by_subscription={21: ready_at},
|
||||||
|
)
|
||||||
|
accelerated = repository.claim_next(owner="worker-b")
|
||||||
|
assert accelerated.subscription_id == 21
|
||||||
|
assert accelerated.available_at == ready_at
|
||||||
|
assert accelerated.priority == 100
|
||||||
|
|
||||||
|
|
||||||
def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path):
|
def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path):
|
||||||
"""进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。"""
|
"""进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。"""
|
||||||
repository, engine = _repository(tmp_path)
|
repository, engine = _repository(tmp_path)
|
||||||
@@ -160,6 +196,38 @@ def test_search_queue_finishes_batch_with_aggregated_failure(tmp_path):
|
|||||||
assert batch.last_error == "site timeout"
|
assert batch.last_error == "site timeout"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_queue_aggregates_skipped_tasks_without_marking_success(tmp_path):
|
||||||
|
"""跳过任务应单独计数并让批次暴露 skipped 聚合终态。"""
|
||||||
|
repository, _engine = _repository(tmp_path)
|
||||||
|
enqueued = repository.enqueue(
|
||||||
|
subscription_ids=(10, 11),
|
||||||
|
source="fallback",
|
||||||
|
priority=10,
|
||||||
|
)
|
||||||
|
first = repository.claim_next(owner="worker-a")
|
||||||
|
assert repository.finish_task(
|
||||||
|
task_id=first.task_id,
|
||||||
|
lease_token=first.lease_token,
|
||||||
|
state="skipped",
|
||||||
|
error="同一订阅正在由其他通道处理,本轮搜索已跳过",
|
||||||
|
) is True
|
||||||
|
second = repository.claim_next(owner="worker-a")
|
||||||
|
assert repository.finish_task(
|
||||||
|
task_id=second.task_id,
|
||||||
|
lease_token=second.lease_token,
|
||||||
|
state="completed",
|
||||||
|
) is True
|
||||||
|
|
||||||
|
batch = repository.get_batch(enqueued.batch.batch_id)
|
||||||
|
|
||||||
|
assert batch.state == "skipped"
|
||||||
|
assert batch.finished_count == 1
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert batch.cancelled_count == 0
|
||||||
|
assert batch.skipped_count == 1
|
||||||
|
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
|
||||||
|
|
||||||
|
|
||||||
def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
|
def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
|
||||||
"""手工任务可优先,但等待超过公平窗口的兜底任务不得持续饥饿。"""
|
"""手工任务可优先,但等待超过公平窗口的兜底任务不得持续饥饿。"""
|
||||||
repository, engine = _repository(tmp_path)
|
repository, engine = _repository(tmp_path)
|
||||||
|
|||||||
@@ -0,0 +1,61 @@
|
|||||||
|
"""订阅搜索批次跳过计数的 Alembic 可逆迁移测试。"""
|
||||||
|
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic.migration import MigrationContext
|
||||||
|
from alembic.operations import Operations
|
||||||
|
|
||||||
|
MIGRATION = "database.versions.b6c1d9e4a7f2_3_0_24"
|
||||||
|
|
||||||
|
|
||||||
|
def _bind_migration(monkeypatch, connection):
|
||||||
|
"""把跳过计数迁移绑定到隔离 SQLite 连接。"""
|
||||||
|
migration = importlib.import_module(MIGRATION)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
migration,
|
||||||
|
"op",
|
||||||
|
Operations(MigrationContext.configure(connection)),
|
||||||
|
)
|
||||||
|
return migration
|
||||||
|
|
||||||
|
|
||||||
|
def test_subscription_search_skipped_count_migration_is_reversible(monkeypatch) -> None:
|
||||||
|
"""存量批次获得零默认值,迁移可重复执行并完整回滚。"""
|
||||||
|
engine = sa.create_engine("sqlite://")
|
||||||
|
metadata = sa.MetaData()
|
||||||
|
batch = sa.Table(
|
||||||
|
"subscriptionsearchbatch",
|
||||||
|
metadata,
|
||||||
|
sa.Column("id", sa.Integer(), primary_key=True),
|
||||||
|
sa.Column("batch_id", sa.String(64), nullable=False),
|
||||||
|
)
|
||||||
|
with engine.begin() as connection:
|
||||||
|
metadata.create_all(connection)
|
||||||
|
connection.execute(batch.insert(), {"id": 1, "batch_id": "batch-1"})
|
||||||
|
migration = _bind_migration(monkeypatch, connection)
|
||||||
|
|
||||||
|
migration.upgrade()
|
||||||
|
migration.upgrade()
|
||||||
|
|
||||||
|
columns = {
|
||||||
|
column["name"]
|
||||||
|
for column in sa.inspect(connection).get_columns("subscriptionsearchbatch")
|
||||||
|
}
|
||||||
|
assert "skipped_count" in columns
|
||||||
|
assert connection.execute(
|
||||||
|
sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1")
|
||||||
|
).scalar_one() == 0
|
||||||
|
|
||||||
|
migration.downgrade()
|
||||||
|
downgraded = {
|
||||||
|
column["name"]
|
||||||
|
for column in sa.inspect(connection).get_columns("subscriptionsearchbatch")
|
||||||
|
}
|
||||||
|
assert "skipped_count" not in downgraded
|
||||||
|
|
||||||
|
migration.upgrade()
|
||||||
|
assert connection.execute(
|
||||||
|
sa.text("SELECT skipped_count FROM subscriptionsearchbatch WHERE id = 1")
|
||||||
|
).scalar_one() == 0
|
||||||
|
engine.dispose()
|
||||||
@@ -14,12 +14,13 @@ from app.application.subscription.sitebudget import (
|
|||||||
SiteBudgetClaim,
|
SiteBudgetClaim,
|
||||||
SubscriptionSearchCancelled,
|
SubscriptionSearchCancelled,
|
||||||
SubscriptionSiteBudget,
|
SubscriptionSiteBudget,
|
||||||
|
SubscriptionSiteBudgetUnavailable,
|
||||||
)
|
)
|
||||||
from app.chain.search.facade import SearchChain
|
from app.chain.search.facade import SearchChain
|
||||||
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
||||||
from app.db.base import Base
|
from app.db.base import Base
|
||||||
from app.db.models.subscriptionsearch import SubscriptionSiteBudget as SiteBudgetRecord
|
from app.db.models.subscriptionsearch import SubscriptionSiteBudget as SiteBudgetRecord
|
||||||
from app.modules.indexer import _classify_search_failure
|
from app.modules.indexer import IndexerModule, _classify_search_failure
|
||||||
from app.runtime.stop import ProcessStopState
|
from app.runtime.stop import ProcessStopState
|
||||||
|
|
||||||
|
|
||||||
@@ -76,6 +77,8 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat
|
|||||||
next_allowed_at=(now + timedelta(minutes=5)).isoformat(timespec="seconds"),
|
next_allowed_at=(now + timedelta(minutes=5)).isoformat(timespec="seconds"),
|
||||||
error="request timeout",
|
error="request timeout",
|
||||||
) is True
|
) is True
|
||||||
|
cooled = repository.claim_site(site_id=3, owner="worker-b", lease_seconds=900)
|
||||||
|
assert cooled.acquired is False
|
||||||
with Session(engine) as session:
|
with Session(engine) as session:
|
||||||
failed = session.execute(
|
failed = session.execute(
|
||||||
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3)
|
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 3)
|
||||||
@@ -103,6 +106,29 @@ def test_site_budget_applies_error_cooldown_and_gradual_success_recovery(tmp_pat
|
|||||||
assert healthy.last_outcome == "success"
|
assert healthy.last_outcome == "success"
|
||||||
|
|
||||||
|
|
||||||
|
def test_site_budget_ignores_legacy_success_interval(tmp_path):
|
||||||
|
"""升级前留下的正常成功间隔不能阻止新语义下的到期订阅。"""
|
||||||
|
repository, engine = _repository(tmp_path)
|
||||||
|
first = repository.claim_site(site_id=4, owner="worker-a", lease_seconds=900)
|
||||||
|
future = (datetime.now(timezone.utc) + timedelta(minutes=10)).isoformat(timespec="seconds")
|
||||||
|
assert repository.finish_site(
|
||||||
|
site_id=4,
|
||||||
|
lease_token=first.lease_token,
|
||||||
|
outcome="success",
|
||||||
|
next_allowed_at=future,
|
||||||
|
) is True
|
||||||
|
|
||||||
|
recovered = repository.claim_site(site_id=4, owner="worker-b", lease_seconds=900)
|
||||||
|
|
||||||
|
assert recovered.acquired is True
|
||||||
|
assert recovered.lease_token is not None
|
||||||
|
with Session(engine) as session:
|
||||||
|
record = session.execute(
|
||||||
|
select(SiteBudgetRecord).where(SiteBudgetRecord.site_id == 4)
|
||||||
|
).scalar_one()
|
||||||
|
assert record.next_allowed_at < future
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("error", "outcome"),
|
("error", "outcome"),
|
||||||
[
|
[
|
||||||
@@ -135,40 +161,50 @@ class _WaitingRepository:
|
|||||||
raise AssertionError("未取得站点预算时不应收口")
|
raise AssertionError("未取得站点预算时不应收口")
|
||||||
|
|
||||||
|
|
||||||
def test_site_budget_wait_is_cancellable_without_business_lock():
|
def test_unavailable_site_budget_does_not_sleep_in_worker():
|
||||||
"""批次取消应在一次短等待后终止预算获取。"""
|
"""错误冷却中的站点应立即留待下轮,不占用同步 worker 等待。"""
|
||||||
cancelled = False
|
|
||||||
|
|
||||||
def sleeper(_seconds: float) -> None:
|
|
||||||
"""模拟等待一次后收到取消请求。"""
|
|
||||||
nonlocal cancelled
|
|
||||||
cancelled = True
|
|
||||||
|
|
||||||
budget = SubscriptionSiteBudget(
|
budget = SubscriptionSiteBudget(
|
||||||
repository=_WaitingRepository(),
|
repository=_WaitingRepository(),
|
||||||
owner="worker-a",
|
owner="worker-a",
|
||||||
cancelled=lambda: cancelled,
|
cancelled=lambda: False,
|
||||||
|
stop_state=ProcessStopState(),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(SubscriptionSiteBudgetUnavailable):
|
||||||
|
budget.acquire(9)
|
||||||
|
|
||||||
|
|
||||||
|
def test_site_budget_checks_cancellation_before_repository_claim():
|
||||||
|
"""取消或停机必须在创建任何站点租约前终止当前任务。"""
|
||||||
|
class _ForbiddenRepository(_WaitingRepository):
|
||||||
|
"""取消路径不允许触发持久认领。"""
|
||||||
|
|
||||||
|
def claim_site(self, **_kwargs) -> SiteBudgetClaim:
|
||||||
|
"""若取消检查失效则立即暴露。"""
|
||||||
|
raise AssertionError("取消任务不应认领站点预算")
|
||||||
|
|
||||||
|
budget = SubscriptionSiteBudget(
|
||||||
|
repository=_ForbiddenRepository(),
|
||||||
|
owner="worker-a",
|
||||||
|
cancelled=lambda: True,
|
||||||
stop_state=ProcessStopState(),
|
stop_state=ProcessStopState(),
|
||||||
sleeper=sleeper,
|
|
||||||
max_wait_seconds=600,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
with pytest.raises(SubscriptionSearchCancelled):
|
with pytest.raises(SubscriptionSearchCancelled):
|
||||||
budget.acquire(9)
|
budget.acquire(9)
|
||||||
|
|
||||||
|
|
||||||
def test_site_budget_delay_keeps_manual_requests_under_same_budget():
|
def test_site_budget_only_delays_external_failures():
|
||||||
"""来源优先级不参与站点冷却计算,手工任务不能获得旁路。"""
|
"""正常完成立即恢复,外站错误仍按类别进入冷却。"""
|
||||||
budget = SubscriptionSiteBudget(
|
budget = SubscriptionSiteBudget(
|
||||||
repository=_WaitingRepository(),
|
repository=_WaitingRepository(),
|
||||||
owner="manual-worker",
|
owner="manual-worker",
|
||||||
cancelled=lambda: False,
|
cancelled=lambda: False,
|
||||||
stop_state=ProcessStopState(),
|
stop_state=ProcessStopState(),
|
||||||
random_uniform=lambda _low, _high: 60.0,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
assert budget._next_delay("success", consecutive_failures=0) == 60.0 # pylint: disable=protected-access
|
assert budget._next_delay("success", consecutive_failures=0) == 0.0 # pylint: disable=protected-access
|
||||||
assert budget._next_delay("success", consecutive_failures=2) == 120.0 # pylint: disable=protected-access
|
assert budget._next_delay("success", consecutive_failures=2) == 0.0 # pylint: disable=protected-access
|
||||||
assert budget._next_delay("rate_limited", consecutive_failures=0) == 900.0 # pylint: disable=protected-access
|
assert budget._next_delay("rate_limited", consecutive_failures=0) == 900.0 # pylint: disable=protected-access
|
||||||
|
|
||||||
|
|
||||||
@@ -206,14 +242,13 @@ def test_skipped_search_releases_budget_without_external_interval():
|
|||||||
|
|
||||||
|
|
||||||
def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
||||||
"""超出短等待窗口的站点返回空页并记录聚合失败,而非阻塞整个 provider。"""
|
"""错误冷却中的站点返回空页并记录聚合失败,而非阻塞 provider。"""
|
||||||
repository = _WaitingRepository()
|
repository = _WaitingRepository()
|
||||||
budget = SubscriptionSiteBudget(
|
budget = SubscriptionSiteBudget(
|
||||||
repository=repository,
|
repository=repository,
|
||||||
owner="fallback-worker",
|
owner="fallback-worker",
|
||||||
cancelled=lambda: False,
|
cancelled=lambda: False,
|
||||||
stop_state=ProcessStopState(),
|
stop_state=ProcessStopState(),
|
||||||
max_wait_seconds=0,
|
|
||||||
)
|
)
|
||||||
chain = object.__new__(SearchChain)
|
chain = object.__new__(SearchChain)
|
||||||
chain.configure_subscription_site_budget(budget)
|
chain.configure_subscription_site_budget(budget)
|
||||||
@@ -233,7 +268,7 @@ def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
|||||||
|
|
||||||
|
|
||||||
def test_search_provider_releases_successful_site_budget():
|
def test_search_provider_releases_successful_site_budget():
|
||||||
"""真实站点页完成后必须释放租约并写入成功间隔。"""
|
"""真实站点页正常完成后必须释放租约并立即允许下一任务。"""
|
||||||
captured = {}
|
captured = {}
|
||||||
|
|
||||||
class _Repository(_WaitingRepository):
|
class _Repository(_WaitingRepository):
|
||||||
@@ -260,7 +295,6 @@ def test_search_provider_releases_successful_site_budget():
|
|||||||
owner="manual-worker",
|
owner="manual-worker",
|
||||||
cancelled=lambda: False,
|
cancelled=lambda: False,
|
||||||
stop_state=ProcessStopState(),
|
stop_state=ProcessStopState(),
|
||||||
random_uniform=lambda _low, _high: 60.0,
|
|
||||||
)
|
)
|
||||||
chain = object.__new__(SearchChain)
|
chain = object.__new__(SearchChain)
|
||||||
chain.configure_subscription_site_budget(budget)
|
chain.configure_subscription_site_budget(budget)
|
||||||
@@ -283,3 +317,78 @@ def test_search_provider_releases_successful_site_budget():
|
|||||||
assert captured["site_id"] == 12
|
assert captured["site_id"] == 12
|
||||||
assert captured["outcome"] == "success"
|
assert captured["outcome"] == "success"
|
||||||
assert captured["lease_token"] == "lease-token"
|
assert captured["lease_token"] == "lease-token"
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_provider_aggregates_swallowed_indexer_failure(monkeypatch):
|
||||||
|
"""索引器吞掉外站异常返回空页时,provider 仍须暴露站点失败。"""
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class _Repository(_WaitingRepository):
|
||||||
|
"""提供立即可用租约并记录失败收口。"""
|
||||||
|
|
||||||
|
def claim_site(self, *, site_id: int, owner: str, lease_seconds: int) -> SiteBudgetClaim:
|
||||||
|
"""返回当前调用独占的站点租约。"""
|
||||||
|
del owner, lease_seconds
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return SiteBudgetClaim(
|
||||||
|
site_id=site_id,
|
||||||
|
acquired=True,
|
||||||
|
retry_at=now.isoformat(timespec="seconds"),
|
||||||
|
consecutive_failures=0,
|
||||||
|
lease_token="lease-token",
|
||||||
|
)
|
||||||
|
|
||||||
|
def finish_site(self, **kwargs) -> bool:
|
||||||
|
"""记录预算收口,确认错误冷却仍由 budget.finish 负责。"""
|
||||||
|
captured.update(kwargs)
|
||||||
|
return True
|
||||||
|
|
||||||
|
def execute_search(_site, _request):
|
||||||
|
"""模拟 IndexerModule 捕获的 HTTP 429。"""
|
||||||
|
raise RuntimeError("HTTP 429")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__search_check",
|
||||||
|
staticmethod(lambda _site, _keyword=None: True),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__execute_search",
|
||||||
|
staticmethod(execute_search),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__indexer_statistic",
|
||||||
|
staticmethod(lambda **_kwargs: None),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
IndexerModule,
|
||||||
|
"_IndexerModule__parse_result",
|
||||||
|
staticmethod(lambda **_kwargs: []),
|
||||||
|
)
|
||||||
|
|
||||||
|
budget = SubscriptionSiteBudget(
|
||||||
|
repository=_Repository(),
|
||||||
|
owner="fallback-worker",
|
||||||
|
cancelled=lambda: False,
|
||||||
|
stop_state=ProcessStopState(),
|
||||||
|
)
|
||||||
|
chain = object.__new__(SearchChain)
|
||||||
|
chain.configure_subscription_site_budget(budget)
|
||||||
|
chain.search_site_torrents = object.__new__(IndexerModule).search_torrents
|
||||||
|
|
||||||
|
result = chain._search_site_torrents_with_budget( # pylint: disable=protected-access
|
||||||
|
site={"id": 13, "name": "Flaky"},
|
||||||
|
keyword="movie",
|
||||||
|
mtype=None,
|
||||||
|
page=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result == []
|
||||||
|
assert captured["outcome"] == "rate_limited"
|
||||||
|
assert captured["error"] == "HTTP 429"
|
||||||
|
failures = chain.consume_subscription_site_budget_failures()
|
||||||
|
assert len(failures) == 1
|
||||||
|
assert "Flaky" in failures[0]
|
||||||
|
assert "HTTP 429" in failures[0]
|
||||||
|
|||||||
Reference in New Issue
Block a user