mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-09 01:16:50 +08:00
fix: defer subscription search on site budget conflicts
This commit is contained in:
@@ -200,6 +200,16 @@ class SubscriptionSearchRepository(Protocol):
|
|||||||
"""释放尚未完成的任务租约,供停止或取消后恢复。"""
|
"""释放尚未完成的任务租约,供停止或取消后恢复。"""
|
||||||
...
|
...
|
||||||
|
|
||||||
|
def defer_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
available_at: str,
|
||||||
|
) -> bool:
|
||||||
|
"""把临时站点预算冲突任务退回队列,并设置下一次领取时间。"""
|
||||||
|
...
|
||||||
|
|
||||||
def is_cancel_requested(self, task_id: str) -> bool:
|
def is_cancel_requested(self, task_id: str) -> bool:
|
||||||
"""判断任务或所属批次是否已请求取消。"""
|
"""判断任务或所属批次是否已请求取消。"""
|
||||||
...
|
...
|
||||||
|
|||||||
@@ -15,11 +15,29 @@ class SubscriptionSearchCancelled(RuntimeError):
|
|||||||
"""表示订阅搜索在可取消预算等待点终止。"""
|
"""表示订阅搜索在可取消预算等待点终止。"""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class SubscriptionSiteBudgetDeferral:
|
||||||
|
"""记录一次站点预算冲突及该站点最早可再次尝试的时间。"""
|
||||||
|
|
||||||
|
site_id: int
|
||||||
|
retry_at: str
|
||||||
|
|
||||||
|
|
||||||
|
class SubscriptionSearchDeferred(RuntimeError):
|
||||||
|
"""表示订阅搜索未失败,而是应在站点预算可用后重新入队。"""
|
||||||
|
|
||||||
|
def __init__(self, *, retry_at: str, site_ids: tuple[int, ...]) -> None:
|
||||||
|
"""保存队列恢复所需的时间和冲突站点,避免把临时冲突写成错误。"""
|
||||||
|
super().__init__(f"订阅搜索已延后,站点预算最早可重试:{retry_at}")
|
||||||
|
self.retry_at = retry_at
|
||||||
|
self.site_ids = site_ids
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionSiteBudgetUnavailable(RuntimeError):
|
class SubscriptionSiteBudgetUnavailable(RuntimeError):
|
||||||
"""表示站点仍处于错误冷却或已有未释放租约。"""
|
"""表示站点预算暂时不可用,调用方应记录为延后而非失败。"""
|
||||||
|
|
||||||
def __init__(self, *, site_id: int, retry_at: str) -> None:
|
def __init__(self, *, site_id: int, retry_at: str) -> None:
|
||||||
"""保存站点和下一次可尝试时间,供批次聚合失败展示。"""
|
"""保存站点和下一次可尝试时间,供订阅队列恢复。"""
|
||||||
super().__init__(f"站点 {site_id} 冷却或已有在途搜索,最早可重试:{retry_at}")
|
super().__init__(f"站点 {site_id} 冷却或已有在途搜索,最早可重试:{retry_at}")
|
||||||
self.site_id = site_id
|
self.site_id = site_id
|
||||||
self.retry_at = retry_at
|
self.retry_at = retry_at
|
||||||
|
|||||||
@@ -153,6 +153,8 @@ class SubscriptionExecutionStatusService:
|
|||||||
state = phase = "cancelling"
|
state = phase = "cancelling"
|
||||||
elif task.state == "running":
|
elif task.state == "running":
|
||||||
state = phase = task.phase or "running"
|
state = phase = task.phase or "running"
|
||||||
|
elif task.state == "queued" and task.phase == "waiting_site_budget":
|
||||||
|
state = phase = "waiting_site_budget"
|
||||||
else:
|
else:
|
||||||
state = phase = task.state
|
state = phase = task.state
|
||||||
return SubscriptionExecutionStatus(
|
return SubscriptionExecutionStatus(
|
||||||
|
|||||||
@@ -99,6 +99,8 @@ if TYPE_CHECKING:
|
|||||||
process: Callable[..., Any]
|
process: Callable[..., Any]
|
||||||
record_subscription_site_budget_failure: Callable[..., Any]
|
record_subscription_site_budget_failure: Callable[..., Any]
|
||||||
consume_subscription_site_budget_failures: Callable[..., Any]
|
consume_subscription_site_budget_failures: Callable[..., Any]
|
||||||
|
record_subscription_site_budget_deferred: Callable[..., Any]
|
||||||
|
consume_subscription_site_budget_deferrals: Callable[..., Any]
|
||||||
save_last_search_params: Callable[..., Any]
|
save_last_search_params: Callable[..., Any]
|
||||||
search_by_id: Callable[..., Any]
|
search_by_id: Callable[..., Any]
|
||||||
search_by_title: Callable[..., Any]
|
search_by_title: Callable[..., Any]
|
||||||
|
|||||||
@@ -4,7 +4,10 @@ import threading
|
|||||||
from collections.abc import AsyncIterator, Callable
|
from collections.abc import AsyncIterator, Callable
|
||||||
from typing import Any, Optional, TypeVar, cast
|
from typing import Any, Optional, TypeVar, cast
|
||||||
|
|
||||||
from app.application.subscription.sitebudget import SubscriptionSiteBudget
|
from app.application.subscription.sitebudget import (
|
||||||
|
SubscriptionSiteBudget,
|
||||||
|
SubscriptionSiteBudgetDeferral,
|
||||||
|
)
|
||||||
from app.chain.base import ChainBase
|
from app.chain.base import ChainBase
|
||||||
from app.chain.search.cache import SearchCacheOwner
|
from app.chain.search.cache import SearchCacheOwner
|
||||||
from app.chain.search.media import SearchMediaOwner
|
from app.chain.search.media import SearchMediaOwner
|
||||||
@@ -49,6 +52,7 @@ class SearchChain(ChainBase):
|
|||||||
"""仅为订阅搜索启用或清除站点预算,不影响其它搜索入口。"""
|
"""仅为订阅搜索启用或清除站点预算,不影响其它搜索入口。"""
|
||||||
self._subscription_site_budget = budget
|
self._subscription_site_budget = budget
|
||||||
self._subscription_site_budget_failures: list[str] = []
|
self._subscription_site_budget_failures: list[str] = []
|
||||||
|
self._subscription_site_budget_deferrals: list[SubscriptionSiteBudgetDeferral] = []
|
||||||
self._subscription_site_budget_failure_lock = threading.Lock()
|
self._subscription_site_budget_failure_lock = threading.Lock()
|
||||||
|
|
||||||
def record_subscription_site_budget_failure(self, error: str) -> None:
|
def record_subscription_site_budget_failure(self, error: str) -> None:
|
||||||
@@ -69,6 +73,27 @@ class SearchChain(ChainBase):
|
|||||||
self._subscription_site_budget_failures.clear()
|
self._subscription_site_budget_failures.clear()
|
||||||
return failures
|
return failures
|
||||||
|
|
||||||
|
def record_subscription_site_budget_deferred(
|
||||||
|
self,
|
||||||
|
deferral: SubscriptionSiteBudgetDeferral,
|
||||||
|
) -> None:
|
||||||
|
"""线程安全地记录临时站点冲突,供订阅任务重新入队。"""
|
||||||
|
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
|
||||||
|
if lock is None:
|
||||||
|
return
|
||||||
|
with lock:
|
||||||
|
self._subscription_site_budget_deferrals.append(deferral)
|
||||||
|
|
||||||
|
def consume_subscription_site_budget_deferrals(self) -> tuple[SubscriptionSiteBudgetDeferral, ...]:
|
||||||
|
"""读取并清空当前订阅搜索积累的站点预算延后事实。"""
|
||||||
|
lock = getattr(self, "_subscription_site_budget_failure_lock", None)
|
||||||
|
if lock is None:
|
||||||
|
return ()
|
||||||
|
with lock:
|
||||||
|
deferrals = tuple(self._subscription_site_budget_deferrals)
|
||||||
|
self._subscription_site_budget_deferrals.clear()
|
||||||
|
return deferrals
|
||||||
|
|
||||||
# owner descriptor 经类访问后被 mypy 视为普通 Callable;运行时仍需取回原始
|
# owner descriptor 经类访问后被 mypy 视为普通 Callable;运行时仍需取回原始
|
||||||
# classmethod 函数,才能保持 SearchChain 的直接 MRO 与既有绑定语义。
|
# classmethod 函数,才能保持 SearchChain 的直接 MRO 与既有绑定语义。
|
||||||
music_site_keywords = classmethod( # type: ignore[var-annotated]
|
music_site_keywords = classmethod( # type: ignore[var-annotated]
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from app.application.site.observation import (
|
|||||||
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
|
from app.application.site.sites import SitesHelper # pylint: disable=import-error,no-name-in-module
|
||||||
from app.application.subscription.sitebudget import (
|
from app.application.subscription.sitebudget import (
|
||||||
SubscriptionSiteBudget,
|
SubscriptionSiteBudget,
|
||||||
|
SubscriptionSiteBudgetDeferral,
|
||||||
SubscriptionSiteBudgetUnavailable,
|
SubscriptionSiteBudgetUnavailable,
|
||||||
)
|
)
|
||||||
from app.chain.search.contract import _SearchOwnerBase
|
from app.chain.search.contract import _SearchOwnerBase
|
||||||
@@ -215,7 +216,12 @@ class _SearchProviderSyncOwner(_SearchOwnerBase):
|
|||||||
try:
|
try:
|
||||||
claim = budget.acquire(site_id)
|
claim = budget.acquire(site_id)
|
||||||
except SubscriptionSiteBudgetUnavailable as error:
|
except SubscriptionSiteBudgetUnavailable as error:
|
||||||
self.record_subscription_site_budget_failure(str(error))
|
self.record_subscription_site_budget_deferred(
|
||||||
|
SubscriptionSiteBudgetDeferral(
|
||||||
|
site_id=error.site_id,
|
||||||
|
retry_at=error.retry_at,
|
||||||
|
)
|
||||||
|
)
|
||||||
logger.debug(str(error))
|
logger.debug(str(error))
|
||||||
return []
|
return []
|
||||||
with capture_site_search_observation() as observation:
|
with capture_site_search_observation() as observation:
|
||||||
|
|||||||
@@ -31,7 +31,9 @@ from app.application.subscription.observability import (
|
|||||||
from app.application.subscription.query import SubscriptionQueryService
|
from app.application.subscription.query import SubscriptionQueryService
|
||||||
from app.application.subscription.sitebudget import (
|
from app.application.subscription.sitebudget import (
|
||||||
SubscriptionSearchCancelled,
|
SubscriptionSearchCancelled,
|
||||||
|
SubscriptionSearchDeferred,
|
||||||
SubscriptionSiteBudget,
|
SubscriptionSiteBudget,
|
||||||
|
SubscriptionSiteBudgetDeferral,
|
||||||
)
|
)
|
||||||
from app.chain.media import MediaChain
|
from app.chain.media import MediaChain
|
||||||
from app.chain.search.facade import SearchChain
|
from app.chain.search.facade import SearchChain
|
||||||
@@ -323,6 +325,12 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
|
|||||||
reason = "ttl_timeout" if execution_context.is_expired() else "cancelled"
|
reason = "ttl_timeout" if execution_context.is_expired() else "cancelled"
|
||||||
outcome = "failed" if reason == "ttl_timeout" else "cancelled"
|
outcome = "failed" if reason == "ttl_timeout" else "cancelled"
|
||||||
logger.debug(f"订阅 {subscribe.name} 搜索已在安全边界取消")
|
logger.debug(f"订阅 {subscribe.name} 搜索已在安全边界取消")
|
||||||
|
except SubscriptionSearchDeferred as deferred:
|
||||||
|
outcome = "skipped"
|
||||||
|
reason = "site_budget_deferred"
|
||||||
|
logger.debug(
|
||||||
|
f"订阅 {subscribe.name} 站点预算冲突,兼容搜索将在 {deferred.retry_at} 后重试"
|
||||||
|
)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
outcome = "failed"
|
outcome = "failed"
|
||||||
reason = "error"
|
reason = "error"
|
||||||
@@ -610,6 +618,20 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
|
|||||||
"requeued" if system_stopped else "cancelled",
|
"requeued" if system_stopped else "cancelled",
|
||||||
"system_stop" if system_stopped else "cancelled",
|
"system_stop" if system_stopped else "cancelled",
|
||||||
)
|
)
|
||||||
|
except SubscriptionSearchDeferred as deferred:
|
||||||
|
requeued = queue.defer_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
available_at=deferred.retry_at,
|
||||||
|
)
|
||||||
|
if requeued:
|
||||||
|
logger.debug(
|
||||||
|
f"订阅 {subscribe.name} 站点预算冲突,已排队至 {deferred.retry_at} 后重试,"
|
||||||
|
f"sites={','.join(str(site_id) for site_id in deferred.site_ids)}"
|
||||||
|
)
|
||||||
|
summary.record("requeued", "site_budget_deferred")
|
||||||
|
else:
|
||||||
|
logger.debug(f"订阅搜索任务 {task_id} 租约已变化,跳过重复站点预算重排队")
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
|
logger.error(f"订阅 {subscribe.name} 搜索失败:{str(err)}", exc_info=True)
|
||||||
queue.finish_task(
|
queue.finish_task(
|
||||||
@@ -821,9 +843,12 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
filter_params=self.get_params(subscribe),
|
filter_params=self.get_params(subscribe),
|
||||||
)
|
)
|
||||||
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
|
site_budget_failures = searchchain.consume_subscription_site_budget_failures()
|
||||||
|
site_budget_deferrals = searchchain.consume_subscription_site_budget_deferrals()
|
||||||
_ensure_execution_active(execution_context)
|
_ensure_execution_active(execution_context)
|
||||||
if not contexts:
|
if not contexts:
|
||||||
logger.debug(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
|
logger.debug(f"订阅 {subscribe.keyword or subscribe.name} 未搜索到资源")
|
||||||
|
if not site_budget_failures:
|
||||||
|
self._raise_site_budget_deferral(site_budget_deferrals, execution_context)
|
||||||
self.finish_subscribe_or_not(
|
self.finish_subscribe_or_not(
|
||||||
subscribe=subscribe,
|
subscribe=subscribe,
|
||||||
meta=meta,
|
meta=meta,
|
||||||
@@ -835,6 +860,8 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
matched = self._filter_search_contexts(subscribe, contexts)
|
matched = self._filter_search_contexts(subscribe, contexts)
|
||||||
if not matched:
|
if not matched:
|
||||||
logger.debug(f"订阅 {subscribe.name} 没有符合过滤条件的资源")
|
logger.debug(f"订阅 {subscribe.name} 没有符合过滤条件的资源")
|
||||||
|
if not site_budget_failures:
|
||||||
|
self._raise_site_budget_deferral(site_budget_deferrals, execution_context)
|
||||||
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
|
self.finish_subscribe_or_not(subscribe=subscribe, meta=meta, mediainfo=mediainfo, lefts=no_exists)
|
||||||
self._raise_site_budget_failures(site_budget_failures)
|
self._raise_site_budget_failures(site_budget_failures)
|
||||||
return subscribe
|
return subscribe
|
||||||
@@ -862,6 +889,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
lefts=lefts,
|
lefts=lefts,
|
||||||
)
|
)
|
||||||
self._raise_site_budget_failures(site_budget_failures)
|
self._raise_site_budget_failures(site_budget_failures)
|
||||||
|
self._raise_site_budget_deferral(site_budget_deferrals, execution_context)
|
||||||
return cast(Optional[SubscriptionSnapshot], current)
|
return cast(Optional[SubscriptionSnapshot], current)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -870,6 +898,18 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
|
|||||||
if failures:
|
if failures:
|
||||||
raise RuntimeError(";".join(failures))
|
raise RuntimeError(";".join(failures))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise_site_budget_deferral(
|
||||||
|
deferrals: tuple[SubscriptionSiteBudgetDeferral, ...],
|
||||||
|
execution_context: Optional[SubscriptionExecutionContext],
|
||||||
|
) -> None:
|
||||||
|
"""在没有下载副作用时,将临时站点冲突转换为持久队列延后。"""
|
||||||
|
if not deferrals or (execution_context and execution_context.download_started):
|
||||||
|
return
|
||||||
|
retry_at = min(deferrals, key=lambda item: item.retry_at).retry_at
|
||||||
|
site_ids = tuple(dict.fromkeys(item.site_id for item in deferrals))
|
||||||
|
raise SubscriptionSearchDeferred(retry_at=retry_at, site_ids=site_ids)
|
||||||
|
|
||||||
def _filter_search_contexts(
|
def _filter_search_contexts(
|
||||||
self,
|
self,
|
||||||
subscribe: SubscriptionSnapshot,
|
subscribe: SubscriptionSnapshot,
|
||||||
|
|||||||
@@ -177,6 +177,22 @@ class TransactionalSubscriptionSearchRepository:
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def defer_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
available_at: str,
|
||||||
|
) -> bool:
|
||||||
|
"""以站点预算的下一次可用时间重新排队任务。"""
|
||||||
|
return self._write(
|
||||||
|
lambda repository: repository.defer_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
available_at=available_at,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def is_cancel_requested(self, task_id: str) -> bool:
|
def is_cancel_requested(self, task_id: str) -> bool:
|
||||||
"""查询任务或批次的取消请求。"""
|
"""查询任务或批次的取消请求。"""
|
||||||
return self._read(lambda repository: repository.is_cancel_requested(task_id))
|
return self._read(lambda repository: repository.is_cancel_requested(task_id))
|
||||||
@@ -208,11 +224,9 @@ class TransactionalSubscriptionSearchRepository:
|
|||||||
owner=owner,
|
owner=owner,
|
||||||
lease_seconds=lease_seconds,
|
lease_seconds=lease_seconds,
|
||||||
)
|
)
|
||||||
retry_at = (
|
retry_at = record.next_allowed_at
|
||||||
record.lease_expires_at
|
if not acquired and record.lease_token and record.lease_expires_at:
|
||||||
if record.lease_token and not acquired
|
retry_at = max(retry_at, record.lease_expires_at)
|
||||||
else record.next_allowed_at
|
|
||||||
) or record.next_allowed_at
|
|
||||||
return SiteBudgetClaim(
|
return SiteBudgetClaim(
|
||||||
site_id=record.site_id,
|
site_id=record.site_id,
|
||||||
acquired=acquired,
|
acquired=acquired,
|
||||||
|
|||||||
@@ -363,6 +363,60 @@ class SubscriptionSearchOper(DbOper):
|
|||||||
self._refresh_batch(task.batch_id, now=now, error=None)
|
self._refresh_batch(task.batch_id, now=now, error=None)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def defer_task(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
lease_token: str,
|
||||||
|
available_at: str,
|
||||||
|
) -> bool:
|
||||||
|
"""释放当前租约并在站点预算时间到达后恢复同一任务。"""
|
||||||
|
if not isinstance(self._db, Session):
|
||||||
|
raise RuntimeError("订阅搜索延后需要调用方提供同步 Session")
|
||||||
|
task = self._db.execute(
|
||||||
|
select(SubscriptionSearchTask).where(
|
||||||
|
SubscriptionSearchTask.task_id == task_id,
|
||||||
|
SubscriptionSearchTask.state == "running",
|
||||||
|
SubscriptionSearchTask.lease_token == lease_token,
|
||||||
|
)
|
||||||
|
).scalars().first()
|
||||||
|
if task is None:
|
||||||
|
return False
|
||||||
|
if bool(task.cancel_requested) or self._batch_cancel_requested(task.batch_id):
|
||||||
|
return self.finish_task(
|
||||||
|
task_id=task_id,
|
||||||
|
lease_token=lease_token,
|
||||||
|
state="cancelled",
|
||||||
|
error=None,
|
||||||
|
)
|
||||||
|
now = utc_now_text()
|
||||||
|
updated = execute_dml(
|
||||||
|
self._db,
|
||||||
|
update(SubscriptionSearchTask)
|
||||||
|
.where(
|
||||||
|
SubscriptionSearchTask.id == task.id,
|
||||||
|
SubscriptionSearchTask.state == "running",
|
||||||
|
SubscriptionSearchTask.lease_token == lease_token,
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
state="queued",
|
||||||
|
phase="waiting_site_budget",
|
||||||
|
current_site_id=None,
|
||||||
|
lease_owner=None,
|
||||||
|
lease_token=None,
|
||||||
|
lease_expires_at=None,
|
||||||
|
available_at=available_at,
|
||||||
|
updated_at=now,
|
||||||
|
finished_at=None,
|
||||||
|
last_error=None,
|
||||||
|
),
|
||||||
|
execution_options={"synchronize_session": False},
|
||||||
|
)
|
||||||
|
if not updated:
|
||||||
|
return False
|
||||||
|
self._refresh_batch(task.batch_id, now=now, error=None)
|
||||||
|
return True
|
||||||
|
|
||||||
def is_cancel_requested(self, task_id: str) -> bool:
|
def is_cancel_requested(self, task_id: str) -> bool:
|
||||||
"""读取任务和批次取消标记。"""
|
"""读取任务和批次取消标记。"""
|
||||||
if not isinstance(self._db, Session):
|
if not isinstance(self._db, Session):
|
||||||
|
|||||||
@@ -379,7 +379,7 @@ def _run_site_pressure_case(
|
|||||||
start_barrier = threading.Barrier(2)
|
start_barrier = threading.Barrier(2)
|
||||||
owners = ("scale-owner-a", "scale-owner-b")
|
owners = ("scale-owner-a", "scale-owner-b")
|
||||||
chains: dict[str, SearchChain] = {}
|
chains: dict[str, SearchChain] = {}
|
||||||
owner_failures: dict[str, list[str]] = {owner: [] for owner in owners}
|
owner_deferrals: dict[str, list[Any]] = {owner: [] for owner in owners}
|
||||||
|
|
||||||
for owner in owners:
|
for owner in owners:
|
||||||
controller = SubscriptionSiteBudgetController(
|
controller = SubscriptionSiteBudgetController(
|
||||||
@@ -391,22 +391,21 @@ def _run_site_pressure_case(
|
|||||||
)
|
)
|
||||||
chain = object.__new__(SearchChain)
|
chain = object.__new__(SearchChain)
|
||||||
chain.configure_subscription_site_budget(controller)
|
chain.configure_subscription_site_budget(controller)
|
||||||
original_record_failure = chain.record_subscription_site_budget_failure
|
original_record_deferred = chain.record_subscription_site_budget_deferred
|
||||||
|
|
||||||
def record_failure(
|
def record_deferred(
|
||||||
error: str,
|
deferral: Any,
|
||||||
*,
|
*,
|
||||||
original=original_record_failure,
|
original=original_record_deferred,
|
||||||
owner_name=owner,
|
owner_name=owner,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""保留 wrapper 聚合失败并同步压力轮次观测。"""
|
"""保留 wrapper 延后事实并同步压力轮次观测。"""
|
||||||
original(error)
|
original(deferral)
|
||||||
owner_failures[owner_name].append(error)
|
owner_deferrals[owner_name].append(deferral)
|
||||||
if "冷却或已有在途搜索" in error:
|
|
||||||
boundary.record_budget_rejection()
|
boundary.record_budget_rejection()
|
||||||
pressure_event.set()
|
pressure_event.set()
|
||||||
|
|
||||||
chain.record_subscription_site_budget_failure = record_failure
|
chain.record_subscription_site_budget_deferred = record_deferred
|
||||||
|
|
||||||
def search_site_torrents(*, _owner=owner, **_kwargs: Any) -> list[str]:
|
def search_site_torrents(*, _owner=owner, **_kwargs: Any) -> list[str]:
|
||||||
"""将 SearchChain 的真实站点请求委托给固定边界。"""
|
"""将 SearchChain 的真实站点请求委托给固定边界。"""
|
||||||
@@ -453,11 +452,10 @@ def _run_site_pressure_case(
|
|||||||
thread.join(timeout=_SITE_PRESSURE_SYNC_TIMEOUT)
|
thread.join(timeout=_SITE_PRESSURE_SYNC_TIMEOUT)
|
||||||
owners_finished = all(not thread.is_alive() for thread in threads)
|
owners_finished = all(not thread.is_alive() for thread in threads)
|
||||||
|
|
||||||
initial_failures = [
|
initial_deferrals = [
|
||||||
error
|
deferral
|
||||||
for owner in owners
|
for owner in owners
|
||||||
for error in owner_failures[owner]
|
for deferral in owner_deferrals[owner]
|
||||||
if "冷却或已有在途搜索" in error
|
|
||||||
]
|
]
|
||||||
successful_owners = [
|
successful_owners = [
|
||||||
owner
|
owner
|
||||||
@@ -469,10 +467,7 @@ def _run_site_pressure_case(
|
|||||||
for owner in owners
|
for owner in owners
|
||||||
if owner in owner_results
|
if owner in owner_results
|
||||||
and not owner_results[owner]
|
and not owner_results[owner]
|
||||||
and any(
|
and owner_deferrals[owner]
|
||||||
"冷却或已有在途搜索" in error
|
|
||||||
for error in owner_failures[owner]
|
|
||||||
)
|
|
||||||
]
|
]
|
||||||
successful_owner = successful_owners[0] if len(successful_owners) == 1 else None
|
successful_owner = successful_owners[0] if len(successful_owners) == 1 else None
|
||||||
blocked_owner = blocked_owners[0] if len(blocked_owners) == 1 else None
|
blocked_owner = blocked_owners[0] if len(blocked_owners) == 1 else None
|
||||||
@@ -482,7 +477,7 @@ def _run_site_pressure_case(
|
|||||||
and boundary.active_at_rejection == 1
|
and boundary.active_at_rejection == 1
|
||||||
and boundary.peak == 1
|
and boundary.peak == 1
|
||||||
and boundary.active == 0
|
and boundary.active == 0
|
||||||
and len(initial_failures) == 1
|
and len(initial_deferrals) == 1
|
||||||
and len(successful_owners) == 1
|
and len(successful_owners) == 1
|
||||||
and len(blocked_owners) == 1
|
and len(blocked_owners) == 1
|
||||||
and all(count == 1 for count in owner_invocations.values())
|
and all(count == 1 for count in owner_invocations.values())
|
||||||
@@ -501,8 +496,8 @@ def _run_site_pressure_case(
|
|||||||
if successful_owner and blocked_owner:
|
if successful_owner and blocked_owner:
|
||||||
winner = chains[successful_owner]
|
winner = chains[successful_owner]
|
||||||
loser = chains[blocked_owner]
|
loser = chains[blocked_owner]
|
||||||
winner.consume_subscription_site_budget_failures()
|
winner.consume_subscription_site_budget_deferrals()
|
||||||
loser.consume_subscription_site_budget_failures()
|
loser.consume_subscription_site_budget_deferrals()
|
||||||
|
|
||||||
if site_id != case.site_count:
|
if site_id != case.site_count:
|
||||||
boundary.set_outcome("success")
|
boundary.set_outcome("success")
|
||||||
@@ -552,11 +547,11 @@ def _run_site_pressure_case(
|
|||||||
page=0,
|
page=0,
|
||||||
)
|
)
|
||||||
cooldown_calls = boundary.calls - calls_before_cooldown
|
cooldown_calls = boundary.calls - calls_before_cooldown
|
||||||
cooldown_failures = loser.consume_subscription_site_budget_failures()
|
cooldown_deferrals = loser.consume_subscription_site_budget_deferrals()
|
||||||
error_cooldown_blocked = bool(
|
error_cooldown_blocked = bool(
|
||||||
not cooldown_result
|
not cooldown_result
|
||||||
and cooldown_calls == 0
|
and cooldown_calls == 0
|
||||||
and cooldown_failures
|
and cooldown_deferrals
|
||||||
)
|
)
|
||||||
error_cooldown_persisted = bool(
|
error_cooldown_persisted = bool(
|
||||||
error_record
|
error_record
|
||||||
@@ -576,7 +571,7 @@ def _run_site_pressure_case(
|
|||||||
and error_cooldown_persisted
|
and error_cooldown_persisted
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
duplicate_site_claims_blocked += len(initial_failures)
|
duplicate_site_claims_blocked += len(initial_deferrals)
|
||||||
successful_site_claims_reused += int(success_reused)
|
successful_site_claims_reused += int(success_reused)
|
||||||
error_cooldown_claims_blocked += int(error_cooldown_blocked)
|
error_cooldown_claims_blocked += int(error_cooldown_blocked)
|
||||||
site_observations.append(
|
site_observations.append(
|
||||||
@@ -589,7 +584,7 @@ def _run_site_pressure_case(
|
|||||||
"request_peak": boundary.peak,
|
"request_peak": boundary.peak,
|
||||||
"request_calls": boundary.calls,
|
"request_calls": boundary.calls,
|
||||||
"request_active_at_rejection": boundary.active_at_rejection,
|
"request_active_at_rejection": boundary.active_at_rejection,
|
||||||
"budget_rejections": len(initial_failures),
|
"budget_rejections": len(initial_deferrals),
|
||||||
"success_reused": success_reused,
|
"success_reused": success_reused,
|
||||||
"error_observed": error_observed,
|
"error_observed": error_observed,
|
||||||
"error_cooldown_blocked": error_cooldown_blocked,
|
"error_cooldown_blocked": error_cooldown_blocked,
|
||||||
|
|||||||
@@ -84,6 +84,19 @@ def test_execution_status_exposes_site_wait_and_cancel_capability():
|
|||||||
assert statuses[1].can_cancel is True
|
assert statuses[1].can_cancel is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_execution_status_exposes_queued_site_wait_without_error():
|
||||||
|
"""重新入队的站点预算冲突应显示等待状态而不是失败。"""
|
||||||
|
repository = _Repository()
|
||||||
|
repository.tasks[2] = _task(2, state="queued", phase="waiting_site_budget")
|
||||||
|
|
||||||
|
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,)))
|
||||||
|
|
||||||
|
assert statuses[2].state == "waiting_site_budget"
|
||||||
|
assert statuses[2].phase == "waiting_site_budget"
|
||||||
|
assert statuses[2].error is None
|
||||||
|
assert statuses[2].can_cancel is True
|
||||||
|
|
||||||
|
|
||||||
def test_failed_search_exposes_safe_error():
|
def test_failed_search_exposes_safe_error():
|
||||||
"""搜索失败文本必须压平且不暴露内部错误细节。"""
|
"""搜索失败文本必须压平且不暴露内部错误细节。"""
|
||||||
repository = _Repository()
|
repository = _Repository()
|
||||||
|
|||||||
@@ -52,8 +52,8 @@ class _MixedBudgetRepository:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def test_cooled_site_does_not_block_independent_site_or_hide_batch_failure():
|
def test_cooled_site_does_not_block_independent_site_or_mark_batch_failed():
|
||||||
"""慢站点跳过后其它站点仍完成,调用方同时收到可聚合失败。"""
|
"""冷却站点延后后其它站点仍完成,调用方只收到可重新入队的事实。"""
|
||||||
repository = _MixedBudgetRepository()
|
repository = _MixedBudgetRepository()
|
||||||
budget = SubscriptionSiteBudget(
|
budget = SubscriptionSiteBudget(
|
||||||
repository=repository,
|
repository=repository,
|
||||||
@@ -91,7 +91,9 @@ def test_cooled_site_does_not_block_independent_site_or_hide_batch_failure():
|
|||||||
|
|
||||||
assert results == [2]
|
assert results == [2]
|
||||||
assert repository.finished_sites == [2]
|
assert repository.finished_sites == [2]
|
||||||
|
deferrals = chain.consume_subscription_site_budget_deferrals()
|
||||||
|
assert len(deferrals) == 1
|
||||||
|
assert deferrals[0].site_id == 1
|
||||||
failures = chain.consume_subscription_site_budget_failures()
|
failures = chain.consume_subscription_site_budget_failures()
|
||||||
assert len(failures) == 1
|
assert failures == ()
|
||||||
assert "站点 1" in failures[0]
|
|
||||||
assert progress.values[-1] == 100
|
assert progress.values[-1] == 100
|
||||||
|
|||||||
@@ -179,22 +179,21 @@ def test_scale_validator_rejects_fake_site_pressure(monkeypatch, tmp_path, mutat
|
|||||||
|
|
||||||
|
|
||||||
def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path):
|
def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path):
|
||||||
"""失败已登记但 wrapper 尚未返回时,压力门禁必须拒绝并暴露未收口 owner。"""
|
"""延后已登记但 wrapper 尚未返回时,压力门禁必须拒绝并暴露未收口 owner。"""
|
||||||
original_wrapper = SearchChain._search_site_torrents_with_budget
|
original_wrapper = SearchChain._search_site_torrents_with_budget
|
||||||
release_stalled = threading.Event()
|
release_stalled = threading.Event()
|
||||||
threads_before = set(threading.enumerate())
|
threads_before = set(threading.enumerate())
|
||||||
|
|
||||||
def stall_after_rejection(self, *, site, keyword, mtype, page):
|
def stall_after_rejection(self, *, site, keyword, mtype, page):
|
||||||
"""模拟预算拒绝已记录、调用方却未取得返回值的挂起路径。"""
|
"""模拟预算拒绝已记录、调用方却未取得返回值的挂起路径。"""
|
||||||
original_record_failure = self.record_subscription_site_budget_failure
|
original_record_deferred = self.record_subscription_site_budget_deferred
|
||||||
|
|
||||||
def record_failure_and_stall(error: str) -> None:
|
def record_deferred_and_stall(deferral) -> None:
|
||||||
"""在拒绝已登记后阻塞原始 wrapper 的返回。"""
|
"""在拒绝已登记后阻塞原始 wrapper 的返回。"""
|
||||||
original_record_failure(error)
|
original_record_deferred(deferral)
|
||||||
if "冷却或已有在途搜索" in error:
|
|
||||||
release_stalled.wait()
|
release_stalled.wait()
|
||||||
|
|
||||||
self.record_subscription_site_budget_failure = record_failure_and_stall
|
self.record_subscription_site_budget_deferred = record_deferred_and_stall
|
||||||
try:
|
try:
|
||||||
return original_wrapper(
|
return original_wrapper(
|
||||||
self,
|
self,
|
||||||
@@ -204,7 +203,7 @@ def test_scale_validator_rejects_unfinished_site_wrapper(monkeypatch, tmp_path):
|
|||||||
page=page,
|
page=page,
|
||||||
)
|
)
|
||||||
finally:
|
finally:
|
||||||
self.record_subscription_site_budget_failure = original_record_failure
|
self.record_subscription_site_budget_deferred = original_record_deferred
|
||||||
|
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
SearchChain,
|
SearchChain,
|
||||||
|
|||||||
@@ -13,13 +13,17 @@ from sqlalchemy.orm import sessionmaker
|
|||||||
from app.application.site.observation import report_site_search_outcome
|
from app.application.site.observation import report_site_search_outcome
|
||||||
from app.application.subscription.contract import SubscriptionSnapshot
|
from app.application.subscription.contract import SubscriptionSnapshot
|
||||||
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
from app.application.subscription.execution import SubscriptionExecutionAdmission
|
||||||
from app.application.subscription.sitebudget import SubscriptionSearchCancelled
|
from app.application.subscription.sitebudget import (
|
||||||
|
SubscriptionSearchCancelled,
|
||||||
|
SubscriptionSearchDeferred,
|
||||||
|
)
|
||||||
from app.chain.search.facade import SearchChain
|
from app.chain.search.facade import SearchChain
|
||||||
from app.chain.subscribe import search as subscribe_search
|
from app.chain.subscribe import search as subscribe_search
|
||||||
from app.chain.subscribe.facade import SubscribeChain
|
from app.chain.subscribe.facade import SubscribeChain
|
||||||
from app.chain.subscribe.search import _search_task_available_at
|
from app.chain.subscribe.search import _search_task_available_at
|
||||||
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
from app.db.adapters.subscriptionsearch import TransactionalSubscriptionSearchRepository
|
||||||
from app.db.base import Base
|
from app.db.base import Base
|
||||||
|
from app.db.models.subscriptionsearch import SubscriptionSearchTask
|
||||||
from app.modules.indexer import IndexerModule
|
from app.modules.indexer import IndexerModule
|
||||||
from app.schemas.types import MediaType
|
from app.schemas.types import MediaType
|
||||||
|
|
||||||
@@ -249,6 +253,37 @@ def test_fallback_queue_continues_after_one_subscription_failure(tmp_path, monke
|
|||||||
assert chain._subscription_execution_admission.release(lease) is True
|
assert chain._subscription_execution_admission.release(lease) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_site_budget_conflict_requeues_task_without_batch_failure(tmp_path, monkeypatch):
|
||||||
|
"""站点预算冲突应自动排队重试,不能把订阅批次置为失败。"""
|
||||||
|
subscribe = _subscribe(5)
|
||||||
|
chain = _chain(tmp_path, [subscribe])
|
||||||
|
_make_tasks_ready(monkeypatch)
|
||||||
|
retry_at = (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(timespec="seconds")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
chain,
|
||||||
|
"_process_search_subscription",
|
||||||
|
Mock(side_effect=SubscriptionSearchDeferred(retry_at=retry_at, site_ids=(31,))),
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
|
||||||
|
batch_id = chain.search(state="R")
|
||||||
|
|
||||||
|
batch = chain.get_search_batch(batch_id)
|
||||||
|
assert batch.state == "queued"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert batch.last_error is None
|
||||||
|
with chain.subscription_search_repository._session_factory() as session:
|
||||||
|
task = session.query(SubscriptionSearchTask).filter_by(
|
||||||
|
subscription_id=subscribe.id,
|
||||||
|
).one()
|
||||||
|
assert task.state == "queued"
|
||||||
|
assert task.phase == "waiting_site_budget"
|
||||||
|
assert task.available_at == retry_at
|
||||||
|
assert task.last_error is None
|
||||||
|
assert chain.subscription_search_repository.claim_next(owner="worker-after-retry") is None
|
||||||
|
|
||||||
|
|
||||||
def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch):
|
def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch):
|
||||||
"""Search INFO 只保留轮次摘要,并携带任务终态与耗时字段。"""
|
"""Search INFO 只保留轮次摘要,并携带任务终态与耗时字段。"""
|
||||||
subscribes = [_subscribe(50), _subscribe(51)]
|
subscribes = [_subscribe(50), _subscribe(51)]
|
||||||
|
|||||||
@@ -132,6 +132,47 @@ def test_search_queue_phase_update_requires_current_lease(tmp_path):
|
|||||||
) is True
|
) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_search_queue_defers_site_budget_conflict_until_retry_time(tmp_path):
|
||||||
|
"""站点预算冲突应释放任务租约并保留同一任务等待后续恢复。"""
|
||||||
|
repository, engine = _repository(tmp_path)
|
||||||
|
enqueued = repository.enqueue(subscription_ids=(31,), source="fallback", priority=10)
|
||||||
|
running = repository.claim_next(owner="worker-a")
|
||||||
|
retry_at = (datetime.now(timezone.utc) + timedelta(minutes=5)).isoformat(timespec="seconds")
|
||||||
|
|
||||||
|
assert repository.defer_task(
|
||||||
|
task_id=running.task_id,
|
||||||
|
lease_token=running.lease_token,
|
||||||
|
available_at=retry_at,
|
||||||
|
) is True
|
||||||
|
|
||||||
|
batch = repository.get_batch(enqueued.batch.batch_id)
|
||||||
|
assert batch.state == "queued"
|
||||||
|
assert batch.finished_count == 0
|
||||||
|
assert batch.failed_count == 0
|
||||||
|
assert repository.claim_next(owner="worker-b") is None
|
||||||
|
|
||||||
|
with Session(engine) as session:
|
||||||
|
task = session.execute(
|
||||||
|
select(SubscriptionSearchTask).where(
|
||||||
|
SubscriptionSearchTask.task_id == running.task_id
|
||||||
|
)
|
||||||
|
).scalar_one()
|
||||||
|
assert task.state == "queued"
|
||||||
|
assert task.phase == "waiting_site_budget"
|
||||||
|
assert task.available_at == retry_at
|
||||||
|
assert task.last_error is None
|
||||||
|
session.execute(
|
||||||
|
update(SubscriptionSearchTask)
|
||||||
|
.where(SubscriptionSearchTask.task_id == running.task_id)
|
||||||
|
.values(available_at="1970-01-01T00:00:00+00:00")
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
recovered = repository.claim_next(owner="worker-c")
|
||||||
|
assert recovered.task_id == running.task_id
|
||||||
|
assert recovered.attempt_count == 2
|
||||||
|
|
||||||
|
|
||||||
def test_search_queue_cancel_finishes_queued_and_running_tasks(tmp_path):
|
def test_search_queue_cancel_finishes_queued_and_running_tasks(tmp_path):
|
||||||
"""取消立即终止未发请求任务,运行中任务在租约边界收口。"""
|
"""取消立即终止未发请求任务,运行中任务在租约边界收口。"""
|
||||||
repository, engine = _repository(tmp_path)
|
repository, engine = _repository(tmp_path)
|
||||||
|
|||||||
@@ -243,7 +243,7 @@ def test_skipped_search_releases_budget_without_external_interval():
|
|||||||
|
|
||||||
|
|
||||||
def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
||||||
"""错误冷却中的站点返回空页并记录聚合失败,而非阻塞 provider。"""
|
"""错误冷却中的站点返回空页并记录延后,而非阻塞 provider 或制造失败。"""
|
||||||
repository = _WaitingRepository()
|
repository = _WaitingRepository()
|
||||||
metrics = SubscriptionSiteBudgetMetrics()
|
metrics = SubscriptionSiteBudgetMetrics()
|
||||||
budget = SubscriptionSiteBudget(
|
budget = SubscriptionSiteBudget(
|
||||||
@@ -265,9 +265,12 @@ def test_search_provider_reports_cooled_site_without_blocking_other_results():
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert result == []
|
assert result == []
|
||||||
|
deferrals = chain.consume_subscription_site_budget_deferrals()
|
||||||
|
assert len(deferrals) == 1
|
||||||
|
assert deferrals[0].site_id == 11
|
||||||
|
assert deferrals[0].retry_at
|
||||||
failures = chain.consume_subscription_site_budget_failures()
|
failures = chain.consume_subscription_site_budget_failures()
|
||||||
assert len(failures) == 1
|
assert failures == ()
|
||||||
assert "站点 11" in failures[0]
|
|
||||||
snapshot = metrics.snapshot()
|
snapshot = metrics.snapshot()
|
||||||
assert snapshot.request_count == 0
|
assert snapshot.request_count == 0
|
||||||
assert snapshot.cooldown_skip_count == 1
|
assert snapshot.cooldown_skip_count == 1
|
||||||
|
|||||||
Reference in New Issue
Block a user