feat(subscribe): improve search scheduling feedback

This commit is contained in:
jxxghp
2026-09-05 17:23:26 +08:00
parent e003a31064
commit 0a732be485
33 changed files with 1080 additions and 365 deletions
+38 -28
View File
@@ -42,7 +42,10 @@ from app.application.subscription.mutation import (
SubscriptionMutationService,
)
from app.application.subscription.query import SubscriptionQueryService
from app.application.subscription.search import SearchSubscriptionsCommand
from app.application.subscription.search import (
SearchSubscriptionsCommand,
SubscriptionSearchSubmission,
)
from app.application.subscription.status import (
SubscriptionExecutionReadRepository,
SubscriptionExecutionStatusService,
@@ -120,24 +123,14 @@ def get_delete_subscriptions_by_identity_command(
)
def _start_subscription_search_batch(
subscribe_ids: tuple[int, ...] | None,
state: str | None,
def _resume_submitted_subscription_search(
subscribe_ids: tuple[int, ...],
) -> None:
"""把一个请求的搜索目标作为同一调度任务提交"""
if subscribe_ids is None:
"""唤醒搜索队列,并优先处理本次手工选择的订阅"""
start_scheduler_job(
"subscribe_search",
sid=None,
state=state,
manual=True,
)
return
start_scheduler_job(
"subscribe_search",
sids=subscribe_ids,
state=None,
manual=True,
"subscribe_search_queue",
limit=max(1, len(subscribe_ids)),
manual_sids=subscribe_ids,
)
@@ -146,23 +139,40 @@ def get_search_subscriptions_command(
db: AsyncSession = Depends(get_async_session),
runtime: HostRuntime = Depends(get_host_runtime),
) -> SearchSubscriptionsCommand:
"""组装手工订阅搜索用例,并把调度延迟到响应后的后台任务"""
"""组装手工搜索用例,请求内只入队,实际搜索交给后台继续"""
def schedule_search(
subscribe_ids: tuple[int, ...] | None,
state: str | None,
) -> None:
"""把当前用户的搜索目标提交为一个顺序后台批次。"""
resolve_background_task_registry(task_registry).create_sync(
_start_subscription_search_batch,
registry = resolve_background_task_registry(task_registry)
search_repository = get_subscription_search_repository(runtime)
async def submit_search(
subscribe_ids: tuple[int, ...],
single: bool,
) -> SubscriptionSearchSubmission:
"""在线程 owner 中完成轻量入队,并返回前端可立即跟踪的批次。"""
enqueue_task = registry.create_sync(
search_repository.enqueue,
subscription_ids=subscribe_ids,
source="manual",
priority=120 if single else 100,
owner="api.subscribe.search.enqueue",
)
enqueued = await enqueue_task
registry.create_sync(
_resume_submitted_subscription_search,
subscribe_ids,
state,
owner="api.subscribe.search",
owner="api.subscribe.search.run",
)
return SubscriptionSearchSubmission(
batch_ids=enqueued.active_batch_ids,
target_count=len(subscribe_ids),
queued_count=enqueued.created_count,
ongoing_count=enqueued.coalesced_count,
single=single,
)
return SearchSubscriptionsCommand(
repository=runtime.subscription.repository(db),
schedule_search=schedule_search,
submit_search=submit_search,
)
+6 -6
View File
@@ -32,7 +32,7 @@ async def _accessible_subscription_ids(
@router.get( # type: ignore[misc]
"/execution/batches",
summary="订阅搜索批次状态",
summary="订阅搜索进度",
response_model=List[_SchemaSubscriptionBatchStatus],
)
async def list_subscription_execution_batches(
@@ -52,7 +52,7 @@ async def list_subscription_execution_batches(
@router.get( # type: ignore[misc]
"/execution/batches/{batch_id}",
summary="订阅搜索批次",
summary="看一次订阅搜索",
response_model=_SchemaSubscriptionBatchStatus,
)
async def get_subscription_execution_batch(
@@ -68,13 +68,13 @@ async def get_subscription_execution_batch(
accessible_subscription_ids=accessible_ids,
)
if batch is None:
raise HTTPException(status_code=404, detail="订阅搜索批次不存在")
raise HTTPException(status_code=404, detail="没有找到这次搜索,请刷新后重试")
return _SchemaSubscriptionBatchStatus.model_validate(batch)
@router.put( # type: ignore[misc]
"/execution/batches/{batch_id}/cancel",
summary="取消订阅搜索批次",
summary="停止一次订阅搜索",
response_model=_SchemaResponse[None],
)
async def cancel_subscription_execution_batch(
@@ -90,9 +90,9 @@ async def cancel_subscription_execution_batch(
accessible_subscription_ids=accessible_ids,
)
if batch is None:
return _SchemaResponse(success=False, message="订阅搜索批次不存在")
return _SchemaResponse(success=False, message="没有找到这次搜索,请刷新后重试")
cancelled = await status_service.request_cancel(batch_id)
return _SchemaResponse(
success=bool(cancelled),
message="" if cancelled else "订阅搜索批次已结束或无法取消",
message="" if cancelled else "这次搜索已经结束,暂时无法停止",
)
+59 -9
View File
@@ -22,12 +22,50 @@ from app.application.subscription.mutation import (
from app.application.subscription.search import (
SearchSubscriptionsCommand,
SubscribeSearchActor,
SubscriptionSearchSubmission,
)
from app.schemas.response import Response
from app.schemas.subscribe import (
SubscriptionSearchSubmission as SubscriptionSearchSubmissionSchema,
)
router = ResponseAPIRouter()
def _search_submission_message(submission: SubscriptionSearchSubmission) -> str:
"""把搜索安排结果转换为用户能直接理解的提示。"""
if submission.target_count == 0:
return "没有需要搜索的订阅"
if submission.queued_count == 0:
return (
"这个订阅已经在搜索中,请稍候"
if submission.single
else f"{submission.ongoing_count} 个订阅已经在搜索中,无需重复提交"
)
if submission.ongoing_count:
return (
f"已安排 {submission.queued_count} 个订阅搜索,"
f"另有 {submission.ongoing_count} 个正在处理中"
)
if submission.single:
return "已安排搜索,很快开始"
return f"已安排 {submission.queued_count} 个订阅搜索,系统会依次处理"
def _search_submission_schema(
submission: SubscriptionSearchSubmission,
) -> SubscriptionSearchSubmissionSchema:
"""构造稳定的手工搜索响应数据。"""
return SubscriptionSearchSubmissionSchema(
batch_id=submission.batch_id,
batch_ids=list(submission.batch_ids),
target_count=submission.target_count,
queued_count=submission.queued_count,
ongoing_count=submission.ongoing_count,
single=submission.single,
)
@router.get( # type: ignore[misc]
"/refresh",
summary="刷新订阅(兼容入口)",
@@ -97,38 +135,46 @@ def check_subscribes(
@router.get( # type: ignore[misc]
"/search",
summary="搜索所有订阅(兼容入口)",
response_model=Response[None],
response_model=Response[SubscriptionSearchSubmissionSchema],
include_in_schema=False,
deprecated=True,
)
@router.post( # type: ignore[misc]
"/search", summary="搜索所有订阅", response_model=Response[None]
"/search",
summary="搜索所有订阅",
response_model=Response[SubscriptionSearchSubmissionSchema],
)
async def search_subscribes(
command: SearchSubscriptionsCommand = Depends(get_search_subscriptions_command),
current_user: ApiPrincipal = Depends(get_current_active_user_async),
) -> Any:
"""搜索当前用户可管理的全部订阅。"""
await command.execute(
submission = await command.execute(
SubscribeSearchActor(
username=current_user.name,
is_superuser=current_user.is_superuser,
)
)
return Response(success=True)
if submission is None:
return Response(success=False, message="没有需要搜索的订阅")
return Response(
success=True,
message=_search_submission_message(submission),
data=_search_submission_schema(submission),
)
@router.get( # type: ignore[misc]
"/search/{subscribe_id}",
summary="搜索订阅(兼容入口)",
response_model=Response[None],
response_model=Response[SubscriptionSearchSubmissionSchema],
include_in_schema=False,
deprecated=True,
)
@router.post( # type: ignore[misc]
"/search/{subscribe_id}",
summary="搜索订阅",
response_model=Response[None],
response_model=Response[SubscriptionSearchSubmissionSchema],
)
async def search_subscribe(
subscribe_id: int,
@@ -136,13 +182,17 @@ async def search_subscribe(
current_user: ApiPrincipal = Depends(get_current_active_user_async),
) -> Any:
"""根据订阅编号搜索一个订阅。"""
found = await command.execute(
submission = await command.execute(
SubscribeSearchActor(
username=current_user.name,
is_superuser=current_user.is_superuser,
),
subscribe_id=subscribe_id,
)
if not found:
if submission is None:
return Response(success=False, message="订阅不存在")
return Response(success=True)
return Response(
success=True,
message=_search_submission_message(submission),
data=_search_submission_schema(submission),
)
+6 -2
View File
@@ -596,8 +596,12 @@ class SubscriptionStagingPort(Protocol):
"""异步按媒体身份读取删除候选快照。"""
...
async def list_search_ids(self, username: str, state: str) -> builtins.list[int]:
"""异步读取用户可搜索订阅主键。"""
async def list_search_ids(
self,
username: Optional[str],
state: str,
) -> builtins.list[int]:
"""异步读取用户或管理员全局范围内可搜索的订阅主键。"""
...
async def stage_delete(self, subscribe_id: int) -> None:
+7 -2
View File
@@ -125,11 +125,13 @@ def handle_subscription_search_deferred(
deferred: SubscriptionSearchDeferred,
record: Callable[..., None],
) -> None:
"""把站点预算冲突重新入队,并记录为可恢复而非失败的任务结果"""
"""把站点暂时不可用的任务重新入队,而不是记录为搜索失败"""
requeued = queue.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=deferred.retry_at,
phase="waiting_site_budget",
message="站点暂时忙,系统会自动继续搜索",
)
if requeued:
record("requeued", "site_budget_deferred")
@@ -187,6 +189,7 @@ class SearchEnqueueResult:
batch: SearchBatchSnapshot
created_count: int
coalesced_count: int
active_batch_ids: tuple[str, ...]
class SubscriptionSearchRepository(Protocol):
@@ -245,8 +248,10 @@ class SubscriptionSearchRepository(Protocol):
task_id: str,
lease_token: str,
available_at: str,
phase: str = "waiting_site_budget",
message: Optional[str] = None,
) -> bool:
"""把临时站点预算冲突任务退回队列,并设置下一次领取时间"""
"""把临时不可执行任务退回队列,并保留用户可理解的等待原因"""
...
def is_cancel_requested(self, task_id: str) -> bool:
+94 -50
View File
@@ -16,30 +16,73 @@ from app.application.subscription.sitebudget import SubscriptionSiteBudgetMetric
SearchTaskOutcome = Literal["completed", "skipped", "failed", "cancelled", "requeued"]
_SEARCH_SOURCE_NAMES = {
"manual": "手动订阅搜索",
"targeted": "指定订阅搜索",
"new": "新订阅自动搜索",
"fallback": "订阅定时检查",
"resume": "等待中的订阅搜索",
"inline": "订阅搜索",
}
_SEARCH_STATE_NAMES = {
"completed": "已完成",
"failed": "部分订阅没有完成",
"cancelled": "已停止",
"skipped": "部分订阅这次未搜索",
"queued": "部分订阅稍后继续",
"running": "仍在处理中",
"cancelling": "正在停止",
"stopped": "部分订阅稍后继续",
}
_MATCH_STATE_NAMES = {
"completed": "检查完成",
"failed": "部分订阅没有完成",
"cancelled": "已停止",
"skipped": "部分订阅这次未检查",
"stopped": "已停止,部分订阅这次未检查",
}
def _search_source_name(source: str) -> str:
"""返回搜索来源对应的可读名称。"""
return _SEARCH_SOURCE_NAMES.get(source, "订阅搜索")
def _search_state_name(state: str) -> str:
"""返回搜索结束状态对应的可读说明。"""
return _SEARCH_STATE_NAMES.get(state, "已结束")
def _match_state_name(state: str) -> str:
"""返回订阅资源检查状态对应的可读说明。"""
return _MATCH_STATE_NAMES.get(state, "已结束")
def batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
"""把批次聚合终态转为兼容进度文案。"""
if batch is None:
return "订阅搜索任务已提交"
return "搜索已安排"
if batch.state == "failed":
return "订阅搜索完成,部分任务失败"
return "搜索结束,部分订阅没有完成"
if batch.state == "cancelled":
return "订阅搜索已取消"
return "搜索已停止"
if batch.state == "skipped":
return "订阅搜索完成,部分任务本轮已跳过"
return "搜索结束,部分订阅这次未搜索"
if batch.state in {"queued", "running", "cancelling"}:
return "订阅搜索任务已排队"
return "搜索已安排,系统正在依次处理"
if batch.skipped_count:
return "订阅搜索完成,部分任务本轮已跳过"
return "订阅搜索完成"
return "搜索结束,部分订阅这次未搜索"
return "搜索完成"
def inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]:
"""返回兼容搜索的真实终态文案与计数。"""
text = (
"订阅搜索完成"
"搜索完成"
if finished == total
else "订阅搜索结束,部分订阅本轮未执行或未完成"
else "搜索结束,部分订阅这次没有完成"
)
return text, {"total": total, "finished": finished}
@@ -77,14 +120,14 @@ def finish_returned_search_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="执行截止时间晚于下载提交边界,已按实际结果完成",
error="下载已经提交,虽然搜索用时较长,结果仍然有效",
)
return subscription_id, "completed", "ttl_timeout"
queue.finish_task(
task_id=task_id,
lease_token=lease_token,
state="failed",
error="订阅执行已超过协作截止时间",
error="这次搜索用时过长,已停止,可稍后重试",
)
return None, "failed", "ttl_timeout"
if system_stopped:
@@ -93,7 +136,7 @@ def finish_returned_search_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="停机请求晚于下载提交边界,已按实际结果完成",
error="下载已经提交,系统停止后仍保留这次结果",
)
return subscription_id, "completed", "system_stop"
queue.release_task(task_id=task_id, lease_token=lease_token)
@@ -104,7 +147,7 @@ def finish_returned_search_task(
task_id=task_id,
lease_token=lease_token,
state="completed",
error="取消请求晚于下载提交边界,已按实际结果完成",
error="下载已经提交,无法撤回,已保留这次结果",
)
return subscription_id, "completed", "cancelled"
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
@@ -173,26 +216,31 @@ class MatchExecutionSummary:
return "completed"
def start_log(self) -> str:
"""构造 Match 轮次开始日志。"""
"""构造不暴露内部标记的资源检查开始日志。"""
return (
"订阅治理轮次开始: operation=match "
f"run_id={self.run_id} subscriptions={self.total} "
f"sites={self.site_count} candidates={self.candidate_count}"
f"开始检查订阅资源,共 {self.total} 个订阅、{self.candidate_count} 个资源,"
f"来自 {self.site_count} 个站点。"
)
def finish_log(self) -> str:
"""构造 Match 轮次结束日志,completed 表示任务处理完成而非订阅成功"""
elapsed_ms = (time.perf_counter() - self.started_at) * 1000
return (
"订阅治理轮次结束: operation=match "
f"run_id={self.run_id} state={self.state} subscriptions={self.total} "
f"processed={self.finished} task_completed={self.completed} "
f"task_skipped={self.skipped} task_failed={self.failed} "
f"admission_conflicts={self.admission_conflicts} cancelled={self.cancelled} "
f"ttl_timeouts={self.ttl_timeouts} release_failures={self.release_failures} "
f"sites={self.site_count} candidates={self.candidate_count} "
f"duration_ms={elapsed_ms:.1f}"
"""构造可直接阅读的资源检查结束日志"""
elapsed_seconds = time.perf_counter() - self.started_at
message = (
f"订阅资源检查结束:{_match_state_name(self.state)}"
f"本次检查 {self.finished}/{self.total} 个订阅,完成 {self.completed} 个,"
f"这次未检查 {self.skipped} 个,失败 {self.failed} 个;"
f"共检查 {self.candidate_count} 个资源,来自 {self.site_count} 个站点,"
f"用时 {elapsed_seconds:.1f} 秒。"
)
if self.admission_conflicts:
message += f"其中 {self.admission_conflicts} 个订阅正在处理中,本次没有重复检查。"
if self.cancelled:
message += f"另有 {self.cancelled} 个订阅已停止。"
if self.ttl_timeouts:
message += f"另有 {self.ttl_timeouts} 个订阅检查时间过长,已停止。"
if self.release_failures:
message += f"另有 {self.release_failures} 个订阅的搜索状态没有正常恢复,系统稍后会继续检查。"
return message
def as_data(self, current: Optional[int] = None) -> dict[str, int]:
"""构造供进度消费者读取的实际计数。"""
@@ -256,15 +304,14 @@ class SearchExecutionSummary:
self.ttl_timeouts += 1
def start_log(self) -> str:
"""构造 Search 轮次开始日志。"""
return (
"订阅治理轮次开始: operation=search "
f"run_id={self.run_id} batch_id={self.batch_id or '-'} source={self.source} "
f"subscriptions={self.requested} coalesced={self.coalesced}"
)
"""构造不暴露内部术语的 Search 开始日志。"""
message = f"开始{_search_source_name(self.source)},共 {self.requested} 个订阅"
if self.coalesced:
message += f",其中 {self.coalesced} 个已经在处理中"
return f"{message}"
def finish_log(self, batch: Optional[SearchBatchSnapshot] = None) -> str:
"""构造 Search 轮次结束日志,任务完成与订阅完成保持分离"""
"""构造可直接阅读的 Search 结束日志。"""
sites = self.site_metrics.snapshot()
if self.round_failed:
state = "failed"
@@ -282,19 +329,16 @@ class SearchExecutionSummary:
state = "skipped"
else:
state = "completed"
elapsed_ms = (time.perf_counter() - self.started_at) * 1000
return (
"订阅治理轮次结束: operation=search "
f"run_id={self.run_id} batch_id={self.batch_id or '-'} source={self.source} "
f"state={state} subscriptions={self.requested} processed={self.processed} "
f"task_completed={self.completed} task_skipped={self.skipped} "
f"task_failed={self.failed} task_cancelled={self.cancelled} task_requeued={self.requeued} "
f"admission_conflicts={self.admission_conflicts} cancelled={self.cancelled} "
f"ttl_timeouts={self.ttl_timeouts} consumer_conflicts={self.consumer_conflicts} "
f"round_failed={int(self.round_failed)} "
f"sites={sites.site_count} site_requests={sites.request_count} "
f"site_failures={sites.failure_count} site_cooldown_skips={sites.cooldown_skip_count} "
f"candidates={sites.candidate_count} cooldown_seconds={sites.cooldown_seconds:.1f} "
f"release_failures={self.release_failures + sites.release_failure_count} "
f"duration_ms={elapsed_ms:.1f}"
elapsed_seconds = time.perf_counter() - self.started_at
message = (
f"{_search_source_name(self.source)}结束:{_search_state_name(state)}"
f"本次处理 {self.processed}/{self.requested} 个,完成 {self.completed} 个,"
f"这次未搜索 {self.skipped} 个,失败 {self.failed} 个,"
f"稍后继续 {self.requeued} 个,已停止 {self.cancelled} 个;"
f"访问 {sites.site_count} 个站点,发出 {sites.request_count} 次请求,"
f"找到 {sites.candidate_count} 个资源,用时 {elapsed_seconds:.1f} 秒。"
)
unfinished_cleanup = self.release_failures + sites.release_failure_count
if unfinished_cleanup:
message += f"另有 {unfinished_cleanup} 个搜索状态没有正常恢复,系统稍后会继续处理。"
return message
+40 -19
View File
@@ -1,7 +1,8 @@
"""手工订阅搜索应用用例。"""
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from typing import Callable
from typing import Optional
from app.application.subscription.contract import (
SubscribeDeletionCandidate,
@@ -17,45 +18,65 @@ class SubscribeSearchActor:
is_superuser: bool
SubscribeSearchScheduler = Callable[[tuple[int, ...] | None, str | None], None]
@dataclass(frozen=True, slots=True)
class SubscriptionSearchSubmission:
"""一次手工搜索请求的入队结果。"""
batch_ids: tuple[str, ...]
target_count: int
queued_count: int
ongoing_count: int
single: bool
@property
def batch_id(self) -> Optional[str]:
"""返回最适合前端立即跟踪的批次编号。"""
return self.batch_ids[0] if self.batch_ids else None
SubscribeSearchSubmitter = Callable[
[tuple[int, ...], bool],
Awaitable[SubscriptionSearchSubmission],
]
class SearchSubscriptionsCommand:
"""按用户权限生成并提交手工订阅搜索任务"""
"""按用户权限读取目标,并把手工搜索轻量提交到持久队列"""
def __init__(
self,
repository: SubscriptionStagingPort,
schedule_search: SubscribeSearchScheduler,
submit_search: SubscribeSearchSubmitter,
) -> None:
"""注入订阅读取端口和后台任务提交端口。"""
"""注入订阅读取端口和持久队列提交端口。"""
self._repository = repository
self._schedule_search = schedule_search
self._submit_search = submit_search
async def execute(
self,
actor: SubscribeSearchActor,
subscribe_id: int | None = None,
) -> bool:
"""提交单条或当前用户全部可搜索订阅,返回目标是否存在"""
) -> Optional[SubscriptionSearchSubmission]:
"""提交单条或当前用户全部可搜索订阅,目标不可访问时返回空"""
if subscribe_id is not None:
candidate = await self._repository.get_candidate(subscribe_id)
if not self._can_access(candidate, actor):
return False
self._schedule_search((subscribe_id,), None)
return True
if actor.is_superuser:
self._schedule_search(None, "R")
return True
return None
return await self._submit_search((subscribe_id,), True)
subscribe_ids = await self._repository.list_search_ids(
actor.username,
None if actor.is_superuser else actor.username,
"R",
)
if subscribe_ids:
self._schedule_search(tuple(subscribe_ids), None)
return True
if not subscribe_ids:
return SubscriptionSearchSubmission(
batch_ids=(),
target_count=0,
queued_count=0,
ongoing_count=0,
single=False,
)
return await self._submit_search(tuple(subscribe_ids), False)
@staticmethod
def _can_access(
+16 -6
View File
@@ -27,20 +27,28 @@ class SubscriptionSearchDeferred(RuntimeError):
"""表示订阅搜索未失败,而是应在站点预算可用后重新入队。"""
def __init__(self, *, retry_at: str, site_ids: tuple[int, ...]) -> None:
"""保存队列恢复所需的时间和冲突站点,避免把临时冲突写成错误。"""
super().__init__(f"订阅搜索已延后,站点预算最早可重试:{retry_at}")
"""保存队列恢复所需的时间和站点,避免把临时等待写成错误。"""
super().__init__("站点暂时忙,系统会自动继续搜索")
self.retry_at = retry_at
self.site_ids = site_ids
class SubscriptionSiteBudgetUnavailable(RuntimeError):
"""表示站点预算暂时不可用,调用方应记录为延后而非失败。"""
"""表示站点暂时不可用,调用方应记录为等待而非失败。"""
def __init__(self, *, site_id: int, retry_at: str) -> None:
def __init__(
self,
*,
site_id: int,
retry_at: str,
wait_reason: Optional[str] = None,
) -> None:
"""保存站点和下一次可尝试时间,供订阅队列恢复。"""
super().__init__(f"站点 {site_id} 冷却或已有在途搜索,最早可重试:{retry_at}")
message = "站点正在处理其他搜索,稍后会自动继续" if wait_reason == "busy" else "站点暂时不可用,稍后会自动重试"
super().__init__(message)
self.site_id = site_id
self.retry_at = retry_at
self.wait_reason = wait_reason
@dataclass(frozen=True, slots=True)
@@ -52,6 +60,7 @@ class SiteBudgetClaim:
retry_at: str
consecutive_failures: int
lease_token: Optional[str] = None
wait_reason: Optional[str] = None
@dataclass(frozen=True, slots=True)
@@ -195,6 +204,7 @@ class SubscriptionSiteBudget:
raise SubscriptionSiteBudgetUnavailable(
site_id=site_id,
retry_at=claim.retry_at,
wait_reason=claim.wait_reason,
)
def _report_phase(self, phase: str, site_id: Optional[int]) -> None:
@@ -251,4 +261,4 @@ class SubscriptionSiteBudget:
def _raise_if_cancelled(self) -> None:
"""在创建站点租约前传播取消或停机。"""
if self._stop_state.is_system_stopped or self._cancelled():
raise SubscriptionSearchCancelled("订阅搜索已取消")
raise SubscriptionSearchCancelled("搜索已停止")
+6 -2
View File
@@ -18,6 +18,7 @@ class SubscriptionExecutionStatus:
batch_id: Optional[str] = None
task_id: Optional[str] = None
current_site_id: Optional[int] = None
next_run_at: Optional[str] = None
error: Optional[str] = None
can_cancel: bool = False
@@ -79,6 +80,8 @@ class SubscriptionExecutionStatusService:
"running",
"matching",
"searching",
"scheduled",
"waiting_subscription",
"waiting_site_budget",
"preparing",
"submitting",
@@ -153,8 +156,8 @@ class SubscriptionExecutionStatusService:
state = phase = "cancelling"
elif task.state == "running":
state = phase = task.phase or "running"
elif task.state == "queued" and task.phase == "waiting_site_budget":
state = phase = "waiting_site_budget"
elif task.state == "queued" and task.phase in cls._ACTIVE_STATES:
state = phase = task.phase
else:
state = phase = task.state
return SubscriptionExecutionStatus(
@@ -164,6 +167,7 @@ class SubscriptionExecutionStatusService:
batch_id=task.batch_id,
task_id=task.task_id,
current_site_id=task.current_site_id,
next_run_at=task.available_at if task.state == "queued" else None,
updated_at=task.updated_at,
error=cls._safe_error(task.last_error),
can_cancel=state in cls._ACTIVE_STATES,
+2 -1
View File
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
_SubscribeChain__is_full_season_best_version_resource: Callable[..., Any]
_SubscribeChain__notify_subscribe_create_failure: Callable[..., Any]
_SubscribeChain__post_subscribe_added: Callable[..., Any]
_SubscribeChain__queue_new_subscription_search: Callable[..., Any]
_SubscribeChain__prepare_best_version_tv_candidate: Callable[..., Any]
_SubscribeChain__prepare_subscribe_progress_fields: Callable[..., Any]
_SubscribeChain__prepare_total_episode_change_fields: Callable[..., Any]
@@ -71,7 +72,7 @@ if TYPE_CHECKING:
_report_search_progress: Callable[..., Any]
_search_music_subscribe: Callable[..., Any]
_subscription_query: Callable[..., Any]
_defer_recent_subscription: Callable[..., Any]
_recent_subscription_retry_at: Callable[..., Any]
_validate_music_subscribe_target: Callable[..., Any]
_wait_before_scheduled_search: Callable[..., Any]
add: Callable[..., Any]
+23 -28
View File
@@ -85,14 +85,11 @@ def _release_match_admission(
subscription_id: int,
summary: MatchExecutionSummary,
) -> None:
"""释放 Match 订阅准入,并让 token 冲突即时可见"""
"""结束当前订阅资源检查,并记录未能恢复的状态"""
if admission.release(lease):
return
summary.release_failures += 1
logger.error(
"订阅准入释放失败: operation=match "
f"subscription_id={subscription_id} run_id={summary.run_id}"
)
logger.error(f"订阅 {subscription_id} 的搜索状态没有正常恢复,系统稍后会继续检查")
def _report_match_finished(
@@ -102,13 +99,13 @@ def _report_match_finished(
"""按实际计数发布 Match 最终进度。"""
if not progress_callback:
return
final_text = "订阅资源匹配完成"
final_text = "订阅资源检查完成"
if summary.finished < summary.total:
final_text = "订阅资源匹配已停止,部分订阅未执行"
final_text = "订阅资源检查已停止,部分订阅这次未检查"
elif summary.failed:
final_text = "订阅资源匹配完成,部分订阅失败"
final_text = "订阅资源检查结束,部分订阅没有完成"
elif summary.skipped:
final_text = "订阅资源匹配完成,部分订阅跳过"
final_text = "订阅资源检查完成,部分订阅这次未检查"
_report_match_progress(progress_callback, summary, value=100, text=final_text)
@@ -234,7 +231,7 @@ def _prepare_subscription_match(
try:
meta = build_subscribe_meta(subscribe)
except ValueError:
logger.error(f"订阅 {subscribe.name} 类型错误:{subscribe.type}")
logger.error(f"订阅{subscribe.name}》的媒体类型不受支持,暂时无法检查资源")
return None
domains = owner.site_repository.get_domains_by_ids(subscribe.sites) if subscribe.sites else []
sub_sites = owner.get_sub_sites(subscribe)
@@ -337,12 +334,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
该入口保持订阅刷新、定时任务和插件调用的稳定签名,具体匹配流程由内部阶段执行。
"""
if not torrents:
logger.warn("没有缓存资源,无法匹配订阅")
logger.warn("当前没有可检查的订阅资源")
if progress_callback:
progress_callback(value=100, text="没有缓存资源,跳过订阅匹配")
progress_callback(value=100, text="当前没有可检查的订阅资源")
return
if progress_callback:
progress_callback(value=0, text="正在预处理订阅资源 ...")
progress_callback(value=0, text="正在理订阅资源 ...")
return self._run_match(
torrents=torrents,
progress_callback=progress_callback,
@@ -360,13 +357,13 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
:param progress_callback: 订阅匹配进度更新回调
"""
if not torrents:
logger.warn("没有缓存资源,无法匹配订阅")
logger.warn("当前没有可检查的订阅资源")
if progress_callback:
progress_callback(value=100, text="没有缓存资源,跳过订阅匹配")
progress_callback(value=100, text="当前没有可检查的订阅资源")
return
if progress_callback:
progress_callback(value=0, text="正在预处理订阅资源 ...")
progress_callback(value=0, text="正在理订阅资源 ...")
lock_acquired = False
summary = MatchExecutionSummary.from_candidates(torrents)
@@ -390,7 +387,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback,
summary,
value=20,
text=f"资源预处理完成,开始匹配 {total_num} 个订阅 ...",
text=f"资源理完成,开始检查 {total_num} 个订阅 ...",
)
try:
for index, listed_subscribe in enumerate(subscribes, start=1):
@@ -401,7 +398,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback,
summary,
value=20 + ((index - 1) / total_num * 80 if total_num else 80),
text=(f"正在匹配订阅({index}/{total_num}{listed_subscribe.name} ..."),
text=(f"正在检查订阅({index}/{total_num}{listed_subscribe.name} ..."),
current=listed_subscribe.id,
)
outcome: Optional[str] = "skipped"
@@ -413,7 +410,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
)
if lease is None:
reason = "admission_conflict"
logger.debug(f"订阅 {listed_subscribe.name} 正在由其他通道处理,本轮匹配已跳过")
logger.debug(f"订阅 {listed_subscribe.name} 正在处理,本次不再重复检查资源")
else:
current_subscribe = None
execution_context = SubscriptionExecutionContext(
@@ -424,14 +421,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
try:
current_subscribe = self.subscription_repository.get(listed_subscribe.id)
if current_subscribe is None:
logger.debug(f"订阅 {listed_subscribe.id}不存在,本轮匹配跳过")
logger.debug(f"订阅 {listed_subscribe.id}删除,本次不再检查资源")
elif current_subscribe.state not in {"R", "P"}:
logger.debug(
f"订阅 {current_subscribe.name} 当前状态为 {current_subscribe.state},本轮匹配跳过"
)
logger.debug(f"订阅 {current_subscribe.name} 当前不需要检查资源,本次跳过")
elif execution_context.should_stop():
reason = _match_stop_reason(execution_context)
logger.debug(f"订阅 {current_subscribe.name} 已取消或超时,本轮匹配跳过")
logger.debug(f"订阅 {current_subscribe.name} 的资源检查已停止")
else:
outcome = self._match_subscription(
subscribe=current_subscribe,
@@ -444,14 +439,14 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
except SubscriptionSearchCancelled as err:
outcome = "skipped"
reason = _match_stop_reason(execution_context) or "cancelled"
logger.debug(f"订阅 {listed_subscribe.name} 匹配已取消{str(err)}")
logger.debug(f"订阅 {listed_subscribe.name} 的资源检查已停止{str(err)}")
except Exception as err:
outcome = "failed"
reason = "error"
subscribe_name = (
current_subscribe.name if current_subscribe is not None else listed_subscribe.name
)
logger.error(f"订阅 {subscribe_name} 匹配失败{str(err)}", exc_info=True)
logger.error(f"订阅 {subscribe_name} 检查资源时出错{str(err)}", exc_info=True)
finally:
_release_match_admission(
self._subscription_execution_admission,
@@ -464,7 +459,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback,
summary,
value=20 + (summary.finished / total_num * 80 if total_num else 80),
text=(f"订阅匹配{index}/{total_num}处理完成"),
text=(f"已检查订阅({index}/{total_num}"),
)
finally:
processed_torrents.clear()
@@ -477,7 +472,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
logger.info(summary.finish_log())
if lock_acquired:
self._match_lock.release()
logger.debug(f"match Lock released at {datetime.now()}")
logger.debug(f"订阅资源检查已结束:{datetime.now()}")
def _match_subscription(
self,
+18
View File
@@ -18,6 +18,7 @@ from app.domain.context import (
MediaInfo,
)
from app.domain.meta.metabase import MetaBase
from app.runtime.execution import run_in_threadpool
from app.runtime.log import logger
from app.schemas.common import JsonData
from app.schemas.message import Message as _SchemaMessage
@@ -142,6 +143,13 @@ class SubscribeNotificationOwner(_SubscribeOwnerBase):
"mediainfo": context.mediainfo.to_dict(),
},
)
try:
self._SubscribeChain__queue_new_subscription_search(subscribe_id)
except Exception as error:
logger.warning(
"订阅已保存,但自动搜索暂时没有安排成功,"
f"系统会在下一次检查时重试:{error}"
)
try:
report_delivered = _subscription_share_snapshot().report_added(
self._SubscribeChain__subscribe_report_payload(context)
@@ -177,6 +185,16 @@ class SubscribeNotificationOwner(_SubscribeOwnerBase):
"mediainfo": context.mediainfo.to_dict(),
},
)
try:
await run_in_threadpool(
self._SubscribeChain__queue_new_subscription_search,
subscribe_id,
)
except Exception as error:
logger.warning(
"订阅已保存,但自动搜索暂时没有安排成功,"
f"系统会在下一次检查时重试:{error}"
)
try:
report_delivered = await _subscription_share_snapshot().async_report_added(
self._SubscribeChain__subscribe_report_payload(context)
+157 -52
View File
@@ -53,6 +53,10 @@ from app.schemas.types import (
SystemConfigKey,
)
_NEW_SUBSCRIPTION_EDIT_SECONDS = 60
_FOREGROUND_RETRY_SECONDS = 5
_BACKGROUND_RETRY_SECONDS = 10
def _ensure_execution_active(
execution_context: Optional[SubscriptionExecutionContext],
@@ -61,9 +65,9 @@ def _ensure_execution_active(
if execution_context is None:
return
if execution_context.is_cancel_requested():
raise SubscriptionSearchCancelled("订阅搜索已取消")
raise SubscriptionSearchCancelled("搜索已停止")
if execution_context.is_expired():
raise TimeoutError("订阅执行已超过协作截止时间")
raise TimeoutError("这次搜索用时过长,已停止")
def _update_search_task_phase(
@@ -91,7 +95,8 @@ def _search_source_and_priority(
) -> tuple[str, int]:
"""把兼容入口归一为持久来源和公平队列优先级。"""
if manual:
return "manual", 100
target_count = 1 if sid else len(sids or ())
return "manual", 120 if target_count == 1 else 100
if sid or sids is not None:
return "targeted", 80
if state in {"R", "P"}:
@@ -99,6 +104,13 @@ def _search_source_and_priority(
return "new", 50
def _retry_at_after(seconds: int) -> str:
"""返回指定秒数后的 UTC 时间,供短暂等待任务重新入队。"""
return (datetime.now(timezone.utc) + timedelta(seconds=max(1, seconds))).isoformat(
timespec="seconds"
)
def _search_task_available_at(
source: str,
subscription_ids: tuple[int, ...],
@@ -179,16 +191,16 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
progress_callback: Optional[Callable[..., None]],
) -> bool:
"""获取 Search 或 Match 通道锁,保持各通道内部串行。"""
operation_label = {"search": "搜索", "match": "资源检查"}[operation]
lock = self._match_lock if operation == "match" else self._search_queue_lock
if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL):
logger.debug(f"{operation} lock acquired at {datetime.now()}")
logger.debug(f"订阅{operation_label}已开始:{datetime.now()}")
return True
operation_label = {"search": "搜索", "match": "匹配"}[operation]
progress_text = {
"search": "订阅搜索锁等待超时,已跳过本轮",
"match": "订阅匹配锁等待超时,已跳过本轮",
"search": "订阅搜索正在处理中,本次不再重复开始",
"match": "订阅资源检查正在进行,本次不再重复开始",
}[operation]
logger.error(f"订阅{operation_label}等待超时,已中止本轮执行")
logger.error(f"订阅{operation_label}等待时间过长,本次不再重复开始")
if progress_callback:
progress_callback(
value=100,
@@ -273,6 +285,12 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
try:
subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state)
total = len(subscribes)
source, _priority = _search_source_and_priority(
sid=sid,
sids=sids,
state=state,
manual=manual,
)
summary = SearchExecutionSummary(source="inline", requested=total)
logger.info(summary.start_log())
if progress_callback:
@@ -287,7 +305,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
summary.stopped = True
break
self._report_search_progress(progress_callback, subscribe, index, total)
if self._defer_recent_subscription(subscribe):
if self._recent_subscription_retry_at(subscribe, source):
summary.record("skipped", "recent_subscription")
continue
self._wait_before_scheduled_search(sid, sids, state, progress_callback)
@@ -297,7 +315,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
)
if lease is None:
logger.debug(f"订阅 {subscribe.name} 正在由其他通道处理,本搜索已跳过")
logger.debug(f"订阅{subscribe.name}正在处理,本搜索先不重复执行")
summary.record("skipped", "admission_conflict")
continue
execution_context = SubscriptionExecutionContext(
@@ -312,7 +330,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
current = self.subscription_repository.get(subscribe.id)
if current is None or current.state == "S":
if current and current.state == "S":
logger.debug(f"订阅 {current.name} 已暂停,本轮搜索已跳过")
logger.debug(f"订阅{current.name}已暂停,本次没有搜索")
continue
processed_result = self._process_search_subscription(
current,
@@ -326,12 +344,12 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
except SubscriptionSearchCancelled:
reason = "ttl_timeout" if execution_context.is_expired() else "cancelled"
outcome = "failed" if reason == "ttl_timeout" else "cancelled"
logger.debug(f"订阅 {subscribe.name} 搜索已在安全边界取消")
logger.debug(f"订阅{subscribe.name}搜索已停止")
except SubscriptionSearchDeferred as deferred:
outcome = "skipped"
reason = "site_budget_deferred"
logger.debug(
f"订阅 {subscribe.name} 站点预算冲突,兼容搜索将在 {deferred.retry_at} 后重试"
f"订阅{subscribe.name}》遇到站点繁忙,将在稍后自动重试:{deferred.retry_at}"
)
except Exception as err:
outcome = "failed"
@@ -347,7 +365,8 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
)
except Exception as err:
logger.error(
f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}",
f"订阅{subscribe.name}搜索结束后没有恢复到正常状态,"
f"系统稍后会继续处理:{str(err)}",
exc_info=True,
)
finally:
@@ -355,8 +374,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
if not released:
summary.release_failures += 1
logger.error(
"订阅准入释放失败: operation=search "
f"subscription_id={subscribe.id} run_id={summary.run_id}"
f"订阅{subscribe.name}》的搜索状态没有正常恢复,系统稍后会继续处理"
)
summary.record(outcome, reason)
self._report_search_progress(
@@ -375,7 +393,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
self._search_queue_lock.release()
if summary is not None:
logger.info(summary.finish_log())
logger.debug(f"search Lock released at {datetime.now()}")
logger.debug(f"订阅搜索已结束:{datetime.now()}")
def _execute_queued_search(
self,
@@ -463,7 +481,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
) -> set[int]:
"""有界消费可恢复任务;单任务失败不得阻止后续订阅。"""
if not self._search_queue_lock.acquire(blocking=False):
logger.debug("订阅搜索队列已有消费者,本轮仅保留持久任务")
logger.debug("订阅搜索已经在后台进行,本次安排会接着处理")
summary.consumer_conflicts += 1
return set()
owner = f"subscribe-search:{uuid4().hex}"
@@ -511,7 +529,7 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
) -> Optional[int]:
"""执行一条已认领任务,返回实际进入搜索处理的订阅 ID。"""
if task.lease_token is None:
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行")
logger.error("这次订阅搜索暂时无法开始,系统稍后会重新处理")
summary.record("failed", "missing_lease")
return None
task_id = str(task.task_id)
@@ -528,14 +546,21 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id,
lease_token=lease_token,
state="cancelled",
error="订阅已不存在",
error="订阅已删除",
)
summary.record("cancelled", "missing_subscription")
return None
self._report_search_progress(progress_callback, subscribe, index, limit)
if self._defer_recent_subscription(subscribe):
_skip_search_task(queue, task, "订阅仍在新增保护期,本轮搜索已跳过")
summary.record("skipped", "recent_subscription")
recent_retry_at = self._recent_subscription_retry_at(subscribe, task.source)
if recent_retry_at:
queue.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=recent_retry_at,
phase="scheduled",
message="订阅刚刚创建,保存好设置后会自动开始搜索",
)
summary.record("requeued", "recent_subscription")
return None
lease = self._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
@@ -543,8 +568,27 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
)
if lease is None:
logger.debug(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过")
_skip_search_task(queue, task, "同一订阅正在由其他通道处理,本轮搜索已跳过")
if task.source in {"manual", "targeted", "new"}:
retry_seconds = (
_FOREGROUND_RETRY_SECONDS
if task.source in {"manual", "targeted"}
else _BACKGROUND_RETRY_SECONDS
)
queue.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=_retry_at_after(retry_seconds),
phase="waiting_subscription",
message="这个订阅正在处理,结束后会自动继续搜索",
)
summary.record("requeued", "admission_conflict")
logger.debug(f"订阅《{subscribe.name}》正在处理,搜索会在结束后自动继续")
else:
_skip_search_task(
queue,
task,
"这个订阅正在处理,本次自动检查无需重复执行",
)
summary.record("skipped", "admission_conflict")
return None
phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token)
@@ -563,12 +607,12 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id,
lease_token=lease_token,
state="cancelled",
error="订阅已不存在",
error="订阅已删除",
)
summary.record("cancelled", "missing_subscription")
return None
if current.state == "S":
_skip_search_task(queue, task, "订阅已暂停,本轮搜索已跳过")
_skip_search_task(queue, task, "订阅已暂停,这次没有搜索")
summary.record("skipped", "paused")
return None
searchchain.configure_subscription_site_budget(
@@ -605,7 +649,7 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id,
lease_token=lease_token,
state="failed",
error="订阅执行已超过协作截止时间",
error="这次搜索用时过长,已停止,可稍后重试",
)
summary.record("failed", "ttl_timeout")
else:
@@ -664,7 +708,11 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
try:
searchchain.configure_subscription_site_budget(None)
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索站点预算清理失败:{str(err)}", exc_info=True)
logger.error(
f"订阅《{subscribe.name}》结束站点访问时遇到问题,"
f"系统稍后会继续处理:{str(err)}",
exc_info=True,
)
try:
if current and current.state == "N":
self._SubscribeChain__apply_subscribe_update(
@@ -673,15 +721,12 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
scene="search_reset",
)
except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索后状态重置失败{str(err)}", exc_info=True)
logger.error(f"订阅{subscribe.name}搜索结束后未能恢复正常状态{str(err)}", exc_info=True)
finally:
released = self._subscription_execution_admission.release(lease)
if not released:
summary.release_failures += 1
logger.error(
"订阅准入释放失败: operation=search "
f"subscription_id={subscribe.id} run_id={summary.run_id}"
)
logger.error(f"订阅《{subscribe.name}》的搜索状态没有正常恢复,系统稍后会继续处理")
self._report_search_progress(
progress_callback,
subscribe,
@@ -694,26 +739,49 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
self,
progress_callback: Optional[Callable[..., None]] = None,
limit: int = 50,
manual_sids: Optional[tuple[int, ...]] = None,
) -> None:
"""短周期恢复排队或租约过期任务,不创建新的 24 小时兜底批次"""
"""短周期恢复等待任务,并可优先反馈本次手工搜索结果"""
queue: Optional[SubscriptionSearchRepository] = getattr(
self, "subscription_search_repository", None
)
if queue is None:
return
summary = SearchExecutionSummary(source="resume", requested=max(1, limit))
manual_ids = tuple(dict.fromkeys(manual_sids or ()))
subscribes = (
self._load_search_subscriptions(sid=None, sids=manual_ids, state=None)
if manual_ids
else []
)
summary = SearchExecutionSummary(
source="manual" if manual_ids else "resume",
requested=len(manual_ids) if manual_ids else max(1, limit),
)
if manual_ids:
logger.info(summary.start_log())
try:
self._drain_search_queue(
processed = self._drain_search_queue(
queue=queue,
limit=max(1, limit),
limit=max(1, limit, len(manual_ids)),
progress_callback=progress_callback,
summary=summary,
)
if manual_ids:
processed_subscribes = [item for item in subscribes if item.id in processed]
self._notify_manual_search(
True,
manual_ids[0] if len(manual_ids) == 1 else None,
None if len(manual_ids) == 1 else manual_ids,
subscribes,
processed_subscribes,
)
except Exception:
summary.round_failed = True
raise
finally:
if not manual_ids and summary.processed:
summary.requested = summary.processed
if manual_ids or summary.processed or summary.round_failed:
logger.info(summary.finish_log())
def cancel_search_batch(self, batch_id: str) -> bool:
@@ -750,15 +818,52 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
return cast(list[SubscriptionSnapshot], repository.list(self.get_states_for_search(state or "N")))
@staticmethod
def _defer_recent_subscription(subscribe: SubscriptionSnapshot) -> bool:
"""新增一分钟内保留 N 状态,为用户编辑筛选条件留出窗口。"""
if not subscribe.date:
return False
def _recent_subscription_retry_at(
subscribe: SubscriptionSnapshot,
source: str,
) -> Optional[str]:
"""新订阅保留一分钟编辑时间,并返回自动开始搜索的时间。"""
if source != "new" or not subscribe.date:
return None
try:
subscribe_time = datetime.strptime(subscribe.date, "%Y-%m-%d %H:%M:%S")
if (datetime.now() - subscribe_time).total_seconds() >= 60:
return False
logger.debug(f"订阅标题:{subscribe.name} 新增小于1分钟,暂不搜索...")
return True
except ValueError:
logger.warning(f"订阅《{subscribe.name}》的添加时间无法识别,将直接开始搜索")
return None
remaining = _NEW_SUBSCRIPTION_EDIT_SECONDS - (
datetime.now() - subscribe_time
).total_seconds()
if remaining <= 0:
return None
retry_seconds = max(1, int(remaining) + 1)
logger.debug(f"新订阅《{subscribe.name}》将在约 {retry_seconds} 秒后自动开始搜索")
return _retry_at_after(retry_seconds)
def _SubscribeChain__queue_new_subscription_search(
self,
subscribe_id: int,
) -> Optional[str]:
"""在订阅保存后安排自动搜索,定时扫描仍作为恢复保障。"""
queue: Optional[SubscriptionSearchRepository] = getattr(
self, "subscription_search_repository", None
)
if queue is None:
return None
subscribe = self.subscription_repository.get(subscribe_id)
if subscribe is None or subscribe.state != "N":
return None
available_at = self._recent_subscription_retry_at(subscribe, "new")
enqueued = queue.enqueue(
subscription_ids=(subscribe_id,),
source="new",
priority=50,
available_at_by_subscription={
subscribe_id: available_at or datetime.now(timezone.utc).isoformat(timespec="seconds")
},
)
if enqueued.created_count:
logger.info(f"已安排新订阅《{subscribe.name}》自动搜索,保存好设置后会自动开始")
return enqueued.active_batch_ids[0] if enqueued.active_batch_ids else None
@staticmethod
def _wait_before_scheduled_search(
@@ -771,9 +876,9 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
if sid or sids is not None or state not in {"R", "P"}:
return
sleep_time = random.randint(60, 300)
logger.debug(f"订阅搜索随机休眠 {sleep_time} ...")
logger.debug(f"为了避免连续访问站点,约 {sleep_time}后继续搜索")
if progress_callback:
progress_callback(text=f"订阅搜索随机休眠 {sleep_time} 秒后继续 ...")
progress_callback(text=f"为了避免连续访问站点,约 {sleep_time} 秒后继续搜索 ...")
time.sleep(sleep_time)
def _process_search_subscription(
@@ -791,7 +896,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
try:
meta = build_subscribe_meta(subscribe)
except ValueError:
logger.error(f"订阅 {subscribe.name} 类型错误:{subscribe.type}")
logger.error(f"订阅{subscribe.name}》的媒体类型不受支持,暂时无法搜索")
return subscribe
mediainfo: MediaInfo = MediaChain().recognize_media(
meta=meta,
@@ -965,7 +1070,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
subscribes: list[SubscriptionSnapshot],
processed: list[SubscriptionSnapshot],
) -> None:
"""为手动触发的搜索发布保持旧文案的系统消息"""
"""用清晰文案反馈手动搜索的完成或等待状态"""
if not manual:
return
if not subscribes:
@@ -974,7 +1079,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
message = (
f"{subscribes[0].name} 搜索完成!"
if processed
else f"{subscribes[0].name} 本轮未执行或未完成,将等待下一次正常调度"
else f"{subscribes[0].name} 已安排搜索,系统会自动继续处理"
)
self.messagehelper.put(message, title="订阅搜索", role="system")
elif sids is not None:
@@ -984,6 +1089,6 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
message = (
"所有订阅搜索完成!"
if len(processed) == len(subscribes)
else "订阅搜索完成,部分订阅本轮未执行或未完成"
else "订阅搜索已安排,暂未完成的项目会自动继续处理"
)
self.messagehelper.put(message, title="订阅搜索", role="system")
+12 -3
View File
@@ -769,9 +769,18 @@ class SessionSubscriptionRepository:
candidates.append(candidate)
return candidates
async def list_search_ids(self, username: str, state: str) -> builtins.list[int]:
"""异步读取用户指定状态下的订阅主键。"""
return [snapshot.id for snapshot in await self.async_list_by_username(username, state)]
async def list_search_ids(
self,
username: Optional[str],
state: str,
) -> builtins.list[int]:
"""异步读取用户或管理员全局范围内指定状态的订阅主键。"""
snapshots = (
await self.async_list_by_username(username, state)
if username is not None
else await self.async_list(state)
)
return [snapshot.id for snapshot in snapshots]
async def stage_delete(self, subscribe_id: int) -> None:
"""异步暂存删除订阅。"""
+29 -4
View File
@@ -1,6 +1,7 @@
"""订阅搜索持久队列的 SQLAlchemy 适配器。"""
from collections.abc import Callable, Mapping
from datetime import datetime, timedelta, timezone
from typing import Optional, TypeVar
from sqlalchemy.orm import Session
@@ -19,6 +20,7 @@ from app.db.oper.subscriptionsearch import SubscriptionSearchOper
from app.db.uow import SqlAlchemyUnitOfWork
T = TypeVar("T")
_BUSY_SITE_RETRY_SECONDS = 10
def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
@@ -101,7 +103,7 @@ class TransactionalSubscriptionSearchRepository:
"""创建批次并返回 single-flight 合并计数。"""
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
"""在同一事务内创建批次和任务。"""
record, created, coalesced = repository.enqueue(
record, created, coalesced, active_batch_ids = repository.enqueue(
subscription_ids=subscription_ids,
source=source,
priority=priority,
@@ -111,6 +113,7 @@ class TransactionalSubscriptionSearchRepository:
batch=_batch(record),
created_count=created,
coalesced_count=coalesced,
active_batch_ids=active_batch_ids,
)
return self._write(operation)
@@ -183,13 +186,17 @@ class TransactionalSubscriptionSearchRepository:
task_id: str,
lease_token: str,
available_at: str,
phase: str = "waiting_site_budget",
message: Optional[str] = None,
) -> bool:
"""以站点预算的下一次可用时间重新排队任务。"""
"""按指定时间和可见原因重新排队任务。"""
return self._write(
lambda repository: repository.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=available_at,
phase=phase,
message=message,
)
)
@@ -225,14 +232,32 @@ class TransactionalSubscriptionSearchRepository:
lease_seconds=lease_seconds,
)
retry_at = record.next_allowed_at
if not acquired and record.lease_token and record.lease_expires_at:
retry_at = max(retry_at, record.lease_expires_at)
wait_reason = None
now = datetime.now(timezone.utc)
cooldown_active = bool(
record.last_outcome not in {None, "success", "skipped"}
and record.next_allowed_at > now.isoformat(timespec="seconds")
)
lease_busy = bool(
record.lease_token
and record.lease_expires_at
and record.lease_expires_at > now.isoformat(timespec="seconds")
)
if not acquired and cooldown_active:
wait_reason = "cooldown"
elif not acquired and lease_busy:
wait_reason = "busy"
short_retry = (now + timedelta(seconds=_BUSY_SITE_RETRY_SECONDS)).isoformat(
timespec="seconds"
)
retry_at = min(record.lease_expires_at, short_retry)
return SiteBudgetClaim(
site_id=record.site_id,
acquired=acquired,
retry_at=retry_at,
consecutive_failures=record.consecutive_failures,
lease_token=record.lease_token if acquired else None,
wait_reason=wait_reason,
)
return self._write(operation)
+1
View File
@@ -83,6 +83,7 @@ class SessionSubscriptionExecutionStatusRepository:
"""返回最近更新的批次,访问范围由应用服务依据任务校验。"""
result = await self._session.execute(
select(SubscriptionSearchBatch)
.where(SubscriptionSearchBatch.total_count > 0)
.order_by(
SubscriptionSearchBatch.updated_at.desc(),
SubscriptionSearchBatch.id.desc(),
+7 -3
View File
@@ -556,9 +556,13 @@ class SubscribeOper(DbOper):
)
return candidates
async def list_search_ids(self, username: str, state: str) -> List[int]:
"""返回用户指定状态的订阅编号,不向应用用例暴露 ORM 列表"""
subscribes = await self.async_list_by_username(username, state=state)
async def list_search_ids(self, username: Optional[str], state: str) -> List[int]:
"""返回用户或管理员全局范围内指定状态的订阅编号。"""
subscribes = (
await self.async_list_by_username(username, state=state)
if username is not None
else await self.async_list(state=state)
)
return [subscribe.id for subscribe in subscribes if subscribe.id]
def get_by(
+40 -6
View File
@@ -31,7 +31,7 @@ class SubscriptionSearchOper(DbOper):
source: str,
priority: int,
available_at_by_subscription: Optional[Mapping[int, str]],
) -> tuple[SubscriptionSearchBatch, int, int]:
) -> tuple[SubscriptionSearchBatch, int, int, tuple[str, ...]]:
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅搜索入队需要调用方提供同步 Session")
@@ -49,6 +49,7 @@ class SubscriptionSearchOper(DbOper):
self._db.flush()
created = 0
coalesced = 0
active_batch_ids: list[str] = []
for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)):
active_key = f"subscription:{subscription_id}"
available_at = (
@@ -56,6 +57,11 @@ class SubscriptionSearchOper(DbOper):
if available_at_by_subscription
else now
)
initial_phase = (
"scheduled"
if source == "new" and available_at > now
else "queued"
)
task = SubscriptionSearchTask(
task_id=uuid4().hex,
batch_id=batch.batch_id,
@@ -65,7 +71,7 @@ class SubscriptionSearchOper(DbOper):
priority=priority,
position=position,
state="queued",
phase="queued",
phase=initial_phase,
available_at=available_at,
created_at=now,
updated_at=now,
@@ -77,15 +83,31 @@ class SubscriptionSearchOper(DbOper):
created += 1
except IntegrityError:
coalesced += 1
promote_queued_task = and_(
SubscriptionSearchTask.priority < priority,
SubscriptionSearchTask.state == "queued",
)
execute_dml(
self._db,
update(SubscriptionSearchTask)
.where(SubscriptionSearchTask.active_key == active_key)
.values(
source=case(
(SubscriptionSearchTask.priority < priority, source),
else_=SubscriptionSearchTask.source,
),
priority=case(
(SubscriptionSearchTask.priority < priority, priority),
else_=SubscriptionSearchTask.priority,
),
phase=case(
(promote_queued_task, "queued"),
else_=SubscriptionSearchTask.phase,
),
last_error=case(
(promote_queued_task, None),
else_=SubscriptionSearchTask.last_error,
),
available_at=case(
(
or_(
@@ -100,11 +122,20 @@ class SubscriptionSearchOper(DbOper):
),
execution_options={"synchronize_session": False},
)
active_task = self._db.execute(
select(SubscriptionSearchTask).where(
SubscriptionSearchTask.active_key == active_key
)
).scalars().first()
if active_task is not None:
active_batch_ids.append(active_task.batch_id)
batch.total_count = created
if created == 0:
batch.state = "completed"
batch.finished_at = now
return batch, created, coalesced
else:
active_batch_ids.insert(0, batch.batch_id)
return batch, created, coalesced, tuple(dict.fromkeys(active_batch_ids))
def claim_next(self, *, owner: str, lease_seconds: int) -> Optional[SubscriptionSearchTask]:
"""使用 CAS 认领最高优先级任务,过期 running 任务可被恢复。"""
@@ -139,6 +170,7 @@ class SubscriptionSearchOper(DbOper):
)
.order_by(
case(
(SubscriptionSearchTask.priority >= 100, 2),
(SubscriptionSearchTask.created_at <= fairness_before, 1),
else_=0,
).desc(),
@@ -369,8 +401,10 @@ class SubscriptionSearchOper(DbOper):
task_id: str,
lease_token: str,
available_at: str,
phase: str,
message: Optional[str],
) -> bool:
"""释放当前租约并在站点预算时间到达后恢复同一任务。"""
"""释放当前租约并在指定时间后按可见原因恢复同一任务。"""
if not isinstance(self._db, Session):
raise RuntimeError("订阅搜索延后需要调用方提供同步 Session")
task = self._db.execute(
@@ -400,7 +434,7 @@ class SubscriptionSearchOper(DbOper):
)
.values(
state="queued",
phase="waiting_site_budget",
phase=phase,
current_site_id=None,
lease_owner=None,
lease_token=None,
@@ -408,7 +442,7 @@ class SubscriptionSearchOper(DbOper):
available_at=available_at,
updated_at=now,
finished_at=None,
last_error=None,
last_error=message,
),
execution_options={"synchronize_session": False},
)
+41 -2
View File
@@ -137,6 +137,17 @@
"当前管理用户缺少可审计身份": "The current administrator does not have an auditable identity",
"人工复核任务不存在": "The manual review task does not exist",
"任务添加失败": "Failed to add task",
"没有需要搜索的订阅": "No subscriptions need searching",
"这个订阅已经在搜索中,请稍候": "This subscription is already being searched. Please wait",
"已安排搜索,很快开始": "Search scheduled and will start soon",
"当前没有可检查的订阅资源": "No subscription resources are currently available to check",
"正在整理订阅资源 ...": "Organizing subscription resources ...",
"订阅资源检查完成": "Subscription resource check complete",
"订阅资源检查已停止,部分订阅这次未检查": "Subscription resource check stopped; some subscriptions were not checked this time",
"订阅资源检查结束,部分订阅没有完成": "Subscription resource check finished; some subscriptions did not complete",
"订阅资源检查完成,部分订阅这次未检查": "Subscription resource check complete; some subscriptions were not checked this time",
"订阅搜索正在处理中,本次不再重复开始": "A subscription search is already in progress. This run will not start again",
"订阅资源检查正在进行,本次不再重复开始": "A subscription resource check is already in progress. This run will not start again",
"无法识别媒体信息": "Unable to recognize media information",
"未识别到媒体信息": "Unable to recognize media information",
"未识别到音乐信息": "Unable to recognize music information",
@@ -382,8 +393,8 @@
"微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "WeChat ClawBot notification is not enabled or the configuration has not been saved. Please save and enable this channel first",
"请输入至少一个有效的站点 ID": "Enter at least one valid site ID",
"所有订阅搜索完成": "All subscription searches are complete",
"订阅搜索批次不存在": "The subscription search batch does not exist",
"订阅搜索批次已结束或无法取消": "The subscription search batch has ended or cannot be cancelled",
"没有找到这次搜索,请刷新后重试": "This search could not be found. Refresh and try again",
"这次搜索已经结束,暂时无法停止": "This search has already ended and cannot be stopped",
"订阅搜索锁等待超时,已跳过本轮": "Subscription search lock timed out, this round was skipped",
"订阅匹配锁等待超时,已跳过本轮": "Subscription matching lock timed out, this round was skipped",
"请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "Enter subscription IDs separated by spaces, or enter all",
@@ -765,6 +776,18 @@
"source": "已完成 {count} 个订阅搜索",
"target": "Completed searches for {count} subscriptions"
},
{
"source": "{count} 个订阅已经在搜索中,无需重复提交",
"target": "{count} subscriptions are already being searched. No need to submit again"
},
{
"source": "已安排 {queued} 个订阅搜索,另有 {ongoing} 个正在处理中",
"target": "Scheduled searches for {queued} subscriptions; {ongoing} are already being processed"
},
{
"source": "已安排 {count} 个订阅搜索,系统会依次处理",
"target": "Scheduled searches for {count} subscriptions. They will be processed in order"
},
{
"source": "请输入至少一个有效的订阅 ID",
"target": "Enter at least one valid subscription ID"
@@ -1421,6 +1444,10 @@
"source": "订阅搜索({index}/{total})处理完成",
"target": "Subscription search ({index}/{total}) completed"
},
{
"source": "为了避免连续访问站点,约 {seconds} 秒后继续搜索 ...",
"target": "To avoid repeated site requests, search will continue in about {seconds} seconds ..."
},
{
"source": "开始刷新订阅,共 {count} 个订阅 ...",
"target": "Starting subscription refresh, {count} subscriptions ..."
@@ -1433,6 +1460,18 @@
"source": "正在匹配订阅({index}/{total}{name} ...",
"target": "Matching subscription ({index}/{total}) {name} ..."
},
{
"source": "资源整理完成,开始检查 {count} 个订阅 ...",
"target": "Resource organization complete. Checking {count} subscriptions ..."
},
{
"source": "正在检查订阅({index}/{total}{name} ...",
"target": "Checking subscription ({index}/{total}) {name} ..."
},
{
"source": "已检查订阅({index}/{total}",
"target": "Checked subscription ({index}/{total})"
},
{
"source": "开始更新订阅元数据,共 {count} 个订阅 ...",
"target": "Starting subscription metadata update, {count} subscriptions ..."
+41 -2
View File
@@ -129,6 +129,17 @@
"当前管理用户缺少可审计身份": "目前管理使用者缺少可稽核身分",
"人工复核任务不存在": "人工複核任務不存在",
"任务添加失败": "任務新增失敗",
"没有需要搜索的订阅": "沒有需要搜尋的訂閱",
"这个订阅已经在搜索中,请稍候": "這個訂閱已在搜尋中,請稍候",
"已安排搜索,很快开始": "已安排搜尋,很快開始",
"当前没有可检查的订阅资源": "目前沒有可檢查的訂閱資源",
"正在整理订阅资源 ...": "正在整理訂閱資源 ...",
"订阅资源检查完成": "訂閱資源檢查完成",
"订阅资源检查已停止,部分订阅这次未检查": "訂閱資源檢查已停止,部分訂閱這次未檢查",
"订阅资源检查结束,部分订阅没有完成": "訂閱資源檢查結束,部分訂閱沒有完成",
"订阅资源检查完成,部分订阅这次未检查": "訂閱資源檢查完成,部分訂閱這次未檢查",
"订阅搜索正在处理中,本次不再重复开始": "訂閱搜尋正在處理中,本次不再重複開始",
"订阅资源检查正在进行,本次不再重复开始": "訂閱資源檢查正在進行,本次不再重複開始",
"无法识别媒体信息": "無法識別媒體資訊",
"未识别到媒体信息": "未識別到媒體資訊",
"未识别到音乐信息": "未識別到音樂資訊",
@@ -374,8 +385,8 @@
"微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "微信 ClawBot 通知未啟用或設定尚未儲存,請先儲存並啟用目前渠道",
"请输入至少一个有效的站点 ID": "請輸入至少一個有效的站點 ID",
"所有订阅搜索完成": "所有訂閱搜尋完成",
"订阅搜索批次不存在": "訂閱搜尋批次不存在",
"订阅搜索批次已结束或无法取消": "訂閱搜尋批次已結束或無法取消",
"没有找到这次搜索,请刷新后重试": "找不到這次搜尋,請重新整理後再試",
"这次搜索已经结束,暂时无法停止": "這次搜尋已經結束,暫時無法停止",
"请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "請輸入訂閱 ID,多個 ID 以空格分隔,或輸入 all",
"请输入至少一个有效的订阅 ID": "請輸入至少一個有效的訂閱 ID",
"格式错误,请输入:cookie <id> <username> <password> [2fa_code/secret]": "格式錯誤,請輸入:cookie <id> <username> <password> [2fa_code/secret]",
@@ -751,6 +762,18 @@
"source": "已完成 {count} 个订阅搜索",
"target": "已完成 {count} 個訂閱搜尋"
},
{
"source": "{count} 个订阅已经在搜索中,无需重复提交",
"target": "{count} 個訂閱已在搜尋中,無需重複提交"
},
{
"source": "已安排 {queued} 个订阅搜索,另有 {ongoing} 个正在处理中",
"target": "已安排 {queued} 個訂閱搜尋,另有 {ongoing} 個正在處理中"
},
{
"source": "已安排 {count} 个订阅搜索,系统会依次处理",
"target": "已安排 {count} 個訂閱搜尋,系統會依序處理"
},
{
"source": "请输入至少一个有效的订阅 ID",
"target": "請輸入至少一個有效的訂閱 ID"
@@ -1407,6 +1430,10 @@
"source": "订阅搜索({index}/{total})处理完成",
"target": "訂閱搜尋({index}/{total})處理完成"
},
{
"source": "为了避免连续访问站点,约 {seconds} 秒后继续搜索 ...",
"target": "為了避免連續存取站點,約 {seconds} 秒後繼續搜尋 ..."
},
{
"source": "开始刷新订阅,共 {count} 个订阅 ...",
"target": "開始重新整理訂閱,共 {count} 個訂閱 ..."
@@ -1419,6 +1446,18 @@
"source": "正在匹配订阅({index}/{total}{name} ...",
"target": "正在匹配訂閱({index}/{total}{name} ..."
},
{
"source": "资源整理完成,开始检查 {count} 个订阅 ...",
"target": "資源整理完成,開始檢查 {count} 個訂閱 ..."
},
{
"source": "正在检查订阅({index}/{total}{name} ...",
"target": "正在檢查訂閱({index}/{total}{name} ..."
},
{
"source": "已检查订阅({index}/{total}",
"target": "已檢查訂閱({index}/{total}"
},
{
"source": "开始更新订阅元数据,共 {count} 个订阅 ...",
"target": "開始更新訂閱元資料,共 {count} 個訂閱 ..."
+2 -2
View File
@@ -154,8 +154,8 @@ class SchedulerCatalogOwner(_SchedulerOwnerBase):
"interval",
id="subscribe_search_queue",
name="恢复订阅搜索队列",
minutes=1,
next_run_time=datetime.now(pytz.timezone(config.timezone)) + timedelta(seconds=10),
seconds=10,
next_run_time=datetime.now(pytz.timezone(config.timezone)) + timedelta(seconds=5),
kwargs={"job_id": "subscribe_search_queue"},
)
+1
View File
@@ -463,6 +463,7 @@ SCHEMA_EXPORTS = {
'SubscriptionBatchStatus': ('app.schemas.subscribe', 'SubscriptionBatchStatus'),
'SubscriptionExecutionStatus': ('app.schemas.subscribe', 'SubscriptionExecutionStatus'),
'SubscriptionMessage': ('app.schemas.message', 'SubscriptionMessage'),
'SubscriptionSearchSubmission': ('app.schemas.subscribe', 'SubscriptionSearchSubmission'),
'SubtitleDownloadData': ('app.schemas.download', 'SubtitleDownloadData'),
'SubtitleInfo': ('app.schemas.search', 'SubtitleInfo'),
'SystemEnvironmentUpdateData': ('app.schemas.system', 'SystemEnvironmentUpdateData'),
+12
View File
@@ -55,6 +55,7 @@ class SubscriptionExecutionStatus(BaseModel): # type: ignore[misc]
batch_id: Optional[str] = None
task_id: Optional[str] = None
current_site_id: Optional[int] = None
next_run_at: Optional[str] = None
error: Optional[str] = None
can_cancel: bool = False
@@ -84,6 +85,17 @@ class SubscriptionBatchStatus(BaseModel): # type: ignore[misc]
model_config = ConfigDict(from_attributes=True)
class SubscriptionSearchSubmission(BaseModel): # type: ignore[misc]
"""手工订阅搜索已安排后的轻量跟踪信息。"""
batch_id: Optional[str] = None
batch_ids: List[str] = Field(default_factory=list)
target_count: int = 0
queued_count: int = 0
ongoing_count: int = 0
single: bool = False
class Subscribe(OptionalMediaIdentityMixin, BaseModel):
"""订阅输入与响应模型,媒体身份必须为空对或完整有效对。"""
@@ -799,7 +799,7 @@ def _run_match_execution_case(case: ScaleCase) -> dict[str, Any]:
@staticmethod
def info(message: Any, *_args: Any, **_kwargs: Any) -> None:
"""保存结构化批次摘要。"""
"""保存开始和结束摘要。"""
info_logs.append(str(message))
class _ScaleMediaChain:
@@ -958,8 +958,10 @@ def _run_match_execution_case(case: ScaleCase) -> dict[str, Any]:
"info_log_count": len(info_logs),
"info_log_bounded": bool(
len(info_logs) == 2
and info_logs[0].startswith("订阅治理轮次开始: operation=match ")
and info_logs[1].startswith("订阅治理轮次结束: operation=match ")
and info_logs[0].startswith("开始检查订阅资源,共 ")
and info_logs[1].startswith("订阅资源检查结束:")
and "operation=" not in "\n".join(info_logs)
and "run_id=" not in "\n".join(info_logs)
),
"first_settlement_local_ms": round(settlement_ms[0], 3),
"subscription_settlement_local_p50_ms": _percentile(settlement_ms, 50),
+53 -9
View File
@@ -72,6 +72,34 @@ class _RunningTaskRegistry(TaskRegistry):
)
class _CompletedSearchTaskRegistry(_TaskRegistry):
"""返回已完成结果,验证手工搜索先保存安排再启动后台处理。"""
def create_sync(self, function, *args, owner: str, **kwargs) -> asyncio.Future:
"""记录同步任务,并为测试提供可等待的安排结果。"""
super().create_sync(function, *args, owner=owner, **kwargs)
future = asyncio.get_running_loop().create_future()
result = None
if owner == "api.subscribe.search.enqueue":
result = SimpleNamespace(
active_batch_ids=("batch-1", "batch-2"),
created_count=2,
coalesced_count=0,
)
future.set_result(result)
return future
class _SubscriptionSearchTargets:
"""为手工搜索命令提供当前用户可访问的订阅编号。"""
async def list_search_ids(self, username, state) -> list[int]:
"""返回稳定目标,并校验超级用户读取全部运行中订阅。"""
assert username is None
assert state == "R"
return [11, 12]
class _ProtocolManager:
"""提供兼容协议流结束时需要的最小 AgentManager 接口。"""
@@ -206,10 +234,14 @@ def test_seerr_subscribe_uses_task_registry(monkeypatch) -> None:
def test_manual_subscription_search_uses_task_registry() -> None:
"""手工订阅搜索命令应以稳定 owner 提交顺序搜索批次。"""
registry = _TaskRegistry()
repository = object()
registry = _CompletedSearchTaskRegistry()
repository = _SubscriptionSearchTargets()
search_repository = SimpleNamespace(enqueue=lambda **_kwargs: None)
runtime = SimpleNamespace(
subscription=SimpleNamespace(repository=lambda _db: repository)
subscription=SimpleNamespace(
repository=lambda _db: repository,
search_repository=search_repository,
)
)
command = subscription_dependencies.get_search_subscriptions_command(
task_registry=registry,
@@ -221,12 +253,24 @@ def test_manual_subscription_search_uses_task_registry() -> None:
command.execute(SubscribeSearchActor(username="admin", is_superuser=True))
)
function, args, kwargs, owner = registry.calls[0]
assert found is True
assert function is subscription_dependencies._start_subscription_search_batch
assert args == (None, "R")
assert kwargs == {}
assert owner == "api.subscribe.search"
enqueue_function, enqueue_args, enqueue_kwargs, enqueue_owner = registry.calls[0]
run_function, run_args, run_kwargs, run_owner = registry.calls[1]
assert found is not None
assert found.batch_ids == ("batch-1", "batch-2")
assert found.queued_count == 2
assert found.ongoing_count == 0
assert enqueue_function is search_repository.enqueue
assert enqueue_args == ()
assert enqueue_kwargs == {
"subscription_ids": (11, 12),
"source": "manual",
"priority": 100,
}
assert enqueue_owner == "api.subscribe.search.enqueue"
assert run_function is subscription_dependencies._resume_submitted_subscription_search
assert run_args == ((11, 12),)
assert run_kwargs == {}
assert run_owner == "api.subscribe.search.run"
def test_history_ai_redo_uses_task_registry() -> None:
+14 -1
View File
@@ -653,8 +653,19 @@ class TestSubscribeEndpoint:
普通用户批量搜索把用户身份交给应用命令
"""
from app.api.endpoints.subscribe import search_subscribes
from app.application.subscription.search import SubscriptionSearchSubmission
command = SimpleNamespace(execute=AsyncMock(return_value=True))
command = SimpleNamespace(
execute=AsyncMock(
return_value=SubscriptionSearchSubmission(
batch_ids=("batch-1",),
target_count=2,
queued_count=1,
ongoing_count=1,
single=False,
)
)
)
response = asyncio.run(
search_subscribes(
command=command,
@@ -663,6 +674,8 @@ class TestSubscribeEndpoint:
)
assert response.success
assert response.message == "已安排 1 个订阅搜索,另有 1 个正在处理中"
assert response.data.queued_count == 1
command.execute.assert_awaited_once()
actor = command.execute.await_args.args[0]
assert actor.username == "alice"
+116 -49
View File
@@ -1,3 +1,4 @@
import asyncio
from types import SimpleNamespace
from unittest.mock import Mock
@@ -8,6 +9,7 @@ from app.application.subscription.delete import SubscribeDeletionCandidate
from app.application.subscription.search import (
SearchSubscriptionsCommand,
SubscribeSearchActor,
SubscriptionSearchSubmission,
)
from app.runtime.tasks import TaskRegistry
@@ -19,15 +21,15 @@ class _Repository:
"""保存预设单条候选和批量编号。"""
self.candidate = candidate
self.subscribe_ids = subscribe_ids or []
self.list_calls = []
async def get_candidate(self, _subscribe_id):
"""返回预设订阅候选。"""
return self.candidate
async def list_search_ids(self, username, state):
"""校验普通用户搜索状态并返回预设编号。"""
assert username == "alice"
assert state == "R"
"""记录访问范围并返回预设编号。"""
self.list_calls.append((username, state))
return self.subscribe_ids
@@ -40,85 +42,117 @@ def _candidate(username):
)
@pytest.mark.asyncio
async def test_superuser_search_all_uses_single_global_scheduler_request():
"""管理员搜索全部订阅时保持一次 state=R 的全局调度语义。"""
scheduled = []
command = SearchSubscriptionsCommand(
repository=_Repository(),
schedule_search=lambda sid, state: scheduled.append((sid, state)),
def _submitter(calls):
"""构造记录目标并返回稳定安排结果的异步提交器。"""
async def submit(subscribe_ids, single):
"""记录一次轻量入队请求。"""
calls.append((subscribe_ids, single))
return SubscriptionSearchSubmission(
batch_ids=("batch-1",),
target_count=len(subscribe_ids),
queued_count=len(subscribe_ids),
ongoing_count=0,
single=single,
)
assert await command.execute(
return submit
@pytest.mark.asyncio
async def test_superuser_search_all_queues_every_active_subscription():
"""管理员搜索全部订阅时读取全局活动订阅并一次提交。"""
submitted = []
repository = _Repository(subscribe_ids=[2, 5])
command = SearchSubscriptionsCommand(
repository=repository,
submit_search=_submitter(submitted),
)
result = await command.execute(
SubscribeSearchActor(username="admin", is_superuser=True)
) is True
assert scheduled == [(None, "R")]
)
assert result is not None
assert result.target_count == 2
assert repository.list_calls == [(None, "R")]
assert submitted == [((2, 5), False)]
@pytest.mark.asyncio
async def test_regular_user_search_all_schedules_only_owned_subscriptions():
"""普通用户搜索全部时把归属订阅合并为一次后台批次。"""
scheduled = []
submitted = []
repository = _Repository(subscribe_ids=[2, 5])
command = SearchSubscriptionsCommand(
repository=_Repository(subscribe_ids=[2, 5]),
schedule_search=lambda sid, state: scheduled.append((sid, state)),
repository=repository,
submit_search=_submitter(submitted),
)
assert await command.execute(
result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False)
) is True
assert scheduled == [((2, 5), None)]
)
assert result is not None
assert repository.list_calls == [("alice", "R")]
assert submitted == [((2, 5), False)]
@pytest.mark.asyncio
async def test_regular_user_search_all_with_no_targets_does_not_schedule():
"""普通用户没有可搜索订阅时不创建空后台任务。"""
scheduled = []
submitted = []
command = SearchSubscriptionsCommand(
repository=_Repository(),
schedule_search=lambda ids, state: scheduled.append((ids, state)),
submit_search=_submitter(submitted),
)
assert await command.execute(
result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False)
) is True
assert scheduled == []
)
assert result is not None
assert result.target_count == 0
assert result.batch_ids == ()
assert submitted == []
@pytest.mark.asyncio
async def test_targeted_search_rejects_missing_or_other_users_subscription():
"""单条搜索不得泄漏订阅是否属于其他普通用户。"""
scheduled = []
submitted = []
command = SearchSubscriptionsCommand(
repository=_Repository(candidate=_candidate("bob")),
schedule_search=lambda sid, state: scheduled.append((sid, state)),
submit_search=_submitter(submitted),
)
assert await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False),
subscribe_id=7,
) is False
assert scheduled == []
) is None
assert submitted == []
@pytest.mark.asyncio
async def test_targeted_search_schedules_accessible_subscription():
"""归属用户搜索单条订阅时提交历史兼容参数。"""
scheduled = []
submitted = []
command = SearchSubscriptionsCommand(
repository=_Repository(candidate=_candidate("alice")),
schedule_search=lambda sid, state: scheduled.append((sid, state)),
submit_search=_submitter(submitted),
)
assert await command.execute(
result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False),
subscribe_id=7,
) is True
assert scheduled == [((7,), None)]
)
assert result is not None
assert result.single is True
assert submitted == [((7,), True)]
def test_subscription_search_batch_uses_one_scheduler_generation(monkeypatch):
"""一个后台批次只占用一次调度任务运行权"""
def test_submitted_search_wakes_short_cycle_queue(monkeypatch):
"""手工搜索入队后立即唤醒短周期恢复任务"""
calls = []
monkeypatch.setattr(
subscription_dependencies,
@@ -126,24 +160,46 @@ def test_subscription_search_batch_uses_one_scheduler_generation(monkeypatch):
lambda job_id, **kwargs: calls.append((job_id, kwargs)),
)
subscription_dependencies._start_subscription_search_batch((2, 5), None)
subscription_dependencies._resume_submitted_subscription_search((2, 5))
assert calls == [
(
"subscribe_search",
{"sids": (2, 5), "state": None, "manual": True},
"subscribe_search_queue",
{"limit": 2, "manual_sids": (2, 5)},
),
]
@pytest.mark.asyncio
async def test_search_dependency_registers_one_owned_background_batch():
"""请求适配器只登记一个具名后台批次"""
async def test_search_dependency_persists_before_waking_background_worker():
"""请求适配器先等待持久入队完成,再登记后台恢复任务"""
registry = TaskRegistry()
registry.create_sync = Mock()
calls = []
def create_sync(function, *args, owner, **kwargs):
"""同步执行测试函数,并返回已经完成的 Future。"""
calls.append((function, args, owner, kwargs))
future = asyncio.get_running_loop().create_future()
result = function(*args, **kwargs) if owner.endswith("enqueue") else None
future.set_result(result)
return future
registry.create_sync = Mock(side_effect=create_sync)
repository = _Repository(subscribe_ids=[2, 5])
search_repository = SimpleNamespace(
enqueue=Mock(
return_value=SimpleNamespace(
active_batch_ids=("batch-new", "batch-existing"),
created_count=1,
coalesced_count=1,
)
)
)
runtime = SimpleNamespace(
subscription=SimpleNamespace(repository=lambda _db: repository),
subscription=SimpleNamespace(
repository=lambda _db: repository,
search_repository=search_repository,
),
)
command = subscription_dependencies.get_search_subscriptions_command(
task_registry=registry,
@@ -151,12 +207,23 @@ async def test_search_dependency_registers_one_owned_background_batch():
runtime=runtime,
)
assert await command.execute(
result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False)
) is True
registry.create_sync.assert_called_once_with(
subscription_dependencies._start_subscription_search_batch,
(2, 5),
None,
owner="api.subscribe.search",
)
assert result is not None
assert result.batch_ids == ("batch-new", "batch-existing")
assert result.queued_count == 1
assert result.ongoing_count == 1
assert calls[0] == (
search_repository.enqueue,
(),
"api.subscribe.search.enqueue",
{"subscription_ids": (2, 5), "source": "manual", "priority": 100},
)
assert calls[1] == (
subscription_dependencies._resume_submitted_subscription_search,
((2, 5),),
"api.subscribe.search.run",
{},
)
+3 -3
View File
@@ -175,7 +175,7 @@ def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None:
subscribe_oper.assert_not_called()
progress.assert_called_once_with(
value=100,
text="订阅搜索锁等待超时,已跳过本轮",
text="订阅搜索正在处理中,本次不再重复开始",
)
@@ -265,7 +265,7 @@ def test_inline_search_conflict_does_not_report_false_completion(monkeypatch) ->
assert progress.call_args.kwargs == {
"value": 100,
"text": "订阅搜索结束,部分订阅本轮未执行或未完成",
"text": "搜索结束,部分订阅这次没有完成",
"data": {"total": 1, "finished": 0},
}
assert chain._subscription_execution_admission.release(match_lease) is True
@@ -279,4 +279,4 @@ def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None:
chain = object.__new__(SubscribeChain)
chain.match({"example.org": []}, progress_callback=progress)
progress.assert_any_call(value=100, text="订阅匹配锁等待超时,已跳过本轮")
progress.assert_any_call(value=100, text="订阅资源检查正在进行,本次不再重复开始")
+40 -5
View File
@@ -14,13 +14,15 @@ def _task(
phase: str = "searching",
updated_at: str = "2026-09-01T01:00:00+00:00",
batch_id: str = "batch-1",
source: str = "manual",
available_at: str | None = None,
) -> SearchTaskSnapshot:
"""构造最小搜索任务快照。"""
return SearchTaskSnapshot(
task_id=f"task-{subscription_id}",
batch_id=batch_id,
subscription_id=subscription_id,
source="manual",
source=source,
priority=100,
position=subscription_id,
state=state,
@@ -30,8 +32,15 @@ def _task(
lease_token="lease" if state == "running" else None,
created_at="2026-09-01T00:00:00+00:00",
updated_at=updated_at,
available_at=available_at,
current_site_id=9 if phase == "waiting_site_budget" else None,
last_error=" provider\n timeout " if state == "failed" else None,
last_error=(
" provider\n timeout "
if state == "failed"
else "站点暂时忙,系统会自动继续搜索"
if state == "queued" and phase == "waiting_site_budget"
else None
),
)
@@ -85,18 +94,44 @@ def test_execution_status_exposes_site_wait_and_cancel_capability():
def test_execution_status_exposes_queued_site_wait_without_error():
"""重新入队的站点预算冲突应显示等待状态而不是失败"""
"""重新入队的站点繁忙应显示等待状态、说明和继续时间"""
repository = _Repository()
repository.tasks[2] = _task(2, state="queued", phase="waiting_site_budget")
retry_at = "2026-09-01T01:00:10+00:00"
repository.tasks[2] = _task(
2,
state="queued",
phase="waiting_site_budget",
available_at=retry_at,
)
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,)))
assert statuses[2].state == "waiting_site_budget"
assert statuses[2].phase == "waiting_site_budget"
assert statuses[2].error is None
assert statuses[2].error == "站点暂时忙,系统会自动继续搜索"
assert statuses[2].next_run_at == retry_at
assert statuses[2].can_cancel is True
def test_execution_status_exposes_scheduled_new_search_without_failure():
"""新订阅编辑等待期应显示为已安排,而不是跳过或失败。"""
repository = _Repository()
retry_at = "2026-09-01T01:01:00+00:00"
repository.tasks[4] = _task(
4,
state="queued",
phase="scheduled",
source="new",
available_at=retry_at,
)
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((4,)))
assert statuses[4].state == "scheduled"
assert statuses[4].next_run_at == retry_at
assert statuses[4].error is None
def test_failed_search_exposes_safe_error():
"""搜索失败文本必须压平且不暴露内部错误细节。"""
repository = _Repository()
+13 -19
View File
@@ -169,7 +169,7 @@ def test_match_skips_subscription_paused_after_admission() -> None:
"skipped": 1,
"failed": 0,
}
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅跳过"
assert progress.call_args.kwargs["text"] == "订阅资源检查完成,部分订阅这次未检查"
_assert_channel_and_subscription_released(chain, (listed.id,))
@@ -200,7 +200,7 @@ def test_match_progress_reports_completed_skipped_and_failed_counts() -> None:
"skipped": 1,
"failed": 1,
}
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅失败"
assert progress.call_args.kwargs["text"] == "订阅资源检查结束,部分订阅没有完成"
assert chain._subscription_execution_admission.release(skipped_lease) is True
_assert_channel_and_subscription_released(chain, (completed.id, failed.id))
@@ -220,7 +220,7 @@ def test_match_stop_does_not_report_unvisited_subscriptions_as_completed(monkeyp
assert progress.call_args.kwargs == {
"value": 100,
"text": "订阅资源匹配已停止,部分订阅未执行",
"text": "订阅资源检查已停止,部分订阅这次未检查",
"data": {
"total": 2,
"finished": 0,
@@ -257,17 +257,12 @@ def test_match_logs_one_bounded_start_and_finish_summary(monkeypatch) -> None:
chain.match({"site-a.example": [object()], "site-b.example": [object(), object()]})
assert len(info_logs) == 2
assert info_logs[0].startswith("订阅治理轮次开始: operation=match ")
assert "subscriptions=3" in info_logs[0]
assert "sites=2" in info_logs[0]
assert "candidates=3" in info_logs[0]
assert info_logs[1].startswith("订阅治理轮次结束: operation=match ")
assert "state=skipped" in info_logs[1]
assert "task_completed=1" in info_logs[1]
assert "task_skipped=2" in info_logs[1]
assert "admission_conflicts=1" in info_logs[1]
assert "cancelled=1" in info_logs[1]
assert "ttl_timeouts=0" in info_logs[1]
assert info_logs[0] == "开始检查订阅资源,共 3 个订阅、3 个资源,来自 2 个站点。"
assert info_logs[1].startswith("订阅资源检查结束:部分订阅这次未检查。")
assert "本次检查 3/3 个订阅,完成 1 个,这次未检查 2 个,失败 0 个" in info_logs[1]
assert "其中 1 个订阅正在处理中,本次没有重复检查" in info_logs[1]
assert "另有 1 个订阅已停止" in info_logs[1]
assert "run_id" not in info_logs[1]
assert "订阅成功" not in "\n".join(info_logs)
assert chain._subscription_execution_admission.release(search_lease) is True
@@ -293,9 +288,8 @@ def test_match_summary_distinguishes_ttl_timeout(monkeypatch) -> None:
chain.match({"site.example": [object()]})
assert len(info_logs) == 2
assert "task_skipped=1" in info_logs[1]
assert "cancelled=0" in info_logs[1]
assert "ttl_timeouts=1" in info_logs[1]
assert "这次未检查 1 个" in info_logs[1]
assert "另有 1 个订阅检查时间过长,已停止" in info_logs[1]
def test_match_logs_subscription_admission_release_failure(monkeypatch) -> None:
@@ -323,5 +317,5 @@ def test_match_logs_subscription_admission_release_failure(monkeypatch) -> None:
chain.match({"site.example": [object()]})
assert any("订阅准入释放失败: operation=match subscription_id=17" in item for item in error_logs)
assert "release_failures=1" in info_logs[-1]
assert any("订阅 17 的搜索状态没有正常恢复,系统稍后会继续检查" in item for item in error_logs)
assert "另有 1 个订阅的搜索状态没有正常恢复" in info_logs[-1]
+134 -43
View File
@@ -82,6 +82,7 @@ def _chain(tmp_path, subscribes: list[SubscriptionSnapshot]):
chain._match_lock = _ForbiddenLock()
chain._search_queue_lock = threading.Lock()
chain._subscription_execution_admission = SubscriptionExecutionAdmission()
chain.messagehelper = Mock()
return chain
@@ -179,11 +180,9 @@ def test_successful_sites_remain_available_to_next_due_subscription(tmp_path, mo
assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)]
assert chain.get_search_batch(batch_id).state == "completed"
assert "sites=2" in info_logs[-1]
assert "site_requests=4" in info_logs[-1]
assert "site_failures=0" in info_logs[-1]
assert "site_cooldown_skips=0" in info_logs[-1]
assert "candidates=4" in info_logs[-1]
assert "访问 2 个站点" in info_logs[-1]
assert "发出 4 次请求" in info_logs[-1]
assert "找到 4 个资源" in info_logs[-1]
def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch):
@@ -280,12 +279,12 @@ def test_site_budget_conflict_requeues_task_without_batch_failure(tmp_path, monk
assert task.state == "queued"
assert task.phase == "waiting_site_budget"
assert task.available_at == retry_at
assert task.last_error is None
assert task.last_error == "站点暂时忙,系统会自动继续搜索"
assert chain.subscription_search_repository.claim_next(owner="worker-after-retry") is None
def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch):
"""Search INFO 只保留轮次摘要,并携带任务终态与耗时字段"""
def test_search_logs_one_readable_start_and_finish_summary(tmp_path, monkeypatch):
"""Search INFO 只保留用户能直接读懂的开始和结束摘要"""
subscribes = [_subscribe(50), _subscribe(51)]
chain = _chain(tmp_path, subscribes)
_make_tasks_ready(monkeypatch)
@@ -305,19 +304,14 @@ def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch)
assert batch_id
assert len(info_logs) == 2
assert info_logs[0].startswith("订阅治理轮次开始: operation=search ")
assert f"batch_id={batch_id}" in info_logs[0]
assert "subscriptions=2" in info_logs[0]
assert info_logs[1].startswith("订阅治理轮次结束: operation=search ")
assert "state=failed" in info_logs[1]
assert "processed=2" in info_logs[1]
assert "task_completed=1" in info_logs[1]
assert "task_failed=1" in info_logs[1]
assert "admission_conflicts=0" in info_logs[1]
assert "ttl_timeouts=0" in info_logs[1]
assert "site_requests=0" in info_logs[1]
assert "duration_ms=" in info_logs[1]
assert "订阅成功" not in "\n".join(info_logs)
assert info_logs[0] == "开始订阅定时检查,共 2 个订阅。"
assert info_logs[1].startswith("订阅定时检查结束:部分订阅没有完成。")
assert "本次处理 2/2 个" in info_logs[1]
assert "完成 1 个" in info_logs[1]
assert "失败 1 个" in info_logs[1]
assert "发出 0 次请求" in info_logs[1]
assert "用时 " in info_logs[1]
assert "operation=" not in "\n".join(info_logs)
def test_search_logs_subscription_admission_release_failure(tmp_path, monkeypatch):
@@ -343,8 +337,8 @@ def test_search_logs_subscription_admission_release_failure(tmp_path, monkeypatc
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.search(state="R")
assert any("订阅准入释放失败: operation=search subscription_id=52" in item for item in error_logs)
assert "release_failures=1" in info_logs[-1]
assert any("订阅《治理电影 52》的搜索状态没有正常恢复" in item for item in error_logs)
assert "另有 1 个搜索状态没有正常恢复" in info_logs[-1]
def test_queued_search_logs_failed_finish_summary_when_callback_raises(tmp_path, monkeypatch):
@@ -361,9 +355,7 @@ def test_queued_search_logs_failed_finish_summary_when_callback_raises(tmp_path,
)
assert len(info_logs) == 2
assert info_logs[-1].startswith("订阅治理轮次结束: operation=search ")
assert "state=failed" in info_logs[-1]
assert "round_failed=1" in info_logs[-1]
assert info_logs[-1].startswith("订阅定时检查结束:部分订阅没有完成。")
def test_resume_search_logs_failed_finish_summary_when_drain_raises(tmp_path, monkeypatch):
@@ -380,11 +372,8 @@ def test_resume_search_logs_failed_finish_summary_when_drain_raises(tmp_path, mo
with pytest.raises(RuntimeError, match="claim failed"):
chain.resume_search_queue()
assert len(info_logs) == 2
assert info_logs[-1].startswith("订阅治理轮次结束: operation=search ")
assert "source=resume" in info_logs[-1]
assert "state=failed" in info_logs[-1]
assert "round_failed=1" in info_logs[-1]
assert len(info_logs) == 1
assert info_logs[-1].startswith("等待中的订阅搜索结束:部分订阅没有完成。")
def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch):
@@ -457,10 +446,9 @@ def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp
assert batch.failed_count == 1
assert "Flaky" in batch.last_error
assert "HTTP 429" in batch.last_error
assert "sites=2" in info_logs[-1]
assert "site_requests=2" in info_logs[-1]
assert "site_failures=1" in info_logs[-1]
assert "cooldown_seconds=900.0" in info_logs[-1]
assert "失败 1 个" in info_logs[-1]
assert "访问 2 个站点" in info_logs[-1]
assert "发出 2 次请求" in info_logs[-1]
def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch):
@@ -487,12 +475,116 @@ def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeyp
assert batch.state == "skipped"
assert batch.finished_count == 0
assert batch.skipped_count == 1
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
assert "task_skipped=1" in info_logs[-1]
assert "admission_conflicts=1" in info_logs[-1]
assert batch.last_error == "这个订阅正在处理,本次自动检查无需重复执行"
assert "这次未搜索 1 个" in info_logs[-1]
assert chain._subscription_execution_admission.release(match_lease) is True
def test_manual_search_waits_for_same_subscription_then_continues(tmp_path, monkeypatch):
"""手动搜索遇到同一订阅正在处理时应稍后自动继续,而不是结束为跳过。"""
subscribe = _subscribe(61)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(return_value=subscribe)
monkeypatch.setattr(chain, "_process_search_subscription", process)
match_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert match_lease is not None
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(sid=subscribe.id, state=None, manual=True)
batch = chain.get_search_batch(batch_id)
assert batch.state == "queued"
process.assert_not_called()
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.phase == "waiting_subscription"
assert task.last_error == "这个订阅正在处理,结束后会自动继续搜索"
task.available_at = "1970-01-01T00:00:00+00:00"
session.commit()
assert chain._subscription_execution_admission.release(match_lease) is True
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.resume_search_queue(limit=1)
process.assert_called_once()
assert chain.get_search_batch(batch_id).state == "completed"
def test_recent_new_subscription_is_scheduled_instead_of_skipped(tmp_path, monkeypatch):
"""新订阅的一分钟编辑窗口应保留任务并在到时后自动继续。"""
subscribe = replace(
_subscribe(62),
state="N",
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(side_effect=AssertionError("编辑等待期内不应开始搜索"))
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="N")
batch = chain.get_search_batch(batch_id)
assert batch.state == "queued"
assert batch.skipped_count == 0
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.phase == "scheduled"
assert task.last_error == "订阅刚刚创建,保存好设置后会自动开始搜索"
assert task.available_at > datetime.now(timezone.utc).isoformat(timespec="seconds")
process.assert_not_called()
def test_new_subscription_post_commit_creates_scheduled_search(tmp_path):
"""订阅保存后应立即写入自动搜索计划,无需等待五分钟扫描。"""
subscribe = replace(
_subscribe(64),
state="N",
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
batch_id = chain._SubscribeChain__queue_new_subscription_search(subscribe.id)
assert batch_id
assert chain.get_search_batch(batch_id).state == "queued"
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.source == "new"
assert task.phase == "scheduled"
assert task.available_at > datetime.now(timezone.utc).isoformat(timespec="seconds")
def test_manual_search_bypasses_new_subscription_edit_wait(tmp_path, monkeypatch):
"""用户主动点击搜索时不受新订阅编辑等待时间限制。"""
subscribe = replace(
_subscribe(63),
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(return_value=subscribe)
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(sid=subscribe.id, state=None, manual=True)
process.assert_called_once()
assert chain.get_search_batch(batch_id).state == "completed"
def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch):
"""准入后重新读取到暂停订阅时不得开始搜索或伪造完成。"""
subscribe = _subscribe(9)
@@ -511,7 +603,7 @@ def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkey
assert batch.state == "skipped"
assert batch.finished_count == 0
assert batch.skipped_count == 1
assert batch.last_error == "订阅已暂停,本轮搜索已跳过"
assert batch.last_error == "订阅已暂停,这次没有搜索"
def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch):
@@ -699,9 +791,8 @@ def test_ttl_expiry_after_normal_return_marks_task_and_batch_failed(tmp_path, mo
assert batch.finished_count == 0
assert batch.failed_count == 1
assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间"
assert "task_failed=1" in info_logs[-1]
assert "ttl_timeouts=1" in info_logs[-1]
assert batch.last_error == "这次搜索用时过长,已停止,可稍后重试"
assert "失败 1 个" in info_logs[-1]
def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch):
@@ -758,4 +849,4 @@ def test_site_budget_ttl_expiry_marks_task_and_batch_failed(tmp_path, monkeypatc
assert batch.finished_count == 0
assert batch.failed_count == 1
assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间"
assert batch.last_error == "这次搜索用时过长,已停止,可稍后重试"
+39 -4
View File
@@ -40,7 +40,9 @@ def test_search_queue_coalesces_active_subscription_and_raises_priority(tmp_path
assert manual.created_count == 0
assert manual.coalesced_count == 1
assert manual.batch.state == "completed"
assert manual.active_batch_ids == (scheduled.batch.batch_id,)
assert first.subscription_id == 1
assert first.source == "manual"
assert first.priority == 100
assert second.subscription_id == 2
assert first.task_id != second.task_id
@@ -82,6 +84,39 @@ def test_search_queue_claims_each_subscription_only_after_its_available_at(tmp_p
assert accelerated.priority == 100
def test_manual_search_promotes_scheduled_new_subscription(tmp_path):
"""用户主动搜索应立即唤醒仍在编辑等待期的新订阅任务。"""
repository, engine = _repository(tmp_path)
later_at = (datetime.now(timezone.utc) + timedelta(minutes=1)).isoformat(timespec="seconds")
automatic = repository.enqueue(
subscription_ids=(22,),
source="new",
priority=50,
available_at_by_subscription={22: later_at},
)
with Session(engine) as session:
scheduled = session.execute(
select(SubscriptionSearchTask).where(SubscriptionSearchTask.subscription_id == 22)
).scalar_one()
assert scheduled.phase == "scheduled"
manual = repository.enqueue(
subscription_ids=(22,),
source="manual",
priority=120,
available_at_by_subscription={22: "1970-01-01T00:00:00+00:00"},
)
claimed = repository.claim_next(owner="worker-manual")
assert manual.created_count == 0
assert manual.active_batch_ids == (automatic.batch.batch_id,)
assert claimed is not None
assert claimed.source == "manual"
assert claimed.priority == 120
assert claimed.phase == "matching"
def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path):
"""进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。"""
repository, engine = _repository(tmp_path)
@@ -269,8 +304,8 @@ def test_search_queue_aggregates_skipped_tasks_without_marking_success(tmp_path)
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
"""手工任务可优先,但等待超过公平窗口的兜底任务不得持续饥饿"""
def test_search_queue_keeps_manual_work_ahead_of_aged_fallback(tmp_path):
"""用户主动搜索始终先于定时检查,避免点击后长时间没有反馈"""
repository, engine = _repository(tmp_path)
repository.enqueue(subscription_ids=(8,), source="fallback", priority=10)
aged_at = (datetime.now(timezone.utc) - timedelta(minutes=16)).isoformat(timespec="seconds")
@@ -285,5 +320,5 @@ def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
claimed = repository.claim_next(owner="worker-a")
assert claimed.subscription_id == 8
assert claimed.source == "fallback"
assert claimed.subscription_id == 9
assert claimed.source == "manual"
+3
View File
@@ -35,6 +35,7 @@ def _repository(tmp_path):
def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path):
"""同站点第二个调用必须等待,不同站点可立即并行。"""
repository, _engine = _repository(tmp_path)
before_retry = datetime.now(timezone.utc)
first = repository.claim_site(site_id=1, owner="worker-a", lease_seconds=900)
same_site = repository.claim_site(site_id=1, owner="worker-b", lease_seconds=900)
@@ -43,6 +44,8 @@ def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path
assert first.acquired is True
assert same_site.acquired is False
assert same_site.lease_token is None
assert same_site.wait_reason == "busy"
assert datetime.fromisoformat(same_site.retry_at) <= before_retry + timedelta(seconds=11)
assert other_site.acquired is True