feat(subscribe): improve search scheduling feedback

This commit is contained in:
jxxghp
2026-09-05 17:23:26 +08:00
parent e003a31064
commit 0a732be485
33 changed files with 1080 additions and 365 deletions
+38 -28
View File
@@ -42,7 +42,10 @@ from app.application.subscription.mutation import (
SubscriptionMutationService, SubscriptionMutationService,
) )
from app.application.subscription.query import SubscriptionQueryService from app.application.subscription.query import SubscriptionQueryService
from app.application.subscription.search import SearchSubscriptionsCommand from app.application.subscription.search import (
SearchSubscriptionsCommand,
SubscriptionSearchSubmission,
)
from app.application.subscription.status import ( from app.application.subscription.status import (
SubscriptionExecutionReadRepository, SubscriptionExecutionReadRepository,
SubscriptionExecutionStatusService, SubscriptionExecutionStatusService,
@@ -120,24 +123,14 @@ def get_delete_subscriptions_by_identity_command(
) )
def _start_subscription_search_batch( def _resume_submitted_subscription_search(
subscribe_ids: tuple[int, ...] | None, subscribe_ids: tuple[int, ...],
state: str | None,
) -> None: ) -> None:
"""把一个请求的搜索目标作为同一调度任务提交""" """唤醒搜索队列,并优先处理本次手工选择的订阅"""
if subscribe_ids is None:
start_scheduler_job( start_scheduler_job(
"subscribe_search", "subscribe_search_queue",
sid=None, limit=max(1, len(subscribe_ids)),
state=state, manual_sids=subscribe_ids,
manual=True,
)
return
start_scheduler_job(
"subscribe_search",
sids=subscribe_ids,
state=None,
manual=True,
) )
@@ -146,23 +139,40 @@ def get_search_subscriptions_command(
db: AsyncSession = Depends(get_async_session), db: AsyncSession = Depends(get_async_session),
runtime: HostRuntime = Depends(get_host_runtime), runtime: HostRuntime = Depends(get_host_runtime),
) -> SearchSubscriptionsCommand: ) -> SearchSubscriptionsCommand:
"""组装手工订阅搜索用例,并把调度延迟到响应后的后台任务""" """组装手工搜索用例,请求内只入队,实际搜索交给后台继续"""
def schedule_search( registry = resolve_background_task_registry(task_registry)
subscribe_ids: tuple[int, ...] | None, search_repository = get_subscription_search_repository(runtime)
state: str | None,
) -> None: async def submit_search(
"""把当前用户的搜索目标提交为一个顺序后台批次。""" subscribe_ids: tuple[int, ...],
resolve_background_task_registry(task_registry).create_sync( single: bool,
_start_subscription_search_batch, ) -> SubscriptionSearchSubmission:
"""在线程 owner 中完成轻量入队,并返回前端可立即跟踪的批次。"""
enqueue_task = registry.create_sync(
search_repository.enqueue,
subscription_ids=subscribe_ids,
source="manual",
priority=120 if single else 100,
owner="api.subscribe.search.enqueue",
)
enqueued = await enqueue_task
registry.create_sync(
_resume_submitted_subscription_search,
subscribe_ids, subscribe_ids,
state, owner="api.subscribe.search.run",
owner="api.subscribe.search", )
return SubscriptionSearchSubmission(
batch_ids=enqueued.active_batch_ids,
target_count=len(subscribe_ids),
queued_count=enqueued.created_count,
ongoing_count=enqueued.coalesced_count,
single=single,
) )
return SearchSubscriptionsCommand( return SearchSubscriptionsCommand(
repository=runtime.subscription.repository(db), repository=runtime.subscription.repository(db),
schedule_search=schedule_search, submit_search=submit_search,
) )
+6 -6
View File
@@ -32,7 +32,7 @@ async def _accessible_subscription_ids(
@router.get( # type: ignore[misc] @router.get( # type: ignore[misc]
"/execution/batches", "/execution/batches",
summary="订阅搜索批次状态", summary="订阅搜索进度",
response_model=List[_SchemaSubscriptionBatchStatus], response_model=List[_SchemaSubscriptionBatchStatus],
) )
async def list_subscription_execution_batches( async def list_subscription_execution_batches(
@@ -52,7 +52,7 @@ async def list_subscription_execution_batches(
@router.get( # type: ignore[misc] @router.get( # type: ignore[misc]
"/execution/batches/{batch_id}", "/execution/batches/{batch_id}",
summary="订阅搜索批次", summary="看一次订阅搜索",
response_model=_SchemaSubscriptionBatchStatus, response_model=_SchemaSubscriptionBatchStatus,
) )
async def get_subscription_execution_batch( async def get_subscription_execution_batch(
@@ -68,13 +68,13 @@ async def get_subscription_execution_batch(
accessible_subscription_ids=accessible_ids, accessible_subscription_ids=accessible_ids,
) )
if batch is None: if batch is None:
raise HTTPException(status_code=404, detail="订阅搜索批次不存在") raise HTTPException(status_code=404, detail="没有找到这次搜索,请刷新后重试")
return _SchemaSubscriptionBatchStatus.model_validate(batch) return _SchemaSubscriptionBatchStatus.model_validate(batch)
@router.put( # type: ignore[misc] @router.put( # type: ignore[misc]
"/execution/batches/{batch_id}/cancel", "/execution/batches/{batch_id}/cancel",
summary="取消订阅搜索批次", summary="停止一次订阅搜索",
response_model=_SchemaResponse[None], response_model=_SchemaResponse[None],
) )
async def cancel_subscription_execution_batch( async def cancel_subscription_execution_batch(
@@ -90,9 +90,9 @@ async def cancel_subscription_execution_batch(
accessible_subscription_ids=accessible_ids, accessible_subscription_ids=accessible_ids,
) )
if batch is None: if batch is None:
return _SchemaResponse(success=False, message="订阅搜索批次不存在") return _SchemaResponse(success=False, message="没有找到这次搜索,请刷新后重试")
cancelled = await status_service.request_cancel(batch_id) cancelled = await status_service.request_cancel(batch_id)
return _SchemaResponse( return _SchemaResponse(
success=bool(cancelled), success=bool(cancelled),
message="" if cancelled else "订阅搜索批次已结束或无法取消", message="" if cancelled else "这次搜索已经结束,暂时无法停止",
) )
+59 -9
View File
@@ -22,12 +22,50 @@ from app.application.subscription.mutation import (
from app.application.subscription.search import ( from app.application.subscription.search import (
SearchSubscriptionsCommand, SearchSubscriptionsCommand,
SubscribeSearchActor, SubscribeSearchActor,
SubscriptionSearchSubmission,
) )
from app.schemas.response import Response from app.schemas.response import Response
from app.schemas.subscribe import (
SubscriptionSearchSubmission as SubscriptionSearchSubmissionSchema,
)
router = ResponseAPIRouter() router = ResponseAPIRouter()
def _search_submission_message(submission: SubscriptionSearchSubmission) -> str:
"""把搜索安排结果转换为用户能直接理解的提示。"""
if submission.target_count == 0:
return "没有需要搜索的订阅"
if submission.queued_count == 0:
return (
"这个订阅已经在搜索中,请稍候"
if submission.single
else f"{submission.ongoing_count} 个订阅已经在搜索中,无需重复提交"
)
if submission.ongoing_count:
return (
f"已安排 {submission.queued_count} 个订阅搜索,"
f"另有 {submission.ongoing_count} 个正在处理中"
)
if submission.single:
return "已安排搜索,很快开始"
return f"已安排 {submission.queued_count} 个订阅搜索,系统会依次处理"
def _search_submission_schema(
submission: SubscriptionSearchSubmission,
) -> SubscriptionSearchSubmissionSchema:
"""构造稳定的手工搜索响应数据。"""
return SubscriptionSearchSubmissionSchema(
batch_id=submission.batch_id,
batch_ids=list(submission.batch_ids),
target_count=submission.target_count,
queued_count=submission.queued_count,
ongoing_count=submission.ongoing_count,
single=submission.single,
)
@router.get( # type: ignore[misc] @router.get( # type: ignore[misc]
"/refresh", "/refresh",
summary="刷新订阅(兼容入口)", summary="刷新订阅(兼容入口)",
@@ -97,38 +135,46 @@ def check_subscribes(
@router.get( # type: ignore[misc] @router.get( # type: ignore[misc]
"/search", "/search",
summary="搜索所有订阅(兼容入口)", summary="搜索所有订阅(兼容入口)",
response_model=Response[None], response_model=Response[SubscriptionSearchSubmissionSchema],
include_in_schema=False, include_in_schema=False,
deprecated=True, deprecated=True,
) )
@router.post( # type: ignore[misc] @router.post( # type: ignore[misc]
"/search", summary="搜索所有订阅", response_model=Response[None] "/search",
summary="搜索所有订阅",
response_model=Response[SubscriptionSearchSubmissionSchema],
) )
async def search_subscribes( async def search_subscribes(
command: SearchSubscriptionsCommand = Depends(get_search_subscriptions_command), command: SearchSubscriptionsCommand = Depends(get_search_subscriptions_command),
current_user: ApiPrincipal = Depends(get_current_active_user_async), current_user: ApiPrincipal = Depends(get_current_active_user_async),
) -> Any: ) -> Any:
"""搜索当前用户可管理的全部订阅。""" """搜索当前用户可管理的全部订阅。"""
await command.execute( submission = await command.execute(
SubscribeSearchActor( SubscribeSearchActor(
username=current_user.name, username=current_user.name,
is_superuser=current_user.is_superuser, is_superuser=current_user.is_superuser,
) )
) )
return Response(success=True) if submission is None:
return Response(success=False, message="没有需要搜索的订阅")
return Response(
success=True,
message=_search_submission_message(submission),
data=_search_submission_schema(submission),
)
@router.get( # type: ignore[misc] @router.get( # type: ignore[misc]
"/search/{subscribe_id}", "/search/{subscribe_id}",
summary="搜索订阅(兼容入口)", summary="搜索订阅(兼容入口)",
response_model=Response[None], response_model=Response[SubscriptionSearchSubmissionSchema],
include_in_schema=False, include_in_schema=False,
deprecated=True, deprecated=True,
) )
@router.post( # type: ignore[misc] @router.post( # type: ignore[misc]
"/search/{subscribe_id}", "/search/{subscribe_id}",
summary="搜索订阅", summary="搜索订阅",
response_model=Response[None], response_model=Response[SubscriptionSearchSubmissionSchema],
) )
async def search_subscribe( async def search_subscribe(
subscribe_id: int, subscribe_id: int,
@@ -136,13 +182,17 @@ async def search_subscribe(
current_user: ApiPrincipal = Depends(get_current_active_user_async), current_user: ApiPrincipal = Depends(get_current_active_user_async),
) -> Any: ) -> Any:
"""根据订阅编号搜索一个订阅。""" """根据订阅编号搜索一个订阅。"""
found = await command.execute( submission = await command.execute(
SubscribeSearchActor( SubscribeSearchActor(
username=current_user.name, username=current_user.name,
is_superuser=current_user.is_superuser, is_superuser=current_user.is_superuser,
), ),
subscribe_id=subscribe_id, subscribe_id=subscribe_id,
) )
if not found: if submission is None:
return Response(success=False, message="订阅不存在") return Response(success=False, message="订阅不存在")
return Response(success=True) return Response(
success=True,
message=_search_submission_message(submission),
data=_search_submission_schema(submission),
)
+6 -2
View File
@@ -596,8 +596,12 @@ class SubscriptionStagingPort(Protocol):
"""异步按媒体身份读取删除候选快照。""" """异步按媒体身份读取删除候选快照。"""
... ...
async def list_search_ids(self, username: str, state: str) -> builtins.list[int]: async def list_search_ids(
"""异步读取用户可搜索订阅主键。""" self,
username: Optional[str],
state: str,
) -> builtins.list[int]:
"""异步读取用户或管理员全局范围内可搜索的订阅主键。"""
... ...
async def stage_delete(self, subscribe_id: int) -> None: async def stage_delete(self, subscribe_id: int) -> None:
+7 -2
View File
@@ -125,11 +125,13 @@ def handle_subscription_search_deferred(
deferred: SubscriptionSearchDeferred, deferred: SubscriptionSearchDeferred,
record: Callable[..., None], record: Callable[..., None],
) -> None: ) -> None:
"""把站点预算冲突重新入队,并记录为可恢复而非失败的任务结果""" """把站点暂时不可用的任务重新入队,而不是记录为搜索失败"""
requeued = queue.defer_task( requeued = queue.defer_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
available_at=deferred.retry_at, available_at=deferred.retry_at,
phase="waiting_site_budget",
message="站点暂时忙,系统会自动继续搜索",
) )
if requeued: if requeued:
record("requeued", "site_budget_deferred") record("requeued", "site_budget_deferred")
@@ -187,6 +189,7 @@ class SearchEnqueueResult:
batch: SearchBatchSnapshot batch: SearchBatchSnapshot
created_count: int created_count: int
coalesced_count: int coalesced_count: int
active_batch_ids: tuple[str, ...]
class SubscriptionSearchRepository(Protocol): class SubscriptionSearchRepository(Protocol):
@@ -245,8 +248,10 @@ class SubscriptionSearchRepository(Protocol):
task_id: str, task_id: str,
lease_token: str, lease_token: str,
available_at: str, available_at: str,
phase: str = "waiting_site_budget",
message: Optional[str] = None,
) -> bool: ) -> bool:
"""把临时站点预算冲突任务退回队列,并设置下一次领取时间""" """把临时不可执行任务退回队列,并保留用户可理解的等待原因"""
... ...
def is_cancel_requested(self, task_id: str) -> bool: def is_cancel_requested(self, task_id: str) -> bool:
+94 -50
View File
@@ -16,30 +16,73 @@ from app.application.subscription.sitebudget import SubscriptionSiteBudgetMetric
SearchTaskOutcome = Literal["completed", "skipped", "failed", "cancelled", "requeued"] SearchTaskOutcome = Literal["completed", "skipped", "failed", "cancelled", "requeued"]
_SEARCH_SOURCE_NAMES = {
"manual": "手动订阅搜索",
"targeted": "指定订阅搜索",
"new": "新订阅自动搜索",
"fallback": "订阅定时检查",
"resume": "等待中的订阅搜索",
"inline": "订阅搜索",
}
_SEARCH_STATE_NAMES = {
"completed": "已完成",
"failed": "部分订阅没有完成",
"cancelled": "已停止",
"skipped": "部分订阅这次未搜索",
"queued": "部分订阅稍后继续",
"running": "仍在处理中",
"cancelling": "正在停止",
"stopped": "部分订阅稍后继续",
}
_MATCH_STATE_NAMES = {
"completed": "检查完成",
"failed": "部分订阅没有完成",
"cancelled": "已停止",
"skipped": "部分订阅这次未检查",
"stopped": "已停止,部分订阅这次未检查",
}
def _search_source_name(source: str) -> str:
"""返回搜索来源对应的可读名称。"""
return _SEARCH_SOURCE_NAMES.get(source, "订阅搜索")
def _search_state_name(state: str) -> str:
"""返回搜索结束状态对应的可读说明。"""
return _SEARCH_STATE_NAMES.get(state, "已结束")
def _match_state_name(state: str) -> str:
"""返回订阅资源检查状态对应的可读说明。"""
return _MATCH_STATE_NAMES.get(state, "已结束")
def batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str: def batch_progress_text(batch: Optional[SearchBatchSnapshot]) -> str:
"""把批次聚合终态转为兼容进度文案。""" """把批次聚合终态转为兼容进度文案。"""
if batch is None: if batch is None:
return "订阅搜索任务已提交" return "搜索已安排"
if batch.state == "failed": if batch.state == "failed":
return "订阅搜索完成,部分任务失败" return "搜索结束,部分订阅没有完成"
if batch.state == "cancelled": if batch.state == "cancelled":
return "订阅搜索已取消" return "搜索已停止"
if batch.state == "skipped": if batch.state == "skipped":
return "订阅搜索完成,部分任务本轮已跳过" return "搜索结束,部分订阅这次未搜索"
if batch.state in {"queued", "running", "cancelling"}: if batch.state in {"queued", "running", "cancelling"}:
return "订阅搜索任务已排队" return "搜索已安排,系统正在依次处理"
if batch.skipped_count: if batch.skipped_count:
return "订阅搜索完成,部分任务本轮已跳过" return "搜索结束,部分订阅这次未搜索"
return "订阅搜索完成" return "搜索完成"
def inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]: def inline_search_result(total: int, finished: int) -> tuple[str, dict[str, int]]:
"""返回兼容搜索的真实终态文案与计数。""" """返回兼容搜索的真实终态文案与计数。"""
text = ( text = (
"订阅搜索完成" "搜索完成"
if finished == total if finished == total
else "订阅搜索结束,部分订阅本轮未执行或未完成" else "搜索结束,部分订阅这次没有完成"
) )
return text, {"total": total, "finished": finished} return text, {"total": total, "finished": finished}
@@ -77,14 +120,14 @@ def finish_returned_search_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="completed", state="completed",
error="执行截止时间晚于下载提交边界,已按实际结果完成", error="下载已经提交,虽然搜索用时较长,结果仍然有效",
) )
return subscription_id, "completed", "ttl_timeout" return subscription_id, "completed", "ttl_timeout"
queue.finish_task( queue.finish_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="failed", state="failed",
error="订阅执行已超过协作截止时间", error="这次搜索用时过长,已停止,可稍后重试",
) )
return None, "failed", "ttl_timeout" return None, "failed", "ttl_timeout"
if system_stopped: if system_stopped:
@@ -93,7 +136,7 @@ def finish_returned_search_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="completed", state="completed",
error="停机请求晚于下载提交边界,已按实际结果完成", error="下载已经提交,系统停止后仍保留这次结果",
) )
return subscription_id, "completed", "system_stop" return subscription_id, "completed", "system_stop"
queue.release_task(task_id=task_id, lease_token=lease_token) queue.release_task(task_id=task_id, lease_token=lease_token)
@@ -104,7 +147,7 @@ def finish_returned_search_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="completed", state="completed",
error="取消请求晚于下载提交边界,已按实际结果完成", error="下载已经提交,无法撤回,已保留这次结果",
) )
return subscription_id, "completed", "cancelled" return subscription_id, "completed", "cancelled"
queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True) queue.release_task(task_id=task_id, lease_token=lease_token, cancelled=True)
@@ -173,26 +216,31 @@ class MatchExecutionSummary:
return "completed" return "completed"
def start_log(self) -> str: def start_log(self) -> str:
"""构造 Match 轮次开始日志。""" """构造不暴露内部标记的资源检查开始日志。"""
return ( return (
"订阅治理轮次开始: operation=match " f"开始检查订阅资源,共 {self.total} 个订阅、{self.candidate_count} 个资源,"
f"run_id={self.run_id} subscriptions={self.total} " f"来自 {self.site_count} 个站点。"
f"sites={self.site_count} candidates={self.candidate_count}"
) )
def finish_log(self) -> str: def finish_log(self) -> str:
"""构造 Match 轮次结束日志,completed 表示任务处理完成而非订阅成功""" """构造可直接阅读的资源检查结束日志"""
elapsed_ms = (time.perf_counter() - self.started_at) * 1000 elapsed_seconds = time.perf_counter() - self.started_at
return ( message = (
"订阅治理轮次结束: operation=match " f"订阅资源检查结束:{_match_state_name(self.state)}"
f"run_id={self.run_id} state={self.state} subscriptions={self.total} " f"本次检查 {self.finished}/{self.total} 个订阅,完成 {self.completed} 个,"
f"processed={self.finished} task_completed={self.completed} " f"这次未检查 {self.skipped} 个,失败 {self.failed} 个;"
f"task_skipped={self.skipped} task_failed={self.failed} " f"共检查 {self.candidate_count} 个资源,来自 {self.site_count} 个站点,"
f"admission_conflicts={self.admission_conflicts} cancelled={self.cancelled} " f"用时 {elapsed_seconds:.1f} 秒。"
f"ttl_timeouts={self.ttl_timeouts} release_failures={self.release_failures} "
f"sites={self.site_count} candidates={self.candidate_count} "
f"duration_ms={elapsed_ms:.1f}"
) )
if self.admission_conflicts:
message += f"其中 {self.admission_conflicts} 个订阅正在处理中,本次没有重复检查。"
if self.cancelled:
message += f"另有 {self.cancelled} 个订阅已停止。"
if self.ttl_timeouts:
message += f"另有 {self.ttl_timeouts} 个订阅检查时间过长,已停止。"
if self.release_failures:
message += f"另有 {self.release_failures} 个订阅的搜索状态没有正常恢复,系统稍后会继续检查。"
return message
def as_data(self, current: Optional[int] = None) -> dict[str, int]: def as_data(self, current: Optional[int] = None) -> dict[str, int]:
"""构造供进度消费者读取的实际计数。""" """构造供进度消费者读取的实际计数。"""
@@ -256,15 +304,14 @@ class SearchExecutionSummary:
self.ttl_timeouts += 1 self.ttl_timeouts += 1
def start_log(self) -> str: def start_log(self) -> str:
"""构造 Search 轮次开始日志。""" """构造不暴露内部术语的 Search 开始日志。"""
return ( message = f"开始{_search_source_name(self.source)},共 {self.requested} 个订阅"
"订阅治理轮次开始: operation=search " if self.coalesced:
f"run_id={self.run_id} batch_id={self.batch_id or '-'} source={self.source} " message += f",其中 {self.coalesced} 个已经在处理中"
f"subscriptions={self.requested} coalesced={self.coalesced}" return f"{message}"
)
def finish_log(self, batch: Optional[SearchBatchSnapshot] = None) -> str: def finish_log(self, batch: Optional[SearchBatchSnapshot] = None) -> str:
"""构造 Search 轮次结束日志,任务完成与订阅完成保持分离""" """构造可直接阅读的 Search 结束日志。"""
sites = self.site_metrics.snapshot() sites = self.site_metrics.snapshot()
if self.round_failed: if self.round_failed:
state = "failed" state = "failed"
@@ -282,19 +329,16 @@ class SearchExecutionSummary:
state = "skipped" state = "skipped"
else: else:
state = "completed" state = "completed"
elapsed_ms = (time.perf_counter() - self.started_at) * 1000 elapsed_seconds = time.perf_counter() - self.started_at
return ( message = (
"订阅治理轮次结束: operation=search " f"{_search_source_name(self.source)}结束:{_search_state_name(state)}"
f"run_id={self.run_id} batch_id={self.batch_id or '-'} source={self.source} " f"本次处理 {self.processed}/{self.requested} 个,完成 {self.completed} 个,"
f"state={state} subscriptions={self.requested} processed={self.processed} " f"这次未搜索 {self.skipped} 个,失败 {self.failed} 个,"
f"task_completed={self.completed} task_skipped={self.skipped} " f"稍后继续 {self.requeued} 个,已停止 {self.cancelled} 个;"
f"task_failed={self.failed} task_cancelled={self.cancelled} task_requeued={self.requeued} " f"访问 {sites.site_count} 个站点,发出 {sites.request_count} 次请求,"
f"admission_conflicts={self.admission_conflicts} cancelled={self.cancelled} " f"找到 {sites.candidate_count} 个资源,用时 {elapsed_seconds:.1f} 秒。"
f"ttl_timeouts={self.ttl_timeouts} consumer_conflicts={self.consumer_conflicts} "
f"round_failed={int(self.round_failed)} "
f"sites={sites.site_count} site_requests={sites.request_count} "
f"site_failures={sites.failure_count} site_cooldown_skips={sites.cooldown_skip_count} "
f"candidates={sites.candidate_count} cooldown_seconds={sites.cooldown_seconds:.1f} "
f"release_failures={self.release_failures + sites.release_failure_count} "
f"duration_ms={elapsed_ms:.1f}"
) )
unfinished_cleanup = self.release_failures + sites.release_failure_count
if unfinished_cleanup:
message += f"另有 {unfinished_cleanup} 个搜索状态没有正常恢复,系统稍后会继续处理。"
return message
+40 -19
View File
@@ -1,7 +1,8 @@
"""手工订阅搜索应用用例。""" """手工订阅搜索应用用例。"""
from collections.abc import Awaitable, Callable
from dataclasses import dataclass from dataclasses import dataclass
from typing import Callable from typing import Optional
from app.application.subscription.contract import ( from app.application.subscription.contract import (
SubscribeDeletionCandidate, SubscribeDeletionCandidate,
@@ -17,45 +18,65 @@ class SubscribeSearchActor:
is_superuser: bool is_superuser: bool
SubscribeSearchScheduler = Callable[[tuple[int, ...] | None, str | None], None] @dataclass(frozen=True, slots=True)
class SubscriptionSearchSubmission:
"""一次手工搜索请求的入队结果。"""
batch_ids: tuple[str, ...]
target_count: int
queued_count: int
ongoing_count: int
single: bool
@property
def batch_id(self) -> Optional[str]:
"""返回最适合前端立即跟踪的批次编号。"""
return self.batch_ids[0] if self.batch_ids else None
SubscribeSearchSubmitter = Callable[
[tuple[int, ...], bool],
Awaitable[SubscriptionSearchSubmission],
]
class SearchSubscriptionsCommand: class SearchSubscriptionsCommand:
"""按用户权限生成并提交手工订阅搜索任务""" """按用户权限读取目标,并把手工搜索轻量提交到持久队列"""
def __init__( def __init__(
self, self,
repository: SubscriptionStagingPort, repository: SubscriptionStagingPort,
schedule_search: SubscribeSearchScheduler, submit_search: SubscribeSearchSubmitter,
) -> None: ) -> None:
"""注入订阅读取端口和后台任务提交端口。""" """注入订阅读取端口和持久队列提交端口。"""
self._repository = repository self._repository = repository
self._schedule_search = schedule_search self._submit_search = submit_search
async def execute( async def execute(
self, self,
actor: SubscribeSearchActor, actor: SubscribeSearchActor,
subscribe_id: int | None = None, subscribe_id: int | None = None,
) -> bool: ) -> Optional[SubscriptionSearchSubmission]:
"""提交单条或当前用户全部可搜索订阅,返回目标是否存在""" """提交单条或当前用户全部可搜索订阅,目标不可访问时返回空"""
if subscribe_id is not None: if subscribe_id is not None:
candidate = await self._repository.get_candidate(subscribe_id) candidate = await self._repository.get_candidate(subscribe_id)
if not self._can_access(candidate, actor): if not self._can_access(candidate, actor):
return False return None
self._schedule_search((subscribe_id,), None) return await self._submit_search((subscribe_id,), True)
return True
if actor.is_superuser:
self._schedule_search(None, "R")
return True
subscribe_ids = await self._repository.list_search_ids( subscribe_ids = await self._repository.list_search_ids(
actor.username, None if actor.is_superuser else actor.username,
"R", "R",
) )
if subscribe_ids: if not subscribe_ids:
self._schedule_search(tuple(subscribe_ids), None) return SubscriptionSearchSubmission(
return True batch_ids=(),
target_count=0,
queued_count=0,
ongoing_count=0,
single=False,
)
return await self._submit_search(tuple(subscribe_ids), False)
@staticmethod @staticmethod
def _can_access( def _can_access(
+16 -6
View File
@@ -27,20 +27,28 @@ class SubscriptionSearchDeferred(RuntimeError):
"""表示订阅搜索未失败,而是应在站点预算可用后重新入队。""" """表示订阅搜索未失败,而是应在站点预算可用后重新入队。"""
def __init__(self, *, retry_at: str, site_ids: tuple[int, ...]) -> None: def __init__(self, *, retry_at: str, site_ids: tuple[int, ...]) -> None:
"""保存队列恢复所需的时间和冲突站点,避免把临时冲突写成错误。""" """保存队列恢复所需的时间和站点,避免把临时等待写成错误。"""
super().__init__(f"订阅搜索已延后,站点预算最早可重试:{retry_at}") super().__init__("站点暂时忙,系统会自动继续搜索")
self.retry_at = retry_at self.retry_at = retry_at
self.site_ids = site_ids self.site_ids = site_ids
class SubscriptionSiteBudgetUnavailable(RuntimeError): class SubscriptionSiteBudgetUnavailable(RuntimeError):
"""表示站点预算暂时不可用,调用方应记录为延后而非失败。""" """表示站点暂时不可用,调用方应记录为等待而非失败。"""
def __init__(self, *, site_id: int, retry_at: str) -> None: def __init__(
self,
*,
site_id: int,
retry_at: str,
wait_reason: Optional[str] = None,
) -> None:
"""保存站点和下一次可尝试时间,供订阅队列恢复。""" """保存站点和下一次可尝试时间,供订阅队列恢复。"""
super().__init__(f"站点 {site_id} 冷却或已有在途搜索,最早可重试:{retry_at}") message = "站点正在处理其他搜索,稍后会自动继续" if wait_reason == "busy" else "站点暂时不可用,稍后会自动重试"
super().__init__(message)
self.site_id = site_id self.site_id = site_id
self.retry_at = retry_at self.retry_at = retry_at
self.wait_reason = wait_reason
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
@@ -52,6 +60,7 @@ class SiteBudgetClaim:
retry_at: str retry_at: str
consecutive_failures: int consecutive_failures: int
lease_token: Optional[str] = None lease_token: Optional[str] = None
wait_reason: Optional[str] = None
@dataclass(frozen=True, slots=True) @dataclass(frozen=True, slots=True)
@@ -195,6 +204,7 @@ class SubscriptionSiteBudget:
raise SubscriptionSiteBudgetUnavailable( raise SubscriptionSiteBudgetUnavailable(
site_id=site_id, site_id=site_id,
retry_at=claim.retry_at, retry_at=claim.retry_at,
wait_reason=claim.wait_reason,
) )
def _report_phase(self, phase: str, site_id: Optional[int]) -> None: def _report_phase(self, phase: str, site_id: Optional[int]) -> None:
@@ -251,4 +261,4 @@ class SubscriptionSiteBudget:
def _raise_if_cancelled(self) -> None: def _raise_if_cancelled(self) -> None:
"""在创建站点租约前传播取消或停机。""" """在创建站点租约前传播取消或停机。"""
if self._stop_state.is_system_stopped or self._cancelled(): if self._stop_state.is_system_stopped or self._cancelled():
raise SubscriptionSearchCancelled("订阅搜索已取消") raise SubscriptionSearchCancelled("搜索已停止")
+6 -2
View File
@@ -18,6 +18,7 @@ class SubscriptionExecutionStatus:
batch_id: Optional[str] = None batch_id: Optional[str] = None
task_id: Optional[str] = None task_id: Optional[str] = None
current_site_id: Optional[int] = None current_site_id: Optional[int] = None
next_run_at: Optional[str] = None
error: Optional[str] = None error: Optional[str] = None
can_cancel: bool = False can_cancel: bool = False
@@ -79,6 +80,8 @@ class SubscriptionExecutionStatusService:
"running", "running",
"matching", "matching",
"searching", "searching",
"scheduled",
"waiting_subscription",
"waiting_site_budget", "waiting_site_budget",
"preparing", "preparing",
"submitting", "submitting",
@@ -153,8 +156,8 @@ class SubscriptionExecutionStatusService:
state = phase = "cancelling" state = phase = "cancelling"
elif task.state == "running": elif task.state == "running":
state = phase = task.phase or "running" state = phase = task.phase or "running"
elif task.state == "queued" and task.phase == "waiting_site_budget": elif task.state == "queued" and task.phase in cls._ACTIVE_STATES:
state = phase = "waiting_site_budget" state = phase = task.phase
else: else:
state = phase = task.state state = phase = task.state
return SubscriptionExecutionStatus( return SubscriptionExecutionStatus(
@@ -164,6 +167,7 @@ class SubscriptionExecutionStatusService:
batch_id=task.batch_id, batch_id=task.batch_id,
task_id=task.task_id, task_id=task.task_id,
current_site_id=task.current_site_id, current_site_id=task.current_site_id,
next_run_at=task.available_at if task.state == "queued" else None,
updated_at=task.updated_at, updated_at=task.updated_at,
error=cls._safe_error(task.last_error), error=cls._safe_error(task.last_error),
can_cancel=state in cls._ACTIVE_STATES, can_cancel=state in cls._ACTIVE_STATES,
+2 -1
View File
@@ -50,6 +50,7 @@ if TYPE_CHECKING:
_SubscribeChain__is_full_season_best_version_resource: Callable[..., Any] _SubscribeChain__is_full_season_best_version_resource: Callable[..., Any]
_SubscribeChain__notify_subscribe_create_failure: Callable[..., Any] _SubscribeChain__notify_subscribe_create_failure: Callable[..., Any]
_SubscribeChain__post_subscribe_added: Callable[..., Any] _SubscribeChain__post_subscribe_added: Callable[..., Any]
_SubscribeChain__queue_new_subscription_search: Callable[..., Any]
_SubscribeChain__prepare_best_version_tv_candidate: Callable[..., Any] _SubscribeChain__prepare_best_version_tv_candidate: Callable[..., Any]
_SubscribeChain__prepare_subscribe_progress_fields: Callable[..., Any] _SubscribeChain__prepare_subscribe_progress_fields: Callable[..., Any]
_SubscribeChain__prepare_total_episode_change_fields: Callable[..., Any] _SubscribeChain__prepare_total_episode_change_fields: Callable[..., Any]
@@ -71,7 +72,7 @@ if TYPE_CHECKING:
_report_search_progress: Callable[..., Any] _report_search_progress: Callable[..., Any]
_search_music_subscribe: Callable[..., Any] _search_music_subscribe: Callable[..., Any]
_subscription_query: Callable[..., Any] _subscription_query: Callable[..., Any]
_defer_recent_subscription: Callable[..., Any] _recent_subscription_retry_at: Callable[..., Any]
_validate_music_subscribe_target: Callable[..., Any] _validate_music_subscribe_target: Callable[..., Any]
_wait_before_scheduled_search: Callable[..., Any] _wait_before_scheduled_search: Callable[..., Any]
add: Callable[..., Any] add: Callable[..., Any]
+23 -28
View File
@@ -85,14 +85,11 @@ def _release_match_admission(
subscription_id: int, subscription_id: int,
summary: MatchExecutionSummary, summary: MatchExecutionSummary,
) -> None: ) -> None:
"""释放 Match 订阅准入,并让 token 冲突即时可见""" """结束当前订阅资源检查,并记录未能恢复的状态"""
if admission.release(lease): if admission.release(lease):
return return
summary.release_failures += 1 summary.release_failures += 1
logger.error( logger.error(f"订阅 {subscription_id} 的搜索状态没有正常恢复,系统稍后会继续检查")
"订阅准入释放失败: operation=match "
f"subscription_id={subscription_id} run_id={summary.run_id}"
)
def _report_match_finished( def _report_match_finished(
@@ -102,13 +99,13 @@ def _report_match_finished(
"""按实际计数发布 Match 最终进度。""" """按实际计数发布 Match 最终进度。"""
if not progress_callback: if not progress_callback:
return return
final_text = "订阅资源匹配完成" final_text = "订阅资源检查完成"
if summary.finished < summary.total: if summary.finished < summary.total:
final_text = "订阅资源匹配已停止,部分订阅未执行" final_text = "订阅资源检查已停止,部分订阅这次未检查"
elif summary.failed: elif summary.failed:
final_text = "订阅资源匹配完成,部分订阅失败" final_text = "订阅资源检查结束,部分订阅没有完成"
elif summary.skipped: elif summary.skipped:
final_text = "订阅资源匹配完成,部分订阅跳过" final_text = "订阅资源检查完成,部分订阅这次未检查"
_report_match_progress(progress_callback, summary, value=100, text=final_text) _report_match_progress(progress_callback, summary, value=100, text=final_text)
@@ -234,7 +231,7 @@ def _prepare_subscription_match(
try: try:
meta = build_subscribe_meta(subscribe) meta = build_subscribe_meta(subscribe)
except ValueError: except ValueError:
logger.error(f"订阅 {subscribe.name} 类型错误:{subscribe.type}") logger.error(f"订阅{subscribe.name}》的媒体类型不受支持,暂时无法检查资源")
return None return None
domains = owner.site_repository.get_domains_by_ids(subscribe.sites) if subscribe.sites else [] domains = owner.site_repository.get_domains_by_ids(subscribe.sites) if subscribe.sites else []
sub_sites = owner.get_sub_sites(subscribe) sub_sites = owner.get_sub_sites(subscribe)
@@ -337,12 +334,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
该入口保持订阅刷新、定时任务和插件调用的稳定签名,具体匹配流程由内部阶段执行。 该入口保持订阅刷新、定时任务和插件调用的稳定签名,具体匹配流程由内部阶段执行。
""" """
if not torrents: if not torrents:
logger.warn("没有缓存资源,无法匹配订阅") logger.warn("当前没有可检查的订阅资源")
if progress_callback: if progress_callback:
progress_callback(value=100, text="没有缓存资源,跳过订阅匹配") progress_callback(value=100, text="当前没有可检查的订阅资源")
return return
if progress_callback: if progress_callback:
progress_callback(value=0, text="正在预处理订阅资源 ...") progress_callback(value=0, text="正在理订阅资源 ...")
return self._run_match( return self._run_match(
torrents=torrents, torrents=torrents,
progress_callback=progress_callback, progress_callback=progress_callback,
@@ -360,13 +357,13 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
:param progress_callback: 订阅匹配进度更新回调 :param progress_callback: 订阅匹配进度更新回调
""" """
if not torrents: if not torrents:
logger.warn("没有缓存资源,无法匹配订阅") logger.warn("当前没有可检查的订阅资源")
if progress_callback: if progress_callback:
progress_callback(value=100, text="没有缓存资源,跳过订阅匹配") progress_callback(value=100, text="当前没有可检查的订阅资源")
return return
if progress_callback: if progress_callback:
progress_callback(value=0, text="正在预处理订阅资源 ...") progress_callback(value=0, text="正在理订阅资源 ...")
lock_acquired = False lock_acquired = False
summary = MatchExecutionSummary.from_candidates(torrents) summary = MatchExecutionSummary.from_candidates(torrents)
@@ -390,7 +387,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback, progress_callback,
summary, summary,
value=20, value=20,
text=f"资源预处理完成,开始匹配 {total_num} 个订阅 ...", text=f"资源理完成,开始检查 {total_num} 个订阅 ...",
) )
try: try:
for index, listed_subscribe in enumerate(subscribes, start=1): for index, listed_subscribe in enumerate(subscribes, start=1):
@@ -401,7 +398,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback, progress_callback,
summary, summary,
value=20 + ((index - 1) / total_num * 80 if total_num else 80), value=20 + ((index - 1) / total_num * 80 if total_num else 80),
text=(f"正在匹配订阅({index}/{total_num}{listed_subscribe.name} ..."), text=(f"正在检查订阅({index}/{total_num}{listed_subscribe.name} ..."),
current=listed_subscribe.id, current=listed_subscribe.id,
) )
outcome: Optional[str] = "skipped" outcome: Optional[str] = "skipped"
@@ -413,7 +410,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
) )
if lease is None: if lease is None:
reason = "admission_conflict" reason = "admission_conflict"
logger.debug(f"订阅 {listed_subscribe.name} 正在由其他通道处理,本轮匹配已跳过") logger.debug(f"订阅 {listed_subscribe.name} 正在处理,本次不再重复检查资源")
else: else:
current_subscribe = None current_subscribe = None
execution_context = SubscriptionExecutionContext( execution_context = SubscriptionExecutionContext(
@@ -424,14 +421,12 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
try: try:
current_subscribe = self.subscription_repository.get(listed_subscribe.id) current_subscribe = self.subscription_repository.get(listed_subscribe.id)
if current_subscribe is None: if current_subscribe is None:
logger.debug(f"订阅 {listed_subscribe.id}不存在,本轮匹配跳过") logger.debug(f"订阅 {listed_subscribe.id}删除,本次不再检查资源")
elif current_subscribe.state not in {"R", "P"}: elif current_subscribe.state not in {"R", "P"}:
logger.debug( logger.debug(f"订阅 {current_subscribe.name} 当前不需要检查资源,本次跳过")
f"订阅 {current_subscribe.name} 当前状态为 {current_subscribe.state},本轮匹配跳过"
)
elif execution_context.should_stop(): elif execution_context.should_stop():
reason = _match_stop_reason(execution_context) reason = _match_stop_reason(execution_context)
logger.debug(f"订阅 {current_subscribe.name} 已取消或超时,本轮匹配跳过") logger.debug(f"订阅 {current_subscribe.name} 的资源检查已停止")
else: else:
outcome = self._match_subscription( outcome = self._match_subscription(
subscribe=current_subscribe, subscribe=current_subscribe,
@@ -444,14 +439,14 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
except SubscriptionSearchCancelled as err: except SubscriptionSearchCancelled as err:
outcome = "skipped" outcome = "skipped"
reason = _match_stop_reason(execution_context) or "cancelled" reason = _match_stop_reason(execution_context) or "cancelled"
logger.debug(f"订阅 {listed_subscribe.name} 匹配已取消{str(err)}") logger.debug(f"订阅 {listed_subscribe.name} 的资源检查已停止{str(err)}")
except Exception as err: except Exception as err:
outcome = "failed" outcome = "failed"
reason = "error" reason = "error"
subscribe_name = ( subscribe_name = (
current_subscribe.name if current_subscribe is not None else listed_subscribe.name current_subscribe.name if current_subscribe is not None else listed_subscribe.name
) )
logger.error(f"订阅 {subscribe_name} 匹配失败{str(err)}", exc_info=True) logger.error(f"订阅 {subscribe_name} 检查资源时出错{str(err)}", exc_info=True)
finally: finally:
_release_match_admission( _release_match_admission(
self._subscription_execution_admission, self._subscription_execution_admission,
@@ -464,7 +459,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
progress_callback, progress_callback,
summary, summary,
value=20 + (summary.finished / total_num * 80 if total_num else 80), value=20 + (summary.finished / total_num * 80 if total_num else 80),
text=(f"订阅匹配{index}/{total_num}处理完成"), text=(f"已检查订阅({index}/{total_num}"),
) )
finally: finally:
processed_torrents.clear() processed_torrents.clear()
@@ -477,7 +472,7 @@ class SubscribeMatchOwner(_SubscribeOwnerBase):
logger.info(summary.finish_log()) logger.info(summary.finish_log())
if lock_acquired: if lock_acquired:
self._match_lock.release() self._match_lock.release()
logger.debug(f"match Lock released at {datetime.now()}") logger.debug(f"订阅资源检查已结束:{datetime.now()}")
def _match_subscription( def _match_subscription(
self, self,
+18
View File
@@ -18,6 +18,7 @@ from app.domain.context import (
MediaInfo, MediaInfo,
) )
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.runtime.execution import run_in_threadpool
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.common import JsonData from app.schemas.common import JsonData
from app.schemas.message import Message as _SchemaMessage from app.schemas.message import Message as _SchemaMessage
@@ -142,6 +143,13 @@ class SubscribeNotificationOwner(_SubscribeOwnerBase):
"mediainfo": context.mediainfo.to_dict(), "mediainfo": context.mediainfo.to_dict(),
}, },
) )
try:
self._SubscribeChain__queue_new_subscription_search(subscribe_id)
except Exception as error:
logger.warning(
"订阅已保存,但自动搜索暂时没有安排成功,"
f"系统会在下一次检查时重试:{error}"
)
try: try:
report_delivered = _subscription_share_snapshot().report_added( report_delivered = _subscription_share_snapshot().report_added(
self._SubscribeChain__subscribe_report_payload(context) self._SubscribeChain__subscribe_report_payload(context)
@@ -177,6 +185,16 @@ class SubscribeNotificationOwner(_SubscribeOwnerBase):
"mediainfo": context.mediainfo.to_dict(), "mediainfo": context.mediainfo.to_dict(),
}, },
) )
try:
await run_in_threadpool(
self._SubscribeChain__queue_new_subscription_search,
subscribe_id,
)
except Exception as error:
logger.warning(
"订阅已保存,但自动搜索暂时没有安排成功,"
f"系统会在下一次检查时重试:{error}"
)
try: try:
report_delivered = await _subscription_share_snapshot().async_report_added( report_delivered = await _subscription_share_snapshot().async_report_added(
self._SubscribeChain__subscribe_report_payload(context) self._SubscribeChain__subscribe_report_payload(context)
+157 -52
View File
@@ -53,6 +53,10 @@ from app.schemas.types import (
SystemConfigKey, SystemConfigKey,
) )
_NEW_SUBSCRIPTION_EDIT_SECONDS = 60
_FOREGROUND_RETRY_SECONDS = 5
_BACKGROUND_RETRY_SECONDS = 10
def _ensure_execution_active( def _ensure_execution_active(
execution_context: Optional[SubscriptionExecutionContext], execution_context: Optional[SubscriptionExecutionContext],
@@ -61,9 +65,9 @@ def _ensure_execution_active(
if execution_context is None: if execution_context is None:
return return
if execution_context.is_cancel_requested(): if execution_context.is_cancel_requested():
raise SubscriptionSearchCancelled("订阅搜索已取消") raise SubscriptionSearchCancelled("搜索已停止")
if execution_context.is_expired(): if execution_context.is_expired():
raise TimeoutError("订阅执行已超过协作截止时间") raise TimeoutError("这次搜索用时过长,已停止")
def _update_search_task_phase( def _update_search_task_phase(
@@ -91,7 +95,8 @@ def _search_source_and_priority(
) -> tuple[str, int]: ) -> tuple[str, int]:
"""把兼容入口归一为持久来源和公平队列优先级。""" """把兼容入口归一为持久来源和公平队列优先级。"""
if manual: if manual:
return "manual", 100 target_count = 1 if sid else len(sids or ())
return "manual", 120 if target_count == 1 else 100
if sid or sids is not None: if sid or sids is not None:
return "targeted", 80 return "targeted", 80
if state in {"R", "P"}: if state in {"R", "P"}:
@@ -99,6 +104,13 @@ def _search_source_and_priority(
return "new", 50 return "new", 50
def _retry_at_after(seconds: int) -> str:
"""返回指定秒数后的 UTC 时间,供短暂等待任务重新入队。"""
return (datetime.now(timezone.utc) + timedelta(seconds=max(1, seconds))).isoformat(
timespec="seconds"
)
def _search_task_available_at( def _search_task_available_at(
source: str, source: str,
subscription_ids: tuple[int, ...], subscription_ids: tuple[int, ...],
@@ -179,16 +191,16 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
progress_callback: Optional[Callable[..., None]], progress_callback: Optional[Callable[..., None]],
) -> bool: ) -> bool:
"""获取 Search 或 Match 通道锁,保持各通道内部串行。""" """获取 Search 或 Match 通道锁,保持各通道内部串行。"""
operation_label = {"search": "搜索", "match": "资源检查"}[operation]
lock = self._match_lock if operation == "match" else self._search_queue_lock lock = self._match_lock if operation == "match" else self._search_queue_lock
if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL): if lock.acquire(blocking=True, timeout=self._SUBSCRIPTION_EXECUTION_TTL):
logger.debug(f"{operation} lock acquired at {datetime.now()}") logger.debug(f"订阅{operation_label}已开始:{datetime.now()}")
return True return True
operation_label = {"search": "搜索", "match": "匹配"}[operation]
progress_text = { progress_text = {
"search": "订阅搜索锁等待超时,已跳过本轮", "search": "订阅搜索正在处理中,本次不再重复开始",
"match": "订阅匹配锁等待超时,已跳过本轮", "match": "订阅资源检查正在进行,本次不再重复开始",
}[operation] }[operation]
logger.error(f"订阅{operation_label}等待超时,已中止本轮执行") logger.error(f"订阅{operation_label}等待时间过长,本次不再重复开始")
if progress_callback: if progress_callback:
progress_callback( progress_callback(
value=100, value=100,
@@ -273,6 +285,12 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
try: try:
subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state) subscribes = self._load_search_subscriptions(sid=sid, sids=sids, state=state)
total = len(subscribes) total = len(subscribes)
source, _priority = _search_source_and_priority(
sid=sid,
sids=sids,
state=state,
manual=manual,
)
summary = SearchExecutionSummary(source="inline", requested=total) summary = SearchExecutionSummary(source="inline", requested=total)
logger.info(summary.start_log()) logger.info(summary.start_log())
if progress_callback: if progress_callback:
@@ -287,7 +305,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
summary.stopped = True summary.stopped = True
break break
self._report_search_progress(progress_callback, subscribe, index, total) self._report_search_progress(progress_callback, subscribe, index, total)
if self._defer_recent_subscription(subscribe): if self._recent_subscription_retry_at(subscribe, source):
summary.record("skipped", "recent_subscription") summary.record("skipped", "recent_subscription")
continue continue
self._wait_before_scheduled_search(sid, sids, state, progress_callback) self._wait_before_scheduled_search(sid, sids, state, progress_callback)
@@ -297,7 +315,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL, ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
) )
if lease is None: if lease is None:
logger.debug(f"订阅 {subscribe.name} 正在由其他通道处理,本搜索已跳过") logger.debug(f"订阅{subscribe.name}正在处理,本搜索先不重复执行")
summary.record("skipped", "admission_conflict") summary.record("skipped", "admission_conflict")
continue continue
execution_context = SubscriptionExecutionContext( execution_context = SubscriptionExecutionContext(
@@ -312,7 +330,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
current = self.subscription_repository.get(subscribe.id) current = self.subscription_repository.get(subscribe.id)
if current is None or current.state == "S": if current is None or current.state == "S":
if current and current.state == "S": if current and current.state == "S":
logger.debug(f"订阅 {current.name} 已暂停,本轮搜索已跳过") logger.debug(f"订阅{current.name}已暂停,本次没有搜索")
continue continue
processed_result = self._process_search_subscription( processed_result = self._process_search_subscription(
current, current,
@@ -326,12 +344,12 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
except SubscriptionSearchCancelled: except SubscriptionSearchCancelled:
reason = "ttl_timeout" if execution_context.is_expired() else "cancelled" reason = "ttl_timeout" if execution_context.is_expired() else "cancelled"
outcome = "failed" if reason == "ttl_timeout" else "cancelled" outcome = "failed" if reason == "ttl_timeout" else "cancelled"
logger.debug(f"订阅 {subscribe.name} 搜索已在安全边界取消") logger.debug(f"订阅{subscribe.name}搜索已停止")
except SubscriptionSearchDeferred as deferred: except SubscriptionSearchDeferred as deferred:
outcome = "skipped" outcome = "skipped"
reason = "site_budget_deferred" reason = "site_budget_deferred"
logger.debug( logger.debug(
f"订阅 {subscribe.name} 站点预算冲突,兼容搜索将在 {deferred.retry_at} 后重试" f"订阅{subscribe.name}》遇到站点繁忙,将在稍后自动重试:{deferred.retry_at}"
) )
except Exception as err: except Exception as err:
outcome = "failed" outcome = "failed"
@@ -347,7 +365,8 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
) )
except Exception as err: except Exception as err:
logger.error( logger.error(
f"订阅 {subscribe.name} 搜索后状态重置失败:{str(err)}", f"订阅{subscribe.name}搜索结束后没有恢复到正常状态,"
f"系统稍后会继续处理:{str(err)}",
exc_info=True, exc_info=True,
) )
finally: finally:
@@ -355,8 +374,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
if not released: if not released:
summary.release_failures += 1 summary.release_failures += 1
logger.error( logger.error(
"订阅准入释放失败: operation=search " f"订阅{subscribe.name}》的搜索状态没有正常恢复,系统稍后会继续处理"
f"subscription_id={subscribe.id} run_id={summary.run_id}"
) )
summary.record(outcome, reason) summary.record(outcome, reason)
self._report_search_progress( self._report_search_progress(
@@ -375,7 +393,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
self._search_queue_lock.release() self._search_queue_lock.release()
if summary is not None: if summary is not None:
logger.info(summary.finish_log()) logger.info(summary.finish_log())
logger.debug(f"search Lock released at {datetime.now()}") logger.debug(f"订阅搜索已结束:{datetime.now()}")
def _execute_queued_search( def _execute_queued_search(
self, self,
@@ -463,7 +481,7 @@ class _SubscribeSearchQueueCoordinator(_SubscribeOwnerBase):
) -> set[int]: ) -> set[int]:
"""有界消费可恢复任务;单任务失败不得阻止后续订阅。""" """有界消费可恢复任务;单任务失败不得阻止后续订阅。"""
if not self._search_queue_lock.acquire(blocking=False): if not self._search_queue_lock.acquire(blocking=False):
logger.debug("订阅搜索队列已有消费者,本轮仅保留持久任务") logger.debug("订阅搜索已经在后台进行,本次安排会接着处理")
summary.consumer_conflicts += 1 summary.consumer_conflicts += 1
return set() return set()
owner = f"subscribe-search:{uuid4().hex}" owner = f"subscribe-search:{uuid4().hex}"
@@ -511,7 +529,7 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
) -> Optional[int]: ) -> Optional[int]:
"""执行一条已认领任务,返回实际进入搜索处理的订阅 ID。""" """执行一条已认领任务,返回实际进入搜索处理的订阅 ID。"""
if task.lease_token is None: if task.lease_token is None:
logger.error(f"订阅搜索任务 {task.task_id} 缺少租约令牌,跳过执行") logger.error("这次订阅搜索暂时无法开始,系统稍后会重新处理")
summary.record("failed", "missing_lease") summary.record("failed", "missing_lease")
return None return None
task_id = str(task.task_id) task_id = str(task.task_id)
@@ -528,14 +546,21 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="cancelled", state="cancelled",
error="订阅已不存在", error="订阅已删除",
) )
summary.record("cancelled", "missing_subscription") summary.record("cancelled", "missing_subscription")
return None return None
self._report_search_progress(progress_callback, subscribe, index, limit) self._report_search_progress(progress_callback, subscribe, index, limit)
if self._defer_recent_subscription(subscribe): recent_retry_at = self._recent_subscription_retry_at(subscribe, task.source)
_skip_search_task(queue, task, "订阅仍在新增保护期,本轮搜索已跳过") if recent_retry_at:
summary.record("skipped", "recent_subscription") queue.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=recent_retry_at,
phase="scheduled",
message="订阅刚刚创建,保存好设置后会自动开始搜索",
)
summary.record("requeued", "recent_subscription")
return None return None
lease = self._subscription_execution_admission.try_acquire( lease = self._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id, subscription_id=subscribe.id,
@@ -543,8 +568,27 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL, ttl_seconds=self._SUBSCRIPTION_EXECUTION_TTL,
) )
if lease is None: if lease is None:
logger.debug(f"订阅 {subscribe.name} 正在由其他通道处理,本轮搜索已跳过") if task.source in {"manual", "targeted", "new"}:
_skip_search_task(queue, task, "同一订阅正在由其他通道处理,本轮搜索已跳过") retry_seconds = (
_FOREGROUND_RETRY_SECONDS
if task.source in {"manual", "targeted"}
else _BACKGROUND_RETRY_SECONDS
)
queue.defer_task(
task_id=task_id,
lease_token=lease_token,
available_at=_retry_at_after(retry_seconds),
phase="waiting_subscription",
message="这个订阅正在处理,结束后会自动继续搜索",
)
summary.record("requeued", "admission_conflict")
logger.debug(f"订阅《{subscribe.name}》正在处理,搜索会在结束后自动继续")
else:
_skip_search_task(
queue,
task,
"这个订阅正在处理,本次自动检查无需重复执行",
)
summary.record("skipped", "admission_conflict") summary.record("skipped", "admission_conflict")
return None return None
phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token) phase_changed = partial(_update_search_task_phase, queue, task_id, lease_token)
@@ -563,12 +607,12 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="cancelled", state="cancelled",
error="订阅已不存在", error="订阅已删除",
) )
summary.record("cancelled", "missing_subscription") summary.record("cancelled", "missing_subscription")
return None return None
if current.state == "S": if current.state == "S":
_skip_search_task(queue, task, "订阅已暂停,本轮搜索已跳过") _skip_search_task(queue, task, "订阅已暂停,这次没有搜索")
summary.record("skipped", "paused") summary.record("skipped", "paused")
return None return None
searchchain.configure_subscription_site_budget( searchchain.configure_subscription_site_budget(
@@ -605,7 +649,7 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
state="failed", state="failed",
error="订阅执行已超过协作截止时间", error="这次搜索用时过长,已停止,可稍后重试",
) )
summary.record("failed", "ttl_timeout") summary.record("failed", "ttl_timeout")
else: else:
@@ -664,7 +708,11 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
try: try:
searchchain.configure_subscription_site_budget(None) searchchain.configure_subscription_site_budget(None)
except Exception as err: except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索站点预算清理失败:{str(err)}", exc_info=True) logger.error(
f"订阅《{subscribe.name}》结束站点访问时遇到问题,"
f"系统稍后会继续处理:{str(err)}",
exc_info=True,
)
try: try:
if current and current.state == "N": if current and current.state == "N":
self._SubscribeChain__apply_subscribe_update( self._SubscribeChain__apply_subscribe_update(
@@ -673,15 +721,12 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
scene="search_reset", scene="search_reset",
) )
except Exception as err: except Exception as err:
logger.error(f"订阅 {subscribe.name} 搜索后状态重置失败{str(err)}", exc_info=True) logger.error(f"订阅{subscribe.name}搜索结束后未能恢复正常状态{str(err)}", exc_info=True)
finally: finally:
released = self._subscription_execution_admission.release(lease) released = self._subscription_execution_admission.release(lease)
if not released: if not released:
summary.release_failures += 1 summary.release_failures += 1
logger.error( logger.error(f"订阅《{subscribe.name}》的搜索状态没有正常恢复,系统稍后会继续处理")
"订阅准入释放失败: operation=search "
f"subscription_id={subscribe.id} run_id={summary.run_id}"
)
self._report_search_progress( self._report_search_progress(
progress_callback, progress_callback,
subscribe, subscribe,
@@ -694,26 +739,49 @@ class _SubscribeSearchQueueOwner(_SubscribeSearchQueueCoordinator):
self, self,
progress_callback: Optional[Callable[..., None]] = None, progress_callback: Optional[Callable[..., None]] = None,
limit: int = 50, limit: int = 50,
manual_sids: Optional[tuple[int, ...]] = None,
) -> None: ) -> None:
"""短周期恢复排队或租约过期任务,不创建新的 24 小时兜底批次""" """短周期恢复等待任务,并可优先反馈本次手工搜索结果"""
queue: Optional[SubscriptionSearchRepository] = getattr( queue: Optional[SubscriptionSearchRepository] = getattr(
self, "subscription_search_repository", None self, "subscription_search_repository", None
) )
if queue is None: if queue is None:
return return
summary = SearchExecutionSummary(source="resume", requested=max(1, limit)) manual_ids = tuple(dict.fromkeys(manual_sids or ()))
subscribes = (
self._load_search_subscriptions(sid=None, sids=manual_ids, state=None)
if manual_ids
else []
)
summary = SearchExecutionSummary(
source="manual" if manual_ids else "resume",
requested=len(manual_ids) if manual_ids else max(1, limit),
)
if manual_ids:
logger.info(summary.start_log()) logger.info(summary.start_log())
try: try:
self._drain_search_queue( processed = self._drain_search_queue(
queue=queue, queue=queue,
limit=max(1, limit), limit=max(1, limit, len(manual_ids)),
progress_callback=progress_callback, progress_callback=progress_callback,
summary=summary, summary=summary,
) )
if manual_ids:
processed_subscribes = [item for item in subscribes if item.id in processed]
self._notify_manual_search(
True,
manual_ids[0] if len(manual_ids) == 1 else None,
None if len(manual_ids) == 1 else manual_ids,
subscribes,
processed_subscribes,
)
except Exception: except Exception:
summary.round_failed = True summary.round_failed = True
raise raise
finally: finally:
if not manual_ids and summary.processed:
summary.requested = summary.processed
if manual_ids or summary.processed or summary.round_failed:
logger.info(summary.finish_log()) logger.info(summary.finish_log())
def cancel_search_batch(self, batch_id: str) -> bool: def cancel_search_batch(self, batch_id: str) -> bool:
@@ -750,15 +818,52 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
return cast(list[SubscriptionSnapshot], repository.list(self.get_states_for_search(state or "N"))) return cast(list[SubscriptionSnapshot], repository.list(self.get_states_for_search(state or "N")))
@staticmethod @staticmethod
def _defer_recent_subscription(subscribe: SubscriptionSnapshot) -> bool: def _recent_subscription_retry_at(
"""新增一分钟内保留 N 状态,为用户编辑筛选条件留出窗口。""" subscribe: SubscriptionSnapshot,
if not subscribe.date: source: str,
return False ) -> Optional[str]:
"""新订阅保留一分钟编辑时间,并返回自动开始搜索的时间。"""
if source != "new" or not subscribe.date:
return None
try:
subscribe_time = datetime.strptime(subscribe.date, "%Y-%m-%d %H:%M:%S") subscribe_time = datetime.strptime(subscribe.date, "%Y-%m-%d %H:%M:%S")
if (datetime.now() - subscribe_time).total_seconds() >= 60: except ValueError:
return False logger.warning(f"订阅《{subscribe.name}》的添加时间无法识别,将直接开始搜索")
logger.debug(f"订阅标题:{subscribe.name} 新增小于1分钟,暂不搜索...") return None
return True remaining = _NEW_SUBSCRIPTION_EDIT_SECONDS - (
datetime.now() - subscribe_time
).total_seconds()
if remaining <= 0:
return None
retry_seconds = max(1, int(remaining) + 1)
logger.debug(f"新订阅《{subscribe.name}》将在约 {retry_seconds} 秒后自动开始搜索")
return _retry_at_after(retry_seconds)
def _SubscribeChain__queue_new_subscription_search(
self,
subscribe_id: int,
) -> Optional[str]:
"""在订阅保存后安排自动搜索,定时扫描仍作为恢复保障。"""
queue: Optional[SubscriptionSearchRepository] = getattr(
self, "subscription_search_repository", None
)
if queue is None:
return None
subscribe = self.subscription_repository.get(subscribe_id)
if subscribe is None or subscribe.state != "N":
return None
available_at = self._recent_subscription_retry_at(subscribe, "new")
enqueued = queue.enqueue(
subscription_ids=(subscribe_id,),
source="new",
priority=50,
available_at_by_subscription={
subscribe_id: available_at or datetime.now(timezone.utc).isoformat(timespec="seconds")
},
)
if enqueued.created_count:
logger.info(f"已安排新订阅《{subscribe.name}》自动搜索,保存好设置后会自动开始")
return enqueued.active_batch_ids[0] if enqueued.active_batch_ids else None
@staticmethod @staticmethod
def _wait_before_scheduled_search( def _wait_before_scheduled_search(
@@ -771,9 +876,9 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
if sid or sids is not None or state not in {"R", "P"}: if sid or sids is not None or state not in {"R", "P"}:
return return
sleep_time = random.randint(60, 300) sleep_time = random.randint(60, 300)
logger.debug(f"订阅搜索随机休眠 {sleep_time} ...") logger.debug(f"为了避免连续访问站点,约 {sleep_time}后继续搜索")
if progress_callback: if progress_callback:
progress_callback(text=f"订阅搜索随机休眠 {sleep_time} 秒后继续 ...") progress_callback(text=f"为了避免连续访问站点,约 {sleep_time} 秒后继续搜索 ...")
time.sleep(sleep_time) time.sleep(sleep_time)
def _process_search_subscription( def _process_search_subscription(
@@ -791,7 +896,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
try: try:
meta = build_subscribe_meta(subscribe) meta = build_subscribe_meta(subscribe)
except ValueError: except ValueError:
logger.error(f"订阅 {subscribe.name} 类型错误:{subscribe.type}") logger.error(f"订阅{subscribe.name}》的媒体类型不受支持,暂时无法搜索")
return subscribe return subscribe
mediainfo: MediaInfo = MediaChain().recognize_media( mediainfo: MediaInfo = MediaChain().recognize_media(
meta=meta, meta=meta,
@@ -965,7 +1070,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
subscribes: list[SubscriptionSnapshot], subscribes: list[SubscriptionSnapshot],
processed: list[SubscriptionSnapshot], processed: list[SubscriptionSnapshot],
) -> None: ) -> None:
"""为手动触发的搜索发布保持旧文案的系统消息""" """用清晰文案反馈手动搜索的完成或等待状态"""
if not manual: if not manual:
return return
if not subscribes: if not subscribes:
@@ -974,7 +1079,7 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
message = ( message = (
f"{subscribes[0].name} 搜索完成!" f"{subscribes[0].name} 搜索完成!"
if processed if processed
else f"{subscribes[0].name} 本轮未执行或未完成,将等待下一次正常调度" else f"{subscribes[0].name} 已安排搜索,系统会自动继续处理"
) )
self.messagehelper.put(message, title="订阅搜索", role="system") self.messagehelper.put(message, title="订阅搜索", role="system")
elif sids is not None: elif sids is not None:
@@ -984,6 +1089,6 @@ class SubscribeSearchOwner(_SubscribeSearchQueueOwner):
message = ( message = (
"所有订阅搜索完成!" "所有订阅搜索完成!"
if len(processed) == len(subscribes) if len(processed) == len(subscribes)
else "订阅搜索完成,部分订阅本轮未执行或未完成" else "订阅搜索已安排,暂未完成的项目会自动继续处理"
) )
self.messagehelper.put(message, title="订阅搜索", role="system") self.messagehelper.put(message, title="订阅搜索", role="system")
+12 -3
View File
@@ -769,9 +769,18 @@ class SessionSubscriptionRepository:
candidates.append(candidate) candidates.append(candidate)
return candidates return candidates
async def list_search_ids(self, username: str, state: str) -> builtins.list[int]: async def list_search_ids(
"""异步读取用户指定状态下的订阅主键。""" self,
return [snapshot.id for snapshot in await self.async_list_by_username(username, state)] username: Optional[str],
state: str,
) -> builtins.list[int]:
"""异步读取用户或管理员全局范围内指定状态的订阅主键。"""
snapshots = (
await self.async_list_by_username(username, state)
if username is not None
else await self.async_list(state)
)
return [snapshot.id for snapshot in snapshots]
async def stage_delete(self, subscribe_id: int) -> None: async def stage_delete(self, subscribe_id: int) -> None:
"""异步暂存删除订阅。""" """异步暂存删除订阅。"""
+29 -4
View File
@@ -1,6 +1,7 @@
"""订阅搜索持久队列的 SQLAlchemy 适配器。""" """订阅搜索持久队列的 SQLAlchemy 适配器。"""
from collections.abc import Callable, Mapping from collections.abc import Callable, Mapping
from datetime import datetime, timedelta, timezone
from typing import Optional, TypeVar from typing import Optional, TypeVar
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -19,6 +20,7 @@ from app.db.oper.subscriptionsearch import SubscriptionSearchOper
from app.db.uow import SqlAlchemyUnitOfWork from app.db.uow import SqlAlchemyUnitOfWork
T = TypeVar("T") T = TypeVar("T")
_BUSY_SITE_RETRY_SECONDS = 10
def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot: def _batch(record: SubscriptionSearchBatch) -> SearchBatchSnapshot:
@@ -101,7 +103,7 @@ class TransactionalSubscriptionSearchRepository:
"""创建批次并返回 single-flight 合并计数。""" """创建批次并返回 single-flight 合并计数。"""
def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult: def operation(repository: SubscriptionSearchOper) -> SearchEnqueueResult:
"""在同一事务内创建批次和任务。""" """在同一事务内创建批次和任务。"""
record, created, coalesced = repository.enqueue( record, created, coalesced, active_batch_ids = repository.enqueue(
subscription_ids=subscription_ids, subscription_ids=subscription_ids,
source=source, source=source,
priority=priority, priority=priority,
@@ -111,6 +113,7 @@ class TransactionalSubscriptionSearchRepository:
batch=_batch(record), batch=_batch(record),
created_count=created, created_count=created,
coalesced_count=coalesced, coalesced_count=coalesced,
active_batch_ids=active_batch_ids,
) )
return self._write(operation) return self._write(operation)
@@ -183,13 +186,17 @@ class TransactionalSubscriptionSearchRepository:
task_id: str, task_id: str,
lease_token: str, lease_token: str,
available_at: str, available_at: str,
phase: str = "waiting_site_budget",
message: Optional[str] = None,
) -> bool: ) -> bool:
"""以站点预算的下一次可用时间重新排队任务。""" """按指定时间和可见原因重新排队任务。"""
return self._write( return self._write(
lambda repository: repository.defer_task( lambda repository: repository.defer_task(
task_id=task_id, task_id=task_id,
lease_token=lease_token, lease_token=lease_token,
available_at=available_at, available_at=available_at,
phase=phase,
message=message,
) )
) )
@@ -225,14 +232,32 @@ class TransactionalSubscriptionSearchRepository:
lease_seconds=lease_seconds, lease_seconds=lease_seconds,
) )
retry_at = record.next_allowed_at retry_at = record.next_allowed_at
if not acquired and record.lease_token and record.lease_expires_at: wait_reason = None
retry_at = max(retry_at, record.lease_expires_at) now = datetime.now(timezone.utc)
cooldown_active = bool(
record.last_outcome not in {None, "success", "skipped"}
and record.next_allowed_at > now.isoformat(timespec="seconds")
)
lease_busy = bool(
record.lease_token
and record.lease_expires_at
and record.lease_expires_at > now.isoformat(timespec="seconds")
)
if not acquired and cooldown_active:
wait_reason = "cooldown"
elif not acquired and lease_busy:
wait_reason = "busy"
short_retry = (now + timedelta(seconds=_BUSY_SITE_RETRY_SECONDS)).isoformat(
timespec="seconds"
)
retry_at = min(record.lease_expires_at, short_retry)
return SiteBudgetClaim( return SiteBudgetClaim(
site_id=record.site_id, site_id=record.site_id,
acquired=acquired, acquired=acquired,
retry_at=retry_at, retry_at=retry_at,
consecutive_failures=record.consecutive_failures, consecutive_failures=record.consecutive_failures,
lease_token=record.lease_token if acquired else None, lease_token=record.lease_token if acquired else None,
wait_reason=wait_reason,
) )
return self._write(operation) return self._write(operation)
+1
View File
@@ -83,6 +83,7 @@ class SessionSubscriptionExecutionStatusRepository:
"""返回最近更新的批次,访问范围由应用服务依据任务校验。""" """返回最近更新的批次,访问范围由应用服务依据任务校验。"""
result = await self._session.execute( result = await self._session.execute(
select(SubscriptionSearchBatch) select(SubscriptionSearchBatch)
.where(SubscriptionSearchBatch.total_count > 0)
.order_by( .order_by(
SubscriptionSearchBatch.updated_at.desc(), SubscriptionSearchBatch.updated_at.desc(),
SubscriptionSearchBatch.id.desc(), SubscriptionSearchBatch.id.desc(),
+7 -3
View File
@@ -556,9 +556,13 @@ class SubscribeOper(DbOper):
) )
return candidates return candidates
async def list_search_ids(self, username: str, state: str) -> List[int]: async def list_search_ids(self, username: Optional[str], state: str) -> List[int]:
"""返回用户指定状态的订阅编号,不向应用用例暴露 ORM 列表""" """返回用户或管理员全局范围内指定状态的订阅编号。"""
subscribes = await self.async_list_by_username(username, state=state) subscribes = (
await self.async_list_by_username(username, state=state)
if username is not None
else await self.async_list(state=state)
)
return [subscribe.id for subscribe in subscribes if subscribe.id] return [subscribe.id for subscribe in subscribes if subscribe.id]
def get_by( def get_by(
+40 -6
View File
@@ -31,7 +31,7 @@ class SubscriptionSearchOper(DbOper):
source: str, source: str,
priority: int, priority: int,
available_at_by_subscription: Optional[Mapping[int, str]], available_at_by_subscription: Optional[Mapping[int, str]],
) -> tuple[SubscriptionSearchBatch, int, int]: ) -> tuple[SubscriptionSearchBatch, int, int, tuple[str, ...]]:
"""创建批次,并以活动键合并同一订阅的重叠搜索入口。""" """创建批次,并以活动键合并同一订阅的重叠搜索入口。"""
if not isinstance(self._db, Session): if not isinstance(self._db, Session):
raise RuntimeError("订阅搜索入队需要调用方提供同步 Session") raise RuntimeError("订阅搜索入队需要调用方提供同步 Session")
@@ -49,6 +49,7 @@ class SubscriptionSearchOper(DbOper):
self._db.flush() self._db.flush()
created = 0 created = 0
coalesced = 0 coalesced = 0
active_batch_ids: list[str] = []
for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)): for position, subscription_id in enumerate(dict.fromkeys(subscription_ids)):
active_key = f"subscription:{subscription_id}" active_key = f"subscription:{subscription_id}"
available_at = ( available_at = (
@@ -56,6 +57,11 @@ class SubscriptionSearchOper(DbOper):
if available_at_by_subscription if available_at_by_subscription
else now else now
) )
initial_phase = (
"scheduled"
if source == "new" and available_at > now
else "queued"
)
task = SubscriptionSearchTask( task = SubscriptionSearchTask(
task_id=uuid4().hex, task_id=uuid4().hex,
batch_id=batch.batch_id, batch_id=batch.batch_id,
@@ -65,7 +71,7 @@ class SubscriptionSearchOper(DbOper):
priority=priority, priority=priority,
position=position, position=position,
state="queued", state="queued",
phase="queued", phase=initial_phase,
available_at=available_at, available_at=available_at,
created_at=now, created_at=now,
updated_at=now, updated_at=now,
@@ -77,15 +83,31 @@ class SubscriptionSearchOper(DbOper):
created += 1 created += 1
except IntegrityError: except IntegrityError:
coalesced += 1 coalesced += 1
promote_queued_task = and_(
SubscriptionSearchTask.priority < priority,
SubscriptionSearchTask.state == "queued",
)
execute_dml( execute_dml(
self._db, self._db,
update(SubscriptionSearchTask) update(SubscriptionSearchTask)
.where(SubscriptionSearchTask.active_key == active_key) .where(SubscriptionSearchTask.active_key == active_key)
.values( .values(
source=case(
(SubscriptionSearchTask.priority < priority, source),
else_=SubscriptionSearchTask.source,
),
priority=case( priority=case(
(SubscriptionSearchTask.priority < priority, priority), (SubscriptionSearchTask.priority < priority, priority),
else_=SubscriptionSearchTask.priority, else_=SubscriptionSearchTask.priority,
), ),
phase=case(
(promote_queued_task, "queued"),
else_=SubscriptionSearchTask.phase,
),
last_error=case(
(promote_queued_task, None),
else_=SubscriptionSearchTask.last_error,
),
available_at=case( available_at=case(
( (
or_( or_(
@@ -100,11 +122,20 @@ class SubscriptionSearchOper(DbOper):
), ),
execution_options={"synchronize_session": False}, execution_options={"synchronize_session": False},
) )
active_task = self._db.execute(
select(SubscriptionSearchTask).where(
SubscriptionSearchTask.active_key == active_key
)
).scalars().first()
if active_task is not None:
active_batch_ids.append(active_task.batch_id)
batch.total_count = created batch.total_count = created
if created == 0: if created == 0:
batch.state = "completed" batch.state = "completed"
batch.finished_at = now batch.finished_at = now
return batch, created, coalesced else:
active_batch_ids.insert(0, batch.batch_id)
return batch, created, coalesced, tuple(dict.fromkeys(active_batch_ids))
def claim_next(self, *, owner: str, lease_seconds: int) -> Optional[SubscriptionSearchTask]: def claim_next(self, *, owner: str, lease_seconds: int) -> Optional[SubscriptionSearchTask]:
"""使用 CAS 认领最高优先级任务,过期 running 任务可被恢复。""" """使用 CAS 认领最高优先级任务,过期 running 任务可被恢复。"""
@@ -139,6 +170,7 @@ class SubscriptionSearchOper(DbOper):
) )
.order_by( .order_by(
case( case(
(SubscriptionSearchTask.priority >= 100, 2),
(SubscriptionSearchTask.created_at <= fairness_before, 1), (SubscriptionSearchTask.created_at <= fairness_before, 1),
else_=0, else_=0,
).desc(), ).desc(),
@@ -369,8 +401,10 @@ class SubscriptionSearchOper(DbOper):
task_id: str, task_id: str,
lease_token: str, lease_token: str,
available_at: str, available_at: str,
phase: str,
message: Optional[str],
) -> bool: ) -> bool:
"""释放当前租约并在站点预算时间到达后恢复同一任务。""" """释放当前租约并在指定时间后按可见原因恢复同一任务。"""
if not isinstance(self._db, Session): if not isinstance(self._db, Session):
raise RuntimeError("订阅搜索延后需要调用方提供同步 Session") raise RuntimeError("订阅搜索延后需要调用方提供同步 Session")
task = self._db.execute( task = self._db.execute(
@@ -400,7 +434,7 @@ class SubscriptionSearchOper(DbOper):
) )
.values( .values(
state="queued", state="queued",
phase="waiting_site_budget", phase=phase,
current_site_id=None, current_site_id=None,
lease_owner=None, lease_owner=None,
lease_token=None, lease_token=None,
@@ -408,7 +442,7 @@ class SubscriptionSearchOper(DbOper):
available_at=available_at, available_at=available_at,
updated_at=now, updated_at=now,
finished_at=None, finished_at=None,
last_error=None, last_error=message,
), ),
execution_options={"synchronize_session": False}, execution_options={"synchronize_session": False},
) )
+41 -2
View File
@@ -137,6 +137,17 @@
"当前管理用户缺少可审计身份": "The current administrator does not have an auditable identity", "当前管理用户缺少可审计身份": "The current administrator does not have an auditable identity",
"人工复核任务不存在": "The manual review task does not exist", "人工复核任务不存在": "The manual review task does not exist",
"任务添加失败": "Failed to add task", "任务添加失败": "Failed to add task",
"没有需要搜索的订阅": "No subscriptions need searching",
"这个订阅已经在搜索中,请稍候": "This subscription is already being searched. Please wait",
"已安排搜索,很快开始": "Search scheduled and will start soon",
"当前没有可检查的订阅资源": "No subscription resources are currently available to check",
"正在整理订阅资源 ...": "Organizing subscription resources ...",
"订阅资源检查完成": "Subscription resource check complete",
"订阅资源检查已停止,部分订阅这次未检查": "Subscription resource check stopped; some subscriptions were not checked this time",
"订阅资源检查结束,部分订阅没有完成": "Subscription resource check finished; some subscriptions did not complete",
"订阅资源检查完成,部分订阅这次未检查": "Subscription resource check complete; some subscriptions were not checked this time",
"订阅搜索正在处理中,本次不再重复开始": "A subscription search is already in progress. This run will not start again",
"订阅资源检查正在进行,本次不再重复开始": "A subscription resource check is already in progress. This run will not start again",
"无法识别媒体信息": "Unable to recognize media information", "无法识别媒体信息": "Unable to recognize media information",
"未识别到媒体信息": "Unable to recognize media information", "未识别到媒体信息": "Unable to recognize media information",
"未识别到音乐信息": "Unable to recognize music information", "未识别到音乐信息": "Unable to recognize music information",
@@ -382,8 +393,8 @@
"微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "WeChat ClawBot notification is not enabled or the configuration has not been saved. Please save and enable this channel first", "微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "WeChat ClawBot notification is not enabled or the configuration has not been saved. Please save and enable this channel first",
"请输入至少一个有效的站点 ID": "Enter at least one valid site ID", "请输入至少一个有效的站点 ID": "Enter at least one valid site ID",
"所有订阅搜索完成": "All subscription searches are complete", "所有订阅搜索完成": "All subscription searches are complete",
"订阅搜索批次不存在": "The subscription search batch does not exist", "没有找到这次搜索,请刷新后重试": "This search could not be found. Refresh and try again",
"订阅搜索批次已结束或无法取消": "The subscription search batch has ended or cannot be cancelled", "这次搜索已经结束,暂时无法停止": "This search has already ended and cannot be stopped",
"订阅搜索锁等待超时,已跳过本轮": "Subscription search lock timed out, this round was skipped", "订阅搜索锁等待超时,已跳过本轮": "Subscription search lock timed out, this round was skipped",
"订阅匹配锁等待超时,已跳过本轮": "Subscription matching lock timed out, this round was skipped", "订阅匹配锁等待超时,已跳过本轮": "Subscription matching lock timed out, this round was skipped",
"请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "Enter subscription IDs separated by spaces, or enter all", "请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "Enter subscription IDs separated by spaces, or enter all",
@@ -765,6 +776,18 @@
"source": "已完成 {count} 个订阅搜索", "source": "已完成 {count} 个订阅搜索",
"target": "Completed searches for {count} subscriptions" "target": "Completed searches for {count} subscriptions"
}, },
{
"source": "{count} 个订阅已经在搜索中,无需重复提交",
"target": "{count} subscriptions are already being searched. No need to submit again"
},
{
"source": "已安排 {queued} 个订阅搜索,另有 {ongoing} 个正在处理中",
"target": "Scheduled searches for {queued} subscriptions; {ongoing} are already being processed"
},
{
"source": "已安排 {count} 个订阅搜索,系统会依次处理",
"target": "Scheduled searches for {count} subscriptions. They will be processed in order"
},
{ {
"source": "请输入至少一个有效的订阅 ID", "source": "请输入至少一个有效的订阅 ID",
"target": "Enter at least one valid subscription ID" "target": "Enter at least one valid subscription ID"
@@ -1421,6 +1444,10 @@
"source": "订阅搜索({index}/{total})处理完成", "source": "订阅搜索({index}/{total})处理完成",
"target": "Subscription search ({index}/{total}) completed" "target": "Subscription search ({index}/{total}) completed"
}, },
{
"source": "为了避免连续访问站点,约 {seconds} 秒后继续搜索 ...",
"target": "To avoid repeated site requests, search will continue in about {seconds} seconds ..."
},
{ {
"source": "开始刷新订阅,共 {count} 个订阅 ...", "source": "开始刷新订阅,共 {count} 个订阅 ...",
"target": "Starting subscription refresh, {count} subscriptions ..." "target": "Starting subscription refresh, {count} subscriptions ..."
@@ -1433,6 +1460,18 @@
"source": "正在匹配订阅({index}/{total}{name} ...", "source": "正在匹配订阅({index}/{total}{name} ...",
"target": "Matching subscription ({index}/{total}) {name} ..." "target": "Matching subscription ({index}/{total}) {name} ..."
}, },
{
"source": "资源整理完成,开始检查 {count} 个订阅 ...",
"target": "Resource organization complete. Checking {count} subscriptions ..."
},
{
"source": "正在检查订阅({index}/{total}{name} ...",
"target": "Checking subscription ({index}/{total}) {name} ..."
},
{
"source": "已检查订阅({index}/{total}",
"target": "Checked subscription ({index}/{total})"
},
{ {
"source": "开始更新订阅元数据,共 {count} 个订阅 ...", "source": "开始更新订阅元数据,共 {count} 个订阅 ...",
"target": "Starting subscription metadata update, {count} subscriptions ..." "target": "Starting subscription metadata update, {count} subscriptions ..."
+41 -2
View File
@@ -129,6 +129,17 @@
"当前管理用户缺少可审计身份": "目前管理使用者缺少可稽核身分", "当前管理用户缺少可审计身份": "目前管理使用者缺少可稽核身分",
"人工复核任务不存在": "人工複核任務不存在", "人工复核任务不存在": "人工複核任務不存在",
"任务添加失败": "任務新增失敗", "任务添加失败": "任務新增失敗",
"没有需要搜索的订阅": "沒有需要搜尋的訂閱",
"这个订阅已经在搜索中,请稍候": "這個訂閱已在搜尋中,請稍候",
"已安排搜索,很快开始": "已安排搜尋,很快開始",
"当前没有可检查的订阅资源": "目前沒有可檢查的訂閱資源",
"正在整理订阅资源 ...": "正在整理訂閱資源 ...",
"订阅资源检查完成": "訂閱資源檢查完成",
"订阅资源检查已停止,部分订阅这次未检查": "訂閱資源檢查已停止,部分訂閱這次未檢查",
"订阅资源检查结束,部分订阅没有完成": "訂閱資源檢查結束,部分訂閱沒有完成",
"订阅资源检查完成,部分订阅这次未检查": "訂閱資源檢查完成,部分訂閱這次未檢查",
"订阅搜索正在处理中,本次不再重复开始": "訂閱搜尋正在處理中,本次不再重複開始",
"订阅资源检查正在进行,本次不再重复开始": "訂閱資源檢查正在進行,本次不再重複開始",
"无法识别媒体信息": "無法識別媒體資訊", "无法识别媒体信息": "無法識別媒體資訊",
"未识别到媒体信息": "未識別到媒體資訊", "未识别到媒体信息": "未識別到媒體資訊",
"未识别到音乐信息": "未識別到音樂資訊", "未识别到音乐信息": "未識別到音樂資訊",
@@ -374,8 +385,8 @@
"微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "微信 ClawBot 通知未啟用或設定尚未儲存,請先儲存並啟用目前渠道", "微信 ClawBot 通知未启用或配置尚未保存,请先保存并启用当前渠道": "微信 ClawBot 通知未啟用或設定尚未儲存,請先儲存並啟用目前渠道",
"请输入至少一个有效的站点 ID": "請輸入至少一個有效的站點 ID", "请输入至少一个有效的站点 ID": "請輸入至少一個有效的站點 ID",
"所有订阅搜索完成": "所有訂閱搜尋完成", "所有订阅搜索完成": "所有訂閱搜尋完成",
"订阅搜索批次不存在": "訂閱搜尋批次不存在", "没有找到这次搜索,请刷新后重试": "找不到這次搜尋,請重新整理後再試",
"订阅搜索批次已结束或无法取消": "訂閱搜尋批次已結束或無法取消", "这次搜索已经结束,暂时无法停止": "這次搜尋已經結束,暫時無法停止",
"请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "請輸入訂閱 ID,多個 ID 以空格分隔,或輸入 all", "请输入订阅 ID,多个 ID 用空格分隔,或输入 all": "請輸入訂閱 ID,多個 ID 以空格分隔,或輸入 all",
"请输入至少一个有效的订阅 ID": "請輸入至少一個有效的訂閱 ID", "请输入至少一个有效的订阅 ID": "請輸入至少一個有效的訂閱 ID",
"格式错误,请输入:cookie <id> <username> <password> [2fa_code/secret]": "格式錯誤,請輸入:cookie <id> <username> <password> [2fa_code/secret]", "格式错误,请输入:cookie <id> <username> <password> [2fa_code/secret]": "格式錯誤,請輸入:cookie <id> <username> <password> [2fa_code/secret]",
@@ -751,6 +762,18 @@
"source": "已完成 {count} 个订阅搜索", "source": "已完成 {count} 个订阅搜索",
"target": "已完成 {count} 個訂閱搜尋" "target": "已完成 {count} 個訂閱搜尋"
}, },
{
"source": "{count} 个订阅已经在搜索中,无需重复提交",
"target": "{count} 個訂閱已在搜尋中,無需重複提交"
},
{
"source": "已安排 {queued} 个订阅搜索,另有 {ongoing} 个正在处理中",
"target": "已安排 {queued} 個訂閱搜尋,另有 {ongoing} 個正在處理中"
},
{
"source": "已安排 {count} 个订阅搜索,系统会依次处理",
"target": "已安排 {count} 個訂閱搜尋,系統會依序處理"
},
{ {
"source": "请输入至少一个有效的订阅 ID", "source": "请输入至少一个有效的订阅 ID",
"target": "請輸入至少一個有效的訂閱 ID" "target": "請輸入至少一個有效的訂閱 ID"
@@ -1407,6 +1430,10 @@
"source": "订阅搜索({index}/{total})处理完成", "source": "订阅搜索({index}/{total})处理完成",
"target": "訂閱搜尋({index}/{total})處理完成" "target": "訂閱搜尋({index}/{total})處理完成"
}, },
{
"source": "为了避免连续访问站点,约 {seconds} 秒后继续搜索 ...",
"target": "為了避免連續存取站點,約 {seconds} 秒後繼續搜尋 ..."
},
{ {
"source": "开始刷新订阅,共 {count} 个订阅 ...", "source": "开始刷新订阅,共 {count} 个订阅 ...",
"target": "開始重新整理訂閱,共 {count} 個訂閱 ..." "target": "開始重新整理訂閱,共 {count} 個訂閱 ..."
@@ -1419,6 +1446,18 @@
"source": "正在匹配订阅({index}/{total}{name} ...", "source": "正在匹配订阅({index}/{total}{name} ...",
"target": "正在匹配訂閱({index}/{total}{name} ..." "target": "正在匹配訂閱({index}/{total}{name} ..."
}, },
{
"source": "资源整理完成,开始检查 {count} 个订阅 ...",
"target": "資源整理完成,開始檢查 {count} 個訂閱 ..."
},
{
"source": "正在检查订阅({index}/{total}{name} ...",
"target": "正在檢查訂閱({index}/{total}{name} ..."
},
{
"source": "已检查订阅({index}/{total}",
"target": "已檢查訂閱({index}/{total}"
},
{ {
"source": "开始更新订阅元数据,共 {count} 个订阅 ...", "source": "开始更新订阅元数据,共 {count} 个订阅 ...",
"target": "開始更新訂閱元資料,共 {count} 個訂閱 ..." "target": "開始更新訂閱元資料,共 {count} 個訂閱 ..."
+2 -2
View File
@@ -154,8 +154,8 @@ class SchedulerCatalogOwner(_SchedulerOwnerBase):
"interval", "interval",
id="subscribe_search_queue", id="subscribe_search_queue",
name="恢复订阅搜索队列", name="恢复订阅搜索队列",
minutes=1, seconds=10,
next_run_time=datetime.now(pytz.timezone(config.timezone)) + timedelta(seconds=10), next_run_time=datetime.now(pytz.timezone(config.timezone)) + timedelta(seconds=5),
kwargs={"job_id": "subscribe_search_queue"}, kwargs={"job_id": "subscribe_search_queue"},
) )
+1
View File
@@ -463,6 +463,7 @@ SCHEMA_EXPORTS = {
'SubscriptionBatchStatus': ('app.schemas.subscribe', 'SubscriptionBatchStatus'), 'SubscriptionBatchStatus': ('app.schemas.subscribe', 'SubscriptionBatchStatus'),
'SubscriptionExecutionStatus': ('app.schemas.subscribe', 'SubscriptionExecutionStatus'), 'SubscriptionExecutionStatus': ('app.schemas.subscribe', 'SubscriptionExecutionStatus'),
'SubscriptionMessage': ('app.schemas.message', 'SubscriptionMessage'), 'SubscriptionMessage': ('app.schemas.message', 'SubscriptionMessage'),
'SubscriptionSearchSubmission': ('app.schemas.subscribe', 'SubscriptionSearchSubmission'),
'SubtitleDownloadData': ('app.schemas.download', 'SubtitleDownloadData'), 'SubtitleDownloadData': ('app.schemas.download', 'SubtitleDownloadData'),
'SubtitleInfo': ('app.schemas.search', 'SubtitleInfo'), 'SubtitleInfo': ('app.schemas.search', 'SubtitleInfo'),
'SystemEnvironmentUpdateData': ('app.schemas.system', 'SystemEnvironmentUpdateData'), 'SystemEnvironmentUpdateData': ('app.schemas.system', 'SystemEnvironmentUpdateData'),
+12
View File
@@ -55,6 +55,7 @@ class SubscriptionExecutionStatus(BaseModel): # type: ignore[misc]
batch_id: Optional[str] = None batch_id: Optional[str] = None
task_id: Optional[str] = None task_id: Optional[str] = None
current_site_id: Optional[int] = None current_site_id: Optional[int] = None
next_run_at: Optional[str] = None
error: Optional[str] = None error: Optional[str] = None
can_cancel: bool = False can_cancel: bool = False
@@ -84,6 +85,17 @@ class SubscriptionBatchStatus(BaseModel): # type: ignore[misc]
model_config = ConfigDict(from_attributes=True) model_config = ConfigDict(from_attributes=True)
class SubscriptionSearchSubmission(BaseModel): # type: ignore[misc]
"""手工订阅搜索已安排后的轻量跟踪信息。"""
batch_id: Optional[str] = None
batch_ids: List[str] = Field(default_factory=list)
target_count: int = 0
queued_count: int = 0
ongoing_count: int = 0
single: bool = False
class Subscribe(OptionalMediaIdentityMixin, BaseModel): class Subscribe(OptionalMediaIdentityMixin, BaseModel):
"""订阅输入与响应模型,媒体身份必须为空对或完整有效对。""" """订阅输入与响应模型,媒体身份必须为空对或完整有效对。"""
@@ -799,7 +799,7 @@ def _run_match_execution_case(case: ScaleCase) -> dict[str, Any]:
@staticmethod @staticmethod
def info(message: Any, *_args: Any, **_kwargs: Any) -> None: def info(message: Any, *_args: Any, **_kwargs: Any) -> None:
"""保存结构化批次摘要。""" """保存开始和结束摘要。"""
info_logs.append(str(message)) info_logs.append(str(message))
class _ScaleMediaChain: class _ScaleMediaChain:
@@ -958,8 +958,10 @@ def _run_match_execution_case(case: ScaleCase) -> dict[str, Any]:
"info_log_count": len(info_logs), "info_log_count": len(info_logs),
"info_log_bounded": bool( "info_log_bounded": bool(
len(info_logs) == 2 len(info_logs) == 2
and info_logs[0].startswith("订阅治理轮次开始: operation=match ") and info_logs[0].startswith("开始检查订阅资源,共 ")
and info_logs[1].startswith("订阅治理轮次结束: operation=match ") and info_logs[1].startswith("订阅资源检查结束:")
and "operation=" not in "\n".join(info_logs)
and "run_id=" not in "\n".join(info_logs)
), ),
"first_settlement_local_ms": round(settlement_ms[0], 3), "first_settlement_local_ms": round(settlement_ms[0], 3),
"subscription_settlement_local_p50_ms": _percentile(settlement_ms, 50), "subscription_settlement_local_p50_ms": _percentile(settlement_ms, 50),
+53 -9
View File
@@ -72,6 +72,34 @@ class _RunningTaskRegistry(TaskRegistry):
) )
class _CompletedSearchTaskRegistry(_TaskRegistry):
"""返回已完成结果,验证手工搜索先保存安排再启动后台处理。"""
def create_sync(self, function, *args, owner: str, **kwargs) -> asyncio.Future:
"""记录同步任务,并为测试提供可等待的安排结果。"""
super().create_sync(function, *args, owner=owner, **kwargs)
future = asyncio.get_running_loop().create_future()
result = None
if owner == "api.subscribe.search.enqueue":
result = SimpleNamespace(
active_batch_ids=("batch-1", "batch-2"),
created_count=2,
coalesced_count=0,
)
future.set_result(result)
return future
class _SubscriptionSearchTargets:
"""为手工搜索命令提供当前用户可访问的订阅编号。"""
async def list_search_ids(self, username, state) -> list[int]:
"""返回稳定目标,并校验超级用户读取全部运行中订阅。"""
assert username is None
assert state == "R"
return [11, 12]
class _ProtocolManager: class _ProtocolManager:
"""提供兼容协议流结束时需要的最小 AgentManager 接口。""" """提供兼容协议流结束时需要的最小 AgentManager 接口。"""
@@ -206,10 +234,14 @@ def test_seerr_subscribe_uses_task_registry(monkeypatch) -> None:
def test_manual_subscription_search_uses_task_registry() -> None: def test_manual_subscription_search_uses_task_registry() -> None:
"""手工订阅搜索命令应以稳定 owner 提交顺序搜索批次。""" """手工订阅搜索命令应以稳定 owner 提交顺序搜索批次。"""
registry = _TaskRegistry() registry = _CompletedSearchTaskRegistry()
repository = object() repository = _SubscriptionSearchTargets()
search_repository = SimpleNamespace(enqueue=lambda **_kwargs: None)
runtime = SimpleNamespace( runtime = SimpleNamespace(
subscription=SimpleNamespace(repository=lambda _db: repository) subscription=SimpleNamespace(
repository=lambda _db: repository,
search_repository=search_repository,
)
) )
command = subscription_dependencies.get_search_subscriptions_command( command = subscription_dependencies.get_search_subscriptions_command(
task_registry=registry, task_registry=registry,
@@ -221,12 +253,24 @@ def test_manual_subscription_search_uses_task_registry() -> None:
command.execute(SubscribeSearchActor(username="admin", is_superuser=True)) command.execute(SubscribeSearchActor(username="admin", is_superuser=True))
) )
function, args, kwargs, owner = registry.calls[0] enqueue_function, enqueue_args, enqueue_kwargs, enqueue_owner = registry.calls[0]
assert found is True run_function, run_args, run_kwargs, run_owner = registry.calls[1]
assert function is subscription_dependencies._start_subscription_search_batch assert found is not None
assert args == (None, "R") assert found.batch_ids == ("batch-1", "batch-2")
assert kwargs == {} assert found.queued_count == 2
assert owner == "api.subscribe.search" assert found.ongoing_count == 0
assert enqueue_function is search_repository.enqueue
assert enqueue_args == ()
assert enqueue_kwargs == {
"subscription_ids": (11, 12),
"source": "manual",
"priority": 100,
}
assert enqueue_owner == "api.subscribe.search.enqueue"
assert run_function is subscription_dependencies._resume_submitted_subscription_search
assert run_args == ((11, 12),)
assert run_kwargs == {}
assert run_owner == "api.subscribe.search.run"
def test_history_ai_redo_uses_task_registry() -> None: def test_history_ai_redo_uses_task_registry() -> None:
+14 -1
View File
@@ -653,8 +653,19 @@ class TestSubscribeEndpoint:
普通用户批量搜索把用户身份交给应用命令 普通用户批量搜索把用户身份交给应用命令
""" """
from app.api.endpoints.subscribe import search_subscribes from app.api.endpoints.subscribe import search_subscribes
from app.application.subscription.search import SubscriptionSearchSubmission
command = SimpleNamespace(execute=AsyncMock(return_value=True)) command = SimpleNamespace(
execute=AsyncMock(
return_value=SubscriptionSearchSubmission(
batch_ids=("batch-1",),
target_count=2,
queued_count=1,
ongoing_count=1,
single=False,
)
)
)
response = asyncio.run( response = asyncio.run(
search_subscribes( search_subscribes(
command=command, command=command,
@@ -663,6 +674,8 @@ class TestSubscribeEndpoint:
) )
assert response.success assert response.success
assert response.message == "已安排 1 个订阅搜索,另有 1 个正在处理中"
assert response.data.queued_count == 1
command.execute.assert_awaited_once() command.execute.assert_awaited_once()
actor = command.execute.await_args.args[0] actor = command.execute.await_args.args[0]
assert actor.username == "alice" assert actor.username == "alice"
+116 -49
View File
@@ -1,3 +1,4 @@
import asyncio
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import Mock from unittest.mock import Mock
@@ -8,6 +9,7 @@ from app.application.subscription.delete import SubscribeDeletionCandidate
from app.application.subscription.search import ( from app.application.subscription.search import (
SearchSubscriptionsCommand, SearchSubscriptionsCommand,
SubscribeSearchActor, SubscribeSearchActor,
SubscriptionSearchSubmission,
) )
from app.runtime.tasks import TaskRegistry from app.runtime.tasks import TaskRegistry
@@ -19,15 +21,15 @@ class _Repository:
"""保存预设单条候选和批量编号。""" """保存预设单条候选和批量编号。"""
self.candidate = candidate self.candidate = candidate
self.subscribe_ids = subscribe_ids or [] self.subscribe_ids = subscribe_ids or []
self.list_calls = []
async def get_candidate(self, _subscribe_id): async def get_candidate(self, _subscribe_id):
"""返回预设订阅候选。""" """返回预设订阅候选。"""
return self.candidate return self.candidate
async def list_search_ids(self, username, state): async def list_search_ids(self, username, state):
"""校验普通用户搜索状态并返回预设编号。""" """记录访问范围并返回预设编号。"""
assert username == "alice" self.list_calls.append((username, state))
assert state == "R"
return self.subscribe_ids return self.subscribe_ids
@@ -40,85 +42,117 @@ def _candidate(username):
) )
@pytest.mark.asyncio def _submitter(calls):
async def test_superuser_search_all_uses_single_global_scheduler_request(): """构造记录目标并返回稳定安排结果的异步提交器。"""
"""管理员搜索全部订阅时保持一次 state=R 的全局调度语义。""" async def submit(subscribe_ids, single):
scheduled = [] """记录一次轻量入队请求。"""
command = SearchSubscriptionsCommand( calls.append((subscribe_ids, single))
repository=_Repository(), return SubscriptionSearchSubmission(
schedule_search=lambda sid, state: scheduled.append((sid, state)), batch_ids=("batch-1",),
target_count=len(subscribe_ids),
queued_count=len(subscribe_ids),
ongoing_count=0,
single=single,
) )
assert await command.execute( return submit
@pytest.mark.asyncio
async def test_superuser_search_all_queues_every_active_subscription():
"""管理员搜索全部订阅时读取全局活动订阅并一次提交。"""
submitted = []
repository = _Repository(subscribe_ids=[2, 5])
command = SearchSubscriptionsCommand(
repository=repository,
submit_search=_submitter(submitted),
)
result = await command.execute(
SubscribeSearchActor(username="admin", is_superuser=True) SubscribeSearchActor(username="admin", is_superuser=True)
) is True )
assert scheduled == [(None, "R")]
assert result is not None
assert result.target_count == 2
assert repository.list_calls == [(None, "R")]
assert submitted == [((2, 5), False)]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_regular_user_search_all_schedules_only_owned_subscriptions(): async def test_regular_user_search_all_schedules_only_owned_subscriptions():
"""普通用户搜索全部时把归属订阅合并为一次后台批次。""" """普通用户搜索全部时把归属订阅合并为一次后台批次。"""
scheduled = [] submitted = []
repository = _Repository(subscribe_ids=[2, 5])
command = SearchSubscriptionsCommand( command = SearchSubscriptionsCommand(
repository=_Repository(subscribe_ids=[2, 5]), repository=repository,
schedule_search=lambda sid, state: scheduled.append((sid, state)), submit_search=_submitter(submitted),
) )
assert await command.execute( result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False) SubscribeSearchActor(username="alice", is_superuser=False)
) is True )
assert scheduled == [((2, 5), None)]
assert result is not None
assert repository.list_calls == [("alice", "R")]
assert submitted == [((2, 5), False)]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_regular_user_search_all_with_no_targets_does_not_schedule(): async def test_regular_user_search_all_with_no_targets_does_not_schedule():
"""普通用户没有可搜索订阅时不创建空后台任务。""" """普通用户没有可搜索订阅时不创建空后台任务。"""
scheduled = [] submitted = []
command = SearchSubscriptionsCommand( command = SearchSubscriptionsCommand(
repository=_Repository(), repository=_Repository(),
schedule_search=lambda ids, state: scheduled.append((ids, state)), submit_search=_submitter(submitted),
) )
assert await command.execute( result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False) SubscribeSearchActor(username="alice", is_superuser=False)
) is True )
assert scheduled == []
assert result is not None
assert result.target_count == 0
assert result.batch_ids == ()
assert submitted == []
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_targeted_search_rejects_missing_or_other_users_subscription(): async def test_targeted_search_rejects_missing_or_other_users_subscription():
"""单条搜索不得泄漏订阅是否属于其他普通用户。""" """单条搜索不得泄漏订阅是否属于其他普通用户。"""
scheduled = [] submitted = []
command = SearchSubscriptionsCommand( command = SearchSubscriptionsCommand(
repository=_Repository(candidate=_candidate("bob")), repository=_Repository(candidate=_candidate("bob")),
schedule_search=lambda sid, state: scheduled.append((sid, state)), submit_search=_submitter(submitted),
) )
assert await command.execute( assert await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False), SubscribeSearchActor(username="alice", is_superuser=False),
subscribe_id=7, subscribe_id=7,
) is False ) is None
assert scheduled == [] assert submitted == []
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_targeted_search_schedules_accessible_subscription(): async def test_targeted_search_schedules_accessible_subscription():
"""归属用户搜索单条订阅时提交历史兼容参数。""" """归属用户搜索单条订阅时提交历史兼容参数。"""
scheduled = [] submitted = []
command = SearchSubscriptionsCommand( command = SearchSubscriptionsCommand(
repository=_Repository(candidate=_candidate("alice")), repository=_Repository(candidate=_candidate("alice")),
schedule_search=lambda sid, state: scheduled.append((sid, state)), submit_search=_submitter(submitted),
) )
assert await command.execute( result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False), SubscribeSearchActor(username="alice", is_superuser=False),
subscribe_id=7, subscribe_id=7,
) is True )
assert scheduled == [((7,), None)]
assert result is not None
assert result.single is True
assert submitted == [((7,), True)]
def test_subscription_search_batch_uses_one_scheduler_generation(monkeypatch): def test_submitted_search_wakes_short_cycle_queue(monkeypatch):
"""一个后台批次只占用一次调度任务运行权""" """手工搜索入队后立即唤醒短周期恢复任务"""
calls = [] calls = []
monkeypatch.setattr( monkeypatch.setattr(
subscription_dependencies, subscription_dependencies,
@@ -126,24 +160,46 @@ def test_subscription_search_batch_uses_one_scheduler_generation(monkeypatch):
lambda job_id, **kwargs: calls.append((job_id, kwargs)), lambda job_id, **kwargs: calls.append((job_id, kwargs)),
) )
subscription_dependencies._start_subscription_search_batch((2, 5), None) subscription_dependencies._resume_submitted_subscription_search((2, 5))
assert calls == [ assert calls == [
( (
"subscribe_search", "subscribe_search_queue",
{"sids": (2, 5), "state": None, "manual": True}, {"limit": 2, "manual_sids": (2, 5)},
), ),
] ]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_search_dependency_registers_one_owned_background_batch(): async def test_search_dependency_persists_before_waking_background_worker():
"""请求适配器只登记一个具名后台批次""" """请求适配器先等待持久入队完成,再登记后台恢复任务"""
registry = TaskRegistry() registry = TaskRegistry()
registry.create_sync = Mock() calls = []
def create_sync(function, *args, owner, **kwargs):
"""同步执行测试函数,并返回已经完成的 Future。"""
calls.append((function, args, owner, kwargs))
future = asyncio.get_running_loop().create_future()
result = function(*args, **kwargs) if owner.endswith("enqueue") else None
future.set_result(result)
return future
registry.create_sync = Mock(side_effect=create_sync)
repository = _Repository(subscribe_ids=[2, 5]) repository = _Repository(subscribe_ids=[2, 5])
search_repository = SimpleNamespace(
enqueue=Mock(
return_value=SimpleNamespace(
active_batch_ids=("batch-new", "batch-existing"),
created_count=1,
coalesced_count=1,
)
)
)
runtime = SimpleNamespace( runtime = SimpleNamespace(
subscription=SimpleNamespace(repository=lambda _db: repository), subscription=SimpleNamespace(
repository=lambda _db: repository,
search_repository=search_repository,
),
) )
command = subscription_dependencies.get_search_subscriptions_command( command = subscription_dependencies.get_search_subscriptions_command(
task_registry=registry, task_registry=registry,
@@ -151,12 +207,23 @@ async def test_search_dependency_registers_one_owned_background_batch():
runtime=runtime, runtime=runtime,
) )
assert await command.execute( result = await command.execute(
SubscribeSearchActor(username="alice", is_superuser=False) SubscribeSearchActor(username="alice", is_superuser=False)
) is True )
registry.create_sync.assert_called_once_with(
subscription_dependencies._start_subscription_search_batch, assert result is not None
(2, 5), assert result.batch_ids == ("batch-new", "batch-existing")
None, assert result.queued_count == 1
owner="api.subscribe.search", assert result.ongoing_count == 1
assert calls[0] == (
search_repository.enqueue,
(),
"api.subscribe.search.enqueue",
{"subscription_ids": (2, 5), "source": "manual", "priority": 100},
)
assert calls[1] == (
subscription_dependencies._resume_submitted_subscription_search,
((2, 5),),
"api.subscribe.search.run",
{},
) )
+3 -3
View File
@@ -175,7 +175,7 @@ def test_subscribe_search_aborts_when_lock_times_out(monkeypatch) -> None:
subscribe_oper.assert_not_called() subscribe_oper.assert_not_called()
progress.assert_called_once_with( progress.assert_called_once_with(
value=100, value=100,
text="订阅搜索锁等待超时,已跳过本轮", text="订阅搜索正在处理中,本次不再重复开始",
) )
@@ -265,7 +265,7 @@ def test_inline_search_conflict_does_not_report_false_completion(monkeypatch) ->
assert progress.call_args.kwargs == { assert progress.call_args.kwargs == {
"value": 100, "value": 100,
"text": "订阅搜索结束,部分订阅本轮未执行或未完成", "text": "搜索结束,部分订阅这次没有完成",
"data": {"total": 1, "finished": 0}, "data": {"total": 1, "finished": 0},
} }
assert chain._subscription_execution_admission.release(match_lease) is True assert chain._subscription_execution_admission.release(match_lease) is True
@@ -279,4 +279,4 @@ def test_subscribe_match_aborts_when_lock_times_out(monkeypatch) -> None:
chain = object.__new__(SubscribeChain) chain = object.__new__(SubscribeChain)
chain.match({"example.org": []}, progress_callback=progress) chain.match({"example.org": []}, progress_callback=progress)
progress.assert_any_call(value=100, text="订阅匹配锁等待超时,已跳过本轮") progress.assert_any_call(value=100, text="订阅资源检查正在进行,本次不再重复开始")
+40 -5
View File
@@ -14,13 +14,15 @@ def _task(
phase: str = "searching", phase: str = "searching",
updated_at: str = "2026-09-01T01:00:00+00:00", updated_at: str = "2026-09-01T01:00:00+00:00",
batch_id: str = "batch-1", batch_id: str = "batch-1",
source: str = "manual",
available_at: str | None = None,
) -> SearchTaskSnapshot: ) -> SearchTaskSnapshot:
"""构造最小搜索任务快照。""" """构造最小搜索任务快照。"""
return SearchTaskSnapshot( return SearchTaskSnapshot(
task_id=f"task-{subscription_id}", task_id=f"task-{subscription_id}",
batch_id=batch_id, batch_id=batch_id,
subscription_id=subscription_id, subscription_id=subscription_id,
source="manual", source=source,
priority=100, priority=100,
position=subscription_id, position=subscription_id,
state=state, state=state,
@@ -30,8 +32,15 @@ def _task(
lease_token="lease" if state == "running" else None, lease_token="lease" if state == "running" else None,
created_at="2026-09-01T00:00:00+00:00", created_at="2026-09-01T00:00:00+00:00",
updated_at=updated_at, updated_at=updated_at,
available_at=available_at,
current_site_id=9 if phase == "waiting_site_budget" else None, current_site_id=9 if phase == "waiting_site_budget" else None,
last_error=" provider\n timeout " if state == "failed" else None, last_error=(
" provider\n timeout "
if state == "failed"
else "站点暂时忙,系统会自动继续搜索"
if state == "queued" and phase == "waiting_site_budget"
else None
),
) )
@@ -85,18 +94,44 @@ def test_execution_status_exposes_site_wait_and_cancel_capability():
def test_execution_status_exposes_queued_site_wait_without_error(): def test_execution_status_exposes_queued_site_wait_without_error():
"""重新入队的站点预算冲突应显示等待状态而不是失败""" """重新入队的站点繁忙应显示等待状态、说明和继续时间"""
repository = _Repository() repository = _Repository()
repository.tasks[2] = _task(2, state="queued", phase="waiting_site_budget") retry_at = "2026-09-01T01:00:10+00:00"
repository.tasks[2] = _task(
2,
state="queued",
phase="waiting_site_budget",
available_at=retry_at,
)
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,))) statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((2,)))
assert statuses[2].state == "waiting_site_budget" assert statuses[2].state == "waiting_site_budget"
assert statuses[2].phase == "waiting_site_budget" assert statuses[2].phase == "waiting_site_budget"
assert statuses[2].error is None assert statuses[2].error == "站点暂时忙,系统会自动继续搜索"
assert statuses[2].next_run_at == retry_at
assert statuses[2].can_cancel is True assert statuses[2].can_cancel is True
def test_execution_status_exposes_scheduled_new_search_without_failure():
"""新订阅编辑等待期应显示为已安排,而不是跳过或失败。"""
repository = _Repository()
retry_at = "2026-09-01T01:01:00+00:00"
repository.tasks[4] = _task(
4,
state="queued",
phase="scheduled",
source="new",
available_at=retry_at,
)
statuses = asyncio.run(SubscriptionExecutionStatusService(repository).for_subscriptions((4,)))
assert statuses[4].state == "scheduled"
assert statuses[4].next_run_at == retry_at
assert statuses[4].error is None
def test_failed_search_exposes_safe_error(): def test_failed_search_exposes_safe_error():
"""搜索失败文本必须压平且不暴露内部错误细节。""" """搜索失败文本必须压平且不暴露内部错误细节。"""
repository = _Repository() repository = _Repository()
+13 -19
View File
@@ -169,7 +169,7 @@ def test_match_skips_subscription_paused_after_admission() -> None:
"skipped": 1, "skipped": 1,
"failed": 0, "failed": 0,
} }
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅跳过" assert progress.call_args.kwargs["text"] == "订阅资源检查完成,部分订阅这次未检查"
_assert_channel_and_subscription_released(chain, (listed.id,)) _assert_channel_and_subscription_released(chain, (listed.id,))
@@ -200,7 +200,7 @@ def test_match_progress_reports_completed_skipped_and_failed_counts() -> None:
"skipped": 1, "skipped": 1,
"failed": 1, "failed": 1,
} }
assert progress.call_args.kwargs["text"] == "订阅资源匹配完成,部分订阅失败" assert progress.call_args.kwargs["text"] == "订阅资源检查结束,部分订阅没有完成"
assert chain._subscription_execution_admission.release(skipped_lease) is True assert chain._subscription_execution_admission.release(skipped_lease) is True
_assert_channel_and_subscription_released(chain, (completed.id, failed.id)) _assert_channel_and_subscription_released(chain, (completed.id, failed.id))
@@ -220,7 +220,7 @@ def test_match_stop_does_not_report_unvisited_subscriptions_as_completed(monkeyp
assert progress.call_args.kwargs == { assert progress.call_args.kwargs == {
"value": 100, "value": 100,
"text": "订阅资源匹配已停止,部分订阅未执行", "text": "订阅资源检查已停止,部分订阅这次未检查",
"data": { "data": {
"total": 2, "total": 2,
"finished": 0, "finished": 0,
@@ -257,17 +257,12 @@ def test_match_logs_one_bounded_start_and_finish_summary(monkeypatch) -> None:
chain.match({"site-a.example": [object()], "site-b.example": [object(), object()]}) chain.match({"site-a.example": [object()], "site-b.example": [object(), object()]})
assert len(info_logs) == 2 assert len(info_logs) == 2
assert info_logs[0].startswith("订阅治理轮次开始: operation=match ") assert info_logs[0] == "开始检查订阅资源,共 3 个订阅、3 个资源,来自 2 个站点。"
assert "subscriptions=3" in info_logs[0] assert info_logs[1].startswith("订阅资源检查结束:部分订阅这次未检查。")
assert "sites=2" in info_logs[0] assert "本次检查 3/3 个订阅,完成 1 个,这次未检查 2 个,失败 0 个" in info_logs[1]
assert "candidates=3" in info_logs[0] assert "其中 1 个订阅正在处理中,本次没有重复检查" in info_logs[1]
assert info_logs[1].startswith("订阅治理轮次结束: operation=match ") assert "另有 1 个订阅已停止" in info_logs[1]
assert "state=skipped" in info_logs[1] assert "run_id" not in info_logs[1]
assert "task_completed=1" in info_logs[1]
assert "task_skipped=2" in info_logs[1]
assert "admission_conflicts=1" in info_logs[1]
assert "cancelled=1" in info_logs[1]
assert "ttl_timeouts=0" in info_logs[1]
assert "订阅成功" not in "\n".join(info_logs) assert "订阅成功" not in "\n".join(info_logs)
assert chain._subscription_execution_admission.release(search_lease) is True assert chain._subscription_execution_admission.release(search_lease) is True
@@ -293,9 +288,8 @@ def test_match_summary_distinguishes_ttl_timeout(monkeypatch) -> None:
chain.match({"site.example": [object()]}) chain.match({"site.example": [object()]})
assert len(info_logs) == 2 assert len(info_logs) == 2
assert "task_skipped=1" in info_logs[1] assert "这次未检查 1 个" in info_logs[1]
assert "cancelled=0" in info_logs[1] assert "另有 1 个订阅检查时间过长,已停止" in info_logs[1]
assert "ttl_timeouts=1" in info_logs[1]
def test_match_logs_subscription_admission_release_failure(monkeypatch) -> None: def test_match_logs_subscription_admission_release_failure(monkeypatch) -> None:
@@ -323,5 +317,5 @@ def test_match_logs_subscription_admission_release_failure(monkeypatch) -> None:
chain.match({"site.example": [object()]}) chain.match({"site.example": [object()]})
assert any("订阅准入释放失败: operation=match subscription_id=17" in item for item in error_logs) assert any("订阅 17 的搜索状态没有正常恢复,系统稍后会继续检查" in item for item in error_logs)
assert "release_failures=1" in info_logs[-1] assert "另有 1 个订阅的搜索状态没有正常恢复" in info_logs[-1]
+134 -43
View File
@@ -82,6 +82,7 @@ def _chain(tmp_path, subscribes: list[SubscriptionSnapshot]):
chain._match_lock = _ForbiddenLock() chain._match_lock = _ForbiddenLock()
chain._search_queue_lock = threading.Lock() chain._search_queue_lock = threading.Lock()
chain._subscription_execution_admission = SubscriptionExecutionAdmission() chain._subscription_execution_admission = SubscriptionExecutionAdmission()
chain.messagehelper = Mock()
return chain return chain
@@ -179,11 +180,9 @@ def test_successful_sites_remain_available_to_next_due_subscription(tmp_path, mo
assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)] assert calls == [(40, 11), (40, 12), (41, 11), (41, 12)]
assert chain.get_search_batch(batch_id).state == "completed" assert chain.get_search_batch(batch_id).state == "completed"
assert "sites=2" in info_logs[-1] assert "访问 2 个站点" in info_logs[-1]
assert "site_requests=4" in info_logs[-1] assert "发出 4 次请求" in info_logs[-1]
assert "site_failures=0" in info_logs[-1] assert "找到 4 个资源" in info_logs[-1]
assert "site_cooldown_skips=0" in info_logs[-1]
assert "candidates=4" in info_logs[-1]
def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch): def test_fallback_queue_executes_without_match_global_lock(tmp_path, monkeypatch):
@@ -280,12 +279,12 @@ def test_site_budget_conflict_requeues_task_without_batch_failure(tmp_path, monk
assert task.state == "queued" assert task.state == "queued"
assert task.phase == "waiting_site_budget" assert task.phase == "waiting_site_budget"
assert task.available_at == retry_at assert task.available_at == retry_at
assert task.last_error is None assert task.last_error == "站点暂时忙,系统会自动继续搜索"
assert chain.subscription_search_repository.claim_next(owner="worker-after-retry") is None assert chain.subscription_search_repository.claim_next(owner="worker-after-retry") is None
def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch): def test_search_logs_one_readable_start_and_finish_summary(tmp_path, monkeypatch):
"""Search INFO 只保留轮次摘要,并携带任务终态与耗时字段""" """Search INFO 只保留用户能直接读懂的开始和结束摘要"""
subscribes = [_subscribe(50), _subscribe(51)] subscribes = [_subscribe(50), _subscribe(51)]
chain = _chain(tmp_path, subscribes) chain = _chain(tmp_path, subscribes)
_make_tasks_ready(monkeypatch) _make_tasks_ready(monkeypatch)
@@ -305,19 +304,14 @@ def test_search_logs_one_bounded_start_and_finish_summary(tmp_path, monkeypatch)
assert batch_id assert batch_id
assert len(info_logs) == 2 assert len(info_logs) == 2
assert info_logs[0].startswith("订阅治理轮次开始: operation=search ") assert info_logs[0] == "开始订阅定时检查,共 2 个订阅。"
assert f"batch_id={batch_id}" in info_logs[0] assert info_logs[1].startswith("订阅定时检查结束:部分订阅没有完成。")
assert "subscriptions=2" in info_logs[0] assert "本次处理 2/2 个" in info_logs[1]
assert info_logs[1].startswith("订阅治理轮次结束: operation=search ") assert "完成 1 个" in info_logs[1]
assert "state=failed" in info_logs[1] assert "失败 1 个" in info_logs[1]
assert "processed=2" in info_logs[1] assert "发出 0 次请求" in info_logs[1]
assert "task_completed=1" in info_logs[1] assert "用时 " in info_logs[1]
assert "task_failed=1" in info_logs[1] assert "operation=" not in "\n".join(info_logs)
assert "admission_conflicts=0" in info_logs[1]
assert "ttl_timeouts=0" in info_logs[1]
assert "site_requests=0" in info_logs[1]
assert "duration_ms=" in info_logs[1]
assert "订阅成功" not in "\n".join(info_logs)
def test_search_logs_subscription_admission_release_failure(tmp_path, monkeypatch): def test_search_logs_subscription_admission_release_failure(tmp_path, monkeypatch):
@@ -343,8 +337,8 @@ def test_search_logs_subscription_admission_release_failure(tmp_path, monkeypatc
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()): with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.search(state="R") chain.search(state="R")
assert any("订阅准入释放失败: operation=search subscription_id=52" in item for item in error_logs) assert any("订阅《治理电影 52》的搜索状态没有正常恢复" in item for item in error_logs)
assert "release_failures=1" in info_logs[-1] assert "另有 1 个搜索状态没有正常恢复" in info_logs[-1]
def test_queued_search_logs_failed_finish_summary_when_callback_raises(tmp_path, monkeypatch): def test_queued_search_logs_failed_finish_summary_when_callback_raises(tmp_path, monkeypatch):
@@ -361,9 +355,7 @@ def test_queued_search_logs_failed_finish_summary_when_callback_raises(tmp_path,
) )
assert len(info_logs) == 2 assert len(info_logs) == 2
assert info_logs[-1].startswith("订阅治理轮次结束: operation=search ") assert info_logs[-1].startswith("订阅定时检查结束:部分订阅没有完成。")
assert "state=failed" in info_logs[-1]
assert "round_failed=1" in info_logs[-1]
def test_resume_search_logs_failed_finish_summary_when_drain_raises(tmp_path, monkeypatch): def test_resume_search_logs_failed_finish_summary_when_drain_raises(tmp_path, monkeypatch):
@@ -380,11 +372,8 @@ def test_resume_search_logs_failed_finish_summary_when_drain_raises(tmp_path, mo
with pytest.raises(RuntimeError, match="claim failed"): with pytest.raises(RuntimeError, match="claim failed"):
chain.resume_search_queue() chain.resume_search_queue()
assert len(info_logs) == 2 assert len(info_logs) == 1
assert info_logs[-1].startswith("订阅治理轮次结束: operation=search ") assert info_logs[-1].startswith("等待中的订阅搜索结束:部分订阅没有完成。")
assert "source=resume" in info_logs[-1]
assert "state=failed" in info_logs[-1]
assert "round_failed=1" in info_logs[-1]
def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch): def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp_path, monkeypatch):
@@ -457,10 +446,9 @@ def test_swallowed_indexer_failure_marks_task_and_batch_failed_but_continues(tmp
assert batch.failed_count == 1 assert batch.failed_count == 1
assert "Flaky" in batch.last_error assert "Flaky" in batch.last_error
assert "HTTP 429" in batch.last_error assert "HTTP 429" in batch.last_error
assert "sites=2" in info_logs[-1] assert "失败 1 个" in info_logs[-1]
assert "site_requests=2" in info_logs[-1] assert "访问 2 个站点" in info_logs[-1]
assert "site_failures=1" in info_logs[-1] assert "发出 2 次请求" in info_logs[-1]
assert "cooldown_seconds=900.0" in info_logs[-1]
def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch): def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeypatch):
@@ -487,12 +475,116 @@ def test_same_subscription_conflict_is_skipped_without_waiting(tmp_path, monkeyp
assert batch.state == "skipped" assert batch.state == "skipped"
assert batch.finished_count == 0 assert batch.finished_count == 0
assert batch.skipped_count == 1 assert batch.skipped_count == 1
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过" assert batch.last_error == "这个订阅正在处理,本次自动检查无需重复执行"
assert "task_skipped=1" in info_logs[-1] assert "这次未搜索 1 个" in info_logs[-1]
assert "admission_conflicts=1" in info_logs[-1]
assert chain._subscription_execution_admission.release(match_lease) is True assert chain._subscription_execution_admission.release(match_lease) is True
def test_manual_search_waits_for_same_subscription_then_continues(tmp_path, monkeypatch):
"""手动搜索遇到同一订阅正在处理时应稍后自动继续,而不是结束为跳过。"""
subscribe = _subscribe(61)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(return_value=subscribe)
monkeypatch.setattr(chain, "_process_search_subscription", process)
match_lease = chain._subscription_execution_admission.try_acquire(
subscription_id=subscribe.id,
operation="match",
ttl_seconds=60,
)
assert match_lease is not None
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(sid=subscribe.id, state=None, manual=True)
batch = chain.get_search_batch(batch_id)
assert batch.state == "queued"
process.assert_not_called()
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.phase == "waiting_subscription"
assert task.last_error == "这个订阅正在处理,结束后会自动继续搜索"
task.available_at = "1970-01-01T00:00:00+00:00"
session.commit()
assert chain._subscription_execution_admission.release(match_lease) is True
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
chain.resume_search_queue(limit=1)
process.assert_called_once()
assert chain.get_search_batch(batch_id).state == "completed"
def test_recent_new_subscription_is_scheduled_instead_of_skipped(tmp_path, monkeypatch):
"""新订阅的一分钟编辑窗口应保留任务并在到时后自动继续。"""
subscribe = replace(
_subscribe(62),
state="N",
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(side_effect=AssertionError("编辑等待期内不应开始搜索"))
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(state="N")
batch = chain.get_search_batch(batch_id)
assert batch.state == "queued"
assert batch.skipped_count == 0
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.phase == "scheduled"
assert task.last_error == "订阅刚刚创建,保存好设置后会自动开始搜索"
assert task.available_at > datetime.now(timezone.utc).isoformat(timespec="seconds")
process.assert_not_called()
def test_new_subscription_post_commit_creates_scheduled_search(tmp_path):
"""订阅保存后应立即写入自动搜索计划,无需等待五分钟扫描。"""
subscribe = replace(
_subscribe(64),
state="N",
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
batch_id = chain._SubscribeChain__queue_new_subscription_search(subscribe.id)
assert batch_id
assert chain.get_search_batch(batch_id).state == "queued"
with chain.subscription_search_repository._session_factory() as session:
task = session.query(SubscriptionSearchTask).filter_by(
subscription_id=subscribe.id,
).one()
assert task.source == "new"
assert task.phase == "scheduled"
assert task.available_at > datetime.now(timezone.utc).isoformat(timespec="seconds")
def test_manual_search_bypasses_new_subscription_edit_wait(tmp_path, monkeypatch):
"""用户主动点击搜索时不受新订阅编辑等待时间限制。"""
subscribe = replace(
_subscribe(63),
date=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
chain = _chain(tmp_path, [subscribe])
_make_tasks_ready(monkeypatch)
process = Mock(return_value=subscribe)
monkeypatch.setattr(chain, "_process_search_subscription", process)
with patch("app.chain.subscribe.search.SearchChain", return_value=Mock()):
batch_id = chain.search(sid=subscribe.id, state=None, manual=True)
process.assert_called_once()
assert chain.get_search_batch(batch_id).state == "completed"
def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch): def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkeypatch):
"""准入后重新读取到暂停订阅时不得开始搜索或伪造完成。""" """准入后重新读取到暂停订阅时不得开始搜索或伪造完成。"""
subscribe = _subscribe(9) subscribe = _subscribe(9)
@@ -511,7 +603,7 @@ def test_paused_subscription_is_skipped_after_admission_refresh(tmp_path, monkey
assert batch.state == "skipped" assert batch.state == "skipped"
assert batch.finished_count == 0 assert batch.finished_count == 0
assert batch.skipped_count == 1 assert batch.skipped_count == 1
assert batch.last_error == "订阅已暂停,本轮搜索已跳过" assert batch.last_error == "订阅已暂停,这次没有搜索"
def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch): def test_cleanup_failures_cannot_leak_subscription_admission(tmp_path, monkeypatch):
@@ -699,9 +791,8 @@ def test_ttl_expiry_after_normal_return_marks_task_and_batch_failed(tmp_path, mo
assert batch.finished_count == 0 assert batch.finished_count == 0
assert batch.failed_count == 1 assert batch.failed_count == 1
assert batch.cancelled_count == 0 assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间" assert batch.last_error == "这次搜索用时过长,已停止,可稍后重试"
assert "task_failed=1" in info_logs[-1] assert "失败 1 个" in info_logs[-1]
assert "ttl_timeouts=1" in info_logs[-1]
def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch): def test_ttl_expiry_after_download_started_completes_with_actual_result(tmp_path, monkeypatch):
@@ -758,4 +849,4 @@ def test_site_budget_ttl_expiry_marks_task_and_batch_failed(tmp_path, monkeypatc
assert batch.finished_count == 0 assert batch.finished_count == 0
assert batch.failed_count == 1 assert batch.failed_count == 1
assert batch.cancelled_count == 0 assert batch.cancelled_count == 0
assert batch.last_error == "订阅执行已超过协作截止时间" assert batch.last_error == "这次搜索用时过长,已停止,可稍后重试"
+39 -4
View File
@@ -40,7 +40,9 @@ def test_search_queue_coalesces_active_subscription_and_raises_priority(tmp_path
assert manual.created_count == 0 assert manual.created_count == 0
assert manual.coalesced_count == 1 assert manual.coalesced_count == 1
assert manual.batch.state == "completed" assert manual.batch.state == "completed"
assert manual.active_batch_ids == (scheduled.batch.batch_id,)
assert first.subscription_id == 1 assert first.subscription_id == 1
assert first.source == "manual"
assert first.priority == 100 assert first.priority == 100
assert second.subscription_id == 2 assert second.subscription_id == 2
assert first.task_id != second.task_id assert first.task_id != second.task_id
@@ -82,6 +84,39 @@ def test_search_queue_claims_each_subscription_only_after_its_available_at(tmp_p
assert accelerated.priority == 100 assert accelerated.priority == 100
def test_manual_search_promotes_scheduled_new_subscription(tmp_path):
"""用户主动搜索应立即唤醒仍在编辑等待期的新订阅任务。"""
repository, engine = _repository(tmp_path)
later_at = (datetime.now(timezone.utc) + timedelta(minutes=1)).isoformat(timespec="seconds")
automatic = repository.enqueue(
subscription_ids=(22,),
source="new",
priority=50,
available_at_by_subscription={22: later_at},
)
with Session(engine) as session:
scheduled = session.execute(
select(SubscriptionSearchTask).where(SubscriptionSearchTask.subscription_id == 22)
).scalar_one()
assert scheduled.phase == "scheduled"
manual = repository.enqueue(
subscription_ids=(22,),
source="manual",
priority=120,
available_at_by_subscription={22: "1970-01-01T00:00:00+00:00"},
)
claimed = repository.claim_next(owner="worker-manual")
assert manual.created_count == 0
assert manual.active_batch_ids == (automatic.batch.batch_id,)
assert claimed is not None
assert claimed.source == "manual"
assert claimed.priority == 120
assert claimed.phase == "matching"
def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path): def test_search_queue_recovers_expired_lease_with_same_task_identity(tmp_path):
"""进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。""" """进程遗留的过期 running 任务应以新 token 恢复且 attempt 单调递增。"""
repository, engine = _repository(tmp_path) repository, engine = _repository(tmp_path)
@@ -269,8 +304,8 @@ def test_search_queue_aggregates_skipped_tasks_without_marking_success(tmp_path)
assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过" assert batch.last_error == "同一订阅正在由其他通道处理,本轮搜索已跳过"
def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path): def test_search_queue_keeps_manual_work_ahead_of_aged_fallback(tmp_path):
"""手工任务可优先,但等待超过公平窗口的兜底任务不得持续饥饿""" """用户主动搜索始终先于定时检查,避免点击后长时间没有反馈"""
repository, engine = _repository(tmp_path) repository, engine = _repository(tmp_path)
repository.enqueue(subscription_ids=(8,), source="fallback", priority=10) repository.enqueue(subscription_ids=(8,), source="fallback", priority=10)
aged_at = (datetime.now(timezone.utc) - timedelta(minutes=16)).isoformat(timespec="seconds") aged_at = (datetime.now(timezone.utc) - timedelta(minutes=16)).isoformat(timespec="seconds")
@@ -285,5 +320,5 @@ def test_search_queue_ages_old_fallback_ahead_of_new_manual_work(tmp_path):
claimed = repository.claim_next(owner="worker-a") claimed = repository.claim_next(owner="worker-a")
assert claimed.subscription_id == 8 assert claimed.subscription_id == 9
assert claimed.source == "fallback" assert claimed.source == "manual"
+3
View File
@@ -35,6 +35,7 @@ def _repository(tmp_path):
def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path): def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path):
"""同站点第二个调用必须等待,不同站点可立即并行。""" """同站点第二个调用必须等待,不同站点可立即并行。"""
repository, _engine = _repository(tmp_path) repository, _engine = _repository(tmp_path)
before_retry = datetime.now(timezone.utc)
first = repository.claim_site(site_id=1, owner="worker-a", lease_seconds=900) first = repository.claim_site(site_id=1, owner="worker-a", lease_seconds=900)
same_site = repository.claim_site(site_id=1, owner="worker-b", lease_seconds=900) same_site = repository.claim_site(site_id=1, owner="worker-b", lease_seconds=900)
@@ -43,6 +44,8 @@ def test_site_budget_allows_one_inflight_per_site_and_independent_sites(tmp_path
assert first.acquired is True assert first.acquired is True
assert same_site.acquired is False assert same_site.acquired is False
assert same_site.lease_token is None assert same_site.lease_token is None
assert same_site.wait_reason == "busy"
assert datetime.fromisoformat(same_site.retry_at) <= before_retry + timedelta(seconds=11)
assert other_site.acquired is True assert other_site.acquired is True