fix(runtime): 收口调度与事件异步任务生命周期 (#6415)

This commit is contained in:
InfinityPacer
2026-08-23 17:17:59 +08:00
committed by GitHub
parent e7bfc9dd43
commit e1d7918297
18 changed files with 1937 additions and 214 deletions
+33 -4
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6451,
"edge_sha256": "9075d8717e384580cd6a41bc36685438770db4a8d8f18c57e6c494f32937113a",
"edge_count": 6479,
"edge_sha256": "a65f8d4024e2299b37510359c7ffea91219f0aa0ed9673b74a3eb22d51d6c67f",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -1510,6 +1510,8 @@
"app.api.context -> app.application.subscription.delete",
"app.api.context -> app.application.subscription.identity",
"app.api.context -> app.application.subscription.mutation",
"app.api.context -> app.runtime",
"app.api.context -> app.runtime.tasks",
"app.api.context -> app.startup",
"app.api.context -> app.startup.context",
"app.api.dependencies.agent -> app.api",
@@ -1599,6 +1601,7 @@
"app.api.dependencies.subscription -> app.runtime",
"app.api.dependencies.subscription -> app.runtime.events",
"app.api.dependencies.subscription -> app.runtime.log",
"app.api.dependencies.subscription -> app.runtime.tasks",
"app.api.dependencies.subscription -> app.schemas",
"app.api.dependencies.subscription -> app.schemas.types",
"app.api.dependencies.subscription -> app.startup",
@@ -1682,6 +1685,7 @@
"app.api.endpoints.anthropic -> app.agent",
"app.api.endpoints.anthropic -> app.agent.runtime_loader",
"app.api.endpoints.anthropic -> app.api",
"app.api.endpoints.anthropic -> app.api.context",
"app.api.endpoints.anthropic -> app.api.endpoints",
"app.api.endpoints.anthropic -> app.api.endpoints.openai",
"app.api.endpoints.anthropic -> app.api.openai_utils",
@@ -1689,6 +1693,8 @@
"app.api.endpoints.anthropic -> app.api.presentation.sse",
"app.api.endpoints.anthropic -> app.application",
"app.api.endpoints.anthropic -> app.application.configuration",
"app.api.endpoints.anthropic -> app.runtime",
"app.api.endpoints.anthropic -> app.runtime.tasks",
"app.api.endpoints.anthropic -> app.schemas",
"app.api.endpoints.anthropic -> app.schemas.openai",
"app.api.endpoints.auth -> app.api",
@@ -1833,6 +1839,7 @@
"app.api.endpoints.history -> app.runtime.config",
"app.api.endpoints.history -> app.runtime.log",
"app.api.endpoints.history -> app.runtime.progress",
"app.api.endpoints.history -> app.runtime.tasks",
"app.api.endpoints.history -> app.schemas",
"app.api.endpoints.history -> app.schemas.common",
"app.api.endpoints.history -> app.schemas.history",
@@ -1945,6 +1952,7 @@
"app.api.endpoints.message -> app.adapters.web.security",
"app.api.endpoints.message -> app.adapters.web.security.access",
"app.api.endpoints.message -> app.api",
"app.api.endpoints.message -> app.api.context",
"app.api.endpoints.message -> app.api.dependencies",
"app.api.endpoints.message -> app.api.dependencies.agent",
"app.api.endpoints.message -> app.api.dependencies.auth",
@@ -1961,6 +1969,7 @@
"app.api.endpoints.message -> app.runtime.extensions",
"app.api.endpoints.message -> app.runtime.extensions.service_config",
"app.api.endpoints.message -> app.runtime.log",
"app.api.endpoints.message -> app.runtime.tasks",
"app.api.endpoints.message -> app.schemas",
"app.api.endpoints.message -> app.schemas.message",
"app.api.endpoints.message -> app.schemas.response",
@@ -2029,11 +2038,14 @@
"app.api.endpoints.openai -> app.agent.contracts",
"app.api.endpoints.openai -> app.agent.runtime_loader",
"app.api.endpoints.openai -> app.api",
"app.api.endpoints.openai -> app.api.context",
"app.api.endpoints.openai -> app.api.openai_utils",
"app.api.endpoints.openai -> app.api.presentation",
"app.api.endpoints.openai -> app.api.presentation.sse",
"app.api.endpoints.openai -> app.application",
"app.api.endpoints.openai -> app.application.configuration",
"app.api.endpoints.openai -> app.runtime",
"app.api.endpoints.openai -> app.runtime.tasks",
"app.api.endpoints.openai -> app.schemas",
"app.api.endpoints.openai -> app.schemas.openai",
"app.api.endpoints.openai -> app.schemas.types",
@@ -2048,6 +2060,7 @@
"app.api.endpoints.plugin -> app.adapters.web.security",
"app.api.endpoints.plugin -> app.adapters.web.security.access",
"app.api.endpoints.plugin -> app.api",
"app.api.endpoints.plugin -> app.api.context",
"app.api.endpoints.plugin -> app.api.dependencies",
"app.api.endpoints.plugin -> app.api.dependencies.auth",
"app.api.endpoints.plugin -> app.api.dependencies.plugin",
@@ -2069,6 +2082,7 @@
"app.api.endpoints.plugin -> app.runtime.extensions.plugin",
"app.api.endpoints.plugin -> app.runtime.extensions.plugin.contracts",
"app.api.endpoints.plugin -> app.runtime.log",
"app.api.endpoints.plugin -> app.runtime.tasks",
"app.api.endpoints.plugin -> app.schemas",
"app.api.endpoints.plugin -> app.schemas.common",
"app.api.endpoints.plugin -> app.schemas.exception",
@@ -2122,6 +2136,7 @@
"app.api.endpoints.site -> app.adapters.web.security",
"app.api.endpoints.site -> app.adapters.web.security.access",
"app.api.endpoints.site -> app.api",
"app.api.endpoints.site -> app.api.context",
"app.api.endpoints.site -> app.api.dependencies",
"app.api.endpoints.site -> app.api.dependencies.auth",
"app.api.endpoints.site -> app.api.dependencies.site",
@@ -2145,6 +2160,7 @@
"app.api.endpoints.site -> app.domain.site",
"app.api.endpoints.site -> app.runtime",
"app.api.endpoints.site -> app.runtime.log",
"app.api.endpoints.site -> app.runtime.tasks",
"app.api.endpoints.site -> app.schemas",
"app.api.endpoints.site -> app.schemas.common",
"app.api.endpoints.site -> app.schemas.response",
@@ -2180,6 +2196,7 @@
"app.api.endpoints.subscribe -> app.adapters.web.security",
"app.api.endpoints.subscribe -> app.adapters.web.security.access",
"app.api.endpoints.subscribe -> app.api",
"app.api.endpoints.subscribe -> app.api.context",
"app.api.endpoints.subscribe -> app.api.dependencies",
"app.api.endpoints.subscribe -> app.api.dependencies.auth",
"app.api.endpoints.subscribe -> app.api.dependencies.subscription",
@@ -2201,6 +2218,7 @@
"app.api.endpoints.subscribe -> app.domain.metainfo",
"app.api.endpoints.subscribe -> app.runtime",
"app.api.endpoints.subscribe -> app.runtime.events",
"app.api.endpoints.subscribe -> app.runtime.tasks",
"app.api.endpoints.subscribe -> app.schemas",
"app.api.endpoints.subscribe -> app.schemas.common",
"app.api.endpoints.subscribe -> app.schemas.event",
@@ -2347,9 +2365,12 @@
"app.api.endpoints.webhook -> app.adapters.web.security",
"app.api.endpoints.webhook -> app.adapters.web.security.access",
"app.api.endpoints.webhook -> app.api",
"app.api.endpoints.webhook -> app.api.context",
"app.api.endpoints.webhook -> app.api.response",
"app.api.endpoints.webhook -> app.chain",
"app.api.endpoints.webhook -> app.chain.webhook",
"app.api.endpoints.webhook -> app.runtime",
"app.api.endpoints.webhook -> app.runtime.tasks",
"app.api.endpoints.webhook -> app.schemas",
"app.api.endpoints.webhook -> app.schemas.response",
"app.api.endpoints.workflow -> app.adapters",
@@ -2555,6 +2576,8 @@
"app.application.mediaserver -> app.schemas.mediaserver",
"app.application.mediaserver -> app.schemas.system",
"app.application.mediaserver -> app.schemas.types",
"app.application.messaging.agent -> app.runtime",
"app.application.messaging.agent -> app.runtime.tasks",
"app.application.messaging.agent -> app.schemas",
"app.application.messaging.agent -> app.schemas.types",
"app.application.messaging.chat -> app.application",
@@ -3619,7 +3642,6 @@
"app.db.oper.passkey -> app.db.base",
"app.db.oper.passkey -> app.db.models",
"app.db.oper.passkey -> app.db.models.passkey",
"app.db.oper.passkey -> app.db.uow",
"app.db.oper.plugindata -> app.db",
"app.db.oper.plugindata -> app.db.base",
"app.db.oper.plugindata -> app.db.models",
@@ -4300,6 +4322,7 @@
"app.modules.imdb.api -> app.runtime.cache",
"app.modules.imdb.api -> app.runtime.log",
"app.modules.imdb.api -> app.runtime.settings",
"app.modules.imdb.api -> app.runtime.tasks",
"app.modules.indexer -> app.application",
"app.modules.indexer -> app.application.site",
"app.modules.indexer -> app.application.site.health",
@@ -5626,6 +5649,7 @@
"app.runtime.extensions.plugin_manager -> app.foundation.version",
"app.runtime.extensions.plugin_manager -> app.runtime",
"app.runtime.extensions.plugin_manager -> app.runtime.events",
"app.runtime.extensions.plugin_manager -> app.runtime.execution",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions.plugin",
"app.runtime.extensions.plugin_manager -> app.runtime.extensions.plugin.access",
@@ -5994,6 +6018,8 @@
"app.startup.context -> app.application.subscription.identity",
"app.startup.context -> app.application.subscription.mutation",
"app.startup.context -> app.application.workflow",
"app.startup.context -> app.runtime",
"app.startup.context -> app.runtime.tasks",
"app.startup.database -> app.adapters",
"app.startup.database -> app.adapters.system",
"app.startup.database -> app.adapters.system.backup",
@@ -6057,6 +6083,7 @@
"app.startup.lifecycle -> app.runtime.log",
"app.startup.lifecycle -> app.runtime.settings",
"app.startup.lifecycle -> app.runtime.state",
"app.startup.lifecycle -> app.runtime.tasks",
"app.startup.lifecycle -> app.runtime.topology",
"app.startup.lifecycle -> app.startup",
"app.startup.lifecycle -> app.startup.cache_initializer",
@@ -6171,6 +6198,7 @@
"app.startup.modules_initializer -> app.runtime.observability",
"app.startup.modules_initializer -> app.runtime.settings",
"app.startup.modules_initializer -> app.runtime.state",
"app.startup.modules_initializer -> app.runtime.tasks",
"app.startup.modules_initializer -> app.runtime.thread",
"app.startup.modules_initializer -> app.scheduler",
"app.startup.modules_initializer -> app.schemas",
@@ -6468,7 +6496,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 799,
"module_count": 800,
"modules": [
"app",
"app.adapters",
@@ -7161,6 +7189,7 @@
"app.runtime.scheduling",
"app.runtime.settings",
"app.runtime.state",
"app.runtime.tasks",
"app.runtime.thread",
"app.runtime.topology",
"app.scheduler",
+73
View File
@@ -127,9 +127,15 @@ def _add_agent_task(trigger_type: str, trigger_value: str, prefix: str):
def _build_agent_task_scheduler(reconcile: bool = False) -> Scheduler:
"""构造不启动后台线程的 Agent 任务调度器。"""
scheduler = object.__new__(Scheduler)
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._scheduler = BackgroundScheduler(timezone=settings.TZ)
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler._agent_task_interruptions_reconciled = False
if reconcile:
scheduler._reconcile_agent_task_interruptions()
@@ -316,6 +322,11 @@ def test_scheduler_registers_and_removes_agent_task_job() -> None:
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._scheduler = BackgroundScheduler(timezone=settings.TZ)
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
next_run_at = scheduler.update_agent_task_job(task.id)
job_id = scheduler._get_agent_task_job_id(task.id)
@@ -642,6 +653,63 @@ async def test_scheduler_config_reload_does_not_interrupt_running_agent_task() -
manager.process_message.assert_not_awaited()
@pytest.mark.anyio
async def test_scheduler_config_reload_preserves_active_agent_task(
monkeypatch,
) -> None:
"""配置热重载只替换后续计划,已开始的 AgentTask 仍按真实结果收口。"""
task = _add_agent_task("cron", "0 * * * *", "reload-active")
scheduler = _build_agent_task_scheduler()
scheduler.init_agent_task_jobs()
started = asyncio.Event()
release = asyncio.Event()
async def process_message(**_kwargs) -> str:
started.set()
await release.wait()
return "执行完成"
manager = SimpleNamespace(
execute_scheduled_task=AgentManager.execute_scheduled_task,
process_message=process_message,
)
manager.execute_scheduled_task = AgentManager.execute_scheduled_task.__get__(manager)
monkeypatch.setattr(
"app.agent.runtime_loader.get_running_agent_manager",
lambda: manager,
)
monkeypatch.setattr(
scheduler,
"init",
Mock(side_effect=lambda **_kwargs: setattr(
scheduler,
"_lifecycle_state",
"running",
)),
)
job_id = scheduler._get_agent_task_job_id(task.id)
assert scheduler.start(job_id) is True
await asyncio.wait_for(started.wait(), timeout=1)
assert AgentTaskOper().get(task.id).last_status == "running"
await scheduler.on_config_changed()
assert AgentTaskOper().get(task.id).last_status == "running"
assert scheduler._handles
release.set()
async def wait_until_released() -> None:
while scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
completed = AgentTaskOper().get(task.id)
assert completed.last_status == "success"
assert completed.last_result == "执行完成"
def test_scheduler_restart_keeps_interrupted_cron_future_schedule() -> None:
"""周期任务中断后只保留下次正常调度,不抹掉本轮中断事实。"""
task = _add_agent_task("cron", "0 * * * *", "restart-cron")
@@ -699,6 +767,11 @@ def test_scheduler_starts_registered_agent_task_without_waiting() -> None:
"running": False,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler.start = Mock()
assert scheduler.start_agent_task(7) is True
+7
View File
@@ -1,6 +1,7 @@
"""数据库备份与宿主调度器的接入合同。"""
import ast
import threading
from dataclasses import replace
from pathlib import Path
from unittest.mock import Mock
@@ -27,6 +28,12 @@ def _scheduler() -> Scheduler:
scheduler = object.__new__(Scheduler)
scheduler._scheduler = _SchedulerStub()
scheduler._jobs = {}
scheduler._lock = threading.RLock()
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
+172 -1
View File
@@ -1,7 +1,12 @@
"""事件调度订阅快照的并发回归测试。"""
"""事件调度订阅快照和生命周期回归测试。"""
import asyncio
import threading
import pytest
from app.runtime.config import global_vars
from app.runtime import events as events_module
from app.runtime.events import Event, eventmanager
from app.schemas.types import ChainEventType, EventType
@@ -37,6 +42,26 @@ def isolated_eventmanager(monkeypatch):
"_EventManager__executor",
_ImmediateExecutor(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__event",
threading.Event(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__consumer_threads",
[],
)
monkeypatch.setattr(
eventmanager,
"_EventManager__lifecycle_state",
"new",
)
monkeypatch.setattr(
eventmanager,
"_EventManager__async_handles",
{},
)
return eventmanager
@@ -158,3 +183,149 @@ async def test_async_chain_dispatch_uses_subscription_snapshot(
calls.clear()
assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True
assert calls == ["mutating", "late"]
@pytest.mark.asyncio
async def test_async_broadcast_handles_are_cancelled_on_shutdown(isolated_eventmanager):
"""事件总线关闭时必须取消并收口已投递的异步广播处理器。"""
global_vars.set_loop(asyncio.get_running_loop())
handler_count = 5
active = 0
cancelled = 0
all_active = asyncio.Event()
async def handler(_event):
nonlocal active, cancelled
active += 1
if active == handler_count:
all_active.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled += 1
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
consumer_threads = tuple(isolated_eventmanager._EventManager__consumer_threads)
for _ in range(handler_count):
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"shutdown"}},
)
await asyncio.wait_for(all_active.wait(), timeout=2)
assert len(isolated_eventmanager._EventManager__async_handles) == handler_count
await isolated_eventmanager.stop_async()
await asyncio.sleep(0)
assert cancelled == handler_count
assert isolated_eventmanager._EventManager__async_handles == {}
assert not any(
thread.is_alive()
for thread in consumer_threads
)
@pytest.mark.asyncio
async def test_async_broadcast_shutdown_waits_for_handler_cleanup(
isolated_eventmanager,
) -> None:
"""提交代理变为 cancelled 后仍须等待处理器 finally 真正完成。"""
global_vars.set_loop(asyncio.get_running_loop())
started = asyncio.Event()
cancelling = asyncio.Event()
cleanup_release = asyncio.Event()
async def handler(_event):
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelling.set()
await cleanup_release.wait()
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
isolated_eventmanager.send_event(EventType.ConfigChanged, {"key": {"shutdown"}})
await asyncio.wait_for(started.wait(), timeout=2)
stop_task = asyncio.create_task(isolated_eventmanager.stop_async())
await asyncio.wait_for(cancelling.wait(), timeout=1)
await asyncio.sleep(0)
assert not stop_task.done()
assert isolated_eventmanager._EventManager__lifecycle_state == "stopping"
stop_task.cancel()
with pytest.raises(asyncio.CancelledError):
await stop_task
assert isolated_eventmanager._EventManager__async_handles
assert isolated_eventmanager._EventManager__lifecycle_state == "stopping"
cleanup_release.set()
async def wait_until_released() -> None:
while isolated_eventmanager._EventManager__async_handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
await isolated_eventmanager.stop_async()
assert isolated_eventmanager._EventManager__async_handles == {}
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
@pytest.mark.asyncio
async def test_async_broadcast_submission_is_registered_before_stop_snapshot(
isolated_eventmanager,
monkeypatch,
) -> None:
"""事件处理器提交和 owner 登记不得被关闭快照从中切开。"""
global_vars.set_loop(asyncio.get_running_loop())
submission_entered = threading.Event()
submission_release = threading.Event()
real_submit = events_module.asyncio.run_coroutine_threadsafe
def delayed_submit(coroutine, loop):
handle = real_submit(coroutine, loop)
submission_entered.set()
submission_release.wait(timeout=1)
return handle
monkeypatch.setattr(
events_module.asyncio,
"run_coroutine_threadsafe",
delayed_submit,
)
isolated_eventmanager.start()
async def handler() -> None:
await asyncio.Event().wait()
submit_thread = threading.Thread(
target=isolated_eventmanager._EventManager__register_async_handle,
args=(handler(),),
)
submit_thread.start()
assert await asyncio.to_thread(submission_entered.wait, 1)
stop_result = []
stop_thread = threading.Thread(
target=lambda: stop_result.append(
isolated_eventmanager._EventManager__begin_stop()
)
)
stop_thread.start()
await asyncio.sleep(0.02)
assert stop_thread.is_alive()
submission_release.set()
await asyncio.to_thread(submit_thread.join, 1)
await asyncio.to_thread(stop_thread.join, 1)
assert not submit_thread.is_alive()
assert not stop_thread.is_alive()
assert len(isolated_eventmanager._EventManager__async_handles) == 1
await isolated_eventmanager.stop_async()
assert isolated_eventmanager._EventManager__async_handles == {}
+1 -1
View File
@@ -694,7 +694,7 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
dependencies = {}
for name, method_name in (
("ModuleManager", "shutdown"),
("EventManager", "stop"),
("EventManager", "stop_async"),
("DohHelper", "shutdown"),
("ThreadHelper", "shutdown"),
("RedisHelper", "close"),
+5
View File
@@ -122,6 +122,11 @@ def _build_scheduler_for_plugin_reload(jobs: dict, backend) -> Scheduler:
scheduler._lock = threading.RLock()
scheduler._jobs = jobs
scheduler._scheduler = backend
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
+8 -3
View File
@@ -54,13 +54,13 @@ def test_scheduler_initializer_stop_preserves_sync_abi(monkeypatch):
assert scheduler_initializer.stop_scheduler() is None
scheduler.stop.assert_called_once_with()
scheduler.async_stop.assert_not_called()
scheduler.stop_async.assert_not_called()
def test_scheduler_initializer_stop_awaits_in_running_loop(monkeypatch):
"""生命周期事件循环中的停止入口应返回可等待的异步收口。"""
scheduler = Mock()
scheduler.async_stop = AsyncMock()
scheduler.stop_async = AsyncMock()
monkeypatch.setattr(scheduler_initializer, "Scheduler", Mock(return_value=scheduler))
async def scenario():
@@ -69,7 +69,7 @@ def test_scheduler_initializer_stop_awaits_in_running_loop(monkeypatch):
await result
asyncio.run(scenario())
scheduler.async_stop.assert_awaited_once_with()
scheduler.stop_async.assert_awaited_once_with()
scheduler.stop.assert_not_called()
@@ -117,6 +117,11 @@ def test_clear_cache_is_manual_only(monkeypatch):
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._lifecycle_state = "new"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
scheduler._agent_task_interruptions_reconciled = True
scheduler._auth_count = 0
scheduler._auth_message = False
+724
View File
@@ -0,0 +1,724 @@
"""Scheduler 任务句柄、generation 与 AgentTask reservation 回归。"""
import asyncio
import gc
import threading
import warnings
import pytest
from app import scheduler as scheduler_module
from app.runtime.config import global_vars
from app.scheduler import Scheduler
class _ProgressStub:
"""隔离 scheduler 生命周期测试的同步进度后端。"""
def __init__(self, _key: str) -> None:
"""接收进度键但不连接外部后端。"""
def start(self) -> None:
"""记录进度开始。"""
def update(self, **_kwargs) -> None:
"""忽略中间进度。"""
def get(self):
"""返回空的历史进度。"""
return None
class _AsyncProgressStub:
"""隔离 scheduler 生命周期测试的异步进度后端。"""
def __init__(self, _key: str) -> None:
"""接收进度键但不连接外部后端。"""
async def get(self):
"""返回空的历史进度。"""
return None
async def update(self, **_kwargs) -> None:
"""忽略中间进度。"""
async def end(self, **_kwargs) -> None:
"""记录终态但不访问外部缓存。"""
def _scheduler(job_id: str, func) -> Scheduler:
"""构造已启动但不拥有 APScheduler 线程的实例。"""
scheduler = object.__new__(Scheduler)
scheduler._scheduler = None
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._jobs = {
job_id: {
"name": "生命周期测试",
"provider_name": "测试",
"func": func,
"running": False,
"_generation": 1,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {job_id: 1}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
@pytest.mark.anyio
async def test_stop_async_cancels_and_awaits_scheduler_owned_job(monkeypatch) -> None:
"""关闭后已投递协程必须取消并完成收尾,不得遗留 owner 句柄。"""
started = asyncio.Event()
cleaned = asyncio.Event()
async def job():
started.set()
try:
await asyncio.Event().wait()
finally:
cleaned.set()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("lifecycle-job", job)
assert scheduler.start("lifecycle-job") is True
await asyncio.wait_for(started.wait(), timeout=1)
assert scheduler._handles
await scheduler.stop_async()
assert cleaned.is_set()
assert scheduler._jobs["lifecycle-job"]["running"] is False
assert scheduler._jobs["lifecycle-job"]["last_error"] == "任务已取消"
assert scheduler._handles == {}
assert scheduler._lifecycle_state == "stopped"
@pytest.mark.anyio
async def test_foreign_loop_submission_runs_on_main_loop_and_finishes_before_stop(
monkeypatch,
) -> None:
"""自建事件循环提交的任务仍由应用主循环拥有并完成取消收尾。"""
main_loop = asyncio.get_running_loop()
started = asyncio.Event()
cancelling = asyncio.Event()
cleanup_release = asyncio.Event()
execution_loop = None
async def job() -> None:
nonlocal execution_loop
execution_loop = asyncio.get_running_loop()
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelling.set()
await cleanup_release.wait()
raise
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
scheduler = _scheduler("foreign-loop-job", job)
def submit_from_foreign_loop() -> bool:
async def submit() -> bool:
return scheduler.start("foreign-loop-job")
return asyncio.run(submit())
assert await asyncio.to_thread(submit_from_foreign_loop) is True
await asyncio.wait_for(started.wait(), timeout=1)
assert execution_loop is main_loop
stop_task = asyncio.create_task(scheduler.stop_async())
await asyncio.wait_for(cancelling.wait(), timeout=1)
await asyncio.sleep(0)
assert not stop_task.done()
assert scheduler._lifecycle_state == "stopping"
stop_task.cancel()
with pytest.raises(asyncio.CancelledError):
await stop_task
assert scheduler._handles
assert scheduler._lifecycle_state == "stopping"
cleanup_release.set()
async def wait_until_released() -> None:
while scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
await scheduler.stop_async()
assert scheduler._handles == {}
assert scheduler._lifecycle_state == "stopped"
@pytest.mark.anyio
async def test_cross_thread_submission_is_registered_before_stop_snapshot(
monkeypatch,
) -> None:
"""跨线程提交与 owner 登记必须对关闭快照表现为同一原子操作。"""
main_loop = asyncio.get_running_loop()
registration_entered = threading.Event()
registration_release = threading.Event()
async def job() -> None:
await asyncio.Event().wait()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
scheduler = _scheduler("atomic-submit", job)
register_handle = scheduler._register_handle
def delayed_register(**kwargs) -> bool:
registration_entered.set()
registration_release.wait(timeout=1)
return register_handle(**kwargs)
monkeypatch.setattr(scheduler, "_register_handle", delayed_register)
submit_thread = threading.Thread(target=scheduler.start, args=("atomic-submit",))
submit_thread.start()
assert await asyncio.to_thread(registration_entered.wait, 1)
stop_result = []
stop_thread = threading.Thread(target=lambda: stop_result.append(scheduler._begin_stop()))
stop_thread.start()
await asyncio.sleep(0.02)
assert stop_thread.is_alive()
registration_release.set()
await asyncio.to_thread(submit_thread.join, 1)
await asyncio.to_thread(stop_thread.join, 1)
assert not submit_thread.is_alive()
assert not stop_thread.is_alive()
assert len(stop_result[0][1]) == 1
for handle in stop_result[0][1]:
scheduler._cancel_handle(handle)
await scheduler._await_cancelled_handles(stop_result[0][1])
@pytest.mark.anyio
async def test_submit_to_loop_tracks_internal_progress_or_finish_tasks() -> None:
"""进度和收尾协程也必须归 Scheduler 所有并可在关闭时收口。"""
started = asyncio.Event()
cancelled = asyncio.Event()
async def pending() -> None:
started.set()
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancelled.set()
raise
scheduler = _scheduler("internal-task", lambda: None)
scheduler._submit_to_loop(
pending(),
job_id="internal-task",
generation=1,
)
await asyncio.wait_for(started.wait(), timeout=1)
assert len(scheduler._handles) == 1
await scheduler.stop_async()
assert cancelled.is_set()
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_sync_job_callback_and_finish_handles_are_owned(monkeypatch) -> None:
"""同步任务回投的进度与收尾句柄都必须纳入关闭收口。"""
update_started = asyncio.Event()
finish_started = asyncio.Event()
gate = asyncio.Event()
cancelled = 0
class BlockingProgress:
"""让进度和收尾停在异步后端,便于验证 owner registry。"""
def __init__(self, _key: str) -> None:
pass
async def update(self, **_kwargs) -> None:
nonlocal cancelled
update_started.set()
try:
await gate.wait()
except asyncio.CancelledError:
cancelled += 1
raise
async def get(self):
nonlocal cancelled
finish_started.set()
try:
await gate.wait()
except asyncio.CancelledError:
cancelled += 1
raise
return None
async def end(self, **_kwargs) -> None:
pass
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", BlockingProgress)
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", asyncio.get_running_loop())
def job(progress_callback) -> None:
progress_callback(value=50)
scheduler = _scheduler("callback-handles", job)
await asyncio.to_thread(scheduler.start, "callback-handles")
await asyncio.wait_for(
asyncio.gather(update_started.wait(), finish_started.wait()),
timeout=1,
)
assert len(scheduler._handles) == 2
await scheduler.stop_async()
assert cancelled == 2
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_stale_progress_cannot_update_replaced_job(monkeypatch) -> None:
"""旧 generation 的延迟进度不得写入新注册的同 ID 任务。"""
updates = []
class RecordingProgress:
def __init__(self, _key: str) -> None:
pass
async def update(self, **kwargs) -> None:
updates.append(kwargs)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", RecordingProgress)
scheduler = _scheduler("generation-progress", lambda: None)
old_job = scheduler._jobs["generation-progress"]
callback = scheduler._Scheduler__build_progress_callback(
"generation-progress",
old_job,
)
scheduler._jobs["generation-progress"] = {
"name": "新一代",
"provider_name": "测试",
"running": True,
"_generation": 2,
}
callback(value=42, text="旧进度")
await asyncio.sleep(0)
await asyncio.sleep(0)
assert updates == []
assert scheduler._handles == {}
@pytest.mark.anyio
async def test_replaced_job_keeps_active_state_without_stale_progress(monkeypatch) -> None:
"""同 ID 新 generation 显示真实运行态,但不继承旧任务进度详情。"""
detail = {}
class RecordingProgress:
def __init__(self, _key: str) -> None:
pass
def start(self) -> None:
pass
def update(self, **kwargs) -> None:
detail.update(kwargs)
def get(self):
return detail
class RecordingAsyncProgress:
def __init__(self, _key: str) -> None:
pass
async def get(self):
return detail
monkeypatch.setattr(scheduler_module, "ProgressHelper", RecordingProgress)
monkeypatch.setattr(
scheduler_module,
"AsyncProgressHelper",
RecordingAsyncProgress,
)
scheduler = _scheduler("generation-cache", lambda: None)
old_job = scheduler._Scheduler__prepare_job("generation-cache")
assert old_job is not None
assert detail["data"]["_generation"] == 1
scheduler._jobs["generation-cache"] = {
"name": "新一代",
"provider_name": "测试",
"running": False,
"_generation": 2,
}
progress = scheduler.get_progress("generation-cache")
assert progress is not None
assert progress.status == "running"
assert progress.enable is True
assert progress.value == 0
assert "_generation" not in progress.data
async_progress = await scheduler.aget_progress("generation-cache")
assert async_progress is not None
assert async_progress.status == "running"
assert async_progress.enable is True
assert async_progress.value == 0
assert "_generation" not in async_progress.data
@pytest.mark.anyio
async def test_stale_generation_cannot_finish_replaced_job(monkeypatch) -> None:
"""旧 generation 收尾不得改写同 ID 的新任务状态或进度。"""
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("generation-job", lambda: None)
old_job = scheduler._jobs["generation-job"]
old_job["running"] = True
new_job = {
"name": "新一代",
"provider_name": "测试",
"running": True,
"_generation": 2,
}
scheduler._jobs["generation-job"] = new_job
await scheduler._Scheduler__finish_job(
job_id="generation-job",
job=old_job,
generation=1,
success=True,
)
assert new_job["running"] is True
assert "last_finished_at" not in new_job
assert old_job["running"] is True
def test_agent_task_manual_start_has_single_reservation() -> None:
"""并发手动触发同一 AgentTask 时只能有一个调用获得 reservation。"""
scheduler = _scheduler("agent-task-1", lambda: None)
scheduler._jobs["agent-task-1"].update(
name="AgentTask",
owner="agent",
)
entered = threading.Event()
release = threading.Event()
results = []
def start(*_args, **_kwargs):
entered.set()
release.wait(timeout=1)
return True
scheduler.start = start
first = threading.Thread(
target=lambda: results.append(scheduler.start_agent_task(1)),
)
first.start()
assert entered.wait(timeout=1)
second = scheduler.start_agent_task(1)
release.set()
first.join(timeout=1)
assert second is False
assert results == [True]
assert scheduler._agent_task_reservations == {}
def test_scheduler_rejects_new_submission_after_stop() -> None:
"""进入 stopping/stopped 后不得再从旧 scheduler 提交任务。"""
scheduler = _scheduler("stopped-job", lambda: None)
scheduler._lifecycle_state = "stopping"
assert scheduler.start("stopped-job") is False
assert scheduler._jobs["stopped-job"]["running"] is False
@pytest.mark.anyio
async def test_config_reload_does_not_restart_scheduler_during_shutdown() -> None:
"""系统关闭开始后到达的配置事件不得重新打开调度入口。"""
scheduler = _scheduler("shutdown-reload", lambda: None)
scheduler._lifecycle_state = "stopping"
scheduler.init = lambda **_kwargs: pytest.fail("关闭阶段不得重新初始化调度器")
await scheduler.on_config_changed()
assert scheduler._lifecycle_state == "stopping"
@pytest.mark.anyio
async def test_concurrent_config_reload_waits_for_old_scheduler_shutdown(
monkeypatch,
) -> None:
"""并发配置事件合并为一次重建,旧调度线程池结束前不得启动新实例。"""
shutdown_started = threading.Event()
shutdown_release = threading.Event()
class BlockingScheduler:
running = True
@staticmethod
def remove_all_jobs() -> None:
pass
@staticmethod
def shutdown() -> None:
shutdown_started.set()
shutdown_release.wait(timeout=1)
scheduler = _scheduler("reload-once", lambda: None)
scheduler._scheduler = BlockingScheduler()
init_calls = 0
def init(**_kwargs) -> None:
nonlocal init_calls
init_calls += 1
scheduler._lifecycle_state = "running"
monkeypatch.setattr(scheduler, "init", init)
first = asyncio.create_task(scheduler.on_config_changed())
assert await asyncio.to_thread(shutdown_started.wait, 1)
await scheduler.on_config_changed()
assert init_calls == 0
assert scheduler._lifecycle_state == "reloading"
shutdown_release.set()
await asyncio.wait_for(first, timeout=1)
assert init_calls == 1
assert scheduler._lifecycle_state == "running"
@pytest.mark.anyio
async def test_config_reload_preserves_overlap_guard_across_job_generations(
monkeypatch,
) -> None:
"""热重载替换任务定义后,同 ID 旧任务结束前不得启动新 generation。"""
started = asyncio.Event()
release = asyncio.Event()
finished = asyncio.Event()
run_count = 0
async def job() -> None:
nonlocal run_count
run_count += 1
if run_count == 1:
started.set()
await release.wait()
finished.set()
monkeypatch.setattr(scheduler_module, "ProgressHelper", _ProgressStub)
monkeypatch.setattr(scheduler_module, "AsyncProgressHelper", _AsyncProgressStub)
scheduler = _scheduler("reload-overlap", job)
class ActiveScheduler:
"""提供列表接口所需的最小 APScheduler 状态。"""
running = True
@staticmethod
def get_jobs() -> list:
"""当前用例只关注正在运行任务,不提供后续计划。"""
return []
def init(**_kwargs) -> None:
replacement = {
"name": "生命周期测试",
"provider_name": "测试",
"func": job,
"running": False,
}
scheduler._assign_job_generation("reload-overlap", replacement)
scheduler._jobs = {"reload-overlap": replacement}
scheduler._scheduler = ActiveScheduler()
scheduler._lifecycle_state = "running"
monkeypatch.setattr(scheduler, "init", init)
assert scheduler.start("reload-overlap") is True
await asyncio.wait_for(started.wait(), timeout=1)
await scheduler.on_config_changed()
progress = scheduler.get_progress("reload-overlap")
assert progress is not None
assert progress.status == "running"
assert progress.enable is True
listed = scheduler.list()
assert len(listed) == 1
assert listed[0].id == "reload-overlap"
assert listed[0].status == "正在运行"
assert scheduler.start("reload-overlap") is False
assert run_count == 1
assert len(scheduler._handles) == 1
release.set()
await asyncio.wait_for(finished.wait(), timeout=1)
async def wait_until_released() -> None:
while scheduler._active_job_generations or scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_released(), timeout=1)
assert scheduler.start("reload-overlap") is True
async def wait_until_second_run_finishes() -> None:
while scheduler._active_job_generations or scheduler._handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_until_second_run_finishes(), timeout=1)
assert run_count == 2
def test_stop_between_prepare_and_submission_releases_active_generation(
monkeypatch,
) -> None:
"""关闭插入准备与提交之间时,不得遗留未实际运行的 generation。"""
calls = 0
scheduler = _scheduler("stop-race", None)
async def job() -> None:
nonlocal calls
calls += 1
scheduler._jobs["stop-race"]["func"] = job
original_prepare = scheduler._Scheduler__prepare_job
def prepare_then_stop(job_id: str):
prepared = original_prepare(job_id)
scheduler._begin_stop()
return prepared
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", prepare_then_stop)
assert scheduler.start("stop-race") is False
assert calls == 0
assert scheduler._handles == {}
assert scheduler._active_job_generations == {}
assert scheduler._jobs["stop-race"]["running"] is False
assert scheduler._jobs["stop-race"]["last_error"] == "任务未提交"
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", original_prepare)
scheduler._lifecycle_state = "running"
assert scheduler.start("stop-race") is True
assert calls == 1
assert scheduler._active_job_generations == {}
@pytest.mark.anyio
async def test_cross_thread_rejection_closes_unstarted_business_coroutine(
monkeypatch,
) -> None:
"""跨线程提交被关闭门禁拒绝时,包装与业务协程都必须释放。"""
main_loop = asyncio.get_running_loop()
prepared = threading.Event()
release = threading.Event()
calls = 0
scheduler = _scheduler("cross-thread-stop-race", None)
async def job() -> None:
nonlocal calls
calls += 1
scheduler._jobs["cross-thread-stop-race"]["func"] = job
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", main_loop)
original_prepare = scheduler._Scheduler__prepare_job
def prepare_then_wait(job_id: str):
result = original_prepare(job_id)
prepared.set()
release.wait(timeout=1)
return result
monkeypatch.setattr(scheduler, "_Scheduler__prepare_job", prepare_then_wait)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always", RuntimeWarning)
start_task = asyncio.create_task(
asyncio.to_thread(scheduler.start, "cross-thread-stop-race")
)
assert await asyncio.to_thread(prepared.wait, 1)
scheduler._begin_stop()
release.set()
assert await asyncio.wait_for(start_task, timeout=1) is False
gc.collect()
assert calls == 0
assert scheduler._active_job_generations == {}
assert scheduler._handles == {}
assert not any("was never awaited" in str(item.message) for item in captured)
def test_cancelled_cross_thread_proxy_waits_for_target_loop_cleanup(
monkeypatch,
) -> None:
"""跨线程代理提前取消后,真实完成信号必须等待目标循环清理。"""
target_loop = asyncio.new_event_loop()
loop_blocked = threading.Event()
loop_release = threading.Event()
loop_drained = threading.Event()
loop_errors = []
loop_thread = threading.Thread(target=target_loop.run_forever)
loop_thread.start()
def block_target_loop() -> None:
loop_blocked.set()
loop_release.wait(timeout=1)
target_loop.set_exception_handler(
lambda _loop, context: loop_errors.append(context)
)
target_loop.call_soon_threadsafe(block_target_loop)
assert loop_blocked.wait(timeout=1)
scheduler = _scheduler("cancel-before-start", None)
calls = 0
async def business() -> None:
nonlocal calls
calls += 1
scheduler._jobs["cancel-before-start"]["func"] = business
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", target_loop)
with warnings.catch_warnings(record=True) as captured:
warnings.simplefilter("always", RuntimeWarning)
try:
assert scheduler.start("cancel-before-start") is True
scheduler_handle = next(iter(scheduler._handles.values()))
scheduler._cancel_handle(scheduler_handle)
assert not scheduler_handle.completion.done()
loop_release.set()
target_loop.call_soon_threadsafe(loop_drained.set)
assert loop_drained.wait(timeout=1)
assert scheduler_handle.completion.done()
gc.collect()
finally:
target_loop.call_soon_threadsafe(target_loop.stop)
loop_thread.join(timeout=1)
target_loop.close()
assert calls == 0
assert loop_errors == []
assert scheduler._active_job_generations == {}
assert scheduler._handles == {}
assert not any("was never awaited" in str(item.message) for item in captured)
+15 -7
View File
@@ -14,8 +14,6 @@ def _build_scheduler(job_id, func):
scheduler._scheduler = None
scheduler._event = threading.Event()
scheduler._lock = threading.RLock()
scheduler._async_tasks = set()
scheduler._accepting_async_tasks = True
scheduler._jobs = {
job_id: {
"name": "测试定时服务",
@@ -24,6 +22,11 @@ def _build_scheduler(job_id, func):
"running": False,
}
}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
return scheduler
@@ -161,7 +164,7 @@ def test_scheduler_records_cancelled_async_job_as_failed():
async def run_task():
job = scheduler._Scheduler__prepare_job(job_id)
with pytest.raises(asyncio.CancelledError):
await scheduler._Scheduler__run_coro_job(task(), job_id, job)
await scheduler._Scheduler__run_coro_job(task, job_id, job)
scheduler = _build_scheduler(job_id, task)
asyncio.run(run_task())
@@ -173,7 +176,7 @@ def test_scheduler_records_cancelled_async_job_as_failed():
assert progress.error == "任务已取消"
def test_scheduler_async_stop_cancels_owned_async_jobs():
def test_scheduler_stop_async_cancels_owned_async_jobs():
"""Scheduler 关停应取消并等待自身登记的异步作业。"""
job_id = f"test-owned-task-{uuid4()}"
started = asyncio.Event()
@@ -194,13 +197,13 @@ def test_scheduler_async_stop_cancels_owned_async_jobs():
"""在当前事件循环启动并收口异步作业。"""
scheduler.start(job_id)
await started.wait()
assert len(scheduler._async_tasks) == 1
await scheduler.async_stop(timeout_seconds=1)
assert len(scheduler._handles) == 1
await scheduler.stop_async()
asyncio.run(run_task())
assert cancelled.is_set()
assert scheduler._async_tasks == set()
assert scheduler._handles == {}
def test_scheduler_returns_none_for_unknown_job():
@@ -209,5 +212,10 @@ def test_scheduler_returns_none_for_unknown_job():
scheduler = object.__new__(Scheduler)
scheduler._lock = threading.RLock()
scheduler._jobs = {}
scheduler._lifecycle_state = "running"
scheduler._handles = {}
scheduler._job_generations = {}
scheduler._active_job_generations = {}
scheduler._agent_task_reservations = {}
assert scheduler.get_progress(job_id) is None
+60
View File
@@ -0,0 +1,60 @@
from unittest.mock import AsyncMock, Mock
import pytest
from app.api.endpoints import site as site_endpoint
from app.application.site.mutation import SiteMutationResult
from app.runtime.tasks import TaskRegistry
class _TaskRegistry(TaskRegistry):
"""记录站点重置提交的同步后台任务。"""
def __init__(self) -> None:
"""初始化任务记录。"""
super().__init__()
self.calls: list[tuple] = []
def create_sync(self, function, *args, owner: str, **kwargs) -> None:
"""保存函数、参数和 owner,避免端点测试启动真实任务。"""
self.calls.append((function, args, kwargs, owner))
@pytest.mark.asyncio
async def test_reset_submits_cookiecloud_after_site_transaction(monkeypatch):
"""站点重置提交 CookieCloud 后台任务,不在请求事件循环内直接执行。"""
command = Mock()
command.reset = AsyncMock(return_value=SiteMutationResult(success=True))
task_registry = _TaskRegistry()
scheduler = Mock()
system_config = Mock()
system_config.async_set = AsyncMock()
monkeypatch.setattr(site_endpoint, "Scheduler", Mock(return_value=scheduler))
monkeypatch.setattr(
site_endpoint,
"get_configured_system_config",
Mock(return_value=system_config),
)
response = await site_endpoint.reset(
task_registry=task_registry,
command=command,
_=Mock(),
)
assert response.success is True
command.reset.assert_awaited_once_with()
scheduler.start.assert_not_called()
system_config.async_set.assert_any_await(
site_endpoint.SystemConfigKey.IndexerSites, []
)
system_config.async_set.assert_any_await(site_endpoint.SystemConfigKey.RssSites, [])
assert task_registry.calls == [
(
scheduler.start,
(),
{"job_id": "cookiecloud", "manual": True},
"api.site.reset",
)
]