refactor: own shutdown lifecycle boundaries

This commit is contained in:
jxxghp
2026-08-23 20:20:26 +08:00
parent 59f020f226
commit 7f09927c47
59 changed files with 6393 additions and 958 deletions
+5 -1
View File
@@ -155,7 +155,11 @@ class AgentServiceAdapter:
raise CapabilityAdapterContractError(
f"{spec.entrypoint}.close() 必须返回 awaitable"
)
await result
converged = await result
if converged is False:
raise CapabilityAdapterContractError(
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
)
@staticmethod
async def cleanup(
+37 -3
View File
@@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
)
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
self._accepting_tasks = True
self._close_cancel_requested: set[asyncio.Task] = set()
self.tools = [
StructuredTool.from_function(
coroutine=self._control_task,
@@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
"""记录任务完成时间并取出异常避免未读取告警。"""
cancel_requested = getattr(self, "_close_cancel_requested", None)
if cancel_requested is not None:
cancel_requested.discard(task)
record = self._tasks.get(task_id)
if record:
record.finished_at = datetime.now()
@@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
@staticmethod
async def _cancel_records(
records: list[_SubAgentRuntimeTask],
*,
cancel_requested: Optional[set[asyncio.Task]] = None,
) -> list[_SubAgentRuntimeTask]:
"""取消一组任务,并返回等待上限内仍未收敛的记录。"""
cancellable_tasks = [
@@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
if cancellable_tasks:
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
for task in cancellable_tasks:
if cancel_requested is not None and task in cancel_requested:
continue
task.cancel()
if cancel_requested is not None:
cancel_requested.add(task)
if not cancellable_tasks:
return []
@@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
return [record for record in records if record.task in pending]
async def close(self) -> None:
"""取消脱离当前 Agent 回合的子代理任务"""
def seal(self) -> None:
"""封住新的 detached 子代理提交,既有任务继续由记录表持有"""
self._accepting_tasks = False
async def close(self) -> bool:
"""有限等待 detached 子代理;超时保留记录并返回 False。"""
self.seal()
if not hasattr(self, "_close_cancel_requested"):
self._close_cancel_requested = set()
unfinished_records = [
record for record in self._tasks.values() if not record.task.done()
]
@@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
logger.info(
f"关闭子代理任务控制器,取消未完成任务: tasks={len(unfinished_records)}"
)
await self._cancel_records(unfinished_records)
pending_records = await self._cancel_records(
unfinished_records,
cancel_requested=self._close_cancel_requested,
)
if pending_records:
self._tasks = {
record.task_id: record
for record in pending_records
}
return False
self._tasks.clear()
self._close_cancel_requested.clear()
return True
@staticmethod
def _pipeline_description(
@@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
timeout = normalized_timeout_ms / 1000
for step_index, spec in enumerate(specs, start=1):
if not self._accepting_tasks:
return records, "子代理任务控制器正在关闭,不能再启动新任务。"
record = self._create_pipeline_record(spec)
records.append(record)
pipeline_description = self._pipeline_description(
@@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
"""管理异步子代理任务。"""
logger.info(f"收到子代理管控操作: action={action}")
if action in {"start", "run", "pipeline"}:
if not self._accepting_tasks:
error = "子代理任务控制器正在关闭,不能再启动新任务。"
return self._json_response({"success": False, "error": error})
specs, error = self._normalize_specs(
description=description,
subagent_type=subagent_type,
+134 -33
View File
@@ -436,6 +436,7 @@ class MoviePilotAgent:
self._agent_started_at: Optional[datetime] = None
self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None
self._subagent_middlewares: tuple[Any, ...] = ()
self._shutdown_started = False
self._last_agent_cache_hit = False
# 流式token管理
@@ -1751,10 +1752,25 @@ class MoviePilotAgent:
return None
@staticmethod
async def _close_subagent_middleware_instances(
def _seal_subagent_middleware_instances(
middlewares: tuple[Any, ...],
) -> None:
"""释放不再由 Agent 图持有的子代理控制器"""
"""同步封住子代理控制器的新任务入口,不等待既有任务退出"""
for middleware in middlewares:
seal = getattr(middleware, "seal", None)
if not callable(seal):
continue
try:
seal()
except Exception as error:
logger.debug(f"封住子代理中间件失败: {error}")
@staticmethod
async def _close_subagent_middleware_instances(
middlewares: tuple[Any, ...],
) -> tuple[Any, ...]:
"""关闭子代理控制器,并返回仍持有未收敛任务的实例。"""
pending_middlewares = []
for middleware in middlewares:
close = getattr(middleware, "close", None)
if not callable(close):
@@ -1762,9 +1778,30 @@ class MoviePilotAgent:
try:
result = close()
if inspect.isawaitable(result):
await result
result = await result
if result is False:
pending_middlewares.append(middleware)
except Exception as error:
logger.debug(f"关闭子代理中间件失败: {error}")
pending_middlewares.append(middleware)
return tuple(pending_middlewares)
@staticmethod
def _merge_subagent_middleware_owners(
*groups: tuple[Any, ...],
) -> tuple[Any, ...]:
"""按对象身份合并当前图与延迟收敛控制器的 owner 集合。"""
merged = []
for group in groups:
for middleware in group:
if not any(middleware is existing for existing in merged):
merged.append(middleware)
return tuple(merged)
def begin_shutdown(self) -> None:
"""在任何异步等待前封住当前 Agent 的 detached 子代理提交。"""
self._shutdown_started = True
self._seal_subagent_middleware_instances(self._subagent_middlewares)
async def _cache_agent(
self,
@@ -1786,7 +1823,11 @@ class MoviePilotAgent:
for replacement in subagent_middlewares
)
)
await self._close_subagent_middleware_instances(previous_middlewares)
if self._shutdown_started:
self._seal_subagent_middleware_instances(subagent_middlewares)
pending_middlewares = await self._close_subagent_middleware_instances(
previous_middlewares
)
self._compiled_agent_bundle = _CompiledAgentBundle(
signature=signature,
agent=agent,
@@ -1798,15 +1839,21 @@ class MoviePilotAgent:
mcp_config_signature=mcp_config_signature,
catalog_checked_at=datetime.now(),
)
self._subagent_middlewares = subagent_middlewares
self._subagent_middlewares = self._merge_subagent_middleware_owners(
pending_middlewares,
subagent_middlewares,
)
return agent
async def _invalidate_cached_agent(self) -> None:
"""使当前图失效,并释放只属于该图的子代理控制器"""
async def _invalidate_cached_agent(self) -> bool:
"""使当前图失效,未收敛的子代理控制器继续由 Agent 持有"""
subagent_middlewares = self._subagent_middlewares
self._subagent_middlewares = ()
self._compiled_agent_bundle = None
await self._close_subagent_middleware_instances(subagent_middlewares)
pending_middlewares = await self._close_subagent_middleware_instances(
subagent_middlewares
)
self._subagent_middlewares = pending_middlewares
return not pending_middlewares
@staticmethod
def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]:
@@ -2003,9 +2050,13 @@ class MoviePilotAgent:
if cached_agent:
# 签名相同表示已编译图中的精确工具实例仍有效;新建快照仅用于复核。
cached_bundle.catalog_checked_at = datetime.now()
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
temporary_subagent_middlewares = ()
logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}")
return cached_agent
@@ -2123,14 +2174,22 @@ class MoviePilotAgent:
temporary_subagent_middlewares = ()
return cached_agent
except asyncio.CancelledError:
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
raise
except Exception as e:
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
logger.error(f"创建 Agent 失败: {e}")
raise
@@ -2539,14 +2598,20 @@ class MoviePilotAgent:
)
)
async def cleanup(self):
async def cleanup(self) -> bool:
"""
清理智能体资源
清理智能体资源;detached 子代理未收敛时保留 owner 并返回 False。
"""
await self._invalidate_cached_agent()
self.begin_shutdown()
if not await self._invalidate_cached_agent():
logger.error(
f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}"
)
return False
self._pending_secret_confirmation = None
self.protected_output_callback = None
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
return True
@dataclass
@@ -2626,6 +2691,7 @@ class AgentManager:
self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {}
self._session_cancel_requested: set[str] = set()
self._close_finalizer_task: Optional[asyncio.Task] = None
self._closed = False
self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT
# 接收门禁与队列写入共用一把锁,确保关闭开始后不会再创建 worker。
self._lifecycle_lock = asyncio.Lock()
@@ -2693,22 +2759,33 @@ class AgentManager:
async with self._lifecycle_lock:
if self._accepting_tasks:
return
if self._close_finalizer_task and not self._close_finalizer_task.done():
raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭")
memory_manager.initialize()
if not self._idle_cleanup_task or self._idle_cleanup_task.done():
self._idle_cleanup_task = asyncio.create_task(
self._cleanup_idle_sessions()
)
self._accepting_tasks = True
self._closed = False
async def close(self):
async def close(self) -> bool:
"""
关闭管理器
关闭管理器,并诚实返回全部会话 owner 是否已经收敛。
"""
async with self._lifecycle_lock:
if self._closed:
return True
if self._close_finalizer_task and not self._close_finalizer_task.done():
return
return False
# 门禁必须先关闭;锁内完成清理可阻止等待中的请求在收口期间重新入队。
self._accepting_tasks = False
# 子代理提交门禁同样必须在第一次 await 前关闭,否则 detached task
# 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。
for agent in self.active_agents.values():
begin_shutdown = getattr(agent, "begin_shutdown", None)
if callable(begin_shutdown):
begin_shutdown()
if self._idle_cleanup_task:
self._idle_cleanup_task.cancel()
try:
@@ -2751,8 +2828,8 @@ class AgentManager:
self._session_workers.pop(session_id, None)
for session_id, agent in list(self.active_agents.items()):
if session_id not in timed_out_session_ids:
await agent.cleanup()
self.active_agents.pop(session_id, None)
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
logger.error(
"AgentManager 关闭时仍有 worker 未收敛,"
f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束"
@@ -2760,13 +2837,21 @@ class AgentManager:
self._close_finalizer_task = asyncio.create_task(
self._finish_deferred_close(timed_out_workers)
)
return
return False
self._session_workers.clear()
for agent in list(self.active_agents.values()):
await agent.cleanup()
self.active_agents.clear()
for session_id, agent in list(self.active_agents.items()):
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
if self.active_agents:
logger.error(
"AgentManager 仍有 %d 个 detached 子代理 owner 未收敛",
len(self.active_agents),
)
return False
await memory_manager.close()
self._closed = True
return True
def _record_session_activity(self, session_id: str, user_id: str) -> None:
"""
@@ -3072,7 +3157,10 @@ class AgentManager:
and isinstance(task.agent_factory, type)
and not isinstance(existing_agent, task.agent_factory)
):
await existing_agent.cleanup()
if await existing_agent.cleanup() is False:
raise AgentManagerUnavailableError(
f"Agent 会话 {session_id} 仍有子代理任务在停止"
)
self.active_agents.pop(session_id, None)
if session_id not in self.active_agents:
@@ -3249,7 +3337,11 @@ class AgentManager:
# 清理agent
if session_id in self.active_agents:
agent = self.active_agents[session_id]
await agent.cleanup()
if await agent.cleanup() is False:
logger.error(
f"会话 {session_id} 仍有子代理 owner 未收敛,保留会话与记忆"
)
return
del self.active_agents[session_id]
memory_manager.clear_memory(session_id, user_id)
logger.info(f"会话 {session_id} 的记忆已清空")
@@ -3290,9 +3382,14 @@ class AgentManager:
self._session_deferred_cleanup_tasks.pop(session_id, None)
self._session_queue_rejections.pop(session_id, None)
self._session_last_queue_wait_ms.pop(session_id, None)
agent = self.active_agents.pop(session_id, None)
agent = self.active_agents.get(session_id)
if agent:
await agent.cleanup()
if await agent.cleanup() is False:
logger.error(
f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛"
)
return
self.active_agents.pop(session_id, None)
memory_manager.clear_memory(session_id, user_id)
logger.info(f"会话 {session_id} 的记忆已清空")
@@ -3310,15 +3407,19 @@ class AgentManager:
for session_id, worker in workers:
if self._session_workers.get(session_id) is worker:
self._session_workers.pop(session_id, None)
agent = self.active_agents.pop(session_id, None)
if agent:
await agent.cleanup()
for session_id, agent in list(self.active_agents.items()):
await agent.cleanup()
self.active_agents.pop(session_id, None)
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
self._session_shutdown_pending.clear()
self._session_cancel_requested.clear()
if self.active_agents:
logger.error(
"AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛",
len(self.active_agents),
)
return
await memory_manager.close()
self._closed = True
finally:
self._close_finalizer_task = None
+12 -7
View File
@@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import (
build_agent_capability_registry,
should_run_agent_service,
)
from app.runtime.capabilities.model import CapabilityMaterializationState
from app.runtime.capabilities.model import (
CapabilityLifecycleState,
CapabilityMaterializationState,
)
from app.runtime.capabilities.runtime import CapabilityRuntime
@@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None:
await close()
async def begin_agent_shutdown() -> None:
"""不可逆关闭首用闸门,并等待全部同步及异步能力释放"""
try:
await _ensure_runtime().shutdown_async(reason="application_shutdown")
finally:
await close_materialized_terminal_sessions()
async def begin_agent_shutdown() -> bool:
"""不可逆关闭首用闸门,并返回 Agent service 是否真实收敛"""
runtime = _ensure_runtime()
await runtime.shutdown_async(reason="application_shutdown")
return (
runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle
is CapabilityLifecycleState.STOPPED
)
+117 -26
View File
@@ -3,7 +3,7 @@ import inspect
import json
import threading
from abc import ABCMeta, abstractmethod
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
from contextvars import Context, copy_context
from functools import partial
from pathlib import Path
@@ -168,33 +168,130 @@ _blocking_semaphores = {
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
}
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
_blocking_executor_lock = threading.Lock()
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
_blocking_executor_lock = threading.RLock()
_blocking_executor_accepting = True
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
"""按桶懒加载线程池,避免在导入阶段创建过多 worker。"""
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
with _blocking_executor_lock:
executor = _blocking_futures.pop(future, None)
if executor is None or executor not in _blocking_retiring_executors:
return
if executor not in _blocking_futures.values():
_blocking_retiring_executors.discard(executor)
def _submit_blocking_call(
bucket: str,
bound_call: Callable[[], Any],
) -> ConcurrentFuture[Any]:
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
context = copy_context()
with _blocking_executor_lock:
if not _blocking_executor_accepting:
raise RuntimeError("Agent 工具阻塞执行器正在关闭,不能再提交新任务")
executor = _blocking_executors.get(bucket)
if executor:
return executor
limit = _BLOCKING_BUCKET_LIMITS[bucket]
executor = ThreadPoolExecutor(
max_workers=limit,
thread_name_prefix=f"agent-tool-{bucket}",
)
_blocking_executors[bucket] = executor
return executor
if executor is None:
limit = _BLOCKING_BUCKET_LIMITS[bucket]
executor = ThreadPoolExecutor(
max_workers=limit,
thread_name_prefix=f"agent-tool-{bucket}",
)
_blocking_executors[bucket] = executor
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
future = Context().run(executor.submit, context.run, bound_call)
_blocking_futures[future] = executor
future.add_done_callback(_discard_blocking_future)
return future
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
"""关闭 Agent 工具阻塞线程池,释放长期运行进程或测试环境中的 worker。"""
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
"""撤销活动 executor 的提交资格,并保留其运行 Future 对应的 owner。"""
with _blocking_executor_lock:
executors = list(_blocking_executors.values())
executors = tuple(_blocking_executors.values())
_blocking_executors.clear()
_blocking_retiring_executors.update(executors)
for executor in executors:
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
executor.shutdown(wait=False, cancel_futures=cancel_futures)
with _blocking_executor_lock:
owned_executors = set(_blocking_futures.values())
_blocking_retiring_executors.intersection_update(owned_executors)
return executors
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
"""原子封住新阻塞工具提交,并请求取消尚未开始的同步调用。"""
global _blocking_executor_accepting
with _blocking_executor_lock:
_blocking_executor_accepting = False
_retire_blocking_executors(cancel_futures=cancel_futures)
def reopen_blocking_executors() -> bool:
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
global _blocking_executor_accepting
with _blocking_executor_lock:
if _blocking_futures or _blocking_retiring_executors:
return False
_blocking_executor_accepting = True
return True
async def close_blocking_executors(
*,
timeout_seconds: float,
cancel_futures: bool = True,
) -> bool:
"""有限等待全部阻塞工具 Future,超时保留 Future 与 executor owner。"""
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
with _blocking_executor_lock:
futures = tuple(_blocking_futures)
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
if wrapped_futures:
done, pending = await asyncio.wait(
wrapped_futures,
timeout=max(0.0, timeout_seconds),
)
if done:
await asyncio.gather(*done, return_exceptions=True)
for pending_future in pending:
pending_future.add_done_callback(
lambda completed: completed.exception()
if not completed.cancelled()
else None
)
with _blocking_executor_lock:
unfinished = tuple(
future for future in _blocking_futures if not future.done()
)
retiring_count = len(_blocking_retiring_executors)
if unfinished:
logger.error(
"Agent 阻塞工具未在 %.1f 秒内收敛:futures=%dexecutors=%d",
max(0.0, timeout_seconds),
len(unfinished),
retiring_count,
)
return False
return True
def shutdown_blocking_executors(
*,
wait: bool = True,
cancel_futures: bool = False,
) -> bool:
"""同步清理测试 owner;非等待模式下保留尚未收敛的 executor 句柄。"""
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
for executor in executors:
if wait:
executor.shutdown(wait=True, cancel_futures=cancel_futures)
with _blocking_executor_lock:
return not _blocking_futures and not _blocking_retiring_executors
class ToolExecutionTimeoutError(TimeoutError):
@@ -226,13 +323,7 @@ async def run_agent_blocking(
await semaphore.acquire()
try:
context = copy_context()
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
future = Context().run(
_get_blocking_executor(bucket_name).submit,
context.run,
bound_call,
)
future = _submit_blocking_call(bucket_name, bound_call)
except Exception:
semaphore.release()
raise
+81 -69
View File
@@ -2,6 +2,7 @@
import json
import shutil
from contextvars import copy_context
from pathlib import Path
from typing import Any, Optional
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
"""重载插件实例并重新注册其命令、定时任务和 API。"""
runtime_status = get_plugin_manager().reload_plugin(plugin_id)
refresh_plugin_registrations(plugin_id)
return runtime_status
plugin_manager = get_plugin_manager()
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
runtime_status = plugin_manager.reload_plugin(plugin_id)
refresh_plugin_registrations(plugin_id)
return runtime_status
def summarize_plugin(plugin: Any) -> dict[str, Any]:
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
async def reload_runtime(target_id: str) -> object:
"""通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
mutation_context = copy_context()
return await run_agent_blocking(
"plugin",
mutation_context.run,
plugin_manager.reload_plugin_tree,
target_id,
)
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
)
return result
with plugin_manager.suppress_plugin_monitor(plugin_id):
result = await PluginInstallCommand(
installed_plugins_reader=lambda: SystemConfigOper().get(
SystemConfigKey.UserInstalledPlugins
) or [],
installed_plugins_writer=save_installed_plugins,
plugin_ids_provider=plugin_manager.get_plugin_ids,
compatibility_checker=skip_compatibility_check,
package_installer=install_package,
package_checkpointer=package_manager.async_checkpoint,
package_committer=package_manager.async_commit,
package_rollback=package_manager.async_rollback,
install_reporter=lambda target_id, target_repo: (
MoviePilotServerHelper.async_install_plugin_reg(
plugin_id=target_id,
repo_url=target_repo,
)
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
).execute(
plugin_id=plugin_id,
repo_url=repo_url,
force=force,
)
result = await PluginInstallCommand(
installed_plugins_reader=lambda: SystemConfigOper().get(
SystemConfigKey.UserInstalledPlugins
) or [],
installed_plugins_writer=save_installed_plugins,
plugin_ids_provider=plugin_manager.get_plugin_ids,
compatibility_checker=skip_compatibility_check,
package_installer=install_package,
package_checkpointer=package_manager.async_checkpoint,
package_committer=package_manager.async_commit,
package_rollback=package_manager.async_rollback,
install_reporter=lambda target_id, target_repo: (
MoviePilotServerHelper.async_install_plugin_reg(
plugin_id=target_id,
repo_url=target_repo,
)
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
mutation=plugin_manager.mutation,
package_write_guard=plugin_manager.suppress_plugin_monitor,
).execute(
plugin_id=plugin_id,
repo_url=repo_url,
force=force,
)
return result.success, result.message, result.refreshed_only
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
from app.agent.tools.base import run_agent_blocking
plugin_manager = get_plugin_manager()
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
config_oper = SystemConfigOper()
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
if plugin_id in install_plugins:
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
remove_plugin_api(plugin_id)
remove_plugin_job(plugin_id)
plugin_class = plugin_manager.plugins.get(plugin_id)
was_clone = bool(getattr(plugin_class, "is_clone", False))
clone_files_removed = False
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif was_clone:
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
try:
clone_files_removed = await run_agent_blocking(
"plugin",
_remove_plugin_directory,
plugin_base_dir,
config_oper = SystemConfigOper()
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
if plugin_id in install_plugins:
install_plugins = [
plugin for plugin in install_plugins if plugin != plugin_id
]
await config_oper.async_set(
SystemConfigKey.UserInstalledPlugins,
install_plugins,
)
if clone_files_removed:
plugin_manager.plugins.pop(plugin_id, None)
except Exception:
clone_files_removed = False
remove_plugin_from_folders(plugin_id)
plugin_manager.remove_plugin(plugin_id)
remove_plugin_api(plugin_id)
remove_plugin_job(plugin_id)
return {
"was_clone": was_clone,
"clone_files_removed": clone_files_removed,
}
plugin_class = plugin_manager.plugins.get(plugin_id)
was_clone = bool(getattr(plugin_class, "is_clone", False))
clone_files_removed = False
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif was_clone:
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
try:
clone_files_removed = await run_agent_blocking(
"plugin",
_remove_plugin_directory,
plugin_base_dir,
)
if clone_files_removed:
plugin_manager.plugins.pop(plugin_id, None)
except Exception:
clone_files_removed = False
remove_plugin_from_folders(plugin_id)
plugin_manager.remove_plugin(plugin_id)
return {
"was_clone": was_clone,
"clone_files_removed": clone_files_removed,
}
+41 -37
View File
@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
)
plugin_manager = get_plugin_manager()
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
next_config = {} if replace else dict(current_config)
if updates:
next_config.update(updates)
for key in remove_keys:
next_config.pop(key, None)
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
next_config = {} if replace else dict(current_config)
if updates:
next_config.update(updates)
for key in remove_keys:
next_config.pop(key, None)
changed_keys = sorted(
key
for key in set(current_config.keys()) | set(next_config.keys())
if current_config.get(key) != next_config.get(key)
or (key in current_config) != (key in next_config)
)
if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
return json.dumps(
{
"success": False,
"message": f"保存插件 {plugin_id} 配置失败",
},
ensure_ascii=False,
changed_keys = sorted(
key
for key in set(current_config.keys()) | set(next_config.keys())
if current_config.get(key) != next_config.get(key)
or (key in current_config) != (key in next_config)
)
return json.dumps(
{
"success": True,
**plugin_info,
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
"replace": replace,
"changed_keys": changed_keys,
"removed_keys": remove_keys,
"config_requires_reload": True,
"previous_config": current_config,
"saved_config": next_config,
},
ensure_ascii=False,
indent=2,
default=str,
)
if not await plugin_manager.async_save_plugin_config(
plugin_id,
next_config,
):
return json.dumps(
{
"success": False,
"message": f"保存插件 {plugin_id} 配置失败",
},
ensure_ascii=False,
)
return json.dumps(
{
"success": True,
**plugin_info,
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
"replace": replace,
"changed_keys": changed_keys,
"removed_keys": remove_keys,
"config_requires_reload": True,
"previous_config": current_config,
"saved_config": next_config,
},
ensure_ascii=False,
indent=2,
default=str,
)
async def run(
self,
+1
View File
@@ -40,4 +40,5 @@ def get_plugin_config_command() -> PluginConfigCommand:
reload_runtime=manager.reload_plugin,
publish_reset=publish_reset,
refresh_registrations=refresh_registrations,
mutation=manager.mutation,
)
+153 -107
View File
@@ -57,7 +57,10 @@ from app.api.dependencies.plugin import (
from app.adapters.external.server import MoviePilotServerHelper
from app.adapters.external.market import PluginHelper
from app.adapters.system.plugin.package import PluginPackageManager
from app.schemas.exception import PersistenceUnavailableError
from app.schemas.exception import (
PersistenceUnavailableError,
PluginMutationRejectedError,
)
from app.runtime.log import logger
from app.schemas.types import SystemConfigKey
from app.api.context import get_background_task_registry, resolve_background_task_registry
@@ -520,10 +523,15 @@ def reload_plugin(
"""
重新加载插件
"""
# 重新加载插件
runtime_status = PluginManager().reload_plugin(plugin_id)
# 注册插件服务
register_plugin(plugin_id)
plugin_manager = PluginManager()
try:
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
# 重新加载插件
runtime_status = plugin_manager.reload_plugin(plugin_id)
# 注册插件服务
register_plugin(plugin_id)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
if runtime_status is _SchemaPluginRuntimeStatus.ACTIVE:
return _SchemaResponse(success=True)
return _SchemaResponse(
@@ -600,14 +608,15 @@ async def install(
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
mutation=plugin_manager.mutation,
package_write_guard=plugin_manager.suppress_plugin_monitor,
)
result = await command.execute(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=bool(force),
)
with plugin_manager.suppress_plugin_monitor(plugin_id):
result = await command.execute(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=bool(force),
)
if not result.success:
return _SchemaResponse(success=False, message=result.message)
return _SchemaResponse(success=True)
@@ -888,11 +897,12 @@ async def save_plugin_folders(
保存插件文件夹分组配置
"""
try:
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(success=True)
with PluginManager().mutation("保存插件文件夹配置"):
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(success=True)
except PersistenceUnavailableError:
raise
except Exception as e:
@@ -909,18 +919,26 @@ async def create_plugin_folder(
"""
创建新的插件文件夹
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
if folder_name not in folders:
folders[folder_name] = []
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 创建成功"
)
else:
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 已存在")
try:
with PluginManager().mutation(f"创建插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
if folder_name not in folders:
folders[folder_name] = []
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 创建成功"
)
return _SchemaResponse(
success=False,
message=f"文件夹 '{folder_name}' 已存在",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.delete(
@@ -932,15 +950,26 @@ async def delete_plugin_folder(
"""
删除插件文件夹
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
if folder_name in folders:
del folders[folder_name]
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 删除成功"
)
else:
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 不存在")
try:
with PluginManager().mutation(f"删除插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
if folder_name in folders:
del folders[folder_name]
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 删除成功"
)
return _SchemaResponse(
success=False,
message=f"文件夹 '{folder_name}' 不存在",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.put(
@@ -956,12 +985,22 @@ async def update_folder_plugins(
"""
更新指定文件夹中的插件列表
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
folders[folder_name] = plugin_ids
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 中的插件已更新"
)
try:
with PluginManager().mutation(f"更新插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
folders[folder_name] = plugin_ids
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True,
message=f"文件夹 '{folder_name}' 中的插件已更新",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.post(
@@ -975,23 +1014,24 @@ def clone_plugin(
"""
创建插件分身
"""
plugin_manager = PluginManager()
try:
success, message = PluginManager().clone_plugin(
plugin_id=plugin_id,
suffix=clone_data.suffix,
name=clone_data.name,
description=clone_data.description,
version=clone_data.version,
icon=clone_data.icon,
)
with plugin_manager.mutation(f"创建插件 {plugin_id} 分身"):
success, message = plugin_manager.clone_plugin(
plugin_id=plugin_id,
suffix=clone_data.suffix,
name=clone_data.name,
description=clone_data.description,
version=clone_data.version,
icon=clone_data.icon,
)
if success:
# 分身服务已完成运行态加载,此处只补齐宿主注册。
register_plugin(message)
# 将分身插件添加到原插件所在的文件夹中
_add_clone_to_plugin_folder(plugin_id, message)
return _SchemaResponse(success=True, message="插件分身创建成功")
else:
if success:
# 分身服务已完成运行态加载,此处只补齐宿主注册。
register_plugin(message)
# 将分身插件添加到原插件所在的文件夹中
_add_clone_to_plugin_folder(plugin_id, message)
return _SchemaResponse(success=True, message="插件分身创建成功")
return _SchemaResponse(success=False, message=message)
except Exception as e:
logger.error(f"创建插件分身失败:{str(e)}")
@@ -1034,54 +1074,60 @@ def uninstall_plugin(
卸载插件
"""
plugin_manager = PluginManager()
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
return _SchemaResponse(
success=False,
message=f"请先卸载该插件的分身:{instance_ids}",
)
config_oper = get_configured_system_config()
# 删除已安装信息
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
for plugin in install_plugins:
if plugin == plugin_id:
install_plugins.remove(plugin)
break
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
# 移除插件API
remove_plugin_api(plugin_id)
# 移除插件服务
remove_plugin_job(plugin_id)
# 判断是否为分身
plugin_class = plugin_manager.plugins.get(plugin_id)
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif getattr(plugin_class, "is_clone", False):
# 如果是分身插件,则删除分身数据和配置
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
# 删除分身文件
plugin_base_dir = (
get_api_runtime_config_snapshot().root_path
/ "app"
/ "plugins"
/ plugin_id.lower()
)
if plugin_base_dir.exists():
try:
shutil.rmtree(plugin_base_dir)
plugin_manager.plugins.pop(plugin_id, None)
except Exception as e:
logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}")
# 从插件文件夹中移除该插件
remove_plugin_from_folders(plugin_id)
# 移除插件
plugin_manager.remove_plugin(plugin_id)
return _SchemaResponse(success=True)
try:
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
return _SchemaResponse(
success=False,
message=f"请先卸载该插件的分身:{instance_ids}",
)
config_oper = get_configured_system_config()
# 删除已安装信息
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
for plugin in install_plugins:
if plugin == plugin_id:
install_plugins.remove(plugin)
break
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
# 移除插件API
remove_plugin_api(plugin_id)
# 移除插件服务
remove_plugin_job(plugin_id)
# 判断是否为分身
plugin_class = plugin_manager.plugins.get(plugin_id)
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif getattr(plugin_class, "is_clone", False):
# 如果是分身插件,则删除分身数据和配置
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
# 删除分身文件
plugin_base_dir = (
get_api_runtime_config_snapshot().root_path
/ "app"
/ "plugins"
/ plugin_id.lower()
)
if plugin_base_dir.exists():
try:
shutil.rmtree(plugin_base_dir)
plugin_manager.plugins.pop(plugin_id, None)
except Exception as e:
logger.error(
f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}"
)
# 从插件文件夹中移除该插件
remove_plugin_from_folders(plugin_id)
# 移除插件
plugin_manager.remove_plugin(plugin_id)
return _SchemaResponse(success=True)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str):
+25 -13
View File
@@ -4,7 +4,9 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from typing import Any, ContextManager
from app.schemas.exception import PluginMutationRejectedError
@dataclass(frozen=True, slots=True)
@@ -29,6 +31,7 @@ class PluginConfigCommand:
reload_runtime: Callable[[str], Any],
publish_reset: Callable[[str], Any],
refresh_registrations: Callable[[str], Any],
mutation: Callable[[str], ContextManager[None]],
) -> None:
"""保存插件管理 Facade 和运行时注册刷新端口。"""
self._save_config = save_config
@@ -39,21 +42,30 @@ class PluginConfigCommand:
self._reload_runtime = reload_runtime
self._publish_reset = publish_reset
self._refresh_registrations = refresh_registrations
self._mutation = mutation
def update(self, plugin_id: str, config: dict) -> PluginConfigResult:
"""保存配置并按既有顺序重新初始化实例及运行时注册。"""
if not self._save_config(plugin_id, config, False):
return PluginConfigResult(False, "插件配置保存失败")
self._initialize(plugin_id, config)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
try:
with self._mutation(f"更新插件 {plugin_id} 配置"):
if not self._save_config(plugin_id, config, False):
return PluginConfigResult(False, "插件配置保存失败")
self._initialize(plugin_id, config)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
except PluginMutationRejectedError as error:
return PluginConfigResult(False, str(error))
def reset(self, plugin_id: str) -> PluginConfigResult:
"""通知插件补偿后停止实例、删除配置数据并重建运行态。"""
self._publish_reset(plugin_id)
self._stop(plugin_id)
self._delete_config(plugin_id, True)
self._delete_data(plugin_id, True)
self._reload_runtime(plugin_id)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
try:
with self._mutation(f"重置插件 {plugin_id} 配置和数据"):
self._publish_reset(plugin_id)
self._stop(plugin_id)
self._delete_config(plugin_id, True)
self._delete_data(plugin_id, True)
self._reload_runtime(plugin_id)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
except PluginMutationRejectedError as error:
return PluginConfigResult(False, str(error))
+30 -14
View File
@@ -5,11 +5,12 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any, Optional
from typing import Any, ContextManager, Optional
from app.schemas.exception import PersistenceUnavailableError
from app.application.plugin.lifecycle import plugin_lifecycle
from app.runtime.log import logger
from app.schemas.exception import PluginMutationRejectedError
InstalledPluginsReader = Callable[[], list[str]]
@@ -25,6 +26,8 @@ PackageCheckpointAction = Callable[[Any], Awaitable[object]]
InstallReporter = Callable[[str, Optional[str]], Awaitable[object]]
PluginReloader = Callable[[str], Awaitable[object]]
PluginRegistrationRefresher = Callable[[str], Awaitable[object]]
PluginMutationAdmission = Callable[[str], ContextManager[None]]
PluginPackageWriteGuard = Callable[[str], ContextManager[None]]
@dataclass(frozen=True, slots=True)
@@ -94,6 +97,8 @@ class PluginInstallCommand:
install_reporter: InstallReporter,
plugin_reloader: PluginReloader,
registration_refresher: PluginRegistrationRefresher,
mutation: PluginMutationAdmission,
package_write_guard: PluginPackageWriteGuard,
) -> None:
"""保存安装用例所需端口,不绑定数据库、网络或运行时实现。"""
self._installed_plugins_reader = installed_plugins_reader
@@ -107,6 +112,8 @@ class PluginInstallCommand:
self._install_reporter = install_reporter
self._plugin_reloader = plugin_reloader
self._registration_refresher = registration_refresher
self._mutation = mutation
self._package_write_guard = package_write_guard
async def execute(
self,
@@ -120,20 +127,29 @@ class PluginInstallCommand:
state = _InstallState()
async with plugin_lifecycle.hold(plugin_id):
try:
return await self._execute_locked(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=force,
state=state,
with self._mutation(f"安装插件 {plugin_id}"):
with self._package_write_guard(plugin_id):
try:
return await self._execute_locked(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=force,
state=state,
)
except asyncio.CancelledError:
await self._rollback_cancelled(
plugin_id=plugin_id,
original_plugins=state.original_plugins,
state=state,
)
raise
except PluginMutationRejectedError as error:
return PluginInstallResult(
success=False,
message=str(error),
failure_stage="admission",
)
except asyncio.CancelledError:
await self._rollback_cancelled(
plugin_id=plugin_id,
original_plugins=state.original_plugins,
state=state,
)
raise
async def _execute_locked(
self,
+5 -2
View File
@@ -3,12 +3,15 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Protocol
from typing import Any, ContextManager, Protocol
class PluginRuntime(Protocol):
"""声明入口层消费的插件宿主能力。"""
def mutation(self, operation: str) -> ContextManager[None]:
"""为完整插件可变事务取得停机准入 lease。"""
...
def __getattr__(self, name: str) -> Any:
"""允许兼容门面按既有 V3 方法名访问插件宿主能力。"""
+170 -29
View File
@@ -191,9 +191,16 @@ class TransferFailureNotificationAggregator:
NOTIFICATION_DEBOUNCE_SECONDS = 30
def __init__(self) -> None:
"""初始化分组缓冲和定时器"""
"""初始化分组缓冲、回调、定时器与关闭状态"""
self._buffers: dict[str, list[TransferFailureNotification]] = {}
self._callbacks: dict[
str,
Callable[[list[TransferFailureNotification]], None],
] = {}
self._timers: dict[str, asyncio.TimerHandle] = {}
self._generations: dict[str, int] = {}
self._lock = threading.Lock()
self._closed = False
def schedule(
self,
@@ -204,48 +211,128 @@ class TransferFailureNotificationAggregator:
loop: asyncio.AbstractEventLoop,
) -> None:
"""从整理线程安全地把失败快照加入事件循环中的聚合缓冲。"""
loop.call_soon_threadsafe(
self._schedule_on_loop,
group_key,
notification,
callback,
loop,
)
# 先在调用线程登记快照,关闭流程才能覆盖已接收但尚未进入事件循环的通知。
with self._lock:
if self._closed:
raise RuntimeError("整理失败通知聚合器正在关闭,不能再接收通知")
self._buffers.setdefault(group_key, []).append(notification)
self._callbacks[group_key] = callback
generation = self._generations.get(group_key, 0) + 1
self._generations[group_key] = generation
try:
loop.call_soon_threadsafe(
self._schedule_on_loop,
group_key,
generation,
callback,
loop,
)
except Exception as err:
logger.error(
f"创建整理失败通知聚合定时器失败,将立即发送 "
f"(group={group_key}): {err}"
)
self.flush(group_key, generation, callback)
def _schedule_on_loop(
self,
group_key: str,
notification: TransferFailureNotification,
generation: int,
callback: Callable[[list[TransferFailureNotification]], None],
loop: asyncio.AbstractEventLoop,
) -> None:
"""在所属事件循环中更新缓冲重置静默窗口。"""
self._buffers.setdefault(group_key, []).append(notification)
timer = self._timers.pop(group_key, None)
if timer:
timer.cancel()
self._timers[group_key] = loop.call_later(
self.NOTIFICATION_DEBOUNCE_SECONDS,
self.flush,
group_key,
callback,
)
"""在所属事件循环中为已登记缓冲重置静默窗口。"""
schedule_error: Optional[Exception] = None
with self._lock:
if (
self._closed
or group_key not in self._buffers
or self._generations.get(group_key) != generation
):
return
timer = self._timers.pop(group_key, None)
if timer:
timer.cancel()
try:
self._timers[group_key] = loop.call_later(
self.NOTIFICATION_DEBOUNCE_SECONDS,
self.flush,
group_key,
generation,
callback,
)
except Exception as err:
schedule_error = err
if schedule_error is not None:
logger.error(
f"创建整理失败通知聚合定时器失败,将立即发送 "
f"(group={group_key}): {schedule_error}"
)
self.flush(group_key, generation, callback)
def flush(
self,
group_key: str,
generation: int,
callback: Callable[[list[TransferFailureNotification]], None],
) -> None:
"""发送一个分组内的聚合结果并释放缓冲。"""
notifications = self._buffers.pop(group_key, [])
self._timers.pop(group_key, None)
with self._lock:
# 新通知已登记但 timer 重置回调尚未执行时,旧代不得提前发送新批次。
if self._generations.get(group_key) != generation:
return
notifications = self._buffers.pop(group_key, [])
self._callbacks.pop(group_key, None)
timer = self._timers.pop(group_key, None)
self._generations.pop(group_key, None)
if timer:
timer.cancel()
if not notifications:
return
self._deliver(group_key, notifications, callback)
@staticmethod
def _deliver(
group_key: str,
notifications: list[TransferFailureNotification],
callback: Callable[[list[TransferFailureNotification]], None],
) -> None:
"""调用聚合通知回调,并统一观察发送异常。"""
try:
callback(notifications)
except Exception as err:
logger.error(f"发送整理失败聚合通知失败 (group={group_key}): {err}")
def close(self) -> None:
"""停止接收新通知,取消定时器并同步发送全部已缓冲通知。"""
with self._lock:
if self._closed and not self._buffers and not self._timers:
return
self._closed = True
timers = list(self._timers.values())
pending = []
orphaned = []
for group_key, notifications in self._buffers.items():
callback = self._callbacks.get(group_key)
if callback is None:
orphaned.append((group_key, len(notifications)))
continue
pending.append((group_key, notifications, callback))
self._timers.clear()
self._generations.clear()
self._buffers.clear()
self._callbacks.clear()
for timer in timers:
timer.cancel()
for group_key, notification_count in orphaned:
logger.error(
f"整理失败通知聚合缓冲缺少发送回调,无法刷新 "
f"(group={group_key}, count={notification_count})"
)
for group_key, notifications, callback in pending:
self._deliver(group_key, notifications, callback)
# 作业锁:JobManager 与 TransferChain 共享,保护整理作业视图。
job_lock = threading.Lock()
@@ -267,7 +354,8 @@ class JobManager:
# 记录仍由主程序整理线程直接执行的任务,避免把阻塞中的本地任务误判为失活
_active_executions: set[Tuple[str, str]] = set()
def __init__(self):
def __init__(self) -> None:
"""初始化当前进程内的整理作业状态。"""
self._job_view = {}
self._season_episodes = {}
self._meta_to_media_ids = {}
@@ -1011,25 +1099,68 @@ class JobManager:
class FailedRetryScheduler:
"""
负责失败整理记录的 debounce 聚合与 AI 重试调度
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度
缓冲不提供持久化保证关闭时会取消尚未触发的记录由上层 durable
工作流在后续阶段承接需要跨进程保证的重试意图
"""
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
def __init__(self):
def __init__(self) -> None:
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
super().__init__()
self._retry_transfer_buffer: dict[str, list[int]] = {}
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
self._retry_transfer_generations: dict[str, int] = {}
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
self._retry_transfer_lock = asyncio.Lock()
self._closed = False
async def close(self):
async def close(self) -> None:
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
async with self._retry_transfer_lock:
self._closed = True
timers = list(self._retry_transfer_timers.values())
buffered_count = sum(
len(history_ids)
for history_ids in self._retry_transfer_buffer.values()
)
self._retry_transfer_timers.clear()
self._retry_transfer_generations.clear()
self._retry_transfer_buffer.clear()
tasks = tuple(self._retry_transfer_tasks)
for timer in timers:
timer.cancel()
if buffered_count:
logger.warning(
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
)
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
if self._closed:
return
task = asyncio.create_task(
self._flush_retry_transfer(group_key, generation),
name="transfer.failed_retry.flush",
)
self._retry_transfer_tasks.add(task)
task.add_done_callback(self._observe_retry_transfer_task)
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
self._retry_transfer_tasks.discard(task)
if task.cancelled():
return
exception = task.exception()
if exception is not None:
logger.error(f"智能体重试整理后台任务异常: {exception}")
@staticmethod
def _build_retry_transfer_template_context(
@@ -1054,7 +1185,7 @@ class FailedRetryScheduler:
template_context=template_context,
)
async def schedule_retry(self, history_id: int, group_key: str = ""):
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
"""
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用
"""
@@ -1062,6 +1193,8 @@ class FailedRetryScheduler:
group_key = f"_default_{history_id}"
async with self._retry_transfer_lock:
if self._closed:
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
if group_key not in self._retry_transfer_buffer:
self._retry_transfer_buffer[group_key] = []
if history_id not in self._retry_transfer_buffer[group_key]:
@@ -1075,18 +1208,26 @@ class FailedRetryScheduler:
self._retry_transfer_timers[group_key].cancel()
loop = asyncio.get_running_loop()
generation = self._retry_transfer_generations.get(group_key, 0) + 1
self._retry_transfer_generations[group_key] = generation
self._retry_transfer_timers[group_key] = loop.call_later(
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)),
self._start_retry_transfer_task,
group_key,
generation,
)
async def _flush_retry_transfer(self, group_key: str):
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
"""
延迟定时器到期后取出该分组的所有 history_id 并合并为一次 agent 调用
"""
async with self._retry_transfer_lock:
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
if self._retry_transfer_generations.get(group_key) != generation:
return
history_ids = self._retry_transfer_buffer.pop(group_key, [])
self._retry_transfer_timers.pop(group_key, None)
self._retry_transfer_generations.pop(group_key, None)
if not history_ids:
return
+335 -72
View File
@@ -2,9 +2,12 @@ import asyncio
import queue
import re
import threading
import time
import traceback
import uuid
from collections import Counter
from concurrent.futures import CancelledError as FutureCancelledError
from concurrent.futures import Future
from copy import deepcopy
from pathlib import Path
from typing import List, Optional, Tuple, Union, Dict, Callable, Any
@@ -87,10 +90,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
文件整理处理链
"""
# worker 在构造期启动;若中途失败,单例仍需先发布给 lifespan 清理入口。
_retain_failed_singleton = True
CONFIG_WATCH = {
"TRANSFER_THREADS",
}
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
_QUEUE_STOP_SENTINEL = object()
@staticmethod
def _transfer_result_payload(
task: TransferTask,
@@ -108,7 +118,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
"transfer_history_id": history_id,
}
def __init__(self):
def __init__(self) -> None:
"""初始化文件整理处理链。"""
super().__init__()
# 主要媒体文件后缀
@@ -141,7 +151,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._progress = ProgressHelper(ProgressKey.FileTransfer)
# 队列相关状态
self._threads = []
self._retiring_threads: List[threading.Thread] = []
self._queue_active = False
# 每一代 worker 使用独立停止信号,避免热更新启动新 worker 后旧线程重新取任务
self._worker_stop_event = threading.Event()
# 生命周期操作串行化;状态锁只保护短临界区,不覆盖同步文件 I/O 等待
self._worker_lifecycle_lock = threading.RLock()
self._worker_state_lock = threading.RLock()
self._closing = False
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
self._replay_thread: Optional[threading.Thread] = None
self._replay_stop_event = threading.Event()
self._active_tasks = 0
self._processed_num = 0
self._fail_num = 0
@@ -149,33 +169,210 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
# 启动整理任务
self.__init()
def __init(self):
"""
启动文件整理线程
"""
self._queue_active = True
for i in range(self.runtime_config.transfer_threads):
logger.info(f"启动文件整理线程 {i + 1} ...")
thread = threading.Thread(
target=self.__start_transfer, name=f"transfer-{i}", daemon=True
def __init(self) -> bool:
"""启动一代文件整理线程,并返回是否成功取得 worker 所有权。"""
with self._worker_lifecycle_lock:
with self._worker_state_lock:
if self._closing:
logger.warning("文件整理链已进入关闭状态,拒绝重新启动 worker")
return False
self._retiring_threads = [
thread for thread in self._retiring_threads if thread.is_alive()
]
alive_threads = [thread for thread in self._threads if thread.is_alive()]
if alive_threads:
logger.error(
"上一代文件整理线程尚未收敛,拒绝并行启动新 worker:%s",
", ".join(thread.name for thread in alive_threads),
)
self._threads = alive_threads
return False
stop_event = threading.Event()
threads = [
threading.Thread(
target=self.__start_transfer,
args=(stop_event,),
name=f"transfer-{index}",
daemon=True,
)
for index in range(self.runtime_config.transfer_threads)
]
self._worker_stop_event = stop_event
self._threads = threads
self._queue_active = True
for index, thread in enumerate(threads):
logger.info(f"启动文件整理线程 {index + 1} ...")
thread.start()
return True
@staticmethod
def __join_threads(
threads: List[threading.Thread], deadline: float
) -> List[threading.Thread]:
"""在统一截止时间内等待线程,返回仍未收敛且继续由调用方持有的线程。"""
current_thread = threading.current_thread()
for thread in threads:
if thread is current_thread or not thread.is_alive():
continue
thread.join(timeout=max(0.0, deadline - time.monotonic()))
return [thread for thread in threads if thread.is_alive()]
def __acquire_worker_lifecycle_lock(self, deadline: float) -> bool:
"""在统一截止时间内取得 worker 生命周期锁,并兼容同线程 RLock 重入。"""
return self._worker_lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
)
def __request_worker_stop(self) -> List[threading.Thread]:
"""发布当前 worker 代的停止信号,并用哨兵唤醒空闲线程。"""
with self._worker_state_lock:
self._queue_active = False
self._worker_stop_event.set()
current_threads = list(self._threads)
threads = [*self._retiring_threads, *current_threads]
for _ in current_threads:
self._queue.put(self._QUEUE_STOP_SENTINEL)
return threads
def __stop(self, timeout_seconds: float = _WORKER_RESTART_TIMEOUT_SECONDS) -> bool:
"""在锁等待与线程 join 的共享预算内停止当前 worker 代。"""
deadline = time.monotonic() + max(0.0, timeout_seconds)
if not self.__acquire_worker_lifecycle_lock(deadline):
logger.error(
"未在 %.1f 秒内取得文件整理 worker 生命周期锁",
max(0.0, timeout_seconds),
)
self._threads.append(thread)
thread.start()
return False
try:
threads = self.__request_worker_stop()
alive_threads = self.__join_threads(threads, deadline)
with self._worker_state_lock:
self._threads = []
self._retiring_threads = alive_threads
if alive_threads:
logger.error(
"文件整理线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_threads),
)
return False
logger.info("文件整理线程已停止")
return True
finally:
self._worker_lifecycle_lock.release()
def __stop(self):
def close_workers(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
"""
停止文件整理进程
"""
self._queue_active = False
for thread in self._threads:
thread.join()
self._threads = []
logger.info("文件整理线程已停止")
关闭整理 worker pending 回放并拒绝后续队列写入
def on_config_changed(self):
这是宿主生命周期使用的同步边界停止信号只能阻止线程领取下一项工作不能
取消已经进入同步文件或数据库 I/O 的调用超过预算时保留活线程句柄并返回
False调用方据此避免过早释放仍被使用的数据库等下游资源
:param timeout_seconds: 生命周期锁worker 与回放线程共享的最大等待秒数
:return: 全部后台线程均已收敛时返回 True否则返回 False
"""
deadline = time.monotonic() + max(0.0, timeout_seconds)
if not self.__acquire_worker_lifecycle_lock(deadline):
logger.error(
"未在 %.1f 秒内取得整理后台生命周期锁,关闭未开始",
max(0.0, timeout_seconds),
)
return False
try:
with self._worker_state_lock:
self._closing = True
worker_threads = self.__request_worker_stop()
replay_thread = self._replay_thread
self._replay_stop_event.set()
alive_workers = self.__join_threads(worker_threads, deadline)
alive_replays = self.__join_threads(
[replay_thread] if replay_thread else [], deadline
)
with self._worker_state_lock:
self._threads = []
self._retiring_threads = alive_workers
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
self._replay_thread = None
alive_threads = [*alive_workers, *alive_replays]
if alive_threads:
logger.error(
"整理后台线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_threads),
)
return False
logger.info("文件整理 worker 与待处理回放线程已关闭")
return True
finally:
self._worker_lifecycle_lock.release()
async def close(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
"""
收口整理线程失败通知和 AI 重试并返回依赖是否可以安全释放
同步文件 I/O 在线程内无法被 asyncio 取消因此先在线程池中执行有界
``close_workers``只有 worker replay 全部退出后才关闭通知和重试
超时则保留这些依赖供仍在运行的整理回调继续使用
:param timeout_seconds: worker replay 共享的最大等待秒数
:return: 所有整理后台 owner 均已收敛时返回 True
"""
workers_closed = await asyncio.to_thread(
self.close_workers,
timeout_seconds,
)
if not workers_closed:
return False
self.failure_notification_aggregator.close()
await self.retry_scheduler.close()
return True
@staticmethod
def _observe_failed_retry_schedule(future: Future[None]) -> None:
"""观察跨线程 AI 重试调度的完成结果,避免关闭竞态变成静默异常。"""
try:
future.result()
except FutureCancelledError:
return
except Exception as err:
logger.error(f"触发AI智能体重试整理失败: {err}")
def _schedule_failed_transfer_retry(
self,
history_id: int,
group_key: str,
) -> None:
"""把失败历史提交到主事件循环,并持续持有和观察调度结果。"""
retry_coroutine = self.retry_scheduler.schedule_retry(
history_id,
group_key=group_key,
)
try:
future = asyncio.run_coroutine_threadsafe(
retry_coroutine,
global_vars.loop,
)
except Exception as err:
retry_coroutine.close()
logger.error(f"触发AI智能体重试整理失败: {err}")
return
future.add_done_callback(self._observe_failed_retry_schedule)
logger.info(f"已触发AI智能体重试整理历史记录 #{history_id}")
def on_config_changed(self) -> None:
"""配置变更时重启文件整理线程。"""
self.__stop()
self.__init()
with self._worker_lifecycle_lock:
if self._closing:
logger.info("文件整理链正在关闭,忽略 worker 配置热更新")
return
if not self.__stop(
timeout_seconds=self._WORKER_RESTART_TIMEOUT_SECONDS
):
logger.warning(
"旧文件整理 worker 仍在收尾;其停止信号保持有效,新一代接管后续队列"
)
self.__init()
def __default_callback(
self, task: TransferTask, transferinfo: TransferInfo, /
@@ -344,19 +541,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
and self.runtime_config.ai_agent_enable
and self.runtime_config.ai_agent_retry_transfer
):
try:
# 使用 download_hash 或源文件父目录作为分组键,
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
group_key = build_transfer_failure_group_key(task)
asyncio.run_coroutine_threadsafe(
self.retry_scheduler.schedule_retry(
history.id, group_key=group_key
),
global_vars.loop,
)
logger.info(f"已触发AI智能体重试整理历史记录 #{history.id}")
except Exception as e:
logger.error(f"触发AI智能体重试整理失败: {e}")
# 使用 download_hash 或源文件父目录作为分组键,
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
self._schedule_failed_transfer_retry(
history.id,
build_transfer_failure_group_key(task),
)
# 返回失败
ret_status = False
@@ -653,7 +843,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
:param task: 任务信息
:return: True表示任务已添加到队列False表示任务无效或已存在重复
"""
return self._transfer_queue_service().put(task, self.__default_callback)
with self._worker_state_lock:
if self._closing:
logger.warning("文件整理链已关闭,拒绝新的队列任务")
return False
return self._transfer_queue_service().put(task, self.__default_callback)
def _transfer_queue_service(self) -> TransferQueueService:
"""构建保持旧队列对象和私有兼容接缝的应用服务。"""
@@ -667,26 +861,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
expire_tasks=self.__expire_stale_transfer_tasks,
)
def replay_pending(self):
def replay_pending(self) -> None:
"""
回放上次进程退出时仍未整理完的文件
在后台线程执行回放要 stat 源文件而启动期挂载可能尚未就绪甚至处于
挂死状态同步执行会把整个启动流程堵住
"""
threading.Thread(
target=self.__replay_pending,
name="MoviePilot-TransferReplay",
daemon=True
).start()
with self._worker_state_lock:
if self._closing:
logger.info("文件整理链正在关闭,跳过待处理文件回放")
return
if self._replay_thread and self._replay_thread.is_alive():
logger.info("待处理文件回放已在运行,跳过重复启动")
return
stop_event = threading.Event()
thread = threading.Thread(
target=self.__run_replay_pending,
args=(stop_event,),
name="MoviePilot-TransferReplay",
daemon=True,
)
self._replay_stop_event = stop_event
self._replay_thread = thread
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
thread.start()
def __replay_pending(self):
def __run_replay_pending(self, stop_event: threading.Event) -> None:
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
try:
self.__replay_pending(stop_event)
finally:
with self._worker_state_lock:
if self._replay_thread is threading.current_thread():
self._replay_thread = None
def __replay_pending(
self, stop_event: Optional[threading.Event] = None
) -> None:
"""
把落盘登记的待整理文件重新送回整理入口
只回放存储 + 源路径这一最小事实重新走完整的识别与整理流程
已经整理完成的由整理历史查重挡掉因此不存在重复整理的问题
:param stop_event: 宿主关闭信号只阻止处理下一条登记不取消运行中的同步 I/O
"""
stop_event = stop_event or threading.Event()
if stop_event.is_set():
return
try:
pendings = self._pendingoper.list_all()
except Exception as err:
@@ -697,8 +919,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
replayed = 0
for storage, src_path in pendings:
if stop_event.is_set():
break
try:
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
if stop_event.is_set():
break
if not fileitem:
if should_discard:
# 源文件确认已消失,注销登记避免每次启动重复回放
@@ -708,7 +935,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
replayed += 1
except Exception as err:
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
if stop_event.is_set():
logger.info(
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
replayed,
)
else:
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
@staticmethod
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
@@ -876,21 +1109,69 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self.jobview.try_remove_job(task)
self._finish_scrape_batch_task(task)
def __start_transfer(self):
def __settle_transfer_progress_if_idle(self) -> None:
"""在没有 active 或未结算真实任务时结束进度并重置本批计数。"""
with task_lock:
# unfinished_tasks 同时覆盖队列内任务和已被其他 worker 取走、尚未来得及
# 登记 active 的任务。持有 Queue 自身互斥锁完成判断和计数重置,使并发
# enqueue 只能发生在旧批次归零之后;仍在 deque 中的停止哨兵不算真实任务。
with self._queue.all_tasks_done:
queued_stop_sentinels = sum(
item is self._QUEUE_STOP_SENTINEL
for item in self._queue.queue
)
has_unsettled_tasks = (
self._queue.unfinished_tasks > queued_stop_sentinels
)
if (
self._active_tasks != 0
or self._processed_num <= 0
or has_unsettled_tasks
):
return
processed_num = self._processed_num
fail_num = self._fail_num
self._total_num = 0
self._processed_num = 0
self._fail_num = 0
__end_msg = (
f"整理队列处理完成,共整理 {processed_num} 个文件,"
f"失败 {fail_num}"
)
logger.info(__end_msg)
self._progress.update(value=100, text=__end_msg)
self._progress.end()
def __start_transfer(self, stop_event: threading.Event) -> None:
"""
处理队列
处理当前 worker 代的队列停止后不领取下一项任务
:param stop_event: 当前 worker 代专属停止信号热更新后不会被重新清除
"""
while not global_vars.is_system_stopped and self._queue_active:
while not global_vars.is_system_stopped and not stop_event.is_set():
try:
item: TransferQueue = self._queue.get(
block=True, timeout=self._transfer_interval
)
if item is self._QUEUE_STOP_SENTINEL:
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
if stop_event.is_set() or global_vars.is_system_stopped:
break
continue
if stop_event.is_set() or global_vars.is_system_stopped:
# 关闭信号与 queue.get 竞态时,把尚未处理的任务放回队列;其
# TransferPending 登记保持不变,供同进程重启 worker 或下次启动回放。
self._queue.put(item)
self._queue.task_done()
break
if not item:
continue
task = item.task
if not task:
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
continue
# 文件信息
@@ -966,18 +1247,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
with task_lock:
# 减少运行中的任务数
self._active_tasks -= 1
# 检查是否所有任务都已完成且队列为空
if self._active_tasks == 0 and self._queue.empty():
# 结束进度
__end_msg = f"整理队列处理完成,共整理 {self._processed_num} 个文件,失败 {self._fail_num}"
logger.info(__end_msg)
self._progress.update(value=100, text=__end_msg)
self._progress.end()
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
self._total_num = 0
self._processed_num = 0
self._fail_num = 0
self.__settle_transfer_progress_if_idle()
except queue.Empty:
# 即使队列空了,如果还有任务在运行,也不应该结束进度
@@ -1113,18 +1383,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
and self.runtime_config.ai_agent_enable
and self.runtime_config.ai_agent_retry_transfer
):
try:
# 使用 download_hash 或源文件父目录作为分组键
group_key = build_transfer_failure_group_key(task)
asyncio.run_coroutine_threadsafe(
self.retry_scheduler.schedule_retry(
his.id, group_key=group_key
),
global_vars.loop,
)
logger.info(f"已触发AI智能体重试整理历史记录 #{his.id}")
except Exception as e:
logger.error(f"触发AI智能体重试整理失败: {e}")
# 使用 download_hash 或源文件父目录作为分组键
self._schedule_failed_transfer_retry(
his.id,
build_transfer_failure_group_key(task),
)
return False, "未识别到媒体信息"
+24 -4
View File
@@ -14,14 +14,24 @@ class Singleton(abc.ABCMeta, type):
def get_existing_instance(cls, *args, **kwargs):
"""按相同参数返回已创建实例,不触发初始化"""
key = (cls, args, frozenset(kwargs.items()))
return cls._instances.get(key)
with cls._lock:
return cls._instances.get(key)
def __call__(cls, *args, **kwargs):
"""按类和构造参数创建或复用实例。"""
key = (cls, args, frozenset(kwargs.items()))
with cls._lock:
if key not in cls._instances:
cls._instances[key] = super().__call__(*args, **kwargs)
if getattr(cls, "_retain_failed_singleton", False):
# 启动线程的 lifecycle owner 必须先发布身份再执行 __init__;
# 构造中途抛错时保留实例,启动失败清理才能找到已创建的 owner。
instance = cls.__new__(cls, *args, **kwargs)
if not isinstance(instance, cls):
return instance
cls._instances[key] = instance
cls.__init__(instance, *args, **kwargs)
else:
cls._instances[key] = super().__call__(*args, **kwargs)
return cls._instances[key]
@@ -42,13 +52,23 @@ class SingletonClass(abc.ABCMeta, type):
def get_existing_instance(cls):
"""返回已创建实例,不触发初始化"""
return cls._instances.get(cls)
with cls._lock:
return cls._instances.get(cls)
def __call__(cls, *args, **kwargs):
"""按类创建或复用唯一实例。"""
with cls._lock:
if cls not in cls._instances:
cls._instances[cls] = super().__call__(*args, **kwargs)
if getattr(cls, "_retain_failed_singleton", False):
# 与参数化单例保持相同的 owner 发布顺序;锁会阻止其他线程
# 在 __init__ 返回或抛错前读取半构造实例。
instance = cls.__new__(cls, *args, **kwargs)
if not isinstance(instance, cls):
return instance
cls._instances[cls] = instance
cls.__init__(instance, *args, **kwargs)
else:
cls._instances[cls] = super().__call__(*args, **kwargs)
return cls._instances[cls]
+399 -57
View File
@@ -1,4 +1,5 @@
import time
import threading
import traceback
from functools import partial
from pathlib import Path
@@ -27,6 +28,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
目录监控门面单例模式装配本地/远程监控维护生命周期与健康检查
"""
# watcher/scheduler 在构造期启动;异常时保留实例供启动失败屏障收口。
_retain_failed_singleton = True
# 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
@@ -54,11 +57,25 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
REBUILD_KEY_PREFIX = "rebuild:"
PROBE_KEY = "probe"
PENDING_KEY = "pending"
RELOAD_STOP_TIMEOUT = 30.0
LIFECYCLE_CLOSE_TIMEOUT = 90.0
def __init__(self):
def __init__(self) -> None:
"""初始化目录监控依赖、owner 注册表与当前 lifespan 状态。"""
super().__init__()
# 生命周期操作串行化;永久关闭请求用 Event 提前封口,不能被阻塞 I/O 挡住。
self._lifecycle_lock = threading.RLock()
self._owner_lock = threading.Lock()
self._work_stop_event = threading.Event()
self._shutdown_event = threading.Event()
self._closed = False
self._compensation_threads: Dict[int, threading.Thread] = {}
self._scheduler_shutdown_thread: Optional[threading.Thread] = None
self._scheduler_shutdown_succeeded = False
# 本地目录监控服务
self._watchers = []
# 已请求停止但尚未退出的 watcher,只保留 owner 身份,不再参与健康检查
self._retired_watchers = []
# 本地目录监控列表读写锁
self._watcher_lock = Lock()
# 启动失败待重试的本地监控配置
@@ -87,9 +104,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
alert_cb=self.__poller_alert)
# 启动目录监控和文件整理
self.init()
if not self.init():
raise RuntimeError("目录监控 owner 初始化未收敛")
def on_config_changed(self):
def on_config_changed(self) -> None:
"""配置变化时重建监控;lifespan 封口后拒绝重新启动。"""
if self._shutdown_event.is_set():
logger.info("目录监控已进入生命周期封口,忽略配置热更新")
return
self.init()
def get_reload_name(self):
@@ -118,6 +140,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
强制全量扫描并处理所有文件包括已存在的文件
"""
if not self.__accepting_work():
return False
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
@staticmethod
@@ -134,24 +158,61 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
return SnapshotStore.compare(old_snapshot, new_snapshot)
def init(self):
def init(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""在旧 owner 收敛后启动一代监控,关闭中的生命周期拒绝重开。"""
if self._shutdown_event.is_set():
logger.info("目录监控生命周期已关闭,跳过启动")
return False
deadline = time.monotonic() + max(0.0, timeout)
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("目录监控未在 %.1f 秒内取得重载所有权", max(0.0, timeout))
return False
try:
if self._shutdown_event.is_set():
return False
if not self.__stop_owned(deadline=deadline, close=False):
logger.error("旧目录监控 owner 未收敛,取消本次配置热更新")
return False
if self._shutdown_event.is_set():
return False
if not self._recovery.reopen():
logger.error("目录监控恢复线程仍未退出,取消本次配置热更新")
return False
if self._shutdown_event.is_set():
self._recovery.request_stop()
return False
self._work_stop_event.clear()
return self.__initialize_monitors()
finally:
self._lifecycle_lock.release()
def __initialize_monitors(self) -> bool:
"""
启动监控
在已取得生命周期所有权后启动监控
永久关闭请求可以在本方法阻塞于 FUSE 时从其他线程提前设置每个可能产生
owner 的边界都重新检查确保解冻后只能退出不能穿透旧生命周期
"""
# 停止现有任务
self.stop()
if not self.__accepting_work():
return False
# 读取目录配置
monitor_dirs = DirectoryHelper().get_download_dirs()
if not self.__accepting_work():
return False
if not monitor_dirs:
logger.info("未找到任何目录监控配置")
return
return True
messagehelper = MessageHelper()
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
valid_dirs = []
for mon_dir in monitor_dirs:
if not self.__accepting_work():
return False
if not mon_dir.library_path:
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
continue
@@ -172,6 +233,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
# 启动定时服务进程
if not self.__accepting_work():
return False
self._scheduler = BackgroundScheduler(timezone=get_runtime_setting("TZ"))
mon_storages: Dict[str, List[Path]] = {}
@@ -179,6 +242,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
local_started = 0
local_failed = 0
for mon_dir in monitor_dirs:
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
# 检查媒体库目录是不是下载目录的子目录
mon_path = Path(mon_dir.download_path)
target_path = Path(mon_dir.library_path)
@@ -197,6 +263,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
for storage, paths in mon_storages.items():
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
# 远程目录监控 - 使用智能间隔
# 先尝试加载已有快照获取文件数量
snapshot_data = self._store.load(storage)
@@ -238,6 +307,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
# 启动定时服务
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
if self._scheduler.get_jobs():
self._scheduler.print_jobs()
self._scheduler.start()
@@ -253,6 +325,47 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(summary)
else:
logger.info(summary)
return True
def __accepting_work(self) -> bool:
"""判断当前 monitor 代是否仍允许创建 owner 和派发整理工作。"""
return not self._work_stop_event.is_set() and not self._shutdown_event.is_set()
@property
def lifecycle_closed(self) -> bool:
"""返回当前 Monitor 是否已经被应用生命周期永久封口。"""
return self._closed or self._shutdown_event.is_set()
def reopen(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""确认旧 owner 已收敛后,为新的应用 lifespan 显式解除永久封口。"""
if not self.lifecycle_closed:
return True
deadline = time.monotonic() + max(0.0, timeout)
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("旧目录监控生命周期仍在收尾,无法重新开启")
return False
try:
if not self.__stop_owned(deadline=deadline, close=True):
logger.error("旧目录监控 owner 仍未收敛,无法重新开启")
return False
if not self._recovery.reopen():
logger.error("旧目录监控恢复线程仍存活,无法重新开启")
return False
self._closed = False
self._shutdown_event.clear()
return True
finally:
self._lifecycle_lock.release()
def __abort_interrupted_startup(self) -> None:
"""关闭请求穿透阻塞启动后,在当前生命周期锁内收拢已经创建的 owner。"""
logger.info("目录监控启动期间收到关闭请求,正在回收本次已创建 owner")
self.__stop_owned(
deadline=time.monotonic() + self.RELOAD_STOP_TIMEOUT,
close=self._shutdown_event.is_set(),
)
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
"""
@@ -261,6 +374,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param monitor_mode: 配置的监控模式
:return: 是否启动成功
"""
if not self.__accepting_work():
return False
logger.info(f"正在启动本地目录监控: {mon_path}")
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
@@ -298,6 +413,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 启动成功后再登记,避免失败的监控残留在列表中
watcher.start()
with self._watcher_lock:
if not self.__accepting_work():
watcher.stop()
if watcher.is_alive():
self._retired_watchers.append(watcher)
return False
self._watchers.append(watcher)
self._pending_locals = [
pending for pending in self._pending_locals
@@ -312,13 +432,15 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
return False
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception) -> None:
"""
处理本地目录监控启动失败登记待重试并按需告警
:param mon_path: 监控目录
:param monitor_mode: 配置的监控模式
:param err: 启动异常
"""
if not self.__accepting_work():
return
err_msg = str(err)
logger.error(f"启动本地目录监控失败: {mon_path}")
logger.error(f"错误详情: {err_msg}")
@@ -346,7 +468,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
f"将自动退避重试")
def watchdog(self):
def watchdog(self) -> None:
"""
目录监控健康检查检测监控线程状态并驱动恢复
@@ -356,8 +478,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
动作会把看门狗冻死在它自己要修复的挂载上随后停滞检测告警重试驱动
全部静默失效这正是全进程雪崩的起点
"""
if not self.__accepting_work():
return
try:
broken = self.__check_watchers()
if not self.__accepting_work():
return
self.__drive_recovery(broken)
except Exception as e:
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
@@ -372,6 +498,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
with self._watcher_lock:
watchers = list(self._watchers)
self._retired_watchers = [
watcher for watcher in self._retired_watchers if watcher.is_alive()
]
isolated = set(self._isolated)
# 探测已确认挂载恢复的目录,本轮直接送去重建
resumed = list(self._pending_rebuild.values())
@@ -412,7 +541,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
broken.append(watcher)
return broken
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]) -> None:
"""
把所有会触碰挂载的恢复动作派发到一次性工作线程并等待有限时间
@@ -421,6 +550,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
可放弃的子进程探测来确认挂载何时恢复
:param broken: 需要重建的监控列表
"""
if not self.__accepting_work():
return
actions: Dict[str, Callable[[], None]] = {}
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
for watcher in broken:
@@ -432,6 +563,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
actions[self.PENDING_KEY] = self.__drive_pending
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
if not self.__accepting_work():
return
for key, state in results.items():
if state is RecoveryState.COMPLETED:
@@ -445,11 +578,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
f"{state.value}),将在后续健康检查周期重试: {key}")
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
def __enter_isolation(self, watcher: LocalDirectoryWatcher) -> None:
"""
将一个监控目录转入挂载级故障隔离停止对它的一切新访问等待探测恢复
:param watcher: 重建未能返回的监控
"""
if not self.__accepting_work():
return
key = str(watcher.watch_path)
with self._watcher_lock:
if key in self._isolated:
@@ -466,7 +601,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
stage="isolated")
def __probe_isolated(self):
def __probe_isolated(self) -> None:
"""
对隔离中的监控目录做可放弃探测挂载恢复应答后解除隔离并重建监控
@@ -474,11 +609,17 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
超时可被 kill因此本线程不会像内联 stat 那样永久冻死探测通过后的重建
仍有极小概率再次卡住届时本线程会被下一轮的 BUSY 判定跳过不再泄漏
"""
if not self.__accepting_work():
return
with self._watcher_lock:
keys = list(self._isolated)
for key in keys:
if not self.__accepting_work():
return
mon_path = Path(key)
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
if not self.__accepting_work():
return
with self._watcher_lock:
entry = self._isolated.get(key)
if not entry:
@@ -489,6 +630,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
continue
if not self.__accepting_work():
return
with self._watcher_lock:
entry = self._isolated.pop(key, None)
if not entry:
@@ -502,19 +645,24 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
with self._watcher_lock:
self._pending_rebuild[key] = entry["watcher"]
def __drive_pending(self):
def __drive_pending(self) -> None:
"""
驱动两条待重试队列两者都会访问挂载启动重试走目录遍历与 exists
整理重试走 stat必须在恢复工作线程里执行而不是看门狗线程里
"""
if not self.__accepting_work():
return
self.__retry_pending_locals()
self._dispatcher.retry_pending()
if self.__accepting_work():
self._dispatcher.retry_pending()
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher) -> None:
"""
重建一个本地目录监控线程
:param watcher: 需要重建的监控
"""
if not self.__accepting_work():
return
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
watcher.stop()
new_watcher = LocalDirectoryWatcher(
@@ -527,8 +675,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
new_watcher.start()
except Exception as e:
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
if not self.__accepting_work():
return
with self._watcher_lock:
# 旧 watcher 可能仍卡在 FUSE 调用里;只有真实退出后才能移除句柄。
self._watchers = [item for item in self._watchers if item is not watcher]
if watcher.is_alive():
self._retired_watchers.append(watcher)
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
self._pending_locals.append({
"mon_path": watcher.watch_path,
@@ -537,9 +690,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
})
return
with self._watcher_lock:
if not self.__accepting_work():
new_watcher.stop()
if new_watcher.is_alive():
self._retired_watchers.append(new_watcher)
return
registered = any(item is watcher for item in self._watchers)
if registered:
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
if watcher.is_alive():
# stop() 只能发信号,FUSE 上的旧线程可能永久不返回;继续持有它,
# 让生命周期关闭如实失败,而不是把 daemon 泄漏伪装成已收敛。
self._retired_watchers.append(watcher)
if not registered:
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
@@ -547,6 +709,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 派发事件的孤儿线程,必须就地停掉
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
new_watcher.stop()
if new_watcher.is_alive():
with self._watcher_lock:
self._retired_watchers.append(new_watcher)
return
# 新监控的重启计数从零开始,同步重置告警基准
self._restart_marks.pop(str(watcher.watch_path), None)
@@ -557,7 +722,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self.__start_compensation(mon_path=watcher.watch_path,
since=watcher.last_activity_time)
def __start_compensation(self, mon_path: Path, since: float):
def __start_compensation(self, mon_path: Path, since: float) -> None:
"""
在后台线程发起补偿扫描避免遍历目录阻塞健康检查周期
:param mon_path: 监控目录
@@ -567,14 +732,34 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
return
Thread(
target=self.__compensate_scan,
thread = Thread(
target=self.__run_compensation,
kwargs={"mon_path": mon_path, "since": since},
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
daemon=True
).start()
)
with self._owner_lock:
# 与 close() 的 owner 快照共用同一把锁;封口先发生时绝不再启动,
# 登记先发生时 close() 必然能看到并等待这条线程。
if not self.__accepting_work():
return
self._compensation_threads[id(thread)] = thread
try:
thread.start()
except Exception:
self._compensation_threads.pop(id(thread), None)
raise
def __compensate_scan(self, mon_path: Path, since: float):
def __run_compensation(self, mon_path: Path, since: float) -> None:
"""执行补偿扫描,并在真实终态后释放 owner 句柄。"""
current_thread = threading.current_thread()
try:
self.__compensate_scan(mon_path=mon_path, since=since)
finally:
with self._owner_lock:
self._compensation_threads.pop(id(current_thread), None)
def __compensate_scan(self, mon_path: Path, since: float) -> None:
"""
补扫监控停摆期间落地的文件
@@ -586,7 +771,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param mon_path: 监控目录
:param since: 停摆起点墙钟时间戳仅用于统计与日志
"""
if not self.__accepting_work():
return
candidates = self.__collect_compensation_files(mon_path)
if not self.__accepting_work():
return
if candidates is None:
return
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
@@ -599,6 +788,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
handled = 0
for file_path, file_modify_time, file_size in candidates:
if not self.__accepting_work():
return
if self._dispatcher.handle_file(
storage="local",
event_path=file_path,
@@ -634,14 +825,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
return None
return candidates
def __retry_pending_locals(self):
def __retry_pending_locals(self) -> None:
"""
重试启动失败的本地目录监控给网络存储/FUSE 挂载留出就绪时间
"""
if not self.__accepting_work():
return
with self._watcher_lock:
pending = list(self._pending_locals)
isolated = set(self._isolated)
for item in pending:
if not self.__accepting_work():
return
if str(item["mon_path"]) in isolated:
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
# 在「请求永不返回」的挂载上会再冻死一个线程
@@ -655,7 +850,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
item["attempts"] = item.get("attempts", 0) + 1
item["skip_cycles"] = min(item["attempts"], 10)
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault") -> None:
"""
推送目录监控异常告警同一目录在同一阶段仅推送一次
:param mon_path: 监控目录
@@ -664,6 +859,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
监控已暂停访问等待挂载恢复只按目录去重会把这条
关键消息吞掉因此阶段变化时重新推送
"""
if not self.__accepting_work():
return
key = str(mon_path)
with self._watcher_lock:
if self._alerted_paths.get(key) == stage:
@@ -672,7 +869,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
MessageHelper().put(message, title="目录监控")
@staticmethod
def __poller_alert(storage: str, message: str):
def __poller_alert(storage: str, message: str) -> None:
"""
远程轮询监控告警回调复用消息渠道推送
:param storage: 存储名称
@@ -681,12 +878,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"[{storage}] {message}")
MessageHelper().put(message, title="目录监控")
def __clear_alert(self, mon_path: Path, message: str):
def __clear_alert(self, mon_path: Path, message: str) -> None:
"""
清除目录监控异常告警状态并在此前告警过时推送恢复消息
:param mon_path: 监控目录
:param message: 恢复内容
"""
if not self.__accepting_work():
return
key = str(mon_path)
with self._watcher_lock:
if key not in self._alerted_paths:
@@ -695,12 +894,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(message)
MessageHelper().put(message, title="目录监控")
def polling_observer(self, storage: str, mon_paths: List[Path]):
def polling_observer(self, storage: str, mon_paths: List[Path]) -> None:
"""
轮询监控执行一轮快照并按结果动态调整监控间隔
"""
if not self.__accepting_work():
return
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
if file_count is None or not self._scheduler:
if not self.__accepting_work() or file_count is None or not self._scheduler:
return
# 动态调整监控间隔
new_interval = SnapshotStore.adjust_interval(file_count)
@@ -716,7 +917,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
except Exception as e:
logger.error(f"调整监控间隔失败: {storage} - {e}")
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
def event_handler(
self, event: Any, text: str, event_path: str,
file_size: Optional[float] = None
) -> None:
"""
处理文件变化
:param event: 事件
@@ -724,7 +928,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param event_path: 事件文件路径
:param file_size: 文件大小
"""
if event.is_directory:
if not self.__accepting_work() or event.is_directory:
return
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
return
@@ -733,6 +937,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
file_modify_time = Path(event_path).stat().st_mtime
except OSError as err:
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
if not self.__accepting_work():
return
# 整理文件
handle_kwargs = {
"storage": "local",
@@ -743,55 +949,191 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
handle_kwargs["file_modify_time"] = file_modify_time
self._dispatcher.handle_file(**handle_kwargs)
def event_unreadable(self, event_path: Path):
def event_unreadable(self, event_path: Path) -> None:
"""
处理读取失败的监控事件登记待重试
:param event_path: 事件文件路径
"""
if not self.__accepting_work():
return
event_path = Path(event_path)
if not self._dispatcher.is_transfer_candidate_path(event_path):
return
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
def stop(self):
def stop(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""在共享预算内临时停止全部监控 owner,供配置热重载使用。"""
return self.__stop_with_budget(timeout=timeout, close=False)
def close(self, timeout: float = LIFECYCLE_CLOSE_TIMEOUT) -> bool:
"""永久封口当前 lifespan,并在共享预算内等待全部监控 owner。"""
return self.__stop_with_budget(timeout=timeout, close=True)
def __stop_with_budget(self, timeout: float, close: bool) -> bool:
"""从等待生命周期锁开始计算一次停止操作的完整预算。"""
timeout = max(0.0, timeout)
deadline = time.monotonic() + timeout
# 先封住工作入口再等待锁。配置重载若正阻塞在 FUSE 上,解冻后也只能清理,
# 不能在外层生命周期已经超时返回后继续创建 watcher 或派发整理任务。
self._work_stop_event.set()
if close:
self._shutdown_event.set()
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("目录监控未在 %.1f 秒内取得停机所有权", timeout)
return False
try:
return self.__stop_owned(deadline=deadline, close=close)
finally:
self._lifecycle_lock.release()
def __stop_owned(self, deadline: float, close: bool) -> bool:
"""
退出监控
在已取得生命周期锁时等待全部 owner共享调用方给出的绝对截止时间
超时只报告未收敛并保留所有活句柄只有 watcher补偿扫描恢复动作及
scheduler shutdown 都到达终态后才清空这一代的内存状态
"""
# 先停定时服务,避免健康检查在停止过程中重建监控线程
if self._scheduler:
self._scheduler.remove_all_jobs()
if self._scheduler.running:
try:
self._scheduler.shutdown()
logger.info("定时监控服务已停止")
except Exception as e:
logger.error(f"停止定时服务出现了错误:{e}")
self._scheduler = None
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
self._work_stop_event.set()
if close:
self._closed = True
self._shutdown_event.set()
self._recovery.request_stop()
self.__request_scheduler_stop()
with self._watcher_lock:
watchers = tuple(self._watchers) + tuple(self._retired_watchers)
if watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in watchers:
try:
watcher.stop()
except Exception as err:
logger.error(f"请求停止目录监控服务出现了错误:{err}")
recovery_converged = self._recovery.close(deadline=deadline)
current_thread = threading.current_thread()
# 恢复线程可能在封口后才从 FUSE 调用返回,并登记一个已经请求停止的新
# watcherRecoveryExecutor 收敛后重新取快照,确保它也使用剩余预算 join。
with self._watcher_lock:
registered_watchers = tuple(self._watchers) + tuple(self._retired_watchers)
for watcher in registered_watchers:
try:
watcher.stop()
watcher.join(timeout=max(0.0, deadline - time.monotonic()))
except Exception as err:
logger.error(f"等待目录监控服务停止出现了错误:{err}")
with self._owner_lock:
compensation_threads = tuple(self._compensation_threads.values())
scheduler_thread = self._scheduler_shutdown_thread
for thread in compensation_threads:
if thread is not current_thread and thread.is_alive():
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if (
scheduler_thread is not None
and scheduler_thread is not current_thread
and scheduler_thread.is_alive()
):
scheduler_thread.join(timeout=max(0.0, deadline - time.monotonic()))
with self._watcher_lock:
alive_watchers = tuple(
watcher
for watcher in (*self._watchers, *self._retired_watchers)
if watcher.is_alive()
)
with self._owner_lock:
alive_compensations = tuple(
thread for thread in self._compensation_threads.values()
if thread.is_alive()
)
scheduler_thread = self._scheduler_shutdown_thread
scheduler_converged = (
self._scheduler is None
or (
scheduler_thread is not None
and not scheduler_thread.is_alive()
and self._scheduler_shutdown_succeeded
)
)
alive_recoveries = self._recovery.running_threads()
if (
alive_watchers
or alive_compensations
or alive_recoveries
or not recovery_converged
or not scheduler_converged
):
logger.error(
"目录监控 owner 未在截止时间内收敛:watcher=%d,补偿=%d,恢复=%dscheduler=%s",
len(alive_watchers),
len(alive_compensations),
len(alive_recoveries),
"未收敛" if not scheduler_converged else "已收敛",
)
return False
# 仅在所有 owner 真实终止后清理业务状态,确保失败重试仍能找到原句柄。
self._dispatcher.clear_pending()
with self._watcher_lock:
watchers = self._watchers
self._watchers = []
self._retired_watchers = []
self._pending_locals = []
self._alerted_paths = {}
self._restart_marks = {}
self._stable_cycles = {}
self._isolated = {}
self._pending_rebuild = {}
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
# 被残留记录误判为 BUSY 而永远拿不到重建机会
self._recovery.clear()
with self._owner_lock:
self._compensation_threads.clear()
self._scheduler = None
self._scheduler_shutdown_thread = None
self._scheduler_shutdown_succeeded = False
if watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in watchers:
try:
watcher.stop()
watcher.join(timeout=5)
if watcher.is_alive():
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
else:
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
except Exception as e:
logger.error(f"停止目录监控服务出现了错误:{e}")
logger.info("本地目录监控服务已停止")
return True
def __request_scheduler_stop(self) -> None:
"""创建并登记唯一 scheduler shutdown 线程,避免阻塞生命周期调用线程。"""
with self._owner_lock:
running = self._scheduler_shutdown_thread
if running is not None and running.is_alive():
return
if running is not None and self._scheduler_shutdown_succeeded:
return
scheduler = self._scheduler
if scheduler is None:
return
self._scheduler_shutdown_succeeded = False
thread = threading.Thread(
target=self.__shutdown_scheduler,
args=(scheduler,),
name="MoviePilot-MonitorSchedulerShutdown",
daemon=True,
)
self._scheduler_shutdown_thread = thread
try:
thread.start()
except Exception as err:
self._scheduler_shutdown_thread = None
logger.error(f"启动定时监控停止线程失败:{err}")
def __shutdown_scheduler(self, scheduler: BackgroundScheduler) -> None:
"""停止 scheduler 及其在途 job,并把真实终态写回 owner 注册表。"""
succeeded = False
try:
scheduler.remove_all_jobs()
if scheduler.running:
scheduler.shutdown(wait=True)
succeeded = True
logger.info("定时监控服务已停止")
except Exception as err:
logger.error(f"停止定时服务出现了错误:{err}")
finally:
with self._owner_lock:
if self._scheduler is scheduler:
self._scheduler_shutdown_succeeded = succeeded
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
+70 -15
View File
@@ -1,5 +1,5 @@
"""
监控恢复动作的可放弃执行单元
监控恢复动作的有限等待执行单元
FUSE/网络挂载有两种故障形态下游程序的免疫力完全不同
@@ -11,8 +11,8 @@ FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
本模块提供 block 型故障下唯一可行的两种自保手段
1. RecoveryExecutor 把会触碰挂载的动作放进一次性守护线程执行调用方只
等待有限时间超时即放弃该线程它会作为守护线程悬挂到进程退出换取
调用方健康检查这个全局自愈单点永远活着
等待有限时间超时后当前健康检查不再等待但线程句柄仍由生命周期 owner
持有到真实终态换取看门狗可继续检测且停机屏障不会伪装收敛
2. probe_path 用子进程而非线程做挂载探测子进程可以被 kill因此探测
本身是可放弃的隔离期间可以无限次周期重试而不累积不可回收的资源
"""
@@ -41,7 +41,7 @@ class RecoveryState(str, Enum):
"""
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
COMPLETED = "completed"
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
# 超时仍未返回,判定为 block 型挂载故障,本轮不再等待但继续持有线程
TIMEOUT = "timeout"
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
BUSY = "busy"
@@ -55,10 +55,12 @@ class RecoveryExecutor:
健康检查周期都会在同一个死挂载上多泄漏一个线程
"""
def __init__(self):
def __init__(self) -> None:
"""初始化在途线程注册表和当前生命周期的提交状态。"""
# key -> 该 key 最近一次提交的执行线程
self._running: Dict[str, threading.Thread] = {}
self._lock = threading.Lock()
self._accepting = True
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
"""
@@ -85,26 +87,70 @@ class RecoveryExecutor:
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if thread.is_alive():
results[key] = RecoveryState.TIMEOUT
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应,本轮不再等待: {key}")
else:
results[key] = RecoveryState.COMPLETED
return results
def discard(self, key: str):
def discard(self, key: str) -> None:
"""
丢弃一个 key 的在途记录监控停止或配置重载时调用避免残留条目
让重建后的同名目录被误判为 BUSY
清理一个已到终态的 key活线程继续保留供生命周期停机屏障追踪
:param key: 动作标识
"""
with self._lock:
self._running.pop(key, None)
thread = self._running.get(key)
if thread is None or not thread.is_alive():
self._running.pop(key, None)
def clear(self):
def clear(self) -> None:
"""
空全部在途记录已经冻死的线程无法回收这里只是不再跟踪它们
理已经完成的记录仍存活的线程继续由执行器持有
兼容旧调用名但不再丢弃挂死线程遗失句柄会让宿主错误释放它仍可能使用
的数据库和整理链资源
"""
with self._lock:
self._running = {
key: thread
for key, thread in self._running.items()
if thread.is_alive()
}
def request_stop(self) -> None:
"""封住新恢复动作提交,既有线程只能自然完成。"""
with self._lock:
self._accepting = False
def close(self, deadline: float) -> bool:
"""在绝对截止时间内等待全部恢复线程,超时继续保留活线程句柄。"""
self.request_stop()
with self._lock:
threads = tuple(self._running.values())
current_thread = threading.current_thread()
for thread in threads:
if thread is current_thread or not thread.is_alive():
continue
thread.join(timeout=max(0.0, deadline - time.monotonic()))
self.clear()
with self._lock:
return not any(thread.is_alive() for thread in self._running.values())
def reopen(self) -> bool:
"""在旧恢复线程全部结束后,为新的 Monitor 生命周期恢复提交。"""
self.clear()
with self._lock:
if any(thread.is_alive() for thread in self._running.values()):
return False
self._running.clear()
self._accepting = True
return True
def running_threads(self) -> tuple[threading.Thread, ...]:
"""返回仍存活且由执行器持有的恢复线程快照。"""
with self._lock:
return tuple(
thread for thread in self._running.values() if thread.is_alive()
)
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
"""
@@ -114,6 +160,8 @@ class RecoveryExecutor:
:return: 执行线程未启动时为 None
"""
with self._lock:
if not self._accepting:
return None
running = self._running.get(key)
if running is not None and running.is_alive():
return None
@@ -124,11 +172,18 @@ class RecoveryExecutor:
daemon=True
)
self._running[key] = thread
thread.start()
return thread
# 登记与 start 必须处于同一所有权临界区。否则 close() 可能把尚未
# is_alive() 的句柄当成已结束并移除,随后该线程才真正启动。
try:
thread.start()
except BaseException:
if self._running.get(key) is thread:
self._running.pop(key, None)
raise
return thread
@staticmethod
def _execute(key: str, action: Callable[[], None]):
def _execute(key: str, action: Callable[[], None]) -> None:
"""
执行一个恢复动作异常只记录不外抛避免一个目录的失败连累整批恢复
:param key: 动作标识
+10
View File
@@ -30,6 +30,9 @@ class EventDispatcher:
event_factory: Callable[..., Any],
error_handler: Callable[..., None],
async_handle_sink: Callable[[Any], bool] | None = None,
sync_handle_sink: (
Callable[[Callable[..., Any], tuple[Any, ...]], bool] | None
) = None,
) -> None:
"""注入注册表、绑定器、执行器和错误策略回调。"""
self._registry = registry
@@ -39,6 +42,7 @@ class EventDispatcher:
self._event_factory = event_factory
self._error_handler = error_handler
self._async_handle_sink = async_handle_sink
self._sync_handle_sink = sync_handle_sink
def dispatch_chain(self, event: Any) -> bool:
"""同步按优先级顺序执行链式事件快照。"""
@@ -134,6 +138,12 @@ class EventDispatcher:
event.event_type,
)
else:
if self._sync_handle_sink:
self._sync_handle_sink(
self.safe_invoke_sync,
(handler, isolated),
)
continue
self._executor().submit(
self.safe_invoke_sync,
handler,
+180 -26
View File
@@ -4,6 +4,7 @@ import random
import threading
import traceback
import uuid
from contextvars import ContextVar
from dataclasses import dataclass
from queue import Empty, PriorityQueue
from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type
@@ -32,6 +33,10 @@ MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间(秒)
MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间(秒)
_EVENT_STOP_SENTINEL = object()
_CURRENT_EVENT_HANDLER_OWNER: ContextVar[object | None] = ContextVar(
"current_event_handler_owner",
default=None,
)
@dataclass(slots=True)
@@ -121,10 +126,11 @@ class EventManager(metaclass=Singleton):
self.__lock = threading.Lock()
# 退出事件
self.__event = threading.Event()
# 广播异步处理器的生命周期由事件总线自己持有,避免关闭后仍向主循环运行
# 广播处理器由事件总线统一持有,确保插件卸载前可以建立结算屏障
self.__lifecycle_lock = threading.RLock()
self.__lifecycle_state = "new"
self.__async_handles: Dict[int, _EventAsyncHandle] = {}
self.__sync_handles: Dict[object, concurrent.futures.Future[Any]] = {}
self.__async_handles: Dict[object, _EventAsyncHandle] = {}
# 由上层管理器注册的处理器实例解析器
self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {}
# 由启动组合层注入的错误通知回调
@@ -155,6 +161,7 @@ class EventManager(metaclass=Singleton):
event_factory=Event,
error_handler=lambda **kwargs: self.__handle_event_error(**kwargs),
async_handle_sink=self.__register_async_handle,
sync_handle_sink=self.__register_sync_handle,
)
def register_handler_instance_resolver(
@@ -188,6 +195,9 @@ class EventManager(metaclass=Singleton):
if self.__lifecycle_state == "stopping":
logger.warning("事件处理仍在停止,忽略重复启动")
return
if self.__lifecycle_state == "sealed":
logger.warning("事件处理已封口,忽略重复启动")
return
self.__lifecycle_state = "running"
self.__event.set()
self.__consumer_threads = []
@@ -199,14 +209,19 @@ class EventManager(metaclass=Singleton):
def stop(self):
"""
停止广播事件处理线程
兼容同步关闭入口等待同步处理器并请求取消异步处理器
调用线程无法安全等待主事件循环完成异步清理需要完整异步收口时应使用
stop_async()
"""
logger.info("正在停止事件处理...")
consumer_threads = self.__begin_stop()
try:
self.__join_consumer_threads(consumer_threads)
self.__discard_stop_sentinels()
self.__cancel_async_handles()
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
self.__cancel_async_handles(exclude_owner=current_owner)
self.__wait_sync_handles(exclude_owner=current_owner)
logger.info("事件处理停止完成")
except Exception as e:
logger.error(f"停止事件处理线程出错:{str(e)} - {traceback.format_exc()}")
@@ -216,7 +231,7 @@ class EventManager(metaclass=Singleton):
self.__consumer_threads = []
async def stop_async(self) -> None:
"""停止广播消费者等待已投递的异步处理器收口"""
"""停止广播消费者等待同步处理器并取消收口异步处理器"""
logger.info("正在停止事件处理...")
consumer_threads = self.__begin_stop()
try:
@@ -226,15 +241,29 @@ class EventManager(metaclass=Singleton):
consumer_threads,
)
self.__discard_stop_sentinels()
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
for handle in handles:
async_handles = tuple(
handle
for owner, handle in self.__async_handles.items()
if owner is not current_owner
)
sync_handles = tuple(
handle
for owner, handle in self.__sync_handles.items()
if owner is not current_owner
)
for handle in async_handles:
handle.handle.cancel()
if handles:
if async_handles or sync_handles:
await asyncio.gather(
*(
asyncio.shield(asyncio.wrap_future(handle.completion))
for handle in handles
for handle in async_handles
),
*(
asyncio.shield(asyncio.wrap_future(handle))
for handle in sync_handles
),
return_exceptions=True,
)
@@ -272,16 +301,81 @@ class EventManager(metaclass=Singleton):
break
if item[1] is not _EVENT_STOP_SENTINEL:
pending.append(item)
self.__event_queue.task_done()
for item in pending:
self.__event_queue.put(item)
def __cancel_async_handles(self) -> None:
"""请求取消所有仍由事件总线持有的异步处理器。"""
def __cancel_async_handles(
self,
*,
exclude_owner: object | None = None,
) -> None:
"""请求取消异步处理器,并避免 handler 关闭事件总线时取消自身。"""
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
handles = tuple(
handle
for owner, handle in self.__async_handles.items()
if owner is not exclude_owner
)
for handle in handles:
handle.handle.cancel()
def __wait_sync_handles(
self,
*,
exclude_owner: object | None = None,
) -> None:
"""等待同步处理器完成,并避免 handler 关闭事件总线时等待自身。"""
with self.__lifecycle_lock:
handles = tuple(
handle
for owner, handle in self.__sync_handles.items()
if owner is not exclude_owner
)
if handles:
concurrent.futures.wait(handles)
def __register_sync_handle(
self,
callback: Callable[..., Any],
args: tuple[Any, ...],
) -> bool:
"""在同一生命周期临界区提交并登记同步广播处理器。"""
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
logger.warning(
"事件处理处于 %s 状态,拒绝同步广播处理器",
self.__lifecycle_state,
)
return False
try:
owner = object()
def _tracked_sync() -> Any:
"""在同步 handler 调用栈中发布当前事件 owner。"""
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
try:
return callback(*args)
finally:
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
handle = self.__executor.submit(_tracked_sync)
except RuntimeError:
logger.warning("同步事件处理器无法投递,线程池已停止")
return False
self.__sync_handles[owner] = handle
handle.add_done_callback(
lambda _completed, current_owner=owner: (
self.__remove_sync_handle(current_owner)
)
)
return True
def __remove_sync_handle(self, owner: object) -> None:
"""同步处理器完成后移除其 owner 句柄。"""
with self.__lifecycle_lock:
self.__sync_handles.pop(owner, None)
def __register_async_handle(
self,
coroutine: Any,
@@ -290,12 +384,18 @@ class EventManager(metaclass=Singleton):
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
coroutine.close()
logger.warning(
"事件处理处于 %s 状态,拒绝异步广播处理器",
self.__lifecycle_state,
)
return False
completion: concurrent.futures.Future[Any] = concurrent.futures.Future()
started = threading.Event()
owner = object()
async def _tracked() -> None:
started.set()
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
try:
result = await coroutine
except asyncio.CancelledError:
@@ -307,6 +407,8 @@ class EventManager(metaclass=Singleton):
else:
if not completion.done():
completion.set_result(result)
finally:
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
tracked = _tracked()
try:
@@ -316,7 +418,7 @@ class EventManager(metaclass=Singleton):
coroutine.close()
logger.warning("异步事件处理器无法投递,事件循环已停止")
return False
self.__async_handles[id(completion)] = _EventAsyncHandle(
self.__async_handles[owner] = _EventAsyncHandle(
handle=handle,
completion=completion,
)
@@ -329,13 +431,54 @@ class EventManager(metaclass=Singleton):
completion.cancel()
handle.add_done_callback(_complete_unstarted_submission)
completion.add_done_callback(self.__remove_async_handle)
completion.add_done_callback(
lambda _completed, current_owner=owner: (
self.__remove_async_handle(current_owner)
)
)
return True
def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None:
def __remove_async_handle(self, owner: object) -> None:
"""异步处理器完成后移除其 owner 句柄。"""
with self.__lifecycle_lock:
self.__async_handles.pop(id(handle), None)
self.__async_handles.pop(owner, None)
async def drain_async(
self,
timeout: Optional[float] = None,
*,
seal: bool = False,
) -> bool:
"""
等待已接纳的广播事件及其派生事件自然结算
seal=True 会在确认稳定空闲的同一临界区关闭后续广播提交供插件卸载前
建立不可穿透的投递屏障超时或停止交错时保持原提交状态并返回 False
广播处理器调用栈内无法等待自身完成因此会立即返回 False且不执行封口
"""
if _CURRENT_EVENT_HANDLER_OWNER.get() is not None:
logger.warning("事件处理器内部不能建立事件投递屏障")
return False
loop = asyncio.get_running_loop()
deadline = None if timeout is None else loop.time() + max(timeout, 0)
while True:
with self.__lifecycle_lock:
if self.__lifecycle_state not in {"running", "sealed"}:
return False
with self.__event_queue.all_tasks_done:
queue_idle = self.__event_queue.unfinished_tasks == 0
handlers_idle = not self.__sync_handles and not self.__async_handles
if queue_idle and handlers_idle:
if seal:
self.__lifecycle_state = "sealed"
return True
if deadline is not None:
remaining = deadline - loop.time()
if remaining <= 0:
return False
await asyncio.sleep(min(0.01, remaining))
else:
await asyncio.sleep(0.01)
def check(self, etype: Union[EventType, ChainEventType]) -> bool:
"""
@@ -470,7 +613,15 @@ class EventManager(metaclass=Singleton):
:param event: 要处理的事件对象
"""
logger.debug(f"Triggering broadcast event: {event}")
self.__event_queue.put((event.priority, event))
with self.__lifecycle_lock:
if self.__lifecycle_state in {"sealed", "stopping", "stopped"}:
logger.warning(
"事件处理处于 %s 状态,拒绝广播事件 %s",
self.__lifecycle_state,
event.event_type,
)
return None
self.__event_queue.put((event.priority, event))
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
@@ -580,15 +731,18 @@ class EventManager(metaclass=Singleton):
while self.__event.is_set():
try:
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
if event is _EVENT_STOP_SENTINEL:
break
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
rate_limiter.reset()
self.__dispatch_broadcast_event(event)
try:
if event is _EVENT_STOP_SENTINEL:
break
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
rate_limiter.reset()
self.__dispatch_broadcast_event(event)
finally:
self.__event_queue.task_done()
except Empty:
rate_limiter.current_wait = rate_limiter.current_wait * random.uniform(1, 1 + jitter_factor)
rate_limiter.trigger_limit()
+112
View File
@@ -0,0 +1,112 @@
"""插件宿主可变事务的停机准入。"""
from __future__ import annotations
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from app.schemas.exception import PluginMutationRejectedError
@dataclass(slots=True)
class _MutationContext:
"""记录一个可跨协程和受控线程传播的事务上下文。"""
admission: "PluginMutationAdmission"
holders: int = 0
open: bool = True
class PluginMutationAdmission:
"""在停机封口与插件可变事务之间维护真实 owner 计数。"""
def __init__(self) -> None:
"""初始化开放准入、活动 owner 计数和事务上下文。"""
self._condition = threading.Condition()
self._accepting = True
self._active_count = 0
self._current_context: ContextVar[_MutationContext | None] = ContextVar(
"plugin_mutation_context",
default=None,
)
@property
def active_count(self) -> int:
"""返回尚未退出的 lease 数量,用于停机诊断和严格卸载判断。"""
with self._condition:
return self._active_count
@property
def accepting(self) -> bool:
"""返回当前生命周期是否仍接纳新的根事务。"""
with self._condition:
return self._accepting
def is_held(self) -> bool:
"""判断当前执行上下文是否持有仍有效的本 admission lease。"""
context = self._current_context.get()
return bool(
context
and context.admission is self
and context.open
and context.holders > 0
)
@contextmanager
def hold(self, operation: str) -> Iterator[None]:
"""取得可变事务 lease;封口后仅允许已获准事务的嵌套调用。"""
context = self._current_context.get()
nested = bool(
context
and context.admission is self
and context.open
and context.holders > 0
)
context_token = None
if not nested:
context = _MutationContext(admission=self)
context_token = self._current_context.set(context)
assert context is not None
acquired = False
try:
with self._condition:
if not self._accepting and not nested:
raise PluginMutationRejectedError(operation)
self._active_count += 1
context.holders += 1
acquired = True
yield
finally:
if acquired:
with self._condition:
self._active_count -= 1
context.holders -= 1
if context.holders == 0:
context.open = False
self._condition.notify_all()
if context_token is not None:
self._current_context.reset(context_token)
def seal(self) -> int:
"""原子停止接纳根事务,并返回封口瞬间的活动 lease 数量。"""
with self._condition:
self._accepting = False
return self._active_count
def wait_until_idle(self) -> None:
"""自然等待全部已获准 lease 退出,不取消或遗失其 owner。"""
with self._condition:
while self._active_count:
self._condition.wait()
def reopen(self) -> bool:
"""仅在没有遗留 lease 时为新的应用生命周期重新开放准入。"""
with self._condition:
if self._active_count:
return False
self._accepting = True
return True
+161 -29
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import traceback
from collections.abc import Callable
from functools import wraps
import threading
import time
from typing import Any, Optional, ParamSpec, TypeVar, cast
@@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
try:
result = func(*args, **kwargs)
statuses = result.values() if isinstance(result, dict) else (result,)
if PluginRuntimeStatus.LOAD_FAILED in statuses:
if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses:
outcome = "error"
return result
except BaseException:
@@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
class PluginLifecycle:
"""管理插件发现、初始化、启停和热重载,不持有市场或 HTTP 路由职责。"""
_EVENT_HANDLERS_QUIESCED = "__event_handlers__"
def __init__(
self,
*,
@@ -83,6 +86,8 @@ class PluginLifecycle:
self._runtime_status_writer = runtime_status_writer
self._logger = log
self._event_sender = event_sender
self._lifecycle_lock = threading.RLock()
self._quiesced_hooks: dict[str, set[str]] = {}
@observe_plugin_lifecycle("start")
def start(
@@ -116,6 +121,7 @@ class PluginLifecycle:
self._classes[current_id] = plugin
instance = plugin()
instance.init_plugin(self._plugin_config(current_id))
self._quiesced_hooks.pop(current_id, None)
self._running[current_id] = instance
self._logger.info(
f"加载插件:{current_id} 版本:{instance.plugin_version}"
@@ -156,31 +162,168 @@ class PluginLifecycle:
@observe_plugin_lifecycle("stop")
def stop(self, plugin_id: Optional[str] = None) -> None:
"""停止指定插件或全部插件,并清理模块缓存"""
"""按旧单阶段 ABI 先解绑 handler,再停止并强制卸载插件"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
self._quiesce_selected(plugins)
self._finalize(
plugin_id,
require_quiesced=False,
disable_events=not self._handlers_quiesced(plugins),
)
@observe_plugin_lifecycle("quiesce")
def quiesce(self, plugin_id: Optional[str] = None) -> bool:
"""先解绑事件 handler,再按旧 hook 顺序停止生产者并保留实例。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
return self._quiesce_selected(plugins)
@observe_plugin_lifecycle("quiesce_handlers")
def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool:
"""禁止目标插件接收新事件,保留实例供在途 handler 和后续 hook 使用。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
return self._disable_selected_handlers(plugins)
@observe_plugin_lifecycle("quiesce_services")
def quiesce_services(self, plugin_id: Optional[str] = None) -> bool:
"""在事件结算屏障后执行旧 close、stop_service hook。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
if not self._handlers_quiesced(plugins):
self._logger.warning("插件事件 handler 尚未全部停用,拒绝关闭插件资源")
return False
return self._quiesce_hooks(plugins)
def _quiesce_selected(self, plugins: dict[str, Any]) -> bool:
"""兼容单阶段调用:先停用全部 handler,再执行稳定快照的旧 hooks。"""
if not self._disable_selected_handlers(plugins):
return False
return self._quiesce_hooks(plugins)
def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool:
"""先停用稳定快照的全部事件入口,任一失败时不执行破坏性 hook。"""
all_converged = True
for current_id, plugin in plugins.items():
completed = self._quiesced_hooks.setdefault(current_id, set())
if self._EVENT_HANDLERS_QUIESCED in completed:
continue
try:
self._disable_events(type(plugin))
except Exception as error: # noqa: BLE001 插件边界必须隔离
all_converged = False
self._logger.warning(
f"停用插件 {current_id} 的事件 handler 时发生错误: {error}"
)
continue
completed.add(self._EVENT_HANDLERS_QUIESCED)
return all_converged
def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool:
"""执行旧 ABI hooks,并只重试尚未成功的步骤。"""
all_converged = True
for current_id, plugin in plugins.items():
completed = self._quiesced_hooks.setdefault(current_id, set())
for hook_name in ("close", "stop_service"):
if hook_name in completed:
continue
hook = getattr(plugin, hook_name, None)
if not callable(hook):
completed.add(hook_name)
continue
try:
result = hook()
except Exception as error: # noqa: BLE001 插件边界必须隔离
all_converged = False
self._logger.warning(
f"停止插件 {current_id}{hook_name} 时发生错误: {error}"
)
continue
if result is False:
all_converged = False
self._logger.warning(
f"停止插件 {current_id}{hook_name} 未收敛"
)
continue
completed.add(hook_name)
return all_converged
@observe_plugin_lifecycle("finalize")
def finalize(self, plugin_id: Optional[str] = None) -> bool:
"""在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。"""
return self._finalize(
plugin_id,
require_quiesced=True,
disable_events=False,
)
def _finalize(
self,
plugin_id: Optional[str],
*,
require_quiesced: bool,
disable_events: bool = True,
) -> bool:
"""按严格或兼容策略卸载插件,并在清理失败时保留实例所有权。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
if require_quiesced and any(
not self._is_quiesced(current_id, plugin)
for current_id, plugin in plugins.items()
):
self._logger.warning("插件后台服务尚未全部收敛,拒绝卸载运行实例")
return False
try:
if disable_events:
for plugin in plugins.values():
self._disable_events(type(plugin))
self._clear_modules(plugin_id)
self._clear_tools()
except Exception as error: # noqa: BLE001 保留实例所有权供后续重试
self._logger.warning(f"卸载插件运行实例时发生错误: {error}")
return False
if plugin_id:
self._classes.pop(plugin_id, None)
self._running.pop(plugin_id, None)
self._quiesced_hooks.pop(plugin_id, None)
else:
self._classes.clear()
self._running.clear()
self._quiesced_hooks.clear()
self._logger.info("插件停止完成")
return True
def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]:
"""返回本阶段处理的稳定实例快照,并保持旧停机日志语义。"""
if plugin_id:
self._logger.info(f"正在停止插件 {plugin_id}...")
plugin = self._running.get(plugin_id)
plugins = {plugin_id: plugin} if plugin else {}
if not plugin:
self._logger.debug(f"插件 {plugin_id} 不存在或未加载")
else:
self._logger.info("正在停止所有插件...")
plugins = dict(self._running)
return plugins
self._logger.info("正在停止所有插件...")
return dict(self._running)
for current_id, plugin in plugins.items():
self._disable_events(type(plugin))
self._stop_plugin(plugin)
def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool:
"""判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。"""
required = {self._EVENT_HANDLERS_QUIESCED} | {
hook_name
for hook_name in ("close", "stop_service")
if callable(getattr(plugin, hook_name, None))
}
return required.issubset(self._quiesced_hooks.get(plugin_id, set()))
if plugin_id:
self._classes.pop(plugin_id, None)
self._running.pop(plugin_id, None)
self._clear_modules(plugin_id)
else:
self._classes.clear()
self._running.clear()
self._clear_modules(None)
self._clear_tools()
self._logger.info("插件停止完成")
def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool:
"""判断稳定快照中的全部插件是否已经停用事件入口。"""
return all(
self._EVENT_HANDLERS_QUIESCED
in self._quiesced_hooks.get(plugin_id, set())
for plugin_id in plugins
)
@observe_plugin_lifecycle("reload")
def reload(
@@ -194,14 +337,3 @@ class PluginLifecycle:
status = self.start(plugin_id)[plugin_id]
self._event_sender(reload_event, data={"plugin_id": plugin_id})
return status
def _stop_plugin(self, plugin: Any) -> None:
"""按插件旧 ABI 顺序关闭资源和服务。"""
try:
if hasattr(plugin, "close"):
plugin.close()
if hasattr(plugin, "stop_service"):
plugin.stop_service()
except Exception as error: # noqa: BLE001
name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__
self._logger.warning(f"停止插件 {name} 时发生错误: {error}")
+64 -21
View File
@@ -2,8 +2,8 @@
from __future__ import annotations
import time
import threading
import time
from collections.abc import Callable
from pathlib import Path
from typing import Any, Optional
@@ -27,6 +27,8 @@ class PluginMonitorController:
self._logger = log
self._thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
self._lifecycle_lock = threading.RLock()
self._closed = False
@property
def stop_event(self) -> threading.Event:
@@ -35,32 +37,73 @@ class PluginMonitorController:
def reload(self, enabled: bool) -> None:
"""按当前配置停止旧线程,并在启用时创建新线程。"""
self.stop()
if enabled:
stopped = self.stop()
if enabled and stopped:
self.start()
def start(self) -> None:
"""启动唯一的守护监控线程。"""
if self._thread and self._thread.is_alive():
self._logger.info("插件文件修改监测已经在运行中...")
return
self._logger.info("开始监测插件文件修改...")
self._stop_event.clear()
self._thread = threading.Thread(target=self._runner, daemon=True)
self._thread.start()
with self._lifecycle_lock:
if self._closed:
self._logger.info("插件文件修改监测已进入停机封口,跳过启动")
return
if self._thread and self._thread.is_alive():
self._logger.info("插件文件修改监测已经在运行中...")
return
self._logger.info("开始监测插件文件修改...")
self._stop_event.clear()
self._thread = threading.Thread(target=self._runner, daemon=True)
self._thread.start()
def stop(self) -> None:
"""请求监控线程退出,并在限定时间内等待其清理"""
if not self._thread or not self._thread.is_alive():
self._logger.info("未启用插件文件修改监测,无需停止")
return
self._logger.info("正在停止插件文件修改监测...")
def reopen(self) -> bool:
"""为新的应用生命周期解除封口,仍有旧线程时拒绝重开"""
with self._lifecycle_lock:
if self._thread and self._thread.is_alive():
self._logger.warning("旧插件文件监测线程仍在运行,无法开启新生命周期")
return False
self._thread = None
self._closed = False
return True
def stop(self, timeout: float = 5.0) -> bool:
"""临时停止监控线程,并返回其是否在预算内真正退出。"""
return self._stop_with_budget(timeout=timeout, close=False)
def close(self, timeout: float = 5.0) -> bool:
"""永久封口当前生命周期,并返回监控线程是否真正退出。"""
return self._stop_with_budget(timeout=timeout, close=True)
def _stop_with_budget(self, *, timeout: float, close: bool) -> bool:
"""在同一预算内取得生命周期锁、设置封口并等待线程退出。"""
timeout = max(0.0, timeout)
deadline = time.monotonic() + timeout
self._stop_event.set()
self._thread.join(timeout=5)
if self._thread.is_alive():
self._logger.warning("插件文件修改监测线程在5秒内未能正常停止")
self._thread = None
self._logger.info("插件文件修改监测停止完成")
if not self._lifecycle_lock.acquire(timeout=timeout):
self._logger.warning(
f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权"
)
return False
try:
if close:
self._closed = True
thread = self._thread
self._stop_event.set()
if not thread or not thread.is_alive():
self._thread = None
self._logger.info("未启用插件文件修改监测,无需停止")
return True
self._logger.info("正在停止插件文件修改监测...")
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if thread.is_alive():
self._logger.warning(
f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。"
)
return False
self._thread = None
self._logger.info("插件文件修改监测停止完成")
return True
finally:
self._lifecycle_lock.release()
class PluginChangeMonitor:
+293 -67
View File
@@ -1,9 +1,21 @@
import asyncio
import concurrent.futures
import inspect
import posixpath
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
from typing import (
Any,
Callable,
ContextManager,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
from watchfiles import watch
@@ -17,6 +29,7 @@ from app.runtime.execution import run_in_threadpool_to_completion
from app.runtime.log import logger
from app.runtime.observability import observe_compat_facade
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.thread import ThreadHelper
settings = RuntimeSettingsCompat()
from app.runtime.events import EventHandlerBinding, eventmanager
@@ -39,6 +52,7 @@ from app.runtime.extensions.plugin.sync import (
)
from app.runtime.extensions.plugin.clone import PluginCloneService
from app.runtime.extensions.plugin.access import PluginAccessPolicy
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
from app.runtime.extensions.plugin.catalog import PluginCatalogFacade
from app.runtime.extensions.plugin.paths import PluginPathResolver
from app.runtime.extensions.plugin.dependency import (
@@ -47,6 +61,7 @@ from app.runtime.extensions.plugin.dependency import (
PluginDependencyService,
)
from app.runtime.extensions.plugin.storage import PluginConfigStore, PluginInstanceStore
from app.schemas.exception import PluginMutationRejectedError
from app.schemas.types import EventType, SystemConfigKey
LegacyDiagnosticsConfigurator = Callable[..., None]
@@ -195,6 +210,15 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
runner=self._run_file_watcher,
log=logger,
)
self._plugin_quiesce_lock = threading.RLock()
self._plugin_quiesce_future: Optional[
concurrent.futures.Future[bool]
] = None
self._plugin_service_quiesce_future: Optional[
concurrent.futures.Future[bool]
] = None
self._plugin_mutation_admission = PluginMutationAdmission()
self._plugin_runtime_closed = False
self._plugin_dependencies = PluginDependencyService(
system=get_plugin_system,
log=logger,
@@ -315,12 +339,16 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
def init_config(self):
"""按最新系统配置完整重启插件。"""
# 停止已有插件
self.stop()
classification = self.classify_plugins()
self.apply_plugin_dependency_classification(classification)
for plugin_id in classification.ready:
self.start(plugin_id)
try:
with self.mutation("配置热重载"):
# 停止已有插件
self.stop()
classification = self.classify_plugins()
self.apply_plugin_dependency_classification(classification)
for plugin_id in classification.ready:
self.start(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def start(self, pid: Optional[str] = None) -> Dict[str, PluginRuntimeStatus]:
"""
@@ -328,8 +356,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID为空加载所有插件
"""
_legacy_diagnostics_configurator(enabled=settings.DEBUG, emitter=logger.warning)
return self._plugin_lifecycle.start(pid)
try:
with self.mutation("启动插件"):
with self._plugin_quiesce_lock:
_legacy_diagnostics_configurator(
enabled=settings.DEBUG,
emitter=logger.warning,
)
return self._plugin_lifecycle.start(pid)
except PluginMutationRejectedError as error:
logger.warning(str(error))
if pid:
return {pid: PluginRuntimeStatus.LOAD_FAILED}
return {}
def init_plugin(self, plugin_id: str, conf: dict):
"""
@@ -337,7 +376,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param plugin_id: 插件ID
:param conf: 插件配置
"""
self._plugin_lifecycle.initialize(plugin_id, conf)
try:
with self.mutation("初始化插件配置"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.initialize(plugin_id, conf)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def clear_plugin_agent_tools_cache(self) -> None:
"""
@@ -356,12 +400,127 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""兼容读取旧私有字段,实际版本由独立工具目录持有。"""
return self._plugin_tool_catalog.revision
def stop(self, pid: Optional[str] = None):
def stop(self, pid: Optional[str] = None) -> None:
"""
停止插件服务
:param pid: 插件ID为空停止所有插件
"""
self._plugin_lifecycle.stop(pid)
try:
with self.mutation("停止插件"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.stop(pid)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def mutation(self, operation: str) -> ContextManager[None]:
"""为一个完整插件可变事务取得可跨异步边界传播的准入 lease。"""
return self._plugin_mutation_admission.hold(operation)
def reopen_plugins(self) -> bool:
"""为新应用生命周期解除运行时封口,仍活跃的 quiesce owner 禁止复用。"""
with self._plugin_quiesce_lock:
futures = (
self._plugin_quiesce_future,
self._plugin_service_quiesce_future,
)
if any(future is not None and not future.done() for future in futures):
logger.warning("插件后台服务仍在停止,无法开启新的应用生命周期")
return False
if self._plugin_runtime_closed and self._running_plugins:
logger.warning("上一应用生命周期仍持有插件实例,拒绝解除运行时封口")
return False
if not self._plugin_mutation_admission.reopen():
logger.warning("上一应用生命周期仍有插件可变事务,拒绝解除运行时封口")
return False
self._plugin_runtime_closed = False
return True
async def quiesce_plugins(self, timeout: float = 240.0) -> bool:
"""封口变更事务并停用插件 handler,超时后保留 Future ownership。"""
if self._plugin_mutation_admission.is_held():
logger.warning("插件可变事务不能等待自身收敛,拒绝在事务内执行停机")
return False
with self._plugin_quiesce_lock:
self._plugin_runtime_closed = True
self._plugin_mutation_admission.seal()
future = self._plugin_quiesce_future
if future is None or future.done():
future = ThreadHelper().submit(self._quiesce_after_mutations)
self._plugin_quiesce_future = future
try:
result = await asyncio.wait_for(
asyncio.shield(asyncio.wrap_future(future)),
timeout=max(0.0, timeout),
)
return bool(result)
except asyncio.TimeoutError:
logger.error(f"插件后台服务未在 {timeout:g} 秒内收敛")
return False
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
logger.error(f"插件后台服务停止失败:{error}", exc_info=True)
return False
finally:
if future.done():
with self._plugin_quiesce_lock:
if self._plugin_quiesce_future is future:
self._plugin_quiesce_future = None
async def quiesce_plugin_services(self, timeout: float = 240.0) -> bool:
"""在事件结算后执行旧插件停机 hook,并有界等待同步 owner。"""
with self._plugin_quiesce_lock:
prepare_future = self._plugin_quiesce_future
if prepare_future is not None and not prepare_future.done():
logger.warning("插件事件入口仍在封口,拒绝提前关闭插件资源")
return False
if not self._plugin_runtime_closed:
logger.warning("插件运行时尚未封口,拒绝关闭插件资源")
return False
if self._plugin_mutation_admission.active_count:
logger.warning("插件可变事务仍在执行,拒绝关闭插件资源")
return False
future = self._plugin_service_quiesce_future
if future is None or future.done():
future = ThreadHelper().submit(
self._plugin_lifecycle.quiesce_services,
)
self._plugin_service_quiesce_future = future
try:
result = await asyncio.wait_for(
asyncio.shield(asyncio.wrap_future(future)),
timeout=max(0.0, timeout),
)
return bool(result)
except asyncio.TimeoutError:
logger.error(f"插件旧停机 hook 未在 {timeout:g} 秒内收敛")
return False
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
logger.error(f"插件旧停机 hook 执行失败:{error}", exc_info=True)
return False
finally:
if future.done():
with self._plugin_quiesce_lock:
if self._plugin_service_quiesce_future is future:
self._plugin_service_quiesce_future = None
def finalize_plugins(self) -> bool:
"""确认 quiesce owner 已结束后禁用 handler 并卸载插件实例。"""
with self._plugin_quiesce_lock:
futures = (
self._plugin_quiesce_future,
self._plugin_service_quiesce_future,
)
if any(future is not None and not future.done() for future in futures):
logger.warning("插件后台服务仍在停止,拒绝释放运行实例")
return False
if self._plugin_mutation_admission.active_count:
logger.warning("插件可变事务仍在执行,拒绝释放运行实例")
return False
return self._plugin_lifecycle.finalize()
def _quiesce_after_mutations(self) -> bool:
"""等待已获准变更自然结束后,再停用插件事件入口。"""
self._plugin_mutation_admission.wait_until_idle()
return self._plugin_lifecycle.quiesce_handlers()
@staticmethod
def _load_selective_plugins(pid: Optional[str], installed_plugins: List[str],
@@ -428,8 +587,10 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""返回配置重载日志使用的功能名称。"""
return "插件文件修改监测"
def start_monitor(self):
"""按当前配置启动插件文件修改监测"""
def start_monitor(self, *, reopen: bool = False) -> None:
"""按当前配置启动监控;新生命周期可显式解除既有封口"""
if reopen and not self._plugin_monitor.reopen():
return
if (
not self.is_plugin_settling()
and (settings.DEV or settings.PLUGIN_AUTO_RELOAD)
@@ -447,11 +608,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
)
)
def stop_monitor(self):
"""
停止监测插件文件修改监测
"""
self._plugin_monitor.stop()
def stop_monitor(self, timeout: float = 5.0) -> bool:
"""停止插件文件监控,并返回线程是否在预算内真正退出。"""
return self._plugin_monitor.stop(timeout=timeout)
def close_monitor(self, timeout: float = 5.0) -> bool:
"""封口当前生命周期的文件监控,并返回线程是否真正退出。"""
return self._plugin_monitor.close(timeout=timeout)
def _run_file_watcher(self):
"""
@@ -504,25 +667,31 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""
已安装本地插件源码变化时同步到运行目录
"""
return self._local_plugin_sync.sync(pid, candidate)
try:
with self.mutation("同步本地插件源码"):
return self._local_plugin_sync.sync(pid, candidate)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
@contextmanager
def suppress_plugin_monitor(self, plugin_id: str):
"""在插件目录原子更新期间阻止文件监控抢先重载半成品。"""
normalized_id = plugin_id.lower()
with self._monitor_suppression_lock:
self._suppressed_monitor_plugins[normalized_id] = (
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
)
try:
yield
finally:
with self.mutation("更新插件包"):
normalized_id = plugin_id.lower()
with self._monitor_suppression_lock:
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
if count <= 1:
self._suppressed_monitor_plugins.pop(normalized_id, None)
else:
self._suppressed_monitor_plugins[normalized_id] = count - 1
self._suppressed_monitor_plugins[normalized_id] = (
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
)
try:
yield
finally:
with self._monitor_suppression_lock:
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
if count <= 1:
self._suppressed_monitor_plugins.pop(normalized_id, None)
else:
self._suppressed_monitor_plugins[normalized_id] = count - 1
def is_plugin_monitor_suppressed(self, plugin_id: str) -> bool:
"""判断指定插件是否处于安装或替换写入阶段。"""
@@ -534,23 +703,43 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
从内存中移除一个插件
:param plugin_id: 插件ID
"""
self._plugin_lifecycle.stop(plugin_id)
self._plugin_registry.remove(plugin_id)
try:
with self.mutation("移除插件实例"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.stop(plugin_id)
self._plugin_registry.remove(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def reload_plugin(self, plugin_id: str) -> PluginRuntimeStatus:
"""
将一个插件重新加载到内存
:param plugin_id: 插件ID
"""
return self._plugin_lifecycle.reload(plugin_id, EventType.PluginReload)
try:
with self.mutation("重新加载插件"):
with self._plugin_quiesce_lock:
return self._plugin_lifecycle.reload(
plugin_id,
EventType.PluginReload,
)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return PluginRuntimeStatus.LOAD_FAILED
def reload_plugin_tree(self, plugin_id: str) -> PluginRuntimeStatus:
"""重载源码插件,并同步刷新所有引用该源码的虚拟实例。"""
source_plugin_id = self.get_plugin_source_id(plugin_id)
status = self.reload_plugin(source_plugin_id)
for instance in self._plugin_instance_store.for_source(source_plugin_id):
self.reload_plugin(instance.instance_id)
return status
try:
with self.mutation("重载插件实例树"):
with self._plugin_quiesce_lock:
source_plugin_id = self.get_plugin_source_id(plugin_id)
status = self.reload_plugin(source_plugin_id)
for instance in self._plugin_instance_store.for_source(source_plugin_id):
self.reload_plugin(instance.instance_id)
return status
except PluginMutationRejectedError as error:
logger.warning(str(error))
return PluginRuntimeStatus.LOAD_FAILED
def get_plugin_reload_targets(self, plugin_id: str) -> List[str]:
"""返回源码更新后需要刷新注册信息的源插件及其实例 ID。"""
@@ -584,33 +773,40 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
安装本地不存在或需要更新的插件
"""
return self._plugin_sync.sync()
with self.mutation("同步插件包"):
return self._plugin_sync.sync()
@staticmethod
def install_plugin_missing_dependencies() -> List[str]:
"""
安装插件中缺失或不兼容的依赖项
"""
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing()
@staticmethod
def install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
"""安装插件缺失依赖并返回缺失项及安装成功状态。"""
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing_with_status()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing_with_status()
@staticmethod
async def async_install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
"""在异步启动链中恢复插件依赖并保留取消语义。"""
return await PluginDependencyService(
system=get_plugin_system,
log=logger,
).async_install_missing_with_status()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return await PluginDependencyService(
system=get_plugin_system,
log=logger,
).async_install_missing_with_status()
def classify_plugins(self) -> PluginDependencyClassification:
"""按源码依赖状态分类物理插件,并把结果映射到虚拟实例。"""
@@ -706,7 +902,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
def delete_plugin_instance(self, plugin_id: str) -> bool:
"""删除虚拟实例描述;调用方仍负责停止实例和清理业务数据。"""
return self._plugin_instance_store.delete(plugin_id)
try:
with self.mutation("删除插件实例描述"):
return self._plugin_instance_store.delete(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def save_plugin_config(self, pid: str, conf: dict, force: bool = False) -> bool:
"""
@@ -715,7 +916,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param conf: 配置
:param force: 强制保存
"""
return self._plugin_config_store.write(pid, conf, force)
try:
with self.mutation("保存插件配置"):
return self._plugin_config_store.write(pid, conf, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
async def async_save_plugin_config(
self, pid: str, conf: dict, force: bool = False
@@ -726,7 +932,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param conf: 配置
:param force: 强制保存
"""
return await self._plugin_config_store.async_write(pid, conf, force)
try:
with self.mutation("保存插件配置"):
return await self._plugin_config_store.async_write(pid, conf, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def delete_plugin_config(self, pid: str, force: bool = False) -> bool:
"""
@@ -734,7 +945,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID
:param force: 插件停止后仍允许按插件 ID 删除持久化配置
"""
return self._plugin_config_store.delete(pid, force)
try:
with self.mutation("删除插件配置"):
return self._plugin_config_store.delete(pid, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def delete_plugin_data(self, pid: str, force: bool = False) -> bool:
"""
@@ -742,7 +958,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID
:param force: 插件停止后仍允许按插件 ID 删除持久化数据
"""
return self._plugin_config_store.delete_data(pid, force)
try:
with self.mutation("删除插件数据"):
return self._plugin_config_store.delete_data(pid, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def get_plugin_state(self, pid: str) -> bool:
"""
@@ -1130,14 +1351,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param icon: 自定义图标URL
:return: (是否成功, 错误信息)
"""
return self._plugin_clone.clone(
plugin_id=plugin_id,
suffix=suffix,
name=name,
description=description,
version=version,
icon=icon,
)
try:
with self.mutation("创建插件分身"):
return self._plugin_clone.clone(
plugin_id=plugin_id,
suffix=suffix,
name=name,
description=description,
version=version,
icon=icon,
)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False, str(error)
def _modify_plugin_files(self, plugin_dir: Path, original_id: str, suffix: str,
name: str, description: str, version: str = None,
+3 -2
View File
@@ -106,8 +106,8 @@ class TaskRegistry:
}
)
async def shutdown(self, *, timeout_seconds: float = 10.0) -> None:
"""停止接收并有限等待存量任务,超时任务保留登记并报告责任域"""
async def shutdown(self, *, timeout_seconds: float = 10.0) -> bool:
"""停止接收并有限等待存量任务,返回全部 owner 是否真实收敛"""
self._accepting = False
records = self.records
tasks = [record.task for record in records]
@@ -139,6 +139,7 @@ class TaskRegistry:
"timeout_seconds": timeout_seconds,
}
)
return all(record.task.done() for record in records)
_default_registry = TaskRegistry()
+10
View File
@@ -4,6 +4,7 @@ __all__ = (
"APIRateLimitException",
"RateLimitExceededException",
"OperationInterrupted",
"PluginMutationRejectedError",
"StorageQueryError",
"TMDbException",
)
@@ -49,6 +50,15 @@ class OperationInterrupted(KeyboardInterrupt):
pass
class PluginMutationRejectedError(RuntimeError):
"""表示插件运行时已封口,新的可变事务未获准执行。"""
def __init__(self, operation: str) -> None:
"""保存被拒绝的操作名称并生成稳定诊断消息。"""
self.operation = operation
super().__init__(f"插件运行时已进入停机阶段,拒绝{operation}")
class StorageQueryError(Exception):
"""
用于表示存储查询无法确认结果的异常类
+68 -24
View File
@@ -20,6 +20,9 @@ from app.runtime.log import logger
from app.schemas.types import EventType
AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0
def _get_skill_catalog() -> Any:
"""按需返回 Agent 技能目录实现,供消息应用层消费端口。"""
from app.agent.skills.registry import SkillHelper
@@ -106,6 +109,7 @@ class AgentInitializer:
self._initialized = False
self._manager: Any = None
self._compat_injected = False
self._shutdown_started = False
self._shutdown_complete = False
eventmanager.add_event_listener(
EventType.ConfigChanged,
@@ -117,6 +121,7 @@ class AgentInitializer:
初始化AI智能体管理器
"""
try:
self._shutdown_started = False
self._shutdown_complete = False
if agent_manager is not None:
if not settings.AI_AGENT_ENABLE:
@@ -142,7 +147,12 @@ class AgentInitializer:
async def handle_config_changed(self, event: Event) -> None:
"""仅在 manifest watch 命中时协调 service,关闭态保持 fail closed。"""
changed_keys = _event_changed_keys(event)
if not changed_keys or self._compat_injected or self._shutdown_complete:
if (
not changed_keys
or self._compat_injected
or self._shutdown_started
or self._shutdown_complete
):
return
try:
self._manager = await reconcile_agent_service(
@@ -156,24 +166,25 @@ class AgentInitializer:
self._initialized = False
logger.debug(f"配置变更协调AI智能体失败: {error}")
async def cleanup(self) -> None:
"""清理 initializer 引用;显式注入对象同时在此关闭"""
async def cleanup(self) -> bool:
"""清理 initializer 引用;未收敛的显式注入对象继续由本实例持有"""
try:
manager = self._manager
compat_injected = self._compat_injected
if manager is None:
return
try:
if compat_injected:
await manager.close()
logger.info("AI智能体管理器已关闭")
finally:
self._initialized = False
self._manager = None
self._compat_injected = False
return True
if compat_injected and await manager.close() is False:
logger.error("AI智能体管理器仍有会话 owner 未收敛")
return False
logger.info("AI智能体管理器已关闭")
self._initialized = False
self._manager = None
self._compat_injected = False
return True
except Exception as e:
logger.debug(f"关闭AI智能体管理器时发生错误: {e}")
return False
# 全局AI智能体初始化器实例
@@ -204,23 +215,56 @@ async def init_agent() -> bool:
return False
async def stop_agent():
async def stop_agent() -> bool:
"""
停止AI智能体异步版本用于在应用关闭时调用
停止AI智能体并在全部会话和工具资源释放后返回 True
"""
converged = True
close_blocking_executors = None
agent_initializer._shutdown_started = True
try:
if is_tool_factory_materialized():
from app.agent.tools.base import (
begin_blocking_executor_shutdown,
close_blocking_executors as close_executors,
)
# 必须在任何 manager await 之前封口,避免旧会话趁收尾窗口提交新同步调用。
begin_blocking_executor_shutdown(cancel_futures=True)
close_blocking_executors = close_executors
except Exception as e:
logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}")
converged = False
try:
if not agent_initializer._shutdown_complete:
if agent_initializer._compat_injected:
await agent_initializer.cleanup()
service_converged = await agent_initializer.cleanup()
else:
await begin_agent_shutdown()
await agent_initializer.cleanup()
agent_initializer._shutdown_complete = True
if is_tool_factory_materialized():
from app.agent.tools.base import shutdown_blocking_executors
shutdown_blocking_executors(wait=False, cancel_futures=True)
service_converged = await begin_agent_shutdown()
if service_converged is not False:
service_converged = await agent_initializer.cleanup()
converged = converged and service_converged is not False
except Exception as e:
logger.error(f"停止AI智能体时发生错误: {e}")
finally:
await close_materialized_terminal_sessions()
converged = False
if close_blocking_executors is not None:
try:
blocking_converged = await close_blocking_executors(
timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS,
cancel_futures=True,
)
converged = converged and blocking_converged
except Exception as e:
logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}")
converged = False
if converged:
try:
await close_materialized_terminal_sessions()
except Exception as e:
logger.error(f"关闭AI智能体终端会话时发生错误: {e}")
converged = False
agent_initializer._shutdown_complete = converged
return converged
+175 -31
View File
@@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.state import SystemHelper
from app.runtime.log import logger, LoggerManager
from app.startup.command_initializer import init_command, stop_command, restart_command
from app.startup.agent_initializer import stop_agent
from app.startup.domain_initializer import configure_domain_dependencies
from app.startup.modules_initializer import init_modules, stop_modules
from app.startup.modules_initializer import (
drain_events,
init_modules,
settle_events,
stop_modules,
)
from app.startup.monitor_initializer import stop_monitor, init_monitor
from app.startup.plugins_initializer import (
configure_plugin_services,
execute_task,
finalize_plugins,
init_plugins,
stop_plugins,
quiesce_plugin_services,
quiesce_plugins,
stop_plugin_monitor,
sync_plugins,
)
from app.startup.routers_initializer import init_routers
@@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import (
init_plugin_scheduler,
)
from app.db.engine import check_connection_budget, get_engine, get_global_async_engine
from app.startup.transfer_initializer import replay_pending_transfers
from app.startup.transfer_initializer import (
replay_pending_transfers,
stop_transfer_runtime,
)
from app.startup.workflow_initializer import init_workflow, stop_workflow
from app.startup.lifecycle.components import (
LifecycleComponent,
LifecycleFailurePolicy,
LifecycleMode,
lifecycle_manifest,
)
@@ -101,8 +114,8 @@ async def run_shutdown_step(
name: str,
callback: Callable[[], object],
timeout_seconds: float | None = None,
) -> None:
"""在有限预算内执行关闭阶段,并保留未收敛任务的资源所有权"""
) -> bool:
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛"""
try:
result = callback()
if inspect.isawaitable(result):
@@ -120,16 +133,22 @@ async def run_shutdown_step(
task.add_done_callback(_consume_shutdown_result)
if timeout_seconds:
try:
await asyncio.wait_for(
result = await asyncio.wait_for(
asyncio.shield(task), timeout=timeout_seconds
)
except asyncio.TimeoutError:
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
task.cancel()
return False
else:
await task
result = await task
if result is False:
logger.error("关闭%s未收敛,资源所有权保持不变", name)
return False
return True
except Exception as err:
logger.error(f"关闭{name}失败:{err}")
return False
async def run_startup_step(
@@ -152,6 +171,66 @@ async def run_startup_step(
logger.info("启动%s完成,耗时=%.2fms", name, elapsed_ms)
async def stop_lifecycle_components(
components: tuple[LifecycleComponent, ...],
) -> bool:
"""按声明顺序关闭组件,并在关键 owner 未收敛时停止释放依赖。"""
all_converged = True
for component in sorted(
(item for item in components if item.stop is not None),
key=lambda item: item.stop_order or 0,
):
completed = await run_shutdown_step(
component.name,
component.stop,
component.stop_timeout_seconds,
)
if completed:
continue
all_converged = False
if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST:
logger.error(
"关闭%s未收敛,停止释放其后续依赖",
component.name,
)
break
return all_converged
def select_startup_cleanup_components(
components: tuple[LifecycleComponent, ...],
*,
started_names: set[str],
active_component: LifecycleComponent | None,
) -> tuple[LifecycleComponent, ...]:
"""选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。"""
cleanup_names = set(started_names)
if active_component is not None:
cleanup_names.add(active_component.name)
# stop-only owner 没有启动回调,按已激活依赖递归纳入,避免清理时反向
# 实例化尚未触达的插件、模块或外部资源。
changed = True
while changed:
changed = False
for component in components:
if (
component.stop is None
or component.start is not None
or component.name in cleanup_names
or not set(component.dependencies).issubset(cleanup_names)
):
continue
cleanup_names.add(component.name)
changed = True
return tuple(
component
for component in components
if component.stop is not None and component.name in cleanup_names
)
async def initialize_modules_component(app: FastAPI) -> None:
"""启动模块并把其类型化运行时发布到当前 FastAPI AppState。"""
try:
@@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None:
configure_task_registry(task_registry)
async def stop_task_registry(app: FastAPI) -> None:
"""停止接收新后台任务,并取消、等待当前 lifespan 的存量任务"""
async def stop_task_registry(app: FastAPI) -> bool:
"""停止接收新后台任务,并把已封口登记器保留到下一次显式启动"""
task_registry = getattr(app.state, "task_registry", None)
try:
if isinstance(task_registry, TaskRegistry):
await task_registry.shutdown(timeout_seconds=30.0)
finally:
if not isinstance(task_registry, TaskRegistry):
configure_task_registry(None)
app.state.task_registry = None
return True
# 即使 owner 已收敛,也不能在后续插件/模块 stop hook 仍会运行时退回永久
# accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。
return await task_registry.shutdown(timeout_seconds=30.0)
def prepare_plugin_restore() -> None:
@@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=5,
start_timeout_seconds=30,
stop_timeout_seconds=60,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="数据库准备",
@@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
dependencies=("插件备份恢复",),
mode=LifecycleMode.NORMAL_ONLY,
start=init_plugins,
stop=stop_plugins,
stop=finalize_plugins,
start_order=90,
stop_order=60,
start_timeout_seconds=300,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件变更监控",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=stop_plugin_monitor,
stop_order=8,
stop_timeout_seconds=10,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="定时器",
@@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=50,
start_timeout_seconds=120,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="监控器",
@@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=40,
start_timeout_seconds=120,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="整理后台服务",
dependencies=("模块服务",),
stop=stop_transfer_runtime,
stop_order=52,
stop_timeout_seconds=45,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="AI智能体会话",
dependencies=("模块服务",),
stop=stop_agent,
stop_order=51,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件事件入口",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=quiesce_plugins,
stop_order=53,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="事件尾任务结算",
dependencies=("模块服务", "插件"),
mode=LifecycleMode.NORMAL_ONLY,
stop=settle_events,
stop_order=54,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件后台服务",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=quiesce_plugin_services,
stop_order=55,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="事件投递屏障",
dependencies=("模块服务", "整理后台服务"),
stop=drain_events,
stop_order=58,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="待处理整理回放",
dependencies=("监控器",),
dependencies=("监控器", "整理后台服务"),
mode=LifecycleMode.NORMAL_ONLY,
start=replay_pending_transfers,
start_order=120,
@@ -369,6 +513,9 @@ async def lifespan(app: FastAPI):
health = get_application_health(app)
health.begin_startup()
main_loop = asyncio.get_running_loop()
enabled_components: tuple[LifecycleComponent, ...] = ()
started_component_names: set[str] = set()
active_start_component: LifecycleComponent | None = None
try:
validate_process_topology(
workers=settings.API_WORKERS,
@@ -390,11 +537,14 @@ async def lifespan(app: FastAPI):
(item for item in enabled_components if item.start is not None),
key=lambda item: item.start_order or 0,
):
active_start_component = component
await run_startup_step(
component.name,
component.start,
component.start_timeout_seconds,
)
started_component_names.add(component.name)
active_start_component = None
if settings.MOVIEPILOT_SAFE_MODE:
print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.")
# 插件同步到本地
@@ -407,10 +557,15 @@ async def lifespan(app: FastAPI):
except BaseException:
# Uvicorn 在 lifespan 抛错时不会开始接流量;状态仍需供嵌入式入口和测试诊断。
health.mark_failed()
cleanup_components = select_startup_cleanup_components(
enabled_components,
started_names=started_component_names,
active_component=active_start_component,
)
try:
await stop_task_registry(app)
await stop_lifecycle_components(cleanup_components)
except Exception as cleanup_error:
logger.error(f"启动失败后的后台任务清理失败:{cleanup_error}")
logger.error(f"启动失败后的生命周期清理失败:{cleanup_error}")
finally:
global_vars.clear_loop(main_loop)
raise
@@ -421,21 +576,10 @@ async def lifespan(app: FastAPI):
health.mark_stopping()
print("Shutting down...")
global_vars.stop_system()
# 插件恢复会在线程池中修改源码与依赖,必须完成后再进入资源关闭阶段。
try:
await sync_plugins_task
except Exception as e:
print(str(e))
try:
for component in sorted(
(item for item in enabled_components if item.stop is not None),
key=lambda item: item.stop_order or 0,
):
await run_shutdown_step(
component.name,
component.stop,
component.stop_timeout_seconds,
)
# 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner
# 在统一预算内取消/等待,不能在屏障之前无界 await 绕过停机预算。
await stop_lifecycle_components(enabled_components)
finally:
try:
# 日志最后关闭,确保其他组件的收尾信息已写入文件
+17 -2
View File
@@ -129,7 +129,7 @@ from app.command import CommandChain
from app.schemas.message import Message
from app.schemas.message import MessageType
from app.schemas.types import EventType, SystemConfigKey
from app.startup.agent_initializer import init_agent, stop_agent
from app.startup.agent_initializer import init_agent
from app.startup.database import build_database_governance
from app.startup.managed_resources_initializer import (
init_managed_resources,
@@ -560,6 +560,22 @@ def close_browser_sessions() -> None:
BrowserSessionHelper.close_all_sessions()
async def drain_events() -> bool:
"""在插件卸载前等待已接收事件及其同步、异步处理器完成。"""
event_manager = EventManager.get_existing_instance()
if event_manager is None:
return True
return await event_manager.drain_async(seal=True)
async def settle_events() -> bool:
"""在插件 handler 停用后结算在途事件,但保留停机 hook 的尾事件入口。"""
event_manager = EventManager.get_existing_instance()
if event_manager is None:
return True
return await event_manager.drain_async(seal=False)
async def stop_modules():
"""
服务关闭
@@ -578,7 +594,6 @@ async def stop_modules():
logger.error(f"关闭{name}失败:{err}")
return True
await run_step("AI智能体", stop_agent)
await run_step("模块", lambda: ModuleManager().shutdown())
await run_step("事件消费", lambda: EventManager().stop_async())
await run_step("浏览器会话", close_browser_sessions)
+21 -10
View File
@@ -1,15 +1,26 @@
from app.monitor import Monitor
from app.runtime.execution import run_in_threadpool_to_completion
def init_monitor():
"""
初始化监控器
"""
Monitor()
def init_monitor() -> None:
"""初始化监控器;复用单例时必须显式开启新的应用 lifespan。"""
monitor = Monitor.get_existing_instance()
if monitor is None:
Monitor()
return
if not monitor.lifecycle_closed:
return
if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT):
raise RuntimeError("旧目录监控 owner 未收敛,无法开启新生命周期")
if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT):
raise RuntimeError("目录监控初始化失败")
def stop_monitor():
"""
停止监控器
"""
Monitor().stop()
async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool:
"""在线程池里永久关闭监控器,取消后仍等待同步 owner 收敛到终态。"""
monitor = Monitor.get_existing_instance()
if monitor is None:
return True
return bool(
await run_in_threadpool_to_completion(monitor.close, timeout=timeout)
)
+106 -52
View File
@@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import (
configure_plugin_resource_import_preparer,
configure_site_auth_level_provider,
)
from app.runtime.execution import run_in_threadpool_to_completion
from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult
from app.application.plugin.catalog import PluginCatalogService
from app.application.plugin.data import DeletePluginDataCommand
@@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork
from app.runtime.log import logger
from app.foundation.version import compare_version
from app.schemas.plugin import PluginRuntimeStatus
from app.schemas.exception import PluginMutationRejectedError
from app.schemas.types import SystemConfigKey
@@ -142,56 +144,64 @@ async def sync_plugins() -> bool:
"""
plugin_manager = None
try:
configure_plugin_services()
loop = global_vars.loop
plugin_manager = PluginManager()
plugin_manager.set_plugin_settling(True)
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
dependency_result = await (
plugin_manager.async_install_plugin_missing_dependencies_with_status()
)
if dependency_result is None:
return False
if not isinstance(dependency_result, PluginDependencyInstallResult):
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
return False
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
if not dependency_result.success:
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
changed_ids = await execute_task(
loop,
lambda: _activate_ready_plugins(
plugin_manager,
classification.ready,
sync_result or [],
previous_statuses,
),
"插件运行态激活",
)
if changed_ids is None:
return False
if not changed_ids:
logger.debug("没有新的插件进入可运行状态")
return False
for plugin_id in changed_ids:
register_plugin_api(plugin_id)
if dependency_result.success:
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
else:
logger.warning(
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
)
return True
with plugin_manager.mutation("启动后同步插件"):
configure_plugin_services()
plugin_manager.set_plugin_settling(True)
return await _sync_plugins_admitted(plugin_manager, loop)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
except Exception as e:
logger.error(f"插件初始化过程中出现异常: {e}")
return False
async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool:
"""在一个 admission lease 内完成包、依赖、实例和动态路由同步。"""
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
dependency_result = await (
plugin_manager.async_install_plugin_missing_dependencies_with_status()
)
if dependency_result is None:
return False
if not isinstance(dependency_result, PluginDependencyInstallResult):
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
return False
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
if not dependency_result.success:
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
changed_ids = await execute_task(
loop,
lambda: _activate_ready_plugins(
plugin_manager,
classification.ready,
sync_result or [],
previous_statuses,
),
"插件运行态激活",
)
if changed_ids is None:
return False
if not changed_ids:
logger.debug("没有新的插件进入可运行状态")
return False
for plugin_id in changed_ids:
register_plugin_api(plugin_id)
if dependency_result.success:
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
else:
logger.warning(
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
)
return True
def _activate_ready_plugins(
plugin_manager: PluginManager,
ready_ids: tuple[str, ...],
@@ -217,12 +227,39 @@ def _activate_ready_plugins(
return changed_ids
async def quiesce_plugins(timeout: float = 240.0) -> bool:
"""封口插件变更并停用 handler,保留超时 Future 的运行所有权。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return await plugin_manager.quiesce_plugins(timeout=timeout)
async def quiesce_plugin_services(timeout: float = 240.0) -> bool:
"""在事件结算后有界执行旧插件 close、stop_service hook。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return await plugin_manager.quiesce_plugin_services(timeout=timeout)
def finalize_plugins() -> bool:
"""在事件屏障封口后卸载已停用 handler 的插件实例。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return bool(plugin_manager.finalize_plugins())
async def execute_task(loop, task_func, task_name):
"""
执行后台任务
执行后台任务取消调用方时仍持有同步线程直到真实完成
"""
try:
result = await loop.run_in_executor(None, task_func)
# loop 参数属于既有调用 ABI;同步执行改由 completion-aware 适配器持有,
# 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。
del loop
result = await run_in_threadpool_to_completion(task_func)
if isinstance(result, PluginDependencyInstallResult):
processed_count = len(result.missing)
elif isinstance(result, list):
@@ -245,13 +282,15 @@ def init_plugins():
"""
configure_plugin_services()
plugin_manager = PluginManager()
if not plugin_manager.reopen_plugins():
raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛")
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
plugin_manager.set_plugin_settling(True)
for plugin_id in classification.ready:
plugin_manager.start(plugin_id)
register_plugin_api()
plugin_manager.start_monitor()
plugin_manager.start_monitor(reopen=True)
logger.info(
"插件启动分类:立即加载=%s,等待依赖=%s,等待源码=%s",
len(classification.ready),
@@ -260,15 +299,30 @@ def init_plugins():
)
def stop_plugins():
"""
停止插件
"""
def stop_plugin_monitor(timeout: float = 5.0) -> bool:
"""封口已创建管理器的文件监控线程,并返回是否完成收口。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
try:
plugin_manager = PluginManager()
return bool(plugin_manager.close_monitor(timeout=timeout))
except Exception as e:
logger.error(f"停止插件文件监控时发生错误:{e}", exc_info=True)
return False
def stop_plugins() -> bool:
"""停止已创建的插件监控和运行实例,不在停机阶段反向物化管理器。"""
try:
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
monitor_stopped = True
try:
plugin_manager.stop_monitor()
monitor_stopped = plugin_manager.stop_monitor()
finally:
plugin_manager.stop()
return bool(monitor_stopped)
except Exception as e:
logger.error(f"停止插件时发生错误:{e}", exc_info=True)
return False
+12
View File
@@ -11,3 +11,15 @@ def replay_pending_transfers():
回放本身在后台线程执行不阻塞启动流程
"""
TransferChain().replay_pending()
async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool:
"""关闭已存在的整理后台 owner,且不在关停阶段创建新的整理链实例。
:param timeout_seconds: worker pending 回放共享的最大等待秒数
:return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True
"""
transfer_chain = TransferChain.get_existing_instance()
if transfer_chain is None:
return True
return await transfer_chain.close(timeout_seconds=timeout_seconds)
@@ -6,7 +6,7 @@
> 审计范围:宿主后端;排除 `app/plugins/**` 运行时插件副本
> 规范优先级:`AGENTS.md``docs/rules/` 高于本文
> 相关文档:`docs/architecture-overview.md``docs/refactor/backend-architecture-governance.md``docs/refactor/backend-module-refactor-compatibility.md`
> 实施进度:阶段 0~6 的宿主架构能力已完成收口;API/Application 公共复杂度基线已清零,启动组合根的 SystemConfigOper 构造点已由 14 降至 1;API 进程内后台任务已完成首批统一登记,插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线;阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义。
> 实施进度:阶段 0~6 的宿主架构能力已完成收口;API/Application 公共复杂度基线已清零,启动组合根的 SystemConfigOper 构造点已由 14 降至 1;API 进程内后台任务已完成首批统一登记,插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线;阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义;阶段 1b1 已收口整理 worker、pending 回放、失败通知、进程内 AI 重试、插件监控与事件投递的生命周期所有权
## 当前复核结论(2026-08-23
@@ -15,8 +15,8 @@
### 长期整改阶段 0:治理门禁恢复(2026-08-23
- 宿主依赖基线已审查 TaskRegistry 接入后的语义差异:当前为 `800` 个模块、`6479` 条内部导入边,12 组重点禁止边继续全部为 `0`,唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistrynormal/safe 组件数分别为 `16`/`8`,CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序,不再把 Python/平台模块数量当作硬合同。
- 宿主依赖基线已审查 TaskRegistry 与有界后台 owner 接入后的语义差异:当前为 `800` 个模块、`6482` 条内部导入边,12 组重点禁止边继续全部为 `0`,唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistrynormal/safe 组件数分别为 `20`/`10`,CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序,不再把 Python/平台模块数量当作硬合同。
- 官方插件快照覆盖 `plugins.v3``plugins.v2` 以及 V3 实际会从 `package.json` 回退加载的 31 个默认实现;`app/plugins/**` 仍只是宿主运行副本,不进入扫描。
- SDK 快照以各模块显式 `__all__` 为公开合同,能够记录赋值别名;`typing``__future__` 等实现期导入不再被误冻结,既有数据库备份门面已补精确导出清单。
- async 阻塞实际债务已由 fixture 中的 10 项下降到 1 项并固化低水位;剩余项是 Scheduler Agent task 查询,后续阶段迁入异步查询边界后归零。
@@ -36,6 +36,36 @@
- 本子阶段仍只覆盖 TaskRegistry。Transfer worker/replay、Agent blocking executor、Event handler
drain、通道线程和 E2/E3 durable 完成点继续作为阶段 1 后续切片,不能因 owner 门禁通过而宣称完成。
### 长期整改阶段 1b1:整理后台生命周期所有权(2026-08-23)
- `TransferChain` 为每一代整理 worker 使用独立停止信号;配置热更新只让旧代完成已经进入同步 I/O 的
工作,不再让旧线程重新领取新任务。超时旧线程继续由 `_retiring_threads` 持有,重复关闭可以继续等待,
不把无法强制取消的文件操作伪装成已结束。生命周期锁等待与 worker/replay join 共用同一个 deadline
停止哨兵也不再被误算成真实队列任务而阻止最后一批进度结算。
- pending 回放改为单一受管线程,启动重复调用不会并发扫描;关停信号会在查询、`stat` 和逐条回放边界
重新检查,尚未处理的 `TransferPending` 登记保持不变。关闭与 `queue.get()` 竞争时,尚未开始的任务会
原样放回队列并保持 `task_done()` 计数平衡。
- 失败通知聚合器和 AI 重试 scheduler 现在拥有 timer、buffer 与 flush task 的显式 `close()`;分组使用
generation 阻止旧 timer 在新静默窗口尚未 armed 时提前消费新批次。跨线程提交 AI 重试产生的 Future
也会观察最终异常,不再只用提交调用外层的 `try/except` 假设异步执行成功。
- 生命周期新增常驻“整理后台服务” owner:正常模式和安全模式都只关闭已存在的单例,不会在 shutdown
反向创建 worker。插件文件监控、Scheduler、Agent 和整理 owner 先停止宿主生产任务;随后封口插件变更、
停用插件事件入口并结算全部在途 handler,最后才调用插件私有 timer、scheduler、watcher 的旧停机 hook。
任一阶段返回 `False` 或超时,`FAIL_FAST` 屏障都会停止释放仍可能被活线程使用的后续依赖。
- EventManager 同时持有线程池同步 handler Future 和事件循环异步 handler completion;“事件投递屏障”会
等待队列、handler 及 handler 派生事件自然收敛,再原子封住新的广播提交。为兼容无法声明资源依赖的
旧插件,宿主先停用其新 handler 投递,再用非封口屏障等待已经开始的 handler 退出,之后按原顺序调用
`close`/`stop_service`;hook 产生的尾事件仍可由其他宿主 handler 消费,最终屏障后才卸载插件实例。
事件消费、共享线程池和模块资源仍在插件之后关闭。
- lifespan 启动中途失败会记录已完成及当前部分启动的组件,递归纳入已激活依赖对应的 stop-only owner
并复用正常停机的顺序、超时和 `FAIL_FAST` 策略。后段初始化失败不再只停止 TaskRegistry 后遗留 Scheduler、
Transfer、Event、插件或 HTTP 资源。
- 兼容边界没有变化:`do_transfer`、队列与手工整理公开签名、模块方法 kwargs、插件整理事件类型及 payload、
同步插件 ABI 和动态 API 原生返回结构均未改名或包裹;本阶段也没有 schema/Alembic 变更。
- 本阶段只证明进程内 owner、取消、等待和依赖释放顺序。失败通知仍可能在业务提交后、消息接受前随进程
崩溃而丢失;五分钟 AI 重试缓冲仍未持久化;`TransferPending` 仍只有 `storage + src_path`,不能表达文件
副作用 checkpoint、lease 和未知完成状态。它们分别留给阶段 1b2、1b3、1b4,不能据此宣称 E2/E3 完成。
### 总体判断
当前架构总体合理,已经从跨层混合的遗留单体收敛为**边界清晰的模块化单体**:
+6 -2
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6479,
"edge_sha256": "a65f8d4024e2299b37510359c7ffea91219f0aa0ed9673b74a3eb22d51d6c67f",
"edge_count": 6483,
"edge_sha256": "0a2f82ca189468d5e954b06a2c188c4ea5380a76c7c64039c7f6eb843cb60523",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -5671,6 +5671,7 @@
"app.runtime.extensions.plugin_manager -> app.runtime.observability",
"app.runtime.extensions.plugin_manager -> app.runtime.reload",
"app.runtime.extensions.plugin_manager -> app.runtime.settings",
"app.runtime.extensions.plugin_manager -> app.runtime.thread",
"app.runtime.extensions.plugin_manager -> app.schemas",
"app.runtime.extensions.plugin_manager -> app.schemas.plugin",
"app.runtime.extensions.plugin_manager -> app.schemas.types",
@@ -6086,6 +6087,7 @@
"app.startup.lifecycle -> app.runtime.tasks",
"app.startup.lifecycle -> app.runtime.topology",
"app.startup.lifecycle -> app.startup",
"app.startup.lifecycle -> app.startup.agent_initializer",
"app.startup.lifecycle -> app.startup.cache_initializer",
"app.startup.lifecycle -> app.startup.command_initializer",
"app.startup.lifecycle -> app.startup.database_initializer",
@@ -6218,6 +6220,8 @@
"app.startup.modules_initializer -> app.startup.transaction",
"app.startup.modules_initializer -> app.startup.workflow",
"app.startup.monitor_initializer -> app.monitor",
"app.startup.monitor_initializer -> app.runtime",
"app.startup.monitor_initializer -> app.runtime.execution",
"app.startup.outbox -> app.application",
"app.startup.outbox -> app.application.outbox",
"app.startup.outbox -> app.db",
+140 -132
View File
@@ -1,41 +1,41 @@
{
"schema_version": 2,
"generated_at": "2026-08-23T08:11:40.406030+00:00",
"generated_at": "2026-08-23T11:06:42.858188+00:00",
"platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O",
"python": "3.14.3",
"repeat": 3,
"targets": {
"app.startup.lifecycle": {
"loaded_app_module_count": 362,
"max_ms": 944.347,
"median_ms": 935.035,
"min_ms": 933.625,
"max_ms": 1022.231,
"median_ms": 976.39,
"min_ms": 967.395,
"samples_ms": [
944.347,
935.035,
933.625
1022.231,
967.395,
976.39
]
},
"app.factory": {
"loaded_app_module_count": 374,
"max_ms": 950.655,
"median_ms": 950.429,
"min_ms": 947.964,
"max_ms": 989.21,
"median_ms": 980.605,
"min_ms": 951.564,
"samples_ms": [
950.655,
947.964,
950.429
980.605,
989.21,
951.564
]
},
"app.main": {
"loaded_app_module_count": 376,
"max_ms": 1150.269,
"median_ms": 1103.976,
"min_ms": 1097.978,
"max_ms": 1114.04,
"median_ms": 1088.251,
"min_ms": 1080.26,
"samples_ms": [
1103.976,
1097.978,
1150.269
1080.26,
1088.251,
1114.04
]
}
},
@@ -46,57 +46,26 @@
"samples": [
{
"mode": "normal",
"enabled_component_count": 16,
"startup_ms": 0.721,
"full_lifespan_ms": 0.907,
"stage_ms": {
"后台任务登记器": 0.084,
"数据库准备": 0.038,
"HTTP 基础能力": 0.028,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.024,
"路由": 0.023,
"模块服务": 0.024,
"插件备份恢复": 0.023,
"插件": 0.029,
"定时器": 0.023,
"监控器": 0.024,
"待处理整理回放": 0.021,
"命令服务": 0.025,
"工作流": 0.02,
"插件同步与启动收尾": 0.031
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "normal",
"enabled_component_count": 16,
"startup_ms": 0.614,
"full_lifespan_ms": 0.788,
"enabled_component_count": 21,
"startup_ms": 0.619,
"full_lifespan_ms": 0.806,
"stage_ms": {
"后台任务登记器": 0.075,
"数据库准备": 0.037,
"HTTP 基础能力": 0.029,
"HTTP 基础能力": 0.031,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.023,
"数据库连接预算": 0.024,
"路由": 0.023,
"模块服务": 0.023,
"插件备份恢复": 0.023,
"插件": 0.024,
"定时器": 0.025,
"监控器": 0.023,
"待处理整理回放": 0.022,
"命令服务": 0.021,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.025,
"路由": 0.022,
"模块服务": 0.024,
"插件备份恢复": 0.021,
"插件": 0.02,
"定时器": 0.023,
"监控器": 0.021,
"待处理整理回放": 0.019,
"命令服务": 0.022,
"工作流": 0.022,
"插件同步与启动收尾": 0.03
"插件同步与启动收尾": 0.034
},
"threads_before": 2,
"threads_started": 2,
@@ -108,26 +77,57 @@
},
{
"mode": "normal",
"enabled_component_count": 16,
"startup_ms": 0.611,
"full_lifespan_ms": 0.785,
"enabled_component_count": 21,
"startup_ms": 0.628,
"full_lifespan_ms": 0.808,
"stage_ms": {
"后台任务登记器": 0.072,
"数据库准备": 0.033,
"HTTP 基础能力": 0.026,
"后台任务登记器": 0.073,
"数据库准备": 0.037,
"HTTP 基础能力": 0.039,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.025,
"数据库连接预算": 0.025,
"路由": 0.022,
"模块服务": 0.022,
"插件备份恢复": 0.021,
"插件": 0.02,
"定时器": 0.024,
"监控器": 0.022,
"待处理整理回放": 0.029,
"命令服务": 0.02,
"工作流": 0.021,
"插件同步与启动收尾": 0.032
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "normal",
"enabled_component_count": 21,
"startup_ms": 0.664,
"full_lifespan_ms": 0.863,
"stage_ms": {
"后台任务登记器": 0.083,
"数据库准备": 0.047,
"HTTP 基础能力": 0.036,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.023,
"数据库连接预算": 0.022,
"路由": 0.024,
"模块服务": 0.02,
"插件备份恢复": 0.023,
"插件": 0.024,
"数据库引擎预热": 0.029,
"数据库连接预算": 0.025,
"路由": 0.026,
"模块服务": 0.025,
"插件备份恢复": 0.025,
"插件": 0.028,
"定时器": 0.026,
"监控器": 0.024,
"待处理整理回放": 0.023,
"监控器": 0.021,
"待处理整理回放": 0.021,
"命令服务": 0.024,
"工作流": 0.022,
"插件同步与启动收尾": 0.031
"工作流": 0.021,
"插件同步与启动收尾": 0.036
},
"threads_before": 2,
"threads_started": 2,
@@ -138,9 +138,9 @@
"database_connections_started": 0
}
],
"median_startup_ms": 0.614,
"median_full_lifespan_ms": 0.788,
"enabled_component_count": 16,
"median_startup_ms": 0.628,
"median_full_lifespan_ms": 0.808,
"enabled_component_count": 21,
"enabled_components": [
"后台任务登记器",
"数据库准备",
@@ -152,8 +152,13 @@
"模块服务",
"插件备份恢复",
"插件",
"插件变更监控",
"定时器",
"监控器",
"整理后台服务",
"AI智能体会话",
"插件后台服务",
"事件投递屏障",
"待处理整理回放",
"命令服务",
"工作流",
@@ -164,19 +169,19 @@
"samples": [
{
"mode": "safe",
"enabled_component_count": 8,
"startup_ms": 0.463,
"full_lifespan_ms": 0.667,
"enabled_component_count": 11,
"startup_ms": 0.464,
"full_lifespan_ms": 0.637,
"stage_ms": {
"后台任务登记器": 0.075,
"数据库准备": 0.035,
"后台任务登记器": 0.074,
"数据库准备": 0.038,
"HTTP 基础能力": 0.03,
"领域依赖装配": 0.032,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.022,
"路由": 0.025,
"模块服务": 0.022,
"插件同步与启动收尾": 0.058
"领域依赖装配": 0.029,
"数据库引擎预热": 0.026,
"数据库连接预算": 0.025,
"路由": 0.024,
"模块服务": 0.024,
"插件同步与启动收尾": 0.033
},
"threads_before": 2,
"threads_started": 2,
@@ -188,43 +193,43 @@
},
{
"mode": "safe",
"enabled_component_count": 8,
"startup_ms": 0.474,
"full_lifespan_ms": 0.68,
"enabled_component_count": 11,
"startup_ms": 0.451,
"full_lifespan_ms": 0.624,
"stage_ms": {
"后台任务登记器": 0.076,
"数据库准备": 0.043,
"HTTP 基础能力": 0.029,
"领域依赖装配": 0.028,
"后台任务登记器": 0.073,
"数据库准备": 0.035,
"HTTP 基础能力": 0.028,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.023,
"路由": 0.023,
"模块服务": 0.02,
"插件同步与启动收尾": 0.031
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "safe",
"enabled_component_count": 11,
"startup_ms": 0.477,
"full_lifespan_ms": 0.661,
"stage_ms": {
"后台任务登记器": 0.081,
"数据库准备": 0.05,
"HTTP 基础能力": 0.035,
"领域依赖装配": 0.03,
"数据库引擎预热": 0.026,
"数据库连接预算": 0.024,
"路由": 0.024,
"模块服务": 0.024,
"插件同步与启动收尾": 0.057
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "safe",
"enabled_component_count": 8,
"startup_ms": 0.454,
"full_lifespan_ms": 0.657,
"stage_ms": {
"后台任务登记器": 0.075,
"数据库准备": 0.036,
"HTTP 基础能力": 0.028,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.025,
"数据库连接预算": 0.024,
"路由": 0.024,
"模块服务": 0.024,
"插件同步与启动收尾": 0.06
"模块服务": 0.02,
"插件同步与启动收尾": 0.026
},
"threads_before": 2,
"threads_started": 2,
@@ -235,9 +240,9 @@
"database_connections_started": 0
}
],
"median_startup_ms": 0.463,
"median_full_lifespan_ms": 0.667,
"enabled_component_count": 8,
"median_startup_ms": 0.464,
"median_full_lifespan_ms": 0.637,
"enabled_component_count": 11,
"enabled_components": [
"后台任务登记器",
"数据库准备",
@@ -246,7 +251,10 @@
"数据库引擎预热",
"数据库连接预算",
"路由",
"模块服务"
"模块服务",
"整理后台服务",
"AI智能体会话",
"事件投递屏障"
]
}
}
+104 -12
View File
@@ -115,7 +115,7 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager(
initializer._manager = manager
initializer._initialized = True
initializer._compat_injected = False
shutdown = AsyncMock()
shutdown = AsyncMock(return_value=True)
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown)
monkeypatch.setattr(agent_initializer, "agent_initializer", initializer)
monkeypatch.setattr(
@@ -124,13 +124,63 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager(
lambda: False,
)
await agent_initializer.stop_agent()
assert await agent_initializer.stop_agent() is True
shutdown.assert_awaited_once_with()
manager.close.assert_not_awaited()
assert initializer._manager is None
@pytest.mark.anyio
async def test_production_stop_retains_manager_when_runtime_does_not_converge(
monkeypatch,
) -> None:
"""Agent service 未收敛时不得释放 initializer 引用或下游工具资源。"""
manager = AsyncMock()
initializer = agent_initializer.AgentInitializer()
initializer._manager = manager
initializer._initialized = True
initializer._compat_injected = False
shutdown = AsyncMock(return_value=False)
executor_seal = MagicMock()
executor_close = AsyncMock(return_value=True)
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown)
monkeypatch.setattr(agent_initializer, "agent_initializer", initializer)
monkeypatch.setattr(
agent_initializer,
"is_tool_factory_materialized",
lambda: True,
)
fake_base = types.ModuleType("app.agent.tools.base")
fake_base.begin_blocking_executor_shutdown = executor_seal
fake_base.close_blocking_executors = executor_close
monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base)
assert await agent_initializer.stop_agent() is False
shutdown.assert_awaited_once_with()
assert initializer._manager is manager
assert initializer._shutdown_started is True
assert initializer._shutdown_complete is False
executor_seal.assert_called_once_with(cancel_futures=True)
executor_close.assert_awaited_once_with(
timeout_seconds=(
agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS
),
cancel_futures=True,
)
event = agent_initializer.Event(
agent_initializer.EventType.ConfigChanged,
{"key": "AI_AGENT_ENABLE"},
)
reconcile = AsyncMock()
monkeypatch.setattr(agent_initializer, "reconcile_agent_service", reconcile)
await initializer.handle_config_changed(event)
reconcile.assert_not_awaited()
assert initializer._manager is manager
@pytest.mark.anyio
async def test_config_listener_delegates_watch_filter_to_runtime(monkeypatch) -> None:
"""配置监听器只转交 changed keys,不维护第二份启用开关。"""
@@ -227,8 +277,10 @@ async def test_stop_closes_tool_executor_after_factory_materialization(
) -> None:
"""工具能力已解析时,应取消仍排队的阻塞工具任务。"""
fake_base = types.ModuleType("app.agent.tools.base")
cleanup = MagicMock()
fake_base.shutdown_blocking_executors = cleanup
seal = MagicMock()
cleanup = AsyncMock(return_value=True)
fake_base.begin_blocking_executor_shutdown = seal
fake_base.close_blocking_executors = cleanup
monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base)
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock())
monkeypatch.setattr(
@@ -242,15 +294,27 @@ async def test_stop_closes_tool_executor_after_factory_materialization(
agent_initializer.AgentInitializer(),
)
await agent_initializer.stop_agent()
assert await agent_initializer.stop_agent() is True
cleanup.assert_called_once_with(wait=False, cancel_futures=True)
seal.assert_called_once_with(cancel_futures=True)
cleanup.assert_awaited_once_with(
timeout_seconds=(
agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS
),
cancel_futures=True,
)
@pytest.mark.anyio
async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None:
"""应用关闭不得等待已经进入线程池且尚未返回的工具调用"""
from app.agent.tools.base import MoviePilotTool
async def test_stop_retains_running_blocking_tool_until_retry(monkeypatch) -> None:
"""阻塞工具超过预算时 stop 返回 False,真实结束后重试才释放 owner"""
from app.agent.tools.base import (
MoviePilotTool,
_blocking_futures,
_blocking_retiring_executors,
close_blocking_executors,
reopen_blocking_executors,
)
started = threading.Event()
release = threading.Event()
@@ -264,7 +328,21 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None
MoviePilotTool.run_blocking("web", _blocking_call)
)
assert await asyncio.wait_for(asyncio.to_thread(started.wait), timeout=1)
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock())
monkeypatch.setattr(
agent_initializer,
"begin_agent_shutdown",
AsyncMock(return_value=True),
)
monkeypatch.setattr(
agent_initializer,
"close_materialized_terminal_sessions",
AsyncMock(),
)
monkeypatch.setattr(
agent_initializer,
"AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS",
0.01,
)
monkeypatch.setattr(
agent_initializer,
"is_tool_factory_materialized",
@@ -277,8 +355,22 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None
)
try:
await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2)
assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is False
assert worker.done() is False
finally:
assert _blocking_futures
assert _blocking_retiring_executors
with pytest.raises(RuntimeError, match="正在关闭"):
await MoviePilotTool.run_blocking("web", lambda: "late")
release.set()
assert await asyncio.wait_for(worker, timeout=1) == "done"
assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is True
assert not _blocking_futures
assert not _blocking_retiring_executors
finally:
release.set()
if not worker.done():
await asyncio.wait_for(worker, timeout=1)
await close_blocking_executors(timeout_seconds=1, cancel_futures=True)
assert reopen_blocking_executors() is True
+45 -3
View File
@@ -115,8 +115,8 @@ async def test_agent_manager_background_tasks_share_owner_loop(monkeypatch) -> N
assert manager._idle_cleanup_task is idle_cleanup_task
assert memory_manager.cleanup_task is memory_cleanup_task
await manager.close()
await manager.close()
assert await manager.close() is True
assert await manager.close() is True
assert manager._idle_cleanup_task is None
assert memory_manager.cleanup_task is None
assert idle_cleanup_task.done()
@@ -489,7 +489,8 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout(
)
await asyncio.wait_for(started.wait(), timeout=1)
await manager.close()
assert await manager.close() is False
assert await manager.close() is False
assert not cleanup_called.is_set()
assert "close-timeout" in manager.active_agents
assert manager._close_finalizer_task is not None
@@ -503,6 +504,47 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout(
break
await asyncio.sleep(0)
assert manager.active_agents == {}
assert await manager.close() is True
@pytest.mark.anyio
async def test_close_retains_agent_until_detached_subagent_converges(
monkeypatch,
) -> None:
"""Agent cleanup 返回 False 时 manager 必须保留 agent 和共享记忆 owner。"""
manager = AgentManager()
memory_manager = MemoryManager()
monkeypatch.setattr(agent_module, "memory_manager", memory_manager)
class PendingSubagentOwner:
"""首次清理未收敛、第二次清理成功的最小 Agent 替身。"""
def __init__(self) -> None:
"""初始化封口计数与可重试清理结果。"""
self.seal_count = 0
self.cleanup_results = iter((False, True))
def begin_shutdown(self) -> None:
"""记录 manager 在首个 await 前封住了子代理提交。"""
self.seal_count += 1
async def cleanup(self) -> bool:
"""按测试序列返回 detached owner 的收敛状态。"""
return next(self.cleanup_results)
owner = PendingSubagentOwner()
await manager.initialize()
manager.active_agents["detached-owner"] = owner
assert await manager.close() is False
assert manager.active_agents == {"detached-owner": owner}
assert owner.seal_count == 1
assert memory_manager.cleanup_task is not None
assert await manager.close() is True
assert manager.active_agents == {}
assert owner.seal_count == 2
assert memory_manager.cleanup_task is None
@pytest.mark.anyio
+37 -3
View File
@@ -4,7 +4,12 @@ from types import SimpleNamespace
from typing import Optional
from unittest.mock import AsyncMock, MagicMock, patch
from app.agent.tools.impl._plugin_tool_utils import install_plugin_runtime
import pytest
from app.agent.tools.impl._plugin_tool_utils import (
install_plugin_runtime,
uninstall_plugin_runtime,
)
from app.agent.tools.impl.install_plugin import InstallPluginTool
from app.agent.tools.impl.query_installed_plugins import QueryInstalledPluginsTool
from app.agent.tools.impl.query_market_plugins import QueryMarketPluginsTool
@@ -14,6 +19,10 @@ from app.agent.tools.impl.reload_plugin import ReloadPluginTool
from app.schemas.plugin import PluginRuntimeStatus
from app.agent.tools.impl.uninstall_plugin import UninstallPluginTool
from app.agent.tools.impl.update_plugin_config import UpdatePluginConfigTool
from app.runtime.extensions.plugin.admission import (
PluginMutationAdmission,
PluginMutationRejectedError,
)
def _plugin_snapshot(state: bool = True) -> dict:
@@ -334,8 +343,7 @@ def test_install_plugin_runtime_reloads_in_threadpool() -> None:
)
assert len(calls) == 2
assert calls[0][0] == "plugin"
assert calls[0][1] == plugin_manager.reload_plugin_tree
assert calls[0][2] == ("DemoPlugin",)
assert calls[0][2] == (plugin_manager.reload_plugin_tree, "DemoPlugin")
assert calls[0][3] == {}
assert calls[1][0] == "plugin"
assert calls[1][1] == refresh_registrations
@@ -372,6 +380,32 @@ def test_uninstall_plugin_uninstalls_installed_candidate() -> None:
uninstall_runtime.assert_awaited_once_with("DemoPlugin")
def test_sealed_agent_uninstall_rejects_before_persistence() -> None:
"""Agent 卸载未获 admission 时不读取实例或修改安装清单。"""
admission = PluginMutationAdmission()
admission.seal()
plugin_manager = MagicMock()
plugin_manager.mutation.side_effect = admission.hold
config_oper = MagicMock()
with (
patch(
"app.agent.tools.impl._plugin_tool_utils.get_plugin_manager",
return_value=plugin_manager,
),
patch(
"app.agent.tools.impl._plugin_tool_utils.SystemConfigOper",
return_value=config_oper,
) as config_provider,
pytest.raises(PluginMutationRejectedError),
):
asyncio.run(uninstall_plugin_runtime("DemoPlugin"))
plugin_manager.get_plugin_instance.assert_not_called()
config_provider.assert_not_called()
config_oper.async_set.assert_not_called()
def test_query_plugin_data_truncates_large_payload() -> None:
"""
查询插件数据会截断超长内容并返回预览
+36 -1
View File
@@ -42,6 +42,7 @@ class _FakeManager:
self.fail_initialize = False
self.initialize_entered: asyncio.Event | None = None
self.initialize_release: asyncio.Event | None = None
self.close_converged: bool | None = None
async def initialize(self) -> None:
self.initialize_calls += 1
@@ -52,8 +53,42 @@ class _FakeManager:
if self.fail_initialize:
raise RuntimeError("service initialization failed")
async def close(self) -> None:
async def close(self) -> bool | None:
self.close_calls += 1
return self.close_converged
@pytest.mark.anyio
async def test_shutdown_retains_nonconverged_agent_service_for_retry(
runtime_loader,
monkeypatch,
) -> None:
"""Manager 返回未收敛时 Runtime 必须保留 owner,后续关闭可继续等待。"""
manager = _FakeManager()
manager.close_converged = False
modules = _fake_agent_modules(manager)
monkeypatch.setattr(
"app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE",
True,
)
monkeypatch.setattr(
"app.agent.capabilities.adapter.importlib.import_module",
lambda name: (
monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name]
),
)
assert await runtime_loader.activate_agent_service() is manager
assert await runtime_loader.begin_agent_shutdown() is False
snapshot = runtime_loader._agent_runtime.snapshot("agent.service")
assert snapshot.lifecycle is CapabilityLifecycleState.FAILED
assert snapshot.visible is False
manager.close_converged = True
assert await runtime_loader.begin_agent_shutdown() is True
snapshot = runtime_loader._agent_runtime.snapshot("agent.service")
assert snapshot.lifecycle is CapabilityLifecycleState.STOPPED
assert manager.close_calls == 2
def _fake_agent_modules(manager: object | None = None) -> dict[str, types.ModuleType]:
+50 -7
View File
@@ -284,7 +284,7 @@ async def test_agent_cleanup_closes_subagent_middlewares() -> None:
agent._subagent_middlewares = (_Middleware(),)
await agent.cleanup()
assert await agent.cleanup() is True
assert closed == [True]
assert agent._subagent_middlewares == ()
@@ -345,16 +345,16 @@ def test_subagent_control_middleware_close_is_idempotent() -> None:
middleware._tasks = {}
async def _close() -> None:
await middleware.close()
await middleware.close()
assert await middleware.close() is True
assert await middleware.close() is True
asyncio.run(_close())
assert middleware._tasks == {}
@pytest.mark.anyio
async def test_subagent_close_has_bounded_cancel_wait() -> None:
"""子代理忽略首次取消时,控制器关闭仍必须在上限内返回"""
async def test_subagent_close_retains_stubborn_owner_until_retry() -> None:
"""子代理忽略取消时 close 返回 False,任务结束后重试才清理记录"""
middleware = object.__new__(SubAgentTaskControlMiddleware)
release = asyncio.Event()
cancelled = asyncio.Event()
@@ -383,14 +383,57 @@ async def test_subagent_close_has_bounded_cancel_wait() -> None:
"app.agent.middleware.subagents.SUBAGENT_CANCEL_GRACE_SECONDS",
0.01,
):
await asyncio.wait_for(middleware.close(), timeout=0.2)
assert await asyncio.wait_for(middleware.close(), timeout=0.2) is False
assert cancelled.is_set()
assert task.done() is False
assert middleware._tasks == {}
assert middleware._tasks == {record.task_id: record}
assert task in middleware._close_cancel_requested
release.set()
await asyncio.wait_for(task, timeout=0.2)
assert await middleware.close() is True
assert middleware._tasks == {}
assert middleware._close_cancel_requested == set()
@pytest.mark.anyio
async def test_subagent_seal_rejects_new_detached_task() -> None:
"""控制器封口后必须同步拒绝 start,且不能创建新的 asyncio Task。"""
middleware = object.__new__(SubAgentTaskControlMiddleware)
middleware._tasks = {}
middleware._accepting_tasks = True
middleware._close_cancel_requested = set()
middleware.seal()
payload = await middleware._control_task(
action="start",
description="late task",
)
result = json.loads(payload)
assert result["success"] is False
assert "正在关闭" in result["error"]
assert middleware._tasks == {}
@pytest.mark.anyio
async def test_agent_cleanup_retains_nonconverged_subagent_middleware() -> None:
"""Agent cleanup 必须保留返回 False 的 middleware,供重复调用继续收口。"""
agent = MoviePilotAgent(session_id="session-owner", user_id="user-owner")
middleware = SimpleNamespace(
seal=MagicMock(),
close=AsyncMock(side_effect=[False, True]),
)
agent._subagent_middlewares = (middleware,)
assert await agent.cleanup() is False
assert agent._subagent_middlewares == (middleware,)
assert await agent.cleanup() is True
assert agent._subagent_middlewares == ()
assert middleware.seal.call_count == 2
assert middleware.close.await_count == 2
@pytest.mark.anyio
+60
View File
@@ -8,6 +8,10 @@ from app.agent.tools.base import (
MoviePilotTool,
ToolExecutionTimeoutError,
_blocking_executors,
_blocking_futures,
_blocking_retiring_executors,
close_blocking_executors,
reopen_blocking_executors,
shutdown_blocking_executors,
)
from app.agent.tools.manager import MoviePilotToolsManager
@@ -37,6 +41,15 @@ class BlockingAgentTool(MoviePilotTool):
return "unused"
@pytest.fixture(autouse=True)
def _reset_blocking_executor_runtime():
"""每个用例前后恢复阻塞池门禁,避免进程级 owner 状态串扰。"""
assert reopen_blocking_executors() is True
yield
assert shutdown_blocking_executors(cancel_futures=True) is True
assert reopen_blocking_executors() is True
def test_arun_raises_timeout_when_tool_exceeds_limit():
"""底层工具入口应把超时交给宿主策略记录失败终态。"""
tool = SlowAgentTool(session_id="session-1", user_id="10001")
@@ -126,6 +139,8 @@ def test_shutdown_blocking_executors_clears_agent_tool_workers():
shutdown_blocking_executors()
assert _blocking_executors == {}
assert _blocking_futures == {}
assert _blocking_retiring_executors == set()
def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent():
@@ -159,10 +174,55 @@ def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent():
queued_future = asyncio.run(_run_scenario())
assert _blocking_executors == {}
assert _blocking_futures == {}
assert _blocking_retiring_executors == set()
assert queued_future.cancelled()
assert not queued_ran.is_set()
@pytest.mark.asyncio
async def test_close_blocking_executors_retains_owner_until_retry() -> None:
"""同步调用超时后保留 Future/executor,完成后的重复 close 才成功。"""
started = threading.Event()
release = threading.Event()
def _blocking_call() -> str:
"""等待测试释放,稳定制造超过关停预算的运行 Future。"""
started.set()
release.wait()
return "done"
task = asyncio.create_task(
MoviePilotTool.run_blocking("web", _blocking_call)
)
assert await asyncio.to_thread(started.wait, 1)
try:
assert await close_blocking_executors(
timeout_seconds=0.01,
cancel_futures=True,
) is False
assert task.done() is False
assert _blocking_futures
assert _blocking_retiring_executors
with pytest.raises(RuntimeError, match="正在关闭"):
await MoviePilotTool.run_blocking("web", lambda: "late")
release.set()
assert await asyncio.wait_for(task, timeout=1) == "done"
assert await close_blocking_executors(
timeout_seconds=0.01,
cancel_futures=True,
) is True
assert _blocking_futures == {}
assert _blocking_retiring_executors == set()
finally:
release.set()
if not task.done():
await asyncio.wait_for(task, timeout=1)
def test_create_agent_config_uses_llm_max_iterations():
"""Agent 执行配置应把 LLM_MAX_ITERATIONS 传给 LangGraph recursion_limit。"""
from app.agent.orchestrator import MoviePilotAgent
+409 -1
View File
@@ -1,7 +1,9 @@
"""事件调度订阅快照和生命周期回归测试。"""
import asyncio
import concurrent.futures
import threading
from queue import PriorityQueue
import pytest
@@ -16,12 +18,49 @@ class _ImmediateExecutor:
@staticmethod
def submit(func, *args, **kwargs):
return func(*args, **kwargs)
"""立即执行调用并返回符合线程池接口的已完成 Future。"""
handle = concurrent.futures.Future()
try:
handle.set_result(func(*args, **kwargs))
except Exception as err:
handle.set_exception(err)
return handle
class _DaemonThreadExecutor:
"""以守护线程执行回调,让死锁回归测试能够有界失败。"""
def __init__(self) -> None:
"""初始化可观察的回调完成信号。"""
self.finished = threading.Event()
def submit(self, func, *args, **kwargs):
"""在线程中执行调用并返回符合线程池接口的 Future。"""
handle = concurrent.futures.Future()
def run() -> None:
"""执行回调并把结果或异常写入 Future。"""
if not handle.set_running_or_notify_cancel():
return
try:
handle.set_result(func(*args, **kwargs))
except BaseException as err: # pragma: no cover - 交由 Future 消费
handle.set_exception(err)
finally:
self.finished.set()
threading.Thread(target=run, daemon=True).start()
return handle
@pytest.fixture
def isolated_eventmanager(monkeypatch):
"""隔离全局事件总线的订阅表和广播执行器。"""
monkeypatch.setattr(
global_vars,
"CURRENT_EVENT_LOOP",
global_vars.CURRENT_EVENT_LOOP,
)
monkeypatch.setattr(
eventmanager,
"_EventManager__broadcast_subscribers",
@@ -37,6 +76,16 @@ def isolated_eventmanager(monkeypatch):
"_EventManager__handler_instance_resolvers",
{},
)
monkeypatch.setattr(
eventmanager,
"_EventManager__disabled_handlers",
set(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__disabled_classes",
set(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__executor",
@@ -47,6 +96,11 @@ def isolated_eventmanager(monkeypatch):
"_EventManager__event",
threading.Event(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__event_queue",
PriorityQueue(),
)
monkeypatch.setattr(
eventmanager,
"_EventManager__consumer_threads",
@@ -57,6 +111,11 @@ def isolated_eventmanager(monkeypatch):
"_EventManager__lifecycle_state",
"new",
)
monkeypatch.setattr(
eventmanager,
"_EventManager__sync_handles",
{},
)
monkeypatch.setattr(
eventmanager,
"_EventManager__async_handles",
@@ -68,6 +127,7 @@ def isolated_eventmanager(monkeypatch):
def test_broadcast_dispatch_uses_subscription_snapshot(isolated_eventmanager):
"""广播事件中新增或移除的 handler 从下一个事件开始生效。"""
calls = []
isolated_eventmanager._EventManager__lifecycle_state = "running"
def late_handler(_event):
calls.append("late")
@@ -276,6 +336,354 @@ async def test_async_broadcast_shutdown_waits_for_handler_cleanup(
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
@pytest.mark.asyncio
async def test_drain_waits_for_slow_sync_broadcast_handler(
isolated_eventmanager,
monkeypatch,
) -> None:
"""同步广播处理器返回前 drain 不得提前宣称结算完成。"""
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
monkeypatch.setattr(
isolated_eventmanager,
"_EventManager__executor",
executor,
)
started = threading.Event()
release = threading.Event()
def handler(_event) -> None:
started.set()
release.wait(timeout=2)
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"drain-sync"}},
)
assert await asyncio.to_thread(started.wait, 1)
assert len(isolated_eventmanager._EventManager__sync_handles) == 1
drain_task = asyncio.create_task(
isolated_eventmanager.drain_async(timeout=1)
)
await asyncio.sleep(0.02)
assert not drain_task.done()
release.set()
assert await drain_task is True
assert isolated_eventmanager._EventManager__sync_handles == {}
assert (
isolated_eventmanager._EventManager__event_queue.unfinished_tasks
== 0
)
finally:
release.set()
await isolated_eventmanager.stop_async()
executor.shutdown(wait=True)
@pytest.mark.asyncio
async def test_drain_timeout_does_not_cancel_async_broadcast_handler(
isolated_eventmanager,
) -> None:
"""drain 超时只报告未收敛,不取消仍在执行的异步业务处理器。"""
global_vars.set_loop(asyncio.get_running_loop())
started = asyncio.Event()
release = asyncio.Event()
cancelled = False
async def handler(_event) -> None:
nonlocal cancelled
started.set()
try:
await release.wait()
except asyncio.CancelledError:
cancelled = True
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"drain-async"}},
)
await asyncio.wait_for(started.wait(), timeout=1)
assert await isolated_eventmanager.drain_async(timeout=0.01) is False
assert cancelled is False
assert isolated_eventmanager._EventManager__async_handles
release.set()
assert await isolated_eventmanager.drain_async(timeout=1) is True
assert cancelled is False
finally:
release.set()
await isolated_eventmanager.stop_async()
@pytest.mark.asyncio
async def test_sealed_drain_waits_for_cascade_and_rejects_later_broadcasts(
isolated_eventmanager,
) -> None:
"""封口 drain 应结算 handler 派生事件,并拒绝封口后的新广播。"""
calls = []
def first_handler(_event) -> None:
calls.append("first")
isolated_eventmanager.send_event(EventType.ModuleReload, {})
def cascaded_handler(_event) -> None:
calls.append("cascaded")
isolated_eventmanager.add_event_listener(
EventType.ConfigChanged,
first_handler,
)
isolated_eventmanager.add_event_listener(
EventType.ModuleReload,
cascaded_handler,
)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"cascade"}},
)
assert await isolated_eventmanager.drain_async(
timeout=1,
seal=True,
) is True
assert calls == ["first", "cascaded"]
queue_size = isolated_eventmanager._EventManager__event_queue.qsize()
isolated_eventmanager.send_event(EventType.ModuleReload, {})
await asyncio.sleep(0.02)
assert calls == ["first", "cascaded"]
assert (
isolated_eventmanager._EventManager__event_queue.qsize()
== queue_size
)
assert isolated_eventmanager._EventManager__lifecycle_state == "sealed"
finally:
await isolated_eventmanager.stop_async()
@pytest.mark.asyncio
async def test_drain_returns_false_when_stop_interleaves(
isolated_eventmanager,
monkeypatch,
) -> None:
"""stop 开始后 drain 必须报告屏障失败,而非等待残留队列后返回成功。"""
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
monkeypatch.setattr(
isolated_eventmanager,
"_EventManager__executor",
executor,
)
started = threading.Event()
release = threading.Event()
def handler(_event) -> None:
started.set()
release.wait(timeout=2)
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"stop-race"}},
)
assert await asyncio.to_thread(started.wait, 1)
drain_task = asyncio.create_task(isolated_eventmanager.drain_async())
stop_task = asyncio.create_task(isolated_eventmanager.stop_async())
assert await asyncio.wait_for(drain_task, timeout=1) is False
assert not stop_task.done()
release.set()
await asyncio.wait_for(stop_task, timeout=1)
assert isolated_eventmanager._EventManager__sync_handles == {}
finally:
release.set()
if isolated_eventmanager._EventManager__lifecycle_state != "stopped":
await isolated_eventmanager.stop_async()
executor.shutdown(wait=True)
@pytest.mark.asyncio
async def test_repeated_async_stop_balances_stop_sentinels(
isolated_eventmanager,
) -> None:
"""重复异步停止不得遗留哨兵或破坏队列 unfinished_tasks 计数。"""
isolated_eventmanager.start()
await asyncio.gather(
isolated_eventmanager.stop_async(),
isolated_eventmanager.stop_async(),
)
await isolated_eventmanager.stop_async()
assert isolated_eventmanager._EventManager__event_queue.qsize() == 0
assert isolated_eventmanager._EventManager__event_queue.unfinished_tasks == 0
@pytest.mark.asyncio
async def test_sync_stop_waits_for_owned_sync_handler(
isolated_eventmanager,
monkeypatch,
) -> None:
"""同步兼容停止入口也不得遗留仍运行的同步广播处理器。"""
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
monkeypatch.setattr(
isolated_eventmanager,
"_EventManager__executor",
executor,
)
started = threading.Event()
release = threading.Event()
def handler(_event) -> None:
started.set()
release.wait(timeout=2)
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"sync-stop"}},
)
assert await asyncio.to_thread(started.wait, 1)
stop_task = asyncio.create_task(
asyncio.to_thread(isolated_eventmanager.stop)
)
await asyncio.sleep(0.02)
assert not stop_task.done()
release.set()
await asyncio.wait_for(stop_task, timeout=1)
assert isolated_eventmanager._EventManager__sync_handles == {}
assert (
isolated_eventmanager._EventManager__event_queue.unfinished_tasks
== 0
)
finally:
release.set()
if isolated_eventmanager._EventManager__lifecycle_state != "stopped":
await isolated_eventmanager.stop_async()
executor.shutdown(wait=True)
def test_sync_broadcast_handler_can_call_legacy_stop_without_self_wait(
isolated_eventmanager,
monkeypatch,
) -> None:
"""同步 handler 调用旧 stop() 时不得等待承载自己的 Future。"""
executor = _DaemonThreadExecutor()
monkeypatch.setattr(
isolated_eventmanager,
"_EventManager__executor",
executor,
)
stopped = threading.Event()
def handler(_event) -> None:
isolated_eventmanager.stop()
stopped.set()
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"handler-stop"}},
)
assert stopped.wait(timeout=1)
assert executor.finished.wait(timeout=1)
assert isolated_eventmanager._EventManager__sync_handles == {}
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
finally:
if isolated_eventmanager._EventManager__lifecycle_state == "running":
consumer_threads = isolated_eventmanager._EventManager__begin_stop()
for consumer_thread in consumer_threads:
consumer_thread.join(timeout=1)
isolated_eventmanager._EventManager__discard_stop_sentinels()
@pytest.mark.asyncio
async def test_async_broadcast_handler_can_await_stop_without_self_cancel(
isolated_eventmanager,
) -> None:
"""异步 handler 调用 stop_async() 时不得取消或等待自身 completion。"""
global_vars.set_loop(asyncio.get_running_loop())
stopped = asyncio.Event()
cancelled = False
async def handler(_event) -> None:
nonlocal cancelled
try:
await isolated_eventmanager.stop_async()
stopped.set()
except asyncio.CancelledError:
cancelled = True
raise
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"async-handler-stop"}},
)
await asyncio.wait_for(stopped.wait(), timeout=1)
async def wait_for_owner_release() -> None:
"""等待 handler 返回后的 completion 回调移除 owner。"""
while isolated_eventmanager._EventManager__async_handles:
await asyncio.sleep(0)
await asyncio.wait_for(wait_for_owner_release(), timeout=1)
assert cancelled is False
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
@pytest.mark.asyncio
async def test_broadcast_handler_cannot_seal_while_own_completion_is_pending(
isolated_eventmanager,
) -> None:
"""handler 内 drain 应立即失败,且不能把仍运行的事件总线伪装成已封口。"""
global_vars.set_loop(asyncio.get_running_loop())
drain_results = []
handler_done = asyncio.Event()
async def handler(_event) -> None:
drain_results.append(
await isolated_eventmanager.drain_async(seal=True)
)
handler_done.set()
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
isolated_eventmanager.start()
try:
isolated_eventmanager.send_event(
EventType.ConfigChanged,
{"key": {"handler-drain"}},
)
await asyncio.wait_for(handler_done.wait(), timeout=1)
assert drain_results == [False]
assert isolated_eventmanager._EventManager__lifecycle_state == "running"
assert await isolated_eventmanager.drain_async(timeout=1) is True
finally:
await isolated_eventmanager.stop_async()
@pytest.mark.asyncio
async def test_async_broadcast_submission_is_registered_before_stop_snapshot(
isolated_eventmanager,
+438 -20
View File
@@ -8,6 +8,7 @@ from fastapi import FastAPI
from app.startup import lifecycle, modules_initializer
from app.adapters.network import http as http_utils
from app.runtime.tasks import get_task_registry
def _assert_completed_once(mock: MagicMock) -> None:
@@ -59,11 +60,16 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
shutdown_steps = {
"backup_plugins": system_chain.backup_plugins,
"stop_plugin_monitor": MagicMock(return_value=True),
"stop_workflow": MagicMock(),
"stop_command": MagicMock(),
"stop_monitor": MagicMock(),
"stop_scheduler": MagicMock(),
"stop_plugins": MagicMock(),
"stop_agent": AsyncMock(return_value=True),
"stop_transfer": AsyncMock(return_value=True),
"quiesce_plugins": AsyncMock(return_value=True),
"drain_events": AsyncMock(return_value=True),
"finalize_plugins": MagicMock(return_value=True),
"stop_modules": AsyncMock(),
"close_http": AsyncMock(),
}
@@ -72,9 +78,22 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_plugins",
"stop_plugin_monitor",
"finalize_plugins",
):
monkeypatch.setattr(lifecycle, name, shutdown_steps[name])
monkeypatch.setattr(lifecycle, "stop_agent", shutdown_steps["stop_agent"])
monkeypatch.setattr(
lifecycle,
"stop_transfer_runtime",
shutdown_steps["stop_transfer"],
)
monkeypatch.setattr(
lifecycle,
"quiesce_plugins",
shutdown_steps["quiesce_plugins"],
)
monkeypatch.setattr(lifecycle, "drain_events", shutdown_steps["drain_events"])
monkeypatch.setattr(lifecycle, "stop_modules", shutdown_steps["stop_modules"])
monkeypatch.setattr(
lifecycle,
@@ -99,9 +118,6 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_plugins",
"stop_modules",
"close_http",
],
@@ -153,8 +169,258 @@ def test_lifespan_normal_mode_starts_full_runtime(monkeypatch):
_assert_completed_once(step)
def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch):
"""关停必须等待插件恢复线程结束,避免与备份和资源释放并发。"""
@pytest.mark.parametrize(
("failing_step", "completed_steps", "blocked_steps"),
[
(
"stop_plugin_monitor",
("stop_plugin_monitor",),
(
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_monitor",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
),
(
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_scheduler",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
),
(
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_agent",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
),
(
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"quiesce_plugins",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
),
(
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_transfer",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
),
("drain_events", "finalize_plugins", "stop_modules", "close_http"),
),
(
"drain_events",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
),
("finalize_plugins", "stop_modules", "close_http"),
),
(
"finalize_plugins",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
),
("stop_modules", "close_http"),
),
],
)
def test_lifespan_stops_releasing_dependencies_when_owner_does_not_converge(
monkeypatch,
failing_step,
completed_steps,
blocked_steps,
):
"""关键 owner 未收敛时不得关闭仍被活任务使用的后续依赖。"""
shutdown_steps = _patch_lifespan(monkeypatch)
shutdown_steps[failing_step].return_value = False
async def run_lifespan():
"""启动并关闭隔离后的应用生命周期。"""
async with lifecycle.lifespan(FastAPI()):
pass
asyncio.run(run_lifespan())
for name in (*completed_steps, "logger"):
_assert_completed_once(shutdown_steps[name])
for name in blocked_steps:
shutdown_steps[name].assert_not_called()
def test_task_registry_nonconvergence_blocks_all_dependency_release(monkeypatch):
"""最前置任务 owner 超时后不得继续释放插件、模块或 HTTP 依赖。"""
shutdown_steps = _patch_lifespan(monkeypatch)
shutdown = AsyncMock(return_value=False)
monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown)
app = FastAPI()
async def run_lifespan() -> None:
"""运行后台登记器无法收敛的隔离生命周期。"""
async with lifecycle.lifespan(app):
pass
asyncio.run(run_lifespan())
shutdown.assert_awaited_once_with(timeout_seconds=30.0)
for name, step in shutdown_steps.items():
if name == "logger":
_assert_completed_once(step)
else:
step.assert_not_called()
assert isinstance(app.state.task_registry, lifecycle.TaskRegistry)
def test_closed_task_registry_rejects_late_shutdown_tasks(monkeypatch) -> None:
"""首屏障完成后,后续 stop hook 的晚到任务不得落回默认登记器。"""
_patch_lifespan(monkeypatch)
app = FastAPI()
async def run_lifespan() -> None:
"""结束完整 lifespan 后验证当前发布的仍是已封口登记器。"""
async with lifecycle.lifespan(app):
pass
registry = get_task_registry()
assert registry is app.state.task_registry
with pytest.raises(RuntimeError, match="正在关闭"):
registry.create(asyncio.sleep(0), owner="shutdown.late_task")
asyncio.run(run_lifespan())
def test_plugin_settlement_cannot_bypass_task_registry_shutdown_budget(
monkeypatch,
) -> None:
"""未收敛 settlement 必须交给首屏障判定,lifespan 不得提前无界等待。"""
shutdown_steps = _patch_lifespan(monkeypatch)
shutdown = AsyncMock(return_value=False)
monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown)
started = asyncio.Event()
release = asyncio.Event()
async def settle_plugins() -> None:
"""模拟停机时仍未结束的插件同步任务。"""
started.set()
await release.wait()
lifecycle.init_extra.side_effect = settle_plugins
async def run_lifespan() -> None:
"""确认 context 能由 TaskRegistry 的失败结果立即结束。"""
async with lifecycle.lifespan(FastAPI()):
await started.wait()
release.set()
await asyncio.sleep(0)
asyncio.run(asyncio.wait_for(run_lifespan(), timeout=0.5))
shutdown.assert_awaited_once_with(timeout_seconds=30.0)
for name, step in shutdown_steps.items():
if name == "logger":
_assert_completed_once(step)
else:
step.assert_not_called()
def test_lifespan_waits_for_uncancellable_plugin_settlement_before_shutdown(
monkeypatch,
):
"""已进入同步 I/O 的 settlement 必须真实结束,才能备份和释放资源。"""
shutdown_steps = _patch_lifespan(monkeypatch)
order = []
shutdown_steps["backup_plugins"].side_effect = lambda: order.append("backup")
@@ -165,7 +431,12 @@ def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch):
async def settle_plugins():
started.set()
await release.wait()
try:
await release.wait()
except asyncio.CancelledError:
# 模拟 run_in_threadpool_to_completion:外层取消只能封住新工作,
# 已进入同步插件源码/依赖修改的调用仍持有 owner 到真实终态。
await release.wait()
order.append("settled")
lifecycle.init_extra.side_effect = settle_plugins
@@ -225,14 +496,31 @@ def test_lifespan_safe_mode_skips_optional_runtime(monkeypatch):
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_plugins",
"stop_plugin_monitor",
"quiesce_plugins",
"finalize_plugins",
):
shutdown_steps[name].assert_not_called()
_assert_completed_once(shutdown_steps["stop_modules"])
_assert_completed_once(shutdown_steps["stop_agent"])
_assert_completed_once(shutdown_steps["stop_transfer"])
_assert_completed_once(shutdown_steps["drain_events"])
_assert_completed_once(shutdown_steps["close_http"])
_assert_completed_once(shutdown_steps["logger"])
@pytest.mark.asyncio
async def test_event_drain_does_not_materialize_manager(monkeypatch) -> None:
"""模块尚未创建事件总线时,停机屏障应直接收敛而不反向构造。"""
event_manager_type = MagicMock()
event_manager_type.get_existing_instance.return_value = None
monkeypatch.setattr(modules_initializer, "EventManager", event_manager_type)
assert await modules_initializer.drain_events() is True
event_manager_type.get_existing_instance.assert_called_once_with()
event_manager_type.assert_not_called()
def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"""组件清单应显式冻结依赖、模式、启动/关闭顺序和超时预算。"""
app = FastAPI()
@@ -274,11 +562,16 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
]
assert normal_stop == [
"后台任务登记器",
"插件变更监控",
"插件备份",
"工作流",
"命令服务",
"监控器",
"定时器",
"AI智能体会话",
"插件后台服务",
"整理后台服务",
"事件投递屏障",
"插件",
"模块服务",
"HTTP 基础能力",
@@ -292,9 +585,42 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"数据库连接预算",
"路由",
"模块服务",
"AI智能体会话",
"整理后台服务",
"事件投递屏障",
}
assert all(item["start_failure"] == "fail_fast" for item in normal)
assert all(item["stop_failure"] == "continue" for item in normal)
assert {
item["name"]
for item in normal
if item["stop_failure"] == "fail_fast"
} == {
"插件变更监控",
"后台任务登记器",
"监控器",
"定时器",
"AI智能体会话",
"整理后台服务",
"插件后台服务",
"事件投递屏障",
"插件",
}
assert all(
item["stop_failure"] == "continue"
for item in normal
if item["name"]
not in {
"插件变更监控",
"后台任务登记器",
"监控器",
"定时器",
"AI智能体会话",
"整理后台服务",
"插件后台服务",
"事件投递屏障",
"插件",
}
)
assert all(
item["start_timeout_seconds"] or item["stop_timeout_seconds"]
for item in normal
@@ -496,6 +822,80 @@ def test_lifespan_does_not_yield_after_migration_failure(monkeypatch):
assert app.state.moviepilot_health.phase.value == "failed"
def test_lifespan_cleans_started_owners_after_late_startup_failure(monkeypatch):
"""后段启动失败时应按同一停机策略回收已启动及部分启动的 owner。"""
shutdown_steps = _patch_lifespan(monkeypatch)
startup_error = RuntimeError("command startup failed")
lifecycle.init_command.side_effect = startup_error
app = FastAPI()
async def run_lifespan() -> None:
"""运行一个在命令服务阶段失败的隔离生命周期。"""
async with lifecycle.lifespan(app):
pytest.fail("命令服务启动失败后不应发布运行态")
with pytest.raises(RuntimeError) as raised:
asyncio.run(run_lifespan())
assert raised.value is startup_error
lifecycle.global_vars.stop_system.assert_not_called()
for name in (
"stop_plugin_monitor",
"backup_plugins",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
):
_assert_completed_once(shutdown_steps[name])
shutdown_steps["stop_workflow"].assert_not_called()
shutdown_steps["logger"].assert_not_called()
assert isinstance(app.state.task_registry, lifecycle.TaskRegistry)
assert get_task_registry() is app.state.task_registry
assert app.state.moviepilot_health.phase.value == "failed"
def test_startup_failure_cleanup_honors_transfer_fail_fast(monkeypatch):
"""启动失败清理中整理 owner 未收敛时也不得继续释放插件和模块。"""
shutdown_steps = _patch_lifespan(monkeypatch)
lifecycle.init_command.side_effect = RuntimeError("command startup failed")
shutdown_steps["stop_transfer"].return_value = False
async def run_lifespan() -> None:
"""运行后段失败且整理线程无法收敛的隔离生命周期。"""
async with lifecycle.lifespan(FastAPI()):
pytest.fail("命令服务启动失败后不应发布运行态")
with pytest.raises(RuntimeError, match="command startup failed"):
asyncio.run(run_lifespan())
for name in (
"stop_plugin_monitor",
"backup_plugins",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
):
_assert_completed_once(shutdown_steps[name])
for name in (
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
):
shutdown_steps[name].assert_not_called()
shutdown_steps["stop_workflow"].assert_not_called()
def test_uvicorn_preserves_stop_requested_before_serve(monkeypatch):
"""Uvicorn 启动不能清除数据库初始化阶段已经发布的停止请求"""
from app import main
@@ -564,16 +964,13 @@ def test_command_restart_failure_does_not_publish_stop_request(monkeypatch):
assert not stop_event.is_set()
def test_stop_modules_continues_after_internal_owner_failures(monkeypatch):
"""模块关闭编排中的个失败不能阻断其余清理"""
stop_agent = AsyncMock(side_effect=RuntimeError("agent failed"))
monkeypatch.setattr(modules_initializer, "stop_agent", stop_agent)
def test_stop_modules_continues_after_internal_owner_failure(monkeypatch):
"""模块关闭编排中的个失败不能阻断其余清理"""
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
dependencies["module"].side_effect = RuntimeError("module failed")
asyncio.run(modules_initializer.stop_modules())
stop_agent.assert_awaited_once_with()
for dependency in dependencies.values():
_assert_completed_once(dependency)
@@ -581,7 +978,6 @@ def test_stop_modules_continues_after_internal_owner_failures(monkeypatch):
def test_stop_modules_drains_web_agent_tasks_before_persistence(monkeypatch):
"""关闭时先收口 Web Agent,再关闭持久化准入和数据库任务。"""
order = []
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
monkeypatch.setattr(
modules_initializer,
@@ -619,7 +1015,6 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
started.set()
await asyncio.Event().wait()
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
_patch_module_shutdown_dependencies(monkeypatch)
monkeypatch.setattr(
modules_initializer,
@@ -651,8 +1046,9 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
)
)
await started.wait()
await shutdown
completed = await shutdown
assert completed is False
stop_database_worker.assert_awaited_once_with()
@@ -683,9 +1079,10 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No
)
)
await started.wait()
await shutdown
completed = await shutdown
elapsed = asyncio.get_running_loop().time() - started_at
assert completed is False
assert elapsed < 0.2
await asyncio.wait_for(cancel_requested.wait(), timeout=0.2)
assert not settled.is_set()
@@ -694,6 +1091,28 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No
await asyncio.wait_for(settled.wait(), timeout=0.2)
@pytest.mark.asyncio
async def test_shutdown_step_reports_explicit_nonconvergence() -> None:
"""同步和异步 owner 显式返回 False 时都必须向生命周期传播失败。"""
async def async_nonconverging_shutdown() -> bool:
"""模拟已经完成等待但仍持有资源的异步关闭入口。"""
return False
assert await lifecycle.run_shutdown_step(
"同步 owner",
lambda: False,
) is False
assert await lifecycle.run_shutdown_step(
"异步 owner",
async_nonconverging_shutdown,
) is False
assert await lifecycle.run_shutdown_step(
"已收敛 owner",
lambda: None,
) is True
def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
"""替换 stop_modules 的资源所有者,避免测试启动真实后台服务"""
dependencies = {}
@@ -749,7 +1168,6 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
def test_browser_sessions_close_before_managed_resources(monkeypatch) -> None:
"""显示等宿主资源必须晚于浏览器会话释放,避免存活上下文失去依赖。"""
calls: list[str] = []
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
dependencies["close_browser_sessions"].side_effect = lambda: calls.append("browser")
+14 -1
View File
@@ -1,14 +1,16 @@
import os
import threading
import time
from unittest.mock import MagicMock
from watchfiles import Change
from app.runtime.config import settings
from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.dispatcher import TransferDispatcher
from app.monitor.monitor import Monitor
from app.monitor.recovery import RecoveryExecutor
from app.monitor.syslimits import decide_monitor_mode
from app.monitor.watcher import LocalDirectoryWatcher
from app.adapters.system.host import SystemUtils
@@ -24,13 +26,24 @@ def _build_monitor(handle_file: MagicMock = None):
if handle_file is not None:
dispatcher.handle_file = handle_file
monitor._dispatcher = dispatcher
monitor._lifecycle_lock = threading.RLock()
monitor._owner_lock = threading.Lock()
monitor._work_stop_event = threading.Event()
monitor._shutdown_event = threading.Event()
monitor._closed = False
monitor._compensation_threads = {}
monitor._scheduler_shutdown_thread = None
monitor._scheduler_shutdown_succeeded = False
monitor._scheduler = None
monitor._watchers = []
monitor._retired_watchers = []
monitor._watcher_lock = Lock()
monitor._alerted_paths = {}
monitor._restart_marks = {}
monitor._stable_cycles = {}
monitor._isolated = {}
monitor._pending_rebuild = {}
monitor._pending_locals = []
monitor._recovery = RecoveryExecutor()
return monitor, dispatcher
+345
View File
@@ -0,0 +1,345 @@
"""目录监控 owner 的生命周期与停机屏障回归。"""
import asyncio
import threading
import time
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from app.monitor.monitor import Monitor
from app.monitor.recovery import RecoveryExecutor, RecoveryState
from app.foundation.singleton import SingletonClass
from app.startup.monitor_initializer import init_monitor, stop_monitor
def _build_monitor() -> Monitor:
"""构造绕过单例初始化、但具备完整生命周期字段的 Monitor 骨架。"""
monitor = object.__new__(Monitor)
monitor._lifecycle_lock = threading.RLock()
monitor._owner_lock = threading.Lock()
monitor._work_stop_event = threading.Event()
monitor._shutdown_event = threading.Event()
monitor._closed = False
monitor._compensation_threads = {}
monitor._scheduler_shutdown_thread = None
monitor._scheduler_shutdown_succeeded = False
monitor._scheduler = None
monitor._watchers = []
monitor._retired_watchers = []
monitor._watcher_lock = threading.Lock()
monitor._pending_locals = []
monitor._alerted_paths = {}
monitor._restart_marks = {}
monitor._stable_cycles = {}
monitor._isolated = {}
monitor._pending_rebuild = {}
monitor._recovery = RecoveryExecutor()
monitor._dispatcher = MagicMock()
return monitor
def _blocking_watcher(mon_path: Path):
"""构造只有显式 release 后才退出的 watcher 与控制事件。"""
started = threading.Event()
release = threading.Event()
def run() -> None:
"""模拟无法被 stop event 唤醒的 FUSE watcher。"""
started.set()
release.wait()
thread = threading.Thread(target=run, daemon=True, name="test-monitor-watcher")
thread.start()
assert started.wait(1)
watcher = MagicMock()
watcher.watch_path = mon_path
watcher.stop.side_effect = lambda: None
watcher.join.side_effect = thread.join
watcher.is_alive.side_effect = thread.is_alive
return watcher, thread, release
def test_close_budget_includes_lifecycle_lock_wait() -> None:
"""停机 deadline 必须先于生命周期锁等待建立,锁竞争超时后立即返回。"""
monitor = _build_monitor()
lock_acquired = threading.Event()
release_lock = threading.Event()
def hold_lifecycle_lock() -> None:
"""占住生命周期锁,模拟配置重载阻塞在挂载访问中。"""
with monitor._lifecycle_lock:
lock_acquired.set()
release_lock.wait()
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
holder.start()
assert lock_acquired.wait(1)
started_at = time.monotonic()
try:
assert monitor.close(timeout=0.02) is False
assert time.monotonic() - started_at < 0.5
assert monitor.lifecycle_closed is True
finally:
release_lock.set()
holder.join(timeout=1)
assert holder.is_alive() is False
assert monitor.close(timeout=1) is True
def test_close_timeout_retains_watcher_and_seals_config_reload(tmp_path) -> None:
"""挂死 watcher 超时时保留句柄,且配置变化不能重开已封口生命周期。"""
monitor = _build_monitor()
watcher, thread, release = _blocking_watcher(tmp_path)
monitor._watchers = [watcher]
reload_monitor = MagicMock()
monitor.init = reload_monitor
try:
assert monitor.close(timeout=0.02) is False
assert monitor._watchers == [watcher]
monitor._dispatcher.clear_pending.assert_not_called()
monitor.on_config_changed()
reload_monitor.assert_not_called()
finally:
release.set()
assert monitor.close(timeout=1) is True
thread.join(timeout=1)
assert thread.is_alive() is False
assert monitor._watchers == []
monitor._dispatcher.clear_pending.assert_called_once_with()
def test_reopen_allows_only_an_explicit_new_lifespan() -> None:
"""close 后普通 init 被拒绝,显式 reopen 收敛旧 owner 后才允许重新初始化。"""
monitor = _build_monitor()
initialize = MagicMock(return_value=True)
monitor._Monitor__initialize_monitors = initialize
assert monitor.close(timeout=1) is True
assert monitor.init(timeout=0) is False
initialize.assert_not_called()
assert monitor.reopen(timeout=1) is True
assert monitor.init(timeout=1) is True
initialize.assert_called_once_with()
assert monitor.lifecycle_closed is False
def test_close_tracks_recovery_and_blocks_post_seal_dispatch() -> None:
"""恢复动作解冻后只能退出,不能越过封口继续派发整理重试。"""
monitor = _build_monitor()
entered = threading.Event()
release = threading.Event()
def blocking_local_retry() -> None:
"""模拟恢复线程阻塞在 FUSE 本地目录访问。"""
entered.set()
release.wait()
monitor._Monitor__retry_pending_locals = blocking_local_retry
result = monitor._recovery.run(
{monitor.PENDING_KEY: monitor._Monitor__drive_pending}, timeout=0.01
)
assert entered.is_set()
assert result == {monitor.PENDING_KEY: RecoveryState.TIMEOUT}
try:
assert monitor.close(timeout=0.02) is False
assert len(monitor._recovery.running_threads()) == 1
monitor._dispatcher.retry_pending.assert_not_called()
finally:
release.set()
assert monitor.close(timeout=1) is True
assert monitor._recovery.running_threads() == ()
monitor._dispatcher.retry_pending.assert_not_called()
def test_recovery_start_and_registration_are_atomic(monkeypatch) -> None:
"""close 不得在恢复线程登记后、真正 start 前把它误判为已收敛。"""
executor = RecoveryExecutor()
start_entered = threading.Event()
allow_start = threading.Event()
close_results: list[bool] = []
original_start = threading.Thread.start
def delayed_recovery_start(thread: threading.Thread) -> None:
"""只暂停恢复线程的 start,稳定放大登记与启动之间的竞态窗口。"""
if thread.name.startswith("MoviePilot-MonitorRecovery-"):
start_entered.set()
assert allow_start.wait(timeout=1)
original_start(thread)
monkeypatch.setattr(threading.Thread, "start", delayed_recovery_start)
runner = threading.Thread(
target=lambda: executor.run({"atomic": lambda: None}, timeout=1),
name="recovery-submit-test",
)
runner.start()
assert start_entered.wait(timeout=1)
closer = threading.Thread(
target=lambda: close_results.append(
executor.close(deadline=time.monotonic() + 1)
),
name="recovery-close-test",
)
closer.start()
closer.join(timeout=0.05)
assert closer.is_alive(), "close 越过了尚未完成 start 的 owner 临界区"
allow_start.set()
runner.join(timeout=1)
closer.join(timeout=1)
assert not runner.is_alive()
assert not closer.is_alive()
assert close_results == [True]
assert executor.running_threads() == ()
def test_close_tracks_compensation_and_blocks_post_seal_dispatch(tmp_path) -> None:
"""补偿扫描解冻后不得在旧 lifespan 中继续把文件送入整理链。"""
monitor = _build_monitor()
entered = threading.Event()
release = threading.Event()
candidate = tmp_path / "late.mkv"
def blocking_collect(_mon_path: Path):
"""模拟补偿扫描阻塞在目录遍历。"""
entered.set()
release.wait()
return [(candidate, time.time(), 1)]
monitor._Monitor__collect_compensation_files = blocking_collect
monitor._Monitor__start_compensation(mon_path=tmp_path, since=time.time())
assert entered.wait(1)
try:
assert monitor.close(timeout=0.02) is False
assert len(monitor._compensation_threads) == 1
monitor._dispatcher.handle_file.assert_not_called()
finally:
release.set()
assert monitor.close(timeout=1) is True
assert monitor._compensation_threads == {}
monitor._dispatcher.handle_file.assert_not_called()
def test_close_tracks_scheduler_shutdown_thread() -> None:
"""scheduler shutdown 阻塞时保留线程和 scheduler,恢复后才能清空句柄。"""
monitor = _build_monitor()
entered = threading.Event()
release = threading.Event()
scheduler = MagicMock()
scheduler.running = True
def blocking_shutdown(*, wait: bool) -> None:
"""模拟等待在途 APScheduler job 的同步 shutdown。"""
assert wait is True
entered.set()
release.wait()
scheduler.shutdown.side_effect = blocking_shutdown
monitor._scheduler = scheduler
try:
assert monitor.close(timeout=0.02) is False
assert entered.is_set()
assert monitor._scheduler is scheduler
assert monitor._scheduler_shutdown_thread is not None
assert monitor._scheduler_shutdown_thread.is_alive()
finally:
release.set()
assert monitor.close(timeout=1) is True
assert monitor._scheduler is None
assert monitor._scheduler_shutdown_thread is None
@pytest.mark.asyncio
async def test_stop_monitor_cancellation_waits_for_sync_close(monkeypatch) -> None:
"""生命周期取消异步 stop 后,线程池调用仍由任务持有到同步 close 结束。"""
entered = threading.Event()
release = threading.Event()
def blocking_close(timeout: float) -> bool:
"""模拟仍在同步收尾的 Monitor.close。"""
assert timeout == 1
entered.set()
release.wait()
return True
monitor = SimpleNamespace(close=blocking_close)
monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor)
task = asyncio.create_task(stop_monitor(timeout=1))
while not entered.is_set():
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0.01)
assert task.done() is False
release.set()
with pytest.raises(asyncio.CancelledError):
await task
def test_init_monitor_explicitly_reopens_existing_lifespan(monkeypatch) -> None:
"""同进程测试中的新 lifespan 必须显式 reopen,再初始化同一个 Monitor。"""
monitor = SimpleNamespace(
lifecycle_closed=True,
reopen=MagicMock(return_value=True),
init=MagicMock(return_value=True),
)
monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor)
init_monitor()
monitor.reopen.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT)
monitor.init.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT)
@pytest.mark.asyncio
async def test_constructor_failure_publishes_started_watcher_to_cleanup(
monkeypatch,
) -> None:
"""Monitor 构造中途失败后,stop-only 入口仍必须找到已启动 watcher owner。"""
instances = dict(SingletonClass._instances)
instances.pop(Monitor, None)
monkeypatch.setattr(SingletonClass, "_instances", instances)
watcher_started = threading.Event()
watcher_release = threading.Event()
watchers: list[threading.Thread] = []
def failing_init(monitor: Monitor) -> None:
"""模拟 watcher 已启动、后续 scheduler 启动失败的构造过程。"""
watcher = threading.Thread(
target=lambda: (watcher_started.set(), watcher_release.wait()),
name="monitor-partial-construction",
daemon=True,
)
watchers.append(watcher)
watcher.start()
def close(timeout: float) -> bool:
"""模拟真实 close 释放并等待半构造实例已经发布的 watcher。"""
watcher_release.set()
watcher.join(timeout=timeout)
return not watcher.is_alive()
monitor.close = close
raise RuntimeError("scheduler start failed")
monkeypatch.setattr(Monitor, "__init__", failing_init)
with pytest.raises(RuntimeError, match="scheduler start failed"):
Monitor()
assert watcher_started.wait(timeout=1)
retained = Monitor.get_existing_instance()
assert retained is not None
assert await stop_monitor(timeout=1) is True
assert watchers[0].is_alive() is False
+19 -4
View File
@@ -18,8 +18,9 @@ from unittest.mock import MagicMock
import pytest
from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.monitor import Monitor
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
from app.monitor.watcher import LocalDirectoryWatcher
def _build_monitor(monkeypatch, put_recorder=None):
@@ -33,7 +34,17 @@ def _build_monitor(monkeypatch, put_recorder=None):
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
monitor = object.__new__(Monitor)
monitor._dispatcher = MagicMock()
monitor._lifecycle_lock = threading.RLock()
monitor._owner_lock = threading.Lock()
monitor._work_stop_event = threading.Event()
monitor._shutdown_event = threading.Event()
monitor._closed = False
monitor._compensation_threads = {}
monitor._scheduler_shutdown_thread = None
monitor._scheduler_shutdown_succeeded = False
monitor._scheduler = None
monitor._watchers = []
monitor._retired_watchers = []
monitor._watcher_lock = Lock()
monitor._pending_locals = []
monitor._alerted_paths = {}
@@ -114,7 +125,7 @@ def _stop_monitor_threads(monitor, timeout=10.0):
:param monitor: Monitor 骨架
:param timeout: 每个线程的最长等待秒数
"""
for watcher in monitor._watchers:
for watcher in (*monitor._watchers, *monitor._retired_watchers):
if isinstance(watcher, LocalDirectoryWatcher):
watcher.stop()
watcher.join(timeout=timeout)
@@ -262,6 +273,10 @@ def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch
assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离"
finally:
_release_and_join(release, monitor._recovery._running.values())
assert watcher in monitor._retired_watchers
# 重建后的旧 watcher 仍由 retired owner 表持有;测试替身不会自行切换
# is_alive,释放模拟阻塞后显式推进到真实线程应有的退出终态。
watcher._thread.is_alive.return_value = False
_stop_monitor_threads(monitor)
@@ -454,7 +469,7 @@ def test_stuck_transfer_does_not_block_other_files(monkeypatch):
def test_recovery_executor_reports_timeout_without_blocking():
"""
永不返回的动作只应消耗一次 timeout执行器必须放弃它并如实报告
永不返回的动作只应消耗一次 timeout执行器必须结束本轮等待并如实报告
"""
executor = RecoveryExecutor()
release = threading.Event()
@@ -465,7 +480,7 @@ def test_recovery_executor_reports_timeout_without_blocking():
try:
assert results == {"stuck": RecoveryState.TIMEOUT}
assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作"
assert elapsed < 3.0, "执行器没有在超时后结束本轮等待"
finally:
_release_and_join(release, executor._running.values())
+13 -1
View File
@@ -1,8 +1,10 @@
import threading
from pathlib import Path
from unittest.mock import MagicMock
from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.monitor import Monitor
from app.monitor.recovery import RecoveryExecutor
from app.monitor.watcher import LocalDirectoryWatcher
def _build_watcher(tmp_path, force_polling):
@@ -133,7 +135,17 @@ def _build_monitor(monkeypatch, put_recorder):
monitor = object.__new__(Monitor)
# 自动重启后健康检查会发起补偿扫描,骨架需要一个分发器替身
monitor._dispatcher = MagicMock()
monitor._lifecycle_lock = threading.RLock()
monitor._owner_lock = threading.Lock()
monitor._work_stop_event = threading.Event()
monitor._shutdown_event = threading.Event()
monitor._closed = False
monitor._compensation_threads = {}
monitor._scheduler_shutdown_thread = None
monitor._scheduler_shutdown_succeeded = False
monitor._scheduler = None
monitor._watchers = []
monitor._retired_watchers = []
monitor._watcher_lock = Lock()
monitor._pending_locals = []
monitor._alerted_paths = {}
+13 -2
View File
@@ -1,11 +1,13 @@
import threading
from pathlib import Path
from unittest.mock import MagicMock
from watchfiles import Change
from app.monitor import DirectoryChangeEvent, LocalDirectoryWatcher, Monitor
from app.monitor.dispatcher import TransferDispatcher
from app.schemas import TransferDirectoryConf
from app.monitor.monitor import Monitor
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
from app.schemas.system import TransferDirectoryConf
from app.schemas.types import MediaType
@@ -42,6 +44,15 @@ def _build_monitor_with_dispatcher(handle_file: MagicMock = None):
if handle_file is not None:
dispatcher.handle_file = handle_file
monitor._dispatcher = dispatcher
monitor._lifecycle_lock = threading.RLock()
monitor._owner_lock = threading.Lock()
monitor._work_stop_event = threading.Event()
monitor._shutdown_event = threading.Event()
monitor._closed = False
monitor._compensation_threads = {}
monitor._scheduler_shutdown_thread = None
monitor._scheduler_shutdown_succeeded = False
monitor._scheduler = None
return monitor, dispatcher
+29 -1
View File
@@ -1,7 +1,15 @@
from contextlib import nullcontext
from app.application.plugin.config import PluginConfigCommand
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCommand:
def _command(
calls: list[tuple],
*,
save_result: bool = True,
mutation=None,
) -> PluginConfigCommand:
"""构造记录端口调用顺序的插件配置用例。"""
return PluginConfigCommand(
save_config=lambda plugin_id, config, force: (
@@ -22,6 +30,7 @@ def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCom
refresh_registrations=lambda plugin_id: calls.append(
("registrations", plugin_id)
),
mutation=mutation or (lambda _operation: nullcontext()),
)
@@ -65,3 +74,22 @@ def test_reset_preserves_compensation_cleanup_and_reload_order() -> None:
("reload", "DemoPlugin"),
("registrations", "DemoPlugin"),
]
def test_sealed_config_command_rejects_before_first_side_effect() -> None:
"""配置事务在 admission 封口后返回失败,且不得先发布 reset 事件。"""
calls: list[tuple] = []
admission = PluginMutationAdmission()
admission.seal()
update_result = _command(calls, mutation=admission.hold).update(
"DemoPlugin",
{"enabled": True},
)
reset_result = _command(calls, mutation=admission.hold).reset("DemoPlugin")
assert update_result.success is False
assert reset_result.success is False
assert "停机阶段" in update_result.message
assert "停机阶段" in reset_result.message
assert calls == []
+79
View File
@@ -1,4 +1,5 @@
import asyncio
from contextlib import nullcontext
from unittest.mock import ANY, AsyncMock, MagicMock, patch
from app.api.endpoints import plugin as plugin_endpoint
@@ -13,6 +14,7 @@ from app.api.endpoints.plugin import uninstall_plugin
from app.api.endpoints.system import sync_plugin_market_from_wiki
from app.application.plugin.config import PluginConfigCommand
from app.runtime.config import settings
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
from app.runtime.extensions.plugin_manager import PluginManager
from app.runtime.tasks import TaskRegistry
from app.schemas.event import PluginDataResetEventData
@@ -495,6 +497,7 @@ def test_reset_plugin_sends_pre_reset_chain_event_before_deleting_data():
reload_runtime=plugin_manager.reload_plugin,
publish_reset=publish_reset,
refresh_registrations=lambda _plugin_id: None,
mutation=lambda _operation: nullcontext(),
)
result = reset_plugin("SubscribeAssistantEnhanced", None, command)
@@ -592,6 +595,82 @@ def test_uninstall_virtual_instance_never_removes_source_package(monkeypatch):
plugin_manager.remove_plugin.assert_called_once_with("DemoPluginwork")
def test_sealed_http_uninstall_rejects_before_first_side_effect(monkeypatch):
"""HTTP 卸载在封口后明确失败,且不读取或写入插件持久化状态。"""
admission = PluginMutationAdmission()
admission.seal()
plugin_manager = MagicMock()
plugin_manager.mutation.side_effect = admission.hold
config_provider = MagicMock()
remove_api = MagicMock()
remove_job = MagicMock()
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
monkeypatch.setattr(
plugin_endpoint,
"get_configured_system_config",
config_provider,
)
monkeypatch.setattr(plugin_endpoint, "remove_plugin_api", remove_api)
monkeypatch.setattr(plugin_endpoint, "remove_plugin_job", remove_job)
result = uninstall_plugin("DemoPlugin", None)
assert result.success is False
assert "停机阶段" in result.message
plugin_manager.get_plugin_instance.assert_not_called()
config_provider.assert_not_called()
remove_api.assert_not_called()
remove_job.assert_not_called()
def test_sealed_http_clone_rejects_before_runtime_and_registration(monkeypatch):
"""HTTP 分身事务未获 admission 时不创建实例、不刷新注册或文件夹。"""
admission = PluginMutationAdmission()
admission.seal()
plugin_manager = MagicMock()
plugin_manager.mutation.side_effect = admission.hold
register = MagicMock()
add_to_folder = MagicMock()
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
monkeypatch.setattr(plugin_endpoint, "register_plugin", register)
monkeypatch.setattr(plugin_endpoint, "_add_clone_to_plugin_folder", add_to_folder)
result = plugin_endpoint.clone_plugin(
"DemoPlugin",
schemas.PluginCloneRequest(suffix="Work"),
None,
)
assert result.success is False
assert "停机阶段" in result.message
plugin_manager.clone_plugin.assert_not_called()
register.assert_not_called()
add_to_folder.assert_not_called()
def test_sealed_http_folder_update_rejects_before_config_access(monkeypatch):
"""插件文件夹写入口在封口后不读取或改写持久化配置。"""
admission = PluginMutationAdmission()
admission.seal()
plugin_manager = MagicMock()
plugin_manager.mutation.side_effect = admission.hold
config_provider = MagicMock()
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
monkeypatch.setattr(
plugin_endpoint,
"get_configured_system_config",
config_provider,
)
result = asyncio.run(
plugin_endpoint.update_folder_plugins("常用", ["DemoPlugin"], None)
)
assert result.success is False
assert "停机阶段" in result.message
config_provider.assert_not_called()
def test_delete_plugin_data_can_force_delete_after_plugin_is_stopped():
"""
重置入口会先停止插件插件数据删除不能依赖运行态注册仍存在
+31
View File
@@ -1,4 +1,5 @@
import asyncio
from contextlib import nullcontext
from unittest.mock import AsyncMock, Mock, patch
import pytest
@@ -9,6 +10,7 @@ from app.schemas.exception import (
PersistenceUnavailableError,
)
from app.application.plugin.install import PluginInstallCommand
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
def _command(
@@ -24,6 +26,8 @@ def _command(
checkpointer=None,
committer=None,
rollback=None,
mutation=None,
package_write_guard=None,
):
"""构造可观测每一步副作用的插件安装命令。"""
return PluginInstallCommand(
@@ -38,6 +42,9 @@ def _command(
install_reporter=reporter or AsyncMock(),
plugin_reloader=reloader or AsyncMock(),
registration_refresher=refresher or AsyncMock(),
mutation=mutation or (lambda _operation: nullcontext()),
package_write_guard=package_write_guard
or (lambda _plugin_id: nullcontext()),
)
@@ -74,6 +81,30 @@ async def test_install_failure_stops_before_report_persistence_and_reload():
reloader.assert_not_awaited()
@pytest.mark.asyncio
async def test_sealed_install_rejects_before_package_guard_and_checkpoint() -> None:
"""安装事务在封口后不进入监控抑制,也不创建文件快照。"""
admission = PluginMutationAdmission()
admission.seal()
package_guard = Mock(return_value=nullcontext())
checkpointer = AsyncMock()
result = await _command(
mutation=admission.hold,
package_write_guard=package_guard,
checkpointer=checkpointer,
).execute(
plugin_id="DemoPlugin",
repo_url="https://github.com/demo/plugins",
)
assert result.success is False
assert result.failure_stage == "admission"
assert "停机阶段" in result.message
package_guard.assert_not_called()
checkpointer.assert_not_awaited()
@pytest.mark.asyncio
async def test_success_records_completed_install_stages_in_order():
"""成功安装在提交文件快照后再执行非关键远程上报。"""
+223
View File
@@ -98,3 +98,226 @@ def test_lifecycle_records_load_failure_when_loader_returns_no_class():
assert result == {"DemoPlugin": PluginRuntimeStatus.LOAD_FAILED}
assert running == {}
assert statuses["DemoPlugin"] is PluginRuntimeStatus.LOAD_FAILED
def test_quiesce_keeps_instance_and_events_until_finalize():
"""第一阶段仅停止插件生产者,事件 handler 和实例留到屏障后释放。"""
order: list[str] = []
class DemoPlugin:
"""记录旧插件 ABI hook 调用顺序的测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件事件 handler 启用。"""
return True
def close(self):
"""停止插件私有计时器和 watcher。"""
order.append("close")
def stop_service(self):
"""停止插件服务并允许尾事件继续投递。"""
order.append("stop_service")
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
lifecycle._disable_events.reset_mock()
lifecycle._clear_modules.reset_mock()
lifecycle._clear_tools.reset_mock()
assert lifecycle.quiesce() is True
assert order == ["close", "stop_service"]
assert classes["DemoPlugin"] is DemoPlugin
assert isinstance(running["DemoPlugin"], DemoPlugin)
lifecycle._disable_events.assert_not_called()
lifecycle._clear_modules.assert_not_called()
lifecycle._clear_tools.assert_not_called()
assert lifecycle.finalize() is True
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
lifecycle._clear_modules.assert_called_once_with(None)
lifecycle._clear_tools.assert_called_once_with()
assert classes == {}
assert running == {}
def test_quiesce_runs_stop_service_after_close_failure_and_retries_missing_hook():
"""close 异常不能跳过 stop_service,重试时只补偿尚未成功的 hook。"""
order: list[str] = []
close_fails = [True]
class DemoPlugin:
"""首次 close 失败、stop_service 正常完成的测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件为启用状态。"""
return True
def close(self):
"""首次调用模拟插件私有资源关闭异常。"""
order.append("close")
if close_fails[0]:
raise RuntimeError("close failed")
def stop_service(self):
"""记录旧 ABI 的第二个停机 hook。"""
order.append("stop_service")
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
assert lifecycle.quiesce() is False
assert order == ["close", "stop_service"]
assert lifecycle.finalize() is False
assert "DemoPlugin" in classes
assert "DemoPlugin" in running
close_fails[0] = False
assert lifecycle.quiesce() is True
assert order == ["close", "stop_service", "close"]
assert lifecycle.finalize() is True
assert classes == {}
assert running == {}
def test_legacy_stop_entry_remains_idempotent():
"""旧 stop 先解绑事件、保持 None 返回,且重复调用不重复执行 hook。"""
order: list[str] = []
class DemoPlugin:
"""提供可计数停机 hook 的兼容测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件为启用状态。"""
return True
def close(self):
"""记录 close 调用。"""
order.append("close")
def stop_service(self):
"""记录 stop_service 调用。"""
order.append("stop_service")
lifecycle, _classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
lifecycle._disable_events.reset_mock()
lifecycle._disable_events.side_effect = (
lambda _plugin_type: order.append("disable_events")
)
assert lifecycle.stop() is None
assert lifecycle.stop() is None
assert order == ["disable_events", "close", "stop_service"]
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
assert running == {}
def test_legacy_stop_force_finalizes_after_hook_failure():
"""旧 stop 即使 hook 报错也释放实例,并保持历史 None 返回 ABI。"""
order: list[str] = []
class DemoPlugin:
"""close 失败但仍应完成兼容卸载的测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件为启用状态。"""
return True
def close(self):
"""模拟旧插件关闭异常。"""
order.append("close")
raise RuntimeError("close failed")
def stop_service(self):
"""证明 close 异常不会跳过后续 hook。"""
order.append("stop_service")
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
lifecycle._disable_events.reset_mock()
lifecycle._clear_modules.reset_mock()
assert lifecycle.stop("DemoPlugin") is None
assert order == ["close", "stop_service"]
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
lifecycle._clear_modules.assert_called_once_with("DemoPlugin")
assert classes == {}
assert running == {}
def test_legacy_reload_continues_after_old_instance_hook_failure():
"""旧实例 hook 失败只影响诊断,不得阻止 reload 创建新实例。"""
instances: list[object] = []
stop_service = MagicMock()
class DemoPlugin:
"""记录每次构造并让 close 持续失败的重载测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def __init__(self):
"""记录新建的运行实例。"""
instances.append(self)
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件为启用状态。"""
return True
@staticmethod
def close():
"""模拟旧实例退出异常。"""
raise RuntimeError("close failed")
def stop_service(self):
"""记录旧 ABI 的第二个停机 hook。"""
stop_service()
lifecycle, _classes, running, statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
previous = running["DemoPlugin"]
result = lifecycle.reload("DemoPlugin", "plugin-reload")
assert result is PluginRuntimeStatus.ACTIVE
assert statuses["DemoPlugin"] is PluginRuntimeStatus.ACTIVE
assert running["DemoPlugin"] is not previous
assert len(instances) == 2
stop_service.assert_called_once_with()
lifecycle._event_sender.assert_called_once_with(
"plugin-reload",
data={"plugin_id": "DemoPlugin"},
)
+417 -7
View File
@@ -1,6 +1,7 @@
import asyncio
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
@@ -16,6 +17,7 @@ from app.runtime.extensions.plugin.monitor import (
PluginChangeMonitor,
PluginMonitorController,
)
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
from app.runtime.extensions.plugin.system import reset_plugin_system
from app.runtime.extensions.plugin_manager import PluginManager
from app.schemas.plugin import PluginRuntimeStatus
@@ -65,8 +67,9 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No
missing_dependencies=("DependencyPending",),
missing_source=("SourcePending",),
)
manager.reopen_plugins.side_effect = lambda: order.append("reopen") or True
manager.start.side_effect = lambda plugin_id: order.append(f"plugin:{plugin_id}")
manager.start_monitor.side_effect = lambda: order.append("monitor")
manager.start_monitor.side_effect = lambda **_kwargs: order.append("monitor")
monkeypatch.setattr(
plugins_initializer,
"configure_plugin_services",
@@ -81,8 +84,16 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No
plugins_initializer.init_plugins()
assert order == ["services", "plugin:ReadyPlugin", "routes", "monitor"]
assert order == [
"services",
"reopen",
"plugin:ReadyPlugin",
"routes",
"monitor",
]
manager.reopen_plugins.assert_called_once_with()
manager.set_plugin_settling.assert_called_once_with(True)
manager.start_monitor.assert_called_once_with(reopen=True)
def test_plugin_manager_projects_dependency_classification_to_runtime_status() -> None:
@@ -155,6 +166,26 @@ def _patch_sync_plugins(monkeypatch, manager: MagicMock) -> MagicMock:
return register
@pytest.mark.asyncio
async def test_sync_plugins_rejects_before_configuring_mutable_services(
monkeypatch,
) -> None:
"""启动后同步在 admission 封口后不重装配服务、不写包或运行态。"""
admission = PluginMutationAdmission()
admission.seal()
manager = MagicMock()
manager.mutation.side_effect = admission.hold
configure = MagicMock()
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
monkeypatch.setattr(plugins_initializer, "configure_plugin_services", configure)
assert await plugins_initializer.sync_plugins() is False
configure.assert_not_called()
manager.set_plugin_settling.assert_not_called()
manager.sync.assert_not_called()
@pytest.mark.asyncio
async def test_sync_plugins_activates_ready_plugins_when_dependencies_fail(
monkeypatch,
@@ -392,6 +423,303 @@ def test_plugin_monitor_waits_until_dependency_settlement(monkeypatch) -> None:
_reset_plugin_manager()
def test_monitor_stop_keeps_thread_owned_until_it_really_exits() -> None:
"""停止超时后保留活线程引用,使后续停机调用仍能继续等待。"""
started = threading.Event()
release = threading.Event()
def runner() -> None:
"""模拟暂时无法响应停止事件的文件监控循环。"""
started.set()
release.wait()
controller = PluginMonitorController(runner=runner, log=MagicMock())
controller.start()
assert started.wait(1)
owned_thread = controller._thread
try:
assert controller.stop(timeout=0.01) is False
assert controller._thread is owned_thread
assert owned_thread is not None and owned_thread.is_alive()
finally:
release.set()
assert controller.stop(timeout=1) is True
assert controller._thread is None
assert controller.stop(timeout=0) is True
def test_monitor_stop_counts_lifecycle_lock_wait_in_timeout() -> None:
"""并发重建占用生命周期锁时,停止调用不得突破自身预算。"""
controller = PluginMonitorController(runner=lambda: None, log=MagicMock())
lock_acquired = threading.Event()
release = threading.Event()
def hold_lifecycle_lock() -> None:
"""模拟配置重载在停止请求到达时仍持有生命周期锁。"""
with controller._lifecycle_lock:
lock_acquired.set()
release.wait()
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
holder.start()
assert lock_acquired.wait(1)
started_at = time.monotonic()
try:
assert controller.stop(timeout=0.01) is False
assert time.monotonic() - started_at < 0.2
finally:
release.set()
holder.join(timeout=1)
assert holder.is_alive() is False
assert controller.stop(timeout=0) is True
def test_shutdown_seal_blocks_reload_between_stop_and_start(monkeypatch) -> None:
"""停机封口落在热重载停启间隙时,旧重载不得再创建监控线程。"""
reload_stopped = threading.Event()
resume_reload = threading.Event()
runner_started = threading.Event()
controller = PluginMonitorController(
runner=lambda: runner_started.set(),
log=MagicMock(),
)
original_stop = controller.stop
def pause_after_stop(timeout: float = 5.0) -> bool:
"""把配置热重载稳定暂停在旧线程已停、新线程未启的窗口。"""
stopped = original_stop(timeout=timeout)
reload_stopped.set()
resume_reload.wait()
return stopped
monkeypatch.setattr(controller, "stop", pause_after_stop)
reload_thread = threading.Thread(
target=lambda: controller.reload(enabled=True),
daemon=True,
)
reload_thread.start()
assert reload_stopped.wait(1)
try:
assert controller.close(timeout=1) is True
resume_reload.set()
reload_thread.join(timeout=1)
assert reload_thread.is_alive() is False
assert runner_started.is_set() is False
assert controller._thread is None
finally:
resume_reload.set()
reload_thread.join(timeout=1)
assert controller.reopen() is True
controller.start()
assert runner_started.wait(1)
assert controller.close(timeout=1) is True
def test_plugin_manager_stop_monitor_returns_controller_result(monkeypatch) -> None:
"""管理器停止入口透传线程收口结果和调用预算。"""
_reset_plugin_manager()
reset_plugin_system()
manager = PluginManager()
stop = MagicMock(return_value=False)
manager._plugin_monitor.stop = stop
try:
assert manager.stop_monitor(timeout=0.25) is False
stop.assert_called_once_with(timeout=0.25)
finally:
_reset_plugin_manager()
def test_plugin_manager_start_monitor_can_reopen_new_lifespan(monkeypatch) -> None:
"""新应用生命周期可显式解除封口,再按运行配置启动监控。"""
_reset_plugin_manager()
reset_plugin_system()
monkeypatch.setattr(
"app.runtime.extensions.plugin_manager.settings",
SimpleNamespace(
DEV=True,
PLUGIN_AUTO_RELOAD=False,
ROOT_PATH=MagicMock(),
),
)
manager = PluginManager()
reopen = MagicMock(return_value=True)
start = MagicMock()
manager._plugin_monitor.reopen = reopen
manager._plugin_monitor.start = start
try:
manager.start_monitor(reopen=True)
reopen.assert_called_once_with()
start.assert_called_once_with()
finally:
_reset_plugin_manager()
def test_stop_plugin_monitor_does_not_materialize_manager() -> None:
"""插件运行时尚未创建时,独立停机入口直接视为已完成。"""
_reset_plugin_manager()
assert plugins_initializer.stop_plugin_monitor(timeout=0) is True
assert PluginManager.get_existing_instance() is None
def test_stop_plugin_monitor_returns_existing_manager_result(monkeypatch) -> None:
"""启动层入口只操作既有管理器,并透传超时失败。"""
manager = MagicMock()
manager.close_monitor.return_value = False
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
assert plugins_initializer.stop_plugin_monitor(timeout=0.25) is False
manager.close_monitor.assert_called_once_with(timeout=0.25)
@pytest.mark.asyncio
async def test_two_phase_plugin_shutdown_does_not_materialize_manager() -> None:
"""两阶段入口在插件管理器尚未创建时都直接视为已收敛。"""
_reset_plugin_manager()
assert await plugins_initializer.quiesce_plugins(timeout=0) is True
assert plugins_initializer.finalize_plugins() is True
assert PluginManager.get_existing_instance() is None
@pytest.mark.asyncio
async def test_quiesce_timeout_retains_future_owner_until_worker_finishes(
monkeypatch,
) -> None:
"""同步插件 hook 超时后必须保留 Future,且未结束前拒绝卸载实例。"""
_reset_plugin_manager()
reset_plugin_system()
monkeypatch.setattr(
"app.runtime.extensions.plugin_manager.settings",
SimpleNamespace(
DEV=False,
PLUGIN_AUTO_RELOAD=False,
ROOT_PATH=MagicMock(),
),
)
manager = PluginManager()
started = threading.Event()
release = threading.Event()
def blocking_quiesce() -> bool:
"""模拟无法由 asyncio 取消的同步旧插件 hook。"""
started.set()
release.wait(timeout=2)
return True
manager._plugin_lifecycle.quiesce = MagicMock(side_effect=blocking_quiesce)
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
try:
with ThreadPoolExecutor(max_workers=1) as executor:
thread_helper = SimpleNamespace(submit=executor.submit)
monkeypatch.setattr(
"app.runtime.extensions.plugin_manager.ThreadHelper",
lambda: thread_helper,
)
assert await manager.quiesce_plugins(timeout=0.01) is False
assert started.is_set()
owner = manager._plugin_quiesce_future
assert owner is not None
assert owner.done() is False
assert manager.finalize_plugins() is False
manager._plugin_lifecycle.finalize.assert_not_called()
release.set()
assert await asyncio.wrap_future(owner) is True
assert manager.finalize_plugins() is True
manager._plugin_lifecycle.finalize.assert_called_once_with()
finally:
release.set()
_reset_plugin_manager()
@pytest.mark.asyncio
async def test_quiesce_seals_runtime_until_new_lifespan_reopens(monkeypatch) -> None:
"""屏障前封口后 start/reload/config 不能重开 producer,新 lifespan 可显式恢复。"""
_reset_plugin_manager()
reset_plugin_system()
monkeypatch.setattr(
"app.runtime.extensions.plugin_manager.settings",
SimpleNamespace(
DEBUG=False,
DEV=False,
PLUGIN_AUTO_RELOAD=False,
ROOT_PATH=MagicMock(),
),
)
manager = PluginManager()
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
manager._plugin_lifecycle.start = MagicMock(
return_value={"DemoPlugin": PluginRuntimeStatus.ACTIVE}
)
manager._plugin_lifecycle.reload = MagicMock(
return_value=PluginRuntimeStatus.ACTIVE
)
manager._plugin_lifecycle.stop = MagicMock(return_value=True)
manager._plugin_lifecycle.initialize = MagicMock()
manager._plugin_lifecycle._disable_events = MagicMock()
manager._plugin_registry.remove = MagicMock()
manager._plugin_registry.set_runtime_status = MagicMock()
manager.classify_plugins = MagicMock()
class DemoPlugin:
"""代表 quiesce 后仍由严格生命周期持有的运行实例。"""
plugin_instance = DemoPlugin()
manager._plugins["DemoPlugin"] = DemoPlugin
manager._running_plugins["DemoPlugin"] = plugin_instance
try:
assert await manager.quiesce_plugins(timeout=1) is True
assert manager.start("DemoPlugin") == {
"DemoPlugin": PluginRuntimeStatus.LOAD_FAILED
}
assert manager.stop("DemoPlugin") is None
assert manager.remove_plugin("DemoPlugin") is None
assert (
manager.reload_plugin("DemoPlugin")
is PluginRuntimeStatus.LOAD_FAILED
)
manager.init_plugin("DemoPlugin", {})
manager.init_config()
manager._plugin_lifecycle.start.assert_not_called()
manager._plugin_lifecycle.stop.assert_not_called()
manager._plugin_lifecycle.reload.assert_not_called()
manager._plugin_lifecycle.initialize.assert_not_called()
manager._plugin_lifecycle._disable_events.assert_not_called()
manager._plugin_registry.remove.assert_not_called()
manager._plugin_registry.set_runtime_status.assert_not_called()
manager.classify_plugins.assert_not_called()
assert manager._plugins["DemoPlugin"] is DemoPlugin
assert manager._running_plugins["DemoPlugin"] is plugin_instance
assert manager.reopen_plugins() is False
manager._plugins.clear()
manager._running_plugins.clear()
assert manager.reopen_plugins() is True
assert manager.start("DemoPlugin") == {
"DemoPlugin": PluginRuntimeStatus.ACTIVE
}
assert (
manager.reload_plugin("DemoPlugin")
is PluginRuntimeStatus.ACTIVE
)
finally:
_reset_plugin_manager()
def test_plugin_monitor_skips_installing_plugin_until_package_write_finishes(tmp_path) -> None:
"""安装替换目录期间,文件事件不得抢先导入未完成的插件包。"""
reload_plugin = MagicMock()
@@ -466,11 +794,12 @@ def test_stop_plugins_stops_monitor_before_plugin_runtime(monkeypatch) -> None:
"""关闭时先隔离文件变化,再停止插件实例。"""
order: list[str] = []
manager = MagicMock()
manager.stop_monitor.side_effect = lambda: order.append("monitor")
manager.stop_monitor.side_effect = lambda: order.append("monitor") or True
manager.stop.side_effect = lambda: order.append("plugins")
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
plugins_initializer.stop_plugins()
assert plugins_initializer.stop_plugins() is True
assert order == ["monitor", "plugins"]
@@ -479,8 +808,89 @@ def test_stop_plugins_still_stops_runtime_when_monitor_stop_fails(monkeypatch) -
"""监控线程停止异常不得阻止插件实例释放资源。"""
manager = MagicMock()
manager.stop_monitor.side_effect = RuntimeError("monitor stop failed")
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
plugins_initializer.stop_plugins()
assert plugins_initializer.stop_plugins() is False
manager.stop.assert_called_once_with()
def test_stop_plugins_does_not_materialize_manager() -> None:
"""插件初始化尚未创建管理器时,失败清理不得反向构造运行时。"""
_reset_plugin_manager()
assert plugins_initializer.stop_plugins() is True
assert PluginManager.get_existing_instance() is None
def test_stop_plugins_remains_idempotent(monkeypatch) -> None:
"""兼容停机入口可重复调用,并保持每次先停监控再停插件。"""
order: list[str] = []
manager = MagicMock()
manager.stop_monitor.side_effect = lambda: order.append("monitor") or True
manager.stop.side_effect = lambda: order.append("plugins")
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
assert plugins_initializer.stop_plugins() is True
assert plugins_initializer.stop_plugins() is True
assert order == ["monitor", "plugins", "monitor", "plugins"]
def test_plugin_manager_legacy_stop_preserves_none_return() -> None:
"""公共 PluginManager.stop 委托单阶段停机后保持历史 None 返回 ABI。"""
manager = object.__new__(PluginManager)
manager._plugin_quiesce_lock = threading.RLock()
manager._plugin_runtime_closed = False
manager._plugin_quiesce_future = None
manager._plugin_mutation_admission = PluginMutationAdmission()
manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=True))
assert PluginManager.stop(manager, "DemoPlugin") is None
manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin")
def test_config_reload_continues_after_legacy_stop() -> None:
"""配置热重载保持旧行为,stop 的 None 返回不得阻止重新分类和启动。"""
manager = object.__new__(PluginManager)
manager._plugin_quiesce_lock = threading.RLock()
manager._plugin_runtime_closed = False
manager._plugin_mutation_admission = PluginMutationAdmission()
manager.stop = MagicMock(return_value=None)
manager.classify_plugins = MagicMock(
return_value=PluginDependencyClassification(
ready=("ReadyPlugin",),
missing_dependencies=(),
missing_source=(),
)
)
manager.apply_plugin_dependency_classification = MagicMock()
manager.start = MagicMock()
PluginManager.init_config(manager)
manager.stop.assert_called_once_with()
manager.classify_plugins.assert_called_once_with()
manager.apply_plugin_dependency_classification.assert_called_once_with(
manager.classify_plugins.return_value
)
manager.start.assert_called_once_with("ReadyPlugin")
def test_remove_plugin_clears_registry_after_legacy_stop() -> None:
"""卸载路径保持忽略旧 stop 返回值并继续清理注册表。"""
manager = object.__new__(PluginManager)
manager._plugin_quiesce_lock = threading.RLock()
manager._plugin_runtime_closed = False
manager._plugin_mutation_admission = PluginMutationAdmission()
manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=None))
manager._plugin_registry = SimpleNamespace(remove=MagicMock())
PluginManager.remove_plugin(manager, "DemoPlugin")
manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin")
manager._plugin_registry.remove.assert_called_once_with("DemoPlugin")
+155
View File
@@ -0,0 +1,155 @@
"""插件可变事务停机准入的确定性测试。"""
import asyncio
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from app.foundation.singleton import Singleton
from app.runtime.extensions.plugin.admission import (
PluginMutationAdmission,
PluginMutationRejectedError,
)
from app.runtime.extensions.plugin.system import reset_plugin_system
from app.runtime.extensions.plugin_manager import PluginManager
from app.schemas.plugin import PluginRuntimeStatus
from app.schemas.types import EventType
@pytest.fixture
def plugin_manager() -> Iterator[PluginManager]:
"""构造隔离的插件管理器,并在用例结束后清除单例状态。"""
key = (PluginManager, (), frozenset())
Singleton._instances.pop(key, None)
reset_plugin_system()
manager = PluginManager()
try:
yield manager
finally:
Singleton._instances.pop(key, None)
def test_seal_rejects_new_root_but_allows_propagated_nested_lease() -> None:
"""封口拒绝无关调用,但已获准事务跨线程后的嵌套仍能完成。"""
admission = PluginMutationAdmission()
calls: list[str] = []
def mutate(label: str) -> None:
"""在测试线程中尝试取得 lease 并记录副作用。"""
with admission.hold(label):
calls.append(label)
with admission.hold("外层事务"):
propagated = copy_context()
assert admission.active_count == 1
assert admission.seal() == 1
with ThreadPoolExecutor(max_workers=2) as executor:
executor.submit(propagated.run, mutate, "嵌套事务").result(timeout=1)
rejected = executor.submit(mutate, "新事务")
with pytest.raises(PluginMutationRejectedError):
rejected.result(timeout=1)
assert calls == ["嵌套事务"]
assert admission.active_count == 1
assert admission.active_count == 0
assert admission.reopen() is True
@pytest.mark.asyncio
async def test_quiesce_timeout_retains_admitted_owner_and_nested_reload(
plugin_manager: PluginManager,
monkeypatch,
) -> None:
"""停机超时保留 active owner,且封口前获准的事务可完成内部重载。"""
manager = plugin_manager
entered = asyncio.Event()
release = asyncio.Event()
manager._plugin_lifecycle.reload = MagicMock(
return_value=PluginRuntimeStatus.ACTIVE
)
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
async def mutate() -> PluginRuntimeStatus:
"""持有外层 lease,等待封口后再调用嵌套 Manager 写入口。"""
with manager.mutation("安装插件 DemoPlugin"):
entered.set()
await release.wait()
return manager.reload_plugin("DemoPlugin")
with ThreadPoolExecutor(max_workers=1) as executor:
monkeypatch.setattr(
"app.runtime.extensions.plugin_manager.ThreadHelper",
lambda: SimpleNamespace(submit=executor.submit),
)
mutation_task = asyncio.create_task(mutate())
await entered.wait()
assert await manager.quiesce_plugins(timeout=0.01) is False
owner = manager._plugin_quiesce_future
assert owner is not None
assert owner.done() is False
assert manager._plugin_mutation_admission.active_count == 1
assert manager.finalize_plugins() is False
with pytest.raises(PluginMutationRejectedError):
with manager.mutation("新的配置写入"):
pass
release.set()
assert await mutation_task is PluginRuntimeStatus.ACTIVE
assert await asyncio.wrap_future(owner) is True
manager._plugin_lifecycle.reload.assert_called_once_with(
"DemoPlugin",
EventType.PluginReload,
)
assert manager._plugin_mutation_admission.active_count == 0
assert manager.finalize_plugins() is True
@pytest.mark.asyncio
async def test_quiesce_inside_mutation_fails_without_sealing(
plugin_manager: PluginManager,
) -> None:
"""事务不能等待自身退出,快速失败时也不得误封口运行时。"""
with plugin_manager.mutation("配置插件"):
assert await plugin_manager.quiesce_plugins(timeout=0) is False
assert plugin_manager._plugin_mutation_admission.accepting is True
assert plugin_manager._plugin_runtime_closed is False
@pytest.mark.asyncio
async def test_sealed_manager_blocks_direct_persistent_mutations(
plugin_manager: PluginManager,
) -> None:
"""Manager 单项兼容入口在封口后返回既有失败形状且不触发底层副作用。"""
plugin_manager._plugin_config_store = MagicMock()
plugin_manager._plugin_instance_store = MagicMock()
plugin_manager._plugin_sync = MagicMock()
plugin_manager._plugin_clone = MagicMock()
plugin_manager._plugin_mutation_admission.seal()
plugin_manager._plugin_runtime_closed = True
assert plugin_manager.save_plugin_config("DemoPlugin", {}) is False
assert await plugin_manager.async_save_plugin_config("DemoPlugin", {}) is False
assert plugin_manager.delete_plugin_config("DemoPlugin") is False
assert plugin_manager.delete_plugin_data("DemoPlugin") is False
assert plugin_manager.delete_plugin_instance("DemoPlugin") is False
clone_result = plugin_manager.clone_plugin("DemoPlugin", "Work", "", "")
assert clone_result[0] is False
assert "停机阶段" in clone_result[1]
with pytest.raises(PluginMutationRejectedError):
plugin_manager.sync()
plugin_manager._plugin_config_store.write.assert_not_called()
plugin_manager._plugin_config_store.async_write.assert_not_called()
plugin_manager._plugin_config_store.delete.assert_not_called()
plugin_manager._plugin_config_store.delete_data.assert_not_called()
plugin_manager._plugin_instance_store.delete.assert_not_called()
plugin_manager._plugin_sync.sync.assert_not_called()
plugin_manager._plugin_clone.clone.assert_not_called()
+48
View File
@@ -1,3 +1,5 @@
import pytest
from app.foundation.singleton import Singleton, SingletonClass
@@ -27,3 +29,49 @@ def test_parameterized_singleton_can_read_matching_instance_without_creating(mon
instance = Example("first")
assert Example.get_existing_instance("first") is instance
assert Example.get_existing_instance("second") is None
def test_parameterized_singleton_can_retain_failed_lifecycle_owner(monkeypatch):
"""构造中途失败时,可选 owner 单例必须仍能由启动失败清理读取。"""
class Example(metaclass=Singleton):
"""模拟在构造期已创建后台 owner 后抛错的参数化单例。"""
_retain_failed_singleton = True
def __init__(self) -> None:
"""发布可观察 owner 后模拟后续启动失败。"""
self.owner_started = True
raise RuntimeError("startup failed")
monkeypatch.setattr(Singleton, "_instances", {})
with pytest.raises(RuntimeError, match="startup failed"):
Example()
retained = Example.get_existing_instance()
assert retained is not None
assert retained.owner_started is True
def test_class_singleton_can_retain_failed_lifecycle_owner(monkeypatch):
"""按类 owner 单例也必须在 __init__ 抛错后保留可清理身份。"""
class Example(metaclass=SingletonClass):
"""模拟目录监控构造中途失败的按类单例。"""
_retain_failed_singleton = True
def __init__(self) -> None:
"""发布可观察 owner 后模拟后续启动失败。"""
self.owner_started = True
raise RuntimeError("startup failed")
monkeypatch.setattr(SingletonClass, "_instances", {})
with pytest.raises(RuntimeError, match="startup failed"):
Example()
retained = Example.get_existing_instance()
assert retained is not None
assert retained.owner_started is True
+4 -4
View File
@@ -50,7 +50,7 @@ def test_task_registry_cancels_tasks_and_rejects_late_registration() -> None:
task = registry.create(worker(), owner="test.shutdown")
await started.wait()
await registry.shutdown(timeout_seconds=1.0)
assert await registry.shutdown(timeout_seconds=1.0) is True
assert task.cancelled()
assert cancelled.is_set()
@@ -108,7 +108,7 @@ def test_task_registry_keeps_timed_out_sync_owner_until_real_completion() -> Non
task = registry.create_sync(worker, owner="test.sync-timeout")
assert await asyncio.to_thread(started.wait, 1.0)
await registry.shutdown(timeout_seconds=0.001)
assert await registry.shutdown(timeout_seconds=0.001) is False
assert not task.done()
assert [record.owner for record in registry.records] == [
@@ -155,7 +155,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None:
task = registry.create(worker(), owner="test.stubborn")
await started.wait()
try:
await registry.shutdown(timeout_seconds=0.001)
assert await registry.shutdown(timeout_seconds=0.001) is False
assert cleanup_started.is_set()
assert not task.done()
@@ -165,7 +165,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None:
]
assert reports[-1]["owners"] == ("test.stubborn",)
await registry.shutdown(timeout_seconds=0.001)
assert await registry.shutdown(timeout_seconds=0.001) is False
assert not task.done()
assert cancellation_count == 1
assert len(reports) == 1
@@ -0,0 +1,126 @@
"""失败整理 AI 重试调度器的生命周期测试。"""
import asyncio
from unittest.mock import Mock, patch
import pytest
from app.application.transfer import FailedRetryScheduler
def test_retry_scheduler_close_cancels_buffered_timer_and_rejects_new_work():
"""关闭应取消尚未触发的 timer、清空缓冲并拒绝新增记录。"""
async def exercise() -> None:
"""在独立事件循环内验证 timer 与关闭状态。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 60
await scheduler.schedule_retry(11, group_key="media:test")
timer = scheduler._retry_transfer_timers["media:test"]
await scheduler.close()
await scheduler.close()
assert timer.cancelled()
assert scheduler._retry_transfer_buffer == {}
assert scheduler._retry_transfer_timers == {}
with pytest.raises(RuntimeError, match="正在关闭"):
await scheduler.schedule_retry(12, group_key="media:test")
asyncio.run(exercise())
def test_retry_scheduler_close_cancels_and_waits_for_active_flush_task():
"""关闭返回前应等待已经启动的 flush 任务完成取消收尾。"""
async def exercise() -> None:
"""启动一个不会自行结束的 flush,并通过关闭流程取消它。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
started = asyncio.Event()
stopped = asyncio.Event()
async def blocking_flush(_group_key: str, _generation: int) -> None:
"""等待取消信号,并在 finally 中证明收尾已经完成。"""
started.set()
try:
await asyncio.Event().wait()
finally:
stopped.set()
scheduler._flush_retry_transfer = blocking_flush
await scheduler.schedule_retry(11, group_key="media:test")
await asyncio.wait_for(started.wait(), timeout=1)
task = next(iter(scheduler._retry_transfer_tasks))
await scheduler.close()
assert stopped.is_set()
assert task.cancelled()
assert task.get_name() == "transfer.failed_retry.flush"
assert scheduler._retry_transfer_tasks == set()
asyncio.run(exercise())
def test_retry_scheduler_observes_unexpected_background_task_error():
"""flush 协程越过自身防线的异常仍应由任务 owner 统一观察。"""
async def exercise() -> None:
"""让受管 flush 任务直接失败,并等待完成回调处理异常。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
async def failing_flush(_group_key: str, _generation: int) -> None:
"""模拟 flush 外层出现未处理异常。"""
raise RuntimeError("flush failed")
scheduler._flush_retry_transfer = failing_flush
with patch("app.application.transfer.logger.error", Mock()) as log_error:
await scheduler.schedule_retry(11, group_key="media:test")
for _ in range(5):
await asyncio.sleep(0)
if scheduler._retry_transfer_tasks:
break
assert scheduler._retry_transfer_tasks
for _ in range(5):
await asyncio.sleep(0)
if not scheduler._retry_transfer_tasks:
break
assert scheduler._retry_transfer_tasks == set()
log_error.assert_called_once()
assert "flush failed" in log_error.call_args.args[0]
await scheduler.close()
asyncio.run(exercise())
def test_retry_scheduler_old_flush_cannot_consume_renewed_generation():
"""旧 timer 已建 task 后的新失败应续期,不能被旧 flush 提前取走。"""
async def exercise() -> None:
"""稳定复现 timer callback 与同组新 schedule 交错的窗口。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 3600
await scheduler.schedule_retry(11, group_key="media:test")
old_timer = scheduler._retry_transfer_timers["media:test"]
old_generation = scheduler._retry_transfer_generations["media:test"]
# 模拟旧 timer callback 已进入事件循环,但 flush task 尚未取得分组锁。
scheduler._start_retry_transfer_task("media:test", old_generation)
await scheduler.schedule_retry(12, group_key="media:test")
renewed_timer = scheduler._retry_transfer_timers["media:test"]
await asyncio.sleep(0)
await asyncio.sleep(0)
assert old_timer.cancelled()
assert renewed_timer.cancelled() is False
assert scheduler._retry_transfer_buffer["media:test"] == [11, 12]
assert scheduler._retry_transfer_timers["media:test"] is renewed_timer
assert scheduler._retry_transfer_tasks == set()
await scheduler.close()
assert renewed_timer.cancelled()
asyncio.run(exercise())
@@ -1,6 +1,8 @@
from unittest.mock import Mock, patch
from types import SimpleNamespace
import pytest
from app.chain import transfer as transfer_module
from app.chain.transfer import TransferChain
from app.application.transfer import (
@@ -59,6 +61,26 @@ class _Loop:
return False
class _DeferredLoop(_Loop):
"""延迟执行线程安全回调,用于覆盖关闭与入环之间的竞态。"""
def __init__(self):
"""初始化延迟回调和定时器清单。"""
super().__init__()
self.soon_callbacks = []
def call_soon_threadsafe(self, callback, *args):
"""保存线程安全回调,直到测试显式执行。"""
self.soon_callbacks.append((callback, args))
def run_soon_callbacks(self):
"""执行并清空已保存的线程安全回调。"""
callbacks = list(self.soon_callbacks)
self.soon_callbacks.clear()
for callback, args in callbacks:
callback(*args)
def _task(*, episode: int, download_hash: str = "hash-1") -> TransferTask:
"""构造同一媒体不同剧集的整理任务。"""
return TransferTask(
@@ -122,6 +144,116 @@ def test_aggregator_debounces_same_group_and_flushes_once():
callback.assert_called_once_with(notices)
def test_aggregator_old_timer_cannot_flush_before_renewal_is_armed():
"""新通知已接收时,旧 timer 不得抢在事件循环重置静默窗前发送。"""
loop = _DeferredLoop()
aggregator = TransferFailureNotificationAggregator()
callback = Mock()
first = TransferFailureNotification(
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
)
second = TransferFailureNotification(
"测试剧 (2026)", "S01E02", "原因B", 2, None, "tester"
)
aggregator.schedule(
group_key="media:test",
notification=first,
callback=callback,
loop=loop,
)
loop.run_soon_callbacks()
old_timer = loop.timers[0]
aggregator.schedule(
group_key="media:test",
notification=second,
callback=callback,
loop=loop,
)
old_timer.callback(*old_timer.args)
callback.assert_not_called()
loop.run_soon_callbacks()
renewed_timer = loop.timers[1]
assert old_timer.cancelled is True
renewed_timer.callback(*renewed_timer.args)
callback.assert_called_once_with([first, second])
def test_aggregator_close_flushes_accepted_notification_before_timer_is_armed():
"""关闭应发送已接收但尚未进入事件循环的通知,且延迟回调不能重新建 timer。"""
loop = _DeferredLoop()
aggregator = TransferFailureNotificationAggregator()
callback = Mock()
notice = TransferFailureNotification(
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
)
aggregator.schedule(
group_key="media:test",
notification=notice,
callback=callback,
loop=loop,
)
aggregator.close()
aggregator.close()
loop.run_soon_callbacks()
callback.assert_called_once_with([notice])
assert loop.timers == []
def test_aggregator_close_cancels_timer_and_rejects_new_notification():
"""关闭应取消已建 timer,并让调用方明确感知后续投递被拒绝。"""
loop = _Loop()
aggregator = TransferFailureNotificationAggregator()
callback = Mock()
notice = TransferFailureNotification(
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
)
aggregator.schedule(
group_key="media:test",
notification=notice,
callback=callback,
loop=loop,
)
aggregator.close()
assert loop.timers[0].cancelled is True
callback.assert_called_once_with([notice])
with pytest.raises(RuntimeError, match="正在关闭"):
aggregator.schedule(
group_key="media:test",
notification=notice,
callback=callback,
loop=loop,
)
def test_aggregator_close_observes_flush_callback_error():
"""关闭阶段同步刷新失败时应记录异常而不是让通知静默丢失。"""
loop = _DeferredLoop()
aggregator = TransferFailureNotificationAggregator()
callback = Mock(side_effect=RuntimeError("send failed"))
notice = TransferFailureNotification(
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
)
aggregator.schedule(
group_key="media:test",
notification=notice,
callback=callback,
loop=loop,
)
with patch("app.application.transfer.logger.error") as log_error:
aggregator.close()
callback.assert_called_once_with([notice])
log_error.assert_called_once()
def test_aggregated_message_contains_count_reason_stats_and_batch_entry():
"""聚合消息应给出失败数、原因统计、历史 ID 和批量处理入口。"""
chain = object.__new__(TransferChain)
+31 -1
View File
@@ -8,11 +8,12 @@
这些测试固定三项不变量入队即落盘登记终态即注销重启能回放
"""
from pathlib import Path
import threading
from unittest.mock import MagicMock
from app.chain.transfer import TransferChain
from app.schemas import FileItem
from app.application.transfer import TransferTask
from app.schemas.file import FileItem
def _build_chain(pendingoper) -> TransferChain:
@@ -231,3 +232,32 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
chain._TransferChain__replay_pending()
assert handled == ["B.mkv"]
def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
"""
宿主关闭后不得继续检查或注销下一条登记未处理项留给下次启动回放
"""
first = tmp_path / "A.mkv"
first.write_bytes(b"x")
missing_second = tmp_path / "gone.mkv"
pendingoper = MagicMock()
pendingoper.list_all.return_value = [
("local", str(first)),
("local", str(missing_second)),
]
chain = _build_chain(pendingoper)
stop_event = threading.Event()
transferred = []
def transfer_first(**kwargs):
"""首条回放送入整理链后模拟宿主发出关闭信号。"""
transferred.append(kwargs["fileitem"].path)
stop_event.set()
monkeypatch.setattr(chain, "do_transfer", transfer_first)
chain._TransferChain__replay_pending(stop_event)
assert transferred == [first.as_posix()]
pendingoper.discard.assert_not_called()
+485
View File
@@ -0,0 +1,485 @@
"""文件整理 worker 与 pending 回放的宿主生命周期测试。"""
import asyncio
import queue
import threading
import time
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from app.chain.transfer import TransferChain
from app.foundation.singleton import Singleton
from app.runtime.config import global_vars
from app.application.transfer import TransferQueue, TransferTask
from app.schemas.file import FileItem
from app.startup import transfer_initializer
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
chain = object.__new__(TransferChain)
chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads)
chain._queue = queue.Queue()
chain._transfer_interval = 0.1
chain._threads = []
chain._retiring_threads = []
chain._queue_active = False
chain._worker_stop_event = threading.Event()
chain._worker_lifecycle_lock = threading.RLock()
chain._worker_state_lock = threading.RLock()
chain._closing = False
chain._replay_thread = None
chain._replay_stop_event = threading.Event()
return chain
def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None:
"""热更新应等待旧 worker 收敛,再启动使用独立停止信号的新一代。"""
chain = _build_chain(transfer_threads=1)
started_workers: queue.Queue = queue.Queue()
def run_worker(stop_event: threading.Event) -> None:
"""记录 worker 代际并等待该代专属停止信号。"""
started_workers.put((threading.current_thread(), stop_event))
stop_event.wait()
chain._TransferChain__start_transfer = run_worker
assert chain._TransferChain__init() is True
first_thread, first_stop_event = started_workers.get(timeout=1)
chain.on_config_changed()
second_thread, second_stop_event = started_workers.get(timeout=1)
assert first_stop_event.is_set() is True
assert first_thread.is_alive() is False
assert second_thread is not first_thread
assert second_stop_event is not first_stop_event
assert second_stop_event.is_set() is False
service = MagicMock()
service.put.return_value = True
chain._transfer_queue_service = MagicMock(return_value=service)
task = MagicMock()
assert chain.put_to_queue(task) is True
service.put.assert_called_once()
assert chain.close_workers(timeout_seconds=1) is True
assert second_thread.is_alive() is False
def test_config_reload_hands_queue_to_new_generation_while_old_io_finishes() -> None:
"""旧代同步 I/O 超时不应让后续队列永久失去 worker。"""
chain = _build_chain(transfer_threads=1)
chain._WORKER_RESTART_TIMEOUT_SECONDS = 0.01
started_workers: queue.Queue = queue.Queue()
release_old_worker = threading.Event()
invocation_count = 0
invocation_lock = threading.Lock()
def run_worker(stop_event: threading.Event) -> None:
"""首代模拟不可取消 I/O,后续代按各自停止信号正常收敛。"""
nonlocal invocation_count
with invocation_lock:
generation = invocation_count
invocation_count += 1
started_workers.put((threading.current_thread(), stop_event))
if generation == 0:
release_old_worker.wait()
else:
stop_event.wait()
chain._TransferChain__start_transfer = run_worker
assert chain._TransferChain__init() is True
old_thread, old_stop_event = started_workers.get(timeout=1)
chain.on_config_changed()
new_thread, new_stop_event = started_workers.get(timeout=1)
assert old_stop_event.is_set() is True
assert old_thread.is_alive() is True
assert chain._retiring_threads == [old_thread]
assert chain._threads == [new_thread]
assert new_stop_event.is_set() is False
release_old_worker.set()
assert chain.close_workers(timeout_seconds=1) is True
assert old_thread.is_alive() is False
assert new_thread.is_alive() is False
def test_close_workers_is_bounded_and_retains_nonconverging_owner() -> None:
"""同步 I/O 线程超时后应保留句柄并报告失败,不能伪装成已取消。"""
chain = _build_chain()
release = threading.Event()
thread = threading.Thread(
target=release.wait,
name="transfer-blocked-test",
daemon=True,
)
chain._threads = [thread]
thread.start()
started_at = time.monotonic()
assert chain.close_workers(timeout_seconds=0.01) is False
assert time.monotonic() - started_at < 0.5
assert chain._threads == []
assert chain._retiring_threads == [thread]
assert thread.is_alive() is True
service = MagicMock()
chain._transfer_queue_service = MagicMock(return_value=service)
assert chain.put_to_queue(MagicMock()) is False
service.put.assert_not_called()
release.set()
assert chain.close_workers(timeout_seconds=1) is True
assert chain.close_workers(timeout_seconds=0) is True
assert chain._threads == []
assert chain._retiring_threads == []
def test_close_workers_lock_wait_uses_the_same_timeout_budget() -> None:
"""生命周期锁竞争必须耗用关闭预算,超时返回后不得延迟修改 worker 状态。"""
chain = _build_chain()
lock_acquired = threading.Event()
release_lock = threading.Event()
def hold_lifecycle_lock() -> None:
"""在独立线程持锁,稳定制造无法重入的生命周期锁竞争。"""
with chain._worker_lifecycle_lock:
lock_acquired.set()
assert release_lock.wait(timeout=1)
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
holder.start()
assert lock_acquired.wait(timeout=1)
started_at = time.monotonic()
assert chain.close_workers(timeout_seconds=0.01) is False
assert time.monotonic() - started_at < 0.5
assert chain._closing is False
assert chain._worker_stop_event.is_set() is False
assert chain._queue.empty() is True
release_lock.set()
holder.join(timeout=1)
assert holder.is_alive() is False
assert chain.close_workers(timeout_seconds=1) is True
def test_close_keeps_timer_dependencies_when_workers_do_not_converge() -> None:
"""活跃整理线程超时后,通知和重试 owner 必须继续供线程使用。"""
chain = _build_chain()
chain.close_workers = MagicMock(return_value=False)
chain.failure_notification_aggregator = MagicMock()
chain.retry_scheduler = MagicMock(close=AsyncMock())
completed = asyncio.run(chain.close(timeout_seconds=0.01))
assert completed is False
chain.close_workers.assert_called_once_with(0.01)
chain.failure_notification_aggregator.close.assert_not_called()
chain.retry_scheduler.close.assert_not_awaited()
def test_close_releases_timer_dependencies_after_workers_converge() -> None:
"""worker 和回放退出后,整理链应继续刷新通知并关闭 AI 重试。"""
chain = _build_chain()
chain.close_workers = MagicMock(return_value=True)
chain.failure_notification_aggregator = MagicMock()
chain.retry_scheduler = MagicMock(close=AsyncMock())
completed = asyncio.run(chain.close(timeout_seconds=0.01))
assert completed is True
chain.failure_notification_aggregator.close.assert_called_once_with()
chain.retry_scheduler.close.assert_awaited_once_with()
def test_stop_transfer_runtime_does_not_construct_chain(monkeypatch) -> None:
"""关闭入口在整理链从未使用时应直接成功,不能因关停而启动 worker。"""
get_existing_instance = MagicMock(return_value=None)
monkeypatch.setattr(
transfer_initializer.TransferChain,
"get_existing_instance",
get_existing_instance,
)
completed = asyncio.run(
transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01)
)
assert completed is True
get_existing_instance.assert_called_once_with()
def test_stop_transfer_runtime_closes_existing_chain(monkeypatch) -> None:
"""关闭入口应把超时预算和真实收敛结果原样传给既有整理链。"""
chain = MagicMock(close=AsyncMock(return_value=False))
monkeypatch.setattr(
transfer_initializer.TransferChain,
"get_existing_instance",
MagicMock(return_value=chain),
)
completed = asyncio.run(
transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01)
)
assert completed is False
chain.close.assert_awaited_once_with(timeout_seconds=0.01)
def test_constructor_failure_publishes_started_worker_to_cleanup(monkeypatch) -> None:
"""首个 worker 启动后构造失败时,stop-only 入口仍必须找到并等待它。"""
instances = dict(Singleton._instances)
instances.pop((TransferChain, (), frozenset()), None)
monkeypatch.setattr(Singleton, "_instances", instances)
worker_started = threading.Event()
worker_release = threading.Event()
workers: list[threading.Thread] = []
def failing_init(chain: TransferChain) -> None:
"""模拟第二个 owner 启动失败前已经成功启动一个整理线程。"""
worker = threading.Thread(
target=lambda: (worker_started.set(), worker_release.wait()),
name="transfer-partial-construction",
daemon=True,
)
workers.append(worker)
worker.start()
async def close(*, timeout_seconds: float) -> bool:
"""模拟真实 close 释放并等待半构造实例已经发布的 worker。"""
worker_release.set()
worker.join(timeout=timeout_seconds)
return not worker.is_alive()
chain.close = close
raise RuntimeError("second worker failed")
monkeypatch.setattr(TransferChain, "__init__", failing_init)
with pytest.raises(RuntimeError, match="second worker failed"):
TransferChain()
assert worker_started.wait(timeout=1)
retained = TransferChain.get_existing_instance()
assert retained is not None
assert asyncio.run(
transfer_initializer.stop_transfer_runtime(timeout_seconds=1)
) is True
assert workers[0].is_alive() is False
def test_failed_retry_schedule_future_error_is_observed() -> None:
"""跨线程调度协程的延迟异常必须被取回并写入日志。"""
future: Future[None] = Future()
future.set_exception(RuntimeError("scheduler closed"))
with patch("app.chain.transfer.logger.error") as log_error:
TransferChain._observe_failed_retry_schedule(future)
log_error.assert_called_once()
assert "scheduler closed" in log_error.call_args.args[0]
def test_failed_retry_schedule_registers_future_observer(monkeypatch) -> None:
"""整理线程提交 AI 重试后应让 Future 持续连接到异常观察回调。"""
chain = _build_chain()
async def schedule_retry(_history_id: int, *, group_key: str) -> None:
"""提供不会实际执行的调度协程,供跨线程提交边界检查。"""
chain.retry_scheduler = MagicMock(schedule_retry=schedule_retry)
future = MagicMock(spec=Future)
event_loop = MagicMock()
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", event_loop)
def submit(coroutine, loop):
"""关闭测试协程并返回可检查的并发 Future。"""
assert loop is event_loop
coroutine.close()
return future
with patch(
"app.chain.transfer.asyncio.run_coroutine_threadsafe",
side_effect=submit,
):
chain._schedule_failed_transfer_retry(42, "media:test")
future.add_done_callback.assert_called_once()
callback = future.add_done_callback.call_args.args[0]
assert callback is TransferChain._observe_failed_retry_schedule
def test_worker_requeues_item_taken_during_shutdown(monkeypatch) -> None:
"""停止信号与 queue.get 竞态时,未开始处理的任务必须原样放回队列。"""
chain = _build_chain()
work_queue = MagicMock()
chain._queue = work_queue
entered_get = threading.Event()
release_get = threading.Event()
item = TransferQueue()
def get_item(*_args, **_kwargs):
"""让停止信号稳定落在阻塞取队列之后、任务处理之前。"""
entered_get.set()
assert release_get.wait(timeout=1)
return item
work_queue.get.side_effect = get_item
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
stop_event = threading.Event()
thread = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
thread.start()
assert entered_get.wait(timeout=1)
stop_event.set()
release_get.set()
thread.join(timeout=1)
assert thread.is_alive() is False
work_queue.put.assert_called_once_with(item)
work_queue.task_done.assert_called_once_with()
def test_worker_settles_progress_when_only_stop_sentinel_remains(monkeypatch) -> None:
"""真实任务完成时仅剩停止哨兵,仍应结束进度并重置本批计数。"""
chain = _build_chain()
task = TransferTask(
fileitem=FileItem(
storage="local",
path="/downloads/movie.mkv",
type="file",
name="movie.mkv",
basename="movie",
extension="mkv",
)
)
chain.jobview = MagicMock()
chain.jobview.pending_total.return_value = 1
chain._progress = MagicMock()
chain._active_tasks = 0
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
task_started = threading.Event()
release_task = threading.Event()
def handle_transfer(*_args, **_kwargs):
"""阻塞真实任务,让测试能在其完成前稳定插入停止哨兵。"""
task_started.set()
assert release_task.wait(timeout=1)
return True, ""
chain._TransferChain__handle_transfer = handle_transfer
chain._TransferChain__start_job_execution = MagicMock()
chain._TransferChain__finish_job_execution = MagicMock()
chain._queue.put(TransferQueue(task=task))
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
stop_event = threading.Event()
worker = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
worker.start()
assert task_started.wait(timeout=1)
stop_event.set()
chain._queue.put(chain._QUEUE_STOP_SENTINEL)
release_task.set()
worker.join(timeout=1)
assert worker.is_alive() is False
chain._progress.end.assert_called_once_with()
assert chain._active_tasks == 0
assert chain._total_num == 0
assert chain._processed_num == 0
assert chain._fail_num == 0
with chain._queue.mutex:
assert list(chain._queue.queue) == [chain._QUEUE_STOP_SENTINEL]
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
"""其他 worker 已取走真实任务但尚未登记 active 时,当前批次不得提前结算。"""
chain = _build_chain()
task = TransferTask(
fileitem=FileItem(
storage="local",
path="/downloads/claimed.mkv",
type="file",
name="claimed.mkv",
basename="claimed",
extension="mkv",
)
)
chain._progress = MagicMock()
chain._active_tasks = 0
chain._processed_num = 1
chain._fail_num = 0
chain._total_num = 2
claimed = threading.Event()
release_claim = threading.Event()
chain._queue.put(TransferQueue(task=task))
def hold_claimed_task() -> None:
"""模拟 worker 已完成 queue.get、尚未取得 task_lock 登记 active 的窗口。"""
item = chain._queue.get(timeout=1)
assert item.task is task
claimed.set()
assert release_claim.wait(timeout=1)
chain._queue.task_done()
chain._TransferChain__settle_transfer_progress_if_idle()
worker = threading.Thread(target=hold_claimed_task, daemon=True)
worker.start()
assert claimed.wait(timeout=1)
chain._TransferChain__settle_transfer_progress_if_idle()
chain._progress.end.assert_not_called()
assert chain._processed_num == 1
release_claim.set()
worker.join(timeout=1)
assert worker.is_alive() is False
chain._progress.end.assert_called_once_with()
assert chain._total_num == 0
assert chain._processed_num == 0
def test_replay_has_single_owner_and_close_waits_for_it() -> None:
"""重复回放只保留一个线程,关闭会通知并等待该线程退出。"""
chain = _build_chain()
replay_started = threading.Event()
replay_calls = []
def replay(stop_event: threading.Event) -> None:
"""模拟可由逐项检查点收敛的 pending 回放。"""
replay_calls.append(stop_event)
replay_started.set()
stop_event.wait()
chain._TransferChain__replay_pending = replay
chain.replay_pending()
assert replay_started.wait(timeout=1)
replay_thread = chain._replay_thread
chain.replay_pending()
assert chain._replay_thread is replay_thread
assert replay_calls == [chain._replay_stop_event]
assert chain.close_workers(timeout_seconds=1) is True
assert replay_thread.is_alive() is False
assert chain._replay_thread is None