mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-03 22:51:47 +08:00
refactor: own shutdown lifecycle boundaries
This commit is contained in:
@@ -155,7 +155,11 @@ class AgentServiceAdapter:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 必须返回 awaitable"
|
||||
)
|
||||
await result
|
||||
converged = await result
|
||||
if converged is False:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def cleanup(
|
||||
|
||||
@@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
|
||||
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
|
||||
self._accepting_tasks = True
|
||||
self._close_cancel_requested: set[asyncio.Task] = set()
|
||||
self.tools = [
|
||||
StructuredTool.from_function(
|
||||
coroutine=self._control_task,
|
||||
@@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
|
||||
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
|
||||
"""记录任务完成时间并取出异常避免未读取告警。"""
|
||||
cancel_requested = getattr(self, "_close_cancel_requested", None)
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.discard(task)
|
||||
record = self._tasks.get(task_id)
|
||||
if record:
|
||||
record.finished_at = datetime.now()
|
||||
@@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
@staticmethod
|
||||
async def _cancel_records(
|
||||
records: list[_SubAgentRuntimeTask],
|
||||
*,
|
||||
cancel_requested: Optional[set[asyncio.Task]] = None,
|
||||
) -> list[_SubAgentRuntimeTask]:
|
||||
"""取消一组任务,并返回等待上限内仍未收敛的记录。"""
|
||||
cancellable_tasks = [
|
||||
@@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
if cancellable_tasks:
|
||||
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
|
||||
for task in cancellable_tasks:
|
||||
if cancel_requested is not None and task in cancel_requested:
|
||||
continue
|
||||
task.cancel()
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.add(task)
|
||||
if not cancellable_tasks:
|
||||
return []
|
||||
|
||||
@@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
|
||||
return [record for record in records if record.task in pending]
|
||||
|
||||
async def close(self) -> None:
|
||||
"""取消脱离当前 Agent 回合的子代理任务。"""
|
||||
def seal(self) -> None:
|
||||
"""封住新的 detached 子代理提交,既有任务继续由记录表持有。"""
|
||||
self._accepting_tasks = False
|
||||
|
||||
async def close(self) -> bool:
|
||||
"""有限等待 detached 子代理;超时保留记录并返回 False。"""
|
||||
self.seal()
|
||||
if not hasattr(self, "_close_cancel_requested"):
|
||||
self._close_cancel_requested = set()
|
||||
unfinished_records = [
|
||||
record for record in self._tasks.values() if not record.task.done()
|
||||
]
|
||||
@@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(
|
||||
f"关闭子代理任务控制器,取消未完成任务: tasks={len(unfinished_records)}"
|
||||
)
|
||||
await self._cancel_records(unfinished_records)
|
||||
pending_records = await self._cancel_records(
|
||||
unfinished_records,
|
||||
cancel_requested=self._close_cancel_requested,
|
||||
)
|
||||
if pending_records:
|
||||
self._tasks = {
|
||||
record.task_id: record
|
||||
for record in pending_records
|
||||
}
|
||||
return False
|
||||
self._tasks.clear()
|
||||
self._close_cancel_requested.clear()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_description(
|
||||
@@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
|
||||
timeout = normalized_timeout_ms / 1000
|
||||
for step_index, spec in enumerate(specs, start=1):
|
||||
if not self._accepting_tasks:
|
||||
return records, "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
record = self._create_pipeline_record(spec)
|
||||
records.append(record)
|
||||
pipeline_description = self._pipeline_description(
|
||||
@@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
"""管理异步子代理任务。"""
|
||||
logger.info(f"收到子代理管控操作: action={action}")
|
||||
if action in {"start", "run", "pipeline"}:
|
||||
if not self._accepting_tasks:
|
||||
error = "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
return self._json_response({"success": False, "error": error})
|
||||
specs, error = self._normalize_specs(
|
||||
description=description,
|
||||
subagent_type=subagent_type,
|
||||
|
||||
+134
-33
@@ -436,6 +436,7 @@ class MoviePilotAgent:
|
||||
self._agent_started_at: Optional[datetime] = None
|
||||
self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None
|
||||
self._subagent_middlewares: tuple[Any, ...] = ()
|
||||
self._shutdown_started = False
|
||||
self._last_agent_cache_hit = False
|
||||
|
||||
# 流式token管理
|
||||
@@ -1751,10 +1752,25 @@ class MoviePilotAgent:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
def _seal_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> None:
|
||||
"""释放不再由 Agent 图持有的子代理控制器。"""
|
||||
"""同步封住子代理控制器的新任务入口,不等待既有任务退出。"""
|
||||
for middleware in middlewares:
|
||||
seal = getattr(middleware, "seal", None)
|
||||
if not callable(seal):
|
||||
continue
|
||||
try:
|
||||
seal()
|
||||
except Exception as error:
|
||||
logger.debug(f"封住子代理中间件失败: {error}")
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""关闭子代理控制器,并返回仍持有未收敛任务的实例。"""
|
||||
pending_middlewares = []
|
||||
for middleware in middlewares:
|
||||
close = getattr(middleware, "close", None)
|
||||
if not callable(close):
|
||||
@@ -1762,9 +1778,30 @@ class MoviePilotAgent:
|
||||
try:
|
||||
result = close()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
result = await result
|
||||
if result is False:
|
||||
pending_middlewares.append(middleware)
|
||||
except Exception as error:
|
||||
logger.debug(f"关闭子代理中间件失败: {error}")
|
||||
pending_middlewares.append(middleware)
|
||||
return tuple(pending_middlewares)
|
||||
|
||||
@staticmethod
|
||||
def _merge_subagent_middleware_owners(
|
||||
*groups: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""按对象身份合并当前图与延迟收敛控制器的 owner 集合。"""
|
||||
merged = []
|
||||
for group in groups:
|
||||
for middleware in group:
|
||||
if not any(middleware is existing for existing in merged):
|
||||
merged.append(middleware)
|
||||
return tuple(merged)
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""在任何异步等待前封住当前 Agent 的 detached 子代理提交。"""
|
||||
self._shutdown_started = True
|
||||
self._seal_subagent_middleware_instances(self._subagent_middlewares)
|
||||
|
||||
async def _cache_agent(
|
||||
self,
|
||||
@@ -1786,7 +1823,11 @@ class MoviePilotAgent:
|
||||
for replacement in subagent_middlewares
|
||||
)
|
||||
)
|
||||
await self._close_subagent_middleware_instances(previous_middlewares)
|
||||
if self._shutdown_started:
|
||||
self._seal_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
previous_middlewares
|
||||
)
|
||||
self._compiled_agent_bundle = _CompiledAgentBundle(
|
||||
signature=signature,
|
||||
agent=agent,
|
||||
@@ -1798,15 +1839,21 @@ class MoviePilotAgent:
|
||||
mcp_config_signature=mcp_config_signature,
|
||||
catalog_checked_at=datetime.now(),
|
||||
)
|
||||
self._subagent_middlewares = subagent_middlewares
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
pending_middlewares,
|
||||
subagent_middlewares,
|
||||
)
|
||||
return agent
|
||||
|
||||
async def _invalidate_cached_agent(self) -> None:
|
||||
"""使当前图失效,并释放只属于该图的子代理控制器。"""
|
||||
async def _invalidate_cached_agent(self) -> bool:
|
||||
"""使当前图失效,未收敛的子代理控制器继续由 Agent 持有。"""
|
||||
subagent_middlewares = self._subagent_middlewares
|
||||
self._subagent_middlewares = ()
|
||||
self._compiled_agent_bundle = None
|
||||
await self._close_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = pending_middlewares
|
||||
return not pending_middlewares
|
||||
|
||||
@staticmethod
|
||||
def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]:
|
||||
@@ -2003,9 +2050,13 @@ class MoviePilotAgent:
|
||||
if cached_agent:
|
||||
# 签名相同表示已编译图中的精确工具实例仍有效;新建快照仅用于复核。
|
||||
cached_bundle.catalog_checked_at = datetime.now()
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
temporary_subagent_middlewares = ()
|
||||
logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}")
|
||||
return cached_agent
|
||||
@@ -2123,14 +2174,22 @@ class MoviePilotAgent:
|
||||
temporary_subagent_middlewares = ()
|
||||
return cached_agent
|
||||
except asyncio.CancelledError:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
logger.error(f"创建 Agent 失败: {e}")
|
||||
raise
|
||||
|
||||
@@ -2539,14 +2598,20 @@ class MoviePilotAgent:
|
||||
)
|
||||
)
|
||||
|
||||
async def cleanup(self):
|
||||
async def cleanup(self) -> bool:
|
||||
"""
|
||||
清理智能体资源
|
||||
清理智能体资源;detached 子代理未收敛时保留 owner 并返回 False。
|
||||
"""
|
||||
await self._invalidate_cached_agent()
|
||||
self.begin_shutdown()
|
||||
if not await self._invalidate_cached_agent():
|
||||
logger.error(
|
||||
f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}"
|
||||
)
|
||||
return False
|
||||
self._pending_secret_confirmation = None
|
||||
self.protected_output_callback = None
|
||||
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2626,6 +2691,7 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {}
|
||||
self._session_cancel_requested: set[str] = set()
|
||||
self._close_finalizer_task: Optional[asyncio.Task] = None
|
||||
self._closed = False
|
||||
self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT
|
||||
# 接收门禁与队列写入共用一把锁,确保关闭开始后不会再创建 worker。
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
@@ -2693,22 +2759,33 @@ class AgentManager:
|
||||
async with self._lifecycle_lock:
|
||||
if self._accepting_tasks:
|
||||
return
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭")
|
||||
memory_manager.initialize()
|
||||
if not self._idle_cleanup_task or self._idle_cleanup_task.done():
|
||||
self._idle_cleanup_task = asyncio.create_task(
|
||||
self._cleanup_idle_sessions()
|
||||
)
|
||||
self._accepting_tasks = True
|
||||
self._closed = False
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> bool:
|
||||
"""
|
||||
关闭管理器
|
||||
关闭管理器,并诚实返回全部会话 owner 是否已经收敛。
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self._closed:
|
||||
return True
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
return
|
||||
return False
|
||||
# 门禁必须先关闭;锁内完成清理可阻止等待中的请求在收口期间重新入队。
|
||||
self._accepting_tasks = False
|
||||
# 子代理提交门禁同样必须在第一次 await 前关闭,否则 detached task
|
||||
# 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。
|
||||
for agent in self.active_agents.values():
|
||||
begin_shutdown = getattr(agent, "begin_shutdown", None)
|
||||
if callable(begin_shutdown):
|
||||
begin_shutdown()
|
||||
if self._idle_cleanup_task:
|
||||
self._idle_cleanup_task.cancel()
|
||||
try:
|
||||
@@ -2751,8 +2828,8 @@ class AgentManager:
|
||||
self._session_workers.pop(session_id, None)
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if session_id not in timed_out_session_ids:
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
logger.error(
|
||||
"AgentManager 关闭时仍有 worker 未收敛,"
|
||||
f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束"
|
||||
@@ -2760,13 +2837,21 @@ class AgentManager:
|
||||
self._close_finalizer_task = asyncio.create_task(
|
||||
self._finish_deferred_close(timed_out_workers)
|
||||
)
|
||||
return
|
||||
return False
|
||||
|
||||
self._session_workers.clear()
|
||||
for agent in list(self.active_agents.values()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.clear()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 仍有 %d 个 detached 子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return False
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
return True
|
||||
|
||||
def _record_session_activity(self, session_id: str, user_id: str) -> None:
|
||||
"""
|
||||
@@ -3072,7 +3157,10 @@ class AgentManager:
|
||||
and isinstance(task.agent_factory, type)
|
||||
and not isinstance(existing_agent, task.agent_factory)
|
||||
):
|
||||
await existing_agent.cleanup()
|
||||
if await existing_agent.cleanup() is False:
|
||||
raise AgentManagerUnavailableError(
|
||||
f"Agent 会话 {session_id} 仍有子代理任务在停止"
|
||||
)
|
||||
self.active_agents.pop(session_id, None)
|
||||
|
||||
if session_id not in self.active_agents:
|
||||
@@ -3249,7 +3337,11 @@ class AgentManager:
|
||||
# 清理agent
|
||||
if session_id in self.active_agents:
|
||||
agent = self.active_agents[session_id]
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 仍有子代理 owner 未收敛,保留会话与记忆"
|
||||
)
|
||||
return
|
||||
del self.active_agents[session_id]
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
@@ -3290,9 +3382,14 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks.pop(session_id, None)
|
||||
self._session_queue_rejections.pop(session_id, None)
|
||||
self._session_last_queue_wait_ms.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
agent = self.active_agents.get(session_id)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛"
|
||||
)
|
||||
return
|
||||
self.active_agents.pop(session_id, None)
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
|
||||
@@ -3310,15 +3407,19 @@ class AgentManager:
|
||||
for session_id, worker in workers:
|
||||
if self._session_workers.get(session_id) is worker:
|
||||
self._session_workers.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
self._session_shutdown_pending.clear()
|
||||
self._session_cancel_requested.clear()
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
finally:
|
||||
self._close_finalizer_task = None
|
||||
|
||||
|
||||
@@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import (
|
||||
build_agent_capability_registry,
|
||||
should_run_agent_service,
|
||||
)
|
||||
from app.runtime.capabilities.model import CapabilityMaterializationState
|
||||
from app.runtime.capabilities.model import (
|
||||
CapabilityLifecycleState,
|
||||
CapabilityMaterializationState,
|
||||
)
|
||||
from app.runtime.capabilities.runtime import CapabilityRuntime
|
||||
|
||||
|
||||
@@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None:
|
||||
await close()
|
||||
|
||||
|
||||
async def begin_agent_shutdown() -> None:
|
||||
"""不可逆关闭首用闸门,并等待全部同步及异步能力释放。"""
|
||||
try:
|
||||
await _ensure_runtime().shutdown_async(reason="application_shutdown")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
async def begin_agent_shutdown() -> bool:
|
||||
"""不可逆关闭首用闸门,并返回 Agent service 是否真实收敛。"""
|
||||
runtime = _ensure_runtime()
|
||||
await runtime.shutdown_async(reason="application_shutdown")
|
||||
return (
|
||||
runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle
|
||||
is CapabilityLifecycleState.STOPPED
|
||||
)
|
||||
|
||||
+117
-26
@@ -3,7 +3,7 @@ import inspect
|
||||
import json
|
||||
import threading
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
|
||||
from contextvars import Context, copy_context
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
@@ -168,33 +168,130 @@ _blocking_semaphores = {
|
||||
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
|
||||
}
|
||||
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.Lock()
|
||||
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
|
||||
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.RLock()
|
||||
_blocking_executor_accepting = True
|
||||
|
||||
|
||||
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
|
||||
"""按桶懒加载线程池,避免在导入阶段创建过多 worker。"""
|
||||
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
|
||||
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executor = _blocking_futures.pop(future, None)
|
||||
if executor is None or executor not in _blocking_retiring_executors:
|
||||
return
|
||||
if executor not in _blocking_futures.values():
|
||||
_blocking_retiring_executors.discard(executor)
|
||||
|
||||
|
||||
def _submit_blocking_call(
|
||||
bucket: str,
|
||||
bound_call: Callable[[], Any],
|
||||
) -> ConcurrentFuture[Any]:
|
||||
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
|
||||
context = copy_context()
|
||||
with _blocking_executor_lock:
|
||||
if not _blocking_executor_accepting:
|
||||
raise RuntimeError("Agent 工具阻塞执行器正在关闭,不能再提交新任务")
|
||||
executor = _blocking_executors.get(bucket)
|
||||
if executor:
|
||||
return executor
|
||||
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
return executor
|
||||
if executor is None:
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(executor.submit, context.run, bound_call)
|
||||
_blocking_futures[future] = executor
|
||||
future.add_done_callback(_discard_blocking_future)
|
||||
return future
|
||||
|
||||
|
||||
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
|
||||
"""关闭 Agent 工具阻塞线程池,释放长期运行进程或测试环境中的 worker。"""
|
||||
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
|
||||
"""撤销活动 executor 的提交资格,并保留其运行 Future 对应的 owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executors = list(_blocking_executors.values())
|
||||
executors = tuple(_blocking_executors.values())
|
||||
_blocking_executors.clear()
|
||||
|
||||
_blocking_retiring_executors.update(executors)
|
||||
for executor in executors:
|
||||
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
executor.shutdown(wait=False, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
owned_executors = set(_blocking_futures.values())
|
||||
_blocking_retiring_executors.intersection_update(owned_executors)
|
||||
return executors
|
||||
|
||||
|
||||
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
|
||||
"""原子封住新阻塞工具提交,并请求取消尚未开始的同步调用。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
_blocking_executor_accepting = False
|
||||
_retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
|
||||
|
||||
def reopen_blocking_executors() -> bool:
|
||||
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
if _blocking_futures or _blocking_retiring_executors:
|
||||
return False
|
||||
_blocking_executor_accepting = True
|
||||
return True
|
||||
|
||||
|
||||
async def close_blocking_executors(
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
cancel_futures: bool = True,
|
||||
) -> bool:
|
||||
"""有限等待全部阻塞工具 Future,超时保留 Future 与 executor owner。"""
|
||||
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
futures = tuple(_blocking_futures)
|
||||
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
|
||||
if wrapped_futures:
|
||||
done, pending = await asyncio.wait(
|
||||
wrapped_futures,
|
||||
timeout=max(0.0, timeout_seconds),
|
||||
)
|
||||
if done:
|
||||
await asyncio.gather(*done, return_exceptions=True)
|
||||
for pending_future in pending:
|
||||
pending_future.add_done_callback(
|
||||
lambda completed: completed.exception()
|
||||
if not completed.cancelled()
|
||||
else None
|
||||
)
|
||||
|
||||
with _blocking_executor_lock:
|
||||
unfinished = tuple(
|
||||
future for future in _blocking_futures if not future.done()
|
||||
)
|
||||
retiring_count = len(_blocking_retiring_executors)
|
||||
if unfinished:
|
||||
logger.error(
|
||||
"Agent 阻塞工具未在 %.1f 秒内收敛:futures=%d,executors=%d",
|
||||
max(0.0, timeout_seconds),
|
||||
len(unfinished),
|
||||
retiring_count,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def shutdown_blocking_executors(
|
||||
*,
|
||||
wait: bool = True,
|
||||
cancel_futures: bool = False,
|
||||
) -> bool:
|
||||
"""同步清理测试 owner;非等待模式下保留尚未收敛的 executor 句柄。"""
|
||||
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
for executor in executors:
|
||||
if wait:
|
||||
executor.shutdown(wait=True, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
return not _blocking_futures and not _blocking_retiring_executors
|
||||
|
||||
|
||||
class ToolExecutionTimeoutError(TimeoutError):
|
||||
@@ -226,13 +323,7 @@ async def run_agent_blocking(
|
||||
|
||||
await semaphore.acquire()
|
||||
try:
|
||||
context = copy_context()
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(
|
||||
_get_blocking_executor(bucket_name).submit,
|
||||
context.run,
|
||||
bound_call,
|
||||
)
|
||||
future = _submit_blocking_call(bucket_name, bound_call)
|
||||
except Exception:
|
||||
semaphore.release()
|
||||
raise
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from contextvars import copy_context
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
|
||||
|
||||
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载插件实例并重新注册其命令、定时任务和 API。"""
|
||||
runtime_status = get_plugin_manager().reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
plugin_manager = get_plugin_manager()
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
|
||||
|
||||
def summarize_plugin(plugin: Any) -> dict[str, Any]:
|
||||
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
|
||||
|
||||
async def reload_runtime(target_id: str) -> object:
|
||||
"""通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
|
||||
mutation_context = copy_context()
|
||||
return await run_agent_blocking(
|
||||
"plugin",
|
||||
mutation_context.run,
|
||||
plugin_manager.reload_plugin_tree,
|
||||
target_id,
|
||||
)
|
||||
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
|
||||
)
|
||||
return result
|
||||
|
||||
with plugin_manager.suppress_plugin_monitor(plugin_id):
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
mutation=plugin_manager.mutation,
|
||||
package_write_guard=plugin_manager.suppress_plugin_monitor,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
return result.success, result.message, result.refreshed_only
|
||||
|
||||
|
||||
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
|
||||
from app.agent.tools.base import run_agent_blocking
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
|
||||
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [
|
||||
plugin for plugin in install_plugins if plugin != plugin_id
|
||||
]
|
||||
await config_oper.async_set(
|
||||
SystemConfigKey.UserInstalledPlugins,
|
||||
install_plugins,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
|
||||
@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
|
||||
)
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
if not await plugin_manager.async_save_plugin_config(
|
||||
plugin_id,
|
||||
next_config,
|
||||
):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
|
||||
@@ -40,4 +40,5 @@ def get_plugin_config_command() -> PluginConfigCommand:
|
||||
reload_runtime=manager.reload_plugin,
|
||||
publish_reset=publish_reset,
|
||||
refresh_registrations=refresh_registrations,
|
||||
mutation=manager.mutation,
|
||||
)
|
||||
|
||||
+153
-107
@@ -57,7 +57,10 @@ from app.api.dependencies.plugin import (
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.adapters.external.market import PluginHelper
|
||||
from app.adapters.system.plugin.package import PluginPackageManager
|
||||
from app.schemas.exception import PersistenceUnavailableError
|
||||
from app.schemas.exception import (
|
||||
PersistenceUnavailableError,
|
||||
PluginMutationRejectedError,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.api.context import get_background_task_registry, resolve_background_task_registry
|
||||
@@ -520,10 +523,15 @@ def reload_plugin(
|
||||
"""
|
||||
重新加载插件
|
||||
"""
|
||||
# 重新加载插件
|
||||
runtime_status = PluginManager().reload_plugin(plugin_id)
|
||||
# 注册插件服务
|
||||
register_plugin(plugin_id)
|
||||
plugin_manager = PluginManager()
|
||||
try:
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
# 重新加载插件
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
# 注册插件服务
|
||||
register_plugin(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
if runtime_status is _SchemaPluginRuntimeStatus.ACTIVE:
|
||||
return _SchemaResponse(success=True)
|
||||
return _SchemaResponse(
|
||||
@@ -600,14 +608,15 @@ async def install(
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
mutation=plugin_manager.mutation,
|
||||
package_write_guard=plugin_manager.suppress_plugin_monitor,
|
||||
)
|
||||
result = await command.execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=bool(force),
|
||||
)
|
||||
with plugin_manager.suppress_plugin_monitor(plugin_id):
|
||||
result = await command.execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=bool(force),
|
||||
)
|
||||
if not result.success:
|
||||
return _SchemaResponse(success=False, message=result.message)
|
||||
return _SchemaResponse(success=True)
|
||||
@@ -888,11 +897,12 @@ async def save_plugin_folders(
|
||||
保存插件文件夹分组配置
|
||||
"""
|
||||
try:
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(success=True)
|
||||
with PluginManager().mutation("保存插件文件夹配置"):
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(success=True)
|
||||
except PersistenceUnavailableError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -909,18 +919,26 @@ async def create_plugin_folder(
|
||||
"""
|
||||
创建新的插件文件夹
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
if folder_name not in folders:
|
||||
folders[folder_name] = []
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 创建成功"
|
||||
)
|
||||
else:
|
||||
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 已存在")
|
||||
try:
|
||||
with PluginManager().mutation(f"创建插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
if folder_name not in folders:
|
||||
folders[folder_name] = []
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 创建成功"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"文件夹 '{folder_name}' 已存在",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.delete(
|
||||
@@ -932,15 +950,26 @@ async def delete_plugin_folder(
|
||||
"""
|
||||
删除插件文件夹
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
if folder_name in folders:
|
||||
del folders[folder_name]
|
||||
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 删除成功"
|
||||
)
|
||||
else:
|
||||
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 不存在")
|
||||
try:
|
||||
with PluginManager().mutation(f"删除插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
if folder_name in folders:
|
||||
del folders[folder_name]
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 删除成功"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"文件夹 '{folder_name}' 不存在",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.put(
|
||||
@@ -956,12 +985,22 @@ async def update_folder_plugins(
|
||||
"""
|
||||
更新指定文件夹中的插件列表
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
folders[folder_name] = plugin_ids
|
||||
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 中的插件已更新"
|
||||
)
|
||||
try:
|
||||
with PluginManager().mutation(f"更新插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
folders[folder_name] = plugin_ids
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
message=f"文件夹 '{folder_name}' 中的插件已更新",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -975,23 +1014,24 @@ def clone_plugin(
|
||||
"""
|
||||
创建插件分身
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
try:
|
||||
success, message = PluginManager().clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.suffix,
|
||||
name=clone_data.name,
|
||||
description=clone_data.description,
|
||||
version=clone_data.version,
|
||||
icon=clone_data.icon,
|
||||
)
|
||||
with plugin_manager.mutation(f"创建插件 {plugin_id} 分身"):
|
||||
success, message = plugin_manager.clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.suffix,
|
||||
name=clone_data.name,
|
||||
description=clone_data.description,
|
||||
version=clone_data.version,
|
||||
icon=clone_data.icon,
|
||||
)
|
||||
|
||||
if success:
|
||||
# 分身服务已完成运行态加载,此处只补齐宿主注册。
|
||||
register_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return _SchemaResponse(success=True, message="插件分身创建成功")
|
||||
else:
|
||||
if success:
|
||||
# 分身服务已完成运行态加载,此处只补齐宿主注册。
|
||||
register_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return _SchemaResponse(success=True, message="插件分身创建成功")
|
||||
return _SchemaResponse(success=False, message=message)
|
||||
except Exception as e:
|
||||
logger.error(f"创建插件分身失败:{str(e)}")
|
||||
@@ -1034,54 +1074,60 @@ def uninstall_plugin(
|
||||
卸载插件
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"请先卸载该插件的分身:{instance_ids}",
|
||||
)
|
||||
config_oper = get_configured_system_config()
|
||||
# 删除已安装信息
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
for plugin in install_plugins:
|
||||
if plugin == plugin_id:
|
||||
install_plugins.remove(plugin)
|
||||
break
|
||||
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
# 移除插件API
|
||||
remove_plugin_api(plugin_id)
|
||||
# 移除插件服务
|
||||
remove_plugin_job(plugin_id)
|
||||
# 判断是否为分身
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif getattr(plugin_class, "is_clone", False):
|
||||
# 如果是分身插件,则删除分身数据和配置
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
# 删除分身文件
|
||||
plugin_base_dir = (
|
||||
get_api_runtime_config_snapshot().root_path
|
||||
/ "app"
|
||||
/ "plugins"
|
||||
/ plugin_id.lower()
|
||||
)
|
||||
if plugin_base_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(plugin_base_dir)
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception as e:
|
||||
logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}")
|
||||
# 从插件文件夹中移除该插件
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
# 移除插件
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
return _SchemaResponse(success=True)
|
||||
try:
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"请先卸载该插件的分身:{instance_ids}",
|
||||
)
|
||||
config_oper = get_configured_system_config()
|
||||
# 删除已安装信息
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
for plugin in install_plugins:
|
||||
if plugin == plugin_id:
|
||||
install_plugins.remove(plugin)
|
||||
break
|
||||
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
# 移除插件API
|
||||
remove_plugin_api(plugin_id)
|
||||
# 移除插件服务
|
||||
remove_plugin_job(plugin_id)
|
||||
# 判断是否为分身
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif getattr(plugin_class, "is_clone", False):
|
||||
# 如果是分身插件,则删除分身数据和配置
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
# 删除分身文件
|
||||
plugin_base_dir = (
|
||||
get_api_runtime_config_snapshot().root_path
|
||||
/ "app"
|
||||
/ "plugins"
|
||||
/ plugin_id.lower()
|
||||
)
|
||||
if plugin_base_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(plugin_base_dir)
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}"
|
||||
)
|
||||
# 从插件文件夹中移除该插件
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
# 移除插件
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
return _SchemaResponse(success=True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str):
|
||||
|
||||
@@ -4,7 +4,9 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Any, ContextManager
|
||||
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -29,6 +31,7 @@ class PluginConfigCommand:
|
||||
reload_runtime: Callable[[str], Any],
|
||||
publish_reset: Callable[[str], Any],
|
||||
refresh_registrations: Callable[[str], Any],
|
||||
mutation: Callable[[str], ContextManager[None]],
|
||||
) -> None:
|
||||
"""保存插件管理 Facade 和运行时注册刷新端口。"""
|
||||
self._save_config = save_config
|
||||
@@ -39,21 +42,30 @@ class PluginConfigCommand:
|
||||
self._reload_runtime = reload_runtime
|
||||
self._publish_reset = publish_reset
|
||||
self._refresh_registrations = refresh_registrations
|
||||
self._mutation = mutation
|
||||
|
||||
def update(self, plugin_id: str, config: dict) -> PluginConfigResult:
|
||||
"""保存配置并按既有顺序重新初始化实例及运行时注册。"""
|
||||
if not self._save_config(plugin_id, config, False):
|
||||
return PluginConfigResult(False, "插件配置保存失败")
|
||||
self._initialize(plugin_id, config)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
try:
|
||||
with self._mutation(f"更新插件 {plugin_id} 配置"):
|
||||
if not self._save_config(plugin_id, config, False):
|
||||
return PluginConfigResult(False, "插件配置保存失败")
|
||||
self._initialize(plugin_id, config)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginConfigResult(False, str(error))
|
||||
|
||||
def reset(self, plugin_id: str) -> PluginConfigResult:
|
||||
"""通知插件补偿后停止实例、删除配置数据并重建运行态。"""
|
||||
self._publish_reset(plugin_id)
|
||||
self._stop(plugin_id)
|
||||
self._delete_config(plugin_id, True)
|
||||
self._delete_data(plugin_id, True)
|
||||
self._reload_runtime(plugin_id)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
try:
|
||||
with self._mutation(f"重置插件 {plugin_id} 配置和数据"):
|
||||
self._publish_reset(plugin_id)
|
||||
self._stop(plugin_id)
|
||||
self._delete_config(plugin_id, True)
|
||||
self._delete_data(plugin_id, True)
|
||||
self._reload_runtime(plugin_id)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginConfigResult(False, str(error))
|
||||
|
||||
@@ -5,11 +5,12 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
from typing import Any, ContextManager, Optional
|
||||
|
||||
from app.schemas.exception import PersistenceUnavailableError
|
||||
from app.application.plugin.lifecycle import plugin_lifecycle
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
InstalledPluginsReader = Callable[[], list[str]]
|
||||
@@ -25,6 +26,8 @@ PackageCheckpointAction = Callable[[Any], Awaitable[object]]
|
||||
InstallReporter = Callable[[str, Optional[str]], Awaitable[object]]
|
||||
PluginReloader = Callable[[str], Awaitable[object]]
|
||||
PluginRegistrationRefresher = Callable[[str], Awaitable[object]]
|
||||
PluginMutationAdmission = Callable[[str], ContextManager[None]]
|
||||
PluginPackageWriteGuard = Callable[[str], ContextManager[None]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -94,6 +97,8 @@ class PluginInstallCommand:
|
||||
install_reporter: InstallReporter,
|
||||
plugin_reloader: PluginReloader,
|
||||
registration_refresher: PluginRegistrationRefresher,
|
||||
mutation: PluginMutationAdmission,
|
||||
package_write_guard: PluginPackageWriteGuard,
|
||||
) -> None:
|
||||
"""保存安装用例所需端口,不绑定数据库、网络或运行时实现。"""
|
||||
self._installed_plugins_reader = installed_plugins_reader
|
||||
@@ -107,6 +112,8 @@ class PluginInstallCommand:
|
||||
self._install_reporter = install_reporter
|
||||
self._plugin_reloader = plugin_reloader
|
||||
self._registration_refresher = registration_refresher
|
||||
self._mutation = mutation
|
||||
self._package_write_guard = package_write_guard
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
@@ -120,20 +127,29 @@ class PluginInstallCommand:
|
||||
state = _InstallState()
|
||||
async with plugin_lifecycle.hold(plugin_id):
|
||||
try:
|
||||
return await self._execute_locked(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=force,
|
||||
state=state,
|
||||
with self._mutation(f"安装插件 {plugin_id}"):
|
||||
with self._package_write_guard(plugin_id):
|
||||
try:
|
||||
return await self._execute_locked(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=force,
|
||||
state=state,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await self._rollback_cancelled(
|
||||
plugin_id=plugin_id,
|
||||
original_plugins=state.original_plugins,
|
||||
state=state,
|
||||
)
|
||||
raise
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginInstallResult(
|
||||
success=False,
|
||||
message=str(error),
|
||||
failure_stage="admission",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await self._rollback_cancelled(
|
||||
plugin_id=plugin_id,
|
||||
original_plugins=state.original_plugins,
|
||||
state=state,
|
||||
)
|
||||
raise
|
||||
|
||||
async def _execute_locked(
|
||||
self,
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
|
||||
from typing import Any, ContextManager, Protocol
|
||||
|
||||
class PluginRuntime(Protocol):
|
||||
"""声明入口层消费的插件宿主能力。"""
|
||||
|
||||
def mutation(self, operation: str) -> ContextManager[None]:
|
||||
"""为完整插件可变事务取得停机准入 lease。"""
|
||||
...
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""允许兼容门面按既有 V3 方法名访问插件宿主能力。"""
|
||||
|
||||
|
||||
+170
-29
@@ -191,9 +191,16 @@ class TransferFailureNotificationAggregator:
|
||||
NOTIFICATION_DEBOUNCE_SECONDS = 30
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化分组缓冲和定时器。"""
|
||||
"""初始化分组缓冲、回调、定时器与关闭状态。"""
|
||||
self._buffers: dict[str, list[TransferFailureNotification]] = {}
|
||||
self._callbacks: dict[
|
||||
str,
|
||||
Callable[[list[TransferFailureNotification]], None],
|
||||
] = {}
|
||||
self._timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._generations: dict[str, int] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._closed = False
|
||||
|
||||
def schedule(
|
||||
self,
|
||||
@@ -204,48 +211,128 @@ class TransferFailureNotificationAggregator:
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
"""从整理线程安全地把失败快照加入事件循环中的聚合缓冲。"""
|
||||
loop.call_soon_threadsafe(
|
||||
self._schedule_on_loop,
|
||||
group_key,
|
||||
notification,
|
||||
callback,
|
||||
loop,
|
||||
)
|
||||
# 先在调用线程登记快照,关闭流程才能覆盖已接收但尚未进入事件循环的通知。
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("整理失败通知聚合器正在关闭,不能再接收通知")
|
||||
self._buffers.setdefault(group_key, []).append(notification)
|
||||
self._callbacks[group_key] = callback
|
||||
generation = self._generations.get(group_key, 0) + 1
|
||||
self._generations[group_key] = generation
|
||||
try:
|
||||
loop.call_soon_threadsafe(
|
||||
self._schedule_on_loop,
|
||||
group_key,
|
||||
generation,
|
||||
callback,
|
||||
loop,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"创建整理失败通知聚合定时器失败,将立即发送 "
|
||||
f"(group={group_key}): {err}"
|
||||
)
|
||||
self.flush(group_key, generation, callback)
|
||||
|
||||
def _schedule_on_loop(
|
||||
self,
|
||||
group_key: str,
|
||||
notification: TransferFailureNotification,
|
||||
generation: int,
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
"""在所属事件循环中更新缓冲并重置静默窗口。"""
|
||||
self._buffers.setdefault(group_key, []).append(notification)
|
||||
timer = self._timers.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
self._timers[group_key] = loop.call_later(
|
||||
self.NOTIFICATION_DEBOUNCE_SECONDS,
|
||||
self.flush,
|
||||
group_key,
|
||||
callback,
|
||||
)
|
||||
"""在所属事件循环中为已登记缓冲重置静默窗口。"""
|
||||
schedule_error: Optional[Exception] = None
|
||||
with self._lock:
|
||||
if (
|
||||
self._closed
|
||||
or group_key not in self._buffers
|
||||
or self._generations.get(group_key) != generation
|
||||
):
|
||||
return
|
||||
timer = self._timers.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
try:
|
||||
self._timers[group_key] = loop.call_later(
|
||||
self.NOTIFICATION_DEBOUNCE_SECONDS,
|
||||
self.flush,
|
||||
group_key,
|
||||
generation,
|
||||
callback,
|
||||
)
|
||||
except Exception as err:
|
||||
schedule_error = err
|
||||
if schedule_error is not None:
|
||||
logger.error(
|
||||
f"创建整理失败通知聚合定时器失败,将立即发送 "
|
||||
f"(group={group_key}): {schedule_error}"
|
||||
)
|
||||
self.flush(group_key, generation, callback)
|
||||
|
||||
def flush(
|
||||
self,
|
||||
group_key: str,
|
||||
generation: int,
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
) -> None:
|
||||
"""发送一个分组内的聚合结果并释放缓冲。"""
|
||||
notifications = self._buffers.pop(group_key, [])
|
||||
self._timers.pop(group_key, None)
|
||||
with self._lock:
|
||||
# 新通知已登记但 timer 重置回调尚未执行时,旧代不得提前发送新批次。
|
||||
if self._generations.get(group_key) != generation:
|
||||
return
|
||||
notifications = self._buffers.pop(group_key, [])
|
||||
self._callbacks.pop(group_key, None)
|
||||
timer = self._timers.pop(group_key, None)
|
||||
self._generations.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
if not notifications:
|
||||
return
|
||||
self._deliver(group_key, notifications, callback)
|
||||
|
||||
@staticmethod
|
||||
def _deliver(
|
||||
group_key: str,
|
||||
notifications: list[TransferFailureNotification],
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
) -> None:
|
||||
"""调用聚合通知回调,并统一观察发送异常。"""
|
||||
try:
|
||||
callback(notifications)
|
||||
except Exception as err:
|
||||
logger.error(f"发送整理失败聚合通知失败 (group={group_key}): {err}")
|
||||
|
||||
def close(self) -> None:
|
||||
"""停止接收新通知,取消定时器并同步发送全部已缓冲通知。"""
|
||||
with self._lock:
|
||||
if self._closed and not self._buffers and not self._timers:
|
||||
return
|
||||
self._closed = True
|
||||
timers = list(self._timers.values())
|
||||
pending = []
|
||||
orphaned = []
|
||||
for group_key, notifications in self._buffers.items():
|
||||
callback = self._callbacks.get(group_key)
|
||||
if callback is None:
|
||||
orphaned.append((group_key, len(notifications)))
|
||||
continue
|
||||
pending.append((group_key, notifications, callback))
|
||||
self._timers.clear()
|
||||
self._generations.clear()
|
||||
self._buffers.clear()
|
||||
self._callbacks.clear()
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
for group_key, notification_count in orphaned:
|
||||
logger.error(
|
||||
f"整理失败通知聚合缓冲缺少发送回调,无法刷新 "
|
||||
f"(group={group_key}, count={notification_count})"
|
||||
)
|
||||
for group_key, notifications, callback in pending:
|
||||
self._deliver(group_key, notifications, callback)
|
||||
|
||||
|
||||
# 作业锁:JobManager 与 TransferChain 共享,保护整理作业视图。
|
||||
job_lock = threading.Lock()
|
||||
@@ -267,7 +354,8 @@ class JobManager:
|
||||
# 记录仍由主程序整理线程直接执行的任务,避免把阻塞中的本地任务误判为失活
|
||||
_active_executions: set[Tuple[str, str]] = set()
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化当前进程内的整理作业状态。"""
|
||||
self._job_view = {}
|
||||
self._season_episodes = {}
|
||||
self._meta_to_media_ids = {}
|
||||
@@ -1011,25 +1099,68 @@ class JobManager:
|
||||
|
||||
class FailedRetryScheduler:
|
||||
"""
|
||||
负责失败整理记录的 debounce 聚合与 AI 重试调度。
|
||||
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。
|
||||
|
||||
缓冲不提供持久化保证;关闭时会取消尚未触发的记录,由上层 durable
|
||||
工作流在后续阶段承接需要跨进程保证的重试意图。
|
||||
"""
|
||||
|
||||
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
|
||||
super().__init__()
|
||||
self._retry_transfer_buffer: dict[str, list[int]] = {}
|
||||
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._retry_transfer_generations: dict[str, int] = {}
|
||||
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
|
||||
self._retry_transfer_lock = asyncio.Lock()
|
||||
self._closed = False
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> None:
|
||||
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
|
||||
async with self._retry_transfer_lock:
|
||||
self._closed = True
|
||||
timers = list(self._retry_transfer_timers.values())
|
||||
buffered_count = sum(
|
||||
len(history_ids)
|
||||
for history_ids in self._retry_transfer_buffer.values()
|
||||
)
|
||||
self._retry_transfer_timers.clear()
|
||||
self._retry_transfer_generations.clear()
|
||||
self._retry_transfer_buffer.clear()
|
||||
tasks = tuple(self._retry_transfer_tasks)
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
if buffered_count:
|
||||
logger.warning(
|
||||
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
|
||||
)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
|
||||
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
|
||||
if self._closed:
|
||||
return
|
||||
task = asyncio.create_task(
|
||||
self._flush_retry_transfer(group_key, generation),
|
||||
name="transfer.failed_retry.flush",
|
||||
)
|
||||
self._retry_transfer_tasks.add(task)
|
||||
task.add_done_callback(self._observe_retry_transfer_task)
|
||||
|
||||
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
|
||||
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
|
||||
self._retry_transfer_tasks.discard(task)
|
||||
if task.cancelled():
|
||||
return
|
||||
exception = task.exception()
|
||||
if exception is not None:
|
||||
logger.error(f"智能体重试整理后台任务异常: {exception}")
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_transfer_template_context(
|
||||
@@ -1054,7 +1185,7 @@ class FailedRetryScheduler:
|
||||
template_context=template_context,
|
||||
)
|
||||
|
||||
async def schedule_retry(self, history_id: int, group_key: str = ""):
|
||||
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
|
||||
"""
|
||||
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
|
||||
"""
|
||||
@@ -1062,6 +1193,8 @@ class FailedRetryScheduler:
|
||||
group_key = f"_default_{history_id}"
|
||||
|
||||
async with self._retry_transfer_lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
|
||||
if group_key not in self._retry_transfer_buffer:
|
||||
self._retry_transfer_buffer[group_key] = []
|
||||
if history_id not in self._retry_transfer_buffer[group_key]:
|
||||
@@ -1075,18 +1208,26 @@ class FailedRetryScheduler:
|
||||
self._retry_transfer_timers[group_key].cancel()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
generation = self._retry_transfer_generations.get(group_key, 0) + 1
|
||||
self._retry_transfer_generations[group_key] = generation
|
||||
self._retry_transfer_timers[group_key] = loop.call_later(
|
||||
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
|
||||
lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)),
|
||||
self._start_retry_transfer_task,
|
||||
group_key,
|
||||
generation,
|
||||
)
|
||||
|
||||
async def _flush_retry_transfer(self, group_key: str):
|
||||
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
|
||||
"""
|
||||
延迟定时器到期后,取出该分组的所有 history_id 并合并为一次 agent 调用。
|
||||
"""
|
||||
async with self._retry_transfer_lock:
|
||||
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
|
||||
if self._retry_transfer_generations.get(group_key) != generation:
|
||||
return
|
||||
history_ids = self._retry_transfer_buffer.pop(group_key, [])
|
||||
self._retry_transfer_timers.pop(group_key, None)
|
||||
self._retry_transfer_generations.pop(group_key, None)
|
||||
|
||||
if not history_ids:
|
||||
return
|
||||
|
||||
+335
-72
@@ -2,9 +2,12 @@ import asyncio
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from concurrent.futures import CancelledError as FutureCancelledError
|
||||
from concurrent.futures import Future
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union, Dict, Callable, Any
|
||||
@@ -87,10 +90,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
文件整理处理链
|
||||
"""
|
||||
|
||||
# worker 在构造期启动;若中途失败,单例仍需先发布给 lifespan 清理入口。
|
||||
_retain_failed_singleton = True
|
||||
|
||||
CONFIG_WATCH = {
|
||||
"TRANSFER_THREADS",
|
||||
}
|
||||
|
||||
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
|
||||
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
|
||||
_QUEUE_STOP_SENTINEL = object()
|
||||
|
||||
@staticmethod
|
||||
def _transfer_result_payload(
|
||||
task: TransferTask,
|
||||
@@ -108,7 +118,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
"transfer_history_id": history_id,
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化文件整理处理链。"""
|
||||
super().__init__()
|
||||
# 主要媒体文件后缀
|
||||
@@ -141,7 +151,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._progress = ProgressHelper(ProgressKey.FileTransfer)
|
||||
# 队列相关状态
|
||||
self._threads = []
|
||||
self._retiring_threads: List[threading.Thread] = []
|
||||
self._queue_active = False
|
||||
# 每一代 worker 使用独立停止信号,避免热更新启动新 worker 后旧线程重新取任务
|
||||
self._worker_stop_event = threading.Event()
|
||||
# 生命周期操作串行化;状态锁只保护短临界区,不覆盖同步文件 I/O 等待
|
||||
self._worker_lifecycle_lock = threading.RLock()
|
||||
self._worker_state_lock = threading.RLock()
|
||||
self._closing = False
|
||||
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
|
||||
self._replay_thread: Optional[threading.Thread] = None
|
||||
self._replay_stop_event = threading.Event()
|
||||
self._active_tasks = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
@@ -149,33 +169,210 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
# 启动整理任务
|
||||
self.__init()
|
||||
|
||||
def __init(self):
|
||||
"""
|
||||
启动文件整理线程
|
||||
"""
|
||||
self._queue_active = True
|
||||
for i in range(self.runtime_config.transfer_threads):
|
||||
logger.info(f"启动文件整理线程 {i + 1} ...")
|
||||
thread = threading.Thread(
|
||||
target=self.__start_transfer, name=f"transfer-{i}", daemon=True
|
||||
def __init(self) -> bool:
|
||||
"""启动一代文件整理线程,并返回是否成功取得 worker 所有权。"""
|
||||
with self._worker_lifecycle_lock:
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.warning("文件整理链已进入关闭状态,拒绝重新启动 worker")
|
||||
return False
|
||||
self._retiring_threads = [
|
||||
thread for thread in self._retiring_threads if thread.is_alive()
|
||||
]
|
||||
alive_threads = [thread for thread in self._threads if thread.is_alive()]
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"上一代文件整理线程尚未收敛,拒绝并行启动新 worker:%s",
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
self._threads = alive_threads
|
||||
return False
|
||||
stop_event = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=self.__start_transfer,
|
||||
args=(stop_event,),
|
||||
name=f"transfer-{index}",
|
||||
daemon=True,
|
||||
)
|
||||
for index in range(self.runtime_config.transfer_threads)
|
||||
]
|
||||
self._worker_stop_event = stop_event
|
||||
self._threads = threads
|
||||
self._queue_active = True
|
||||
for index, thread in enumerate(threads):
|
||||
logger.info(f"启动文件整理线程 {index + 1} ...")
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def __join_threads(
|
||||
threads: List[threading.Thread], deadline: float
|
||||
) -> List[threading.Thread]:
|
||||
"""在统一截止时间内等待线程,返回仍未收敛且继续由调用方持有的线程。"""
|
||||
current_thread = threading.current_thread()
|
||||
for thread in threads:
|
||||
if thread is current_thread or not thread.is_alive():
|
||||
continue
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
return [thread for thread in threads if thread.is_alive()]
|
||||
|
||||
def __acquire_worker_lifecycle_lock(self, deadline: float) -> bool:
|
||||
"""在统一截止时间内取得 worker 生命周期锁,并兼容同线程 RLock 重入。"""
|
||||
return self._worker_lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
)
|
||||
|
||||
def __request_worker_stop(self) -> List[threading.Thread]:
|
||||
"""发布当前 worker 代的停止信号,并用哨兵唤醒空闲线程。"""
|
||||
with self._worker_state_lock:
|
||||
self._queue_active = False
|
||||
self._worker_stop_event.set()
|
||||
current_threads = list(self._threads)
|
||||
threads = [*self._retiring_threads, *current_threads]
|
||||
for _ in current_threads:
|
||||
self._queue.put(self._QUEUE_STOP_SENTINEL)
|
||||
return threads
|
||||
|
||||
def __stop(self, timeout_seconds: float = _WORKER_RESTART_TIMEOUT_SECONDS) -> bool:
|
||||
"""在锁等待与线程 join 的共享预算内停止当前 worker 代。"""
|
||||
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
||||
if not self.__acquire_worker_lifecycle_lock(deadline):
|
||||
logger.error(
|
||||
"未在 %.1f 秒内取得文件整理 worker 生命周期锁",
|
||||
max(0.0, timeout_seconds),
|
||||
)
|
||||
self._threads.append(thread)
|
||||
thread.start()
|
||||
return False
|
||||
try:
|
||||
threads = self.__request_worker_stop()
|
||||
alive_threads = self.__join_threads(threads, deadline)
|
||||
with self._worker_state_lock:
|
||||
self._threads = []
|
||||
self._retiring_threads = alive_threads
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"文件整理线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
return False
|
||||
logger.info("文件整理线程已停止")
|
||||
return True
|
||||
finally:
|
||||
self._worker_lifecycle_lock.release()
|
||||
|
||||
def __stop(self):
|
||||
def close_workers(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
|
||||
"""
|
||||
停止文件整理进程
|
||||
"""
|
||||
self._queue_active = False
|
||||
for thread in self._threads:
|
||||
thread.join()
|
||||
self._threads = []
|
||||
logger.info("文件整理线程已停止")
|
||||
关闭整理 worker 与 pending 回放,并拒绝后续队列写入。
|
||||
|
||||
def on_config_changed(self):
|
||||
这是宿主生命周期使用的同步边界。停止信号只能阻止线程领取下一项工作,不能
|
||||
取消已经进入同步文件或数据库 I/O 的调用;超过预算时保留活线程句柄并返回
|
||||
False,调用方据此避免过早释放仍被使用的数据库等下游资源。
|
||||
:param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数
|
||||
:return: 全部后台线程均已收敛时返回 True,否则返回 False
|
||||
"""
|
||||
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
||||
if not self.__acquire_worker_lifecycle_lock(deadline):
|
||||
logger.error(
|
||||
"未在 %.1f 秒内取得整理后台生命周期锁,关闭未开始",
|
||||
max(0.0, timeout_seconds),
|
||||
)
|
||||
return False
|
||||
try:
|
||||
with self._worker_state_lock:
|
||||
self._closing = True
|
||||
worker_threads = self.__request_worker_stop()
|
||||
replay_thread = self._replay_thread
|
||||
self._replay_stop_event.set()
|
||||
|
||||
alive_workers = self.__join_threads(worker_threads, deadline)
|
||||
alive_replays = self.__join_threads(
|
||||
[replay_thread] if replay_thread else [], deadline
|
||||
)
|
||||
with self._worker_state_lock:
|
||||
self._threads = []
|
||||
self._retiring_threads = alive_workers
|
||||
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
|
||||
self._replay_thread = None
|
||||
|
||||
alive_threads = [*alive_workers, *alive_replays]
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"整理后台线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
return False
|
||||
logger.info("文件整理 worker 与待处理回放线程已关闭")
|
||||
return True
|
||||
finally:
|
||||
self._worker_lifecycle_lock.release()
|
||||
|
||||
async def close(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
|
||||
"""
|
||||
收口整理线程、失败通知和 AI 重试,并返回依赖是否可以安全释放。
|
||||
|
||||
同步文件 I/O 在线程内无法被 asyncio 取消,因此先在线程池中执行有界
|
||||
``close_workers``。只有 worker 与 replay 全部退出后才关闭通知和重试;若
|
||||
超时则保留这些依赖,供仍在运行的整理回调继续使用。
|
||||
:param timeout_seconds: worker 与 replay 共享的最大等待秒数
|
||||
:return: 所有整理后台 owner 均已收敛时返回 True
|
||||
"""
|
||||
workers_closed = await asyncio.to_thread(
|
||||
self.close_workers,
|
||||
timeout_seconds,
|
||||
)
|
||||
if not workers_closed:
|
||||
return False
|
||||
self.failure_notification_aggregator.close()
|
||||
await self.retry_scheduler.close()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _observe_failed_retry_schedule(future: Future[None]) -> None:
|
||||
"""观察跨线程 AI 重试调度的完成结果,避免关闭竞态变成静默异常。"""
|
||||
try:
|
||||
future.result()
|
||||
except FutureCancelledError:
|
||||
return
|
||||
except Exception as err:
|
||||
logger.error(f"触发AI智能体重试整理失败: {err}")
|
||||
|
||||
def _schedule_failed_transfer_retry(
|
||||
self,
|
||||
history_id: int,
|
||||
group_key: str,
|
||||
) -> None:
|
||||
"""把失败历史提交到主事件循环,并持续持有和观察调度结果。"""
|
||||
retry_coroutine = self.retry_scheduler.schedule_retry(
|
||||
history_id,
|
||||
group_key=group_key,
|
||||
)
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
retry_coroutine,
|
||||
global_vars.loop,
|
||||
)
|
||||
except Exception as err:
|
||||
retry_coroutine.close()
|
||||
logger.error(f"触发AI智能体重试整理失败: {err}")
|
||||
return
|
||||
future.add_done_callback(self._observe_failed_retry_schedule)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{history_id}")
|
||||
|
||||
def on_config_changed(self) -> None:
|
||||
"""配置变更时重启文件整理线程。"""
|
||||
self.__stop()
|
||||
self.__init()
|
||||
with self._worker_lifecycle_lock:
|
||||
if self._closing:
|
||||
logger.info("文件整理链正在关闭,忽略 worker 配置热更新")
|
||||
return
|
||||
if not self.__stop(
|
||||
timeout_seconds=self._WORKER_RESTART_TIMEOUT_SECONDS
|
||||
):
|
||||
logger.warning(
|
||||
"旧文件整理 worker 仍在收尾;其停止信号保持有效,新一代接管后续队列"
|
||||
)
|
||||
self.__init()
|
||||
|
||||
def __default_callback(
|
||||
self, task: TransferTask, transferinfo: TransferInfo, /
|
||||
@@ -344,19 +541,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
and self.runtime_config.ai_agent_enable
|
||||
and self.runtime_config.ai_agent_retry_transfer
|
||||
):
|
||||
try:
|
||||
# 使用 download_hash 或源文件父目录作为分组键,
|
||||
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
|
||||
group_key = build_transfer_failure_group_key(task)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.retry_scheduler.schedule_retry(
|
||||
history.id, group_key=group_key
|
||||
),
|
||||
global_vars.loop,
|
||||
)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{history.id}")
|
||||
except Exception as e:
|
||||
logger.error(f"触发AI智能体重试整理失败: {e}")
|
||||
# 使用 download_hash 或源文件父目录作为分组键,
|
||||
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
|
||||
self._schedule_failed_transfer_retry(
|
||||
history.id,
|
||||
build_transfer_failure_group_key(task),
|
||||
)
|
||||
|
||||
# 返回失败
|
||||
ret_status = False
|
||||
@@ -653,7 +843,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
:param task: 任务信息
|
||||
:return: True表示任务已添加到队列,False表示任务无效或已存在(重复)
|
||||
"""
|
||||
return self._transfer_queue_service().put(task, self.__default_callback)
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.warning("文件整理链已关闭,拒绝新的队列任务")
|
||||
return False
|
||||
return self._transfer_queue_service().put(task, self.__default_callback)
|
||||
|
||||
def _transfer_queue_service(self) -> TransferQueueService:
|
||||
"""构建保持旧队列对象和私有兼容接缝的应用服务。"""
|
||||
@@ -667,26 +861,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
expire_tasks=self.__expire_stale_transfer_tasks,
|
||||
)
|
||||
|
||||
def replay_pending(self):
|
||||
def replay_pending(self) -> None:
|
||||
"""
|
||||
回放上次进程退出时仍未整理完的文件。
|
||||
|
||||
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
|
||||
挂死状态,同步执行会把整个启动流程堵住。
|
||||
"""
|
||||
threading.Thread(
|
||||
target=self.__replay_pending,
|
||||
name="MoviePilot-TransferReplay",
|
||||
daemon=True
|
||||
).start()
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.info("文件整理链正在关闭,跳过待处理文件回放")
|
||||
return
|
||||
if self._replay_thread and self._replay_thread.is_alive():
|
||||
logger.info("待处理文件回放已在运行,跳过重复启动")
|
||||
return
|
||||
stop_event = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self.__run_replay_pending,
|
||||
args=(stop_event,),
|
||||
name="MoviePilot-TransferReplay",
|
||||
daemon=True,
|
||||
)
|
||||
self._replay_stop_event = stop_event
|
||||
self._replay_thread = thread
|
||||
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
|
||||
thread.start()
|
||||
|
||||
def __replay_pending(self):
|
||||
def __run_replay_pending(self, stop_event: threading.Event) -> None:
|
||||
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
|
||||
try:
|
||||
self.__replay_pending(stop_event)
|
||||
finally:
|
||||
with self._worker_state_lock:
|
||||
if self._replay_thread is threading.current_thread():
|
||||
self._replay_thread = None
|
||||
|
||||
def __replay_pending(
|
||||
self, stop_event: Optional[threading.Event] = None
|
||||
) -> None:
|
||||
"""
|
||||
把落盘登记的待整理文件重新送回整理入口。
|
||||
|
||||
只回放「存储 + 源路径」这一最小事实,重新走完整的识别与整理流程,
|
||||
已经整理完成的由整理历史查重挡掉,因此不存在重复整理的问题。
|
||||
:param stop_event: 宿主关闭信号;只阻止处理下一条登记,不取消运行中的同步 I/O
|
||||
"""
|
||||
stop_event = stop_event or threading.Event()
|
||||
if stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
pendings = self._pendingoper.list_all()
|
||||
except Exception as err:
|
||||
@@ -697,8 +919,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
|
||||
replayed = 0
|
||||
for storage, src_path in pendings:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
try:
|
||||
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
|
||||
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
|
||||
if stop_event.is_set():
|
||||
break
|
||||
if not fileitem:
|
||||
if should_discard:
|
||||
# 源文件确认已消失,注销登记避免每次启动重复回放
|
||||
@@ -708,7 +935,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
replayed += 1
|
||||
except Exception as err:
|
||||
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
|
||||
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
|
||||
if stop_event.is_set():
|
||||
logger.info(
|
||||
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
|
||||
replayed,
|
||||
)
|
||||
else:
|
||||
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
|
||||
|
||||
@staticmethod
|
||||
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
|
||||
@@ -876,21 +1109,69 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self.jobview.try_remove_job(task)
|
||||
self._finish_scrape_batch_task(task)
|
||||
|
||||
def __start_transfer(self):
|
||||
def __settle_transfer_progress_if_idle(self) -> None:
|
||||
"""在没有 active 或未结算真实任务时结束进度并重置本批计数。"""
|
||||
with task_lock:
|
||||
# unfinished_tasks 同时覆盖队列内任务和已被其他 worker 取走、尚未来得及
|
||||
# 登记 active 的任务。持有 Queue 自身互斥锁完成判断和计数重置,使并发
|
||||
# enqueue 只能发生在旧批次归零之后;仍在 deque 中的停止哨兵不算真实任务。
|
||||
with self._queue.all_tasks_done:
|
||||
queued_stop_sentinels = sum(
|
||||
item is self._QUEUE_STOP_SENTINEL
|
||||
for item in self._queue.queue
|
||||
)
|
||||
has_unsettled_tasks = (
|
||||
self._queue.unfinished_tasks > queued_stop_sentinels
|
||||
)
|
||||
if (
|
||||
self._active_tasks != 0
|
||||
or self._processed_num <= 0
|
||||
or has_unsettled_tasks
|
||||
):
|
||||
return
|
||||
processed_num = self._processed_num
|
||||
fail_num = self._fail_num
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
__end_msg = (
|
||||
f"整理队列处理完成,共整理 {processed_num} 个文件,"
|
||||
f"失败 {fail_num} 个"
|
||||
)
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
|
||||
def __start_transfer(self, stop_event: threading.Event) -> None:
|
||||
"""
|
||||
处理队列
|
||||
处理当前 worker 代的队列,停止后不领取下一项任务。
|
||||
|
||||
:param stop_event: 当前 worker 代专属停止信号,热更新后不会被重新清除
|
||||
"""
|
||||
while not global_vars.is_system_stopped and self._queue_active:
|
||||
while not global_vars.is_system_stopped and not stop_event.is_set():
|
||||
try:
|
||||
item: TransferQueue = self._queue.get(
|
||||
block=True, timeout=self._transfer_interval
|
||||
)
|
||||
if item is self._QUEUE_STOP_SENTINEL:
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
if stop_event.is_set() or global_vars.is_system_stopped:
|
||||
break
|
||||
continue
|
||||
if stop_event.is_set() or global_vars.is_system_stopped:
|
||||
# 关闭信号与 queue.get 竞态时,把尚未处理的任务放回队列;其
|
||||
# TransferPending 登记保持不变,供同进程重启 worker 或下次启动回放。
|
||||
self._queue.put(item)
|
||||
self._queue.task_done()
|
||||
break
|
||||
if not item:
|
||||
continue
|
||||
|
||||
task = item.task
|
||||
if not task:
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
continue
|
||||
|
||||
# 文件信息
|
||||
@@ -966,18 +1247,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
with task_lock:
|
||||
# 减少运行中的任务数
|
||||
self._active_tasks -= 1
|
||||
# 检查是否所有任务都已完成且队列为空
|
||||
if self._active_tasks == 0 and self._queue.empty():
|
||||
# 结束进度
|
||||
__end_msg = f"整理队列处理完成,共整理 {self._processed_num} 个文件,失败 {self._fail_num} 个"
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
|
||||
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
|
||||
except queue.Empty:
|
||||
# 即使队列空了,如果还有任务在运行,也不应该结束进度
|
||||
@@ -1113,18 +1383,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
and self.runtime_config.ai_agent_enable
|
||||
and self.runtime_config.ai_agent_retry_transfer
|
||||
):
|
||||
try:
|
||||
# 使用 download_hash 或源文件父目录作为分组键
|
||||
group_key = build_transfer_failure_group_key(task)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.retry_scheduler.schedule_retry(
|
||||
his.id, group_key=group_key
|
||||
),
|
||||
global_vars.loop,
|
||||
)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{his.id}")
|
||||
except Exception as e:
|
||||
logger.error(f"触发AI智能体重试整理失败: {e}")
|
||||
# 使用 download_hash 或源文件父目录作为分组键
|
||||
self._schedule_failed_transfer_retry(
|
||||
his.id,
|
||||
build_transfer_failure_group_key(task),
|
||||
)
|
||||
|
||||
return False, "未识别到媒体信息"
|
||||
|
||||
|
||||
@@ -14,14 +14,24 @@ class Singleton(abc.ABCMeta, type):
|
||||
def get_existing_instance(cls, *args, **kwargs):
|
||||
"""按相同参数返回已创建实例,不触发初始化"""
|
||||
key = (cls, args, frozenset(kwargs.items()))
|
||||
return cls._instances.get(key)
|
||||
with cls._lock:
|
||||
return cls._instances.get(key)
|
||||
|
||||
def __call__(cls, *args, **kwargs):
|
||||
"""按类和构造参数创建或复用实例。"""
|
||||
key = (cls, args, frozenset(kwargs.items()))
|
||||
with cls._lock:
|
||||
if key not in cls._instances:
|
||||
cls._instances[key] = super().__call__(*args, **kwargs)
|
||||
if getattr(cls, "_retain_failed_singleton", False):
|
||||
# 启动线程的 lifecycle owner 必须先发布身份再执行 __init__;
|
||||
# 构造中途抛错时保留实例,启动失败清理才能找到已创建的 owner。
|
||||
instance = cls.__new__(cls, *args, **kwargs)
|
||||
if not isinstance(instance, cls):
|
||||
return instance
|
||||
cls._instances[key] = instance
|
||||
cls.__init__(instance, *args, **kwargs)
|
||||
else:
|
||||
cls._instances[key] = super().__call__(*args, **kwargs)
|
||||
return cls._instances[key]
|
||||
|
||||
|
||||
@@ -42,13 +52,23 @@ class SingletonClass(abc.ABCMeta, type):
|
||||
|
||||
def get_existing_instance(cls):
|
||||
"""返回已创建实例,不触发初始化"""
|
||||
return cls._instances.get(cls)
|
||||
with cls._lock:
|
||||
return cls._instances.get(cls)
|
||||
|
||||
def __call__(cls, *args, **kwargs):
|
||||
"""按类创建或复用唯一实例。"""
|
||||
with cls._lock:
|
||||
if cls not in cls._instances:
|
||||
cls._instances[cls] = super().__call__(*args, **kwargs)
|
||||
if getattr(cls, "_retain_failed_singleton", False):
|
||||
# 与参数化单例保持相同的 owner 发布顺序;锁会阻止其他线程
|
||||
# 在 __init__ 返回或抛错前读取半构造实例。
|
||||
instance = cls.__new__(cls, *args, **kwargs)
|
||||
if not isinstance(instance, cls):
|
||||
return instance
|
||||
cls._instances[cls] = instance
|
||||
cls.__init__(instance, *args, **kwargs)
|
||||
else:
|
||||
cls._instances[cls] = super().__call__(*args, **kwargs)
|
||||
return cls._instances[cls]
|
||||
|
||||
|
||||
|
||||
+399
-57
@@ -1,4 +1,5 @@
|
||||
import time
|
||||
import threading
|
||||
import traceback
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
@@ -27,6 +28,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
||||
"""
|
||||
# watcher/scheduler 在构造期启动;异常时保留实例供启动失败屏障收口。
|
||||
_retain_failed_singleton = True
|
||||
# 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
|
||||
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
|
||||
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
|
||||
@@ -54,11 +57,25 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
REBUILD_KEY_PREFIX = "rebuild:"
|
||||
PROBE_KEY = "probe"
|
||||
PENDING_KEY = "pending"
|
||||
RELOAD_STOP_TIMEOUT = 30.0
|
||||
LIFECYCLE_CLOSE_TIMEOUT = 90.0
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化目录监控依赖、owner 注册表与当前 lifespan 状态。"""
|
||||
super().__init__()
|
||||
# 生命周期操作串行化;永久关闭请求用 Event 提前封口,不能被阻塞 I/O 挡住。
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._owner_lock = threading.Lock()
|
||||
self._work_stop_event = threading.Event()
|
||||
self._shutdown_event = threading.Event()
|
||||
self._closed = False
|
||||
self._compensation_threads: Dict[int, threading.Thread] = {}
|
||||
self._scheduler_shutdown_thread: Optional[threading.Thread] = None
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 已请求停止但尚未退出的 watcher,只保留 owner 身份,不再参与健康检查
|
||||
self._retired_watchers = []
|
||||
# 本地目录监控列表读写锁
|
||||
self._watcher_lock = Lock()
|
||||
# 启动失败待重试的本地监控配置
|
||||
@@ -87,9 +104,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
|
||||
alert_cb=self.__poller_alert)
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
if not self.init():
|
||||
raise RuntimeError("目录监控 owner 初始化未收敛")
|
||||
|
||||
def on_config_changed(self):
|
||||
def on_config_changed(self) -> None:
|
||||
"""配置变化时重建监控;lifespan 封口后拒绝重新启动。"""
|
||||
if self._shutdown_event.is_set():
|
||||
logger.info("目录监控已进入生命周期封口,忽略配置热更新")
|
||||
return
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
@@ -118,6 +140,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
|
||||
|
||||
@staticmethod
|
||||
@@ -134,24 +158,61 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
return SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
|
||||
def init(self):
|
||||
def init(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""在旧 owner 收敛后启动一代监控,关闭中的生命周期拒绝重开。"""
|
||||
if self._shutdown_event.is_set():
|
||||
logger.info("目录监控生命周期已关闭,跳过启动")
|
||||
return False
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("目录监控未在 %.1f 秒内取得重载所有权", max(0.0, timeout))
|
||||
return False
|
||||
try:
|
||||
if self._shutdown_event.is_set():
|
||||
return False
|
||||
if not self.__stop_owned(deadline=deadline, close=False):
|
||||
logger.error("旧目录监控 owner 未收敛,取消本次配置热更新")
|
||||
return False
|
||||
if self._shutdown_event.is_set():
|
||||
return False
|
||||
if not self._recovery.reopen():
|
||||
logger.error("目录监控恢复线程仍未退出,取消本次配置热更新")
|
||||
return False
|
||||
if self._shutdown_event.is_set():
|
||||
self._recovery.request_stop()
|
||||
return False
|
||||
self._work_stop_event.clear()
|
||||
return self.__initialize_monitors()
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __initialize_monitors(self) -> bool:
|
||||
"""
|
||||
启动监控
|
||||
在已取得生命周期所有权后启动监控。
|
||||
|
||||
永久关闭请求可以在本方法阻塞于 FUSE 时从其他线程提前设置;每个可能产生
|
||||
新 owner 的边界都重新检查,确保解冻后只能退出,不能穿透旧生命周期。
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
return True
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
|
||||
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
|
||||
valid_dirs = []
|
||||
for mon_dir in monitor_dirs:
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
@@ -172,6 +233,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
self._scheduler = BackgroundScheduler(timezone=get_runtime_setting("TZ"))
|
||||
|
||||
mon_storages: Dict[str, List[Path]] = {}
|
||||
@@ -179,6 +242,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
local_started = 0
|
||||
local_failed = 0
|
||||
for mon_dir in monitor_dirs:
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
@@ -197,6 +263,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self._store.load(storage)
|
||||
@@ -238,6 +307,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
|
||||
|
||||
# 启动定时服务
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
@@ -253,6 +325,47 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(summary)
|
||||
else:
|
||||
logger.info(summary)
|
||||
return True
|
||||
|
||||
def __accepting_work(self) -> bool:
|
||||
"""判断当前 monitor 代是否仍允许创建 owner 和派发整理工作。"""
|
||||
return not self._work_stop_event.is_set() and not self._shutdown_event.is_set()
|
||||
|
||||
@property
|
||||
def lifecycle_closed(self) -> bool:
|
||||
"""返回当前 Monitor 是否已经被应用生命周期永久封口。"""
|
||||
return self._closed or self._shutdown_event.is_set()
|
||||
|
||||
def reopen(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""确认旧 owner 已收敛后,为新的应用 lifespan 显式解除永久封口。"""
|
||||
if not self.lifecycle_closed:
|
||||
return True
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("旧目录监控生命周期仍在收尾,无法重新开启")
|
||||
return False
|
||||
try:
|
||||
if not self.__stop_owned(deadline=deadline, close=True):
|
||||
logger.error("旧目录监控 owner 仍未收敛,无法重新开启")
|
||||
return False
|
||||
if not self._recovery.reopen():
|
||||
logger.error("旧目录监控恢复线程仍存活,无法重新开启")
|
||||
return False
|
||||
self._closed = False
|
||||
self._shutdown_event.clear()
|
||||
return True
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __abort_interrupted_startup(self) -> None:
|
||||
"""关闭请求穿透阻塞启动后,在当前生命周期锁内收拢已经创建的 owner。"""
|
||||
logger.info("目录监控启动期间收到关闭请求,正在回收本次已创建 owner")
|
||||
self.__stop_owned(
|
||||
deadline=time.monotonic() + self.RELOAD_STOP_TIMEOUT,
|
||||
close=self._shutdown_event.is_set(),
|
||||
)
|
||||
|
||||
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
|
||||
"""
|
||||
@@ -261,6 +374,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: 是否启动成功
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
@@ -298,6 +413,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 启动成功后再登记,避免失败的监控残留在列表中
|
||||
watcher.start()
|
||||
with self._watcher_lock:
|
||||
if not self.__accepting_work():
|
||||
watcher.stop()
|
||||
if watcher.is_alive():
|
||||
self._retired_watchers.append(watcher)
|
||||
return False
|
||||
self._watchers.append(watcher)
|
||||
self._pending_locals = [
|
||||
pending for pending in self._pending_locals
|
||||
@@ -312,13 +432,15 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
|
||||
return False
|
||||
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception) -> None:
|
||||
"""
|
||||
处理本地目录监控启动失败,登记待重试并按需告警。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param err: 启动异常
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
err_msg = str(err)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
@@ -346,7 +468,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
|
||||
f"将自动退避重试")
|
||||
|
||||
def watchdog(self):
|
||||
def watchdog(self) -> None:
|
||||
"""
|
||||
目录监控健康检查:检测监控线程状态并驱动恢复。
|
||||
|
||||
@@ -356,8 +478,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
动作会把看门狗冻死在它自己要修复的挂载上,随后停滞检测、告警、重试驱动
|
||||
全部静默失效——这正是全进程雪崩的起点。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
try:
|
||||
broken = self.__check_watchers()
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self.__drive_recovery(broken)
|
||||
except Exception as e:
|
||||
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
||||
@@ -372,6 +498,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
watchers = list(self._watchers)
|
||||
self._retired_watchers = [
|
||||
watcher for watcher in self._retired_watchers if watcher.is_alive()
|
||||
]
|
||||
isolated = set(self._isolated)
|
||||
# 探测已确认挂载恢复的目录,本轮直接送去重建
|
||||
resumed = list(self._pending_rebuild.values())
|
||||
@@ -412,7 +541,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
broken.append(watcher)
|
||||
return broken
|
||||
|
||||
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
|
||||
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]) -> None:
|
||||
"""
|
||||
把所有会触碰挂载的恢复动作派发到一次性工作线程,并等待有限时间。
|
||||
|
||||
@@ -421,6 +550,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
可放弃的子进程探测来确认挂载何时恢复。
|
||||
:param broken: 需要重建的监控列表
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
actions: Dict[str, Callable[[], None]] = {}
|
||||
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
|
||||
for watcher in broken:
|
||||
@@ -432,6 +563,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
actions[self.PENDING_KEY] = self.__drive_pending
|
||||
|
||||
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
|
||||
for key, state in results.items():
|
||||
if state is RecoveryState.COMPLETED:
|
||||
@@ -445,11 +578,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
|
||||
f"({state.value}),将在后续健康检查周期重试: {key}")
|
||||
|
||||
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
|
||||
def __enter_isolation(self, watcher: LocalDirectoryWatcher) -> None:
|
||||
"""
|
||||
将一个监控目录转入挂载级故障隔离:停止对它的一切新访问,等待探测恢复。
|
||||
:param watcher: 重建未能返回的监控
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(watcher.watch_path)
|
||||
with self._watcher_lock:
|
||||
if key in self._isolated:
|
||||
@@ -466,7 +601,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
|
||||
stage="isolated")
|
||||
|
||||
def __probe_isolated(self):
|
||||
def __probe_isolated(self) -> None:
|
||||
"""
|
||||
对隔离中的监控目录做可放弃探测,挂载恢复应答后解除隔离并重建监控。
|
||||
|
||||
@@ -474,11 +609,17 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
超时可被 kill,因此本线程不会像内联 stat 那样永久冻死。探测通过后的重建
|
||||
仍有极小概率再次卡住,届时本线程会被下一轮的 BUSY 判定跳过,不再泄漏。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
keys = list(self._isolated)
|
||||
for key in keys:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
mon_path = Path(key)
|
||||
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
entry = self._isolated.get(key)
|
||||
if not entry:
|
||||
@@ -489,6 +630,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
|
||||
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
|
||||
continue
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
entry = self._isolated.pop(key, None)
|
||||
if not entry:
|
||||
@@ -502,19 +645,24 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
with self._watcher_lock:
|
||||
self._pending_rebuild[key] = entry["watcher"]
|
||||
|
||||
def __drive_pending(self):
|
||||
def __drive_pending(self) -> None:
|
||||
"""
|
||||
驱动两条待重试队列。两者都会访问挂载(启动重试走目录遍历与 exists,
|
||||
整理重试走 stat),必须在恢复工作线程里执行而不是看门狗线程里。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self.__retry_pending_locals()
|
||||
self._dispatcher.retry_pending()
|
||||
if self.__accepting_work():
|
||||
self._dispatcher.retry_pending()
|
||||
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher) -> None:
|
||||
"""
|
||||
重建一个本地目录监控线程。
|
||||
:param watcher: 需要重建的监控
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
|
||||
watcher.stop()
|
||||
new_watcher = LocalDirectoryWatcher(
|
||||
@@ -527,8 +675,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
new_watcher.start()
|
||||
except Exception as e:
|
||||
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
# 旧 watcher 可能仍卡在 FUSE 调用里;只有真实退出后才能移除句柄。
|
||||
self._watchers = [item for item in self._watchers if item is not watcher]
|
||||
if watcher.is_alive():
|
||||
self._retired_watchers.append(watcher)
|
||||
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": watcher.watch_path,
|
||||
@@ -537,9 +690,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
})
|
||||
return
|
||||
with self._watcher_lock:
|
||||
if not self.__accepting_work():
|
||||
new_watcher.stop()
|
||||
if new_watcher.is_alive():
|
||||
self._retired_watchers.append(new_watcher)
|
||||
return
|
||||
registered = any(item is watcher for item in self._watchers)
|
||||
if registered:
|
||||
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
||||
if watcher.is_alive():
|
||||
# stop() 只能发信号,FUSE 上的旧线程可能永久不返回;继续持有它,
|
||||
# 让生命周期关闭如实失败,而不是把 daemon 泄漏伪装成已收敛。
|
||||
self._retired_watchers.append(watcher)
|
||||
if not registered:
|
||||
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
|
||||
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
|
||||
@@ -547,6 +709,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 派发事件的孤儿线程,必须就地停掉
|
||||
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
|
||||
new_watcher.stop()
|
||||
if new_watcher.is_alive():
|
||||
with self._watcher_lock:
|
||||
self._retired_watchers.append(new_watcher)
|
||||
return
|
||||
# 新监控的重启计数从零开始,同步重置告警基准
|
||||
self._restart_marks.pop(str(watcher.watch_path), None)
|
||||
@@ -557,7 +722,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self.__start_compensation(mon_path=watcher.watch_path,
|
||||
since=watcher.last_activity_time)
|
||||
|
||||
def __start_compensation(self, mon_path: Path, since: float):
|
||||
def __start_compensation(self, mon_path: Path, since: float) -> None:
|
||||
"""
|
||||
在后台线程发起补偿扫描,避免遍历目录阻塞健康检查周期。
|
||||
:param mon_path: 监控目录
|
||||
@@ -567,14 +732,34 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
|
||||
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
|
||||
return
|
||||
Thread(
|
||||
target=self.__compensate_scan,
|
||||
thread = Thread(
|
||||
target=self.__run_compensation,
|
||||
kwargs={"mon_path": mon_path, "since": since},
|
||||
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
|
||||
daemon=True
|
||||
).start()
|
||||
)
|
||||
with self._owner_lock:
|
||||
# 与 close() 的 owner 快照共用同一把锁;封口先发生时绝不再启动,
|
||||
# 登记先发生时 close() 必然能看到并等待这条线程。
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self._compensation_threads[id(thread)] = thread
|
||||
try:
|
||||
thread.start()
|
||||
except Exception:
|
||||
self._compensation_threads.pop(id(thread), None)
|
||||
raise
|
||||
|
||||
def __compensate_scan(self, mon_path: Path, since: float):
|
||||
def __run_compensation(self, mon_path: Path, since: float) -> None:
|
||||
"""执行补偿扫描,并在真实终态后释放 owner 句柄。"""
|
||||
current_thread = threading.current_thread()
|
||||
try:
|
||||
self.__compensate_scan(mon_path=mon_path, since=since)
|
||||
finally:
|
||||
with self._owner_lock:
|
||||
self._compensation_threads.pop(id(current_thread), None)
|
||||
|
||||
def __compensate_scan(self, mon_path: Path, since: float) -> None:
|
||||
"""
|
||||
补扫监控停摆期间落地的文件。
|
||||
|
||||
@@ -586,7 +771,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param mon_path: 监控目录
|
||||
:param since: 停摆起点(墙钟时间戳),仅用于统计与日志
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
candidates = self.__collect_compensation_files(mon_path)
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if candidates is None:
|
||||
return
|
||||
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
|
||||
@@ -599,6 +788,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
|
||||
handled = 0
|
||||
for file_path, file_modify_time, file_size in candidates:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if self._dispatcher.handle_file(
|
||||
storage="local",
|
||||
event_path=file_path,
|
||||
@@ -634,14 +825,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
return None
|
||||
return candidates
|
||||
|
||||
def __retry_pending_locals(self):
|
||||
def __retry_pending_locals(self) -> None:
|
||||
"""
|
||||
重试启动失败的本地目录监控,给网络存储/FUSE 挂载留出就绪时间。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
pending = list(self._pending_locals)
|
||||
isolated = set(self._isolated)
|
||||
for item in pending:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if str(item["mon_path"]) in isolated:
|
||||
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
|
||||
# 在「请求永不返回」的挂载上会再冻死一个线程
|
||||
@@ -655,7 +850,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
item["attempts"] = item.get("attempts", 0) + 1
|
||||
item["skip_cycles"] = min(item["attempts"], 10)
|
||||
|
||||
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
|
||||
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault") -> None:
|
||||
"""
|
||||
推送目录监控异常告警,同一目录在同一阶段仅推送一次。
|
||||
:param mon_path: 监控目录
|
||||
@@ -664,6 +859,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
(监控已暂停访问、等待挂载恢复),只按目录去重会把这条
|
||||
关键消息吞掉,因此阶段变化时重新推送
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if self._alerted_paths.get(key) == stage:
|
||||
@@ -672,7 +869,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
@staticmethod
|
||||
def __poller_alert(storage: str, message: str):
|
||||
def __poller_alert(storage: str, message: str) -> None:
|
||||
"""
|
||||
远程轮询监控告警回调,复用消息渠道推送。
|
||||
:param storage: 存储名称
|
||||
@@ -681,12 +878,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"[{storage}] {message}")
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def __clear_alert(self, mon_path: Path, message: str):
|
||||
def __clear_alert(self, mon_path: Path, message: str) -> None:
|
||||
"""
|
||||
清除目录监控异常告警状态,并在此前告警过时推送恢复消息。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 恢复内容
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key not in self._alerted_paths:
|
||||
@@ -695,12 +894,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(message)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]) -> None:
|
||||
"""
|
||||
轮询监控:执行一轮快照并按结果动态调整监控间隔。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
|
||||
if file_count is None or not self._scheduler:
|
||||
if not self.__accepting_work() or file_count is None or not self._scheduler:
|
||||
return
|
||||
# 动态调整监控间隔
|
||||
new_interval = SnapshotStore.adjust_interval(file_count)
|
||||
@@ -716,7 +917,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
except Exception as e:
|
||||
logger.error(f"调整监控间隔失败: {storage} - {e}")
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
def event_handler(
|
||||
self, event: Any, text: str, event_path: str,
|
||||
file_size: Optional[float] = None
|
||||
) -> None:
|
||||
"""
|
||||
处理文件变化。
|
||||
:param event: 事件
|
||||
@@ -724,7 +928,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if event.is_directory:
|
||||
if not self.__accepting_work() or event.is_directory:
|
||||
return
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
@@ -733,6 +937,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
file_modify_time = Path(event_path).stat().st_mtime
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
# 整理文件
|
||||
handle_kwargs = {
|
||||
"storage": "local",
|
||||
@@ -743,55 +949,191 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
handle_kwargs["file_modify_time"] = file_modify_time
|
||||
self._dispatcher.handle_file(**handle_kwargs)
|
||||
|
||||
def event_unreadable(self, event_path: Path):
|
||||
def event_unreadable(self, event_path: Path) -> None:
|
||||
"""
|
||||
处理读取失败的监控事件,登记待重试。
|
||||
:param event_path: 事件文件路径
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
event_path = Path(event_path)
|
||||
if not self._dispatcher.is_transfer_candidate_path(event_path):
|
||||
return
|
||||
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
|
||||
|
||||
def stop(self):
|
||||
def stop(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""在共享预算内临时停止全部监控 owner,供配置热重载使用。"""
|
||||
return self.__stop_with_budget(timeout=timeout, close=False)
|
||||
|
||||
def close(self, timeout: float = LIFECYCLE_CLOSE_TIMEOUT) -> bool:
|
||||
"""永久封口当前 lifespan,并在共享预算内等待全部监控 owner。"""
|
||||
return self.__stop_with_budget(timeout=timeout, close=True)
|
||||
|
||||
def __stop_with_budget(self, timeout: float, close: bool) -> bool:
|
||||
"""从等待生命周期锁开始计算一次停止操作的完整预算。"""
|
||||
timeout = max(0.0, timeout)
|
||||
deadline = time.monotonic() + timeout
|
||||
# 先封住工作入口再等待锁。配置重载若正阻塞在 FUSE 上,解冻后也只能清理,
|
||||
# 不能在外层生命周期已经超时返回后继续创建 watcher 或派发整理任务。
|
||||
self._work_stop_event.set()
|
||||
if close:
|
||||
self._shutdown_event.set()
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("目录监控未在 %.1f 秒内取得停机所有权", timeout)
|
||||
return False
|
||||
try:
|
||||
return self.__stop_owned(deadline=deadline, close=close)
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __stop_owned(self, deadline: float, close: bool) -> bool:
|
||||
"""
|
||||
退出监控
|
||||
在已取得生命周期锁时等待全部 owner,共享调用方给出的绝对截止时间。
|
||||
|
||||
超时只报告未收敛并保留所有活句柄;只有 watcher、补偿扫描、恢复动作及
|
||||
scheduler shutdown 都到达终态后,才清空这一代的内存状态。
|
||||
"""
|
||||
# 先停定时服务,避免健康检查在停止过程中重建监控线程
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
|
||||
self._work_stop_event.set()
|
||||
if close:
|
||||
self._closed = True
|
||||
self._shutdown_event.set()
|
||||
self._recovery.request_stop()
|
||||
self.__request_scheduler_stop()
|
||||
|
||||
with self._watcher_lock:
|
||||
watchers = tuple(self._watchers) + tuple(self._retired_watchers)
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception as err:
|
||||
logger.error(f"请求停止目录监控服务出现了错误:{err}")
|
||||
|
||||
recovery_converged = self._recovery.close(deadline=deadline)
|
||||
current_thread = threading.current_thread()
|
||||
# 恢复线程可能在封口后才从 FUSE 调用返回,并登记一个已经请求停止的新
|
||||
# watcher;RecoveryExecutor 收敛后重新取快照,确保它也使用剩余预算 join。
|
||||
with self._watcher_lock:
|
||||
registered_watchers = tuple(self._watchers) + tuple(self._retired_watchers)
|
||||
for watcher in registered_watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
except Exception as err:
|
||||
logger.error(f"等待目录监控服务停止出现了错误:{err}")
|
||||
|
||||
with self._owner_lock:
|
||||
compensation_threads = tuple(self._compensation_threads.values())
|
||||
scheduler_thread = self._scheduler_shutdown_thread
|
||||
for thread in compensation_threads:
|
||||
if thread is not current_thread and thread.is_alive():
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if (
|
||||
scheduler_thread is not None
|
||||
and scheduler_thread is not current_thread
|
||||
and scheduler_thread.is_alive()
|
||||
):
|
||||
scheduler_thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
|
||||
with self._watcher_lock:
|
||||
alive_watchers = tuple(
|
||||
watcher
|
||||
for watcher in (*self._watchers, *self._retired_watchers)
|
||||
if watcher.is_alive()
|
||||
)
|
||||
with self._owner_lock:
|
||||
alive_compensations = tuple(
|
||||
thread for thread in self._compensation_threads.values()
|
||||
if thread.is_alive()
|
||||
)
|
||||
scheduler_thread = self._scheduler_shutdown_thread
|
||||
scheduler_converged = (
|
||||
self._scheduler is None
|
||||
or (
|
||||
scheduler_thread is not None
|
||||
and not scheduler_thread.is_alive()
|
||||
and self._scheduler_shutdown_succeeded
|
||||
)
|
||||
)
|
||||
alive_recoveries = self._recovery.running_threads()
|
||||
if (
|
||||
alive_watchers
|
||||
or alive_compensations
|
||||
or alive_recoveries
|
||||
or not recovery_converged
|
||||
or not scheduler_converged
|
||||
):
|
||||
logger.error(
|
||||
"目录监控 owner 未在截止时间内收敛:watcher=%d,补偿=%d,恢复=%d,scheduler=%s",
|
||||
len(alive_watchers),
|
||||
len(alive_compensations),
|
||||
len(alive_recoveries),
|
||||
"未收敛" if not scheduler_converged else "已收敛",
|
||||
)
|
||||
return False
|
||||
|
||||
# 仅在所有 owner 真实终止后清理业务状态,确保失败重试仍能找到原句柄。
|
||||
self._dispatcher.clear_pending()
|
||||
with self._watcher_lock:
|
||||
watchers = self._watchers
|
||||
self._watchers = []
|
||||
self._retired_watchers = []
|
||||
self._pending_locals = []
|
||||
self._alerted_paths = {}
|
||||
self._restart_marks = {}
|
||||
self._stable_cycles = {}
|
||||
self._isolated = {}
|
||||
self._pending_rebuild = {}
|
||||
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
|
||||
# 被残留记录误判为 BUSY 而永远拿不到重建机会
|
||||
self._recovery.clear()
|
||||
with self._owner_lock:
|
||||
self._compensation_threads.clear()
|
||||
self._scheduler = None
|
||||
self._scheduler_shutdown_thread = None
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
logger.info("本地目录监控服务已停止")
|
||||
return True
|
||||
|
||||
def __request_scheduler_stop(self) -> None:
|
||||
"""创建并登记唯一 scheduler shutdown 线程,避免阻塞生命周期调用线程。"""
|
||||
with self._owner_lock:
|
||||
running = self._scheduler_shutdown_thread
|
||||
if running is not None and running.is_alive():
|
||||
return
|
||||
if running is not None and self._scheduler_shutdown_succeeded:
|
||||
return
|
||||
scheduler = self._scheduler
|
||||
if scheduler is None:
|
||||
return
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
thread = threading.Thread(
|
||||
target=self.__shutdown_scheduler,
|
||||
args=(scheduler,),
|
||||
name="MoviePilot-MonitorSchedulerShutdown",
|
||||
daemon=True,
|
||||
)
|
||||
self._scheduler_shutdown_thread = thread
|
||||
try:
|
||||
thread.start()
|
||||
except Exception as err:
|
||||
self._scheduler_shutdown_thread = None
|
||||
logger.error(f"启动定时监控停止线程失败:{err}")
|
||||
|
||||
def __shutdown_scheduler(self, scheduler: BackgroundScheduler) -> None:
|
||||
"""停止 scheduler 及其在途 job,并把真实终态写回 owner 注册表。"""
|
||||
succeeded = False
|
||||
try:
|
||||
scheduler.remove_all_jobs()
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=True)
|
||||
succeeded = True
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as err:
|
||||
logger.error(f"停止定时服务出现了错误:{err}")
|
||||
finally:
|
||||
with self._owner_lock:
|
||||
if self._scheduler is scheduler:
|
||||
self._scheduler_shutdown_succeeded = succeeded
|
||||
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
|
||||
|
||||
+70
-15
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
监控恢复动作的可放弃执行单元。
|
||||
监控恢复动作的有限等待执行单元。
|
||||
|
||||
FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
|
||||
|
||||
@@ -11,8 +11,8 @@ FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
|
||||
本模块提供 block 型故障下唯一可行的两种自保手段:
|
||||
|
||||
1. RecoveryExecutor —— 把会触碰挂载的动作放进一次性守护线程执行,调用方只
|
||||
等待有限时间。超时即放弃该线程(它会作为守护线程悬挂到进程退出),换取
|
||||
调用方(健康检查这个全局自愈单点)永远活着。
|
||||
等待有限时间。超时后当前健康检查不再等待,但线程句柄仍由生命周期 owner
|
||||
持有到真实终态,换取看门狗可继续检测且停机屏障不会伪装收敛。
|
||||
2. probe_path —— 用子进程而非线程做挂载探测。子进程可以被 kill,因此探测
|
||||
本身是可放弃的,隔离期间可以无限次周期重试而不累积不可回收的资源。
|
||||
"""
|
||||
@@ -41,7 +41,7 @@ class RecoveryState(str, Enum):
|
||||
"""
|
||||
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
|
||||
COMPLETED = "completed"
|
||||
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
|
||||
# 超时仍未返回,判定为 block 型挂载故障,本轮不再等待但继续持有线程
|
||||
TIMEOUT = "timeout"
|
||||
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
|
||||
BUSY = "busy"
|
||||
@@ -55,10 +55,12 @@ class RecoveryExecutor:
|
||||
健康检查周期都会在同一个死挂载上多泄漏一个线程。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化在途线程注册表和当前生命周期的提交状态。"""
|
||||
# key -> 该 key 最近一次提交的执行线程
|
||||
self._running: Dict[str, threading.Thread] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._accepting = True
|
||||
|
||||
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
|
||||
"""
|
||||
@@ -85,26 +87,70 @@ class RecoveryExecutor:
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if thread.is_alive():
|
||||
results[key] = RecoveryState.TIMEOUT
|
||||
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
|
||||
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应,本轮不再等待: {key}")
|
||||
else:
|
||||
results[key] = RecoveryState.COMPLETED
|
||||
return results
|
||||
|
||||
def discard(self, key: str):
|
||||
def discard(self, key: str) -> None:
|
||||
"""
|
||||
丢弃一个 key 的在途记录。监控停止或配置重载时调用,避免残留条目
|
||||
让重建后的同名目录被误判为 BUSY。
|
||||
清理一个已到终态的 key;活线程继续保留,供生命周期停机屏障追踪。
|
||||
:param key: 动作标识
|
||||
"""
|
||||
with self._lock:
|
||||
self._running.pop(key, None)
|
||||
thread = self._running.get(key)
|
||||
if thread is None or not thread.is_alive():
|
||||
self._running.pop(key, None)
|
||||
|
||||
def clear(self):
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
清空全部在途记录。已经冻死的线程无法回收,这里只是不再跟踪它们。
|
||||
清理已经完成的记录,仍存活的线程继续由执行器持有。
|
||||
|
||||
兼容旧调用名,但不再丢弃挂死线程:遗失句柄会让宿主错误释放它仍可能使用
|
||||
的数据库和整理链资源。
|
||||
"""
|
||||
with self._lock:
|
||||
self._running = {
|
||||
key: thread
|
||||
for key, thread in self._running.items()
|
||||
if thread.is_alive()
|
||||
}
|
||||
|
||||
def request_stop(self) -> None:
|
||||
"""封住新恢复动作提交,既有线程只能自然完成。"""
|
||||
with self._lock:
|
||||
self._accepting = False
|
||||
|
||||
def close(self, deadline: float) -> bool:
|
||||
"""在绝对截止时间内等待全部恢复线程,超时继续保留活线程句柄。"""
|
||||
self.request_stop()
|
||||
with self._lock:
|
||||
threads = tuple(self._running.values())
|
||||
current_thread = threading.current_thread()
|
||||
for thread in threads:
|
||||
if thread is current_thread or not thread.is_alive():
|
||||
continue
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
self.clear()
|
||||
with self._lock:
|
||||
return not any(thread.is_alive() for thread in self._running.values())
|
||||
|
||||
def reopen(self) -> bool:
|
||||
"""在旧恢复线程全部结束后,为新的 Monitor 生命周期恢复提交。"""
|
||||
self.clear()
|
||||
with self._lock:
|
||||
if any(thread.is_alive() for thread in self._running.values()):
|
||||
return False
|
||||
self._running.clear()
|
||||
self._accepting = True
|
||||
return True
|
||||
|
||||
def running_threads(self) -> tuple[threading.Thread, ...]:
|
||||
"""返回仍存活且由执行器持有的恢复线程快照。"""
|
||||
with self._lock:
|
||||
return tuple(
|
||||
thread for thread in self._running.values() if thread.is_alive()
|
||||
)
|
||||
|
||||
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
|
||||
"""
|
||||
@@ -114,6 +160,8 @@ class RecoveryExecutor:
|
||||
:return: 执行线程,未启动时为 None
|
||||
"""
|
||||
with self._lock:
|
||||
if not self._accepting:
|
||||
return None
|
||||
running = self._running.get(key)
|
||||
if running is not None and running.is_alive():
|
||||
return None
|
||||
@@ -124,11 +172,18 @@ class RecoveryExecutor:
|
||||
daemon=True
|
||||
)
|
||||
self._running[key] = thread
|
||||
thread.start()
|
||||
return thread
|
||||
# 登记与 start 必须处于同一所有权临界区。否则 close() 可能把尚未
|
||||
# is_alive() 的句柄当成已结束并移除,随后该线程才真正启动。
|
||||
try:
|
||||
thread.start()
|
||||
except BaseException:
|
||||
if self._running.get(key) is thread:
|
||||
self._running.pop(key, None)
|
||||
raise
|
||||
return thread
|
||||
|
||||
@staticmethod
|
||||
def _execute(key: str, action: Callable[[], None]):
|
||||
def _execute(key: str, action: Callable[[], None]) -> None:
|
||||
"""
|
||||
执行一个恢复动作,异常只记录不外抛,避免一个目录的失败连累整批恢复。
|
||||
:param key: 动作标识
|
||||
|
||||
@@ -30,6 +30,9 @@ class EventDispatcher:
|
||||
event_factory: Callable[..., Any],
|
||||
error_handler: Callable[..., None],
|
||||
async_handle_sink: Callable[[Any], bool] | None = None,
|
||||
sync_handle_sink: (
|
||||
Callable[[Callable[..., Any], tuple[Any, ...]], bool] | None
|
||||
) = None,
|
||||
) -> None:
|
||||
"""注入注册表、绑定器、执行器和错误策略回调。"""
|
||||
self._registry = registry
|
||||
@@ -39,6 +42,7 @@ class EventDispatcher:
|
||||
self._event_factory = event_factory
|
||||
self._error_handler = error_handler
|
||||
self._async_handle_sink = async_handle_sink
|
||||
self._sync_handle_sink = sync_handle_sink
|
||||
|
||||
def dispatch_chain(self, event: Any) -> bool:
|
||||
"""同步按优先级顺序执行链式事件快照。"""
|
||||
@@ -134,6 +138,12 @@ class EventDispatcher:
|
||||
event.event_type,
|
||||
)
|
||||
else:
|
||||
if self._sync_handle_sink:
|
||||
self._sync_handle_sink(
|
||||
self.safe_invoke_sync,
|
||||
(handler, isolated),
|
||||
)
|
||||
continue
|
||||
self._executor().submit(
|
||||
self.safe_invoke_sync,
|
||||
handler,
|
||||
|
||||
+180
-26
@@ -4,6 +4,7 @@ import random
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, PriorityQueue
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type
|
||||
@@ -32,6 +33,10 @@ MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
|
||||
INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间(秒)
|
||||
MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间(秒)
|
||||
_EVENT_STOP_SENTINEL = object()
|
||||
_CURRENT_EVENT_HANDLER_OWNER: ContextVar[object | None] = ContextVar(
|
||||
"current_event_handler_owner",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -121,10 +126,11 @@ class EventManager(metaclass=Singleton):
|
||||
self.__lock = threading.Lock()
|
||||
# 退出事件
|
||||
self.__event = threading.Event()
|
||||
# 广播异步处理器的生命周期由事件总线自己持有,避免关闭后仍向主循环运行。
|
||||
# 广播处理器由事件总线统一持有,确保插件卸载前可以建立结算屏障。
|
||||
self.__lifecycle_lock = threading.RLock()
|
||||
self.__lifecycle_state = "new"
|
||||
self.__async_handles: Dict[int, _EventAsyncHandle] = {}
|
||||
self.__sync_handles: Dict[object, concurrent.futures.Future[Any]] = {}
|
||||
self.__async_handles: Dict[object, _EventAsyncHandle] = {}
|
||||
# 由上层管理器注册的处理器实例解析器
|
||||
self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {}
|
||||
# 由启动组合层注入的错误通知回调
|
||||
@@ -155,6 +161,7 @@ class EventManager(metaclass=Singleton):
|
||||
event_factory=Event,
|
||||
error_handler=lambda **kwargs: self.__handle_event_error(**kwargs),
|
||||
async_handle_sink=self.__register_async_handle,
|
||||
sync_handle_sink=self.__register_sync_handle,
|
||||
)
|
||||
|
||||
def register_handler_instance_resolver(
|
||||
@@ -188,6 +195,9 @@ class EventManager(metaclass=Singleton):
|
||||
if self.__lifecycle_state == "stopping":
|
||||
logger.warning("事件处理仍在停止,忽略重复启动")
|
||||
return
|
||||
if self.__lifecycle_state == "sealed":
|
||||
logger.warning("事件处理已封口,忽略重复启动")
|
||||
return
|
||||
self.__lifecycle_state = "running"
|
||||
self.__event.set()
|
||||
self.__consumer_threads = []
|
||||
@@ -199,14 +209,19 @@ class EventManager(metaclass=Singleton):
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
停止广播事件处理线程
|
||||
兼容同步关闭入口,等待同步处理器并请求取消异步处理器。
|
||||
|
||||
调用线程无法安全等待主事件循环完成异步清理;需要完整异步收口时应使用
|
||||
stop_async()。
|
||||
"""
|
||||
logger.info("正在停止事件处理...")
|
||||
consumer_threads = self.__begin_stop()
|
||||
try:
|
||||
self.__join_consumer_threads(consumer_threads)
|
||||
self.__discard_stop_sentinels()
|
||||
self.__cancel_async_handles()
|
||||
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
|
||||
self.__cancel_async_handles(exclude_owner=current_owner)
|
||||
self.__wait_sync_handles(exclude_owner=current_owner)
|
||||
logger.info("事件处理停止完成")
|
||||
except Exception as e:
|
||||
logger.error(f"停止事件处理线程出错:{str(e)} - {traceback.format_exc()}")
|
||||
@@ -216,7 +231,7 @@ class EventManager(metaclass=Singleton):
|
||||
self.__consumer_threads = []
|
||||
|
||||
async def stop_async(self) -> None:
|
||||
"""停止广播消费者并等待已投递的异步处理器收口。"""
|
||||
"""停止广播消费者,等待同步处理器并取消收口异步处理器。"""
|
||||
logger.info("正在停止事件处理...")
|
||||
consumer_threads = self.__begin_stop()
|
||||
try:
|
||||
@@ -226,15 +241,29 @@ class EventManager(metaclass=Singleton):
|
||||
consumer_threads,
|
||||
)
|
||||
self.__discard_stop_sentinels()
|
||||
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(self.__async_handles.values())
|
||||
for handle in handles:
|
||||
async_handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__async_handles.items()
|
||||
if owner is not current_owner
|
||||
)
|
||||
sync_handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__sync_handles.items()
|
||||
if owner is not current_owner
|
||||
)
|
||||
for handle in async_handles:
|
||||
handle.handle.cancel()
|
||||
if handles:
|
||||
if async_handles or sync_handles:
|
||||
await asyncio.gather(
|
||||
*(
|
||||
asyncio.shield(asyncio.wrap_future(handle.completion))
|
||||
for handle in handles
|
||||
for handle in async_handles
|
||||
),
|
||||
*(
|
||||
asyncio.shield(asyncio.wrap_future(handle))
|
||||
for handle in sync_handles
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
@@ -272,16 +301,81 @@ class EventManager(metaclass=Singleton):
|
||||
break
|
||||
if item[1] is not _EVENT_STOP_SENTINEL:
|
||||
pending.append(item)
|
||||
self.__event_queue.task_done()
|
||||
for item in pending:
|
||||
self.__event_queue.put(item)
|
||||
|
||||
def __cancel_async_handles(self) -> None:
|
||||
"""请求取消所有仍由事件总线持有的异步处理器。"""
|
||||
def __cancel_async_handles(
|
||||
self,
|
||||
*,
|
||||
exclude_owner: object | None = None,
|
||||
) -> None:
|
||||
"""请求取消异步处理器,并避免 handler 关闭事件总线时取消自身。"""
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(self.__async_handles.values())
|
||||
handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__async_handles.items()
|
||||
if owner is not exclude_owner
|
||||
)
|
||||
for handle in handles:
|
||||
handle.handle.cancel()
|
||||
|
||||
def __wait_sync_handles(
|
||||
self,
|
||||
*,
|
||||
exclude_owner: object | None = None,
|
||||
) -> None:
|
||||
"""等待同步处理器完成,并避免 handler 关闭事件总线时等待自身。"""
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__sync_handles.items()
|
||||
if owner is not exclude_owner
|
||||
)
|
||||
if handles:
|
||||
concurrent.futures.wait(handles)
|
||||
|
||||
def __register_sync_handle(
|
||||
self,
|
||||
callback: Callable[..., Any],
|
||||
args: tuple[Any, ...],
|
||||
) -> bool:
|
||||
"""在同一生命周期临界区提交并登记同步广播处理器。"""
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state != "running":
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝同步广播处理器",
|
||||
self.__lifecycle_state,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
owner = object()
|
||||
|
||||
def _tracked_sync() -> Any:
|
||||
"""在同步 handler 调用栈中发布当前事件 owner。"""
|
||||
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
|
||||
try:
|
||||
return callback(*args)
|
||||
finally:
|
||||
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
|
||||
|
||||
handle = self.__executor.submit(_tracked_sync)
|
||||
except RuntimeError:
|
||||
logger.warning("同步事件处理器无法投递,线程池已停止")
|
||||
return False
|
||||
self.__sync_handles[owner] = handle
|
||||
handle.add_done_callback(
|
||||
lambda _completed, current_owner=owner: (
|
||||
self.__remove_sync_handle(current_owner)
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def __remove_sync_handle(self, owner: object) -> None:
|
||||
"""同步处理器完成后移除其 owner 句柄。"""
|
||||
with self.__lifecycle_lock:
|
||||
self.__sync_handles.pop(owner, None)
|
||||
|
||||
def __register_async_handle(
|
||||
self,
|
||||
coroutine: Any,
|
||||
@@ -290,12 +384,18 @@ class EventManager(metaclass=Singleton):
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state != "running":
|
||||
coroutine.close()
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝异步广播处理器",
|
||||
self.__lifecycle_state,
|
||||
)
|
||||
return False
|
||||
completion: concurrent.futures.Future[Any] = concurrent.futures.Future()
|
||||
started = threading.Event()
|
||||
owner = object()
|
||||
|
||||
async def _tracked() -> None:
|
||||
started.set()
|
||||
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
|
||||
try:
|
||||
result = await coroutine
|
||||
except asyncio.CancelledError:
|
||||
@@ -307,6 +407,8 @@ class EventManager(metaclass=Singleton):
|
||||
else:
|
||||
if not completion.done():
|
||||
completion.set_result(result)
|
||||
finally:
|
||||
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
|
||||
|
||||
tracked = _tracked()
|
||||
try:
|
||||
@@ -316,7 +418,7 @@ class EventManager(metaclass=Singleton):
|
||||
coroutine.close()
|
||||
logger.warning("异步事件处理器无法投递,事件循环已停止")
|
||||
return False
|
||||
self.__async_handles[id(completion)] = _EventAsyncHandle(
|
||||
self.__async_handles[owner] = _EventAsyncHandle(
|
||||
handle=handle,
|
||||
completion=completion,
|
||||
)
|
||||
@@ -329,13 +431,54 @@ class EventManager(metaclass=Singleton):
|
||||
completion.cancel()
|
||||
|
||||
handle.add_done_callback(_complete_unstarted_submission)
|
||||
completion.add_done_callback(self.__remove_async_handle)
|
||||
completion.add_done_callback(
|
||||
lambda _completed, current_owner=owner: (
|
||||
self.__remove_async_handle(current_owner)
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None:
|
||||
def __remove_async_handle(self, owner: object) -> None:
|
||||
"""异步处理器完成后移除其 owner 句柄。"""
|
||||
with self.__lifecycle_lock:
|
||||
self.__async_handles.pop(id(handle), None)
|
||||
self.__async_handles.pop(owner, None)
|
||||
|
||||
async def drain_async(
|
||||
self,
|
||||
timeout: Optional[float] = None,
|
||||
*,
|
||||
seal: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
等待已接纳的广播事件及其派生事件自然结算。
|
||||
|
||||
seal=True 会在确认稳定空闲的同一临界区关闭后续广播提交,供插件卸载前
|
||||
建立不可穿透的投递屏障;超时或停止交错时保持原提交状态并返回 False。
|
||||
广播处理器调用栈内无法等待自身完成,因此会立即返回 False,且不执行封口。
|
||||
"""
|
||||
if _CURRENT_EVENT_HANDLER_OWNER.get() is not None:
|
||||
logger.warning("事件处理器内部不能建立事件投递屏障")
|
||||
return False
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = None if timeout is None else loop.time() + max(timeout, 0)
|
||||
while True:
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state not in {"running", "sealed"}:
|
||||
return False
|
||||
with self.__event_queue.all_tasks_done:
|
||||
queue_idle = self.__event_queue.unfinished_tasks == 0
|
||||
handlers_idle = not self.__sync_handles and not self.__async_handles
|
||||
if queue_idle and handlers_idle:
|
||||
if seal:
|
||||
self.__lifecycle_state = "sealed"
|
||||
return True
|
||||
if deadline is not None:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
await asyncio.sleep(min(0.01, remaining))
|
||||
else:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
def check(self, etype: Union[EventType, ChainEventType]) -> bool:
|
||||
"""
|
||||
@@ -470,7 +613,15 @@ class EventManager(metaclass=Singleton):
|
||||
:param event: 要处理的事件对象
|
||||
"""
|
||||
logger.debug(f"Triggering broadcast event: {event}")
|
||||
self.__event_queue.put((event.priority, event))
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state in {"sealed", "stopping", "stopped"}:
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝广播事件 %s",
|
||||
self.__lifecycle_state,
|
||||
event.event_type,
|
||||
)
|
||||
return None
|
||||
self.__event_queue.put((event.priority, event))
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
@@ -580,15 +731,18 @@ class EventManager(metaclass=Singleton):
|
||||
while self.__event.is_set():
|
||||
try:
|
||||
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
|
||||
if event is _EVENT_STOP_SENTINEL:
|
||||
break
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
delivery="broadcast",
|
||||
)
|
||||
rate_limiter.reset()
|
||||
self.__dispatch_broadcast_event(event)
|
||||
try:
|
||||
if event is _EVENT_STOP_SENTINEL:
|
||||
break
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
delivery="broadcast",
|
||||
)
|
||||
rate_limiter.reset()
|
||||
self.__dispatch_broadcast_event(event)
|
||||
finally:
|
||||
self.__event_queue.task_done()
|
||||
except Empty:
|
||||
rate_limiter.current_wait = rate_limiter.current_wait * random.uniform(1, 1 + jitter_factor)
|
||||
rate_limiter.trigger_limit()
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""插件宿主可变事务的停机准入。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _MutationContext:
|
||||
"""记录一个可跨协程和受控线程传播的事务上下文。"""
|
||||
|
||||
admission: "PluginMutationAdmission"
|
||||
holders: int = 0
|
||||
open: bool = True
|
||||
|
||||
|
||||
class PluginMutationAdmission:
|
||||
"""在停机封口与插件可变事务之间维护真实 owner 计数。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化开放准入、活动 owner 计数和事务上下文。"""
|
||||
self._condition = threading.Condition()
|
||||
self._accepting = True
|
||||
self._active_count = 0
|
||||
self._current_context: ContextVar[_MutationContext | None] = ContextVar(
|
||||
"plugin_mutation_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
@property
|
||||
def active_count(self) -> int:
|
||||
"""返回尚未退出的 lease 数量,用于停机诊断和严格卸载判断。"""
|
||||
with self._condition:
|
||||
return self._active_count
|
||||
|
||||
@property
|
||||
def accepting(self) -> bool:
|
||||
"""返回当前生命周期是否仍接纳新的根事务。"""
|
||||
with self._condition:
|
||||
return self._accepting
|
||||
|
||||
def is_held(self) -> bool:
|
||||
"""判断当前执行上下文是否持有仍有效的本 admission lease。"""
|
||||
context = self._current_context.get()
|
||||
return bool(
|
||||
context
|
||||
and context.admission is self
|
||||
and context.open
|
||||
and context.holders > 0
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def hold(self, operation: str) -> Iterator[None]:
|
||||
"""取得可变事务 lease;封口后仅允许已获准事务的嵌套调用。"""
|
||||
context = self._current_context.get()
|
||||
nested = bool(
|
||||
context
|
||||
and context.admission is self
|
||||
and context.open
|
||||
and context.holders > 0
|
||||
)
|
||||
context_token = None
|
||||
if not nested:
|
||||
context = _MutationContext(admission=self)
|
||||
context_token = self._current_context.set(context)
|
||||
assert context is not None
|
||||
|
||||
acquired = False
|
||||
try:
|
||||
with self._condition:
|
||||
if not self._accepting and not nested:
|
||||
raise PluginMutationRejectedError(operation)
|
||||
self._active_count += 1
|
||||
context.holders += 1
|
||||
acquired = True
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
with self._condition:
|
||||
self._active_count -= 1
|
||||
context.holders -= 1
|
||||
if context.holders == 0:
|
||||
context.open = False
|
||||
self._condition.notify_all()
|
||||
if context_token is not None:
|
||||
self._current_context.reset(context_token)
|
||||
|
||||
def seal(self) -> int:
|
||||
"""原子停止接纳根事务,并返回封口瞬间的活动 lease 数量。"""
|
||||
with self._condition:
|
||||
self._accepting = False
|
||||
return self._active_count
|
||||
|
||||
def wait_until_idle(self) -> None:
|
||||
"""自然等待全部已获准 lease 退出,不取消或遗失其 owner。"""
|
||||
with self._condition:
|
||||
while self._active_count:
|
||||
self._condition.wait()
|
||||
|
||||
def reopen(self) -> bool:
|
||||
"""仅在没有遗留 lease 时为新的应用生命周期重新开放准入。"""
|
||||
with self._condition:
|
||||
if self._active_count:
|
||||
return False
|
||||
self._accepting = True
|
||||
return True
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional, ParamSpec, TypeVar, cast
|
||||
|
||||
@@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
statuses = result.values() if isinstance(result, dict) else (result,)
|
||||
if PluginRuntimeStatus.LOAD_FAILED in statuses:
|
||||
if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses:
|
||||
outcome = "error"
|
||||
return result
|
||||
except BaseException:
|
||||
@@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
|
||||
class PluginLifecycle:
|
||||
"""管理插件发现、初始化、启停和热重载,不持有市场或 HTTP 路由职责。"""
|
||||
|
||||
_EVENT_HANDLERS_QUIESCED = "__event_handlers__"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -83,6 +86,8 @@ class PluginLifecycle:
|
||||
self._runtime_status_writer = runtime_status_writer
|
||||
self._logger = log
|
||||
self._event_sender = event_sender
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._quiesced_hooks: dict[str, set[str]] = {}
|
||||
|
||||
@observe_plugin_lifecycle("start")
|
||||
def start(
|
||||
@@ -116,6 +121,7 @@ class PluginLifecycle:
|
||||
self._classes[current_id] = plugin
|
||||
instance = plugin()
|
||||
instance.init_plugin(self._plugin_config(current_id))
|
||||
self._quiesced_hooks.pop(current_id, None)
|
||||
self._running[current_id] = instance
|
||||
self._logger.info(
|
||||
f"加载插件:{current_id} 版本:{instance.plugin_version}"
|
||||
@@ -156,31 +162,168 @@ class PluginLifecycle:
|
||||
|
||||
@observe_plugin_lifecycle("stop")
|
||||
def stop(self, plugin_id: Optional[str] = None) -> None:
|
||||
"""停止指定插件或全部插件,并清理模块缓存。"""
|
||||
"""按旧单阶段 ABI 先解绑 handler,再停止并强制卸载插件。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
self._quiesce_selected(plugins)
|
||||
self._finalize(
|
||||
plugin_id,
|
||||
require_quiesced=False,
|
||||
disable_events=not self._handlers_quiesced(plugins),
|
||||
)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce")
|
||||
def quiesce(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""先解绑事件 handler,再按旧 hook 顺序停止生产者并保留实例。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
return self._quiesce_selected(plugins)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce_handlers")
|
||||
def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""禁止目标插件接收新事件,保留实例供在途 handler 和后续 hook 使用。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
return self._disable_selected_handlers(plugins)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce_services")
|
||||
def quiesce_services(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""在事件结算屏障后执行旧 close、stop_service hook。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
if not self._handlers_quiesced(plugins):
|
||||
self._logger.warning("插件事件 handler 尚未全部停用,拒绝关闭插件资源")
|
||||
return False
|
||||
return self._quiesce_hooks(plugins)
|
||||
|
||||
def _quiesce_selected(self, plugins: dict[str, Any]) -> bool:
|
||||
"""兼容单阶段调用:先停用全部 handler,再执行稳定快照的旧 hooks。"""
|
||||
if not self._disable_selected_handlers(plugins):
|
||||
return False
|
||||
return self._quiesce_hooks(plugins)
|
||||
|
||||
def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool:
|
||||
"""先停用稳定快照的全部事件入口,任一失败时不执行破坏性 hook。"""
|
||||
all_converged = True
|
||||
for current_id, plugin in plugins.items():
|
||||
completed = self._quiesced_hooks.setdefault(current_id, set())
|
||||
if self._EVENT_HANDLERS_QUIESCED in completed:
|
||||
continue
|
||||
try:
|
||||
self._disable_events(type(plugin))
|
||||
except Exception as error: # noqa: BLE001 插件边界必须隔离
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停用插件 {current_id} 的事件 handler 时发生错误: {error}"
|
||||
)
|
||||
continue
|
||||
completed.add(self._EVENT_HANDLERS_QUIESCED)
|
||||
return all_converged
|
||||
|
||||
def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool:
|
||||
"""执行旧 ABI hooks,并只重试尚未成功的步骤。"""
|
||||
all_converged = True
|
||||
for current_id, plugin in plugins.items():
|
||||
completed = self._quiesced_hooks.setdefault(current_id, set())
|
||||
for hook_name in ("close", "stop_service"):
|
||||
if hook_name in completed:
|
||||
continue
|
||||
hook = getattr(plugin, hook_name, None)
|
||||
if not callable(hook):
|
||||
completed.add(hook_name)
|
||||
continue
|
||||
try:
|
||||
result = hook()
|
||||
except Exception as error: # noqa: BLE001 插件边界必须隔离
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停止插件 {current_id} 的 {hook_name} 时发生错误: {error}"
|
||||
)
|
||||
continue
|
||||
if result is False:
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停止插件 {current_id} 的 {hook_name} 未收敛"
|
||||
)
|
||||
continue
|
||||
completed.add(hook_name)
|
||||
return all_converged
|
||||
|
||||
@observe_plugin_lifecycle("finalize")
|
||||
def finalize(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。"""
|
||||
return self._finalize(
|
||||
plugin_id,
|
||||
require_quiesced=True,
|
||||
disable_events=False,
|
||||
)
|
||||
|
||||
def _finalize(
|
||||
self,
|
||||
plugin_id: Optional[str],
|
||||
*,
|
||||
require_quiesced: bool,
|
||||
disable_events: bool = True,
|
||||
) -> bool:
|
||||
"""按严格或兼容策略卸载插件,并在清理失败时保留实例所有权。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
if require_quiesced and any(
|
||||
not self._is_quiesced(current_id, plugin)
|
||||
for current_id, plugin in plugins.items()
|
||||
):
|
||||
self._logger.warning("插件后台服务尚未全部收敛,拒绝卸载运行实例")
|
||||
return False
|
||||
|
||||
try:
|
||||
if disable_events:
|
||||
for plugin in plugins.values():
|
||||
self._disable_events(type(plugin))
|
||||
self._clear_modules(plugin_id)
|
||||
self._clear_tools()
|
||||
except Exception as error: # noqa: BLE001 保留实例所有权供后续重试
|
||||
self._logger.warning(f"卸载插件运行实例时发生错误: {error}")
|
||||
return False
|
||||
|
||||
if plugin_id:
|
||||
self._classes.pop(plugin_id, None)
|
||||
self._running.pop(plugin_id, None)
|
||||
self._quiesced_hooks.pop(plugin_id, None)
|
||||
else:
|
||||
self._classes.clear()
|
||||
self._running.clear()
|
||||
self._quiesced_hooks.clear()
|
||||
self._logger.info("插件停止完成")
|
||||
return True
|
||||
|
||||
def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]:
|
||||
"""返回本阶段处理的稳定实例快照,并保持旧停机日志语义。"""
|
||||
if plugin_id:
|
||||
self._logger.info(f"正在停止插件 {plugin_id}...")
|
||||
plugin = self._running.get(plugin_id)
|
||||
plugins = {plugin_id: plugin} if plugin else {}
|
||||
if not plugin:
|
||||
self._logger.debug(f"插件 {plugin_id} 不存在或未加载")
|
||||
else:
|
||||
self._logger.info("正在停止所有插件...")
|
||||
plugins = dict(self._running)
|
||||
return plugins
|
||||
self._logger.info("正在停止所有插件...")
|
||||
return dict(self._running)
|
||||
|
||||
for current_id, plugin in plugins.items():
|
||||
self._disable_events(type(plugin))
|
||||
self._stop_plugin(plugin)
|
||||
def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool:
|
||||
"""判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。"""
|
||||
required = {self._EVENT_HANDLERS_QUIESCED} | {
|
||||
hook_name
|
||||
for hook_name in ("close", "stop_service")
|
||||
if callable(getattr(plugin, hook_name, None))
|
||||
}
|
||||
return required.issubset(self._quiesced_hooks.get(plugin_id, set()))
|
||||
|
||||
if plugin_id:
|
||||
self._classes.pop(plugin_id, None)
|
||||
self._running.pop(plugin_id, None)
|
||||
self._clear_modules(plugin_id)
|
||||
else:
|
||||
self._classes.clear()
|
||||
self._running.clear()
|
||||
self._clear_modules(None)
|
||||
self._clear_tools()
|
||||
self._logger.info("插件停止完成")
|
||||
def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool:
|
||||
"""判断稳定快照中的全部插件是否已经停用事件入口。"""
|
||||
return all(
|
||||
self._EVENT_HANDLERS_QUIESCED
|
||||
in self._quiesced_hooks.get(plugin_id, set())
|
||||
for plugin_id in plugins
|
||||
)
|
||||
|
||||
@observe_plugin_lifecycle("reload")
|
||||
def reload(
|
||||
@@ -194,14 +337,3 @@ class PluginLifecycle:
|
||||
status = self.start(plugin_id)[plugin_id]
|
||||
self._event_sender(reload_event, data={"plugin_id": plugin_id})
|
||||
return status
|
||||
|
||||
def _stop_plugin(self, plugin: Any) -> None:
|
||||
"""按插件旧 ABI 顺序关闭资源和服务。"""
|
||||
try:
|
||||
if hasattr(plugin, "close"):
|
||||
plugin.close()
|
||||
if hasattr(plugin, "stop_service"):
|
||||
plugin.stop_service()
|
||||
except Exception as error: # noqa: BLE001
|
||||
name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__
|
||||
self._logger.warning(f"停止插件 {name} 时发生错误: {error}")
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
@@ -27,6 +27,8 @@ class PluginMonitorController:
|
||||
self._logger = log
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._stop_event = threading.Event()
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._closed = False
|
||||
|
||||
@property
|
||||
def stop_event(self) -> threading.Event:
|
||||
@@ -35,32 +37,73 @@ class PluginMonitorController:
|
||||
|
||||
def reload(self, enabled: bool) -> None:
|
||||
"""按当前配置停止旧线程,并在启用时创建新线程。"""
|
||||
self.stop()
|
||||
if enabled:
|
||||
stopped = self.stop()
|
||||
if enabled and stopped:
|
||||
self.start()
|
||||
|
||||
def start(self) -> None:
|
||||
"""启动唯一的守护监控线程。"""
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.info("插件文件修改监测已经在运行中...")
|
||||
return
|
||||
self._logger.info("开始监测插件文件修改...")
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._runner, daemon=True)
|
||||
self._thread.start()
|
||||
with self._lifecycle_lock:
|
||||
if self._closed:
|
||||
self._logger.info("插件文件修改监测已进入停机封口,跳过启动")
|
||||
return
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.info("插件文件修改监测已经在运行中...")
|
||||
return
|
||||
self._logger.info("开始监测插件文件修改...")
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._runner, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""请求监控线程退出,并在限定时间内等待其清理。"""
|
||||
if not self._thread or not self._thread.is_alive():
|
||||
self._logger.info("未启用插件文件修改监测,无需停止")
|
||||
return
|
||||
self._logger.info("正在停止插件文件修改监测...")
|
||||
def reopen(self) -> bool:
|
||||
"""为新的应用生命周期解除封口,仍有旧线程时拒绝重开。"""
|
||||
with self._lifecycle_lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.warning("旧插件文件监测线程仍在运行,无法开启新生命周期")
|
||||
return False
|
||||
self._thread = None
|
||||
self._closed = False
|
||||
return True
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> bool:
|
||||
"""临时停止监控线程,并返回其是否在预算内真正退出。"""
|
||||
return self._stop_with_budget(timeout=timeout, close=False)
|
||||
|
||||
def close(self, timeout: float = 5.0) -> bool:
|
||||
"""永久封口当前生命周期,并返回监控线程是否真正退出。"""
|
||||
return self._stop_with_budget(timeout=timeout, close=True)
|
||||
|
||||
def _stop_with_budget(self, *, timeout: float, close: bool) -> bool:
|
||||
"""在同一预算内取得生命周期锁、设置封口并等待线程退出。"""
|
||||
timeout = max(0.0, timeout)
|
||||
deadline = time.monotonic() + timeout
|
||||
self._stop_event.set()
|
||||
self._thread.join(timeout=5)
|
||||
if self._thread.is_alive():
|
||||
self._logger.warning("插件文件修改监测线程在5秒内未能正常停止。")
|
||||
self._thread = None
|
||||
self._logger.info("插件文件修改监测停止完成")
|
||||
if not self._lifecycle_lock.acquire(timeout=timeout):
|
||||
self._logger.warning(
|
||||
f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权。"
|
||||
)
|
||||
return False
|
||||
try:
|
||||
if close:
|
||||
self._closed = True
|
||||
thread = self._thread
|
||||
self._stop_event.set()
|
||||
if not thread or not thread.is_alive():
|
||||
self._thread = None
|
||||
self._logger.info("未启用插件文件修改监测,无需停止")
|
||||
return True
|
||||
self._logger.info("正在停止插件文件修改监测...")
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if thread.is_alive():
|
||||
self._logger.warning(
|
||||
f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。"
|
||||
)
|
||||
return False
|
||||
self._thread = None
|
||||
self._logger.info("插件文件修改监测停止完成")
|
||||
return True
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
|
||||
class PluginChangeMonitor:
|
||||
|
||||
@@ -1,9 +1,21 @@
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
import posixpath
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ContextManager,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
from watchfiles import watch
|
||||
|
||||
@@ -17,6 +29,7 @@ from app.runtime.execution import run_in_threadpool_to_completion
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.observability import observe_compat_facade
|
||||
from app.runtime.settings import RuntimeSettingsCompat
|
||||
from app.runtime.thread import ThreadHelper
|
||||
|
||||
settings = RuntimeSettingsCompat()
|
||||
from app.runtime.events import EventHandlerBinding, eventmanager
|
||||
@@ -39,6 +52,7 @@ from app.runtime.extensions.plugin.sync import (
|
||||
)
|
||||
from app.runtime.extensions.plugin.clone import PluginCloneService
|
||||
from app.runtime.extensions.plugin.access import PluginAccessPolicy
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
from app.runtime.extensions.plugin.catalog import PluginCatalogFacade
|
||||
from app.runtime.extensions.plugin.paths import PluginPathResolver
|
||||
from app.runtime.extensions.plugin.dependency import (
|
||||
@@ -47,6 +61,7 @@ from app.runtime.extensions.plugin.dependency import (
|
||||
PluginDependencyService,
|
||||
)
|
||||
from app.runtime.extensions.plugin.storage import PluginConfigStore, PluginInstanceStore
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
|
||||
LegacyDiagnosticsConfigurator = Callable[..., None]
|
||||
@@ -195,6 +210,15 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
runner=self._run_file_watcher,
|
||||
log=logger,
|
||||
)
|
||||
self._plugin_quiesce_lock = threading.RLock()
|
||||
self._plugin_quiesce_future: Optional[
|
||||
concurrent.futures.Future[bool]
|
||||
] = None
|
||||
self._plugin_service_quiesce_future: Optional[
|
||||
concurrent.futures.Future[bool]
|
||||
] = None
|
||||
self._plugin_mutation_admission = PluginMutationAdmission()
|
||||
self._plugin_runtime_closed = False
|
||||
self._plugin_dependencies = PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
@@ -315,12 +339,16 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
|
||||
def init_config(self):
|
||||
"""按最新系统配置完整重启插件。"""
|
||||
# 停止已有插件
|
||||
self.stop()
|
||||
classification = self.classify_plugins()
|
||||
self.apply_plugin_dependency_classification(classification)
|
||||
for plugin_id in classification.ready:
|
||||
self.start(plugin_id)
|
||||
try:
|
||||
with self.mutation("配置热重载"):
|
||||
# 停止已有插件
|
||||
self.stop()
|
||||
classification = self.classify_plugins()
|
||||
self.apply_plugin_dependency_classification(classification)
|
||||
for plugin_id in classification.ready:
|
||||
self.start(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def start(self, pid: Optional[str] = None) -> Dict[str, PluginRuntimeStatus]:
|
||||
"""
|
||||
@@ -328,8 +356,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID,为空加载所有插件
|
||||
"""
|
||||
|
||||
_legacy_diagnostics_configurator(enabled=settings.DEBUG, emitter=logger.warning)
|
||||
return self._plugin_lifecycle.start(pid)
|
||||
try:
|
||||
with self.mutation("启动插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
_legacy_diagnostics_configurator(
|
||||
enabled=settings.DEBUG,
|
||||
emitter=logger.warning,
|
||||
)
|
||||
return self._plugin_lifecycle.start(pid)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
if pid:
|
||||
return {pid: PluginRuntimeStatus.LOAD_FAILED}
|
||||
return {}
|
||||
|
||||
def init_plugin(self, plugin_id: str, conf: dict):
|
||||
"""
|
||||
@@ -337,7 +376,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param plugin_id: 插件ID
|
||||
:param conf: 插件配置
|
||||
"""
|
||||
self._plugin_lifecycle.initialize(plugin_id, conf)
|
||||
try:
|
||||
with self.mutation("初始化插件配置"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.initialize(plugin_id, conf)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def clear_plugin_agent_tools_cache(self) -> None:
|
||||
"""
|
||||
@@ -356,12 +400,127 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""兼容读取旧私有字段,实际版本由独立工具目录持有。"""
|
||||
return self._plugin_tool_catalog.revision
|
||||
|
||||
def stop(self, pid: Optional[str] = None):
|
||||
def stop(self, pid: Optional[str] = None) -> None:
|
||||
"""
|
||||
停止插件服务
|
||||
:param pid: 插件ID,为空停止所有插件
|
||||
"""
|
||||
self._plugin_lifecycle.stop(pid)
|
||||
try:
|
||||
with self.mutation("停止插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.stop(pid)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def mutation(self, operation: str) -> ContextManager[None]:
|
||||
"""为一个完整插件可变事务取得可跨异步边界传播的准入 lease。"""
|
||||
return self._plugin_mutation_admission.hold(operation)
|
||||
|
||||
def reopen_plugins(self) -> bool:
|
||||
"""为新应用生命周期解除运行时封口,仍活跃的 quiesce owner 禁止复用。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
futures = (
|
||||
self._plugin_quiesce_future,
|
||||
self._plugin_service_quiesce_future,
|
||||
)
|
||||
if any(future is not None and not future.done() for future in futures):
|
||||
logger.warning("插件后台服务仍在停止,无法开启新的应用生命周期")
|
||||
return False
|
||||
if self._plugin_runtime_closed and self._running_plugins:
|
||||
logger.warning("上一应用生命周期仍持有插件实例,拒绝解除运行时封口")
|
||||
return False
|
||||
if not self._plugin_mutation_admission.reopen():
|
||||
logger.warning("上一应用生命周期仍有插件可变事务,拒绝解除运行时封口")
|
||||
return False
|
||||
self._plugin_runtime_closed = False
|
||||
return True
|
||||
|
||||
async def quiesce_plugins(self, timeout: float = 240.0) -> bool:
|
||||
"""封口变更事务并停用插件 handler,超时后保留 Future ownership。"""
|
||||
if self._plugin_mutation_admission.is_held():
|
||||
logger.warning("插件可变事务不能等待自身收敛,拒绝在事务内执行停机")
|
||||
return False
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_runtime_closed = True
|
||||
self._plugin_mutation_admission.seal()
|
||||
future = self._plugin_quiesce_future
|
||||
if future is None or future.done():
|
||||
future = ThreadHelper().submit(self._quiesce_after_mutations)
|
||||
self._plugin_quiesce_future = future
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(asyncio.wrap_future(future)),
|
||||
timeout=max(0.0, timeout),
|
||||
)
|
||||
return bool(result)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"插件后台服务未在 {timeout:g} 秒内收敛")
|
||||
return False
|
||||
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
|
||||
logger.error(f"插件后台服务停止失败:{error}", exc_info=True)
|
||||
return False
|
||||
finally:
|
||||
if future.done():
|
||||
with self._plugin_quiesce_lock:
|
||||
if self._plugin_quiesce_future is future:
|
||||
self._plugin_quiesce_future = None
|
||||
|
||||
async def quiesce_plugin_services(self, timeout: float = 240.0) -> bool:
|
||||
"""在事件结算后执行旧插件停机 hook,并有界等待同步 owner。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
prepare_future = self._plugin_quiesce_future
|
||||
if prepare_future is not None and not prepare_future.done():
|
||||
logger.warning("插件事件入口仍在封口,拒绝提前关闭插件资源")
|
||||
return False
|
||||
if not self._plugin_runtime_closed:
|
||||
logger.warning("插件运行时尚未封口,拒绝关闭插件资源")
|
||||
return False
|
||||
if self._plugin_mutation_admission.active_count:
|
||||
logger.warning("插件可变事务仍在执行,拒绝关闭插件资源")
|
||||
return False
|
||||
future = self._plugin_service_quiesce_future
|
||||
if future is None or future.done():
|
||||
future = ThreadHelper().submit(
|
||||
self._plugin_lifecycle.quiesce_services,
|
||||
)
|
||||
self._plugin_service_quiesce_future = future
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(asyncio.wrap_future(future)),
|
||||
timeout=max(0.0, timeout),
|
||||
)
|
||||
return bool(result)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"插件旧停机 hook 未在 {timeout:g} 秒内收敛")
|
||||
return False
|
||||
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
|
||||
logger.error(f"插件旧停机 hook 执行失败:{error}", exc_info=True)
|
||||
return False
|
||||
finally:
|
||||
if future.done():
|
||||
with self._plugin_quiesce_lock:
|
||||
if self._plugin_service_quiesce_future is future:
|
||||
self._plugin_service_quiesce_future = None
|
||||
|
||||
def finalize_plugins(self) -> bool:
|
||||
"""确认 quiesce owner 已结束后禁用 handler 并卸载插件实例。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
futures = (
|
||||
self._plugin_quiesce_future,
|
||||
self._plugin_service_quiesce_future,
|
||||
)
|
||||
if any(future is not None and not future.done() for future in futures):
|
||||
logger.warning("插件后台服务仍在停止,拒绝释放运行实例")
|
||||
return False
|
||||
if self._plugin_mutation_admission.active_count:
|
||||
logger.warning("插件可变事务仍在执行,拒绝释放运行实例")
|
||||
return False
|
||||
return self._plugin_lifecycle.finalize()
|
||||
|
||||
def _quiesce_after_mutations(self) -> bool:
|
||||
"""等待已获准变更自然结束后,再停用插件事件入口。"""
|
||||
self._plugin_mutation_admission.wait_until_idle()
|
||||
return self._plugin_lifecycle.quiesce_handlers()
|
||||
|
||||
@staticmethod
|
||||
def _load_selective_plugins(pid: Optional[str], installed_plugins: List[str],
|
||||
@@ -428,8 +587,10 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""返回配置重载日志使用的功能名称。"""
|
||||
return "插件文件修改监测"
|
||||
|
||||
def start_monitor(self):
|
||||
"""按当前配置启动插件文件修改监测。"""
|
||||
def start_monitor(self, *, reopen: bool = False) -> None:
|
||||
"""按当前配置启动监控;新生命周期可显式解除既有封口。"""
|
||||
if reopen and not self._plugin_monitor.reopen():
|
||||
return
|
||||
if (
|
||||
not self.is_plugin_settling()
|
||||
and (settings.DEV or settings.PLUGIN_AUTO_RELOAD)
|
||||
@@ -447,11 +608,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
)
|
||||
)
|
||||
|
||||
def stop_monitor(self):
|
||||
"""
|
||||
停止监测插件文件修改监测
|
||||
"""
|
||||
self._plugin_monitor.stop()
|
||||
def stop_monitor(self, timeout: float = 5.0) -> bool:
|
||||
"""停止插件文件监控,并返回线程是否在预算内真正退出。"""
|
||||
return self._plugin_monitor.stop(timeout=timeout)
|
||||
|
||||
def close_monitor(self, timeout: float = 5.0) -> bool:
|
||||
"""封口当前生命周期的文件监控,并返回线程是否真正退出。"""
|
||||
return self._plugin_monitor.close(timeout=timeout)
|
||||
|
||||
def _run_file_watcher(self):
|
||||
"""
|
||||
@@ -504,25 +667,31 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
已安装本地插件源码变化时,同步到运行目录
|
||||
"""
|
||||
return self._local_plugin_sync.sync(pid, candidate)
|
||||
try:
|
||||
with self.mutation("同步本地插件源码"):
|
||||
return self._local_plugin_sync.sync(pid, candidate)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
@contextmanager
|
||||
def suppress_plugin_monitor(self, plugin_id: str):
|
||||
"""在插件目录原子更新期间阻止文件监控抢先重载半成品。"""
|
||||
normalized_id = plugin_id.lower()
|
||||
with self._monitor_suppression_lock:
|
||||
self._suppressed_monitor_plugins[normalized_id] = (
|
||||
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self.mutation("更新插件包"):
|
||||
normalized_id = plugin_id.lower()
|
||||
with self._monitor_suppression_lock:
|
||||
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
|
||||
if count <= 1:
|
||||
self._suppressed_monitor_plugins.pop(normalized_id, None)
|
||||
else:
|
||||
self._suppressed_monitor_plugins[normalized_id] = count - 1
|
||||
self._suppressed_monitor_plugins[normalized_id] = (
|
||||
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self._monitor_suppression_lock:
|
||||
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
|
||||
if count <= 1:
|
||||
self._suppressed_monitor_plugins.pop(normalized_id, None)
|
||||
else:
|
||||
self._suppressed_monitor_plugins[normalized_id] = count - 1
|
||||
|
||||
def is_plugin_monitor_suppressed(self, plugin_id: str) -> bool:
|
||||
"""判断指定插件是否处于安装或替换写入阶段。"""
|
||||
@@ -534,23 +703,43 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
从内存中移除一个插件
|
||||
:param plugin_id: 插件ID
|
||||
"""
|
||||
self._plugin_lifecycle.stop(plugin_id)
|
||||
self._plugin_registry.remove(plugin_id)
|
||||
try:
|
||||
with self.mutation("移除插件实例"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.stop(plugin_id)
|
||||
self._plugin_registry.remove(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def reload_plugin(self, plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""
|
||||
将一个插件重新加载到内存
|
||||
:param plugin_id: 插件ID
|
||||
"""
|
||||
return self._plugin_lifecycle.reload(plugin_id, EventType.PluginReload)
|
||||
try:
|
||||
with self.mutation("重新加载插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
return self._plugin_lifecycle.reload(
|
||||
plugin_id,
|
||||
EventType.PluginReload,
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return PluginRuntimeStatus.LOAD_FAILED
|
||||
|
||||
def reload_plugin_tree(self, plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载源码插件,并同步刷新所有引用该源码的虚拟实例。"""
|
||||
source_plugin_id = self.get_plugin_source_id(plugin_id)
|
||||
status = self.reload_plugin(source_plugin_id)
|
||||
for instance in self._plugin_instance_store.for_source(source_plugin_id):
|
||||
self.reload_plugin(instance.instance_id)
|
||||
return status
|
||||
try:
|
||||
with self.mutation("重载插件实例树"):
|
||||
with self._plugin_quiesce_lock:
|
||||
source_plugin_id = self.get_plugin_source_id(plugin_id)
|
||||
status = self.reload_plugin(source_plugin_id)
|
||||
for instance in self._plugin_instance_store.for_source(source_plugin_id):
|
||||
self.reload_plugin(instance.instance_id)
|
||||
return status
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return PluginRuntimeStatus.LOAD_FAILED
|
||||
|
||||
def get_plugin_reload_targets(self, plugin_id: str) -> List[str]:
|
||||
"""返回源码更新后需要刷新注册信息的源插件及其实例 ID。"""
|
||||
@@ -584,33 +773,40 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
安装本地不存在或需要更新的插件
|
||||
"""
|
||||
|
||||
return self._plugin_sync.sync()
|
||||
with self.mutation("同步插件包"):
|
||||
return self._plugin_sync.sync()
|
||||
|
||||
@staticmethod
|
||||
def install_plugin_missing_dependencies() -> List[str]:
|
||||
"""
|
||||
安装插件中缺失或不兼容的依赖项
|
||||
"""
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing()
|
||||
|
||||
@staticmethod
|
||||
def install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
|
||||
"""安装插件缺失依赖并返回缺失项及安装成功状态。"""
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing_with_status()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing_with_status()
|
||||
|
||||
@staticmethod
|
||||
async def async_install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
|
||||
"""在异步启动链中恢复插件依赖并保留取消语义。"""
|
||||
return await PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).async_install_missing_with_status()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return await PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).async_install_missing_with_status()
|
||||
|
||||
def classify_plugins(self) -> PluginDependencyClassification:
|
||||
"""按源码依赖状态分类物理插件,并把结果映射到虚拟实例。"""
|
||||
@@ -706,7 +902,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
|
||||
def delete_plugin_instance(self, plugin_id: str) -> bool:
|
||||
"""删除虚拟实例描述;调用方仍负责停止实例和清理业务数据。"""
|
||||
return self._plugin_instance_store.delete(plugin_id)
|
||||
try:
|
||||
with self.mutation("删除插件实例描述"):
|
||||
return self._plugin_instance_store.delete(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def save_plugin_config(self, pid: str, conf: dict, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -715,7 +916,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param conf: 配置
|
||||
:param force: 强制保存
|
||||
"""
|
||||
return self._plugin_config_store.write(pid, conf, force)
|
||||
try:
|
||||
with self.mutation("保存插件配置"):
|
||||
return self._plugin_config_store.write(pid, conf, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
async def async_save_plugin_config(
|
||||
self, pid: str, conf: dict, force: bool = False
|
||||
@@ -726,7 +932,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param conf: 配置
|
||||
:param force: 强制保存
|
||||
"""
|
||||
return await self._plugin_config_store.async_write(pid, conf, force)
|
||||
try:
|
||||
with self.mutation("保存插件配置"):
|
||||
return await self._plugin_config_store.async_write(pid, conf, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def delete_plugin_config(self, pid: str, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -734,7 +945,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID
|
||||
:param force: 插件停止后仍允许按插件 ID 删除持久化配置
|
||||
"""
|
||||
return self._plugin_config_store.delete(pid, force)
|
||||
try:
|
||||
with self.mutation("删除插件配置"):
|
||||
return self._plugin_config_store.delete(pid, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def delete_plugin_data(self, pid: str, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -742,7 +958,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID
|
||||
:param force: 插件停止后仍允许按插件 ID 删除持久化数据
|
||||
"""
|
||||
return self._plugin_config_store.delete_data(pid, force)
|
||||
try:
|
||||
with self.mutation("删除插件数据"):
|
||||
return self._plugin_config_store.delete_data(pid, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def get_plugin_state(self, pid: str) -> bool:
|
||||
"""
|
||||
@@ -1130,14 +1351,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param icon: 自定义图标URL
|
||||
:return: (是否成功, 错误信息)
|
||||
"""
|
||||
return self._plugin_clone.clone(
|
||||
plugin_id=plugin_id,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
try:
|
||||
with self.mutation("创建插件分身"):
|
||||
return self._plugin_clone.clone(
|
||||
plugin_id=plugin_id,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False, str(error)
|
||||
|
||||
def _modify_plugin_files(self, plugin_dir: Path, original_id: str, suffix: str,
|
||||
name: str, description: str, version: str = None,
|
||||
|
||||
@@ -106,8 +106,8 @@ class TaskRegistry:
|
||||
}
|
||||
)
|
||||
|
||||
async def shutdown(self, *, timeout_seconds: float = 10.0) -> None:
|
||||
"""停止接收并有限等待存量任务,超时任务保留登记并报告责任域。"""
|
||||
async def shutdown(self, *, timeout_seconds: float = 10.0) -> bool:
|
||||
"""停止接收并有限等待存量任务,返回全部 owner 是否真实收敛。"""
|
||||
self._accepting = False
|
||||
records = self.records
|
||||
tasks = [record.task for record in records]
|
||||
@@ -139,6 +139,7 @@ class TaskRegistry:
|
||||
"timeout_seconds": timeout_seconds,
|
||||
}
|
||||
)
|
||||
return all(record.task.done() for record in records)
|
||||
|
||||
|
||||
_default_registry = TaskRegistry()
|
||||
|
||||
@@ -4,6 +4,7 @@ __all__ = (
|
||||
"APIRateLimitException",
|
||||
"RateLimitExceededException",
|
||||
"OperationInterrupted",
|
||||
"PluginMutationRejectedError",
|
||||
"StorageQueryError",
|
||||
"TMDbException",
|
||||
)
|
||||
@@ -49,6 +50,15 @@ class OperationInterrupted(KeyboardInterrupt):
|
||||
pass
|
||||
|
||||
|
||||
class PluginMutationRejectedError(RuntimeError):
|
||||
"""表示插件运行时已封口,新的可变事务未获准执行。"""
|
||||
|
||||
def __init__(self, operation: str) -> None:
|
||||
"""保存被拒绝的操作名称并生成稳定诊断消息。"""
|
||||
self.operation = operation
|
||||
super().__init__(f"插件运行时已进入停机阶段,拒绝{operation}")
|
||||
|
||||
|
||||
class StorageQueryError(Exception):
|
||||
"""
|
||||
用于表示存储查询无法确认结果的异常类。
|
||||
|
||||
@@ -20,6 +20,9 @@ from app.runtime.log import logger
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
|
||||
def _get_skill_catalog() -> Any:
|
||||
"""按需返回 Agent 技能目录实现,供消息应用层消费端口。"""
|
||||
from app.agent.skills.registry import SkillHelper
|
||||
@@ -106,6 +109,7 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
self._manager: Any = None
|
||||
self._compat_injected = False
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
eventmanager.add_event_listener(
|
||||
EventType.ConfigChanged,
|
||||
@@ -117,6 +121,7 @@ class AgentInitializer:
|
||||
初始化AI智能体管理器
|
||||
"""
|
||||
try:
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
if agent_manager is not None:
|
||||
if not settings.AI_AGENT_ENABLE:
|
||||
@@ -142,7 +147,12 @@ class AgentInitializer:
|
||||
async def handle_config_changed(self, event: Event) -> None:
|
||||
"""仅在 manifest watch 命中时协调 service,关闭态保持 fail closed。"""
|
||||
changed_keys = _event_changed_keys(event)
|
||||
if not changed_keys or self._compat_injected or self._shutdown_complete:
|
||||
if (
|
||||
not changed_keys
|
||||
or self._compat_injected
|
||||
or self._shutdown_started
|
||||
or self._shutdown_complete
|
||||
):
|
||||
return
|
||||
try:
|
||||
self._manager = await reconcile_agent_service(
|
||||
@@ -156,24 +166,25 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
logger.debug(f"配置变更协调AI智能体失败: {error}")
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""清理 initializer 引用;显式注入对象同时在此关闭。"""
|
||||
async def cleanup(self) -> bool:
|
||||
"""清理 initializer 引用;未收敛的显式注入对象继续由本实例持有。"""
|
||||
try:
|
||||
manager = self._manager
|
||||
compat_injected = self._compat_injected
|
||||
if manager is None:
|
||||
return
|
||||
try:
|
||||
if compat_injected:
|
||||
await manager.close()
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
finally:
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
if compat_injected and await manager.close() is False:
|
||||
logger.error("AI智能体管理器仍有会话 owner 未收敛")
|
||||
return False
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"关闭AI智能体管理器时发生错误: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# 全局AI智能体初始化器实例
|
||||
@@ -204,23 +215,56 @@ async def init_agent() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def stop_agent():
|
||||
async def stop_agent() -> bool:
|
||||
"""
|
||||
停止AI智能体(异步版本,用于在应用关闭时调用)
|
||||
停止AI智能体,并在全部会话和工具资源释放后返回 True。
|
||||
"""
|
||||
converged = True
|
||||
close_blocking_executors = None
|
||||
agent_initializer._shutdown_started = True
|
||||
try:
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import (
|
||||
begin_blocking_executor_shutdown,
|
||||
close_blocking_executors as close_executors,
|
||||
)
|
||||
|
||||
# 必须在任何 manager await 之前封口,避免旧会话趁收尾窗口提交新同步调用。
|
||||
begin_blocking_executor_shutdown(cancel_futures=True)
|
||||
close_blocking_executors = close_executors
|
||||
except Exception as e:
|
||||
logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
try:
|
||||
if not agent_initializer._shutdown_complete:
|
||||
if agent_initializer._compat_injected:
|
||||
await agent_initializer.cleanup()
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
else:
|
||||
await begin_agent_shutdown()
|
||||
await agent_initializer.cleanup()
|
||||
agent_initializer._shutdown_complete = True
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import shutdown_blocking_executors
|
||||
|
||||
shutdown_blocking_executors(wait=False, cancel_futures=True)
|
||||
service_converged = await begin_agent_shutdown()
|
||||
if service_converged is not False:
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
converged = converged and service_converged is not False
|
||||
except Exception as e:
|
||||
logger.error(f"停止AI智能体时发生错误: {e}")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
converged = False
|
||||
|
||||
if close_blocking_executors is not None:
|
||||
try:
|
||||
blocking_converged = await close_blocking_executors(
|
||||
timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS,
|
||||
cancel_futures=True,
|
||||
)
|
||||
converged = converged and blocking_converged
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
if converged:
|
||||
try:
|
||||
await close_materialized_terminal_sessions()
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体终端会话时发生错误: {e}")
|
||||
converged = False
|
||||
agent_initializer._shutdown_complete = converged
|
||||
return converged
|
||||
|
||||
@@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.runtime.state import SystemHelper
|
||||
from app.runtime.log import logger, LoggerManager
|
||||
from app.startup.command_initializer import init_command, stop_command, restart_command
|
||||
from app.startup.agent_initializer import stop_agent
|
||||
from app.startup.domain_initializer import configure_domain_dependencies
|
||||
from app.startup.modules_initializer import init_modules, stop_modules
|
||||
from app.startup.modules_initializer import (
|
||||
drain_events,
|
||||
init_modules,
|
||||
settle_events,
|
||||
stop_modules,
|
||||
)
|
||||
from app.startup.monitor_initializer import stop_monitor, init_monitor
|
||||
from app.startup.plugins_initializer import (
|
||||
configure_plugin_services,
|
||||
execute_task,
|
||||
finalize_plugins,
|
||||
init_plugins,
|
||||
stop_plugins,
|
||||
quiesce_plugin_services,
|
||||
quiesce_plugins,
|
||||
stop_plugin_monitor,
|
||||
sync_plugins,
|
||||
)
|
||||
from app.startup.routers_initializer import init_routers
|
||||
@@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import (
|
||||
init_plugin_scheduler,
|
||||
)
|
||||
from app.db.engine import check_connection_budget, get_engine, get_global_async_engine
|
||||
from app.startup.transfer_initializer import replay_pending_transfers
|
||||
from app.startup.transfer_initializer import (
|
||||
replay_pending_transfers,
|
||||
stop_transfer_runtime,
|
||||
)
|
||||
from app.startup.workflow_initializer import init_workflow, stop_workflow
|
||||
from app.startup.lifecycle.components import (
|
||||
LifecycleComponent,
|
||||
LifecycleFailurePolicy,
|
||||
LifecycleMode,
|
||||
lifecycle_manifest,
|
||||
)
|
||||
@@ -101,8 +114,8 @@ async def run_shutdown_step(
|
||||
name: str,
|
||||
callback: Callable[[], object],
|
||||
timeout_seconds: float | None = None,
|
||||
) -> None:
|
||||
"""在有限预算内执行关闭阶段,并保留未收敛任务的资源所有权。"""
|
||||
) -> bool:
|
||||
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛。"""
|
||||
try:
|
||||
result = callback()
|
||||
if inspect.isawaitable(result):
|
||||
@@ -120,16 +133,22 @@ async def run_shutdown_step(
|
||||
task.add_done_callback(_consume_shutdown_result)
|
||||
if timeout_seconds:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(task), timeout=timeout_seconds
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
|
||||
task.cancel()
|
||||
return False
|
||||
else:
|
||||
await task
|
||||
result = await task
|
||||
if result is False:
|
||||
logger.error("关闭%s未收敛,资源所有权保持不变", name)
|
||||
return False
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return False
|
||||
|
||||
|
||||
async def run_startup_step(
|
||||
@@ -152,6 +171,66 @@ async def run_startup_step(
|
||||
logger.info("启动%s完成,耗时=%.2fms", name, elapsed_ms)
|
||||
|
||||
|
||||
async def stop_lifecycle_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
) -> bool:
|
||||
"""按声明顺序关闭组件,并在关键 owner 未收敛时停止释放依赖。"""
|
||||
all_converged = True
|
||||
for component in sorted(
|
||||
(item for item in components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
completed = await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
if completed:
|
||||
continue
|
||||
all_converged = False
|
||||
if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST:
|
||||
logger.error(
|
||||
"关闭%s未收敛,停止释放其后续依赖",
|
||||
component.name,
|
||||
)
|
||||
break
|
||||
return all_converged
|
||||
|
||||
|
||||
def select_startup_cleanup_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
*,
|
||||
started_names: set[str],
|
||||
active_component: LifecycleComponent | None,
|
||||
) -> tuple[LifecycleComponent, ...]:
|
||||
"""选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。"""
|
||||
cleanup_names = set(started_names)
|
||||
if active_component is not None:
|
||||
cleanup_names.add(active_component.name)
|
||||
|
||||
# stop-only owner 没有启动回调,按已激活依赖递归纳入,避免清理时反向
|
||||
# 实例化尚未触达的插件、模块或外部资源。
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for component in components:
|
||||
if (
|
||||
component.stop is None
|
||||
or component.start is not None
|
||||
or component.name in cleanup_names
|
||||
or not set(component.dependencies).issubset(cleanup_names)
|
||||
):
|
||||
continue
|
||||
cleanup_names.add(component.name)
|
||||
changed = True
|
||||
|
||||
return tuple(
|
||||
component
|
||||
for component in components
|
||||
if component.stop is not None and component.name in cleanup_names
|
||||
)
|
||||
|
||||
|
||||
async def initialize_modules_component(app: FastAPI) -> None:
|
||||
"""启动模块并把其类型化运行时发布到当前 FastAPI AppState。"""
|
||||
try:
|
||||
@@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None:
|
||||
configure_task_registry(task_registry)
|
||||
|
||||
|
||||
async def stop_task_registry(app: FastAPI) -> None:
|
||||
"""停止接收新后台任务,并取消、等待当前 lifespan 的存量任务。"""
|
||||
async def stop_task_registry(app: FastAPI) -> bool:
|
||||
"""停止接收新后台任务,并把已封口登记器保留到下一次显式启动。"""
|
||||
task_registry = getattr(app.state, "task_registry", None)
|
||||
try:
|
||||
if isinstance(task_registry, TaskRegistry):
|
||||
await task_registry.shutdown(timeout_seconds=30.0)
|
||||
finally:
|
||||
if not isinstance(task_registry, TaskRegistry):
|
||||
configure_task_registry(None)
|
||||
app.state.task_registry = None
|
||||
return True
|
||||
# 即使 owner 已收敛,也不能在后续插件/模块 stop hook 仍会运行时退回永久
|
||||
# accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。
|
||||
return await task_registry.shutdown(timeout_seconds=30.0)
|
||||
|
||||
|
||||
def prepare_plugin_restore() -> None:
|
||||
@@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=5,
|
||||
start_timeout_seconds=30,
|
||||
stop_timeout_seconds=60,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="数据库准备",
|
||||
@@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
dependencies=("插件备份恢复",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=init_plugins,
|
||||
stop=stop_plugins,
|
||||
stop=finalize_plugins,
|
||||
start_order=90,
|
||||
stop_order=60,
|
||||
start_timeout_seconds=300,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件变更监控",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=stop_plugin_monitor,
|
||||
stop_order=8,
|
||||
stop_timeout_seconds=10,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="定时器",
|
||||
@@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=50,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="监控器",
|
||||
@@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=40,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="整理后台服务",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_transfer_runtime,
|
||||
stop_order=52,
|
||||
stop_timeout_seconds=45,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="AI智能体会话",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_agent,
|
||||
stop_order=51,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件事件入口",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugins,
|
||||
stop_order=53,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件尾任务结算",
|
||||
dependencies=("模块服务", "插件"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=settle_events,
|
||||
stop_order=54,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件后台服务",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugin_services,
|
||||
stop_order=55,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件投递屏障",
|
||||
dependencies=("模块服务", "整理后台服务"),
|
||||
stop=drain_events,
|
||||
stop_order=58,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="待处理整理回放",
|
||||
dependencies=("监控器",),
|
||||
dependencies=("监控器", "整理后台服务"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=replay_pending_transfers,
|
||||
start_order=120,
|
||||
@@ -369,6 +513,9 @@ async def lifespan(app: FastAPI):
|
||||
health = get_application_health(app)
|
||||
health.begin_startup()
|
||||
main_loop = asyncio.get_running_loop()
|
||||
enabled_components: tuple[LifecycleComponent, ...] = ()
|
||||
started_component_names: set[str] = set()
|
||||
active_start_component: LifecycleComponent | None = None
|
||||
try:
|
||||
validate_process_topology(
|
||||
workers=settings.API_WORKERS,
|
||||
@@ -390,11 +537,14 @@ async def lifespan(app: FastAPI):
|
||||
(item for item in enabled_components if item.start is not None),
|
||||
key=lambda item: item.start_order or 0,
|
||||
):
|
||||
active_start_component = component
|
||||
await run_startup_step(
|
||||
component.name,
|
||||
component.start,
|
||||
component.start_timeout_seconds,
|
||||
)
|
||||
started_component_names.add(component.name)
|
||||
active_start_component = None
|
||||
if settings.MOVIEPILOT_SAFE_MODE:
|
||||
print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.")
|
||||
# 插件同步到本地
|
||||
@@ -407,10 +557,15 @@ async def lifespan(app: FastAPI):
|
||||
except BaseException:
|
||||
# Uvicorn 在 lifespan 抛错时不会开始接流量;状态仍需供嵌入式入口和测试诊断。
|
||||
health.mark_failed()
|
||||
cleanup_components = select_startup_cleanup_components(
|
||||
enabled_components,
|
||||
started_names=started_component_names,
|
||||
active_component=active_start_component,
|
||||
)
|
||||
try:
|
||||
await stop_task_registry(app)
|
||||
await stop_lifecycle_components(cleanup_components)
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"启动失败后的后台任务清理失败:{cleanup_error}")
|
||||
logger.error(f"启动失败后的生命周期清理失败:{cleanup_error}")
|
||||
finally:
|
||||
global_vars.clear_loop(main_loop)
|
||||
raise
|
||||
@@ -421,21 +576,10 @@ async def lifespan(app: FastAPI):
|
||||
health.mark_stopping()
|
||||
print("Shutting down...")
|
||||
global_vars.stop_system()
|
||||
# 插件恢复会在线程池中修改源码与依赖,必须完成后再进入资源关闭阶段。
|
||||
try:
|
||||
await sync_plugins_task
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
try:
|
||||
for component in sorted(
|
||||
(item for item in enabled_components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
# 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner
|
||||
# 在统一预算内取消/等待,不能在屏障之前无界 await 绕过停机预算。
|
||||
await stop_lifecycle_components(enabled_components)
|
||||
finally:
|
||||
try:
|
||||
# 日志最后关闭,确保其他组件的收尾信息已写入文件
|
||||
|
||||
@@ -129,7 +129,7 @@ from app.command import CommandChain
|
||||
from app.schemas.message import Message
|
||||
from app.schemas.message import MessageType
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
from app.startup.agent_initializer import init_agent, stop_agent
|
||||
from app.startup.agent_initializer import init_agent
|
||||
from app.startup.database import build_database_governance
|
||||
from app.startup.managed_resources_initializer import (
|
||||
init_managed_resources,
|
||||
@@ -560,6 +560,22 @@ def close_browser_sessions() -> None:
|
||||
BrowserSessionHelper.close_all_sessions()
|
||||
|
||||
|
||||
async def drain_events() -> bool:
|
||||
"""在插件卸载前等待已接收事件及其同步、异步处理器完成。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=True)
|
||||
|
||||
|
||||
async def settle_events() -> bool:
|
||||
"""在插件 handler 停用后结算在途事件,但保留停机 hook 的尾事件入口。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=False)
|
||||
|
||||
|
||||
async def stop_modules():
|
||||
"""
|
||||
服务关闭
|
||||
@@ -578,7 +594,6 @@ async def stop_modules():
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return True
|
||||
|
||||
await run_step("AI智能体", stop_agent)
|
||||
await run_step("模块", lambda: ModuleManager().shutdown())
|
||||
await run_step("事件消费", lambda: EventManager().stop_async())
|
||||
await run_step("浏览器会话", close_browser_sessions)
|
||||
|
||||
@@ -1,15 +1,26 @@
|
||||
from app.monitor import Monitor
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
|
||||
|
||||
def init_monitor():
|
||||
"""
|
||||
初始化监控器
|
||||
"""
|
||||
Monitor()
|
||||
def init_monitor() -> None:
|
||||
"""初始化监控器;复用单例时必须显式开启新的应用 lifespan。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
Monitor()
|
||||
return
|
||||
if not monitor.lifecycle_closed:
|
||||
return
|
||||
if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("旧目录监控 owner 未收敛,无法开启新生命周期")
|
||||
if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("目录监控初始化失败")
|
||||
|
||||
|
||||
def stop_monitor():
|
||||
"""
|
||||
停止监控器
|
||||
"""
|
||||
Monitor().stop()
|
||||
async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool:
|
||||
"""在线程池里永久关闭监控器,取消后仍等待同步 owner 收敛到终态。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
return True
|
||||
return bool(
|
||||
await run_in_threadpool_to_completion(monitor.close, timeout=timeout)
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import (
|
||||
configure_plugin_resource_import_preparer,
|
||||
configure_site_auth_level_provider,
|
||||
)
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult
|
||||
from app.application.plugin.catalog import PluginCatalogService
|
||||
from app.application.plugin.data import DeletePluginDataCommand
|
||||
@@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork
|
||||
from app.runtime.log import logger
|
||||
from app.foundation.version import compare_version
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
@@ -142,56 +144,64 @@ async def sync_plugins() -> bool:
|
||||
"""
|
||||
plugin_manager = None
|
||||
try:
|
||||
configure_plugin_services()
|
||||
loop = global_vars.loop
|
||||
plugin_manager = PluginManager()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
with plugin_manager.mutation("启动后同步插件"):
|
||||
configure_plugin_services()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
return await _sync_plugins_admitted(plugin_manager, loop)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"插件初始化过程中出现异常: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool:
|
||||
"""在一个 admission lease 内完成包、依赖、实例和动态路由同步。"""
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _activate_ready_plugins(
|
||||
plugin_manager: PluginManager,
|
||||
ready_ids: tuple[str, ...],
|
||||
@@ -217,12 +227,39 @@ def _activate_ready_plugins(
|
||||
return changed_ids
|
||||
|
||||
|
||||
async def quiesce_plugins(timeout: float = 240.0) -> bool:
|
||||
"""封口插件变更并停用 handler,保留超时 Future 的运行所有权。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugins(timeout=timeout)
|
||||
|
||||
|
||||
async def quiesce_plugin_services(timeout: float = 240.0) -> bool:
|
||||
"""在事件结算后有界执行旧插件 close、stop_service hook。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugin_services(timeout=timeout)
|
||||
|
||||
|
||||
def finalize_plugins() -> bool:
|
||||
"""在事件屏障封口后卸载已停用 handler 的插件实例。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return bool(plugin_manager.finalize_plugins())
|
||||
|
||||
|
||||
async def execute_task(loop, task_func, task_name):
|
||||
"""
|
||||
执行后台任务
|
||||
执行后台任务;取消调用方时仍持有同步线程直到真实完成。
|
||||
"""
|
||||
try:
|
||||
result = await loop.run_in_executor(None, task_func)
|
||||
# loop 参数属于既有调用 ABI;同步执行改由 completion-aware 适配器持有,
|
||||
# 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。
|
||||
del loop
|
||||
result = await run_in_threadpool_to_completion(task_func)
|
||||
if isinstance(result, PluginDependencyInstallResult):
|
||||
processed_count = len(result.missing)
|
||||
elif isinstance(result, list):
|
||||
@@ -245,13 +282,15 @@ def init_plugins():
|
||||
"""
|
||||
configure_plugin_services()
|
||||
plugin_manager = PluginManager()
|
||||
if not plugin_manager.reopen_plugins():
|
||||
raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛")
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
for plugin_id in classification.ready:
|
||||
plugin_manager.start(plugin_id)
|
||||
register_plugin_api()
|
||||
plugin_manager.start_monitor()
|
||||
plugin_manager.start_monitor(reopen=True)
|
||||
logger.info(
|
||||
"插件启动分类:立即加载=%s,等待依赖=%s,等待源码=%s",
|
||||
len(classification.ready),
|
||||
@@ -260,15 +299,30 @@ def init_plugins():
|
||||
)
|
||||
|
||||
|
||||
def stop_plugins():
|
||||
"""
|
||||
停止插件
|
||||
"""
|
||||
def stop_plugin_monitor(timeout: float = 5.0) -> bool:
|
||||
"""封口已创建管理器的文件监控线程,并返回是否完成收口。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
try:
|
||||
plugin_manager = PluginManager()
|
||||
return bool(plugin_manager.close_monitor(timeout=timeout))
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件文件监控时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def stop_plugins() -> bool:
|
||||
"""停止已创建的插件监控和运行实例,不在停机阶段反向物化管理器。"""
|
||||
try:
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
monitor_stopped = True
|
||||
try:
|
||||
plugin_manager.stop_monitor()
|
||||
monitor_stopped = plugin_manager.stop_monitor()
|
||||
finally:
|
||||
plugin_manager.stop()
|
||||
return bool(monitor_stopped)
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
@@ -11,3 +11,15 @@ def replay_pending_transfers():
|
||||
回放本身在后台线程执行,不阻塞启动流程。
|
||||
"""
|
||||
TransferChain().replay_pending()
|
||||
|
||||
|
||||
async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool:
|
||||
"""关闭已存在的整理后台 owner,且不在关停阶段创建新的整理链实例。
|
||||
|
||||
:param timeout_seconds: worker 与 pending 回放共享的最大等待秒数
|
||||
:return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True
|
||||
"""
|
||||
transfer_chain = TransferChain.get_existing_instance()
|
||||
if transfer_chain is None:
|
||||
return True
|
||||
return await transfer_chain.close(timeout_seconds=timeout_seconds)
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
> 审计范围:宿主后端;排除 `app/plugins/**` 运行时插件副本
|
||||
> 规范优先级:`AGENTS.md` 与 `docs/rules/` 高于本文
|
||||
> 相关文档:`docs/architecture-overview.md`、`docs/refactor/backend-architecture-governance.md`、`docs/refactor/backend-module-refactor-compatibility.md`
|
||||
> 实施进度:阶段 0~6 的宿主架构能力已完成收口;API/Application 公共复杂度基线已清零,启动组合根的 SystemConfigOper 构造点已由 14 降至 1;API 进程内后台任务已完成首批统一登记,插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线;阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义。
|
||||
> 实施进度:阶段 0~6 的宿主架构能力已完成收口;API/Application 公共复杂度基线已清零,启动组合根的 SystemConfigOper 构造点已由 14 降至 1;API 进程内后台任务已完成首批统一登记,插件仓适配、Outbox 外围扩展和 Model 查询兼容面仍按风险切片推进。2026-08-23 的长期整改阶段 0 已恢复宿主、启动性能、官方插件和 SDK 契约门禁的可信基线;阶段 1a 已补齐 TaskRegistry owner 零债务门禁和诚实的关停超时语义;阶段 1b1 已收口整理 worker、pending 回放、失败通知、进程内 AI 重试、插件监控与事件投递的生命周期所有权。
|
||||
|
||||
## 当前复核结论(2026-08-23)
|
||||
|
||||
@@ -15,8 +15,8 @@
|
||||
|
||||
### 长期整改阶段 0:治理门禁恢复(2026-08-23)
|
||||
|
||||
- 宿主依赖基线已审查 TaskRegistry 接入后的语义差异:当前为 `800` 个模块、`6479` 条内部导入边,12 组重点禁止边继续全部为 `0`,唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
|
||||
- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistry;normal/safe 组件数分别为 `16`/`8`,CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序,不再把 Python/平台模块数量当作硬合同。
|
||||
- 宿主依赖基线已审查 TaskRegistry 与有界后台 owner 接入后的语义差异:当前为 `800` 个模块、`6482` 条内部导入边,12 组重点禁止边继续全部为 `0`,唯一非平凡 SCC 仍是隔离的 TMDB 移植包。
|
||||
- 启动性能探针会在隔离生命周期中真实创建并释放 TaskRegistry;normal/safe 组件数分别为 `20`/`10`,CI 只读检查使用稳定的宿主模块集合和生命周期组件顺序,不再把 Python/平台模块数量当作硬合同。
|
||||
- 官方插件快照覆盖 `plugins.v3`、`plugins.v2` 以及 V3 实际会从 `package.json` 回退加载的 31 个默认实现;`app/plugins/**` 仍只是宿主运行副本,不进入扫描。
|
||||
- SDK 快照以各模块显式 `__all__` 为公开合同,能够记录赋值别名;`typing`、`__future__` 等实现期导入不再被误冻结,既有数据库备份门面已补精确导出清单。
|
||||
- async 阻塞实际债务已由 fixture 中的 10 项下降到 1 项并固化低水位;剩余项是 Scheduler Agent task 查询,后续阶段迁入异步查询边界后归零。
|
||||
@@ -36,6 +36,36 @@
|
||||
- 本子阶段仍只覆盖 TaskRegistry。Transfer worker/replay、Agent blocking executor、Event handler
|
||||
drain、通道线程和 E2/E3 durable 完成点继续作为阶段 1 后续切片,不能因 owner 门禁通过而宣称完成。
|
||||
|
||||
### 长期整改阶段 1b1:整理后台生命周期所有权(2026-08-23)
|
||||
|
||||
- `TransferChain` 为每一代整理 worker 使用独立停止信号;配置热更新只让旧代完成已经进入同步 I/O 的
|
||||
工作,不再让旧线程重新领取新任务。超时旧线程继续由 `_retiring_threads` 持有,重复关闭可以继续等待,
|
||||
不把无法强制取消的文件操作伪装成已结束。生命周期锁等待与 worker/replay join 共用同一个 deadline,
|
||||
停止哨兵也不再被误算成真实队列任务而阻止最后一批进度结算。
|
||||
- pending 回放改为单一受管线程,启动重复调用不会并发扫描;关停信号会在查询、`stat` 和逐条回放边界
|
||||
重新检查,尚未处理的 `TransferPending` 登记保持不变。关闭与 `queue.get()` 竞争时,尚未开始的任务会
|
||||
原样放回队列并保持 `task_done()` 计数平衡。
|
||||
- 失败通知聚合器和 AI 重试 scheduler 现在拥有 timer、buffer 与 flush task 的显式 `close()`;分组使用
|
||||
generation 阻止旧 timer 在新静默窗口尚未 armed 时提前消费新批次。跨线程提交 AI 重试产生的 Future
|
||||
也会观察最终异常,不再只用提交调用外层的 `try/except` 假设异步执行成功。
|
||||
- 生命周期新增常驻“整理后台服务” owner:正常模式和安全模式都只关闭已存在的单例,不会在 shutdown
|
||||
反向创建 worker。插件文件监控、Scheduler、Agent 和整理 owner 先停止宿主生产任务;随后封口插件变更、
|
||||
停用插件事件入口并结算全部在途 handler,最后才调用插件私有 timer、scheduler、watcher 的旧停机 hook。
|
||||
任一阶段返回 `False` 或超时,`FAIL_FAST` 屏障都会停止释放仍可能被活线程使用的后续依赖。
|
||||
- EventManager 同时持有线程池同步 handler Future 和事件循环异步 handler completion;“事件投递屏障”会
|
||||
等待队列、handler 及 handler 派生事件自然收敛,再原子封住新的广播提交。为兼容无法声明资源依赖的
|
||||
旧插件,宿主先停用其新 handler 投递,再用非封口屏障等待已经开始的 handler 退出,之后按原顺序调用
|
||||
`close`/`stop_service`;hook 产生的尾事件仍可由其他宿主 handler 消费,最终屏障后才卸载插件实例。
|
||||
事件消费、共享线程池和模块资源仍在插件之后关闭。
|
||||
- lifespan 启动中途失败会记录已完成及当前部分启动的组件,递归纳入已激活依赖对应的 stop-only owner,
|
||||
并复用正常停机的顺序、超时和 `FAIL_FAST` 策略。后段初始化失败不再只停止 TaskRegistry 后遗留 Scheduler、
|
||||
Transfer、Event、插件或 HTTP 资源。
|
||||
- 兼容边界没有变化:`do_transfer`、队列与手工整理公开签名、模块方法 kwargs、插件整理事件类型及 payload、
|
||||
同步插件 ABI 和动态 API 原生返回结构均未改名或包裹;本阶段也没有 schema/Alembic 变更。
|
||||
- 本阶段只证明进程内 owner、取消、等待和依赖释放顺序。失败通知仍可能在业务提交后、消息接受前随进程
|
||||
崩溃而丢失;五分钟 AI 重试缓冲仍未持久化;`TransferPending` 仍只有 `storage + src_path`,不能表达文件
|
||||
副作用 checkpoint、lease 和未知完成状态。它们分别留给阶段 1b2、1b3、1b4,不能据此宣称 E2/E3 完成。
|
||||
|
||||
### 总体判断
|
||||
|
||||
当前架构总体合理,已经从跨层混合的遗留单体收敛为**边界清晰的模块化单体**:
|
||||
|
||||
+6
-2
@@ -13,8 +13,8 @@
|
||||
"runtime_to_db": [],
|
||||
"workflow_to_db": []
|
||||
},
|
||||
"edge_count": 6479,
|
||||
"edge_sha256": "a65f8d4024e2299b37510359c7ffea91219f0aa0ed9673b74a3eb22d51d6c67f",
|
||||
"edge_count": 6483,
|
||||
"edge_sha256": "0a2f82ca189468d5e954b06a2c188c4ea5380a76c7c64039c7f6eb843cb60523",
|
||||
"edges": [
|
||||
"app -> app.runtime",
|
||||
"app -> app.runtime.compat",
|
||||
@@ -5671,6 +5671,7 @@
|
||||
"app.runtime.extensions.plugin_manager -> app.runtime.observability",
|
||||
"app.runtime.extensions.plugin_manager -> app.runtime.reload",
|
||||
"app.runtime.extensions.plugin_manager -> app.runtime.settings",
|
||||
"app.runtime.extensions.plugin_manager -> app.runtime.thread",
|
||||
"app.runtime.extensions.plugin_manager -> app.schemas",
|
||||
"app.runtime.extensions.plugin_manager -> app.schemas.plugin",
|
||||
"app.runtime.extensions.plugin_manager -> app.schemas.types",
|
||||
@@ -6086,6 +6087,7 @@
|
||||
"app.startup.lifecycle -> app.runtime.tasks",
|
||||
"app.startup.lifecycle -> app.runtime.topology",
|
||||
"app.startup.lifecycle -> app.startup",
|
||||
"app.startup.lifecycle -> app.startup.agent_initializer",
|
||||
"app.startup.lifecycle -> app.startup.cache_initializer",
|
||||
"app.startup.lifecycle -> app.startup.command_initializer",
|
||||
"app.startup.lifecycle -> app.startup.database_initializer",
|
||||
@@ -6218,6 +6220,8 @@
|
||||
"app.startup.modules_initializer -> app.startup.transaction",
|
||||
"app.startup.modules_initializer -> app.startup.workflow",
|
||||
"app.startup.monitor_initializer -> app.monitor",
|
||||
"app.startup.monitor_initializer -> app.runtime",
|
||||
"app.startup.monitor_initializer -> app.runtime.execution",
|
||||
"app.startup.outbox -> app.application",
|
||||
"app.startup.outbox -> app.application.outbox",
|
||||
"app.startup.outbox -> app.db",
|
||||
|
||||
+140
-132
@@ -1,41 +1,41 @@
|
||||
{
|
||||
"schema_version": 2,
|
||||
"generated_at": "2026-08-23T08:11:40.406030+00:00",
|
||||
"generated_at": "2026-08-23T11:06:42.858188+00:00",
|
||||
"platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O",
|
||||
"python": "3.14.3",
|
||||
"repeat": 3,
|
||||
"targets": {
|
||||
"app.startup.lifecycle": {
|
||||
"loaded_app_module_count": 362,
|
||||
"max_ms": 944.347,
|
||||
"median_ms": 935.035,
|
||||
"min_ms": 933.625,
|
||||
"max_ms": 1022.231,
|
||||
"median_ms": 976.39,
|
||||
"min_ms": 967.395,
|
||||
"samples_ms": [
|
||||
944.347,
|
||||
935.035,
|
||||
933.625
|
||||
1022.231,
|
||||
967.395,
|
||||
976.39
|
||||
]
|
||||
},
|
||||
"app.factory": {
|
||||
"loaded_app_module_count": 374,
|
||||
"max_ms": 950.655,
|
||||
"median_ms": 950.429,
|
||||
"min_ms": 947.964,
|
||||
"max_ms": 989.21,
|
||||
"median_ms": 980.605,
|
||||
"min_ms": 951.564,
|
||||
"samples_ms": [
|
||||
950.655,
|
||||
947.964,
|
||||
950.429
|
||||
980.605,
|
||||
989.21,
|
||||
951.564
|
||||
]
|
||||
},
|
||||
"app.main": {
|
||||
"loaded_app_module_count": 376,
|
||||
"max_ms": 1150.269,
|
||||
"median_ms": 1103.976,
|
||||
"min_ms": 1097.978,
|
||||
"max_ms": 1114.04,
|
||||
"median_ms": 1088.251,
|
||||
"min_ms": 1080.26,
|
||||
"samples_ms": [
|
||||
1103.976,
|
||||
1097.978,
|
||||
1150.269
|
||||
1080.26,
|
||||
1088.251,
|
||||
1114.04
|
||||
]
|
||||
}
|
||||
},
|
||||
@@ -46,57 +46,26 @@
|
||||
"samples": [
|
||||
{
|
||||
"mode": "normal",
|
||||
"enabled_component_count": 16,
|
||||
"startup_ms": 0.721,
|
||||
"full_lifespan_ms": 0.907,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.084,
|
||||
"数据库准备": 0.038,
|
||||
"HTTP 基础能力": 0.028,
|
||||
"领域依赖装配": 0.027,
|
||||
"数据库引擎预热": 0.024,
|
||||
"数据库连接预算": 0.024,
|
||||
"路由": 0.023,
|
||||
"模块服务": 0.024,
|
||||
"插件备份恢复": 0.023,
|
||||
"插件": 0.029,
|
||||
"定时器": 0.023,
|
||||
"监控器": 0.024,
|
||||
"待处理整理回放": 0.021,
|
||||
"命令服务": 0.025,
|
||||
"工作流": 0.02,
|
||||
"插件同步与启动收尾": 0.031
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
"threads_after": 2,
|
||||
"tasks_before": 1,
|
||||
"tasks_started": 2,
|
||||
"tasks_after": 1,
|
||||
"database_connections_started": 0
|
||||
},
|
||||
{
|
||||
"mode": "normal",
|
||||
"enabled_component_count": 16,
|
||||
"startup_ms": 0.614,
|
||||
"full_lifespan_ms": 0.788,
|
||||
"enabled_component_count": 21,
|
||||
"startup_ms": 0.619,
|
||||
"full_lifespan_ms": 0.806,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.075,
|
||||
"数据库准备": 0.037,
|
||||
"HTTP 基础能力": 0.029,
|
||||
"HTTP 基础能力": 0.031,
|
||||
"领域依赖装配": 0.027,
|
||||
"数据库引擎预热": 0.023,
|
||||
"数据库连接预算": 0.024,
|
||||
"路由": 0.023,
|
||||
"模块服务": 0.023,
|
||||
"插件备份恢复": 0.023,
|
||||
"插件": 0.024,
|
||||
"定时器": 0.025,
|
||||
"监控器": 0.023,
|
||||
"待处理整理回放": 0.022,
|
||||
"命令服务": 0.021,
|
||||
"数据库引擎预热": 0.024,
|
||||
"数据库连接预算": 0.025,
|
||||
"路由": 0.022,
|
||||
"模块服务": 0.024,
|
||||
"插件备份恢复": 0.021,
|
||||
"插件": 0.02,
|
||||
"定时器": 0.023,
|
||||
"监控器": 0.021,
|
||||
"待处理整理回放": 0.019,
|
||||
"命令服务": 0.022,
|
||||
"工作流": 0.022,
|
||||
"插件同步与启动收尾": 0.03
|
||||
"插件同步与启动收尾": 0.034
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
@@ -108,26 +77,57 @@
|
||||
},
|
||||
{
|
||||
"mode": "normal",
|
||||
"enabled_component_count": 16,
|
||||
"startup_ms": 0.611,
|
||||
"full_lifespan_ms": 0.785,
|
||||
"enabled_component_count": 21,
|
||||
"startup_ms": 0.628,
|
||||
"full_lifespan_ms": 0.808,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.072,
|
||||
"数据库准备": 0.033,
|
||||
"HTTP 基础能力": 0.026,
|
||||
"后台任务登记器": 0.073,
|
||||
"数据库准备": 0.037,
|
||||
"HTTP 基础能力": 0.039,
|
||||
"领域依赖装配": 0.027,
|
||||
"数据库引擎预热": 0.025,
|
||||
"数据库连接预算": 0.025,
|
||||
"路由": 0.022,
|
||||
"模块服务": 0.022,
|
||||
"插件备份恢复": 0.021,
|
||||
"插件": 0.02,
|
||||
"定时器": 0.024,
|
||||
"监控器": 0.022,
|
||||
"待处理整理回放": 0.029,
|
||||
"命令服务": 0.02,
|
||||
"工作流": 0.021,
|
||||
"插件同步与启动收尾": 0.032
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
"threads_after": 2,
|
||||
"tasks_before": 1,
|
||||
"tasks_started": 2,
|
||||
"tasks_after": 1,
|
||||
"database_connections_started": 0
|
||||
},
|
||||
{
|
||||
"mode": "normal",
|
||||
"enabled_component_count": 21,
|
||||
"startup_ms": 0.664,
|
||||
"full_lifespan_ms": 0.863,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.083,
|
||||
"数据库准备": 0.047,
|
||||
"HTTP 基础能力": 0.036,
|
||||
"领域依赖装配": 0.028,
|
||||
"数据库引擎预热": 0.023,
|
||||
"数据库连接预算": 0.022,
|
||||
"路由": 0.024,
|
||||
"模块服务": 0.02,
|
||||
"插件备份恢复": 0.023,
|
||||
"插件": 0.024,
|
||||
"数据库引擎预热": 0.029,
|
||||
"数据库连接预算": 0.025,
|
||||
"路由": 0.026,
|
||||
"模块服务": 0.025,
|
||||
"插件备份恢复": 0.025,
|
||||
"插件": 0.028,
|
||||
"定时器": 0.026,
|
||||
"监控器": 0.024,
|
||||
"待处理整理回放": 0.023,
|
||||
"监控器": 0.021,
|
||||
"待处理整理回放": 0.021,
|
||||
"命令服务": 0.024,
|
||||
"工作流": 0.022,
|
||||
"插件同步与启动收尾": 0.031
|
||||
"工作流": 0.021,
|
||||
"插件同步与启动收尾": 0.036
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
@@ -138,9 +138,9 @@
|
||||
"database_connections_started": 0
|
||||
}
|
||||
],
|
||||
"median_startup_ms": 0.614,
|
||||
"median_full_lifespan_ms": 0.788,
|
||||
"enabled_component_count": 16,
|
||||
"median_startup_ms": 0.628,
|
||||
"median_full_lifespan_ms": 0.808,
|
||||
"enabled_component_count": 21,
|
||||
"enabled_components": [
|
||||
"后台任务登记器",
|
||||
"数据库准备",
|
||||
@@ -152,8 +152,13 @@
|
||||
"模块服务",
|
||||
"插件备份恢复",
|
||||
"插件",
|
||||
"插件变更监控",
|
||||
"定时器",
|
||||
"监控器",
|
||||
"整理后台服务",
|
||||
"AI智能体会话",
|
||||
"插件后台服务",
|
||||
"事件投递屏障",
|
||||
"待处理整理回放",
|
||||
"命令服务",
|
||||
"工作流",
|
||||
@@ -164,19 +169,19 @@
|
||||
"samples": [
|
||||
{
|
||||
"mode": "safe",
|
||||
"enabled_component_count": 8,
|
||||
"startup_ms": 0.463,
|
||||
"full_lifespan_ms": 0.667,
|
||||
"enabled_component_count": 11,
|
||||
"startup_ms": 0.464,
|
||||
"full_lifespan_ms": 0.637,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.075,
|
||||
"数据库准备": 0.035,
|
||||
"后台任务登记器": 0.074,
|
||||
"数据库准备": 0.038,
|
||||
"HTTP 基础能力": 0.03,
|
||||
"领域依赖装配": 0.032,
|
||||
"数据库引擎预热": 0.024,
|
||||
"数据库连接预算": 0.022,
|
||||
"路由": 0.025,
|
||||
"模块服务": 0.022,
|
||||
"插件同步与启动收尾": 0.058
|
||||
"领域依赖装配": 0.029,
|
||||
"数据库引擎预热": 0.026,
|
||||
"数据库连接预算": 0.025,
|
||||
"路由": 0.024,
|
||||
"模块服务": 0.024,
|
||||
"插件同步与启动收尾": 0.033
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
@@ -188,43 +193,43 @@
|
||||
},
|
||||
{
|
||||
"mode": "safe",
|
||||
"enabled_component_count": 8,
|
||||
"startup_ms": 0.474,
|
||||
"full_lifespan_ms": 0.68,
|
||||
"enabled_component_count": 11,
|
||||
"startup_ms": 0.451,
|
||||
"full_lifespan_ms": 0.624,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.076,
|
||||
"数据库准备": 0.043,
|
||||
"HTTP 基础能力": 0.029,
|
||||
"领域依赖装配": 0.028,
|
||||
"后台任务登记器": 0.073,
|
||||
"数据库准备": 0.035,
|
||||
"HTTP 基础能力": 0.028,
|
||||
"领域依赖装配": 0.027,
|
||||
"数据库引擎预热": 0.024,
|
||||
"数据库连接预算": 0.023,
|
||||
"路由": 0.023,
|
||||
"模块服务": 0.02,
|
||||
"插件同步与启动收尾": 0.031
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
"threads_after": 2,
|
||||
"tasks_before": 1,
|
||||
"tasks_started": 2,
|
||||
"tasks_after": 1,
|
||||
"database_connections_started": 0
|
||||
},
|
||||
{
|
||||
"mode": "safe",
|
||||
"enabled_component_count": 11,
|
||||
"startup_ms": 0.477,
|
||||
"full_lifespan_ms": 0.661,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.081,
|
||||
"数据库准备": 0.05,
|
||||
"HTTP 基础能力": 0.035,
|
||||
"领域依赖装配": 0.03,
|
||||
"数据库引擎预热": 0.026,
|
||||
"数据库连接预算": 0.024,
|
||||
"路由": 0.024,
|
||||
"模块服务": 0.024,
|
||||
"插件同步与启动收尾": 0.057
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
"threads_after": 2,
|
||||
"tasks_before": 1,
|
||||
"tasks_started": 2,
|
||||
"tasks_after": 1,
|
||||
"database_connections_started": 0
|
||||
},
|
||||
{
|
||||
"mode": "safe",
|
||||
"enabled_component_count": 8,
|
||||
"startup_ms": 0.454,
|
||||
"full_lifespan_ms": 0.657,
|
||||
"stage_ms": {
|
||||
"后台任务登记器": 0.075,
|
||||
"数据库准备": 0.036,
|
||||
"HTTP 基础能力": 0.028,
|
||||
"领域依赖装配": 0.028,
|
||||
"数据库引擎预热": 0.025,
|
||||
"数据库连接预算": 0.024,
|
||||
"路由": 0.024,
|
||||
"模块服务": 0.024,
|
||||
"插件同步与启动收尾": 0.06
|
||||
"模块服务": 0.02,
|
||||
"插件同步与启动收尾": 0.026
|
||||
},
|
||||
"threads_before": 2,
|
||||
"threads_started": 2,
|
||||
@@ -235,9 +240,9 @@
|
||||
"database_connections_started": 0
|
||||
}
|
||||
],
|
||||
"median_startup_ms": 0.463,
|
||||
"median_full_lifespan_ms": 0.667,
|
||||
"enabled_component_count": 8,
|
||||
"median_startup_ms": 0.464,
|
||||
"median_full_lifespan_ms": 0.637,
|
||||
"enabled_component_count": 11,
|
||||
"enabled_components": [
|
||||
"后台任务登记器",
|
||||
"数据库准备",
|
||||
@@ -246,7 +251,10 @@
|
||||
"数据库引擎预热",
|
||||
"数据库连接预算",
|
||||
"路由",
|
||||
"模块服务"
|
||||
"模块服务",
|
||||
"整理后台服务",
|
||||
"AI智能体会话",
|
||||
"事件投递屏障"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,7 +115,7 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager(
|
||||
initializer._manager = manager
|
||||
initializer._initialized = True
|
||||
initializer._compat_injected = False
|
||||
shutdown = AsyncMock()
|
||||
shutdown = AsyncMock(return_value=True)
|
||||
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown)
|
||||
monkeypatch.setattr(agent_initializer, "agent_initializer", initializer)
|
||||
monkeypatch.setattr(
|
||||
@@ -124,13 +124,63 @@ async def test_production_stop_seals_runtime_without_manually_closing_manager(
|
||||
lambda: False,
|
||||
)
|
||||
|
||||
await agent_initializer.stop_agent()
|
||||
assert await agent_initializer.stop_agent() is True
|
||||
|
||||
shutdown.assert_awaited_once_with()
|
||||
manager.close.assert_not_awaited()
|
||||
assert initializer._manager is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_production_stop_retains_manager_when_runtime_does_not_converge(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Agent service 未收敛时不得释放 initializer 引用或下游工具资源。"""
|
||||
manager = AsyncMock()
|
||||
initializer = agent_initializer.AgentInitializer()
|
||||
initializer._manager = manager
|
||||
initializer._initialized = True
|
||||
initializer._compat_injected = False
|
||||
shutdown = AsyncMock(return_value=False)
|
||||
executor_seal = MagicMock()
|
||||
executor_close = AsyncMock(return_value=True)
|
||||
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown)
|
||||
monkeypatch.setattr(agent_initializer, "agent_initializer", initializer)
|
||||
monkeypatch.setattr(
|
||||
agent_initializer,
|
||||
"is_tool_factory_materialized",
|
||||
lambda: True,
|
||||
)
|
||||
fake_base = types.ModuleType("app.agent.tools.base")
|
||||
fake_base.begin_blocking_executor_shutdown = executor_seal
|
||||
fake_base.close_blocking_executors = executor_close
|
||||
monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base)
|
||||
|
||||
assert await agent_initializer.stop_agent() is False
|
||||
|
||||
shutdown.assert_awaited_once_with()
|
||||
assert initializer._manager is manager
|
||||
assert initializer._shutdown_started is True
|
||||
assert initializer._shutdown_complete is False
|
||||
executor_seal.assert_called_once_with(cancel_futures=True)
|
||||
executor_close.assert_awaited_once_with(
|
||||
timeout_seconds=(
|
||||
agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS
|
||||
),
|
||||
cancel_futures=True,
|
||||
)
|
||||
|
||||
event = agent_initializer.Event(
|
||||
agent_initializer.EventType.ConfigChanged,
|
||||
{"key": "AI_AGENT_ENABLE"},
|
||||
)
|
||||
reconcile = AsyncMock()
|
||||
monkeypatch.setattr(agent_initializer, "reconcile_agent_service", reconcile)
|
||||
await initializer.handle_config_changed(event)
|
||||
reconcile.assert_not_awaited()
|
||||
assert initializer._manager is manager
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_config_listener_delegates_watch_filter_to_runtime(monkeypatch) -> None:
|
||||
"""配置监听器只转交 changed keys,不维护第二份启用开关。"""
|
||||
@@ -227,8 +277,10 @@ async def test_stop_closes_tool_executor_after_factory_materialization(
|
||||
) -> None:
|
||||
"""工具能力已解析时,应取消仍排队的阻塞工具任务。"""
|
||||
fake_base = types.ModuleType("app.agent.tools.base")
|
||||
cleanup = MagicMock()
|
||||
fake_base.shutdown_blocking_executors = cleanup
|
||||
seal = MagicMock()
|
||||
cleanup = AsyncMock(return_value=True)
|
||||
fake_base.begin_blocking_executor_shutdown = seal
|
||||
fake_base.close_blocking_executors = cleanup
|
||||
monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base)
|
||||
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock())
|
||||
monkeypatch.setattr(
|
||||
@@ -242,15 +294,27 @@ async def test_stop_closes_tool_executor_after_factory_materialization(
|
||||
agent_initializer.AgentInitializer(),
|
||||
)
|
||||
|
||||
await agent_initializer.stop_agent()
|
||||
assert await agent_initializer.stop_agent() is True
|
||||
|
||||
cleanup.assert_called_once_with(wait=False, cancel_futures=True)
|
||||
seal.assert_called_once_with(cancel_futures=True)
|
||||
cleanup.assert_awaited_once_with(
|
||||
timeout_seconds=(
|
||||
agent_initializer.AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS
|
||||
),
|
||||
cancel_futures=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None:
|
||||
"""应用关闭不得等待已经进入线程池且尚未返回的工具调用。"""
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
async def test_stop_retains_running_blocking_tool_until_retry(monkeypatch) -> None:
|
||||
"""阻塞工具超过预算时 stop 返回 False,真实结束后重试才释放 owner。"""
|
||||
from app.agent.tools.base import (
|
||||
MoviePilotTool,
|
||||
_blocking_futures,
|
||||
_blocking_retiring_executors,
|
||||
close_blocking_executors,
|
||||
reopen_blocking_executors,
|
||||
)
|
||||
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
@@ -264,7 +328,21 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None
|
||||
MoviePilotTool.run_blocking("web", _blocking_call)
|
||||
)
|
||||
assert await asyncio.wait_for(asyncio.to_thread(started.wait), timeout=1)
|
||||
monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock())
|
||||
monkeypatch.setattr(
|
||||
agent_initializer,
|
||||
"begin_agent_shutdown",
|
||||
AsyncMock(return_value=True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent_initializer,
|
||||
"close_materialized_terminal_sessions",
|
||||
AsyncMock(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent_initializer,
|
||||
"AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS",
|
||||
0.01,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
agent_initializer,
|
||||
"is_tool_factory_materialized",
|
||||
@@ -277,8 +355,22 @@ async def test_stop_does_not_wait_for_running_blocking_tool(monkeypatch) -> None
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2)
|
||||
assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is False
|
||||
assert worker.done() is False
|
||||
finally:
|
||||
assert _blocking_futures
|
||||
assert _blocking_retiring_executors
|
||||
|
||||
with pytest.raises(RuntimeError, match="正在关闭"):
|
||||
await MoviePilotTool.run_blocking("web", lambda: "late")
|
||||
|
||||
release.set()
|
||||
assert await asyncio.wait_for(worker, timeout=1) == "done"
|
||||
assert await asyncio.wait_for(agent_initializer.stop_agent(), timeout=0.2) is True
|
||||
assert not _blocking_futures
|
||||
assert not _blocking_retiring_executors
|
||||
finally:
|
||||
release.set()
|
||||
if not worker.done():
|
||||
await asyncio.wait_for(worker, timeout=1)
|
||||
await close_blocking_executors(timeout_seconds=1, cancel_futures=True)
|
||||
assert reopen_blocking_executors() is True
|
||||
|
||||
@@ -115,8 +115,8 @@ async def test_agent_manager_background_tasks_share_owner_loop(monkeypatch) -> N
|
||||
assert manager._idle_cleanup_task is idle_cleanup_task
|
||||
assert memory_manager.cleanup_task is memory_cleanup_task
|
||||
|
||||
await manager.close()
|
||||
await manager.close()
|
||||
assert await manager.close() is True
|
||||
assert await manager.close() is True
|
||||
assert manager._idle_cleanup_task is None
|
||||
assert memory_manager.cleanup_task is None
|
||||
assert idle_cleanup_task.done()
|
||||
@@ -489,7 +489,8 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout(
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), timeout=1)
|
||||
|
||||
await manager.close()
|
||||
assert await manager.close() is False
|
||||
assert await manager.close() is False
|
||||
assert not cleanup_called.is_set()
|
||||
assert "close-timeout" in manager.active_agents
|
||||
assert manager._close_finalizer_task is not None
|
||||
@@ -503,6 +504,47 @@ async def test_close_defers_shared_agent_teardown_after_worker_timeout(
|
||||
break
|
||||
await asyncio.sleep(0)
|
||||
assert manager.active_agents == {}
|
||||
assert await manager.close() is True
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_close_retains_agent_until_detached_subagent_converges(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Agent cleanup 返回 False 时 manager 必须保留 agent 和共享记忆 owner。"""
|
||||
manager = AgentManager()
|
||||
memory_manager = MemoryManager()
|
||||
monkeypatch.setattr(agent_module, "memory_manager", memory_manager)
|
||||
|
||||
class PendingSubagentOwner:
|
||||
"""首次清理未收敛、第二次清理成功的最小 Agent 替身。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化封口计数与可重试清理结果。"""
|
||||
self.seal_count = 0
|
||||
self.cleanup_results = iter((False, True))
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""记录 manager 在首个 await 前封住了子代理提交。"""
|
||||
self.seal_count += 1
|
||||
|
||||
async def cleanup(self) -> bool:
|
||||
"""按测试序列返回 detached owner 的收敛状态。"""
|
||||
return next(self.cleanup_results)
|
||||
|
||||
owner = PendingSubagentOwner()
|
||||
await manager.initialize()
|
||||
manager.active_agents["detached-owner"] = owner
|
||||
|
||||
assert await manager.close() is False
|
||||
assert manager.active_agents == {"detached-owner": owner}
|
||||
assert owner.seal_count == 1
|
||||
assert memory_manager.cleanup_task is not None
|
||||
|
||||
assert await manager.close() is True
|
||||
assert manager.active_agents == {}
|
||||
assert owner.seal_count == 2
|
||||
assert memory_manager.cleanup_task is None
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
|
||||
@@ -4,7 +4,12 @@ from types import SimpleNamespace
|
||||
from typing import Optional
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
from app.agent.tools.impl._plugin_tool_utils import install_plugin_runtime
|
||||
import pytest
|
||||
|
||||
from app.agent.tools.impl._plugin_tool_utils import (
|
||||
install_plugin_runtime,
|
||||
uninstall_plugin_runtime,
|
||||
)
|
||||
from app.agent.tools.impl.install_plugin import InstallPluginTool
|
||||
from app.agent.tools.impl.query_installed_plugins import QueryInstalledPluginsTool
|
||||
from app.agent.tools.impl.query_market_plugins import QueryMarketPluginsTool
|
||||
@@ -14,6 +19,10 @@ from app.agent.tools.impl.reload_plugin import ReloadPluginTool
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.agent.tools.impl.uninstall_plugin import UninstallPluginTool
|
||||
from app.agent.tools.impl.update_plugin_config import UpdatePluginConfigTool
|
||||
from app.runtime.extensions.plugin.admission import (
|
||||
PluginMutationAdmission,
|
||||
PluginMutationRejectedError,
|
||||
)
|
||||
|
||||
|
||||
def _plugin_snapshot(state: bool = True) -> dict:
|
||||
@@ -334,8 +343,7 @@ def test_install_plugin_runtime_reloads_in_threadpool() -> None:
|
||||
)
|
||||
assert len(calls) == 2
|
||||
assert calls[0][0] == "plugin"
|
||||
assert calls[0][1] == plugin_manager.reload_plugin_tree
|
||||
assert calls[0][2] == ("DemoPlugin",)
|
||||
assert calls[0][2] == (plugin_manager.reload_plugin_tree, "DemoPlugin")
|
||||
assert calls[0][3] == {}
|
||||
assert calls[1][0] == "plugin"
|
||||
assert calls[1][1] == refresh_registrations
|
||||
@@ -372,6 +380,32 @@ def test_uninstall_plugin_uninstalls_installed_candidate() -> None:
|
||||
uninstall_runtime.assert_awaited_once_with("DemoPlugin")
|
||||
|
||||
|
||||
def test_sealed_agent_uninstall_rejects_before_persistence() -> None:
|
||||
"""Agent 卸载未获 admission 时不读取实例或修改安装清单。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
plugin_manager = MagicMock()
|
||||
plugin_manager.mutation.side_effect = admission.hold
|
||||
config_oper = MagicMock()
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.agent.tools.impl._plugin_tool_utils.get_plugin_manager",
|
||||
return_value=plugin_manager,
|
||||
),
|
||||
patch(
|
||||
"app.agent.tools.impl._plugin_tool_utils.SystemConfigOper",
|
||||
return_value=config_oper,
|
||||
) as config_provider,
|
||||
pytest.raises(PluginMutationRejectedError),
|
||||
):
|
||||
asyncio.run(uninstall_plugin_runtime("DemoPlugin"))
|
||||
|
||||
plugin_manager.get_plugin_instance.assert_not_called()
|
||||
config_provider.assert_not_called()
|
||||
config_oper.async_set.assert_not_called()
|
||||
|
||||
|
||||
def test_query_plugin_data_truncates_large_payload() -> None:
|
||||
"""
|
||||
查询插件数据会截断超长内容并返回预览。
|
||||
|
||||
@@ -42,6 +42,7 @@ class _FakeManager:
|
||||
self.fail_initialize = False
|
||||
self.initialize_entered: asyncio.Event | None = None
|
||||
self.initialize_release: asyncio.Event | None = None
|
||||
self.close_converged: bool | None = None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
self.initialize_calls += 1
|
||||
@@ -52,8 +53,42 @@ class _FakeManager:
|
||||
if self.fail_initialize:
|
||||
raise RuntimeError("service initialization failed")
|
||||
|
||||
async def close(self) -> None:
|
||||
async def close(self) -> bool | None:
|
||||
self.close_calls += 1
|
||||
return self.close_converged
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_shutdown_retains_nonconverged_agent_service_for_retry(
|
||||
runtime_loader,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Manager 返回未收敛时 Runtime 必须保留 owner,后续关闭可继续等待。"""
|
||||
manager = _FakeManager()
|
||||
manager.close_converged = False
|
||||
modules = _fake_agent_modules(manager)
|
||||
monkeypatch.setattr(
|
||||
"app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE",
|
||||
True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.agent.capabilities.adapter.importlib.import_module",
|
||||
lambda name: (
|
||||
monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name]
|
||||
),
|
||||
)
|
||||
assert await runtime_loader.activate_agent_service() is manager
|
||||
|
||||
assert await runtime_loader.begin_agent_shutdown() is False
|
||||
snapshot = runtime_loader._agent_runtime.snapshot("agent.service")
|
||||
assert snapshot.lifecycle is CapabilityLifecycleState.FAILED
|
||||
assert snapshot.visible is False
|
||||
|
||||
manager.close_converged = True
|
||||
assert await runtime_loader.begin_agent_shutdown() is True
|
||||
snapshot = runtime_loader._agent_runtime.snapshot("agent.service")
|
||||
assert snapshot.lifecycle is CapabilityLifecycleState.STOPPED
|
||||
assert manager.close_calls == 2
|
||||
|
||||
|
||||
def _fake_agent_modules(manager: object | None = None) -> dict[str, types.ModuleType]:
|
||||
|
||||
@@ -284,7 +284,7 @@ async def test_agent_cleanup_closes_subagent_middlewares() -> None:
|
||||
|
||||
agent._subagent_middlewares = (_Middleware(),)
|
||||
|
||||
await agent.cleanup()
|
||||
assert await agent.cleanup() is True
|
||||
|
||||
assert closed == [True]
|
||||
assert agent._subagent_middlewares == ()
|
||||
@@ -345,16 +345,16 @@ def test_subagent_control_middleware_close_is_idempotent() -> None:
|
||||
middleware._tasks = {}
|
||||
|
||||
async def _close() -> None:
|
||||
await middleware.close()
|
||||
await middleware.close()
|
||||
assert await middleware.close() is True
|
||||
assert await middleware.close() is True
|
||||
|
||||
asyncio.run(_close())
|
||||
assert middleware._tasks == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_subagent_close_has_bounded_cancel_wait() -> None:
|
||||
"""子代理忽略首次取消时,控制器关闭仍必须在上限内返回。"""
|
||||
async def test_subagent_close_retains_stubborn_owner_until_retry() -> None:
|
||||
"""子代理忽略取消时 close 返回 False,任务结束后重试才清理记录。"""
|
||||
middleware = object.__new__(SubAgentTaskControlMiddleware)
|
||||
release = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
@@ -383,14 +383,57 @@ async def test_subagent_close_has_bounded_cancel_wait() -> None:
|
||||
"app.agent.middleware.subagents.SUBAGENT_CANCEL_GRACE_SECONDS",
|
||||
0.01,
|
||||
):
|
||||
await asyncio.wait_for(middleware.close(), timeout=0.2)
|
||||
assert await asyncio.wait_for(middleware.close(), timeout=0.2) is False
|
||||
|
||||
assert cancelled.is_set()
|
||||
assert task.done() is False
|
||||
assert middleware._tasks == {}
|
||||
assert middleware._tasks == {record.task_id: record}
|
||||
assert task in middleware._close_cancel_requested
|
||||
|
||||
release.set()
|
||||
await asyncio.wait_for(task, timeout=0.2)
|
||||
assert await middleware.close() is True
|
||||
assert middleware._tasks == {}
|
||||
assert middleware._close_cancel_requested == set()
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_subagent_seal_rejects_new_detached_task() -> None:
|
||||
"""控制器封口后必须同步拒绝 start,且不能创建新的 asyncio Task。"""
|
||||
middleware = object.__new__(SubAgentTaskControlMiddleware)
|
||||
middleware._tasks = {}
|
||||
middleware._accepting_tasks = True
|
||||
middleware._close_cancel_requested = set()
|
||||
|
||||
middleware.seal()
|
||||
payload = await middleware._control_task(
|
||||
action="start",
|
||||
description="late task",
|
||||
)
|
||||
|
||||
result = json.loads(payload)
|
||||
assert result["success"] is False
|
||||
assert "正在关闭" in result["error"]
|
||||
assert middleware._tasks == {}
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
async def test_agent_cleanup_retains_nonconverged_subagent_middleware() -> None:
|
||||
"""Agent cleanup 必须保留返回 False 的 middleware,供重复调用继续收口。"""
|
||||
agent = MoviePilotAgent(session_id="session-owner", user_id="user-owner")
|
||||
middleware = SimpleNamespace(
|
||||
seal=MagicMock(),
|
||||
close=AsyncMock(side_effect=[False, True]),
|
||||
)
|
||||
agent._subagent_middlewares = (middleware,)
|
||||
|
||||
assert await agent.cleanup() is False
|
||||
assert agent._subagent_middlewares == (middleware,)
|
||||
|
||||
assert await agent.cleanup() is True
|
||||
assert agent._subagent_middlewares == ()
|
||||
assert middleware.seal.call_count == 2
|
||||
assert middleware.close.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.anyio
|
||||
|
||||
@@ -8,6 +8,10 @@ from app.agent.tools.base import (
|
||||
MoviePilotTool,
|
||||
ToolExecutionTimeoutError,
|
||||
_blocking_executors,
|
||||
_blocking_futures,
|
||||
_blocking_retiring_executors,
|
||||
close_blocking_executors,
|
||||
reopen_blocking_executors,
|
||||
shutdown_blocking_executors,
|
||||
)
|
||||
from app.agent.tools.manager import MoviePilotToolsManager
|
||||
@@ -37,6 +41,15 @@ class BlockingAgentTool(MoviePilotTool):
|
||||
return "unused"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_blocking_executor_runtime():
|
||||
"""每个用例前后恢复阻塞池门禁,避免进程级 owner 状态串扰。"""
|
||||
assert reopen_blocking_executors() is True
|
||||
yield
|
||||
assert shutdown_blocking_executors(cancel_futures=True) is True
|
||||
assert reopen_blocking_executors() is True
|
||||
|
||||
|
||||
def test_arun_raises_timeout_when_tool_exceeds_limit():
|
||||
"""底层工具入口应把超时交给宿主策略记录失败终态。"""
|
||||
tool = SlowAgentTool(session_id="session-1", user_id="10001")
|
||||
@@ -126,6 +139,8 @@ def test_shutdown_blocking_executors_clears_agent_tool_workers():
|
||||
shutdown_blocking_executors()
|
||||
|
||||
assert _blocking_executors == {}
|
||||
assert _blocking_futures == {}
|
||||
assert _blocking_retiring_executors == set()
|
||||
|
||||
|
||||
def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent():
|
||||
@@ -159,10 +174,55 @@ def test_shutdown_blocking_executors_cancels_queued_workers_and_is_idempotent():
|
||||
queued_future = asyncio.run(_run_scenario())
|
||||
|
||||
assert _blocking_executors == {}
|
||||
assert _blocking_futures == {}
|
||||
assert _blocking_retiring_executors == set()
|
||||
assert queued_future.cancelled()
|
||||
assert not queued_ran.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_blocking_executors_retains_owner_until_retry() -> None:
|
||||
"""同步调用超时后保留 Future/executor,完成后的重复 close 才成功。"""
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def _blocking_call() -> str:
|
||||
"""等待测试释放,稳定制造超过关停预算的运行 Future。"""
|
||||
started.set()
|
||||
release.wait()
|
||||
return "done"
|
||||
|
||||
task = asyncio.create_task(
|
||||
MoviePilotTool.run_blocking("web", _blocking_call)
|
||||
)
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
|
||||
try:
|
||||
assert await close_blocking_executors(
|
||||
timeout_seconds=0.01,
|
||||
cancel_futures=True,
|
||||
) is False
|
||||
assert task.done() is False
|
||||
assert _blocking_futures
|
||||
assert _blocking_retiring_executors
|
||||
|
||||
with pytest.raises(RuntimeError, match="正在关闭"):
|
||||
await MoviePilotTool.run_blocking("web", lambda: "late")
|
||||
|
||||
release.set()
|
||||
assert await asyncio.wait_for(task, timeout=1) == "done"
|
||||
assert await close_blocking_executors(
|
||||
timeout_seconds=0.01,
|
||||
cancel_futures=True,
|
||||
) is True
|
||||
assert _blocking_futures == {}
|
||||
assert _blocking_retiring_executors == set()
|
||||
finally:
|
||||
release.set()
|
||||
if not task.done():
|
||||
await asyncio.wait_for(task, timeout=1)
|
||||
|
||||
|
||||
def test_create_agent_config_uses_llm_max_iterations():
|
||||
"""Agent 执行配置应把 LLM_MAX_ITERATIONS 传给 LangGraph recursion_limit。"""
|
||||
from app.agent.orchestrator import MoviePilotAgent
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""事件调度订阅快照和生命周期回归测试。"""
|
||||
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import threading
|
||||
from queue import PriorityQueue
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -16,12 +18,49 @@ class _ImmediateExecutor:
|
||||
|
||||
@staticmethod
|
||||
def submit(func, *args, **kwargs):
|
||||
return func(*args, **kwargs)
|
||||
"""立即执行调用并返回符合线程池接口的已完成 Future。"""
|
||||
handle = concurrent.futures.Future()
|
||||
try:
|
||||
handle.set_result(func(*args, **kwargs))
|
||||
except Exception as err:
|
||||
handle.set_exception(err)
|
||||
return handle
|
||||
|
||||
|
||||
class _DaemonThreadExecutor:
|
||||
"""以守护线程执行回调,让死锁回归测试能够有界失败。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化可观察的回调完成信号。"""
|
||||
self.finished = threading.Event()
|
||||
|
||||
def submit(self, func, *args, **kwargs):
|
||||
"""在线程中执行调用并返回符合线程池接口的 Future。"""
|
||||
handle = concurrent.futures.Future()
|
||||
|
||||
def run() -> None:
|
||||
"""执行回调并把结果或异常写入 Future。"""
|
||||
if not handle.set_running_or_notify_cancel():
|
||||
return
|
||||
try:
|
||||
handle.set_result(func(*args, **kwargs))
|
||||
except BaseException as err: # pragma: no cover - 交由 Future 消费
|
||||
handle.set_exception(err)
|
||||
finally:
|
||||
self.finished.set()
|
||||
|
||||
threading.Thread(target=run, daemon=True).start()
|
||||
return handle
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def isolated_eventmanager(monkeypatch):
|
||||
"""隔离全局事件总线的订阅表和广播执行器。"""
|
||||
monkeypatch.setattr(
|
||||
global_vars,
|
||||
"CURRENT_EVENT_LOOP",
|
||||
global_vars.CURRENT_EVENT_LOOP,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__broadcast_subscribers",
|
||||
@@ -37,6 +76,16 @@ def isolated_eventmanager(monkeypatch):
|
||||
"_EventManager__handler_instance_resolvers",
|
||||
{},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__disabled_handlers",
|
||||
set(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__disabled_classes",
|
||||
set(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__executor",
|
||||
@@ -47,6 +96,11 @@ def isolated_eventmanager(monkeypatch):
|
||||
"_EventManager__event",
|
||||
threading.Event(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__event_queue",
|
||||
PriorityQueue(),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__consumer_threads",
|
||||
@@ -57,6 +111,11 @@ def isolated_eventmanager(monkeypatch):
|
||||
"_EventManager__lifecycle_state",
|
||||
"new",
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__sync_handles",
|
||||
{},
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
eventmanager,
|
||||
"_EventManager__async_handles",
|
||||
@@ -68,6 +127,7 @@ def isolated_eventmanager(monkeypatch):
|
||||
def test_broadcast_dispatch_uses_subscription_snapshot(isolated_eventmanager):
|
||||
"""广播事件中新增或移除的 handler 从下一个事件开始生效。"""
|
||||
calls = []
|
||||
isolated_eventmanager._EventManager__lifecycle_state = "running"
|
||||
|
||||
def late_handler(_event):
|
||||
calls.append("late")
|
||||
@@ -276,6 +336,354 @@ async def test_async_broadcast_shutdown_waits_for_handler_cleanup(
|
||||
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_waits_for_slow_sync_broadcast_handler(
|
||||
isolated_eventmanager,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""同步广播处理器返回前 drain 不得提前宣称结算完成。"""
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
monkeypatch.setattr(
|
||||
isolated_eventmanager,
|
||||
"_EventManager__executor",
|
||||
executor,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def handler(_event) -> None:
|
||||
started.set()
|
||||
release.wait(timeout=2)
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"drain-sync"}},
|
||||
)
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
assert len(isolated_eventmanager._EventManager__sync_handles) == 1
|
||||
|
||||
drain_task = asyncio.create_task(
|
||||
isolated_eventmanager.drain_async(timeout=1)
|
||||
)
|
||||
await asyncio.sleep(0.02)
|
||||
assert not drain_task.done()
|
||||
|
||||
release.set()
|
||||
assert await drain_task is True
|
||||
assert isolated_eventmanager._EventManager__sync_handles == {}
|
||||
assert (
|
||||
isolated_eventmanager._EventManager__event_queue.unfinished_tasks
|
||||
== 0
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
await isolated_eventmanager.stop_async()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_timeout_does_not_cancel_async_broadcast_handler(
|
||||
isolated_eventmanager,
|
||||
) -> None:
|
||||
"""drain 超时只报告未收敛,不取消仍在执行的异步业务处理器。"""
|
||||
global_vars.set_loop(asyncio.get_running_loop())
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
cancelled = False
|
||||
|
||||
async def handler(_event) -> None:
|
||||
nonlocal cancelled
|
||||
started.set()
|
||||
try:
|
||||
await release.wait()
|
||||
except asyncio.CancelledError:
|
||||
cancelled = True
|
||||
raise
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"drain-async"}},
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), timeout=1)
|
||||
|
||||
assert await isolated_eventmanager.drain_async(timeout=0.01) is False
|
||||
assert cancelled is False
|
||||
assert isolated_eventmanager._EventManager__async_handles
|
||||
|
||||
release.set()
|
||||
assert await isolated_eventmanager.drain_async(timeout=1) is True
|
||||
assert cancelled is False
|
||||
finally:
|
||||
release.set()
|
||||
await isolated_eventmanager.stop_async()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sealed_drain_waits_for_cascade_and_rejects_later_broadcasts(
|
||||
isolated_eventmanager,
|
||||
) -> None:
|
||||
"""封口 drain 应结算 handler 派生事件,并拒绝封口后的新广播。"""
|
||||
calls = []
|
||||
|
||||
def first_handler(_event) -> None:
|
||||
calls.append("first")
|
||||
isolated_eventmanager.send_event(EventType.ModuleReload, {})
|
||||
|
||||
def cascaded_handler(_event) -> None:
|
||||
calls.append("cascaded")
|
||||
|
||||
isolated_eventmanager.add_event_listener(
|
||||
EventType.ConfigChanged,
|
||||
first_handler,
|
||||
)
|
||||
isolated_eventmanager.add_event_listener(
|
||||
EventType.ModuleReload,
|
||||
cascaded_handler,
|
||||
)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"cascade"}},
|
||||
)
|
||||
assert await isolated_eventmanager.drain_async(
|
||||
timeout=1,
|
||||
seal=True,
|
||||
) is True
|
||||
assert calls == ["first", "cascaded"]
|
||||
queue_size = isolated_eventmanager._EventManager__event_queue.qsize()
|
||||
|
||||
isolated_eventmanager.send_event(EventType.ModuleReload, {})
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
assert calls == ["first", "cascaded"]
|
||||
assert (
|
||||
isolated_eventmanager._EventManager__event_queue.qsize()
|
||||
== queue_size
|
||||
)
|
||||
assert isolated_eventmanager._EventManager__lifecycle_state == "sealed"
|
||||
finally:
|
||||
await isolated_eventmanager.stop_async()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_returns_false_when_stop_interleaves(
|
||||
isolated_eventmanager,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""stop 开始后 drain 必须报告屏障失败,而非等待残留队列后返回成功。"""
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
monkeypatch.setattr(
|
||||
isolated_eventmanager,
|
||||
"_EventManager__executor",
|
||||
executor,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def handler(_event) -> None:
|
||||
started.set()
|
||||
release.wait(timeout=2)
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"stop-race"}},
|
||||
)
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
drain_task = asyncio.create_task(isolated_eventmanager.drain_async())
|
||||
stop_task = asyncio.create_task(isolated_eventmanager.stop_async())
|
||||
|
||||
assert await asyncio.wait_for(drain_task, timeout=1) is False
|
||||
assert not stop_task.done()
|
||||
|
||||
release.set()
|
||||
await asyncio.wait_for(stop_task, timeout=1)
|
||||
assert isolated_eventmanager._EventManager__sync_handles == {}
|
||||
finally:
|
||||
release.set()
|
||||
if isolated_eventmanager._EventManager__lifecycle_state != "stopped":
|
||||
await isolated_eventmanager.stop_async()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_async_stop_balances_stop_sentinels(
|
||||
isolated_eventmanager,
|
||||
) -> None:
|
||||
"""重复异步停止不得遗留哨兵或破坏队列 unfinished_tasks 计数。"""
|
||||
isolated_eventmanager.start()
|
||||
|
||||
await asyncio.gather(
|
||||
isolated_eventmanager.stop_async(),
|
||||
isolated_eventmanager.stop_async(),
|
||||
)
|
||||
await isolated_eventmanager.stop_async()
|
||||
|
||||
assert isolated_eventmanager._EventManager__event_queue.qsize() == 0
|
||||
assert isolated_eventmanager._EventManager__event_queue.unfinished_tasks == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_stop_waits_for_owned_sync_handler(
|
||||
isolated_eventmanager,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""同步兼容停止入口也不得遗留仍运行的同步广播处理器。"""
|
||||
executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
monkeypatch.setattr(
|
||||
isolated_eventmanager,
|
||||
"_EventManager__executor",
|
||||
executor,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def handler(_event) -> None:
|
||||
started.set()
|
||||
release.wait(timeout=2)
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"sync-stop"}},
|
||||
)
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
stop_task = asyncio.create_task(
|
||||
asyncio.to_thread(isolated_eventmanager.stop)
|
||||
)
|
||||
await asyncio.sleep(0.02)
|
||||
assert not stop_task.done()
|
||||
|
||||
release.set()
|
||||
await asyncio.wait_for(stop_task, timeout=1)
|
||||
assert isolated_eventmanager._EventManager__sync_handles == {}
|
||||
assert (
|
||||
isolated_eventmanager._EventManager__event_queue.unfinished_tasks
|
||||
== 0
|
||||
)
|
||||
finally:
|
||||
release.set()
|
||||
if isolated_eventmanager._EventManager__lifecycle_state != "stopped":
|
||||
await isolated_eventmanager.stop_async()
|
||||
executor.shutdown(wait=True)
|
||||
|
||||
|
||||
def test_sync_broadcast_handler_can_call_legacy_stop_without_self_wait(
|
||||
isolated_eventmanager,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""同步 handler 调用旧 stop() 时不得等待承载自己的 Future。"""
|
||||
executor = _DaemonThreadExecutor()
|
||||
monkeypatch.setattr(
|
||||
isolated_eventmanager,
|
||||
"_EventManager__executor",
|
||||
executor,
|
||||
)
|
||||
stopped = threading.Event()
|
||||
|
||||
def handler(_event) -> None:
|
||||
isolated_eventmanager.stop()
|
||||
stopped.set()
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"handler-stop"}},
|
||||
)
|
||||
|
||||
assert stopped.wait(timeout=1)
|
||||
assert executor.finished.wait(timeout=1)
|
||||
assert isolated_eventmanager._EventManager__sync_handles == {}
|
||||
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
|
||||
finally:
|
||||
if isolated_eventmanager._EventManager__lifecycle_state == "running":
|
||||
consumer_threads = isolated_eventmanager._EventManager__begin_stop()
|
||||
for consumer_thread in consumer_threads:
|
||||
consumer_thread.join(timeout=1)
|
||||
isolated_eventmanager._EventManager__discard_stop_sentinels()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_broadcast_handler_can_await_stop_without_self_cancel(
|
||||
isolated_eventmanager,
|
||||
) -> None:
|
||||
"""异步 handler 调用 stop_async() 时不得取消或等待自身 completion。"""
|
||||
global_vars.set_loop(asyncio.get_running_loop())
|
||||
stopped = asyncio.Event()
|
||||
cancelled = False
|
||||
|
||||
async def handler(_event) -> None:
|
||||
nonlocal cancelled
|
||||
try:
|
||||
await isolated_eventmanager.stop_async()
|
||||
stopped.set()
|
||||
except asyncio.CancelledError:
|
||||
cancelled = True
|
||||
raise
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"async-handler-stop"}},
|
||||
)
|
||||
|
||||
await asyncio.wait_for(stopped.wait(), timeout=1)
|
||||
|
||||
async def wait_for_owner_release() -> None:
|
||||
"""等待 handler 返回后的 completion 回调移除 owner。"""
|
||||
while isolated_eventmanager._EventManager__async_handles:
|
||||
await asyncio.sleep(0)
|
||||
|
||||
await asyncio.wait_for(wait_for_owner_release(), timeout=1)
|
||||
assert cancelled is False
|
||||
assert isolated_eventmanager._EventManager__lifecycle_state == "stopped"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_broadcast_handler_cannot_seal_while_own_completion_is_pending(
|
||||
isolated_eventmanager,
|
||||
) -> None:
|
||||
"""handler 内 drain 应立即失败,且不能把仍运行的事件总线伪装成已封口。"""
|
||||
global_vars.set_loop(asyncio.get_running_loop())
|
||||
drain_results = []
|
||||
handler_done = asyncio.Event()
|
||||
|
||||
async def handler(_event) -> None:
|
||||
drain_results.append(
|
||||
await isolated_eventmanager.drain_async(seal=True)
|
||||
)
|
||||
handler_done.set()
|
||||
|
||||
isolated_eventmanager.add_event_listener(EventType.ConfigChanged, handler)
|
||||
isolated_eventmanager.start()
|
||||
try:
|
||||
isolated_eventmanager.send_event(
|
||||
EventType.ConfigChanged,
|
||||
{"key": {"handler-drain"}},
|
||||
)
|
||||
await asyncio.wait_for(handler_done.wait(), timeout=1)
|
||||
|
||||
assert drain_results == [False]
|
||||
assert isolated_eventmanager._EventManager__lifecycle_state == "running"
|
||||
assert await isolated_eventmanager.drain_async(timeout=1) is True
|
||||
finally:
|
||||
await isolated_eventmanager.stop_async()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_broadcast_submission_is_registered_before_stop_snapshot(
|
||||
isolated_eventmanager,
|
||||
|
||||
@@ -8,6 +8,7 @@ from fastapi import FastAPI
|
||||
|
||||
from app.startup import lifecycle, modules_initializer
|
||||
from app.adapters.network import http as http_utils
|
||||
from app.runtime.tasks import get_task_registry
|
||||
|
||||
|
||||
def _assert_completed_once(mock: MagicMock) -> None:
|
||||
@@ -59,11 +60,16 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
|
||||
|
||||
shutdown_steps = {
|
||||
"backup_plugins": system_chain.backup_plugins,
|
||||
"stop_plugin_monitor": MagicMock(return_value=True),
|
||||
"stop_workflow": MagicMock(),
|
||||
"stop_command": MagicMock(),
|
||||
"stop_monitor": MagicMock(),
|
||||
"stop_scheduler": MagicMock(),
|
||||
"stop_plugins": MagicMock(),
|
||||
"stop_agent": AsyncMock(return_value=True),
|
||||
"stop_transfer": AsyncMock(return_value=True),
|
||||
"quiesce_plugins": AsyncMock(return_value=True),
|
||||
"drain_events": AsyncMock(return_value=True),
|
||||
"finalize_plugins": MagicMock(return_value=True),
|
||||
"stop_modules": AsyncMock(),
|
||||
"close_http": AsyncMock(),
|
||||
}
|
||||
@@ -72,9 +78,22 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_plugins",
|
||||
"stop_plugin_monitor",
|
||||
"finalize_plugins",
|
||||
):
|
||||
monkeypatch.setattr(lifecycle, name, shutdown_steps[name])
|
||||
monkeypatch.setattr(lifecycle, "stop_agent", shutdown_steps["stop_agent"])
|
||||
monkeypatch.setattr(
|
||||
lifecycle,
|
||||
"stop_transfer_runtime",
|
||||
shutdown_steps["stop_transfer"],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
lifecycle,
|
||||
"quiesce_plugins",
|
||||
shutdown_steps["quiesce_plugins"],
|
||||
)
|
||||
monkeypatch.setattr(lifecycle, "drain_events", shutdown_steps["drain_events"])
|
||||
monkeypatch.setattr(lifecycle, "stop_modules", shutdown_steps["stop_modules"])
|
||||
monkeypatch.setattr(
|
||||
lifecycle,
|
||||
@@ -99,9 +118,6 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
],
|
||||
@@ -153,8 +169,258 @@ def test_lifespan_normal_mode_starts_full_runtime(monkeypatch):
|
||||
_assert_completed_once(step)
|
||||
|
||||
|
||||
def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch):
|
||||
"""关停必须等待插件恢复线程结束,避免与备份和资源释放并发。"""
|
||||
@pytest.mark.parametrize(
|
||||
("failing_step", "completed_steps", "blocked_steps"),
|
||||
[
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
("stop_plugin_monitor",),
|
||||
(
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
),
|
||||
),
|
||||
(
|
||||
"stop_monitor",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
),
|
||||
(
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
),
|
||||
),
|
||||
(
|
||||
"stop_scheduler",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
),
|
||||
(
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
),
|
||||
),
|
||||
(
|
||||
"stop_agent",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
),
|
||||
(
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
),
|
||||
),
|
||||
(
|
||||
"quiesce_plugins",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
),
|
||||
(
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
),
|
||||
),
|
||||
(
|
||||
"stop_transfer",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
),
|
||||
("drain_events", "finalize_plugins", "stop_modules", "close_http"),
|
||||
),
|
||||
(
|
||||
"drain_events",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
),
|
||||
("finalize_plugins", "stop_modules", "close_http"),
|
||||
),
|
||||
(
|
||||
"finalize_plugins",
|
||||
(
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_workflow",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
),
|
||||
("stop_modules", "close_http"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_lifespan_stops_releasing_dependencies_when_owner_does_not_converge(
|
||||
monkeypatch,
|
||||
failing_step,
|
||||
completed_steps,
|
||||
blocked_steps,
|
||||
):
|
||||
"""关键 owner 未收敛时不得关闭仍被活任务使用的后续依赖。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
shutdown_steps[failing_step].return_value = False
|
||||
|
||||
async def run_lifespan():
|
||||
"""启动并关闭隔离后的应用生命周期。"""
|
||||
async with lifecycle.lifespan(FastAPI()):
|
||||
pass
|
||||
|
||||
asyncio.run(run_lifespan())
|
||||
|
||||
for name in (*completed_steps, "logger"):
|
||||
_assert_completed_once(shutdown_steps[name])
|
||||
for name in blocked_steps:
|
||||
shutdown_steps[name].assert_not_called()
|
||||
|
||||
|
||||
def test_task_registry_nonconvergence_blocks_all_dependency_release(monkeypatch):
|
||||
"""最前置任务 owner 超时后不得继续释放插件、模块或 HTTP 依赖。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
shutdown = AsyncMock(return_value=False)
|
||||
monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown)
|
||||
app = FastAPI()
|
||||
|
||||
async def run_lifespan() -> None:
|
||||
"""运行后台登记器无法收敛的隔离生命周期。"""
|
||||
async with lifecycle.lifespan(app):
|
||||
pass
|
||||
|
||||
asyncio.run(run_lifespan())
|
||||
|
||||
shutdown.assert_awaited_once_with(timeout_seconds=30.0)
|
||||
for name, step in shutdown_steps.items():
|
||||
if name == "logger":
|
||||
_assert_completed_once(step)
|
||||
else:
|
||||
step.assert_not_called()
|
||||
assert isinstance(app.state.task_registry, lifecycle.TaskRegistry)
|
||||
|
||||
|
||||
def test_closed_task_registry_rejects_late_shutdown_tasks(monkeypatch) -> None:
|
||||
"""首屏障完成后,后续 stop hook 的晚到任务不得落回默认登记器。"""
|
||||
_patch_lifespan(monkeypatch)
|
||||
app = FastAPI()
|
||||
|
||||
async def run_lifespan() -> None:
|
||||
"""结束完整 lifespan 后验证当前发布的仍是已封口登记器。"""
|
||||
async with lifecycle.lifespan(app):
|
||||
pass
|
||||
registry = get_task_registry()
|
||||
assert registry is app.state.task_registry
|
||||
with pytest.raises(RuntimeError, match="正在关闭"):
|
||||
registry.create(asyncio.sleep(0), owner="shutdown.late_task")
|
||||
|
||||
asyncio.run(run_lifespan())
|
||||
|
||||
|
||||
def test_plugin_settlement_cannot_bypass_task_registry_shutdown_budget(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""未收敛 settlement 必须交给首屏障判定,lifespan 不得提前无界等待。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
shutdown = AsyncMock(return_value=False)
|
||||
monkeypatch.setattr(lifecycle.TaskRegistry, "shutdown", shutdown)
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def settle_plugins() -> None:
|
||||
"""模拟停机时仍未结束的插件同步任务。"""
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
lifecycle.init_extra.side_effect = settle_plugins
|
||||
|
||||
async def run_lifespan() -> None:
|
||||
"""确认 context 能由 TaskRegistry 的失败结果立即结束。"""
|
||||
async with lifecycle.lifespan(FastAPI()):
|
||||
await started.wait()
|
||||
release.set()
|
||||
await asyncio.sleep(0)
|
||||
|
||||
asyncio.run(asyncio.wait_for(run_lifespan(), timeout=0.5))
|
||||
|
||||
shutdown.assert_awaited_once_with(timeout_seconds=30.0)
|
||||
for name, step in shutdown_steps.items():
|
||||
if name == "logger":
|
||||
_assert_completed_once(step)
|
||||
else:
|
||||
step.assert_not_called()
|
||||
|
||||
|
||||
def test_lifespan_waits_for_uncancellable_plugin_settlement_before_shutdown(
|
||||
monkeypatch,
|
||||
):
|
||||
"""已进入同步 I/O 的 settlement 必须真实结束,才能备份和释放资源。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
order = []
|
||||
shutdown_steps["backup_plugins"].side_effect = lambda: order.append("backup")
|
||||
@@ -165,7 +431,12 @@ def test_lifespan_waits_for_plugin_settlement_before_shutdown(monkeypatch):
|
||||
|
||||
async def settle_plugins():
|
||||
started.set()
|
||||
await release.wait()
|
||||
try:
|
||||
await release.wait()
|
||||
except asyncio.CancelledError:
|
||||
# 模拟 run_in_threadpool_to_completion:外层取消只能封住新工作,
|
||||
# 已进入同步插件源码/依赖修改的调用仍持有 owner 到真实终态。
|
||||
await release.wait()
|
||||
order.append("settled")
|
||||
|
||||
lifecycle.init_extra.side_effect = settle_plugins
|
||||
@@ -225,14 +496,31 @@ def test_lifespan_safe_mode_skips_optional_runtime(monkeypatch):
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_plugins",
|
||||
"stop_plugin_monitor",
|
||||
"quiesce_plugins",
|
||||
"finalize_plugins",
|
||||
):
|
||||
shutdown_steps[name].assert_not_called()
|
||||
_assert_completed_once(shutdown_steps["stop_modules"])
|
||||
_assert_completed_once(shutdown_steps["stop_agent"])
|
||||
_assert_completed_once(shutdown_steps["stop_transfer"])
|
||||
_assert_completed_once(shutdown_steps["drain_events"])
|
||||
_assert_completed_once(shutdown_steps["close_http"])
|
||||
_assert_completed_once(shutdown_steps["logger"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_event_drain_does_not_materialize_manager(monkeypatch) -> None:
|
||||
"""模块尚未创建事件总线时,停机屏障应直接收敛而不反向构造。"""
|
||||
event_manager_type = MagicMock()
|
||||
event_manager_type.get_existing_instance.return_value = None
|
||||
monkeypatch.setattr(modules_initializer, "EventManager", event_manager_type)
|
||||
|
||||
assert await modules_initializer.drain_events() is True
|
||||
event_manager_type.get_existing_instance.assert_called_once_with()
|
||||
event_manager_type.assert_not_called()
|
||||
|
||||
|
||||
def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
|
||||
"""组件清单应显式冻结依赖、模式、启动/关闭顺序和超时预算。"""
|
||||
app = FastAPI()
|
||||
@@ -274,11 +562,16 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
|
||||
]
|
||||
assert normal_stop == [
|
||||
"后台任务登记器",
|
||||
"插件变更监控",
|
||||
"插件备份",
|
||||
"工作流",
|
||||
"命令服务",
|
||||
"监控器",
|
||||
"定时器",
|
||||
"AI智能体会话",
|
||||
"插件后台服务",
|
||||
"整理后台服务",
|
||||
"事件投递屏障",
|
||||
"插件",
|
||||
"模块服务",
|
||||
"HTTP 基础能力",
|
||||
@@ -292,9 +585,42 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
|
||||
"数据库连接预算",
|
||||
"路由",
|
||||
"模块服务",
|
||||
"AI智能体会话",
|
||||
"整理后台服务",
|
||||
"事件投递屏障",
|
||||
}
|
||||
assert all(item["start_failure"] == "fail_fast" for item in normal)
|
||||
assert all(item["stop_failure"] == "continue" for item in normal)
|
||||
assert {
|
||||
item["name"]
|
||||
for item in normal
|
||||
if item["stop_failure"] == "fail_fast"
|
||||
} == {
|
||||
"插件变更监控",
|
||||
"后台任务登记器",
|
||||
"监控器",
|
||||
"定时器",
|
||||
"AI智能体会话",
|
||||
"整理后台服务",
|
||||
"插件后台服务",
|
||||
"事件投递屏障",
|
||||
"插件",
|
||||
}
|
||||
assert all(
|
||||
item["stop_failure"] == "continue"
|
||||
for item in normal
|
||||
if item["name"]
|
||||
not in {
|
||||
"插件变更监控",
|
||||
"后台任务登记器",
|
||||
"监控器",
|
||||
"定时器",
|
||||
"AI智能体会话",
|
||||
"整理后台服务",
|
||||
"插件后台服务",
|
||||
"事件投递屏障",
|
||||
"插件",
|
||||
}
|
||||
)
|
||||
assert all(
|
||||
item["start_timeout_seconds"] or item["stop_timeout_seconds"]
|
||||
for item in normal
|
||||
@@ -496,6 +822,80 @@ def test_lifespan_does_not_yield_after_migration_failure(monkeypatch):
|
||||
assert app.state.moviepilot_health.phase.value == "failed"
|
||||
|
||||
|
||||
def test_lifespan_cleans_started_owners_after_late_startup_failure(monkeypatch):
|
||||
"""后段启动失败时应按同一停机策略回收已启动及部分启动的 owner。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
startup_error = RuntimeError("command startup failed")
|
||||
lifecycle.init_command.side_effect = startup_error
|
||||
app = FastAPI()
|
||||
|
||||
async def run_lifespan() -> None:
|
||||
"""运行一个在命令服务阶段失败的隔离生命周期。"""
|
||||
async with lifecycle.lifespan(app):
|
||||
pytest.fail("命令服务启动失败后不应发布运行态")
|
||||
|
||||
with pytest.raises(RuntimeError) as raised:
|
||||
asyncio.run(run_lifespan())
|
||||
|
||||
assert raised.value is startup_error
|
||||
lifecycle.global_vars.stop_system.assert_not_called()
|
||||
for name in (
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
):
|
||||
_assert_completed_once(shutdown_steps[name])
|
||||
shutdown_steps["stop_workflow"].assert_not_called()
|
||||
shutdown_steps["logger"].assert_not_called()
|
||||
assert isinstance(app.state.task_registry, lifecycle.TaskRegistry)
|
||||
assert get_task_registry() is app.state.task_registry
|
||||
assert app.state.moviepilot_health.phase.value == "failed"
|
||||
|
||||
|
||||
def test_startup_failure_cleanup_honors_transfer_fail_fast(monkeypatch):
|
||||
"""启动失败清理中整理 owner 未收敛时也不得继续释放插件和模块。"""
|
||||
shutdown_steps = _patch_lifespan(monkeypatch)
|
||||
lifecycle.init_command.side_effect = RuntimeError("command startup failed")
|
||||
shutdown_steps["stop_transfer"].return_value = False
|
||||
|
||||
async def run_lifespan() -> None:
|
||||
"""运行后段失败且整理线程无法收敛的隔离生命周期。"""
|
||||
async with lifecycle.lifespan(FastAPI()):
|
||||
pytest.fail("命令服务启动失败后不应发布运行态")
|
||||
|
||||
with pytest.raises(RuntimeError, match="command startup failed"):
|
||||
asyncio.run(run_lifespan())
|
||||
|
||||
for name in (
|
||||
"stop_plugin_monitor",
|
||||
"backup_plugins",
|
||||
"stop_command",
|
||||
"stop_monitor",
|
||||
"stop_scheduler",
|
||||
"stop_agent",
|
||||
"quiesce_plugins",
|
||||
"stop_transfer",
|
||||
):
|
||||
_assert_completed_once(shutdown_steps[name])
|
||||
for name in (
|
||||
"drain_events",
|
||||
"finalize_plugins",
|
||||
"stop_modules",
|
||||
"close_http",
|
||||
):
|
||||
shutdown_steps[name].assert_not_called()
|
||||
shutdown_steps["stop_workflow"].assert_not_called()
|
||||
|
||||
|
||||
def test_uvicorn_preserves_stop_requested_before_serve(monkeypatch):
|
||||
"""Uvicorn 启动不能清除数据库初始化阶段已经发布的停止请求"""
|
||||
from app import main
|
||||
@@ -564,16 +964,13 @@ def test_command_restart_failure_does_not_publish_stop_request(monkeypatch):
|
||||
assert not stop_event.is_set()
|
||||
|
||||
|
||||
def test_stop_modules_continues_after_internal_owner_failures(monkeypatch):
|
||||
"""模块关闭编排中的多个失败不能阻断其余清理"""
|
||||
stop_agent = AsyncMock(side_effect=RuntimeError("agent failed"))
|
||||
monkeypatch.setattr(modules_initializer, "stop_agent", stop_agent)
|
||||
def test_stop_modules_continues_after_internal_owner_failure(monkeypatch):
|
||||
"""模块关闭编排中的单个失败不能阻断其余清理。"""
|
||||
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
|
||||
dependencies["module"].side_effect = RuntimeError("module failed")
|
||||
|
||||
asyncio.run(modules_initializer.stop_modules())
|
||||
|
||||
stop_agent.assert_awaited_once_with()
|
||||
for dependency in dependencies.values():
|
||||
_assert_completed_once(dependency)
|
||||
|
||||
@@ -581,7 +978,6 @@ def test_stop_modules_continues_after_internal_owner_failures(monkeypatch):
|
||||
def test_stop_modules_drains_web_agent_tasks_before_persistence(monkeypatch):
|
||||
"""关闭时先收口 Web Agent,再关闭持久化准入和数据库任务。"""
|
||||
order = []
|
||||
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
|
||||
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
modules_initializer,
|
||||
@@ -619,7 +1015,6 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
|
||||
_patch_module_shutdown_dependencies(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
modules_initializer,
|
||||
@@ -651,8 +1046,9 @@ async def test_shutdown_timeout_does_not_skip_database_worker_cleanup(monkeypatc
|
||||
)
|
||||
)
|
||||
await started.wait()
|
||||
await shutdown
|
||||
completed = await shutdown
|
||||
|
||||
assert completed is False
|
||||
stop_database_worker.assert_awaited_once_with()
|
||||
|
||||
|
||||
@@ -683,9 +1079,10 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No
|
||||
)
|
||||
)
|
||||
await started.wait()
|
||||
await shutdown
|
||||
completed = await shutdown
|
||||
|
||||
elapsed = asyncio.get_running_loop().time() - started_at
|
||||
assert completed is False
|
||||
assert elapsed < 0.2
|
||||
await asyncio.wait_for(cancel_requested.wait(), timeout=0.2)
|
||||
assert not settled.is_set()
|
||||
@@ -694,6 +1091,28 @@ async def test_shutdown_timeout_has_hard_bound_for_nonconverging_cleanup() -> No
|
||||
await asyncio.wait_for(settled.wait(), timeout=0.2)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_shutdown_step_reports_explicit_nonconvergence() -> None:
|
||||
"""同步和异步 owner 显式返回 False 时都必须向生命周期传播失败。"""
|
||||
|
||||
async def async_nonconverging_shutdown() -> bool:
|
||||
"""模拟已经完成等待但仍持有资源的异步关闭入口。"""
|
||||
return False
|
||||
|
||||
assert await lifecycle.run_shutdown_step(
|
||||
"同步 owner",
|
||||
lambda: False,
|
||||
) is False
|
||||
assert await lifecycle.run_shutdown_step(
|
||||
"异步 owner",
|
||||
async_nonconverging_shutdown,
|
||||
) is False
|
||||
assert await lifecycle.run_shutdown_step(
|
||||
"已收敛 owner",
|
||||
lambda: None,
|
||||
) is True
|
||||
|
||||
|
||||
def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
|
||||
"""替换 stop_modules 的资源所有者,避免测试启动真实后台服务"""
|
||||
dependencies = {}
|
||||
@@ -749,7 +1168,6 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict:
|
||||
def test_browser_sessions_close_before_managed_resources(monkeypatch) -> None:
|
||||
"""显示等宿主资源必须晚于浏览器会话释放,避免存活上下文失去依赖。"""
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock())
|
||||
dependencies = _patch_module_shutdown_dependencies(monkeypatch)
|
||||
dependencies["close_browser_sessions"].side_effect = lambda: calls.append("browser")
|
||||
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from watchfiles import Change
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.monitor.monitor import Monitor
|
||||
from app.monitor.recovery import RecoveryExecutor
|
||||
from app.monitor.syslimits import decide_monitor_mode
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
from app.adapters.system.host import SystemUtils
|
||||
|
||||
|
||||
@@ -24,13 +26,24 @@ def _build_monitor(handle_file: MagicMock = None):
|
||||
if handle_file is not None:
|
||||
dispatcher.handle_file = handle_file
|
||||
monitor._dispatcher = dispatcher
|
||||
monitor._lifecycle_lock = threading.RLock()
|
||||
monitor._owner_lock = threading.Lock()
|
||||
monitor._work_stop_event = threading.Event()
|
||||
monitor._shutdown_event = threading.Event()
|
||||
monitor._closed = False
|
||||
monitor._compensation_threads = {}
|
||||
monitor._scheduler_shutdown_thread = None
|
||||
monitor._scheduler_shutdown_succeeded = False
|
||||
monitor._scheduler = None
|
||||
monitor._watchers = []
|
||||
monitor._retired_watchers = []
|
||||
monitor._watcher_lock = Lock()
|
||||
monitor._alerted_paths = {}
|
||||
monitor._restart_marks = {}
|
||||
monitor._stable_cycles = {}
|
||||
monitor._isolated = {}
|
||||
monitor._pending_rebuild = {}
|
||||
monitor._pending_locals = []
|
||||
monitor._recovery = RecoveryExecutor()
|
||||
return monitor, dispatcher
|
||||
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
"""目录监控 owner 的生命周期与停机屏障回归。"""
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.monitor.monitor import Monitor
|
||||
from app.monitor.recovery import RecoveryExecutor, RecoveryState
|
||||
from app.foundation.singleton import SingletonClass
|
||||
from app.startup.monitor_initializer import init_monitor, stop_monitor
|
||||
|
||||
|
||||
def _build_monitor() -> Monitor:
|
||||
"""构造绕过单例初始化、但具备完整生命周期字段的 Monitor 骨架。"""
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor._lifecycle_lock = threading.RLock()
|
||||
monitor._owner_lock = threading.Lock()
|
||||
monitor._work_stop_event = threading.Event()
|
||||
monitor._shutdown_event = threading.Event()
|
||||
monitor._closed = False
|
||||
monitor._compensation_threads = {}
|
||||
monitor._scheduler_shutdown_thread = None
|
||||
monitor._scheduler_shutdown_succeeded = False
|
||||
monitor._scheduler = None
|
||||
monitor._watchers = []
|
||||
monitor._retired_watchers = []
|
||||
monitor._watcher_lock = threading.Lock()
|
||||
monitor._pending_locals = []
|
||||
monitor._alerted_paths = {}
|
||||
monitor._restart_marks = {}
|
||||
monitor._stable_cycles = {}
|
||||
monitor._isolated = {}
|
||||
monitor._pending_rebuild = {}
|
||||
monitor._recovery = RecoveryExecutor()
|
||||
monitor._dispatcher = MagicMock()
|
||||
return monitor
|
||||
|
||||
|
||||
def _blocking_watcher(mon_path: Path):
|
||||
"""构造只有显式 release 后才退出的 watcher 与控制事件。"""
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def run() -> None:
|
||||
"""模拟无法被 stop event 唤醒的 FUSE watcher。"""
|
||||
started.set()
|
||||
release.wait()
|
||||
|
||||
thread = threading.Thread(target=run, daemon=True, name="test-monitor-watcher")
|
||||
thread.start()
|
||||
assert started.wait(1)
|
||||
watcher = MagicMock()
|
||||
watcher.watch_path = mon_path
|
||||
watcher.stop.side_effect = lambda: None
|
||||
watcher.join.side_effect = thread.join
|
||||
watcher.is_alive.side_effect = thread.is_alive
|
||||
return watcher, thread, release
|
||||
|
||||
|
||||
def test_close_budget_includes_lifecycle_lock_wait() -> None:
|
||||
"""停机 deadline 必须先于生命周期锁等待建立,锁竞争超时后立即返回。"""
|
||||
monitor = _build_monitor()
|
||||
lock_acquired = threading.Event()
|
||||
release_lock = threading.Event()
|
||||
|
||||
def hold_lifecycle_lock() -> None:
|
||||
"""占住生命周期锁,模拟配置重载阻塞在挂载访问中。"""
|
||||
with monitor._lifecycle_lock:
|
||||
lock_acquired.set()
|
||||
release_lock.wait()
|
||||
|
||||
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
|
||||
holder.start()
|
||||
assert lock_acquired.wait(1)
|
||||
|
||||
started_at = time.monotonic()
|
||||
try:
|
||||
assert monitor.close(timeout=0.02) is False
|
||||
assert time.monotonic() - started_at < 0.5
|
||||
assert monitor.lifecycle_closed is True
|
||||
finally:
|
||||
release_lock.set()
|
||||
holder.join(timeout=1)
|
||||
assert holder.is_alive() is False
|
||||
assert monitor.close(timeout=1) is True
|
||||
|
||||
|
||||
def test_close_timeout_retains_watcher_and_seals_config_reload(tmp_path) -> None:
|
||||
"""挂死 watcher 超时时保留句柄,且配置变化不能重开已封口生命周期。"""
|
||||
monitor = _build_monitor()
|
||||
watcher, thread, release = _blocking_watcher(tmp_path)
|
||||
monitor._watchers = [watcher]
|
||||
reload_monitor = MagicMock()
|
||||
monitor.init = reload_monitor
|
||||
|
||||
try:
|
||||
assert monitor.close(timeout=0.02) is False
|
||||
assert monitor._watchers == [watcher]
|
||||
monitor._dispatcher.clear_pending.assert_not_called()
|
||||
|
||||
monitor.on_config_changed()
|
||||
reload_monitor.assert_not_called()
|
||||
finally:
|
||||
release.set()
|
||||
assert monitor.close(timeout=1) is True
|
||||
thread.join(timeout=1)
|
||||
assert thread.is_alive() is False
|
||||
assert monitor._watchers == []
|
||||
monitor._dispatcher.clear_pending.assert_called_once_with()
|
||||
|
||||
|
||||
def test_reopen_allows_only_an_explicit_new_lifespan() -> None:
|
||||
"""close 后普通 init 被拒绝,显式 reopen 收敛旧 owner 后才允许重新初始化。"""
|
||||
monitor = _build_monitor()
|
||||
initialize = MagicMock(return_value=True)
|
||||
monitor._Monitor__initialize_monitors = initialize
|
||||
|
||||
assert monitor.close(timeout=1) is True
|
||||
assert monitor.init(timeout=0) is False
|
||||
initialize.assert_not_called()
|
||||
|
||||
assert monitor.reopen(timeout=1) is True
|
||||
assert monitor.init(timeout=1) is True
|
||||
initialize.assert_called_once_with()
|
||||
assert monitor.lifecycle_closed is False
|
||||
|
||||
|
||||
def test_close_tracks_recovery_and_blocks_post_seal_dispatch() -> None:
|
||||
"""恢复动作解冻后只能退出,不能越过封口继续派发整理重试。"""
|
||||
monitor = _build_monitor()
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def blocking_local_retry() -> None:
|
||||
"""模拟恢复线程阻塞在 FUSE 本地目录访问。"""
|
||||
entered.set()
|
||||
release.wait()
|
||||
|
||||
monitor._Monitor__retry_pending_locals = blocking_local_retry
|
||||
result = monitor._recovery.run(
|
||||
{monitor.PENDING_KEY: monitor._Monitor__drive_pending}, timeout=0.01
|
||||
)
|
||||
assert entered.is_set()
|
||||
assert result == {monitor.PENDING_KEY: RecoveryState.TIMEOUT}
|
||||
|
||||
try:
|
||||
assert monitor.close(timeout=0.02) is False
|
||||
assert len(monitor._recovery.running_threads()) == 1
|
||||
monitor._dispatcher.retry_pending.assert_not_called()
|
||||
finally:
|
||||
release.set()
|
||||
assert monitor.close(timeout=1) is True
|
||||
assert monitor._recovery.running_threads() == ()
|
||||
monitor._dispatcher.retry_pending.assert_not_called()
|
||||
|
||||
|
||||
def test_recovery_start_and_registration_are_atomic(monkeypatch) -> None:
|
||||
"""close 不得在恢复线程登记后、真正 start 前把它误判为已收敛。"""
|
||||
executor = RecoveryExecutor()
|
||||
start_entered = threading.Event()
|
||||
allow_start = threading.Event()
|
||||
close_results: list[bool] = []
|
||||
original_start = threading.Thread.start
|
||||
|
||||
def delayed_recovery_start(thread: threading.Thread) -> None:
|
||||
"""只暂停恢复线程的 start,稳定放大登记与启动之间的竞态窗口。"""
|
||||
if thread.name.startswith("MoviePilot-MonitorRecovery-"):
|
||||
start_entered.set()
|
||||
assert allow_start.wait(timeout=1)
|
||||
original_start(thread)
|
||||
|
||||
monkeypatch.setattr(threading.Thread, "start", delayed_recovery_start)
|
||||
runner = threading.Thread(
|
||||
target=lambda: executor.run({"atomic": lambda: None}, timeout=1),
|
||||
name="recovery-submit-test",
|
||||
)
|
||||
runner.start()
|
||||
assert start_entered.wait(timeout=1)
|
||||
|
||||
closer = threading.Thread(
|
||||
target=lambda: close_results.append(
|
||||
executor.close(deadline=time.monotonic() + 1)
|
||||
),
|
||||
name="recovery-close-test",
|
||||
)
|
||||
closer.start()
|
||||
closer.join(timeout=0.05)
|
||||
assert closer.is_alive(), "close 越过了尚未完成 start 的 owner 临界区"
|
||||
|
||||
allow_start.set()
|
||||
runner.join(timeout=1)
|
||||
closer.join(timeout=1)
|
||||
|
||||
assert not runner.is_alive()
|
||||
assert not closer.is_alive()
|
||||
assert close_results == [True]
|
||||
assert executor.running_threads() == ()
|
||||
|
||||
|
||||
def test_close_tracks_compensation_and_blocks_post_seal_dispatch(tmp_path) -> None:
|
||||
"""补偿扫描解冻后不得在旧 lifespan 中继续把文件送入整理链。"""
|
||||
monitor = _build_monitor()
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
candidate = tmp_path / "late.mkv"
|
||||
|
||||
def blocking_collect(_mon_path: Path):
|
||||
"""模拟补偿扫描阻塞在目录遍历。"""
|
||||
entered.set()
|
||||
release.wait()
|
||||
return [(candidate, time.time(), 1)]
|
||||
|
||||
monitor._Monitor__collect_compensation_files = blocking_collect
|
||||
monitor._Monitor__start_compensation(mon_path=tmp_path, since=time.time())
|
||||
assert entered.wait(1)
|
||||
|
||||
try:
|
||||
assert monitor.close(timeout=0.02) is False
|
||||
assert len(monitor._compensation_threads) == 1
|
||||
monitor._dispatcher.handle_file.assert_not_called()
|
||||
finally:
|
||||
release.set()
|
||||
assert monitor.close(timeout=1) is True
|
||||
assert monitor._compensation_threads == {}
|
||||
monitor._dispatcher.handle_file.assert_not_called()
|
||||
|
||||
|
||||
def test_close_tracks_scheduler_shutdown_thread() -> None:
|
||||
"""scheduler shutdown 阻塞时保留线程和 scheduler,恢复后才能清空句柄。"""
|
||||
monitor = _build_monitor()
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
scheduler = MagicMock()
|
||||
scheduler.running = True
|
||||
|
||||
def blocking_shutdown(*, wait: bool) -> None:
|
||||
"""模拟等待在途 APScheduler job 的同步 shutdown。"""
|
||||
assert wait is True
|
||||
entered.set()
|
||||
release.wait()
|
||||
|
||||
scheduler.shutdown.side_effect = blocking_shutdown
|
||||
monitor._scheduler = scheduler
|
||||
|
||||
try:
|
||||
assert monitor.close(timeout=0.02) is False
|
||||
assert entered.is_set()
|
||||
assert monitor._scheduler is scheduler
|
||||
assert monitor._scheduler_shutdown_thread is not None
|
||||
assert monitor._scheduler_shutdown_thread.is_alive()
|
||||
finally:
|
||||
release.set()
|
||||
assert monitor.close(timeout=1) is True
|
||||
assert monitor._scheduler is None
|
||||
assert monitor._scheduler_shutdown_thread is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_monitor_cancellation_waits_for_sync_close(monkeypatch) -> None:
|
||||
"""生命周期取消异步 stop 后,线程池调用仍由任务持有到同步 close 结束。"""
|
||||
entered = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def blocking_close(timeout: float) -> bool:
|
||||
"""模拟仍在同步收尾的 Monitor.close。"""
|
||||
assert timeout == 1
|
||||
entered.set()
|
||||
release.wait()
|
||||
return True
|
||||
|
||||
monitor = SimpleNamespace(close=blocking_close)
|
||||
monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor)
|
||||
task = asyncio.create_task(stop_monitor(timeout=1))
|
||||
while not entered.is_set():
|
||||
await asyncio.sleep(0)
|
||||
|
||||
task.cancel()
|
||||
await asyncio.sleep(0.01)
|
||||
assert task.done() is False
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
def test_init_monitor_explicitly_reopens_existing_lifespan(monkeypatch) -> None:
|
||||
"""同进程测试中的新 lifespan 必须显式 reopen,再初始化同一个 Monitor。"""
|
||||
monitor = SimpleNamespace(
|
||||
lifecycle_closed=True,
|
||||
reopen=MagicMock(return_value=True),
|
||||
init=MagicMock(return_value=True),
|
||||
)
|
||||
monkeypatch.setattr(Monitor, "get_existing_instance", lambda: monitor)
|
||||
|
||||
init_monitor()
|
||||
|
||||
monitor.reopen.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT)
|
||||
monitor.init.assert_called_once_with(timeout=Monitor.RELOAD_STOP_TIMEOUT)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_constructor_failure_publishes_started_watcher_to_cleanup(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""Monitor 构造中途失败后,stop-only 入口仍必须找到已启动 watcher owner。"""
|
||||
instances = dict(SingletonClass._instances)
|
||||
instances.pop(Monitor, None)
|
||||
monkeypatch.setattr(SingletonClass, "_instances", instances)
|
||||
watcher_started = threading.Event()
|
||||
watcher_release = threading.Event()
|
||||
watchers: list[threading.Thread] = []
|
||||
|
||||
def failing_init(monitor: Monitor) -> None:
|
||||
"""模拟 watcher 已启动、后续 scheduler 启动失败的构造过程。"""
|
||||
watcher = threading.Thread(
|
||||
target=lambda: (watcher_started.set(), watcher_release.wait()),
|
||||
name="monitor-partial-construction",
|
||||
daemon=True,
|
||||
)
|
||||
watchers.append(watcher)
|
||||
watcher.start()
|
||||
|
||||
def close(timeout: float) -> bool:
|
||||
"""模拟真实 close 释放并等待半构造实例已经发布的 watcher。"""
|
||||
watcher_release.set()
|
||||
watcher.join(timeout=timeout)
|
||||
return not watcher.is_alive()
|
||||
|
||||
monitor.close = close
|
||||
raise RuntimeError("scheduler start failed")
|
||||
|
||||
monkeypatch.setattr(Monitor, "__init__", failing_init)
|
||||
|
||||
with pytest.raises(RuntimeError, match="scheduler start failed"):
|
||||
Monitor()
|
||||
assert watcher_started.wait(timeout=1)
|
||||
retained = Monitor.get_existing_instance()
|
||||
assert retained is not None
|
||||
|
||||
assert await stop_monitor(timeout=1) is True
|
||||
assert watchers[0].is_alive() is False
|
||||
@@ -18,8 +18,9 @@ from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||
from app.monitor.monitor import Monitor
|
||||
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
|
||||
|
||||
def _build_monitor(monkeypatch, put_recorder=None):
|
||||
@@ -33,7 +34,17 @@ def _build_monitor(monkeypatch, put_recorder=None):
|
||||
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
|
||||
monitor = object.__new__(Monitor)
|
||||
monitor._dispatcher = MagicMock()
|
||||
monitor._lifecycle_lock = threading.RLock()
|
||||
monitor._owner_lock = threading.Lock()
|
||||
monitor._work_stop_event = threading.Event()
|
||||
monitor._shutdown_event = threading.Event()
|
||||
monitor._closed = False
|
||||
monitor._compensation_threads = {}
|
||||
monitor._scheduler_shutdown_thread = None
|
||||
monitor._scheduler_shutdown_succeeded = False
|
||||
monitor._scheduler = None
|
||||
monitor._watchers = []
|
||||
monitor._retired_watchers = []
|
||||
monitor._watcher_lock = Lock()
|
||||
monitor._pending_locals = []
|
||||
monitor._alerted_paths = {}
|
||||
@@ -114,7 +125,7 @@ def _stop_monitor_threads(monitor, timeout=10.0):
|
||||
:param monitor: Monitor 骨架
|
||||
:param timeout: 每个线程的最长等待秒数
|
||||
"""
|
||||
for watcher in monitor._watchers:
|
||||
for watcher in (*monitor._watchers, *monitor._retired_watchers):
|
||||
if isinstance(watcher, LocalDirectoryWatcher):
|
||||
watcher.stop()
|
||||
watcher.join(timeout=timeout)
|
||||
@@ -262,6 +273,10 @@ def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch
|
||||
assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离"
|
||||
finally:
|
||||
_release_and_join(release, monitor._recovery._running.values())
|
||||
assert watcher in monitor._retired_watchers
|
||||
# 重建后的旧 watcher 仍由 retired owner 表持有;测试替身不会自行切换
|
||||
# is_alive,释放模拟阻塞后显式推进到真实线程应有的退出终态。
|
||||
watcher._thread.is_alive.return_value = False
|
||||
_stop_monitor_threads(monitor)
|
||||
|
||||
|
||||
@@ -454,7 +469,7 @@ def test_stuck_transfer_does_not_block_other_files(monkeypatch):
|
||||
|
||||
def test_recovery_executor_reports_timeout_without_blocking():
|
||||
"""
|
||||
永不返回的动作只应消耗一次 timeout,执行器必须放弃它并如实报告。
|
||||
永不返回的动作只应消耗一次 timeout,执行器必须结束本轮等待并如实报告。
|
||||
"""
|
||||
executor = RecoveryExecutor()
|
||||
release = threading.Event()
|
||||
@@ -465,7 +480,7 @@ def test_recovery_executor_reports_timeout_without_blocking():
|
||||
|
||||
try:
|
||||
assert results == {"stuck": RecoveryState.TIMEOUT}
|
||||
assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作"
|
||||
assert elapsed < 3.0, "执行器没有在超时后结束本轮等待"
|
||||
finally:
|
||||
_release_and_join(release, executor._running.values())
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||
from app.monitor.monitor import Monitor
|
||||
from app.monitor.recovery import RecoveryExecutor
|
||||
from app.monitor.watcher import LocalDirectoryWatcher
|
||||
|
||||
|
||||
def _build_watcher(tmp_path, force_polling):
|
||||
@@ -133,7 +135,17 @@ def _build_monitor(monkeypatch, put_recorder):
|
||||
monitor = object.__new__(Monitor)
|
||||
# 自动重启后健康检查会发起补偿扫描,骨架需要一个分发器替身
|
||||
monitor._dispatcher = MagicMock()
|
||||
monitor._lifecycle_lock = threading.RLock()
|
||||
monitor._owner_lock = threading.Lock()
|
||||
monitor._work_stop_event = threading.Event()
|
||||
monitor._shutdown_event = threading.Event()
|
||||
monitor._closed = False
|
||||
monitor._compensation_threads = {}
|
||||
monitor._scheduler_shutdown_thread = None
|
||||
monitor._scheduler_shutdown_succeeded = False
|
||||
monitor._scheduler = None
|
||||
monitor._watchers = []
|
||||
monitor._retired_watchers = []
|
||||
monitor._watcher_lock = Lock()
|
||||
monitor._pending_locals = []
|
||||
monitor._alerted_paths = {}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from watchfiles import Change
|
||||
|
||||
from app.monitor import DirectoryChangeEvent, LocalDirectoryWatcher, Monitor
|
||||
from app.monitor.dispatcher import TransferDispatcher
|
||||
from app.schemas import TransferDirectoryConf
|
||||
from app.monitor.monitor import Monitor
|
||||
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
|
||||
from app.schemas.system import TransferDirectoryConf
|
||||
from app.schemas.types import MediaType
|
||||
|
||||
|
||||
@@ -42,6 +44,15 @@ def _build_monitor_with_dispatcher(handle_file: MagicMock = None):
|
||||
if handle_file is not None:
|
||||
dispatcher.handle_file = handle_file
|
||||
monitor._dispatcher = dispatcher
|
||||
monitor._lifecycle_lock = threading.RLock()
|
||||
monitor._owner_lock = threading.Lock()
|
||||
monitor._work_stop_event = threading.Event()
|
||||
monitor._shutdown_event = threading.Event()
|
||||
monitor._closed = False
|
||||
monitor._compensation_threads = {}
|
||||
monitor._scheduler_shutdown_thread = None
|
||||
monitor._scheduler_shutdown_succeeded = False
|
||||
monitor._scheduler = None
|
||||
return monitor, dispatcher
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
from contextlib import nullcontext
|
||||
|
||||
from app.application.plugin.config import PluginConfigCommand
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
|
||||
|
||||
def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCommand:
|
||||
def _command(
|
||||
calls: list[tuple],
|
||||
*,
|
||||
save_result: bool = True,
|
||||
mutation=None,
|
||||
) -> PluginConfigCommand:
|
||||
"""构造记录端口调用顺序的插件配置用例。"""
|
||||
return PluginConfigCommand(
|
||||
save_config=lambda plugin_id, config, force: (
|
||||
@@ -22,6 +30,7 @@ def _command(calls: list[tuple], *, save_result: bool = True) -> PluginConfigCom
|
||||
refresh_registrations=lambda plugin_id: calls.append(
|
||||
("registrations", plugin_id)
|
||||
),
|
||||
mutation=mutation or (lambda _operation: nullcontext()),
|
||||
)
|
||||
|
||||
|
||||
@@ -65,3 +74,22 @@ def test_reset_preserves_compensation_cleanup_and_reload_order() -> None:
|
||||
("reload", "DemoPlugin"),
|
||||
("registrations", "DemoPlugin"),
|
||||
]
|
||||
|
||||
|
||||
def test_sealed_config_command_rejects_before_first_side_effect() -> None:
|
||||
"""配置事务在 admission 封口后返回失败,且不得先发布 reset 事件。"""
|
||||
calls: list[tuple] = []
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
|
||||
update_result = _command(calls, mutation=admission.hold).update(
|
||||
"DemoPlugin",
|
||||
{"enabled": True},
|
||||
)
|
||||
reset_result = _command(calls, mutation=admission.hold).reset("DemoPlugin")
|
||||
|
||||
assert update_result.success is False
|
||||
assert reset_result.success is False
|
||||
assert "停机阶段" in update_result.message
|
||||
assert "停机阶段" in reset_result.message
|
||||
assert calls == []
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
||||
|
||||
from app.api.endpoints import plugin as plugin_endpoint
|
||||
@@ -13,6 +14,7 @@ from app.api.endpoints.plugin import uninstall_plugin
|
||||
from app.api.endpoints.system import sync_plugin_market_from_wiki
|
||||
from app.application.plugin.config import PluginConfigCommand
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.runtime.tasks import TaskRegistry
|
||||
from app.schemas.event import PluginDataResetEventData
|
||||
@@ -495,6 +497,7 @@ def test_reset_plugin_sends_pre_reset_chain_event_before_deleting_data():
|
||||
reload_runtime=plugin_manager.reload_plugin,
|
||||
publish_reset=publish_reset,
|
||||
refresh_registrations=lambda _plugin_id: None,
|
||||
mutation=lambda _operation: nullcontext(),
|
||||
)
|
||||
result = reset_plugin("SubscribeAssistantEnhanced", None, command)
|
||||
|
||||
@@ -592,6 +595,82 @@ def test_uninstall_virtual_instance_never_removes_source_package(monkeypatch):
|
||||
plugin_manager.remove_plugin.assert_called_once_with("DemoPluginwork")
|
||||
|
||||
|
||||
def test_sealed_http_uninstall_rejects_before_first_side_effect(monkeypatch):
|
||||
"""HTTP 卸载在封口后明确失败,且不读取或写入插件持久化状态。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
plugin_manager = MagicMock()
|
||||
plugin_manager.mutation.side_effect = admission.hold
|
||||
config_provider = MagicMock()
|
||||
remove_api = MagicMock()
|
||||
remove_job = MagicMock()
|
||||
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
|
||||
monkeypatch.setattr(
|
||||
plugin_endpoint,
|
||||
"get_configured_system_config",
|
||||
config_provider,
|
||||
)
|
||||
monkeypatch.setattr(plugin_endpoint, "remove_plugin_api", remove_api)
|
||||
monkeypatch.setattr(plugin_endpoint, "remove_plugin_job", remove_job)
|
||||
|
||||
result = uninstall_plugin("DemoPlugin", None)
|
||||
|
||||
assert result.success is False
|
||||
assert "停机阶段" in result.message
|
||||
plugin_manager.get_plugin_instance.assert_not_called()
|
||||
config_provider.assert_not_called()
|
||||
remove_api.assert_not_called()
|
||||
remove_job.assert_not_called()
|
||||
|
||||
|
||||
def test_sealed_http_clone_rejects_before_runtime_and_registration(monkeypatch):
|
||||
"""HTTP 分身事务未获 admission 时不创建实例、不刷新注册或文件夹。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
plugin_manager = MagicMock()
|
||||
plugin_manager.mutation.side_effect = admission.hold
|
||||
register = MagicMock()
|
||||
add_to_folder = MagicMock()
|
||||
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
|
||||
monkeypatch.setattr(plugin_endpoint, "register_plugin", register)
|
||||
monkeypatch.setattr(plugin_endpoint, "_add_clone_to_plugin_folder", add_to_folder)
|
||||
|
||||
result = plugin_endpoint.clone_plugin(
|
||||
"DemoPlugin",
|
||||
schemas.PluginCloneRequest(suffix="Work"),
|
||||
None,
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert "停机阶段" in result.message
|
||||
plugin_manager.clone_plugin.assert_not_called()
|
||||
register.assert_not_called()
|
||||
add_to_folder.assert_not_called()
|
||||
|
||||
|
||||
def test_sealed_http_folder_update_rejects_before_config_access(monkeypatch):
|
||||
"""插件文件夹写入口在封口后不读取或改写持久化配置。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
plugin_manager = MagicMock()
|
||||
plugin_manager.mutation.side_effect = admission.hold
|
||||
config_provider = MagicMock()
|
||||
monkeypatch.setattr(plugin_endpoint, "PluginManager", lambda: plugin_manager)
|
||||
monkeypatch.setattr(
|
||||
plugin_endpoint,
|
||||
"get_configured_system_config",
|
||||
config_provider,
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
plugin_endpoint.update_folder_plugins("常用", ["DemoPlugin"], None)
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert "停机阶段" in result.message
|
||||
config_provider.assert_not_called()
|
||||
|
||||
|
||||
def test_delete_plugin_data_can_force_delete_after_plugin_is_stopped():
|
||||
"""
|
||||
重置入口会先停止插件;插件数据删除不能依赖运行态注册仍存在。
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import asyncio
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
@@ -9,6 +10,7 @@ from app.schemas.exception import (
|
||||
PersistenceUnavailableError,
|
||||
)
|
||||
from app.application.plugin.install import PluginInstallCommand
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
|
||||
|
||||
def _command(
|
||||
@@ -24,6 +26,8 @@ def _command(
|
||||
checkpointer=None,
|
||||
committer=None,
|
||||
rollback=None,
|
||||
mutation=None,
|
||||
package_write_guard=None,
|
||||
):
|
||||
"""构造可观测每一步副作用的插件安装命令。"""
|
||||
return PluginInstallCommand(
|
||||
@@ -38,6 +42,9 @@ def _command(
|
||||
install_reporter=reporter or AsyncMock(),
|
||||
plugin_reloader=reloader or AsyncMock(),
|
||||
registration_refresher=refresher or AsyncMock(),
|
||||
mutation=mutation or (lambda _operation: nullcontext()),
|
||||
package_write_guard=package_write_guard
|
||||
or (lambda _plugin_id: nullcontext()),
|
||||
)
|
||||
|
||||
|
||||
@@ -74,6 +81,30 @@ async def test_install_failure_stops_before_report_persistence_and_reload():
|
||||
reloader.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sealed_install_rejects_before_package_guard_and_checkpoint() -> None:
|
||||
"""安装事务在封口后不进入监控抑制,也不创建文件快照。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
package_guard = Mock(return_value=nullcontext())
|
||||
checkpointer = AsyncMock()
|
||||
|
||||
result = await _command(
|
||||
mutation=admission.hold,
|
||||
package_write_guard=package_guard,
|
||||
checkpointer=checkpointer,
|
||||
).execute(
|
||||
plugin_id="DemoPlugin",
|
||||
repo_url="https://github.com/demo/plugins",
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert result.failure_stage == "admission"
|
||||
assert "停机阶段" in result.message
|
||||
package_guard.assert_not_called()
|
||||
checkpointer.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_records_completed_install_stages_in_order():
|
||||
"""成功安装在提交文件快照后再执行非关键远程上报。"""
|
||||
|
||||
@@ -98,3 +98,226 @@ def test_lifecycle_records_load_failure_when_loader_returns_no_class():
|
||||
assert result == {"DemoPlugin": PluginRuntimeStatus.LOAD_FAILED}
|
||||
assert running == {}
|
||||
assert statuses["DemoPlugin"] is PluginRuntimeStatus.LOAD_FAILED
|
||||
|
||||
|
||||
def test_quiesce_keeps_instance_and_events_until_finalize():
|
||||
"""第一阶段仅停止插件生产者,事件 handler 和实例留到屏障后释放。"""
|
||||
order: list[str] = []
|
||||
|
||||
class DemoPlugin:
|
||||
"""记录旧插件 ABI hook 调用顺序的测试插件。"""
|
||||
|
||||
plugin_name = "演示插件"
|
||||
plugin_version = "1.0.0"
|
||||
|
||||
def init_plugin(self, _config):
|
||||
"""接受宿主初始化配置。"""
|
||||
|
||||
@staticmethod
|
||||
def get_state():
|
||||
"""保持插件事件 handler 启用。"""
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
"""停止插件私有计时器和 watcher。"""
|
||||
order.append("close")
|
||||
|
||||
def stop_service(self):
|
||||
"""停止插件服务并允许尾事件继续投递。"""
|
||||
order.append("stop_service")
|
||||
|
||||
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
|
||||
lifecycle.start("DemoPlugin")
|
||||
lifecycle._disable_events.reset_mock()
|
||||
lifecycle._clear_modules.reset_mock()
|
||||
lifecycle._clear_tools.reset_mock()
|
||||
|
||||
assert lifecycle.quiesce() is True
|
||||
assert order == ["close", "stop_service"]
|
||||
assert classes["DemoPlugin"] is DemoPlugin
|
||||
assert isinstance(running["DemoPlugin"], DemoPlugin)
|
||||
lifecycle._disable_events.assert_not_called()
|
||||
lifecycle._clear_modules.assert_not_called()
|
||||
lifecycle._clear_tools.assert_not_called()
|
||||
|
||||
assert lifecycle.finalize() is True
|
||||
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
|
||||
lifecycle._clear_modules.assert_called_once_with(None)
|
||||
lifecycle._clear_tools.assert_called_once_with()
|
||||
assert classes == {}
|
||||
assert running == {}
|
||||
|
||||
|
||||
def test_quiesce_runs_stop_service_after_close_failure_and_retries_missing_hook():
|
||||
"""close 异常不能跳过 stop_service,重试时只补偿尚未成功的 hook。"""
|
||||
order: list[str] = []
|
||||
close_fails = [True]
|
||||
|
||||
class DemoPlugin:
|
||||
"""首次 close 失败、stop_service 正常完成的测试插件。"""
|
||||
|
||||
plugin_name = "演示插件"
|
||||
plugin_version = "1.0.0"
|
||||
|
||||
def init_plugin(self, _config):
|
||||
"""接受宿主初始化配置。"""
|
||||
|
||||
@staticmethod
|
||||
def get_state():
|
||||
"""保持插件为启用状态。"""
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
"""首次调用模拟插件私有资源关闭异常。"""
|
||||
order.append("close")
|
||||
if close_fails[0]:
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
def stop_service(self):
|
||||
"""记录旧 ABI 的第二个停机 hook。"""
|
||||
order.append("stop_service")
|
||||
|
||||
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
|
||||
lifecycle.start("DemoPlugin")
|
||||
|
||||
assert lifecycle.quiesce() is False
|
||||
assert order == ["close", "stop_service"]
|
||||
assert lifecycle.finalize() is False
|
||||
assert "DemoPlugin" in classes
|
||||
assert "DemoPlugin" in running
|
||||
|
||||
close_fails[0] = False
|
||||
assert lifecycle.quiesce() is True
|
||||
assert order == ["close", "stop_service", "close"]
|
||||
assert lifecycle.finalize() is True
|
||||
assert classes == {}
|
||||
assert running == {}
|
||||
|
||||
|
||||
def test_legacy_stop_entry_remains_idempotent():
|
||||
"""旧 stop 先解绑事件、保持 None 返回,且重复调用不重复执行 hook。"""
|
||||
order: list[str] = []
|
||||
|
||||
class DemoPlugin:
|
||||
"""提供可计数停机 hook 的兼容测试插件。"""
|
||||
|
||||
plugin_name = "演示插件"
|
||||
plugin_version = "1.0.0"
|
||||
|
||||
def init_plugin(self, _config):
|
||||
"""接受宿主初始化配置。"""
|
||||
|
||||
@staticmethod
|
||||
def get_state():
|
||||
"""保持插件为启用状态。"""
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
"""记录 close 调用。"""
|
||||
order.append("close")
|
||||
|
||||
def stop_service(self):
|
||||
"""记录 stop_service 调用。"""
|
||||
order.append("stop_service")
|
||||
|
||||
lifecycle, _classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
|
||||
lifecycle.start("DemoPlugin")
|
||||
lifecycle._disable_events.reset_mock()
|
||||
lifecycle._disable_events.side_effect = (
|
||||
lambda _plugin_type: order.append("disable_events")
|
||||
)
|
||||
|
||||
assert lifecycle.stop() is None
|
||||
assert lifecycle.stop() is None
|
||||
assert order == ["disable_events", "close", "stop_service"]
|
||||
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
|
||||
assert running == {}
|
||||
|
||||
|
||||
def test_legacy_stop_force_finalizes_after_hook_failure():
|
||||
"""旧 stop 即使 hook 报错也释放实例,并保持历史 None 返回 ABI。"""
|
||||
order: list[str] = []
|
||||
|
||||
class DemoPlugin:
|
||||
"""close 失败但仍应完成兼容卸载的测试插件。"""
|
||||
|
||||
plugin_name = "演示插件"
|
||||
plugin_version = "1.0.0"
|
||||
|
||||
def init_plugin(self, _config):
|
||||
"""接受宿主初始化配置。"""
|
||||
|
||||
@staticmethod
|
||||
def get_state():
|
||||
"""保持插件为启用状态。"""
|
||||
return True
|
||||
|
||||
def close(self):
|
||||
"""模拟旧插件关闭异常。"""
|
||||
order.append("close")
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
def stop_service(self):
|
||||
"""证明 close 异常不会跳过后续 hook。"""
|
||||
order.append("stop_service")
|
||||
|
||||
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
|
||||
lifecycle.start("DemoPlugin")
|
||||
lifecycle._disable_events.reset_mock()
|
||||
lifecycle._clear_modules.reset_mock()
|
||||
|
||||
assert lifecycle.stop("DemoPlugin") is None
|
||||
assert order == ["close", "stop_service"]
|
||||
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
|
||||
lifecycle._clear_modules.assert_called_once_with("DemoPlugin")
|
||||
assert classes == {}
|
||||
assert running == {}
|
||||
|
||||
|
||||
def test_legacy_reload_continues_after_old_instance_hook_failure():
|
||||
"""旧实例 hook 失败只影响诊断,不得阻止 reload 创建新实例。"""
|
||||
instances: list[object] = []
|
||||
stop_service = MagicMock()
|
||||
|
||||
class DemoPlugin:
|
||||
"""记录每次构造并让 close 持续失败的重载测试插件。"""
|
||||
|
||||
plugin_name = "演示插件"
|
||||
plugin_version = "1.0.0"
|
||||
|
||||
def __init__(self):
|
||||
"""记录新建的运行实例。"""
|
||||
instances.append(self)
|
||||
|
||||
def init_plugin(self, _config):
|
||||
"""接受宿主初始化配置。"""
|
||||
|
||||
@staticmethod
|
||||
def get_state():
|
||||
"""保持插件为启用状态。"""
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def close():
|
||||
"""模拟旧实例退出异常。"""
|
||||
raise RuntimeError("close failed")
|
||||
|
||||
def stop_service(self):
|
||||
"""记录旧 ABI 的第二个停机 hook。"""
|
||||
stop_service()
|
||||
|
||||
lifecycle, _classes, running, statuses = _lifecycle(plugins=[DemoPlugin])
|
||||
lifecycle.start("DemoPlugin")
|
||||
previous = running["DemoPlugin"]
|
||||
|
||||
result = lifecycle.reload("DemoPlugin", "plugin-reload")
|
||||
|
||||
assert result is PluginRuntimeStatus.ACTIVE
|
||||
assert statuses["DemoPlugin"] is PluginRuntimeStatus.ACTIVE
|
||||
assert running["DemoPlugin"] is not previous
|
||||
assert len(instances) == 2
|
||||
stop_service.assert_called_once_with()
|
||||
lifecycle._event_sender.assert_called_once_with(
|
||||
"plugin-reload",
|
||||
data={"plugin_id": "DemoPlugin"},
|
||||
)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
@@ -16,6 +17,7 @@ from app.runtime.extensions.plugin.monitor import (
|
||||
PluginChangeMonitor,
|
||||
PluginMonitorController,
|
||||
)
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
from app.runtime.extensions.plugin.system import reset_plugin_system
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
@@ -65,8 +67,9 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No
|
||||
missing_dependencies=("DependencyPending",),
|
||||
missing_source=("SourcePending",),
|
||||
)
|
||||
manager.reopen_plugins.side_effect = lambda: order.append("reopen") or True
|
||||
manager.start.side_effect = lambda plugin_id: order.append(f"plugin:{plugin_id}")
|
||||
manager.start_monitor.side_effect = lambda: order.append("monitor")
|
||||
manager.start_monitor.side_effect = lambda **_kwargs: order.append("monitor")
|
||||
monkeypatch.setattr(
|
||||
plugins_initializer,
|
||||
"configure_plugin_services",
|
||||
@@ -81,8 +84,16 @@ def test_init_plugins_starts_monitor_after_runtime_and_routes(monkeypatch) -> No
|
||||
|
||||
plugins_initializer.init_plugins()
|
||||
|
||||
assert order == ["services", "plugin:ReadyPlugin", "routes", "monitor"]
|
||||
assert order == [
|
||||
"services",
|
||||
"reopen",
|
||||
"plugin:ReadyPlugin",
|
||||
"routes",
|
||||
"monitor",
|
||||
]
|
||||
manager.reopen_plugins.assert_called_once_with()
|
||||
manager.set_plugin_settling.assert_called_once_with(True)
|
||||
manager.start_monitor.assert_called_once_with(reopen=True)
|
||||
|
||||
|
||||
def test_plugin_manager_projects_dependency_classification_to_runtime_status() -> None:
|
||||
@@ -155,6 +166,26 @@ def _patch_sync_plugins(monkeypatch, manager: MagicMock) -> MagicMock:
|
||||
return register
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_plugins_rejects_before_configuring_mutable_services(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""启动后同步在 admission 封口后不重装配服务、不写包或运行态。"""
|
||||
admission = PluginMutationAdmission()
|
||||
admission.seal()
|
||||
manager = MagicMock()
|
||||
manager.mutation.side_effect = admission.hold
|
||||
configure = MagicMock()
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
|
||||
monkeypatch.setattr(plugins_initializer, "configure_plugin_services", configure)
|
||||
|
||||
assert await plugins_initializer.sync_plugins() is False
|
||||
|
||||
configure.assert_not_called()
|
||||
manager.set_plugin_settling.assert_not_called()
|
||||
manager.sync.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_plugins_activates_ready_plugins_when_dependencies_fail(
|
||||
monkeypatch,
|
||||
@@ -392,6 +423,303 @@ def test_plugin_monitor_waits_until_dependency_settlement(monkeypatch) -> None:
|
||||
_reset_plugin_manager()
|
||||
|
||||
|
||||
def test_monitor_stop_keeps_thread_owned_until_it_really_exits() -> None:
|
||||
"""停止超时后保留活线程引用,使后续停机调用仍能继续等待。"""
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def runner() -> None:
|
||||
"""模拟暂时无法响应停止事件的文件监控循环。"""
|
||||
started.set()
|
||||
release.wait()
|
||||
|
||||
controller = PluginMonitorController(runner=runner, log=MagicMock())
|
||||
controller.start()
|
||||
assert started.wait(1)
|
||||
owned_thread = controller._thread
|
||||
|
||||
try:
|
||||
assert controller.stop(timeout=0.01) is False
|
||||
assert controller._thread is owned_thread
|
||||
assert owned_thread is not None and owned_thread.is_alive()
|
||||
finally:
|
||||
release.set()
|
||||
assert controller.stop(timeout=1) is True
|
||||
|
||||
assert controller._thread is None
|
||||
assert controller.stop(timeout=0) is True
|
||||
|
||||
|
||||
def test_monitor_stop_counts_lifecycle_lock_wait_in_timeout() -> None:
|
||||
"""并发重建占用生命周期锁时,停止调用不得突破自身预算。"""
|
||||
controller = PluginMonitorController(runner=lambda: None, log=MagicMock())
|
||||
lock_acquired = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def hold_lifecycle_lock() -> None:
|
||||
"""模拟配置重载在停止请求到达时仍持有生命周期锁。"""
|
||||
with controller._lifecycle_lock:
|
||||
lock_acquired.set()
|
||||
release.wait()
|
||||
|
||||
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
|
||||
holder.start()
|
||||
assert lock_acquired.wait(1)
|
||||
started_at = time.monotonic()
|
||||
|
||||
try:
|
||||
assert controller.stop(timeout=0.01) is False
|
||||
assert time.monotonic() - started_at < 0.2
|
||||
finally:
|
||||
release.set()
|
||||
holder.join(timeout=1)
|
||||
|
||||
assert holder.is_alive() is False
|
||||
assert controller.stop(timeout=0) is True
|
||||
|
||||
|
||||
def test_shutdown_seal_blocks_reload_between_stop_and_start(monkeypatch) -> None:
|
||||
"""停机封口落在热重载停启间隙时,旧重载不得再创建监控线程。"""
|
||||
reload_stopped = threading.Event()
|
||||
resume_reload = threading.Event()
|
||||
runner_started = threading.Event()
|
||||
controller = PluginMonitorController(
|
||||
runner=lambda: runner_started.set(),
|
||||
log=MagicMock(),
|
||||
)
|
||||
original_stop = controller.stop
|
||||
|
||||
def pause_after_stop(timeout: float = 5.0) -> bool:
|
||||
"""把配置热重载稳定暂停在旧线程已停、新线程未启的窗口。"""
|
||||
stopped = original_stop(timeout=timeout)
|
||||
reload_stopped.set()
|
||||
resume_reload.wait()
|
||||
return stopped
|
||||
|
||||
monkeypatch.setattr(controller, "stop", pause_after_stop)
|
||||
reload_thread = threading.Thread(
|
||||
target=lambda: controller.reload(enabled=True),
|
||||
daemon=True,
|
||||
)
|
||||
reload_thread.start()
|
||||
assert reload_stopped.wait(1)
|
||||
|
||||
try:
|
||||
assert controller.close(timeout=1) is True
|
||||
resume_reload.set()
|
||||
reload_thread.join(timeout=1)
|
||||
assert reload_thread.is_alive() is False
|
||||
assert runner_started.is_set() is False
|
||||
assert controller._thread is None
|
||||
finally:
|
||||
resume_reload.set()
|
||||
reload_thread.join(timeout=1)
|
||||
|
||||
assert controller.reopen() is True
|
||||
controller.start()
|
||||
assert runner_started.wait(1)
|
||||
assert controller.close(timeout=1) is True
|
||||
|
||||
|
||||
def test_plugin_manager_stop_monitor_returns_controller_result(monkeypatch) -> None:
|
||||
"""管理器停止入口透传线程收口结果和调用预算。"""
|
||||
_reset_plugin_manager()
|
||||
reset_plugin_system()
|
||||
manager = PluginManager()
|
||||
stop = MagicMock(return_value=False)
|
||||
manager._plugin_monitor.stop = stop
|
||||
|
||||
try:
|
||||
assert manager.stop_monitor(timeout=0.25) is False
|
||||
stop.assert_called_once_with(timeout=0.25)
|
||||
finally:
|
||||
_reset_plugin_manager()
|
||||
|
||||
|
||||
def test_plugin_manager_start_monitor_can_reopen_new_lifespan(monkeypatch) -> None:
|
||||
"""新应用生命周期可显式解除封口,再按运行配置启动监控。"""
|
||||
_reset_plugin_manager()
|
||||
reset_plugin_system()
|
||||
monkeypatch.setattr(
|
||||
"app.runtime.extensions.plugin_manager.settings",
|
||||
SimpleNamespace(
|
||||
DEV=True,
|
||||
PLUGIN_AUTO_RELOAD=False,
|
||||
ROOT_PATH=MagicMock(),
|
||||
),
|
||||
)
|
||||
manager = PluginManager()
|
||||
reopen = MagicMock(return_value=True)
|
||||
start = MagicMock()
|
||||
manager._plugin_monitor.reopen = reopen
|
||||
manager._plugin_monitor.start = start
|
||||
|
||||
try:
|
||||
manager.start_monitor(reopen=True)
|
||||
reopen.assert_called_once_with()
|
||||
start.assert_called_once_with()
|
||||
finally:
|
||||
_reset_plugin_manager()
|
||||
|
||||
|
||||
def test_stop_plugin_monitor_does_not_materialize_manager() -> None:
|
||||
"""插件运行时尚未创建时,独立停机入口直接视为已完成。"""
|
||||
_reset_plugin_manager()
|
||||
|
||||
assert plugins_initializer.stop_plugin_monitor(timeout=0) is True
|
||||
assert PluginManager.get_existing_instance() is None
|
||||
|
||||
|
||||
def test_stop_plugin_monitor_returns_existing_manager_result(monkeypatch) -> None:
|
||||
"""启动层入口只操作既有管理器,并透传超时失败。"""
|
||||
manager = MagicMock()
|
||||
manager.close_monitor.return_value = False
|
||||
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
|
||||
|
||||
assert plugins_initializer.stop_plugin_monitor(timeout=0.25) is False
|
||||
manager.close_monitor.assert_called_once_with(timeout=0.25)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_phase_plugin_shutdown_does_not_materialize_manager() -> None:
|
||||
"""两阶段入口在插件管理器尚未创建时都直接视为已收敛。"""
|
||||
_reset_plugin_manager()
|
||||
|
||||
assert await plugins_initializer.quiesce_plugins(timeout=0) is True
|
||||
assert plugins_initializer.finalize_plugins() is True
|
||||
assert PluginManager.get_existing_instance() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quiesce_timeout_retains_future_owner_until_worker_finishes(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""同步插件 hook 超时后必须保留 Future,且未结束前拒绝卸载实例。"""
|
||||
_reset_plugin_manager()
|
||||
reset_plugin_system()
|
||||
monkeypatch.setattr(
|
||||
"app.runtime.extensions.plugin_manager.settings",
|
||||
SimpleNamespace(
|
||||
DEV=False,
|
||||
PLUGIN_AUTO_RELOAD=False,
|
||||
ROOT_PATH=MagicMock(),
|
||||
),
|
||||
)
|
||||
manager = PluginManager()
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def blocking_quiesce() -> bool:
|
||||
"""模拟无法由 asyncio 取消的同步旧插件 hook。"""
|
||||
started.set()
|
||||
release.wait(timeout=2)
|
||||
return True
|
||||
|
||||
manager._plugin_lifecycle.quiesce = MagicMock(side_effect=blocking_quiesce)
|
||||
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
thread_helper = SimpleNamespace(submit=executor.submit)
|
||||
monkeypatch.setattr(
|
||||
"app.runtime.extensions.plugin_manager.ThreadHelper",
|
||||
lambda: thread_helper,
|
||||
)
|
||||
|
||||
assert await manager.quiesce_plugins(timeout=0.01) is False
|
||||
assert started.is_set()
|
||||
owner = manager._plugin_quiesce_future
|
||||
assert owner is not None
|
||||
assert owner.done() is False
|
||||
assert manager.finalize_plugins() is False
|
||||
manager._plugin_lifecycle.finalize.assert_not_called()
|
||||
|
||||
release.set()
|
||||
assert await asyncio.wrap_future(owner) is True
|
||||
assert manager.finalize_plugins() is True
|
||||
manager._plugin_lifecycle.finalize.assert_called_once_with()
|
||||
finally:
|
||||
release.set()
|
||||
_reset_plugin_manager()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quiesce_seals_runtime_until_new_lifespan_reopens(monkeypatch) -> None:
|
||||
"""屏障前封口后 start/reload/config 不能重开 producer,新 lifespan 可显式恢复。"""
|
||||
_reset_plugin_manager()
|
||||
reset_plugin_system()
|
||||
monkeypatch.setattr(
|
||||
"app.runtime.extensions.plugin_manager.settings",
|
||||
SimpleNamespace(
|
||||
DEBUG=False,
|
||||
DEV=False,
|
||||
PLUGIN_AUTO_RELOAD=False,
|
||||
ROOT_PATH=MagicMock(),
|
||||
),
|
||||
)
|
||||
manager = PluginManager()
|
||||
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
|
||||
manager._plugin_lifecycle.start = MagicMock(
|
||||
return_value={"DemoPlugin": PluginRuntimeStatus.ACTIVE}
|
||||
)
|
||||
manager._plugin_lifecycle.reload = MagicMock(
|
||||
return_value=PluginRuntimeStatus.ACTIVE
|
||||
)
|
||||
manager._plugin_lifecycle.stop = MagicMock(return_value=True)
|
||||
manager._plugin_lifecycle.initialize = MagicMock()
|
||||
manager._plugin_lifecycle._disable_events = MagicMock()
|
||||
manager._plugin_registry.remove = MagicMock()
|
||||
manager._plugin_registry.set_runtime_status = MagicMock()
|
||||
manager.classify_plugins = MagicMock()
|
||||
|
||||
class DemoPlugin:
|
||||
"""代表 quiesce 后仍由严格生命周期持有的运行实例。"""
|
||||
|
||||
plugin_instance = DemoPlugin()
|
||||
manager._plugins["DemoPlugin"] = DemoPlugin
|
||||
manager._running_plugins["DemoPlugin"] = plugin_instance
|
||||
|
||||
try:
|
||||
assert await manager.quiesce_plugins(timeout=1) is True
|
||||
|
||||
assert manager.start("DemoPlugin") == {
|
||||
"DemoPlugin": PluginRuntimeStatus.LOAD_FAILED
|
||||
}
|
||||
assert manager.stop("DemoPlugin") is None
|
||||
assert manager.remove_plugin("DemoPlugin") is None
|
||||
assert (
|
||||
manager.reload_plugin("DemoPlugin")
|
||||
is PluginRuntimeStatus.LOAD_FAILED
|
||||
)
|
||||
manager.init_plugin("DemoPlugin", {})
|
||||
manager.init_config()
|
||||
manager._plugin_lifecycle.start.assert_not_called()
|
||||
manager._plugin_lifecycle.stop.assert_not_called()
|
||||
manager._plugin_lifecycle.reload.assert_not_called()
|
||||
manager._plugin_lifecycle.initialize.assert_not_called()
|
||||
manager._plugin_lifecycle._disable_events.assert_not_called()
|
||||
manager._plugin_registry.remove.assert_not_called()
|
||||
manager._plugin_registry.set_runtime_status.assert_not_called()
|
||||
manager.classify_plugins.assert_not_called()
|
||||
assert manager._plugins["DemoPlugin"] is DemoPlugin
|
||||
assert manager._running_plugins["DemoPlugin"] is plugin_instance
|
||||
|
||||
assert manager.reopen_plugins() is False
|
||||
manager._plugins.clear()
|
||||
manager._running_plugins.clear()
|
||||
assert manager.reopen_plugins() is True
|
||||
assert manager.start("DemoPlugin") == {
|
||||
"DemoPlugin": PluginRuntimeStatus.ACTIVE
|
||||
}
|
||||
assert (
|
||||
manager.reload_plugin("DemoPlugin")
|
||||
is PluginRuntimeStatus.ACTIVE
|
||||
)
|
||||
finally:
|
||||
_reset_plugin_manager()
|
||||
|
||||
|
||||
def test_plugin_monitor_skips_installing_plugin_until_package_write_finishes(tmp_path) -> None:
|
||||
"""安装替换目录期间,文件事件不得抢先导入未完成的插件包。"""
|
||||
reload_plugin = MagicMock()
|
||||
@@ -466,11 +794,12 @@ def test_stop_plugins_stops_monitor_before_plugin_runtime(monkeypatch) -> None:
|
||||
"""关闭时先隔离文件变化,再停止插件实例。"""
|
||||
order: list[str] = []
|
||||
manager = MagicMock()
|
||||
manager.stop_monitor.side_effect = lambda: order.append("monitor")
|
||||
manager.stop_monitor.side_effect = lambda: order.append("monitor") or True
|
||||
manager.stop.side_effect = lambda: order.append("plugins")
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
|
||||
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
|
||||
|
||||
plugins_initializer.stop_plugins()
|
||||
assert plugins_initializer.stop_plugins() is True
|
||||
|
||||
assert order == ["monitor", "plugins"]
|
||||
|
||||
@@ -479,8 +808,89 @@ def test_stop_plugins_still_stops_runtime_when_monitor_stop_fails(monkeypatch) -
|
||||
"""监控线程停止异常不得阻止插件实例释放资源。"""
|
||||
manager = MagicMock()
|
||||
manager.stop_monitor.side_effect = RuntimeError("monitor stop failed")
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", lambda: manager)
|
||||
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
|
||||
|
||||
plugins_initializer.stop_plugins()
|
||||
assert plugins_initializer.stop_plugins() is False
|
||||
|
||||
manager.stop.assert_called_once_with()
|
||||
|
||||
|
||||
def test_stop_plugins_does_not_materialize_manager() -> None:
|
||||
"""插件初始化尚未创建管理器时,失败清理不得反向构造运行时。"""
|
||||
_reset_plugin_manager()
|
||||
|
||||
assert plugins_initializer.stop_plugins() is True
|
||||
|
||||
assert PluginManager.get_existing_instance() is None
|
||||
|
||||
|
||||
def test_stop_plugins_remains_idempotent(monkeypatch) -> None:
|
||||
"""兼容停机入口可重复调用,并保持每次先停监控再停插件。"""
|
||||
order: list[str] = []
|
||||
manager = MagicMock()
|
||||
manager.stop_monitor.side_effect = lambda: order.append("monitor") or True
|
||||
manager.stop.side_effect = lambda: order.append("plugins")
|
||||
manager_type = SimpleNamespace(get_existing_instance=lambda: manager)
|
||||
monkeypatch.setattr(plugins_initializer, "PluginManager", manager_type)
|
||||
|
||||
assert plugins_initializer.stop_plugins() is True
|
||||
assert plugins_initializer.stop_plugins() is True
|
||||
|
||||
assert order == ["monitor", "plugins", "monitor", "plugins"]
|
||||
|
||||
|
||||
def test_plugin_manager_legacy_stop_preserves_none_return() -> None:
|
||||
"""公共 PluginManager.stop 委托单阶段停机后保持历史 None 返回 ABI。"""
|
||||
manager = object.__new__(PluginManager)
|
||||
manager._plugin_quiesce_lock = threading.RLock()
|
||||
manager._plugin_runtime_closed = False
|
||||
manager._plugin_quiesce_future = None
|
||||
manager._plugin_mutation_admission = PluginMutationAdmission()
|
||||
manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=True))
|
||||
|
||||
assert PluginManager.stop(manager, "DemoPlugin") is None
|
||||
|
||||
manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin")
|
||||
|
||||
|
||||
def test_config_reload_continues_after_legacy_stop() -> None:
|
||||
"""配置热重载保持旧行为,stop 的 None 返回不得阻止重新分类和启动。"""
|
||||
manager = object.__new__(PluginManager)
|
||||
manager._plugin_quiesce_lock = threading.RLock()
|
||||
manager._plugin_runtime_closed = False
|
||||
manager._plugin_mutation_admission = PluginMutationAdmission()
|
||||
manager.stop = MagicMock(return_value=None)
|
||||
manager.classify_plugins = MagicMock(
|
||||
return_value=PluginDependencyClassification(
|
||||
ready=("ReadyPlugin",),
|
||||
missing_dependencies=(),
|
||||
missing_source=(),
|
||||
)
|
||||
)
|
||||
manager.apply_plugin_dependency_classification = MagicMock()
|
||||
manager.start = MagicMock()
|
||||
|
||||
PluginManager.init_config(manager)
|
||||
|
||||
manager.stop.assert_called_once_with()
|
||||
manager.classify_plugins.assert_called_once_with()
|
||||
manager.apply_plugin_dependency_classification.assert_called_once_with(
|
||||
manager.classify_plugins.return_value
|
||||
)
|
||||
manager.start.assert_called_once_with("ReadyPlugin")
|
||||
|
||||
|
||||
def test_remove_plugin_clears_registry_after_legacy_stop() -> None:
|
||||
"""卸载路径保持忽略旧 stop 返回值并继续清理注册表。"""
|
||||
manager = object.__new__(PluginManager)
|
||||
manager._plugin_quiesce_lock = threading.RLock()
|
||||
manager._plugin_runtime_closed = False
|
||||
manager._plugin_mutation_admission = PluginMutationAdmission()
|
||||
manager._plugin_lifecycle = SimpleNamespace(stop=MagicMock(return_value=None))
|
||||
manager._plugin_registry = SimpleNamespace(remove=MagicMock())
|
||||
|
||||
PluginManager.remove_plugin(manager, "DemoPlugin")
|
||||
|
||||
manager._plugin_lifecycle.stop.assert_called_once_with("DemoPlugin")
|
||||
manager._plugin_registry.remove.assert_called_once_with("DemoPlugin")
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""插件可变事务停机准入的确定性测试。"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextvars import copy_context
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.extensions.plugin.admission import (
|
||||
PluginMutationAdmission,
|
||||
PluginMutationRejectedError,
|
||||
)
|
||||
from app.runtime.extensions.plugin.system import reset_plugin_system
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plugin_manager() -> Iterator[PluginManager]:
|
||||
"""构造隔离的插件管理器,并在用例结束后清除单例状态。"""
|
||||
key = (PluginManager, (), frozenset())
|
||||
Singleton._instances.pop(key, None)
|
||||
reset_plugin_system()
|
||||
manager = PluginManager()
|
||||
try:
|
||||
yield manager
|
||||
finally:
|
||||
Singleton._instances.pop(key, None)
|
||||
|
||||
|
||||
def test_seal_rejects_new_root_but_allows_propagated_nested_lease() -> None:
|
||||
"""封口拒绝无关调用,但已获准事务跨线程后的嵌套仍能完成。"""
|
||||
admission = PluginMutationAdmission()
|
||||
calls: list[str] = []
|
||||
|
||||
def mutate(label: str) -> None:
|
||||
"""在测试线程中尝试取得 lease 并记录副作用。"""
|
||||
with admission.hold(label):
|
||||
calls.append(label)
|
||||
|
||||
with admission.hold("外层事务"):
|
||||
propagated = copy_context()
|
||||
assert admission.active_count == 1
|
||||
assert admission.seal() == 1
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
executor.submit(propagated.run, mutate, "嵌套事务").result(timeout=1)
|
||||
rejected = executor.submit(mutate, "新事务")
|
||||
with pytest.raises(PluginMutationRejectedError):
|
||||
rejected.result(timeout=1)
|
||||
|
||||
assert calls == ["嵌套事务"]
|
||||
assert admission.active_count == 1
|
||||
|
||||
assert admission.active_count == 0
|
||||
assert admission.reopen() is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quiesce_timeout_retains_admitted_owner_and_nested_reload(
|
||||
plugin_manager: PluginManager,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""停机超时保留 active owner,且封口前获准的事务可完成内部重载。"""
|
||||
manager = plugin_manager
|
||||
entered = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
manager._plugin_lifecycle.reload = MagicMock(
|
||||
return_value=PluginRuntimeStatus.ACTIVE
|
||||
)
|
||||
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
|
||||
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
|
||||
|
||||
async def mutate() -> PluginRuntimeStatus:
|
||||
"""持有外层 lease,等待封口后再调用嵌套 Manager 写入口。"""
|
||||
with manager.mutation("安装插件 DemoPlugin"):
|
||||
entered.set()
|
||||
await release.wait()
|
||||
return manager.reload_plugin("DemoPlugin")
|
||||
|
||||
with ThreadPoolExecutor(max_workers=1) as executor:
|
||||
monkeypatch.setattr(
|
||||
"app.runtime.extensions.plugin_manager.ThreadHelper",
|
||||
lambda: SimpleNamespace(submit=executor.submit),
|
||||
)
|
||||
mutation_task = asyncio.create_task(mutate())
|
||||
await entered.wait()
|
||||
|
||||
assert await manager.quiesce_plugins(timeout=0.01) is False
|
||||
owner = manager._plugin_quiesce_future
|
||||
assert owner is not None
|
||||
assert owner.done() is False
|
||||
assert manager._plugin_mutation_admission.active_count == 1
|
||||
assert manager.finalize_plugins() is False
|
||||
with pytest.raises(PluginMutationRejectedError):
|
||||
with manager.mutation("新的配置写入"):
|
||||
pass
|
||||
|
||||
release.set()
|
||||
assert await mutation_task is PluginRuntimeStatus.ACTIVE
|
||||
assert await asyncio.wrap_future(owner) is True
|
||||
|
||||
manager._plugin_lifecycle.reload.assert_called_once_with(
|
||||
"DemoPlugin",
|
||||
EventType.PluginReload,
|
||||
)
|
||||
assert manager._plugin_mutation_admission.active_count == 0
|
||||
assert manager.finalize_plugins() is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_quiesce_inside_mutation_fails_without_sealing(
|
||||
plugin_manager: PluginManager,
|
||||
) -> None:
|
||||
"""事务不能等待自身退出,快速失败时也不得误封口运行时。"""
|
||||
with plugin_manager.mutation("配置插件"):
|
||||
assert await plugin_manager.quiesce_plugins(timeout=0) is False
|
||||
assert plugin_manager._plugin_mutation_admission.accepting is True
|
||||
assert plugin_manager._plugin_runtime_closed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sealed_manager_blocks_direct_persistent_mutations(
|
||||
plugin_manager: PluginManager,
|
||||
) -> None:
|
||||
"""Manager 单项兼容入口在封口后返回既有失败形状且不触发底层副作用。"""
|
||||
plugin_manager._plugin_config_store = MagicMock()
|
||||
plugin_manager._plugin_instance_store = MagicMock()
|
||||
plugin_manager._plugin_sync = MagicMock()
|
||||
plugin_manager._plugin_clone = MagicMock()
|
||||
plugin_manager._plugin_mutation_admission.seal()
|
||||
plugin_manager._plugin_runtime_closed = True
|
||||
|
||||
assert plugin_manager.save_plugin_config("DemoPlugin", {}) is False
|
||||
assert await plugin_manager.async_save_plugin_config("DemoPlugin", {}) is False
|
||||
assert plugin_manager.delete_plugin_config("DemoPlugin") is False
|
||||
assert plugin_manager.delete_plugin_data("DemoPlugin") is False
|
||||
assert plugin_manager.delete_plugin_instance("DemoPlugin") is False
|
||||
clone_result = plugin_manager.clone_plugin("DemoPlugin", "Work", "", "")
|
||||
assert clone_result[0] is False
|
||||
assert "停机阶段" in clone_result[1]
|
||||
with pytest.raises(PluginMutationRejectedError):
|
||||
plugin_manager.sync()
|
||||
|
||||
plugin_manager._plugin_config_store.write.assert_not_called()
|
||||
plugin_manager._plugin_config_store.async_write.assert_not_called()
|
||||
plugin_manager._plugin_config_store.delete.assert_not_called()
|
||||
plugin_manager._plugin_config_store.delete_data.assert_not_called()
|
||||
plugin_manager._plugin_instance_store.delete.assert_not_called()
|
||||
plugin_manager._plugin_sync.sync.assert_not_called()
|
||||
plugin_manager._plugin_clone.clone.assert_not_called()
|
||||
@@ -1,3 +1,5 @@
|
||||
import pytest
|
||||
|
||||
from app.foundation.singleton import Singleton, SingletonClass
|
||||
|
||||
|
||||
@@ -27,3 +29,49 @@ def test_parameterized_singleton_can_read_matching_instance_without_creating(mon
|
||||
instance = Example("first")
|
||||
assert Example.get_existing_instance("first") is instance
|
||||
assert Example.get_existing_instance("second") is None
|
||||
|
||||
|
||||
def test_parameterized_singleton_can_retain_failed_lifecycle_owner(monkeypatch):
|
||||
"""构造中途失败时,可选 owner 单例必须仍能由启动失败清理读取。"""
|
||||
|
||||
class Example(metaclass=Singleton):
|
||||
"""模拟在构造期已创建后台 owner 后抛错的参数化单例。"""
|
||||
|
||||
_retain_failed_singleton = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""发布可观察 owner 后模拟后续启动失败。"""
|
||||
self.owner_started = True
|
||||
raise RuntimeError("startup failed")
|
||||
|
||||
monkeypatch.setattr(Singleton, "_instances", {})
|
||||
|
||||
with pytest.raises(RuntimeError, match="startup failed"):
|
||||
Example()
|
||||
|
||||
retained = Example.get_existing_instance()
|
||||
assert retained is not None
|
||||
assert retained.owner_started is True
|
||||
|
||||
|
||||
def test_class_singleton_can_retain_failed_lifecycle_owner(monkeypatch):
|
||||
"""按类 owner 单例也必须在 __init__ 抛错后保留可清理身份。"""
|
||||
|
||||
class Example(metaclass=SingletonClass):
|
||||
"""模拟目录监控构造中途失败的按类单例。"""
|
||||
|
||||
_retain_failed_singleton = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""发布可观察 owner 后模拟后续启动失败。"""
|
||||
self.owner_started = True
|
||||
raise RuntimeError("startup failed")
|
||||
|
||||
monkeypatch.setattr(SingletonClass, "_instances", {})
|
||||
|
||||
with pytest.raises(RuntimeError, match="startup failed"):
|
||||
Example()
|
||||
|
||||
retained = Example.get_existing_instance()
|
||||
assert retained is not None
|
||||
assert retained.owner_started is True
|
||||
|
||||
@@ -50,7 +50,7 @@ def test_task_registry_cancels_tasks_and_rejects_late_registration() -> None:
|
||||
|
||||
task = registry.create(worker(), owner="test.shutdown")
|
||||
await started.wait()
|
||||
await registry.shutdown(timeout_seconds=1.0)
|
||||
assert await registry.shutdown(timeout_seconds=1.0) is True
|
||||
|
||||
assert task.cancelled()
|
||||
assert cancelled.is_set()
|
||||
@@ -108,7 +108,7 @@ def test_task_registry_keeps_timed_out_sync_owner_until_real_completion() -> Non
|
||||
task = registry.create_sync(worker, owner="test.sync-timeout")
|
||||
assert await asyncio.to_thread(started.wait, 1.0)
|
||||
|
||||
await registry.shutdown(timeout_seconds=0.001)
|
||||
assert await registry.shutdown(timeout_seconds=0.001) is False
|
||||
|
||||
assert not task.done()
|
||||
assert [record.owner for record in registry.records] == [
|
||||
@@ -155,7 +155,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None:
|
||||
task = registry.create(worker(), owner="test.stubborn")
|
||||
await started.wait()
|
||||
try:
|
||||
await registry.shutdown(timeout_seconds=0.001)
|
||||
assert await registry.shutdown(timeout_seconds=0.001) is False
|
||||
|
||||
assert cleanup_started.is_set()
|
||||
assert not task.done()
|
||||
@@ -165,7 +165,7 @@ def test_task_registry_keeps_stubborn_cancelled_task_visible() -> None:
|
||||
]
|
||||
assert reports[-1]["owners"] == ("test.stubborn",)
|
||||
|
||||
await registry.shutdown(timeout_seconds=0.001)
|
||||
assert await registry.shutdown(timeout_seconds=0.001) is False
|
||||
assert not task.done()
|
||||
assert cancellation_count == 1
|
||||
assert len(reports) == 1
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""失败整理 AI 重试调度器的生命周期测试。"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.application.transfer import FailedRetryScheduler
|
||||
|
||||
|
||||
def test_retry_scheduler_close_cancels_buffered_timer_and_rejects_new_work():
|
||||
"""关闭应取消尚未触发的 timer、清空缓冲并拒绝新增记录。"""
|
||||
|
||||
async def exercise() -> None:
|
||||
"""在独立事件循环内验证 timer 与关闭状态。"""
|
||||
scheduler = FailedRetryScheduler()
|
||||
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 60
|
||||
await scheduler.schedule_retry(11, group_key="media:test")
|
||||
timer = scheduler._retry_transfer_timers["media:test"]
|
||||
|
||||
await scheduler.close()
|
||||
await scheduler.close()
|
||||
|
||||
assert timer.cancelled()
|
||||
assert scheduler._retry_transfer_buffer == {}
|
||||
assert scheduler._retry_transfer_timers == {}
|
||||
with pytest.raises(RuntimeError, match="正在关闭"):
|
||||
await scheduler.schedule_retry(12, group_key="media:test")
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_retry_scheduler_close_cancels_and_waits_for_active_flush_task():
|
||||
"""关闭返回前应等待已经启动的 flush 任务完成取消收尾。"""
|
||||
|
||||
async def exercise() -> None:
|
||||
"""启动一个不会自行结束的 flush,并通过关闭流程取消它。"""
|
||||
scheduler = FailedRetryScheduler()
|
||||
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
|
||||
started = asyncio.Event()
|
||||
stopped = asyncio.Event()
|
||||
|
||||
async def blocking_flush(_group_key: str, _generation: int) -> None:
|
||||
"""等待取消信号,并在 finally 中证明收尾已经完成。"""
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
stopped.set()
|
||||
|
||||
scheduler._flush_retry_transfer = blocking_flush
|
||||
await scheduler.schedule_retry(11, group_key="media:test")
|
||||
await asyncio.wait_for(started.wait(), timeout=1)
|
||||
task = next(iter(scheduler._retry_transfer_tasks))
|
||||
|
||||
await scheduler.close()
|
||||
|
||||
assert stopped.is_set()
|
||||
assert task.cancelled()
|
||||
assert task.get_name() == "transfer.failed_retry.flush"
|
||||
assert scheduler._retry_transfer_tasks == set()
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_retry_scheduler_observes_unexpected_background_task_error():
|
||||
"""flush 协程越过自身防线的异常仍应由任务 owner 统一观察。"""
|
||||
|
||||
async def exercise() -> None:
|
||||
"""让受管 flush 任务直接失败,并等待完成回调处理异常。"""
|
||||
scheduler = FailedRetryScheduler()
|
||||
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
|
||||
|
||||
async def failing_flush(_group_key: str, _generation: int) -> None:
|
||||
"""模拟 flush 外层出现未处理异常。"""
|
||||
raise RuntimeError("flush failed")
|
||||
|
||||
scheduler._flush_retry_transfer = failing_flush
|
||||
with patch("app.application.transfer.logger.error", Mock()) as log_error:
|
||||
await scheduler.schedule_retry(11, group_key="media:test")
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
if scheduler._retry_transfer_tasks:
|
||||
break
|
||||
assert scheduler._retry_transfer_tasks
|
||||
for _ in range(5):
|
||||
await asyncio.sleep(0)
|
||||
if not scheduler._retry_transfer_tasks:
|
||||
break
|
||||
|
||||
assert scheduler._retry_transfer_tasks == set()
|
||||
log_error.assert_called_once()
|
||||
assert "flush failed" in log_error.call_args.args[0]
|
||||
await scheduler.close()
|
||||
|
||||
asyncio.run(exercise())
|
||||
|
||||
|
||||
def test_retry_scheduler_old_flush_cannot_consume_renewed_generation():
|
||||
"""旧 timer 已建 task 后的新失败应续期,不能被旧 flush 提前取走。"""
|
||||
|
||||
async def exercise() -> None:
|
||||
"""稳定复现 timer callback 与同组新 schedule 交错的窗口。"""
|
||||
scheduler = FailedRetryScheduler()
|
||||
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 3600
|
||||
await scheduler.schedule_retry(11, group_key="media:test")
|
||||
old_timer = scheduler._retry_transfer_timers["media:test"]
|
||||
old_generation = scheduler._retry_transfer_generations["media:test"]
|
||||
|
||||
# 模拟旧 timer callback 已进入事件循环,但 flush task 尚未取得分组锁。
|
||||
scheduler._start_retry_transfer_task("media:test", old_generation)
|
||||
await scheduler.schedule_retry(12, group_key="media:test")
|
||||
renewed_timer = scheduler._retry_transfer_timers["media:test"]
|
||||
await asyncio.sleep(0)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert old_timer.cancelled()
|
||||
assert renewed_timer.cancelled() is False
|
||||
assert scheduler._retry_transfer_buffer["media:test"] == [11, 12]
|
||||
assert scheduler._retry_transfer_timers["media:test"] is renewed_timer
|
||||
assert scheduler._retry_transfer_tasks == set()
|
||||
|
||||
await scheduler.close()
|
||||
assert renewed_timer.cancelled()
|
||||
|
||||
asyncio.run(exercise())
|
||||
@@ -1,6 +1,8 @@
|
||||
from unittest.mock import Mock, patch
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.chain import transfer as transfer_module
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.application.transfer import (
|
||||
@@ -59,6 +61,26 @@ class _Loop:
|
||||
return False
|
||||
|
||||
|
||||
class _DeferredLoop(_Loop):
|
||||
"""延迟执行线程安全回调,用于覆盖关闭与入环之间的竞态。"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化延迟回调和定时器清单。"""
|
||||
super().__init__()
|
||||
self.soon_callbacks = []
|
||||
|
||||
def call_soon_threadsafe(self, callback, *args):
|
||||
"""保存线程安全回调,直到测试显式执行。"""
|
||||
self.soon_callbacks.append((callback, args))
|
||||
|
||||
def run_soon_callbacks(self):
|
||||
"""执行并清空已保存的线程安全回调。"""
|
||||
callbacks = list(self.soon_callbacks)
|
||||
self.soon_callbacks.clear()
|
||||
for callback, args in callbacks:
|
||||
callback(*args)
|
||||
|
||||
|
||||
def _task(*, episode: int, download_hash: str = "hash-1") -> TransferTask:
|
||||
"""构造同一媒体不同剧集的整理任务。"""
|
||||
return TransferTask(
|
||||
@@ -122,6 +144,116 @@ def test_aggregator_debounces_same_group_and_flushes_once():
|
||||
callback.assert_called_once_with(notices)
|
||||
|
||||
|
||||
def test_aggregator_old_timer_cannot_flush_before_renewal_is_armed():
|
||||
"""新通知已接收时,旧 timer 不得抢在事件循环重置静默窗前发送。"""
|
||||
loop = _DeferredLoop()
|
||||
aggregator = TransferFailureNotificationAggregator()
|
||||
callback = Mock()
|
||||
first = TransferFailureNotification(
|
||||
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
|
||||
)
|
||||
second = TransferFailureNotification(
|
||||
"测试剧 (2026)", "S01E02", "原因B", 2, None, "tester"
|
||||
)
|
||||
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=first,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
loop.run_soon_callbacks()
|
||||
old_timer = loop.timers[0]
|
||||
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=second,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
old_timer.callback(*old_timer.args)
|
||||
callback.assert_not_called()
|
||||
|
||||
loop.run_soon_callbacks()
|
||||
renewed_timer = loop.timers[1]
|
||||
assert old_timer.cancelled is True
|
||||
renewed_timer.callback(*renewed_timer.args)
|
||||
callback.assert_called_once_with([first, second])
|
||||
|
||||
|
||||
def test_aggregator_close_flushes_accepted_notification_before_timer_is_armed():
|
||||
"""关闭应发送已接收但尚未进入事件循环的通知,且延迟回调不能重新建 timer。"""
|
||||
loop = _DeferredLoop()
|
||||
aggregator = TransferFailureNotificationAggregator()
|
||||
callback = Mock()
|
||||
notice = TransferFailureNotification(
|
||||
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
|
||||
)
|
||||
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=notice,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
aggregator.close()
|
||||
aggregator.close()
|
||||
loop.run_soon_callbacks()
|
||||
|
||||
callback.assert_called_once_with([notice])
|
||||
assert loop.timers == []
|
||||
|
||||
|
||||
def test_aggregator_close_cancels_timer_and_rejects_new_notification():
|
||||
"""关闭应取消已建 timer,并让调用方明确感知后续投递被拒绝。"""
|
||||
loop = _Loop()
|
||||
aggregator = TransferFailureNotificationAggregator()
|
||||
callback = Mock()
|
||||
notice = TransferFailureNotification(
|
||||
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
|
||||
)
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=notice,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
|
||||
aggregator.close()
|
||||
|
||||
assert loop.timers[0].cancelled is True
|
||||
callback.assert_called_once_with([notice])
|
||||
with pytest.raises(RuntimeError, match="正在关闭"):
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=notice,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
|
||||
|
||||
def test_aggregator_close_observes_flush_callback_error():
|
||||
"""关闭阶段同步刷新失败时应记录异常而不是让通知静默丢失。"""
|
||||
loop = _DeferredLoop()
|
||||
aggregator = TransferFailureNotificationAggregator()
|
||||
callback = Mock(side_effect=RuntimeError("send failed"))
|
||||
notice = TransferFailureNotification(
|
||||
"测试剧 (2026)", "S01E01", "原因A", 1, None, "tester"
|
||||
)
|
||||
aggregator.schedule(
|
||||
group_key="media:test",
|
||||
notification=notice,
|
||||
callback=callback,
|
||||
loop=loop,
|
||||
)
|
||||
|
||||
with patch("app.application.transfer.logger.error") as log_error:
|
||||
aggregator.close()
|
||||
|
||||
callback.assert_called_once_with([notice])
|
||||
log_error.assert_called_once()
|
||||
|
||||
|
||||
def test_aggregated_message_contains_count_reason_stats_and_batch_entry():
|
||||
"""聚合消息应给出失败数、原因统计、历史 ID 和批量处理入口。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
|
||||
@@ -8,11 +8,12 @@
|
||||
这些测试固定三项不变量:入队即落盘登记、终态即注销、重启能回放。
|
||||
"""
|
||||
from pathlib import Path
|
||||
import threading
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.schemas import FileItem
|
||||
from app.application.transfer import TransferTask
|
||||
from app.schemas.file import FileItem
|
||||
|
||||
|
||||
def _build_chain(pendingoper) -> TransferChain:
|
||||
@@ -231,3 +232,32 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
assert handled == ["B.mkv"]
|
||||
|
||||
|
||||
def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
|
||||
"""
|
||||
宿主关闭后不得继续检查或注销下一条登记,未处理项留给下次启动回放。
|
||||
"""
|
||||
first = tmp_path / "A.mkv"
|
||||
first.write_bytes(b"x")
|
||||
missing_second = tmp_path / "gone.mkv"
|
||||
pendingoper = MagicMock()
|
||||
pendingoper.list_all.return_value = [
|
||||
("local", str(first)),
|
||||
("local", str(missing_second)),
|
||||
]
|
||||
chain = _build_chain(pendingoper)
|
||||
stop_event = threading.Event()
|
||||
transferred = []
|
||||
|
||||
def transfer_first(**kwargs):
|
||||
"""首条回放送入整理链后模拟宿主发出关闭信号。"""
|
||||
transferred.append(kwargs["fileitem"].path)
|
||||
stop_event.set()
|
||||
|
||||
monkeypatch.setattr(chain, "do_transfer", transfer_first)
|
||||
|
||||
chain._TransferChain__replay_pending(stop_event)
|
||||
|
||||
assert transferred == [first.as_posix()]
|
||||
pendingoper.discard.assert_not_called()
|
||||
|
||||
@@ -0,0 +1,485 @@
|
||||
"""文件整理 worker 与 pending 回放的宿主生命周期测试。"""
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.config import global_vars
|
||||
from app.application.transfer import TransferQueue, TransferTask
|
||||
from app.schemas.file import FileItem
|
||||
from app.startup import transfer_initializer
|
||||
|
||||
|
||||
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
|
||||
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads)
|
||||
chain._queue = queue.Queue()
|
||||
chain._transfer_interval = 0.1
|
||||
chain._threads = []
|
||||
chain._retiring_threads = []
|
||||
chain._queue_active = False
|
||||
chain._worker_stop_event = threading.Event()
|
||||
chain._worker_lifecycle_lock = threading.RLock()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._replay_thread = None
|
||||
chain._replay_stop_event = threading.Event()
|
||||
return chain
|
||||
|
||||
|
||||
def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None:
|
||||
"""热更新应等待旧 worker 收敛,再启动使用独立停止信号的新一代。"""
|
||||
chain = _build_chain(transfer_threads=1)
|
||||
started_workers: queue.Queue = queue.Queue()
|
||||
|
||||
def run_worker(stop_event: threading.Event) -> None:
|
||||
"""记录 worker 代际并等待该代专属停止信号。"""
|
||||
started_workers.put((threading.current_thread(), stop_event))
|
||||
stop_event.wait()
|
||||
|
||||
chain._TransferChain__start_transfer = run_worker
|
||||
assert chain._TransferChain__init() is True
|
||||
first_thread, first_stop_event = started_workers.get(timeout=1)
|
||||
|
||||
chain.on_config_changed()
|
||||
|
||||
second_thread, second_stop_event = started_workers.get(timeout=1)
|
||||
assert first_stop_event.is_set() is True
|
||||
assert first_thread.is_alive() is False
|
||||
assert second_thread is not first_thread
|
||||
assert second_stop_event is not first_stop_event
|
||||
assert second_stop_event.is_set() is False
|
||||
|
||||
service = MagicMock()
|
||||
service.put.return_value = True
|
||||
chain._transfer_queue_service = MagicMock(return_value=service)
|
||||
task = MagicMock()
|
||||
assert chain.put_to_queue(task) is True
|
||||
service.put.assert_called_once()
|
||||
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert second_thread.is_alive() is False
|
||||
|
||||
|
||||
def test_config_reload_hands_queue_to_new_generation_while_old_io_finishes() -> None:
|
||||
"""旧代同步 I/O 超时不应让后续队列永久失去 worker。"""
|
||||
chain = _build_chain(transfer_threads=1)
|
||||
chain._WORKER_RESTART_TIMEOUT_SECONDS = 0.01
|
||||
started_workers: queue.Queue = queue.Queue()
|
||||
release_old_worker = threading.Event()
|
||||
invocation_count = 0
|
||||
invocation_lock = threading.Lock()
|
||||
|
||||
def run_worker(stop_event: threading.Event) -> None:
|
||||
"""首代模拟不可取消 I/O,后续代按各自停止信号正常收敛。"""
|
||||
nonlocal invocation_count
|
||||
with invocation_lock:
|
||||
generation = invocation_count
|
||||
invocation_count += 1
|
||||
started_workers.put((threading.current_thread(), stop_event))
|
||||
if generation == 0:
|
||||
release_old_worker.wait()
|
||||
else:
|
||||
stop_event.wait()
|
||||
|
||||
chain._TransferChain__start_transfer = run_worker
|
||||
assert chain._TransferChain__init() is True
|
||||
old_thread, old_stop_event = started_workers.get(timeout=1)
|
||||
|
||||
chain.on_config_changed()
|
||||
|
||||
new_thread, new_stop_event = started_workers.get(timeout=1)
|
||||
assert old_stop_event.is_set() is True
|
||||
assert old_thread.is_alive() is True
|
||||
assert chain._retiring_threads == [old_thread]
|
||||
assert chain._threads == [new_thread]
|
||||
assert new_stop_event.is_set() is False
|
||||
|
||||
release_old_worker.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert old_thread.is_alive() is False
|
||||
assert new_thread.is_alive() is False
|
||||
|
||||
|
||||
def test_close_workers_is_bounded_and_retains_nonconverging_owner() -> None:
|
||||
"""同步 I/O 线程超时后应保留句柄并报告失败,不能伪装成已取消。"""
|
||||
chain = _build_chain()
|
||||
release = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=release.wait,
|
||||
name="transfer-blocked-test",
|
||||
daemon=True,
|
||||
)
|
||||
chain._threads = [thread]
|
||||
thread.start()
|
||||
|
||||
started_at = time.monotonic()
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert time.monotonic() - started_at < 0.5
|
||||
assert chain._threads == []
|
||||
assert chain._retiring_threads == [thread]
|
||||
assert thread.is_alive() is True
|
||||
|
||||
service = MagicMock()
|
||||
chain._transfer_queue_service = MagicMock(return_value=service)
|
||||
assert chain.put_to_queue(MagicMock()) is False
|
||||
service.put.assert_not_called()
|
||||
|
||||
release.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert chain.close_workers(timeout_seconds=0) is True
|
||||
assert chain._threads == []
|
||||
assert chain._retiring_threads == []
|
||||
|
||||
|
||||
def test_close_workers_lock_wait_uses_the_same_timeout_budget() -> None:
|
||||
"""生命周期锁竞争必须耗用关闭预算,超时返回后不得延迟修改 worker 状态。"""
|
||||
chain = _build_chain()
|
||||
lock_acquired = threading.Event()
|
||||
release_lock = threading.Event()
|
||||
|
||||
def hold_lifecycle_lock() -> None:
|
||||
"""在独立线程持锁,稳定制造无法重入的生命周期锁竞争。"""
|
||||
with chain._worker_lifecycle_lock:
|
||||
lock_acquired.set()
|
||||
assert release_lock.wait(timeout=1)
|
||||
|
||||
holder = threading.Thread(target=hold_lifecycle_lock, daemon=True)
|
||||
holder.start()
|
||||
assert lock_acquired.wait(timeout=1)
|
||||
|
||||
started_at = time.monotonic()
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert time.monotonic() - started_at < 0.5
|
||||
assert chain._closing is False
|
||||
assert chain._worker_stop_event.is_set() is False
|
||||
assert chain._queue.empty() is True
|
||||
|
||||
release_lock.set()
|
||||
holder.join(timeout=1)
|
||||
assert holder.is_alive() is False
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
|
||||
|
||||
def test_close_keeps_timer_dependencies_when_workers_do_not_converge() -> None:
|
||||
"""活跃整理线程超时后,通知和重试 owner 必须继续供线程使用。"""
|
||||
chain = _build_chain()
|
||||
chain.close_workers = MagicMock(return_value=False)
|
||||
chain.failure_notification_aggregator = MagicMock()
|
||||
chain.retry_scheduler = MagicMock(close=AsyncMock())
|
||||
|
||||
completed = asyncio.run(chain.close(timeout_seconds=0.01))
|
||||
|
||||
assert completed is False
|
||||
chain.close_workers.assert_called_once_with(0.01)
|
||||
chain.failure_notification_aggregator.close.assert_not_called()
|
||||
chain.retry_scheduler.close.assert_not_awaited()
|
||||
|
||||
|
||||
def test_close_releases_timer_dependencies_after_workers_converge() -> None:
|
||||
"""worker 和回放退出后,整理链应继续刷新通知并关闭 AI 重试。"""
|
||||
chain = _build_chain()
|
||||
chain.close_workers = MagicMock(return_value=True)
|
||||
chain.failure_notification_aggregator = MagicMock()
|
||||
chain.retry_scheduler = MagicMock(close=AsyncMock())
|
||||
|
||||
completed = asyncio.run(chain.close(timeout_seconds=0.01))
|
||||
|
||||
assert completed is True
|
||||
chain.failure_notification_aggregator.close.assert_called_once_with()
|
||||
chain.retry_scheduler.close.assert_awaited_once_with()
|
||||
|
||||
|
||||
def test_stop_transfer_runtime_does_not_construct_chain(monkeypatch) -> None:
|
||||
"""关闭入口在整理链从未使用时应直接成功,不能因关停而启动 worker。"""
|
||||
get_existing_instance = MagicMock(return_value=None)
|
||||
monkeypatch.setattr(
|
||||
transfer_initializer.TransferChain,
|
||||
"get_existing_instance",
|
||||
get_existing_instance,
|
||||
)
|
||||
|
||||
completed = asyncio.run(
|
||||
transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01)
|
||||
)
|
||||
|
||||
assert completed is True
|
||||
get_existing_instance.assert_called_once_with()
|
||||
|
||||
|
||||
def test_stop_transfer_runtime_closes_existing_chain(monkeypatch) -> None:
|
||||
"""关闭入口应把超时预算和真实收敛结果原样传给既有整理链。"""
|
||||
chain = MagicMock(close=AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(
|
||||
transfer_initializer.TransferChain,
|
||||
"get_existing_instance",
|
||||
MagicMock(return_value=chain),
|
||||
)
|
||||
|
||||
completed = asyncio.run(
|
||||
transfer_initializer.stop_transfer_runtime(timeout_seconds=0.01)
|
||||
)
|
||||
|
||||
assert completed is False
|
||||
chain.close.assert_awaited_once_with(timeout_seconds=0.01)
|
||||
|
||||
|
||||
def test_constructor_failure_publishes_started_worker_to_cleanup(monkeypatch) -> None:
|
||||
"""首个 worker 启动后构造失败时,stop-only 入口仍必须找到并等待它。"""
|
||||
instances = dict(Singleton._instances)
|
||||
instances.pop((TransferChain, (), frozenset()), None)
|
||||
monkeypatch.setattr(Singleton, "_instances", instances)
|
||||
worker_started = threading.Event()
|
||||
worker_release = threading.Event()
|
||||
workers: list[threading.Thread] = []
|
||||
|
||||
def failing_init(chain: TransferChain) -> None:
|
||||
"""模拟第二个 owner 启动失败前已经成功启动一个整理线程。"""
|
||||
worker = threading.Thread(
|
||||
target=lambda: (worker_started.set(), worker_release.wait()),
|
||||
name="transfer-partial-construction",
|
||||
daemon=True,
|
||||
)
|
||||
workers.append(worker)
|
||||
worker.start()
|
||||
|
||||
async def close(*, timeout_seconds: float) -> bool:
|
||||
"""模拟真实 close 释放并等待半构造实例已经发布的 worker。"""
|
||||
worker_release.set()
|
||||
worker.join(timeout=timeout_seconds)
|
||||
return not worker.is_alive()
|
||||
|
||||
chain.close = close
|
||||
raise RuntimeError("second worker failed")
|
||||
|
||||
monkeypatch.setattr(TransferChain, "__init__", failing_init)
|
||||
|
||||
with pytest.raises(RuntimeError, match="second worker failed"):
|
||||
TransferChain()
|
||||
assert worker_started.wait(timeout=1)
|
||||
retained = TransferChain.get_existing_instance()
|
||||
assert retained is not None
|
||||
|
||||
assert asyncio.run(
|
||||
transfer_initializer.stop_transfer_runtime(timeout_seconds=1)
|
||||
) is True
|
||||
assert workers[0].is_alive() is False
|
||||
|
||||
|
||||
def test_failed_retry_schedule_future_error_is_observed() -> None:
|
||||
"""跨线程调度协程的延迟异常必须被取回并写入日志。"""
|
||||
future: Future[None] = Future()
|
||||
future.set_exception(RuntimeError("scheduler closed"))
|
||||
|
||||
with patch("app.chain.transfer.logger.error") as log_error:
|
||||
TransferChain._observe_failed_retry_schedule(future)
|
||||
|
||||
log_error.assert_called_once()
|
||||
assert "scheduler closed" in log_error.call_args.args[0]
|
||||
|
||||
|
||||
def test_failed_retry_schedule_registers_future_observer(monkeypatch) -> None:
|
||||
"""整理线程提交 AI 重试后应让 Future 持续连接到异常观察回调。"""
|
||||
chain = _build_chain()
|
||||
|
||||
async def schedule_retry(_history_id: int, *, group_key: str) -> None:
|
||||
"""提供不会实际执行的调度协程,供跨线程提交边界检查。"""
|
||||
|
||||
chain.retry_scheduler = MagicMock(schedule_retry=schedule_retry)
|
||||
future = MagicMock(spec=Future)
|
||||
event_loop = MagicMock()
|
||||
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", event_loop)
|
||||
|
||||
def submit(coroutine, loop):
|
||||
"""关闭测试协程并返回可检查的并发 Future。"""
|
||||
assert loop is event_loop
|
||||
coroutine.close()
|
||||
return future
|
||||
|
||||
with patch(
|
||||
"app.chain.transfer.asyncio.run_coroutine_threadsafe",
|
||||
side_effect=submit,
|
||||
):
|
||||
chain._schedule_failed_transfer_retry(42, "media:test")
|
||||
|
||||
future.add_done_callback.assert_called_once()
|
||||
callback = future.add_done_callback.call_args.args[0]
|
||||
assert callback is TransferChain._observe_failed_retry_schedule
|
||||
|
||||
|
||||
def test_worker_requeues_item_taken_during_shutdown(monkeypatch) -> None:
|
||||
"""停止信号与 queue.get 竞态时,未开始处理的任务必须原样放回队列。"""
|
||||
chain = _build_chain()
|
||||
work_queue = MagicMock()
|
||||
chain._queue = work_queue
|
||||
entered_get = threading.Event()
|
||||
release_get = threading.Event()
|
||||
item = TransferQueue()
|
||||
|
||||
def get_item(*_args, **_kwargs):
|
||||
"""让停止信号稳定落在阻塞取队列之后、任务处理之前。"""
|
||||
entered_get.set()
|
||||
assert release_get.wait(timeout=1)
|
||||
return item
|
||||
|
||||
work_queue.get.side_effect = get_item
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
stop_event = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
assert entered_get.wait(timeout=1)
|
||||
|
||||
stop_event.set()
|
||||
release_get.set()
|
||||
thread.join(timeout=1)
|
||||
|
||||
assert thread.is_alive() is False
|
||||
work_queue.put.assert_called_once_with(item)
|
||||
work_queue.task_done.assert_called_once_with()
|
||||
|
||||
|
||||
def test_worker_settles_progress_when_only_stop_sentinel_remains(monkeypatch) -> None:
|
||||
"""真实任务完成时仅剩停止哨兵,仍应结束进度并重置本批计数。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(
|
||||
fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/movie.mkv",
|
||||
type="file",
|
||||
name="movie.mkv",
|
||||
basename="movie",
|
||||
extension="mkv",
|
||||
)
|
||||
)
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
task_started = threading.Event()
|
||||
release_task = threading.Event()
|
||||
|
||||
def handle_transfer(*_args, **_kwargs):
|
||||
"""阻塞真实任务,让测试能在其完成前稳定插入停止哨兵。"""
|
||||
task_started.set()
|
||||
assert release_task.wait(timeout=1)
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = handle_transfer
|
||||
chain._TransferChain__start_job_execution = MagicMock()
|
||||
chain._TransferChain__finish_job_execution = MagicMock()
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
stop_event = threading.Event()
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
assert task_started.wait(timeout=1)
|
||||
|
||||
stop_event.set()
|
||||
chain._queue.put(chain._QUEUE_STOP_SENTINEL)
|
||||
release_task.set()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._progress.end.assert_called_once_with()
|
||||
assert chain._active_tasks == 0
|
||||
assert chain._total_num == 0
|
||||
assert chain._processed_num == 0
|
||||
assert chain._fail_num == 0
|
||||
with chain._queue.mutex:
|
||||
assert list(chain._queue.queue) == [chain._QUEUE_STOP_SENTINEL]
|
||||
|
||||
|
||||
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
|
||||
"""其他 worker 已取走真实任务但尚未登记 active 时,当前批次不得提前结算。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(
|
||||
fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/claimed.mkv",
|
||||
type="file",
|
||||
name="claimed.mkv",
|
||||
basename="claimed",
|
||||
extension="mkv",
|
||||
)
|
||||
)
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 1
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 2
|
||||
claimed = threading.Event()
|
||||
release_claim = threading.Event()
|
||||
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
|
||||
def hold_claimed_task() -> None:
|
||||
"""模拟 worker 已完成 queue.get、尚未取得 task_lock 登记 active 的窗口。"""
|
||||
item = chain._queue.get(timeout=1)
|
||||
assert item.task is task
|
||||
claimed.set()
|
||||
assert release_claim.wait(timeout=1)
|
||||
chain._queue.task_done()
|
||||
chain._TransferChain__settle_transfer_progress_if_idle()
|
||||
|
||||
worker = threading.Thread(target=hold_claimed_task, daemon=True)
|
||||
worker.start()
|
||||
assert claimed.wait(timeout=1)
|
||||
|
||||
chain._TransferChain__settle_transfer_progress_if_idle()
|
||||
|
||||
chain._progress.end.assert_not_called()
|
||||
assert chain._processed_num == 1
|
||||
|
||||
release_claim.set()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._progress.end.assert_called_once_with()
|
||||
assert chain._total_num == 0
|
||||
assert chain._processed_num == 0
|
||||
|
||||
|
||||
def test_replay_has_single_owner_and_close_waits_for_it() -> None:
|
||||
"""重复回放只保留一个线程,关闭会通知并等待该线程退出。"""
|
||||
chain = _build_chain()
|
||||
replay_started = threading.Event()
|
||||
replay_calls = []
|
||||
|
||||
def replay(stop_event: threading.Event) -> None:
|
||||
"""模拟可由逐项检查点收敛的 pending 回放。"""
|
||||
replay_calls.append(stop_event)
|
||||
replay_started.set()
|
||||
stop_event.wait()
|
||||
|
||||
chain._TransferChain__replay_pending = replay
|
||||
chain.replay_pending()
|
||||
assert replay_started.wait(timeout=1)
|
||||
replay_thread = chain._replay_thread
|
||||
|
||||
chain.replay_pending()
|
||||
|
||||
assert chain._replay_thread is replay_thread
|
||||
assert replay_calls == [chain._replay_stop_event]
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert replay_thread.is_alive() is False
|
||||
assert chain._replay_thread is None
|
||||
Reference in New Issue
Block a user