refactor: own shutdown lifecycle boundaries

This commit is contained in:
jxxghp
2026-08-23 20:20:26 +08:00
parent 59f020f226
commit 7f09927c47
59 changed files with 6393 additions and 958 deletions
+5 -1
View File
@@ -155,7 +155,11 @@ class AgentServiceAdapter:
raise CapabilityAdapterContractError(
f"{spec.entrypoint}.close() 必须返回 awaitable"
)
await result
converged = await result
if converged is False:
raise CapabilityAdapterContractError(
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
)
@staticmethod
async def cleanup(
+37 -3
View File
@@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
)
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
self._accepting_tasks = True
self._close_cancel_requested: set[asyncio.Task] = set()
self.tools = [
StructuredTool.from_function(
coroutine=self._control_task,
@@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
"""记录任务完成时间并取出异常避免未读取告警。"""
cancel_requested = getattr(self, "_close_cancel_requested", None)
if cancel_requested is not None:
cancel_requested.discard(task)
record = self._tasks.get(task_id)
if record:
record.finished_at = datetime.now()
@@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
@staticmethod
async def _cancel_records(
records: list[_SubAgentRuntimeTask],
*,
cancel_requested: Optional[set[asyncio.Task]] = None,
) -> list[_SubAgentRuntimeTask]:
"""取消一组任务,并返回等待上限内仍未收敛的记录。"""
cancellable_tasks = [
@@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
if cancellable_tasks:
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
for task in cancellable_tasks:
if cancel_requested is not None and task in cancel_requested:
continue
task.cancel()
if cancel_requested is not None:
cancel_requested.add(task)
if not cancellable_tasks:
return []
@@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
return [record for record in records if record.task in pending]
async def close(self) -> None:
"""取消脱离当前 Agent 回合的子代理任务"""
def seal(self) -> None:
"""封住新的 detached 子代理提交,既有任务继续由记录表持有"""
self._accepting_tasks = False
async def close(self) -> bool:
"""有限等待 detached 子代理;超时保留记录并返回 False。"""
self.seal()
if not hasattr(self, "_close_cancel_requested"):
self._close_cancel_requested = set()
unfinished_records = [
record for record in self._tasks.values() if not record.task.done()
]
@@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
logger.info(
f"关闭子代理任务控制器,取消未完成任务: tasks={len(unfinished_records)}"
)
await self._cancel_records(unfinished_records)
pending_records = await self._cancel_records(
unfinished_records,
cancel_requested=self._close_cancel_requested,
)
if pending_records:
self._tasks = {
record.task_id: record
for record in pending_records
}
return False
self._tasks.clear()
self._close_cancel_requested.clear()
return True
@staticmethod
def _pipeline_description(
@@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
timeout = normalized_timeout_ms / 1000
for step_index, spec in enumerate(specs, start=1):
if not self._accepting_tasks:
return records, "子代理任务控制器正在关闭,不能再启动新任务。"
record = self._create_pipeline_record(spec)
records.append(record)
pipeline_description = self._pipeline_description(
@@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
"""管理异步子代理任务。"""
logger.info(f"收到子代理管控操作: action={action}")
if action in {"start", "run", "pipeline"}:
if not self._accepting_tasks:
error = "子代理任务控制器正在关闭,不能再启动新任务。"
return self._json_response({"success": False, "error": error})
specs, error = self._normalize_specs(
description=description,
subagent_type=subagent_type,
+134 -33
View File
@@ -436,6 +436,7 @@ class MoviePilotAgent:
self._agent_started_at: Optional[datetime] = None
self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None
self._subagent_middlewares: tuple[Any, ...] = ()
self._shutdown_started = False
self._last_agent_cache_hit = False
# 流式token管理
@@ -1751,10 +1752,25 @@ class MoviePilotAgent:
return None
@staticmethod
async def _close_subagent_middleware_instances(
def _seal_subagent_middleware_instances(
middlewares: tuple[Any, ...],
) -> None:
"""释放不再由 Agent 图持有的子代理控制器"""
"""同步封住子代理控制器的新任务入口,不等待既有任务退出"""
for middleware in middlewares:
seal = getattr(middleware, "seal", None)
if not callable(seal):
continue
try:
seal()
except Exception as error:
logger.debug(f"封住子代理中间件失败: {error}")
@staticmethod
async def _close_subagent_middleware_instances(
middlewares: tuple[Any, ...],
) -> tuple[Any, ...]:
"""关闭子代理控制器,并返回仍持有未收敛任务的实例。"""
pending_middlewares = []
for middleware in middlewares:
close = getattr(middleware, "close", None)
if not callable(close):
@@ -1762,9 +1778,30 @@ class MoviePilotAgent:
try:
result = close()
if inspect.isawaitable(result):
await result
result = await result
if result is False:
pending_middlewares.append(middleware)
except Exception as error:
logger.debug(f"关闭子代理中间件失败: {error}")
pending_middlewares.append(middleware)
return tuple(pending_middlewares)
@staticmethod
def _merge_subagent_middleware_owners(
*groups: tuple[Any, ...],
) -> tuple[Any, ...]:
"""按对象身份合并当前图与延迟收敛控制器的 owner 集合。"""
merged = []
for group in groups:
for middleware in group:
if not any(middleware is existing for existing in merged):
merged.append(middleware)
return tuple(merged)
def begin_shutdown(self) -> None:
"""在任何异步等待前封住当前 Agent 的 detached 子代理提交。"""
self._shutdown_started = True
self._seal_subagent_middleware_instances(self._subagent_middlewares)
async def _cache_agent(
self,
@@ -1786,7 +1823,11 @@ class MoviePilotAgent:
for replacement in subagent_middlewares
)
)
await self._close_subagent_middleware_instances(previous_middlewares)
if self._shutdown_started:
self._seal_subagent_middleware_instances(subagent_middlewares)
pending_middlewares = await self._close_subagent_middleware_instances(
previous_middlewares
)
self._compiled_agent_bundle = _CompiledAgentBundle(
signature=signature,
agent=agent,
@@ -1798,15 +1839,21 @@ class MoviePilotAgent:
mcp_config_signature=mcp_config_signature,
catalog_checked_at=datetime.now(),
)
self._subagent_middlewares = subagent_middlewares
self._subagent_middlewares = self._merge_subagent_middleware_owners(
pending_middlewares,
subagent_middlewares,
)
return agent
async def _invalidate_cached_agent(self) -> None:
"""使当前图失效,并释放只属于该图的子代理控制器"""
async def _invalidate_cached_agent(self) -> bool:
"""使当前图失效,未收敛的子代理控制器继续由 Agent 持有"""
subagent_middlewares = self._subagent_middlewares
self._subagent_middlewares = ()
self._compiled_agent_bundle = None
await self._close_subagent_middleware_instances(subagent_middlewares)
pending_middlewares = await self._close_subagent_middleware_instances(
subagent_middlewares
)
self._subagent_middlewares = pending_middlewares
return not pending_middlewares
@staticmethod
def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]:
@@ -2003,9 +2050,13 @@ class MoviePilotAgent:
if cached_agent:
# 签名相同表示已编译图中的精确工具实例仍有效;新建快照仅用于复核。
cached_bundle.catalog_checked_at = datetime.now()
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
temporary_subagent_middlewares = ()
logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}")
return cached_agent
@@ -2123,14 +2174,22 @@ class MoviePilotAgent:
temporary_subagent_middlewares = ()
return cached_agent
except asyncio.CancelledError:
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
raise
except Exception as e:
await self._close_subagent_middleware_instances(
pending_middlewares = await self._close_subagent_middleware_instances(
temporary_subagent_middlewares
)
self._subagent_middlewares = self._merge_subagent_middleware_owners(
self._subagent_middlewares,
pending_middlewares,
)
logger.error(f"创建 Agent 失败: {e}")
raise
@@ -2539,14 +2598,20 @@ class MoviePilotAgent:
)
)
async def cleanup(self):
async def cleanup(self) -> bool:
"""
清理智能体资源
清理智能体资源;detached 子代理未收敛时保留 owner 并返回 False。
"""
await self._invalidate_cached_agent()
self.begin_shutdown()
if not await self._invalidate_cached_agent():
logger.error(
f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}"
)
return False
self._pending_secret_confirmation = None
self.protected_output_callback = None
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
return True
@dataclass
@@ -2626,6 +2691,7 @@ class AgentManager:
self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {}
self._session_cancel_requested: set[str] = set()
self._close_finalizer_task: Optional[asyncio.Task] = None
self._closed = False
self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT
# 接收门禁与队列写入共用一把锁,确保关闭开始后不会再创建 worker。
self._lifecycle_lock = asyncio.Lock()
@@ -2693,22 +2759,33 @@ class AgentManager:
async with self._lifecycle_lock:
if self._accepting_tasks:
return
if self._close_finalizer_task and not self._close_finalizer_task.done():
raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭")
memory_manager.initialize()
if not self._idle_cleanup_task or self._idle_cleanup_task.done():
self._idle_cleanup_task = asyncio.create_task(
self._cleanup_idle_sessions()
)
self._accepting_tasks = True
self._closed = False
async def close(self):
async def close(self) -> bool:
"""
关闭管理器
关闭管理器,并诚实返回全部会话 owner 是否已经收敛。
"""
async with self._lifecycle_lock:
if self._closed:
return True
if self._close_finalizer_task and not self._close_finalizer_task.done():
return
return False
# 门禁必须先关闭;锁内完成清理可阻止等待中的请求在收口期间重新入队。
self._accepting_tasks = False
# 子代理提交门禁同样必须在第一次 await 前关闭,否则 detached task
# 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。
for agent in self.active_agents.values():
begin_shutdown = getattr(agent, "begin_shutdown", None)
if callable(begin_shutdown):
begin_shutdown()
if self._idle_cleanup_task:
self._idle_cleanup_task.cancel()
try:
@@ -2751,8 +2828,8 @@ class AgentManager:
self._session_workers.pop(session_id, None)
for session_id, agent in list(self.active_agents.items()):
if session_id not in timed_out_session_ids:
await agent.cleanup()
self.active_agents.pop(session_id, None)
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
logger.error(
"AgentManager 关闭时仍有 worker 未收敛,"
f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束"
@@ -2760,13 +2837,21 @@ class AgentManager:
self._close_finalizer_task = asyncio.create_task(
self._finish_deferred_close(timed_out_workers)
)
return
return False
self._session_workers.clear()
for agent in list(self.active_agents.values()):
await agent.cleanup()
self.active_agents.clear()
for session_id, agent in list(self.active_agents.items()):
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
if self.active_agents:
logger.error(
"AgentManager 仍有 %d 个 detached 子代理 owner 未收敛",
len(self.active_agents),
)
return False
await memory_manager.close()
self._closed = True
return True
def _record_session_activity(self, session_id: str, user_id: str) -> None:
"""
@@ -3072,7 +3157,10 @@ class AgentManager:
and isinstance(task.agent_factory, type)
and not isinstance(existing_agent, task.agent_factory)
):
await existing_agent.cleanup()
if await existing_agent.cleanup() is False:
raise AgentManagerUnavailableError(
f"Agent 会话 {session_id} 仍有子代理任务在停止"
)
self.active_agents.pop(session_id, None)
if session_id not in self.active_agents:
@@ -3249,7 +3337,11 @@ class AgentManager:
# 清理agent
if session_id in self.active_agents:
agent = self.active_agents[session_id]
await agent.cleanup()
if await agent.cleanup() is False:
logger.error(
f"会话 {session_id} 仍有子代理 owner 未收敛,保留会话与记忆"
)
return
del self.active_agents[session_id]
memory_manager.clear_memory(session_id, user_id)
logger.info(f"会话 {session_id} 的记忆已清空")
@@ -3290,9 +3382,14 @@ class AgentManager:
self._session_deferred_cleanup_tasks.pop(session_id, None)
self._session_queue_rejections.pop(session_id, None)
self._session_last_queue_wait_ms.pop(session_id, None)
agent = self.active_agents.pop(session_id, None)
agent = self.active_agents.get(session_id)
if agent:
await agent.cleanup()
if await agent.cleanup() is False:
logger.error(
f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛"
)
return
self.active_agents.pop(session_id, None)
memory_manager.clear_memory(session_id, user_id)
logger.info(f"会话 {session_id} 的记忆已清空")
@@ -3310,15 +3407,19 @@ class AgentManager:
for session_id, worker in workers:
if self._session_workers.get(session_id) is worker:
self._session_workers.pop(session_id, None)
agent = self.active_agents.pop(session_id, None)
if agent:
await agent.cleanup()
for session_id, agent in list(self.active_agents.items()):
await agent.cleanup()
self.active_agents.pop(session_id, None)
if await agent.cleanup() is not False:
self.active_agents.pop(session_id, None)
self._session_shutdown_pending.clear()
self._session_cancel_requested.clear()
if self.active_agents:
logger.error(
"AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛",
len(self.active_agents),
)
return
await memory_manager.close()
self._closed = True
finally:
self._close_finalizer_task = None
+12 -7
View File
@@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import (
build_agent_capability_registry,
should_run_agent_service,
)
from app.runtime.capabilities.model import CapabilityMaterializationState
from app.runtime.capabilities.model import (
CapabilityLifecycleState,
CapabilityMaterializationState,
)
from app.runtime.capabilities.runtime import CapabilityRuntime
@@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None:
await close()
async def begin_agent_shutdown() -> None:
"""不可逆关闭首用闸门,并等待全部同步及异步能力释放"""
try:
await _ensure_runtime().shutdown_async(reason="application_shutdown")
finally:
await close_materialized_terminal_sessions()
async def begin_agent_shutdown() -> bool:
"""不可逆关闭首用闸门,并返回 Agent service 是否真实收敛"""
runtime = _ensure_runtime()
await runtime.shutdown_async(reason="application_shutdown")
return (
runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle
is CapabilityLifecycleState.STOPPED
)
+117 -26
View File
@@ -3,7 +3,7 @@ import inspect
import json
import threading
from abc import ABCMeta, abstractmethod
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
from contextvars import Context, copy_context
from functools import partial
from pathlib import Path
@@ -168,33 +168,130 @@ _blocking_semaphores = {
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
}
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
_blocking_executor_lock = threading.Lock()
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
_blocking_executor_lock = threading.RLock()
_blocking_executor_accepting = True
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
"""按桶懒加载线程池,避免在导入阶段创建过多 worker。"""
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
with _blocking_executor_lock:
executor = _blocking_futures.pop(future, None)
if executor is None or executor not in _blocking_retiring_executors:
return
if executor not in _blocking_futures.values():
_blocking_retiring_executors.discard(executor)
def _submit_blocking_call(
bucket: str,
bound_call: Callable[[], Any],
) -> ConcurrentFuture[Any]:
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
context = copy_context()
with _blocking_executor_lock:
if not _blocking_executor_accepting:
raise RuntimeError("Agent 工具阻塞执行器正在关闭,不能再提交新任务")
executor = _blocking_executors.get(bucket)
if executor:
return executor
limit = _BLOCKING_BUCKET_LIMITS[bucket]
executor = ThreadPoolExecutor(
max_workers=limit,
thread_name_prefix=f"agent-tool-{bucket}",
)
_blocking_executors[bucket] = executor
return executor
if executor is None:
limit = _BLOCKING_BUCKET_LIMITS[bucket]
executor = ThreadPoolExecutor(
max_workers=limit,
thread_name_prefix=f"agent-tool-{bucket}",
)
_blocking_executors[bucket] = executor
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
future = Context().run(executor.submit, context.run, bound_call)
_blocking_futures[future] = executor
future.add_done_callback(_discard_blocking_future)
return future
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
"""关闭 Agent 工具阻塞线程池,释放长期运行进程或测试环境中的 worker。"""
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
"""撤销活动 executor 的提交资格,并保留其运行 Future 对应的 owner。"""
with _blocking_executor_lock:
executors = list(_blocking_executors.values())
executors = tuple(_blocking_executors.values())
_blocking_executors.clear()
_blocking_retiring_executors.update(executors)
for executor in executors:
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
executor.shutdown(wait=False, cancel_futures=cancel_futures)
with _blocking_executor_lock:
owned_executors = set(_blocking_futures.values())
_blocking_retiring_executors.intersection_update(owned_executors)
return executors
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
"""原子封住新阻塞工具提交,并请求取消尚未开始的同步调用。"""
global _blocking_executor_accepting
with _blocking_executor_lock:
_blocking_executor_accepting = False
_retire_blocking_executors(cancel_futures=cancel_futures)
def reopen_blocking_executors() -> bool:
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
global _blocking_executor_accepting
with _blocking_executor_lock:
if _blocking_futures or _blocking_retiring_executors:
return False
_blocking_executor_accepting = True
return True
async def close_blocking_executors(
*,
timeout_seconds: float,
cancel_futures: bool = True,
) -> bool:
"""有限等待全部阻塞工具 Future,超时保留 Future 与 executor owner。"""
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
with _blocking_executor_lock:
futures = tuple(_blocking_futures)
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
if wrapped_futures:
done, pending = await asyncio.wait(
wrapped_futures,
timeout=max(0.0, timeout_seconds),
)
if done:
await asyncio.gather(*done, return_exceptions=True)
for pending_future in pending:
pending_future.add_done_callback(
lambda completed: completed.exception()
if not completed.cancelled()
else None
)
with _blocking_executor_lock:
unfinished = tuple(
future for future in _blocking_futures if not future.done()
)
retiring_count = len(_blocking_retiring_executors)
if unfinished:
logger.error(
"Agent 阻塞工具未在 %.1f 秒内收敛:futures=%dexecutors=%d",
max(0.0, timeout_seconds),
len(unfinished),
retiring_count,
)
return False
return True
def shutdown_blocking_executors(
*,
wait: bool = True,
cancel_futures: bool = False,
) -> bool:
"""同步清理测试 owner;非等待模式下保留尚未收敛的 executor 句柄。"""
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
for executor in executors:
if wait:
executor.shutdown(wait=True, cancel_futures=cancel_futures)
with _blocking_executor_lock:
return not _blocking_futures and not _blocking_retiring_executors
class ToolExecutionTimeoutError(TimeoutError):
@@ -226,13 +323,7 @@ async def run_agent_blocking(
await semaphore.acquire()
try:
context = copy_context()
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
future = Context().run(
_get_blocking_executor(bucket_name).submit,
context.run,
bound_call,
)
future = _submit_blocking_call(bucket_name, bound_call)
except Exception:
semaphore.release()
raise
+81 -69
View File
@@ -2,6 +2,7 @@
import json
import shutil
from contextvars import copy_context
from pathlib import Path
from typing import Any, Optional
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
"""重载插件实例并重新注册其命令、定时任务和 API。"""
runtime_status = get_plugin_manager().reload_plugin(plugin_id)
refresh_plugin_registrations(plugin_id)
return runtime_status
plugin_manager = get_plugin_manager()
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
runtime_status = plugin_manager.reload_plugin(plugin_id)
refresh_plugin_registrations(plugin_id)
return runtime_status
def summarize_plugin(plugin: Any) -> dict[str, Any]:
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
async def reload_runtime(target_id: str) -> object:
"""通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
mutation_context = copy_context()
return await run_agent_blocking(
"plugin",
mutation_context.run,
plugin_manager.reload_plugin_tree,
target_id,
)
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
)
return result
with plugin_manager.suppress_plugin_monitor(plugin_id):
result = await PluginInstallCommand(
installed_plugins_reader=lambda: SystemConfigOper().get(
SystemConfigKey.UserInstalledPlugins
) or [],
installed_plugins_writer=save_installed_plugins,
plugin_ids_provider=plugin_manager.get_plugin_ids,
compatibility_checker=skip_compatibility_check,
package_installer=install_package,
package_checkpointer=package_manager.async_checkpoint,
package_committer=package_manager.async_commit,
package_rollback=package_manager.async_rollback,
install_reporter=lambda target_id, target_repo: (
MoviePilotServerHelper.async_install_plugin_reg(
plugin_id=target_id,
repo_url=target_repo,
)
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
).execute(
plugin_id=plugin_id,
repo_url=repo_url,
force=force,
)
result = await PluginInstallCommand(
installed_plugins_reader=lambda: SystemConfigOper().get(
SystemConfigKey.UserInstalledPlugins
) or [],
installed_plugins_writer=save_installed_plugins,
plugin_ids_provider=plugin_manager.get_plugin_ids,
compatibility_checker=skip_compatibility_check,
package_installer=install_package,
package_checkpointer=package_manager.async_checkpoint,
package_committer=package_manager.async_commit,
package_rollback=package_manager.async_rollback,
install_reporter=lambda target_id, target_repo: (
MoviePilotServerHelper.async_install_plugin_reg(
plugin_id=target_id,
repo_url=target_repo,
)
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
mutation=plugin_manager.mutation,
package_write_guard=plugin_manager.suppress_plugin_monitor,
).execute(
plugin_id=plugin_id,
repo_url=repo_url,
force=force,
)
return result.success, result.message, result.refreshed_only
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
from app.agent.tools.base import run_agent_blocking
plugin_manager = get_plugin_manager()
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
config_oper = SystemConfigOper()
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
if plugin_id in install_plugins:
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
remove_plugin_api(plugin_id)
remove_plugin_job(plugin_id)
plugin_class = plugin_manager.plugins.get(plugin_id)
was_clone = bool(getattr(plugin_class, "is_clone", False))
clone_files_removed = False
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif was_clone:
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
try:
clone_files_removed = await run_agent_blocking(
"plugin",
_remove_plugin_directory,
plugin_base_dir,
config_oper = SystemConfigOper()
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
if plugin_id in install_plugins:
install_plugins = [
plugin for plugin in install_plugins if plugin != plugin_id
]
await config_oper.async_set(
SystemConfigKey.UserInstalledPlugins,
install_plugins,
)
if clone_files_removed:
plugin_manager.plugins.pop(plugin_id, None)
except Exception:
clone_files_removed = False
remove_plugin_from_folders(plugin_id)
plugin_manager.remove_plugin(plugin_id)
remove_plugin_api(plugin_id)
remove_plugin_job(plugin_id)
return {
"was_clone": was_clone,
"clone_files_removed": clone_files_removed,
}
plugin_class = plugin_manager.plugins.get(plugin_id)
was_clone = bool(getattr(plugin_class, "is_clone", False))
clone_files_removed = False
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif was_clone:
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
try:
clone_files_removed = await run_agent_blocking(
"plugin",
_remove_plugin_directory,
plugin_base_dir,
)
if clone_files_removed:
plugin_manager.plugins.pop(plugin_id, None)
except Exception:
clone_files_removed = False
remove_plugin_from_folders(plugin_id)
plugin_manager.remove_plugin(plugin_id)
return {
"was_clone": was_clone,
"clone_files_removed": clone_files_removed,
}
+41 -37
View File
@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
)
plugin_manager = get_plugin_manager()
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
next_config = {} if replace else dict(current_config)
if updates:
next_config.update(updates)
for key in remove_keys:
next_config.pop(key, None)
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
next_config = {} if replace else dict(current_config)
if updates:
next_config.update(updates)
for key in remove_keys:
next_config.pop(key, None)
changed_keys = sorted(
key
for key in set(current_config.keys()) | set(next_config.keys())
if current_config.get(key) != next_config.get(key)
or (key in current_config) != (key in next_config)
)
if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
return json.dumps(
{
"success": False,
"message": f"保存插件 {plugin_id} 配置失败",
},
ensure_ascii=False,
changed_keys = sorted(
key
for key in set(current_config.keys()) | set(next_config.keys())
if current_config.get(key) != next_config.get(key)
or (key in current_config) != (key in next_config)
)
return json.dumps(
{
"success": True,
**plugin_info,
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
"replace": replace,
"changed_keys": changed_keys,
"removed_keys": remove_keys,
"config_requires_reload": True,
"previous_config": current_config,
"saved_config": next_config,
},
ensure_ascii=False,
indent=2,
default=str,
)
if not await plugin_manager.async_save_plugin_config(
plugin_id,
next_config,
):
return json.dumps(
{
"success": False,
"message": f"保存插件 {plugin_id} 配置失败",
},
ensure_ascii=False,
)
return json.dumps(
{
"success": True,
**plugin_info,
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
"replace": replace,
"changed_keys": changed_keys,
"removed_keys": remove_keys,
"config_requires_reload": True,
"previous_config": current_config,
"saved_config": next_config,
},
ensure_ascii=False,
indent=2,
default=str,
)
async def run(
self,
+1
View File
@@ -40,4 +40,5 @@ def get_plugin_config_command() -> PluginConfigCommand:
reload_runtime=manager.reload_plugin,
publish_reset=publish_reset,
refresh_registrations=refresh_registrations,
mutation=manager.mutation,
)
+153 -107
View File
@@ -57,7 +57,10 @@ from app.api.dependencies.plugin import (
from app.adapters.external.server import MoviePilotServerHelper
from app.adapters.external.market import PluginHelper
from app.adapters.system.plugin.package import PluginPackageManager
from app.schemas.exception import PersistenceUnavailableError
from app.schemas.exception import (
PersistenceUnavailableError,
PluginMutationRejectedError,
)
from app.runtime.log import logger
from app.schemas.types import SystemConfigKey
from app.api.context import get_background_task_registry, resolve_background_task_registry
@@ -520,10 +523,15 @@ def reload_plugin(
"""
重新加载插件
"""
# 重新加载插件
runtime_status = PluginManager().reload_plugin(plugin_id)
# 注册插件服务
register_plugin(plugin_id)
plugin_manager = PluginManager()
try:
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
# 重新加载插件
runtime_status = plugin_manager.reload_plugin(plugin_id)
# 注册插件服务
register_plugin(plugin_id)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
if runtime_status is _SchemaPluginRuntimeStatus.ACTIVE:
return _SchemaResponse(success=True)
return _SchemaResponse(
@@ -600,14 +608,15 @@ async def install(
),
plugin_reloader=reload_runtime,
registration_refresher=refresh_registrations,
mutation=plugin_manager.mutation,
package_write_guard=plugin_manager.suppress_plugin_monitor,
)
result = await command.execute(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=bool(force),
)
with plugin_manager.suppress_plugin_monitor(plugin_id):
result = await command.execute(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=bool(force),
)
if not result.success:
return _SchemaResponse(success=False, message=result.message)
return _SchemaResponse(success=True)
@@ -888,11 +897,12 @@ async def save_plugin_folders(
保存插件文件夹分组配置
"""
try:
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(success=True)
with PluginManager().mutation("保存插件文件夹配置"):
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(success=True)
except PersistenceUnavailableError:
raise
except Exception as e:
@@ -909,18 +919,26 @@ async def create_plugin_folder(
"""
创建新的插件文件夹
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
if folder_name not in folders:
folders[folder_name] = []
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 创建成功"
)
else:
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 已存在")
try:
with PluginManager().mutation(f"创建插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
if folder_name not in folders:
folders[folder_name] = []
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 创建成功"
)
return _SchemaResponse(
success=False,
message=f"文件夹 '{folder_name}' 已存在",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.delete(
@@ -932,15 +950,26 @@ async def delete_plugin_folder(
"""
删除插件文件夹
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
if folder_name in folders:
del folders[folder_name]
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 删除成功"
)
else:
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 不存在")
try:
with PluginManager().mutation(f"删除插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
if folder_name in folders:
del folders[folder_name]
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 删除成功"
)
return _SchemaResponse(
success=False,
message=f"文件夹 '{folder_name}' 不存在",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.put(
@@ -956,12 +985,22 @@ async def update_folder_plugins(
"""
更新指定文件夹中的插件列表
"""
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
folders[folder_name] = plugin_ids
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
return _SchemaResponse(
success=True, message=f"文件夹 '{folder_name}' 中的插件已更新"
)
try:
with PluginManager().mutation(f"更新插件文件夹 {folder_name}"):
folders = (
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
)
folders[folder_name] = plugin_ids
await get_configured_system_config().async_set(
SystemConfigKey.PluginFolders,
folders,
)
return _SchemaResponse(
success=True,
message=f"文件夹 '{folder_name}' 中的插件已更新",
)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
@router.post(
@@ -975,23 +1014,24 @@ def clone_plugin(
"""
创建插件分身
"""
plugin_manager = PluginManager()
try:
success, message = PluginManager().clone_plugin(
plugin_id=plugin_id,
suffix=clone_data.suffix,
name=clone_data.name,
description=clone_data.description,
version=clone_data.version,
icon=clone_data.icon,
)
with plugin_manager.mutation(f"创建插件 {plugin_id} 分身"):
success, message = plugin_manager.clone_plugin(
plugin_id=plugin_id,
suffix=clone_data.suffix,
name=clone_data.name,
description=clone_data.description,
version=clone_data.version,
icon=clone_data.icon,
)
if success:
# 分身服务已完成运行态加载,此处只补齐宿主注册。
register_plugin(message)
# 将分身插件添加到原插件所在的文件夹中
_add_clone_to_plugin_folder(plugin_id, message)
return _SchemaResponse(success=True, message="插件分身创建成功")
else:
if success:
# 分身服务已完成运行态加载,此处只补齐宿主注册。
register_plugin(message)
# 将分身插件添加到原插件所在的文件夹中
_add_clone_to_plugin_folder(plugin_id, message)
return _SchemaResponse(success=True, message="插件分身创建成功")
return _SchemaResponse(success=False, message=message)
except Exception as e:
logger.error(f"创建插件分身失败:{str(e)}")
@@ -1034,54 +1074,60 @@ def uninstall_plugin(
卸载插件
"""
plugin_manager = PluginManager()
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
return _SchemaResponse(
success=False,
message=f"请先卸载该插件的分身:{instance_ids}",
)
config_oper = get_configured_system_config()
# 删除已安装信息
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
for plugin in install_plugins:
if plugin == plugin_id:
install_plugins.remove(plugin)
break
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
# 移除插件API
remove_plugin_api(plugin_id)
# 移除插件服务
remove_plugin_job(plugin_id)
# 判断是否为分身
plugin_class = plugin_manager.plugins.get(plugin_id)
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif getattr(plugin_class, "is_clone", False):
# 如果是分身插件,则删除分身数据和配置
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
# 删除分身文件
plugin_base_dir = (
get_api_runtime_config_snapshot().root_path
/ "app"
/ "plugins"
/ plugin_id.lower()
)
if plugin_base_dir.exists():
try:
shutil.rmtree(plugin_base_dir)
plugin_manager.plugins.pop(plugin_id, None)
except Exception as e:
logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}")
# 从插件文件夹中移除该插件
remove_plugin_from_folders(plugin_id)
# 移除插件
plugin_manager.remove_plugin(plugin_id)
return _SchemaResponse(success=True)
try:
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
if not virtual_instance and source_instances:
instance_ids = "".join(item.instance_id for item in source_instances)
return _SchemaResponse(
success=False,
message=f"请先卸载该插件的分身:{instance_ids}",
)
config_oper = get_configured_system_config()
# 删除已安装信息
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
for plugin in install_plugins:
if plugin == plugin_id:
install_plugins.remove(plugin)
break
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
# 移除插件API
remove_plugin_api(plugin_id)
# 移除插件服务
remove_plugin_job(plugin_id)
# 判断是否为分身
plugin_class = plugin_manager.plugins.get(plugin_id)
if virtual_instance:
plugin_manager.delete_plugin_config(plugin_id, force=True)
plugin_manager.delete_plugin_data(plugin_id, force=True)
plugin_manager.delete_plugin_instance(plugin_id)
elif getattr(plugin_class, "is_clone", False):
# 如果是分身插件,则删除分身数据和配置
plugin_manager.delete_plugin_config(plugin_id)
plugin_manager.delete_plugin_data(plugin_id)
# 删除分身文件
plugin_base_dir = (
get_api_runtime_config_snapshot().root_path
/ "app"
/ "plugins"
/ plugin_id.lower()
)
if plugin_base_dir.exists():
try:
shutil.rmtree(plugin_base_dir)
plugin_manager.plugins.pop(plugin_id, None)
except Exception as e:
logger.error(
f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}"
)
# 从插件文件夹中移除该插件
remove_plugin_from_folders(plugin_id)
# 移除插件
plugin_manager.remove_plugin(plugin_id)
return _SchemaResponse(success=True)
except PluginMutationRejectedError as error:
return _SchemaResponse(success=False, message=str(error))
def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str):
+25 -13
View File
@@ -4,7 +4,9 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from typing import Any, ContextManager
from app.schemas.exception import PluginMutationRejectedError
@dataclass(frozen=True, slots=True)
@@ -29,6 +31,7 @@ class PluginConfigCommand:
reload_runtime: Callable[[str], Any],
publish_reset: Callable[[str], Any],
refresh_registrations: Callable[[str], Any],
mutation: Callable[[str], ContextManager[None]],
) -> None:
"""保存插件管理 Facade 和运行时注册刷新端口。"""
self._save_config = save_config
@@ -39,21 +42,30 @@ class PluginConfigCommand:
self._reload_runtime = reload_runtime
self._publish_reset = publish_reset
self._refresh_registrations = refresh_registrations
self._mutation = mutation
def update(self, plugin_id: str, config: dict) -> PluginConfigResult:
"""保存配置并按既有顺序重新初始化实例及运行时注册。"""
if not self._save_config(plugin_id, config, False):
return PluginConfigResult(False, "插件配置保存失败")
self._initialize(plugin_id, config)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
try:
with self._mutation(f"更新插件 {plugin_id} 配置"):
if not self._save_config(plugin_id, config, False):
return PluginConfigResult(False, "插件配置保存失败")
self._initialize(plugin_id, config)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
except PluginMutationRejectedError as error:
return PluginConfigResult(False, str(error))
def reset(self, plugin_id: str) -> PluginConfigResult:
"""通知插件补偿后停止实例、删除配置数据并重建运行态。"""
self._publish_reset(plugin_id)
self._stop(plugin_id)
self._delete_config(plugin_id, True)
self._delete_data(plugin_id, True)
self._reload_runtime(plugin_id)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
try:
with self._mutation(f"重置插件 {plugin_id} 配置和数据"):
self._publish_reset(plugin_id)
self._stop(plugin_id)
self._delete_config(plugin_id, True)
self._delete_data(plugin_id, True)
self._reload_runtime(plugin_id)
self._refresh_registrations(plugin_id)
return PluginConfigResult(True)
except PluginMutationRejectedError as error:
return PluginConfigResult(False, str(error))
+30 -14
View File
@@ -5,11 +5,12 @@ from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any, Optional
from typing import Any, ContextManager, Optional
from app.schemas.exception import PersistenceUnavailableError
from app.application.plugin.lifecycle import plugin_lifecycle
from app.runtime.log import logger
from app.schemas.exception import PluginMutationRejectedError
InstalledPluginsReader = Callable[[], list[str]]
@@ -25,6 +26,8 @@ PackageCheckpointAction = Callable[[Any], Awaitable[object]]
InstallReporter = Callable[[str, Optional[str]], Awaitable[object]]
PluginReloader = Callable[[str], Awaitable[object]]
PluginRegistrationRefresher = Callable[[str], Awaitable[object]]
PluginMutationAdmission = Callable[[str], ContextManager[None]]
PluginPackageWriteGuard = Callable[[str], ContextManager[None]]
@dataclass(frozen=True, slots=True)
@@ -94,6 +97,8 @@ class PluginInstallCommand:
install_reporter: InstallReporter,
plugin_reloader: PluginReloader,
registration_refresher: PluginRegistrationRefresher,
mutation: PluginMutationAdmission,
package_write_guard: PluginPackageWriteGuard,
) -> None:
"""保存安装用例所需端口,不绑定数据库、网络或运行时实现。"""
self._installed_plugins_reader = installed_plugins_reader
@@ -107,6 +112,8 @@ class PluginInstallCommand:
self._install_reporter = install_reporter
self._plugin_reloader = plugin_reloader
self._registration_refresher = registration_refresher
self._mutation = mutation
self._package_write_guard = package_write_guard
async def execute(
self,
@@ -120,20 +127,29 @@ class PluginInstallCommand:
state = _InstallState()
async with plugin_lifecycle.hold(plugin_id):
try:
return await self._execute_locked(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=force,
state=state,
with self._mutation(f"安装插件 {plugin_id}"):
with self._package_write_guard(plugin_id):
try:
return await self._execute_locked(
plugin_id=plugin_id,
repo_url=repo_url,
release_version=release_version,
force=force,
state=state,
)
except asyncio.CancelledError:
await self._rollback_cancelled(
plugin_id=plugin_id,
original_plugins=state.original_plugins,
state=state,
)
raise
except PluginMutationRejectedError as error:
return PluginInstallResult(
success=False,
message=str(error),
failure_stage="admission",
)
except asyncio.CancelledError:
await self._rollback_cancelled(
plugin_id=plugin_id,
original_plugins=state.original_plugins,
state=state,
)
raise
async def _execute_locked(
self,
+5 -2
View File
@@ -3,12 +3,15 @@
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Protocol
from typing import Any, ContextManager, Protocol
class PluginRuntime(Protocol):
"""声明入口层消费的插件宿主能力。"""
def mutation(self, operation: str) -> ContextManager[None]:
"""为完整插件可变事务取得停机准入 lease。"""
...
def __getattr__(self, name: str) -> Any:
"""允许兼容门面按既有 V3 方法名访问插件宿主能力。"""
+170 -29
View File
@@ -191,9 +191,16 @@ class TransferFailureNotificationAggregator:
NOTIFICATION_DEBOUNCE_SECONDS = 30
def __init__(self) -> None:
"""初始化分组缓冲和定时器"""
"""初始化分组缓冲、回调、定时器与关闭状态"""
self._buffers: dict[str, list[TransferFailureNotification]] = {}
self._callbacks: dict[
str,
Callable[[list[TransferFailureNotification]], None],
] = {}
self._timers: dict[str, asyncio.TimerHandle] = {}
self._generations: dict[str, int] = {}
self._lock = threading.Lock()
self._closed = False
def schedule(
self,
@@ -204,48 +211,128 @@ class TransferFailureNotificationAggregator:
loop: asyncio.AbstractEventLoop,
) -> None:
"""从整理线程安全地把失败快照加入事件循环中的聚合缓冲。"""
loop.call_soon_threadsafe(
self._schedule_on_loop,
group_key,
notification,
callback,
loop,
)
# 先在调用线程登记快照,关闭流程才能覆盖已接收但尚未进入事件循环的通知。
with self._lock:
if self._closed:
raise RuntimeError("整理失败通知聚合器正在关闭,不能再接收通知")
self._buffers.setdefault(group_key, []).append(notification)
self._callbacks[group_key] = callback
generation = self._generations.get(group_key, 0) + 1
self._generations[group_key] = generation
try:
loop.call_soon_threadsafe(
self._schedule_on_loop,
group_key,
generation,
callback,
loop,
)
except Exception as err:
logger.error(
f"创建整理失败通知聚合定时器失败,将立即发送 "
f"(group={group_key}): {err}"
)
self.flush(group_key, generation, callback)
def _schedule_on_loop(
self,
group_key: str,
notification: TransferFailureNotification,
generation: int,
callback: Callable[[list[TransferFailureNotification]], None],
loop: asyncio.AbstractEventLoop,
) -> None:
"""在所属事件循环中更新缓冲重置静默窗口。"""
self._buffers.setdefault(group_key, []).append(notification)
timer = self._timers.pop(group_key, None)
if timer:
timer.cancel()
self._timers[group_key] = loop.call_later(
self.NOTIFICATION_DEBOUNCE_SECONDS,
self.flush,
group_key,
callback,
)
"""在所属事件循环中为已登记缓冲重置静默窗口。"""
schedule_error: Optional[Exception] = None
with self._lock:
if (
self._closed
or group_key not in self._buffers
or self._generations.get(group_key) != generation
):
return
timer = self._timers.pop(group_key, None)
if timer:
timer.cancel()
try:
self._timers[group_key] = loop.call_later(
self.NOTIFICATION_DEBOUNCE_SECONDS,
self.flush,
group_key,
generation,
callback,
)
except Exception as err:
schedule_error = err
if schedule_error is not None:
logger.error(
f"创建整理失败通知聚合定时器失败,将立即发送 "
f"(group={group_key}): {schedule_error}"
)
self.flush(group_key, generation, callback)
def flush(
self,
group_key: str,
generation: int,
callback: Callable[[list[TransferFailureNotification]], None],
) -> None:
"""发送一个分组内的聚合结果并释放缓冲。"""
notifications = self._buffers.pop(group_key, [])
self._timers.pop(group_key, None)
with self._lock:
# 新通知已登记但 timer 重置回调尚未执行时,旧代不得提前发送新批次。
if self._generations.get(group_key) != generation:
return
notifications = self._buffers.pop(group_key, [])
self._callbacks.pop(group_key, None)
timer = self._timers.pop(group_key, None)
self._generations.pop(group_key, None)
if timer:
timer.cancel()
if not notifications:
return
self._deliver(group_key, notifications, callback)
@staticmethod
def _deliver(
group_key: str,
notifications: list[TransferFailureNotification],
callback: Callable[[list[TransferFailureNotification]], None],
) -> None:
"""调用聚合通知回调,并统一观察发送异常。"""
try:
callback(notifications)
except Exception as err:
logger.error(f"发送整理失败聚合通知失败 (group={group_key}): {err}")
def close(self) -> None:
"""停止接收新通知,取消定时器并同步发送全部已缓冲通知。"""
with self._lock:
if self._closed and not self._buffers and not self._timers:
return
self._closed = True
timers = list(self._timers.values())
pending = []
orphaned = []
for group_key, notifications in self._buffers.items():
callback = self._callbacks.get(group_key)
if callback is None:
orphaned.append((group_key, len(notifications)))
continue
pending.append((group_key, notifications, callback))
self._timers.clear()
self._generations.clear()
self._buffers.clear()
self._callbacks.clear()
for timer in timers:
timer.cancel()
for group_key, notification_count in orphaned:
logger.error(
f"整理失败通知聚合缓冲缺少发送回调,无法刷新 "
f"(group={group_key}, count={notification_count})"
)
for group_key, notifications, callback in pending:
self._deliver(group_key, notifications, callback)
# 作业锁:JobManager 与 TransferChain 共享,保护整理作业视图。
job_lock = threading.Lock()
@@ -267,7 +354,8 @@ class JobManager:
# 记录仍由主程序整理线程直接执行的任务,避免把阻塞中的本地任务误判为失活
_active_executions: set[Tuple[str, str]] = set()
def __init__(self):
def __init__(self) -> None:
"""初始化当前进程内的整理作业状态。"""
self._job_view = {}
self._season_episodes = {}
self._meta_to_media_ids = {}
@@ -1011,25 +1099,68 @@ class JobManager:
class FailedRetryScheduler:
"""
负责失败整理记录的 debounce 聚合与 AI 重试调度
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度
缓冲不提供持久化保证关闭时会取消尚未触发的记录由上层 durable
工作流在后续阶段承接需要跨进程保证的重试意图
"""
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
def __init__(self):
def __init__(self) -> None:
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
super().__init__()
self._retry_transfer_buffer: dict[str, list[int]] = {}
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
self._retry_transfer_generations: dict[str, int] = {}
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
self._retry_transfer_lock = asyncio.Lock()
self._closed = False
async def close(self):
async def close(self) -> None:
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
async with self._retry_transfer_lock:
self._closed = True
timers = list(self._retry_transfer_timers.values())
buffered_count = sum(
len(history_ids)
for history_ids in self._retry_transfer_buffer.values()
)
self._retry_transfer_timers.clear()
self._retry_transfer_generations.clear()
self._retry_transfer_buffer.clear()
tasks = tuple(self._retry_transfer_tasks)
for timer in timers:
timer.cancel()
if buffered_count:
logger.warning(
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
)
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
if self._closed:
return
task = asyncio.create_task(
self._flush_retry_transfer(group_key, generation),
name="transfer.failed_retry.flush",
)
self._retry_transfer_tasks.add(task)
task.add_done_callback(self._observe_retry_transfer_task)
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
self._retry_transfer_tasks.discard(task)
if task.cancelled():
return
exception = task.exception()
if exception is not None:
logger.error(f"智能体重试整理后台任务异常: {exception}")
@staticmethod
def _build_retry_transfer_template_context(
@@ -1054,7 +1185,7 @@ class FailedRetryScheduler:
template_context=template_context,
)
async def schedule_retry(self, history_id: int, group_key: str = ""):
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
"""
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用
"""
@@ -1062,6 +1193,8 @@ class FailedRetryScheduler:
group_key = f"_default_{history_id}"
async with self._retry_transfer_lock:
if self._closed:
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
if group_key not in self._retry_transfer_buffer:
self._retry_transfer_buffer[group_key] = []
if history_id not in self._retry_transfer_buffer[group_key]:
@@ -1075,18 +1208,26 @@ class FailedRetryScheduler:
self._retry_transfer_timers[group_key].cancel()
loop = asyncio.get_running_loop()
generation = self._retry_transfer_generations.get(group_key, 0) + 1
self._retry_transfer_generations[group_key] = generation
self._retry_transfer_timers[group_key] = loop.call_later(
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)),
self._start_retry_transfer_task,
group_key,
generation,
)
async def _flush_retry_transfer(self, group_key: str):
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
"""
延迟定时器到期后取出该分组的所有 history_id 并合并为一次 agent 调用
"""
async with self._retry_transfer_lock:
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
if self._retry_transfer_generations.get(group_key) != generation:
return
history_ids = self._retry_transfer_buffer.pop(group_key, [])
self._retry_transfer_timers.pop(group_key, None)
self._retry_transfer_generations.pop(group_key, None)
if not history_ids:
return
+335 -72
View File
@@ -2,9 +2,12 @@ import asyncio
import queue
import re
import threading
import time
import traceback
import uuid
from collections import Counter
from concurrent.futures import CancelledError as FutureCancelledError
from concurrent.futures import Future
from copy import deepcopy
from pathlib import Path
from typing import List, Optional, Tuple, Union, Dict, Callable, Any
@@ -87,10 +90,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
文件整理处理链
"""
# worker 在构造期启动;若中途失败,单例仍需先发布给 lifespan 清理入口。
_retain_failed_singleton = True
CONFIG_WATCH = {
"TRANSFER_THREADS",
}
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
_QUEUE_STOP_SENTINEL = object()
@staticmethod
def _transfer_result_payload(
task: TransferTask,
@@ -108,7 +118,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
"transfer_history_id": history_id,
}
def __init__(self):
def __init__(self) -> None:
"""初始化文件整理处理链。"""
super().__init__()
# 主要媒体文件后缀
@@ -141,7 +151,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._progress = ProgressHelper(ProgressKey.FileTransfer)
# 队列相关状态
self._threads = []
self._retiring_threads: List[threading.Thread] = []
self._queue_active = False
# 每一代 worker 使用独立停止信号,避免热更新启动新 worker 后旧线程重新取任务
self._worker_stop_event = threading.Event()
# 生命周期操作串行化;状态锁只保护短临界区,不覆盖同步文件 I/O 等待
self._worker_lifecycle_lock = threading.RLock()
self._worker_state_lock = threading.RLock()
self._closing = False
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
self._replay_thread: Optional[threading.Thread] = None
self._replay_stop_event = threading.Event()
self._active_tasks = 0
self._processed_num = 0
self._fail_num = 0
@@ -149,33 +169,210 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
# 启动整理任务
self.__init()
def __init(self):
"""
启动文件整理线程
"""
self._queue_active = True
for i in range(self.runtime_config.transfer_threads):
logger.info(f"启动文件整理线程 {i + 1} ...")
thread = threading.Thread(
target=self.__start_transfer, name=f"transfer-{i}", daemon=True
def __init(self) -> bool:
"""启动一代文件整理线程,并返回是否成功取得 worker 所有权。"""
with self._worker_lifecycle_lock:
with self._worker_state_lock:
if self._closing:
logger.warning("文件整理链已进入关闭状态,拒绝重新启动 worker")
return False
self._retiring_threads = [
thread for thread in self._retiring_threads if thread.is_alive()
]
alive_threads = [thread for thread in self._threads if thread.is_alive()]
if alive_threads:
logger.error(
"上一代文件整理线程尚未收敛,拒绝并行启动新 worker:%s",
", ".join(thread.name for thread in alive_threads),
)
self._threads = alive_threads
return False
stop_event = threading.Event()
threads = [
threading.Thread(
target=self.__start_transfer,
args=(stop_event,),
name=f"transfer-{index}",
daemon=True,
)
for index in range(self.runtime_config.transfer_threads)
]
self._worker_stop_event = stop_event
self._threads = threads
self._queue_active = True
for index, thread in enumerate(threads):
logger.info(f"启动文件整理线程 {index + 1} ...")
thread.start()
return True
@staticmethod
def __join_threads(
threads: List[threading.Thread], deadline: float
) -> List[threading.Thread]:
"""在统一截止时间内等待线程,返回仍未收敛且继续由调用方持有的线程。"""
current_thread = threading.current_thread()
for thread in threads:
if thread is current_thread or not thread.is_alive():
continue
thread.join(timeout=max(0.0, deadline - time.monotonic()))
return [thread for thread in threads if thread.is_alive()]
def __acquire_worker_lifecycle_lock(self, deadline: float) -> bool:
"""在统一截止时间内取得 worker 生命周期锁,并兼容同线程 RLock 重入。"""
return self._worker_lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
)
def __request_worker_stop(self) -> List[threading.Thread]:
"""发布当前 worker 代的停止信号,并用哨兵唤醒空闲线程。"""
with self._worker_state_lock:
self._queue_active = False
self._worker_stop_event.set()
current_threads = list(self._threads)
threads = [*self._retiring_threads, *current_threads]
for _ in current_threads:
self._queue.put(self._QUEUE_STOP_SENTINEL)
return threads
def __stop(self, timeout_seconds: float = _WORKER_RESTART_TIMEOUT_SECONDS) -> bool:
"""在锁等待与线程 join 的共享预算内停止当前 worker 代。"""
deadline = time.monotonic() + max(0.0, timeout_seconds)
if not self.__acquire_worker_lifecycle_lock(deadline):
logger.error(
"未在 %.1f 秒内取得文件整理 worker 生命周期锁",
max(0.0, timeout_seconds),
)
self._threads.append(thread)
thread.start()
return False
try:
threads = self.__request_worker_stop()
alive_threads = self.__join_threads(threads, deadline)
with self._worker_state_lock:
self._threads = []
self._retiring_threads = alive_threads
if alive_threads:
logger.error(
"文件整理线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_threads),
)
return False
logger.info("文件整理线程已停止")
return True
finally:
self._worker_lifecycle_lock.release()
def __stop(self):
def close_workers(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
"""
停止文件整理进程
"""
self._queue_active = False
for thread in self._threads:
thread.join()
self._threads = []
logger.info("文件整理线程已停止")
关闭整理 worker pending 回放并拒绝后续队列写入
def on_config_changed(self):
这是宿主生命周期使用的同步边界停止信号只能阻止线程领取下一项工作不能
取消已经进入同步文件或数据库 I/O 的调用超过预算时保留活线程句柄并返回
False调用方据此避免过早释放仍被使用的数据库等下游资源
:param timeout_seconds: 生命周期锁worker 与回放线程共享的最大等待秒数
:return: 全部后台线程均已收敛时返回 True否则返回 False
"""
deadline = time.monotonic() + max(0.0, timeout_seconds)
if not self.__acquire_worker_lifecycle_lock(deadline):
logger.error(
"未在 %.1f 秒内取得整理后台生命周期锁,关闭未开始",
max(0.0, timeout_seconds),
)
return False
try:
with self._worker_state_lock:
self._closing = True
worker_threads = self.__request_worker_stop()
replay_thread = self._replay_thread
self._replay_stop_event.set()
alive_workers = self.__join_threads(worker_threads, deadline)
alive_replays = self.__join_threads(
[replay_thread] if replay_thread else [], deadline
)
with self._worker_state_lock:
self._threads = []
self._retiring_threads = alive_workers
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
self._replay_thread = None
alive_threads = [*alive_workers, *alive_replays]
if alive_threads:
logger.error(
"整理后台线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_threads),
)
return False
logger.info("文件整理 worker 与待处理回放线程已关闭")
return True
finally:
self._worker_lifecycle_lock.release()
async def close(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
"""
收口整理线程失败通知和 AI 重试并返回依赖是否可以安全释放
同步文件 I/O 在线程内无法被 asyncio 取消因此先在线程池中执行有界
``close_workers``只有 worker replay 全部退出后才关闭通知和重试
超时则保留这些依赖供仍在运行的整理回调继续使用
:param timeout_seconds: worker replay 共享的最大等待秒数
:return: 所有整理后台 owner 均已收敛时返回 True
"""
workers_closed = await asyncio.to_thread(
self.close_workers,
timeout_seconds,
)
if not workers_closed:
return False
self.failure_notification_aggregator.close()
await self.retry_scheduler.close()
return True
@staticmethod
def _observe_failed_retry_schedule(future: Future[None]) -> None:
"""观察跨线程 AI 重试调度的完成结果,避免关闭竞态变成静默异常。"""
try:
future.result()
except FutureCancelledError:
return
except Exception as err:
logger.error(f"触发AI智能体重试整理失败: {err}")
def _schedule_failed_transfer_retry(
self,
history_id: int,
group_key: str,
) -> None:
"""把失败历史提交到主事件循环,并持续持有和观察调度结果。"""
retry_coroutine = self.retry_scheduler.schedule_retry(
history_id,
group_key=group_key,
)
try:
future = asyncio.run_coroutine_threadsafe(
retry_coroutine,
global_vars.loop,
)
except Exception as err:
retry_coroutine.close()
logger.error(f"触发AI智能体重试整理失败: {err}")
return
future.add_done_callback(self._observe_failed_retry_schedule)
logger.info(f"已触发AI智能体重试整理历史记录 #{history_id}")
def on_config_changed(self) -> None:
"""配置变更时重启文件整理线程。"""
self.__stop()
self.__init()
with self._worker_lifecycle_lock:
if self._closing:
logger.info("文件整理链正在关闭,忽略 worker 配置热更新")
return
if not self.__stop(
timeout_seconds=self._WORKER_RESTART_TIMEOUT_SECONDS
):
logger.warning(
"旧文件整理 worker 仍在收尾;其停止信号保持有效,新一代接管后续队列"
)
self.__init()
def __default_callback(
self, task: TransferTask, transferinfo: TransferInfo, /
@@ -344,19 +541,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
and self.runtime_config.ai_agent_enable
and self.runtime_config.ai_agent_retry_transfer
):
try:
# 使用 download_hash 或源文件父目录作为分组键,
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
group_key = build_transfer_failure_group_key(task)
asyncio.run_coroutine_threadsafe(
self.retry_scheduler.schedule_retry(
history.id, group_key=group_key
),
global_vars.loop,
)
logger.info(f"已触发AI智能体重试整理历史记录 #{history.id}")
except Exception as e:
logger.error(f"触发AI智能体重试整理失败: {e}")
# 使用 download_hash 或源文件父目录作为分组键,
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
self._schedule_failed_transfer_retry(
history.id,
build_transfer_failure_group_key(task),
)
# 返回失败
ret_status = False
@@ -653,7 +843,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
:param task: 任务信息
:return: True表示任务已添加到队列False表示任务无效或已存在重复
"""
return self._transfer_queue_service().put(task, self.__default_callback)
with self._worker_state_lock:
if self._closing:
logger.warning("文件整理链已关闭,拒绝新的队列任务")
return False
return self._transfer_queue_service().put(task, self.__default_callback)
def _transfer_queue_service(self) -> TransferQueueService:
"""构建保持旧队列对象和私有兼容接缝的应用服务。"""
@@ -667,26 +861,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
expire_tasks=self.__expire_stale_transfer_tasks,
)
def replay_pending(self):
def replay_pending(self) -> None:
"""
回放上次进程退出时仍未整理完的文件
在后台线程执行回放要 stat 源文件而启动期挂载可能尚未就绪甚至处于
挂死状态同步执行会把整个启动流程堵住
"""
threading.Thread(
target=self.__replay_pending,
name="MoviePilot-TransferReplay",
daemon=True
).start()
with self._worker_state_lock:
if self._closing:
logger.info("文件整理链正在关闭,跳过待处理文件回放")
return
if self._replay_thread and self._replay_thread.is_alive():
logger.info("待处理文件回放已在运行,跳过重复启动")
return
stop_event = threading.Event()
thread = threading.Thread(
target=self.__run_replay_pending,
args=(stop_event,),
name="MoviePilot-TransferReplay",
daemon=True,
)
self._replay_stop_event = stop_event
self._replay_thread = thread
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
thread.start()
def __replay_pending(self):
def __run_replay_pending(self, stop_event: threading.Event) -> None:
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
try:
self.__replay_pending(stop_event)
finally:
with self._worker_state_lock:
if self._replay_thread is threading.current_thread():
self._replay_thread = None
def __replay_pending(
self, stop_event: Optional[threading.Event] = None
) -> None:
"""
把落盘登记的待整理文件重新送回整理入口
只回放存储 + 源路径这一最小事实重新走完整的识别与整理流程
已经整理完成的由整理历史查重挡掉因此不存在重复整理的问题
:param stop_event: 宿主关闭信号只阻止处理下一条登记不取消运行中的同步 I/O
"""
stop_event = stop_event or threading.Event()
if stop_event.is_set():
return
try:
pendings = self._pendingoper.list_all()
except Exception as err:
@@ -697,8 +919,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
replayed = 0
for storage, src_path in pendings:
if stop_event.is_set():
break
try:
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
if stop_event.is_set():
break
if not fileitem:
if should_discard:
# 源文件确认已消失,注销登记避免每次启动重复回放
@@ -708,7 +935,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
replayed += 1
except Exception as err:
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
if stop_event.is_set():
logger.info(
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
replayed,
)
else:
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
@staticmethod
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
@@ -876,21 +1109,69 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self.jobview.try_remove_job(task)
self._finish_scrape_batch_task(task)
def __start_transfer(self):
def __settle_transfer_progress_if_idle(self) -> None:
"""在没有 active 或未结算真实任务时结束进度并重置本批计数。"""
with task_lock:
# unfinished_tasks 同时覆盖队列内任务和已被其他 worker 取走、尚未来得及
# 登记 active 的任务。持有 Queue 自身互斥锁完成判断和计数重置,使并发
# enqueue 只能发生在旧批次归零之后;仍在 deque 中的停止哨兵不算真实任务。
with self._queue.all_tasks_done:
queued_stop_sentinels = sum(
item is self._QUEUE_STOP_SENTINEL
for item in self._queue.queue
)
has_unsettled_tasks = (
self._queue.unfinished_tasks > queued_stop_sentinels
)
if (
self._active_tasks != 0
or self._processed_num <= 0
or has_unsettled_tasks
):
return
processed_num = self._processed_num
fail_num = self._fail_num
self._total_num = 0
self._processed_num = 0
self._fail_num = 0
__end_msg = (
f"整理队列处理完成,共整理 {processed_num} 个文件,"
f"失败 {fail_num}"
)
logger.info(__end_msg)
self._progress.update(value=100, text=__end_msg)
self._progress.end()
def __start_transfer(self, stop_event: threading.Event) -> None:
"""
处理队列
处理当前 worker 代的队列停止后不领取下一项任务
:param stop_event: 当前 worker 代专属停止信号热更新后不会被重新清除
"""
while not global_vars.is_system_stopped and self._queue_active:
while not global_vars.is_system_stopped and not stop_event.is_set():
try:
item: TransferQueue = self._queue.get(
block=True, timeout=self._transfer_interval
)
if item is self._QUEUE_STOP_SENTINEL:
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
if stop_event.is_set() or global_vars.is_system_stopped:
break
continue
if stop_event.is_set() or global_vars.is_system_stopped:
# 关闭信号与 queue.get 竞态时,把尚未处理的任务放回队列;其
# TransferPending 登记保持不变,供同进程重启 worker 或下次启动回放。
self._queue.put(item)
self._queue.task_done()
break
if not item:
continue
task = item.task
if not task:
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
continue
# 文件信息
@@ -966,18 +1247,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
with task_lock:
# 减少运行中的任务数
self._active_tasks -= 1
# 检查是否所有任务都已完成且队列为空
if self._active_tasks == 0 and self._queue.empty():
# 结束进度
__end_msg = f"整理队列处理完成,共整理 {self._processed_num} 个文件,失败 {self._fail_num}"
logger.info(__end_msg)
self._progress.update(value=100, text=__end_msg)
self._progress.end()
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
self._total_num = 0
self._processed_num = 0
self._fail_num = 0
self.__settle_transfer_progress_if_idle()
except queue.Empty:
# 即使队列空了,如果还有任务在运行,也不应该结束进度
@@ -1113,18 +1383,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
and self.runtime_config.ai_agent_enable
and self.runtime_config.ai_agent_retry_transfer
):
try:
# 使用 download_hash 或源文件父目录作为分组键
group_key = build_transfer_failure_group_key(task)
asyncio.run_coroutine_threadsafe(
self.retry_scheduler.schedule_retry(
his.id, group_key=group_key
),
global_vars.loop,
)
logger.info(f"已触发AI智能体重试整理历史记录 #{his.id}")
except Exception as e:
logger.error(f"触发AI智能体重试整理失败: {e}")
# 使用 download_hash 或源文件父目录作为分组键
self._schedule_failed_transfer_retry(
his.id,
build_transfer_failure_group_key(task),
)
return False, "未识别到媒体信息"
+24 -4
View File
@@ -14,14 +14,24 @@ class Singleton(abc.ABCMeta, type):
def get_existing_instance(cls, *args, **kwargs):
"""按相同参数返回已创建实例,不触发初始化"""
key = (cls, args, frozenset(kwargs.items()))
return cls._instances.get(key)
with cls._lock:
return cls._instances.get(key)
def __call__(cls, *args, **kwargs):
"""按类和构造参数创建或复用实例。"""
key = (cls, args, frozenset(kwargs.items()))
with cls._lock:
if key not in cls._instances:
cls._instances[key] = super().__call__(*args, **kwargs)
if getattr(cls, "_retain_failed_singleton", False):
# 启动线程的 lifecycle owner 必须先发布身份再执行 __init__;
# 构造中途抛错时保留实例,启动失败清理才能找到已创建的 owner。
instance = cls.__new__(cls, *args, **kwargs)
if not isinstance(instance, cls):
return instance
cls._instances[key] = instance
cls.__init__(instance, *args, **kwargs)
else:
cls._instances[key] = super().__call__(*args, **kwargs)
return cls._instances[key]
@@ -42,13 +52,23 @@ class SingletonClass(abc.ABCMeta, type):
def get_existing_instance(cls):
"""返回已创建实例,不触发初始化"""
return cls._instances.get(cls)
with cls._lock:
return cls._instances.get(cls)
def __call__(cls, *args, **kwargs):
"""按类创建或复用唯一实例。"""
with cls._lock:
if cls not in cls._instances:
cls._instances[cls] = super().__call__(*args, **kwargs)
if getattr(cls, "_retain_failed_singleton", False):
# 与参数化单例保持相同的 owner 发布顺序;锁会阻止其他线程
# 在 __init__ 返回或抛错前读取半构造实例。
instance = cls.__new__(cls, *args, **kwargs)
if not isinstance(instance, cls):
return instance
cls._instances[cls] = instance
cls.__init__(instance, *args, **kwargs)
else:
cls._instances[cls] = super().__call__(*args, **kwargs)
return cls._instances[cls]
+399 -57
View File
@@ -1,4 +1,5 @@
import time
import threading
import traceback
from functools import partial
from pathlib import Path
@@ -27,6 +28,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
目录监控门面单例模式装配本地/远程监控维护生命周期与健康检查
"""
# watcher/scheduler 在构造期启动;异常时保留实例供启动失败屏障收口。
_retain_failed_singleton = True
# 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
@@ -54,11 +57,25 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
REBUILD_KEY_PREFIX = "rebuild:"
PROBE_KEY = "probe"
PENDING_KEY = "pending"
RELOAD_STOP_TIMEOUT = 30.0
LIFECYCLE_CLOSE_TIMEOUT = 90.0
def __init__(self):
def __init__(self) -> None:
"""初始化目录监控依赖、owner 注册表与当前 lifespan 状态。"""
super().__init__()
# 生命周期操作串行化;永久关闭请求用 Event 提前封口,不能被阻塞 I/O 挡住。
self._lifecycle_lock = threading.RLock()
self._owner_lock = threading.Lock()
self._work_stop_event = threading.Event()
self._shutdown_event = threading.Event()
self._closed = False
self._compensation_threads: Dict[int, threading.Thread] = {}
self._scheduler_shutdown_thread: Optional[threading.Thread] = None
self._scheduler_shutdown_succeeded = False
# 本地目录监控服务
self._watchers = []
# 已请求停止但尚未退出的 watcher,只保留 owner 身份,不再参与健康检查
self._retired_watchers = []
# 本地目录监控列表读写锁
self._watcher_lock = Lock()
# 启动失败待重试的本地监控配置
@@ -87,9 +104,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
alert_cb=self.__poller_alert)
# 启动目录监控和文件整理
self.init()
if not self.init():
raise RuntimeError("目录监控 owner 初始化未收敛")
def on_config_changed(self):
def on_config_changed(self) -> None:
"""配置变化时重建监控;lifespan 封口后拒绝重新启动。"""
if self._shutdown_event.is_set():
logger.info("目录监控已进入生命周期封口,忽略配置热更新")
return
self.init()
def get_reload_name(self):
@@ -118,6 +140,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
强制全量扫描并处理所有文件包括已存在的文件
"""
if not self.__accepting_work():
return False
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
@staticmethod
@@ -134,24 +158,61 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
return SnapshotStore.compare(old_snapshot, new_snapshot)
def init(self):
def init(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""在旧 owner 收敛后启动一代监控,关闭中的生命周期拒绝重开。"""
if self._shutdown_event.is_set():
logger.info("目录监控生命周期已关闭,跳过启动")
return False
deadline = time.monotonic() + max(0.0, timeout)
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("目录监控未在 %.1f 秒内取得重载所有权", max(0.0, timeout))
return False
try:
if self._shutdown_event.is_set():
return False
if not self.__stop_owned(deadline=deadline, close=False):
logger.error("旧目录监控 owner 未收敛,取消本次配置热更新")
return False
if self._shutdown_event.is_set():
return False
if not self._recovery.reopen():
logger.error("目录监控恢复线程仍未退出,取消本次配置热更新")
return False
if self._shutdown_event.is_set():
self._recovery.request_stop()
return False
self._work_stop_event.clear()
return self.__initialize_monitors()
finally:
self._lifecycle_lock.release()
def __initialize_monitors(self) -> bool:
"""
启动监控
在已取得生命周期所有权后启动监控
永久关闭请求可以在本方法阻塞于 FUSE 时从其他线程提前设置每个可能产生
owner 的边界都重新检查确保解冻后只能退出不能穿透旧生命周期
"""
# 停止现有任务
self.stop()
if not self.__accepting_work():
return False
# 读取目录配置
monitor_dirs = DirectoryHelper().get_download_dirs()
if not self.__accepting_work():
return False
if not monitor_dirs:
logger.info("未找到任何目录监控配置")
return
return True
messagehelper = MessageHelper()
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
valid_dirs = []
for mon_dir in monitor_dirs:
if not self.__accepting_work():
return False
if not mon_dir.library_path:
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
continue
@@ -172,6 +233,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
# 启动定时服务进程
if not self.__accepting_work():
return False
self._scheduler = BackgroundScheduler(timezone=get_runtime_setting("TZ"))
mon_storages: Dict[str, List[Path]] = {}
@@ -179,6 +242,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
local_started = 0
local_failed = 0
for mon_dir in monitor_dirs:
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
# 检查媒体库目录是不是下载目录的子目录
mon_path = Path(mon_dir.download_path)
target_path = Path(mon_dir.library_path)
@@ -197,6 +263,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
for storage, paths in mon_storages.items():
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
# 远程目录监控 - 使用智能间隔
# 先尝试加载已有快照获取文件数量
snapshot_data = self._store.load(storage)
@@ -238,6 +307,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
# 启动定时服务
if not self.__accepting_work():
self.__abort_interrupted_startup()
return False
if self._scheduler.get_jobs():
self._scheduler.print_jobs()
self._scheduler.start()
@@ -253,6 +325,47 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(summary)
else:
logger.info(summary)
return True
def __accepting_work(self) -> bool:
"""判断当前 monitor 代是否仍允许创建 owner 和派发整理工作。"""
return not self._work_stop_event.is_set() and not self._shutdown_event.is_set()
@property
def lifecycle_closed(self) -> bool:
"""返回当前 Monitor 是否已经被应用生命周期永久封口。"""
return self._closed or self._shutdown_event.is_set()
def reopen(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""确认旧 owner 已收敛后,为新的应用 lifespan 显式解除永久封口。"""
if not self.lifecycle_closed:
return True
deadline = time.monotonic() + max(0.0, timeout)
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("旧目录监控生命周期仍在收尾,无法重新开启")
return False
try:
if not self.__stop_owned(deadline=deadline, close=True):
logger.error("旧目录监控 owner 仍未收敛,无法重新开启")
return False
if not self._recovery.reopen():
logger.error("旧目录监控恢复线程仍存活,无法重新开启")
return False
self._closed = False
self._shutdown_event.clear()
return True
finally:
self._lifecycle_lock.release()
def __abort_interrupted_startup(self) -> None:
"""关闭请求穿透阻塞启动后,在当前生命周期锁内收拢已经创建的 owner。"""
logger.info("目录监控启动期间收到关闭请求,正在回收本次已创建 owner")
self.__stop_owned(
deadline=time.monotonic() + self.RELOAD_STOP_TIMEOUT,
close=self._shutdown_event.is_set(),
)
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
"""
@@ -261,6 +374,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param monitor_mode: 配置的监控模式
:return: 是否启动成功
"""
if not self.__accepting_work():
return False
logger.info(f"正在启动本地目录监控: {mon_path}")
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
@@ -298,6 +413,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 启动成功后再登记,避免失败的监控残留在列表中
watcher.start()
with self._watcher_lock:
if not self.__accepting_work():
watcher.stop()
if watcher.is_alive():
self._retired_watchers.append(watcher)
return False
self._watchers.append(watcher)
self._pending_locals = [
pending for pending in self._pending_locals
@@ -312,13 +432,15 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
return False
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception) -> None:
"""
处理本地目录监控启动失败登记待重试并按需告警
:param mon_path: 监控目录
:param monitor_mode: 配置的监控模式
:param err: 启动异常
"""
if not self.__accepting_work():
return
err_msg = str(err)
logger.error(f"启动本地目录监控失败: {mon_path}")
logger.error(f"错误详情: {err_msg}")
@@ -346,7 +468,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
f"将自动退避重试")
def watchdog(self):
def watchdog(self) -> None:
"""
目录监控健康检查检测监控线程状态并驱动恢复
@@ -356,8 +478,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
动作会把看门狗冻死在它自己要修复的挂载上随后停滞检测告警重试驱动
全部静默失效这正是全进程雪崩的起点
"""
if not self.__accepting_work():
return
try:
broken = self.__check_watchers()
if not self.__accepting_work():
return
self.__drive_recovery(broken)
except Exception as e:
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
@@ -372,6 +498,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
"""
with self._watcher_lock:
watchers = list(self._watchers)
self._retired_watchers = [
watcher for watcher in self._retired_watchers if watcher.is_alive()
]
isolated = set(self._isolated)
# 探测已确认挂载恢复的目录,本轮直接送去重建
resumed = list(self._pending_rebuild.values())
@@ -412,7 +541,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
broken.append(watcher)
return broken
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]) -> None:
"""
把所有会触碰挂载的恢复动作派发到一次性工作线程并等待有限时间
@@ -421,6 +550,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
可放弃的子进程探测来确认挂载何时恢复
:param broken: 需要重建的监控列表
"""
if not self.__accepting_work():
return
actions: Dict[str, Callable[[], None]] = {}
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
for watcher in broken:
@@ -432,6 +563,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
actions[self.PENDING_KEY] = self.__drive_pending
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
if not self.__accepting_work():
return
for key, state in results.items():
if state is RecoveryState.COMPLETED:
@@ -445,11 +578,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
f"{state.value}),将在后续健康检查周期重试: {key}")
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
def __enter_isolation(self, watcher: LocalDirectoryWatcher) -> None:
"""
将一个监控目录转入挂载级故障隔离停止对它的一切新访问等待探测恢复
:param watcher: 重建未能返回的监控
"""
if not self.__accepting_work():
return
key = str(watcher.watch_path)
with self._watcher_lock:
if key in self._isolated:
@@ -466,7 +601,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
stage="isolated")
def __probe_isolated(self):
def __probe_isolated(self) -> None:
"""
对隔离中的监控目录做可放弃探测挂载恢复应答后解除隔离并重建监控
@@ -474,11 +609,17 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
超时可被 kill因此本线程不会像内联 stat 那样永久冻死探测通过后的重建
仍有极小概率再次卡住届时本线程会被下一轮的 BUSY 判定跳过不再泄漏
"""
if not self.__accepting_work():
return
with self._watcher_lock:
keys = list(self._isolated)
for key in keys:
if not self.__accepting_work():
return
mon_path = Path(key)
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
if not self.__accepting_work():
return
with self._watcher_lock:
entry = self._isolated.get(key)
if not entry:
@@ -489,6 +630,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
continue
if not self.__accepting_work():
return
with self._watcher_lock:
entry = self._isolated.pop(key, None)
if not entry:
@@ -502,19 +645,24 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
with self._watcher_lock:
self._pending_rebuild[key] = entry["watcher"]
def __drive_pending(self):
def __drive_pending(self) -> None:
"""
驱动两条待重试队列两者都会访问挂载启动重试走目录遍历与 exists
整理重试走 stat必须在恢复工作线程里执行而不是看门狗线程里
"""
if not self.__accepting_work():
return
self.__retry_pending_locals()
self._dispatcher.retry_pending()
if self.__accepting_work():
self._dispatcher.retry_pending()
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher) -> None:
"""
重建一个本地目录监控线程
:param watcher: 需要重建的监控
"""
if not self.__accepting_work():
return
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
watcher.stop()
new_watcher = LocalDirectoryWatcher(
@@ -527,8 +675,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
new_watcher.start()
except Exception as e:
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
if not self.__accepting_work():
return
with self._watcher_lock:
# 旧 watcher 可能仍卡在 FUSE 调用里;只有真实退出后才能移除句柄。
self._watchers = [item for item in self._watchers if item is not watcher]
if watcher.is_alive():
self._retired_watchers.append(watcher)
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
self._pending_locals.append({
"mon_path": watcher.watch_path,
@@ -537,9 +690,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
})
return
with self._watcher_lock:
if not self.__accepting_work():
new_watcher.stop()
if new_watcher.is_alive():
self._retired_watchers.append(new_watcher)
return
registered = any(item is watcher for item in self._watchers)
if registered:
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
if watcher.is_alive():
# stop() 只能发信号,FUSE 上的旧线程可能永久不返回;继续持有它,
# 让生命周期关闭如实失败,而不是把 daemon 泄漏伪装成已收敛。
self._retired_watchers.append(watcher)
if not registered:
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
@@ -547,6 +709,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 派发事件的孤儿线程,必须就地停掉
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
new_watcher.stop()
if new_watcher.is_alive():
with self._watcher_lock:
self._retired_watchers.append(new_watcher)
return
# 新监控的重启计数从零开始,同步重置告警基准
self._restart_marks.pop(str(watcher.watch_path), None)
@@ -557,7 +722,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self.__start_compensation(mon_path=watcher.watch_path,
since=watcher.last_activity_time)
def __start_compensation(self, mon_path: Path, since: float):
def __start_compensation(self, mon_path: Path, since: float) -> None:
"""
在后台线程发起补偿扫描避免遍历目录阻塞健康检查周期
:param mon_path: 监控目录
@@ -567,14 +732,34 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
return
Thread(
target=self.__compensate_scan,
thread = Thread(
target=self.__run_compensation,
kwargs={"mon_path": mon_path, "since": since},
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
daemon=True
).start()
)
with self._owner_lock:
# 与 close() 的 owner 快照共用同一把锁;封口先发生时绝不再启动,
# 登记先发生时 close() 必然能看到并等待这条线程。
if not self.__accepting_work():
return
self._compensation_threads[id(thread)] = thread
try:
thread.start()
except Exception:
self._compensation_threads.pop(id(thread), None)
raise
def __compensate_scan(self, mon_path: Path, since: float):
def __run_compensation(self, mon_path: Path, since: float) -> None:
"""执行补偿扫描,并在真实终态后释放 owner 句柄。"""
current_thread = threading.current_thread()
try:
self.__compensate_scan(mon_path=mon_path, since=since)
finally:
with self._owner_lock:
self._compensation_threads.pop(id(current_thread), None)
def __compensate_scan(self, mon_path: Path, since: float) -> None:
"""
补扫监控停摆期间落地的文件
@@ -586,7 +771,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param mon_path: 监控目录
:param since: 停摆起点墙钟时间戳仅用于统计与日志
"""
if not self.__accepting_work():
return
candidates = self.__collect_compensation_files(mon_path)
if not self.__accepting_work():
return
if candidates is None:
return
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
@@ -599,6 +788,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
handled = 0
for file_path, file_modify_time, file_size in candidates:
if not self.__accepting_work():
return
if self._dispatcher.handle_file(
storage="local",
event_path=file_path,
@@ -634,14 +825,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
return None
return candidates
def __retry_pending_locals(self):
def __retry_pending_locals(self) -> None:
"""
重试启动失败的本地目录监控给网络存储/FUSE 挂载留出就绪时间
"""
if not self.__accepting_work():
return
with self._watcher_lock:
pending = list(self._pending_locals)
isolated = set(self._isolated)
for item in pending:
if not self.__accepting_work():
return
if str(item["mon_path"]) in isolated:
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
# 在「请求永不返回」的挂载上会再冻死一个线程
@@ -655,7 +850,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
item["attempts"] = item.get("attempts", 0) + 1
item["skip_cycles"] = min(item["attempts"], 10)
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault") -> None:
"""
推送目录监控异常告警同一目录在同一阶段仅推送一次
:param mon_path: 监控目录
@@ -664,6 +859,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
监控已暂停访问等待挂载恢复只按目录去重会把这条
关键消息吞掉因此阶段变化时重新推送
"""
if not self.__accepting_work():
return
key = str(mon_path)
with self._watcher_lock:
if self._alerted_paths.get(key) == stage:
@@ -672,7 +869,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
MessageHelper().put(message, title="目录监控")
@staticmethod
def __poller_alert(storage: str, message: str):
def __poller_alert(storage: str, message: str) -> None:
"""
远程轮询监控告警回调复用消息渠道推送
:param storage: 存储名称
@@ -681,12 +878,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.warn(f"[{storage}] {message}")
MessageHelper().put(message, title="目录监控")
def __clear_alert(self, mon_path: Path, message: str):
def __clear_alert(self, mon_path: Path, message: str) -> None:
"""
清除目录监控异常告警状态并在此前告警过时推送恢复消息
:param mon_path: 监控目录
:param message: 恢复内容
"""
if not self.__accepting_work():
return
key = str(mon_path)
with self._watcher_lock:
if key not in self._alerted_paths:
@@ -695,12 +894,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.info(message)
MessageHelper().put(message, title="目录监控")
def polling_observer(self, storage: str, mon_paths: List[Path]):
def polling_observer(self, storage: str, mon_paths: List[Path]) -> None:
"""
轮询监控执行一轮快照并按结果动态调整监控间隔
"""
if not self.__accepting_work():
return
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
if file_count is None or not self._scheduler:
if not self.__accepting_work() or file_count is None or not self._scheduler:
return
# 动态调整监控间隔
new_interval = SnapshotStore.adjust_interval(file_count)
@@ -716,7 +917,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
except Exception as e:
logger.error(f"调整监控间隔失败: {storage} - {e}")
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
def event_handler(
self, event: Any, text: str, event_path: str,
file_size: Optional[float] = None
) -> None:
"""
处理文件变化
:param event: 事件
@@ -724,7 +928,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
:param event_path: 事件文件路径
:param file_size: 文件大小
"""
if event.is_directory:
if not self.__accepting_work() or event.is_directory:
return
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
return
@@ -733,6 +937,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
file_modify_time = Path(event_path).stat().st_mtime
except OSError as err:
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
if not self.__accepting_work():
return
# 整理文件
handle_kwargs = {
"storage": "local",
@@ -743,55 +949,191 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
handle_kwargs["file_modify_time"] = file_modify_time
self._dispatcher.handle_file(**handle_kwargs)
def event_unreadable(self, event_path: Path):
def event_unreadable(self, event_path: Path) -> None:
"""
处理读取失败的监控事件登记待重试
:param event_path: 事件文件路径
"""
if not self.__accepting_work():
return
event_path = Path(event_path)
if not self._dispatcher.is_transfer_candidate_path(event_path):
return
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
def stop(self):
def stop(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
"""在共享预算内临时停止全部监控 owner,供配置热重载使用。"""
return self.__stop_with_budget(timeout=timeout, close=False)
def close(self, timeout: float = LIFECYCLE_CLOSE_TIMEOUT) -> bool:
"""永久封口当前 lifespan,并在共享预算内等待全部监控 owner。"""
return self.__stop_with_budget(timeout=timeout, close=True)
def __stop_with_budget(self, timeout: float, close: bool) -> bool:
"""从等待生命周期锁开始计算一次停止操作的完整预算。"""
timeout = max(0.0, timeout)
deadline = time.monotonic() + timeout
# 先封住工作入口再等待锁。配置重载若正阻塞在 FUSE 上,解冻后也只能清理,
# 不能在外层生命周期已经超时返回后继续创建 watcher 或派发整理任务。
self._work_stop_event.set()
if close:
self._shutdown_event.set()
if not self._lifecycle_lock.acquire(
timeout=max(0.0, deadline - time.monotonic())
):
logger.error("目录监控未在 %.1f 秒内取得停机所有权", timeout)
return False
try:
return self.__stop_owned(deadline=deadline, close=close)
finally:
self._lifecycle_lock.release()
def __stop_owned(self, deadline: float, close: bool) -> bool:
"""
退出监控
在已取得生命周期锁时等待全部 owner共享调用方给出的绝对截止时间
超时只报告未收敛并保留所有活句柄只有 watcher补偿扫描恢复动作及
scheduler shutdown 都到达终态后才清空这一代的内存状态
"""
# 先停定时服务,避免健康检查在停止过程中重建监控线程
if self._scheduler:
self._scheduler.remove_all_jobs()
if self._scheduler.running:
try:
self._scheduler.shutdown()
logger.info("定时监控服务已停止")
except Exception as e:
logger.error(f"停止定时服务出现了错误:{e}")
self._scheduler = None
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
self._work_stop_event.set()
if close:
self._closed = True
self._shutdown_event.set()
self._recovery.request_stop()
self.__request_scheduler_stop()
with self._watcher_lock:
watchers = tuple(self._watchers) + tuple(self._retired_watchers)
if watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in watchers:
try:
watcher.stop()
except Exception as err:
logger.error(f"请求停止目录监控服务出现了错误:{err}")
recovery_converged = self._recovery.close(deadline=deadline)
current_thread = threading.current_thread()
# 恢复线程可能在封口后才从 FUSE 调用返回,并登记一个已经请求停止的新
# watcherRecoveryExecutor 收敛后重新取快照,确保它也使用剩余预算 join。
with self._watcher_lock:
registered_watchers = tuple(self._watchers) + tuple(self._retired_watchers)
for watcher in registered_watchers:
try:
watcher.stop()
watcher.join(timeout=max(0.0, deadline - time.monotonic()))
except Exception as err:
logger.error(f"等待目录监控服务停止出现了错误:{err}")
with self._owner_lock:
compensation_threads = tuple(self._compensation_threads.values())
scheduler_thread = self._scheduler_shutdown_thread
for thread in compensation_threads:
if thread is not current_thread and thread.is_alive():
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if (
scheduler_thread is not None
and scheduler_thread is not current_thread
and scheduler_thread.is_alive()
):
scheduler_thread.join(timeout=max(0.0, deadline - time.monotonic()))
with self._watcher_lock:
alive_watchers = tuple(
watcher
for watcher in (*self._watchers, *self._retired_watchers)
if watcher.is_alive()
)
with self._owner_lock:
alive_compensations = tuple(
thread for thread in self._compensation_threads.values()
if thread.is_alive()
)
scheduler_thread = self._scheduler_shutdown_thread
scheduler_converged = (
self._scheduler is None
or (
scheduler_thread is not None
and not scheduler_thread.is_alive()
and self._scheduler_shutdown_succeeded
)
)
alive_recoveries = self._recovery.running_threads()
if (
alive_watchers
or alive_compensations
or alive_recoveries
or not recovery_converged
or not scheduler_converged
):
logger.error(
"目录监控 owner 未在截止时间内收敛:watcher=%d,补偿=%d,恢复=%dscheduler=%s",
len(alive_watchers),
len(alive_compensations),
len(alive_recoveries),
"未收敛" if not scheduler_converged else "已收敛",
)
return False
# 仅在所有 owner 真实终止后清理业务状态,确保失败重试仍能找到原句柄。
self._dispatcher.clear_pending()
with self._watcher_lock:
watchers = self._watchers
self._watchers = []
self._retired_watchers = []
self._pending_locals = []
self._alerted_paths = {}
self._restart_marks = {}
self._stable_cycles = {}
self._isolated = {}
self._pending_rebuild = {}
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
# 被残留记录误判为 BUSY 而永远拿不到重建机会
self._recovery.clear()
with self._owner_lock:
self._compensation_threads.clear()
self._scheduler = None
self._scheduler_shutdown_thread = None
self._scheduler_shutdown_succeeded = False
if watchers:
logger.info("正在停止本地目录监控服务...")
for watcher in watchers:
try:
watcher.stop()
watcher.join(timeout=5)
if watcher.is_alive():
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
else:
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
except Exception as e:
logger.error(f"停止目录监控服务出现了错误:{e}")
logger.info("本地目录监控服务已停止")
return True
def __request_scheduler_stop(self) -> None:
"""创建并登记唯一 scheduler shutdown 线程,避免阻塞生命周期调用线程。"""
with self._owner_lock:
running = self._scheduler_shutdown_thread
if running is not None and running.is_alive():
return
if running is not None and self._scheduler_shutdown_succeeded:
return
scheduler = self._scheduler
if scheduler is None:
return
self._scheduler_shutdown_succeeded = False
thread = threading.Thread(
target=self.__shutdown_scheduler,
args=(scheduler,),
name="MoviePilot-MonitorSchedulerShutdown",
daemon=True,
)
self._scheduler_shutdown_thread = thread
try:
thread.start()
except Exception as err:
self._scheduler_shutdown_thread = None
logger.error(f"启动定时监控停止线程失败:{err}")
def __shutdown_scheduler(self, scheduler: BackgroundScheduler) -> None:
"""停止 scheduler 及其在途 job,并把真实终态写回 owner 注册表。"""
succeeded = False
try:
scheduler.remove_all_jobs()
if scheduler.running:
scheduler.shutdown(wait=True)
succeeded = True
logger.info("定时监控服务已停止")
except Exception as err:
logger.error(f"停止定时服务出现了错误:{err}")
finally:
with self._owner_lock:
if self._scheduler is scheduler:
self._scheduler_shutdown_succeeded = succeeded
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
+70 -15
View File
@@ -1,5 +1,5 @@
"""
监控恢复动作的可放弃执行单元
监控恢复动作的有限等待执行单元
FUSE/网络挂载有两种故障形态下游程序的免疫力完全不同
@@ -11,8 +11,8 @@ FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
本模块提供 block 型故障下唯一可行的两种自保手段
1. RecoveryExecutor 把会触碰挂载的动作放进一次性守护线程执行调用方只
等待有限时间超时即放弃该线程它会作为守护线程悬挂到进程退出换取
调用方健康检查这个全局自愈单点永远活着
等待有限时间超时后当前健康检查不再等待但线程句柄仍由生命周期 owner
持有到真实终态换取看门狗可继续检测且停机屏障不会伪装收敛
2. probe_path 用子进程而非线程做挂载探测子进程可以被 kill因此探测
本身是可放弃的隔离期间可以无限次周期重试而不累积不可回收的资源
"""
@@ -41,7 +41,7 @@ class RecoveryState(str, Enum):
"""
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
COMPLETED = "completed"
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
# 超时仍未返回,判定为 block 型挂载故障,本轮不再等待但继续持有线程
TIMEOUT = "timeout"
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
BUSY = "busy"
@@ -55,10 +55,12 @@ class RecoveryExecutor:
健康检查周期都会在同一个死挂载上多泄漏一个线程
"""
def __init__(self):
def __init__(self) -> None:
"""初始化在途线程注册表和当前生命周期的提交状态。"""
# key -> 该 key 最近一次提交的执行线程
self._running: Dict[str, threading.Thread] = {}
self._lock = threading.Lock()
self._accepting = True
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
"""
@@ -85,26 +87,70 @@ class RecoveryExecutor:
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if thread.is_alive():
results[key] = RecoveryState.TIMEOUT
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应,本轮不再等待: {key}")
else:
results[key] = RecoveryState.COMPLETED
return results
def discard(self, key: str):
def discard(self, key: str) -> None:
"""
丢弃一个 key 的在途记录监控停止或配置重载时调用避免残留条目
让重建后的同名目录被误判为 BUSY
清理一个已到终态的 key活线程继续保留供生命周期停机屏障追踪
:param key: 动作标识
"""
with self._lock:
self._running.pop(key, None)
thread = self._running.get(key)
if thread is None or not thread.is_alive():
self._running.pop(key, None)
def clear(self):
def clear(self) -> None:
"""
空全部在途记录已经冻死的线程无法回收这里只是不再跟踪它们
理已经完成的记录仍存活的线程继续由执行器持有
兼容旧调用名但不再丢弃挂死线程遗失句柄会让宿主错误释放它仍可能使用
的数据库和整理链资源
"""
with self._lock:
self._running = {
key: thread
for key, thread in self._running.items()
if thread.is_alive()
}
def request_stop(self) -> None:
"""封住新恢复动作提交,既有线程只能自然完成。"""
with self._lock:
self._accepting = False
def close(self, deadline: float) -> bool:
"""在绝对截止时间内等待全部恢复线程,超时继续保留活线程句柄。"""
self.request_stop()
with self._lock:
threads = tuple(self._running.values())
current_thread = threading.current_thread()
for thread in threads:
if thread is current_thread or not thread.is_alive():
continue
thread.join(timeout=max(0.0, deadline - time.monotonic()))
self.clear()
with self._lock:
return not any(thread.is_alive() for thread in self._running.values())
def reopen(self) -> bool:
"""在旧恢复线程全部结束后,为新的 Monitor 生命周期恢复提交。"""
self.clear()
with self._lock:
if any(thread.is_alive() for thread in self._running.values()):
return False
self._running.clear()
self._accepting = True
return True
def running_threads(self) -> tuple[threading.Thread, ...]:
"""返回仍存活且由执行器持有的恢复线程快照。"""
with self._lock:
return tuple(
thread for thread in self._running.values() if thread.is_alive()
)
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
"""
@@ -114,6 +160,8 @@ class RecoveryExecutor:
:return: 执行线程未启动时为 None
"""
with self._lock:
if not self._accepting:
return None
running = self._running.get(key)
if running is not None and running.is_alive():
return None
@@ -124,11 +172,18 @@ class RecoveryExecutor:
daemon=True
)
self._running[key] = thread
thread.start()
return thread
# 登记与 start 必须处于同一所有权临界区。否则 close() 可能把尚未
# is_alive() 的句柄当成已结束并移除,随后该线程才真正启动。
try:
thread.start()
except BaseException:
if self._running.get(key) is thread:
self._running.pop(key, None)
raise
return thread
@staticmethod
def _execute(key: str, action: Callable[[], None]):
def _execute(key: str, action: Callable[[], None]) -> None:
"""
执行一个恢复动作异常只记录不外抛避免一个目录的失败连累整批恢复
:param key: 动作标识
+10
View File
@@ -30,6 +30,9 @@ class EventDispatcher:
event_factory: Callable[..., Any],
error_handler: Callable[..., None],
async_handle_sink: Callable[[Any], bool] | None = None,
sync_handle_sink: (
Callable[[Callable[..., Any], tuple[Any, ...]], bool] | None
) = None,
) -> None:
"""注入注册表、绑定器、执行器和错误策略回调。"""
self._registry = registry
@@ -39,6 +42,7 @@ class EventDispatcher:
self._event_factory = event_factory
self._error_handler = error_handler
self._async_handle_sink = async_handle_sink
self._sync_handle_sink = sync_handle_sink
def dispatch_chain(self, event: Any) -> bool:
"""同步按优先级顺序执行链式事件快照。"""
@@ -134,6 +138,12 @@ class EventDispatcher:
event.event_type,
)
else:
if self._sync_handle_sink:
self._sync_handle_sink(
self.safe_invoke_sync,
(handler, isolated),
)
continue
self._executor().submit(
self.safe_invoke_sync,
handler,
+180 -26
View File
@@ -4,6 +4,7 @@ import random
import threading
import traceback
import uuid
from contextvars import ContextVar
from dataclasses import dataclass
from queue import Empty, PriorityQueue
from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type
@@ -32,6 +33,10 @@ MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间(秒)
MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间(秒)
_EVENT_STOP_SENTINEL = object()
_CURRENT_EVENT_HANDLER_OWNER: ContextVar[object | None] = ContextVar(
"current_event_handler_owner",
default=None,
)
@dataclass(slots=True)
@@ -121,10 +126,11 @@ class EventManager(metaclass=Singleton):
self.__lock = threading.Lock()
# 退出事件
self.__event = threading.Event()
# 广播异步处理器的生命周期由事件总线自己持有,避免关闭后仍向主循环运行
# 广播处理器由事件总线统一持有,确保插件卸载前可以建立结算屏障
self.__lifecycle_lock = threading.RLock()
self.__lifecycle_state = "new"
self.__async_handles: Dict[int, _EventAsyncHandle] = {}
self.__sync_handles: Dict[object, concurrent.futures.Future[Any]] = {}
self.__async_handles: Dict[object, _EventAsyncHandle] = {}
# 由上层管理器注册的处理器实例解析器
self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {}
# 由启动组合层注入的错误通知回调
@@ -155,6 +161,7 @@ class EventManager(metaclass=Singleton):
event_factory=Event,
error_handler=lambda **kwargs: self.__handle_event_error(**kwargs),
async_handle_sink=self.__register_async_handle,
sync_handle_sink=self.__register_sync_handle,
)
def register_handler_instance_resolver(
@@ -188,6 +195,9 @@ class EventManager(metaclass=Singleton):
if self.__lifecycle_state == "stopping":
logger.warning("事件处理仍在停止,忽略重复启动")
return
if self.__lifecycle_state == "sealed":
logger.warning("事件处理已封口,忽略重复启动")
return
self.__lifecycle_state = "running"
self.__event.set()
self.__consumer_threads = []
@@ -199,14 +209,19 @@ class EventManager(metaclass=Singleton):
def stop(self):
"""
停止广播事件处理线程
兼容同步关闭入口等待同步处理器并请求取消异步处理器
调用线程无法安全等待主事件循环完成异步清理需要完整异步收口时应使用
stop_async()
"""
logger.info("正在停止事件处理...")
consumer_threads = self.__begin_stop()
try:
self.__join_consumer_threads(consumer_threads)
self.__discard_stop_sentinels()
self.__cancel_async_handles()
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
self.__cancel_async_handles(exclude_owner=current_owner)
self.__wait_sync_handles(exclude_owner=current_owner)
logger.info("事件处理停止完成")
except Exception as e:
logger.error(f"停止事件处理线程出错:{str(e)} - {traceback.format_exc()}")
@@ -216,7 +231,7 @@ class EventManager(metaclass=Singleton):
self.__consumer_threads = []
async def stop_async(self) -> None:
"""停止广播消费者等待已投递的异步处理器收口"""
"""停止广播消费者等待同步处理器并取消收口异步处理器"""
logger.info("正在停止事件处理...")
consumer_threads = self.__begin_stop()
try:
@@ -226,15 +241,29 @@ class EventManager(metaclass=Singleton):
consumer_threads,
)
self.__discard_stop_sentinels()
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
for handle in handles:
async_handles = tuple(
handle
for owner, handle in self.__async_handles.items()
if owner is not current_owner
)
sync_handles = tuple(
handle
for owner, handle in self.__sync_handles.items()
if owner is not current_owner
)
for handle in async_handles:
handle.handle.cancel()
if handles:
if async_handles or sync_handles:
await asyncio.gather(
*(
asyncio.shield(asyncio.wrap_future(handle.completion))
for handle in handles
for handle in async_handles
),
*(
asyncio.shield(asyncio.wrap_future(handle))
for handle in sync_handles
),
return_exceptions=True,
)
@@ -272,16 +301,81 @@ class EventManager(metaclass=Singleton):
break
if item[1] is not _EVENT_STOP_SENTINEL:
pending.append(item)
self.__event_queue.task_done()
for item in pending:
self.__event_queue.put(item)
def __cancel_async_handles(self) -> None:
"""请求取消所有仍由事件总线持有的异步处理器。"""
def __cancel_async_handles(
self,
*,
exclude_owner: object | None = None,
) -> None:
"""请求取消异步处理器,并避免 handler 关闭事件总线时取消自身。"""
with self.__lifecycle_lock:
handles = tuple(self.__async_handles.values())
handles = tuple(
handle
for owner, handle in self.__async_handles.items()
if owner is not exclude_owner
)
for handle in handles:
handle.handle.cancel()
def __wait_sync_handles(
self,
*,
exclude_owner: object | None = None,
) -> None:
"""等待同步处理器完成,并避免 handler 关闭事件总线时等待自身。"""
with self.__lifecycle_lock:
handles = tuple(
handle
for owner, handle in self.__sync_handles.items()
if owner is not exclude_owner
)
if handles:
concurrent.futures.wait(handles)
def __register_sync_handle(
self,
callback: Callable[..., Any],
args: tuple[Any, ...],
) -> bool:
"""在同一生命周期临界区提交并登记同步广播处理器。"""
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
logger.warning(
"事件处理处于 %s 状态,拒绝同步广播处理器",
self.__lifecycle_state,
)
return False
try:
owner = object()
def _tracked_sync() -> Any:
"""在同步 handler 调用栈中发布当前事件 owner。"""
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
try:
return callback(*args)
finally:
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
handle = self.__executor.submit(_tracked_sync)
except RuntimeError:
logger.warning("同步事件处理器无法投递,线程池已停止")
return False
self.__sync_handles[owner] = handle
handle.add_done_callback(
lambda _completed, current_owner=owner: (
self.__remove_sync_handle(current_owner)
)
)
return True
def __remove_sync_handle(self, owner: object) -> None:
"""同步处理器完成后移除其 owner 句柄。"""
with self.__lifecycle_lock:
self.__sync_handles.pop(owner, None)
def __register_async_handle(
self,
coroutine: Any,
@@ -290,12 +384,18 @@ class EventManager(metaclass=Singleton):
with self.__lifecycle_lock:
if self.__lifecycle_state != "running":
coroutine.close()
logger.warning(
"事件处理处于 %s 状态,拒绝异步广播处理器",
self.__lifecycle_state,
)
return False
completion: concurrent.futures.Future[Any] = concurrent.futures.Future()
started = threading.Event()
owner = object()
async def _tracked() -> None:
started.set()
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
try:
result = await coroutine
except asyncio.CancelledError:
@@ -307,6 +407,8 @@ class EventManager(metaclass=Singleton):
else:
if not completion.done():
completion.set_result(result)
finally:
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
tracked = _tracked()
try:
@@ -316,7 +418,7 @@ class EventManager(metaclass=Singleton):
coroutine.close()
logger.warning("异步事件处理器无法投递,事件循环已停止")
return False
self.__async_handles[id(completion)] = _EventAsyncHandle(
self.__async_handles[owner] = _EventAsyncHandle(
handle=handle,
completion=completion,
)
@@ -329,13 +431,54 @@ class EventManager(metaclass=Singleton):
completion.cancel()
handle.add_done_callback(_complete_unstarted_submission)
completion.add_done_callback(self.__remove_async_handle)
completion.add_done_callback(
lambda _completed, current_owner=owner: (
self.__remove_async_handle(current_owner)
)
)
return True
def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None:
def __remove_async_handle(self, owner: object) -> None:
"""异步处理器完成后移除其 owner 句柄。"""
with self.__lifecycle_lock:
self.__async_handles.pop(id(handle), None)
self.__async_handles.pop(owner, None)
async def drain_async(
self,
timeout: Optional[float] = None,
*,
seal: bool = False,
) -> bool:
"""
等待已接纳的广播事件及其派生事件自然结算
seal=True 会在确认稳定空闲的同一临界区关闭后续广播提交供插件卸载前
建立不可穿透的投递屏障超时或停止交错时保持原提交状态并返回 False
广播处理器调用栈内无法等待自身完成因此会立即返回 False且不执行封口
"""
if _CURRENT_EVENT_HANDLER_OWNER.get() is not None:
logger.warning("事件处理器内部不能建立事件投递屏障")
return False
loop = asyncio.get_running_loop()
deadline = None if timeout is None else loop.time() + max(timeout, 0)
while True:
with self.__lifecycle_lock:
if self.__lifecycle_state not in {"running", "sealed"}:
return False
with self.__event_queue.all_tasks_done:
queue_idle = self.__event_queue.unfinished_tasks == 0
handlers_idle = not self.__sync_handles and not self.__async_handles
if queue_idle and handlers_idle:
if seal:
self.__lifecycle_state = "sealed"
return True
if deadline is not None:
remaining = deadline - loop.time()
if remaining <= 0:
return False
await asyncio.sleep(min(0.01, remaining))
else:
await asyncio.sleep(0.01)
def check(self, etype: Union[EventType, ChainEventType]) -> bool:
"""
@@ -470,7 +613,15 @@ class EventManager(metaclass=Singleton):
:param event: 要处理的事件对象
"""
logger.debug(f"Triggering broadcast event: {event}")
self.__event_queue.put((event.priority, event))
with self.__lifecycle_lock:
if self.__lifecycle_state in {"sealed", "stopping", "stopped"}:
logger.warning(
"事件处理处于 %s 状态,拒绝广播事件 %s",
self.__lifecycle_state,
event.event_type,
)
return None
self.__event_queue.put((event.priority, event))
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
@@ -580,15 +731,18 @@ class EventManager(metaclass=Singleton):
while self.__event.is_set():
try:
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
if event is _EVENT_STOP_SENTINEL:
break
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
rate_limiter.reset()
self.__dispatch_broadcast_event(event)
try:
if event is _EVENT_STOP_SENTINEL:
break
record_metric(
"event.queue.depth",
self.__event_queue.qsize(),
delivery="broadcast",
)
rate_limiter.reset()
self.__dispatch_broadcast_event(event)
finally:
self.__event_queue.task_done()
except Empty:
rate_limiter.current_wait = rate_limiter.current_wait * random.uniform(1, 1 + jitter_factor)
rate_limiter.trigger_limit()
+112
View File
@@ -0,0 +1,112 @@
"""插件宿主可变事务的停机准入。"""
from __future__ import annotations
import threading
from collections.abc import Iterator
from contextlib import contextmanager
from contextvars import ContextVar
from dataclasses import dataclass
from app.schemas.exception import PluginMutationRejectedError
@dataclass(slots=True)
class _MutationContext:
"""记录一个可跨协程和受控线程传播的事务上下文。"""
admission: "PluginMutationAdmission"
holders: int = 0
open: bool = True
class PluginMutationAdmission:
"""在停机封口与插件可变事务之间维护真实 owner 计数。"""
def __init__(self) -> None:
"""初始化开放准入、活动 owner 计数和事务上下文。"""
self._condition = threading.Condition()
self._accepting = True
self._active_count = 0
self._current_context: ContextVar[_MutationContext | None] = ContextVar(
"plugin_mutation_context",
default=None,
)
@property
def active_count(self) -> int:
"""返回尚未退出的 lease 数量,用于停机诊断和严格卸载判断。"""
with self._condition:
return self._active_count
@property
def accepting(self) -> bool:
"""返回当前生命周期是否仍接纳新的根事务。"""
with self._condition:
return self._accepting
def is_held(self) -> bool:
"""判断当前执行上下文是否持有仍有效的本 admission lease。"""
context = self._current_context.get()
return bool(
context
and context.admission is self
and context.open
and context.holders > 0
)
@contextmanager
def hold(self, operation: str) -> Iterator[None]:
"""取得可变事务 lease;封口后仅允许已获准事务的嵌套调用。"""
context = self._current_context.get()
nested = bool(
context
and context.admission is self
and context.open
and context.holders > 0
)
context_token = None
if not nested:
context = _MutationContext(admission=self)
context_token = self._current_context.set(context)
assert context is not None
acquired = False
try:
with self._condition:
if not self._accepting and not nested:
raise PluginMutationRejectedError(operation)
self._active_count += 1
context.holders += 1
acquired = True
yield
finally:
if acquired:
with self._condition:
self._active_count -= 1
context.holders -= 1
if context.holders == 0:
context.open = False
self._condition.notify_all()
if context_token is not None:
self._current_context.reset(context_token)
def seal(self) -> int:
"""原子停止接纳根事务,并返回封口瞬间的活动 lease 数量。"""
with self._condition:
self._accepting = False
return self._active_count
def wait_until_idle(self) -> None:
"""自然等待全部已获准 lease 退出,不取消或遗失其 owner。"""
with self._condition:
while self._active_count:
self._condition.wait()
def reopen(self) -> bool:
"""仅在没有遗留 lease 时为新的应用生命周期重新开放准入。"""
with self._condition:
if self._active_count:
return False
self._accepting = True
return True
+161 -29
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import traceback
from collections.abc import Callable
from functools import wraps
import threading
import time
from typing import Any, Optional, ParamSpec, TypeVar, cast
@@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
try:
result = func(*args, **kwargs)
statuses = result.values() if isinstance(result, dict) else (result,)
if PluginRuntimeStatus.LOAD_FAILED in statuses:
if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses:
outcome = "error"
return result
except BaseException:
@@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
class PluginLifecycle:
"""管理插件发现、初始化、启停和热重载,不持有市场或 HTTP 路由职责。"""
_EVENT_HANDLERS_QUIESCED = "__event_handlers__"
def __init__(
self,
*,
@@ -83,6 +86,8 @@ class PluginLifecycle:
self._runtime_status_writer = runtime_status_writer
self._logger = log
self._event_sender = event_sender
self._lifecycle_lock = threading.RLock()
self._quiesced_hooks: dict[str, set[str]] = {}
@observe_plugin_lifecycle("start")
def start(
@@ -116,6 +121,7 @@ class PluginLifecycle:
self._classes[current_id] = plugin
instance = plugin()
instance.init_plugin(self._plugin_config(current_id))
self._quiesced_hooks.pop(current_id, None)
self._running[current_id] = instance
self._logger.info(
f"加载插件:{current_id} 版本:{instance.plugin_version}"
@@ -156,31 +162,168 @@ class PluginLifecycle:
@observe_plugin_lifecycle("stop")
def stop(self, plugin_id: Optional[str] = None) -> None:
"""停止指定插件或全部插件,并清理模块缓存"""
"""按旧单阶段 ABI 先解绑 handler,再停止并强制卸载插件"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
self._quiesce_selected(plugins)
self._finalize(
plugin_id,
require_quiesced=False,
disable_events=not self._handlers_quiesced(plugins),
)
@observe_plugin_lifecycle("quiesce")
def quiesce(self, plugin_id: Optional[str] = None) -> bool:
"""先解绑事件 handler,再按旧 hook 顺序停止生产者并保留实例。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
return self._quiesce_selected(plugins)
@observe_plugin_lifecycle("quiesce_handlers")
def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool:
"""禁止目标插件接收新事件,保留实例供在途 handler 和后续 hook 使用。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
return self._disable_selected_handlers(plugins)
@observe_plugin_lifecycle("quiesce_services")
def quiesce_services(self, plugin_id: Optional[str] = None) -> bool:
"""在事件结算屏障后执行旧 close、stop_service hook。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
if not self._handlers_quiesced(plugins):
self._logger.warning("插件事件 handler 尚未全部停用,拒绝关闭插件资源")
return False
return self._quiesce_hooks(plugins)
def _quiesce_selected(self, plugins: dict[str, Any]) -> bool:
"""兼容单阶段调用:先停用全部 handler,再执行稳定快照的旧 hooks。"""
if not self._disable_selected_handlers(plugins):
return False
return self._quiesce_hooks(plugins)
def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool:
"""先停用稳定快照的全部事件入口,任一失败时不执行破坏性 hook。"""
all_converged = True
for current_id, plugin in plugins.items():
completed = self._quiesced_hooks.setdefault(current_id, set())
if self._EVENT_HANDLERS_QUIESCED in completed:
continue
try:
self._disable_events(type(plugin))
except Exception as error: # noqa: BLE001 插件边界必须隔离
all_converged = False
self._logger.warning(
f"停用插件 {current_id} 的事件 handler 时发生错误: {error}"
)
continue
completed.add(self._EVENT_HANDLERS_QUIESCED)
return all_converged
def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool:
"""执行旧 ABI hooks,并只重试尚未成功的步骤。"""
all_converged = True
for current_id, plugin in plugins.items():
completed = self._quiesced_hooks.setdefault(current_id, set())
for hook_name in ("close", "stop_service"):
if hook_name in completed:
continue
hook = getattr(plugin, hook_name, None)
if not callable(hook):
completed.add(hook_name)
continue
try:
result = hook()
except Exception as error: # noqa: BLE001 插件边界必须隔离
all_converged = False
self._logger.warning(
f"停止插件 {current_id}{hook_name} 时发生错误: {error}"
)
continue
if result is False:
all_converged = False
self._logger.warning(
f"停止插件 {current_id}{hook_name} 未收敛"
)
continue
completed.add(hook_name)
return all_converged
@observe_plugin_lifecycle("finalize")
def finalize(self, plugin_id: Optional[str] = None) -> bool:
"""在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。"""
return self._finalize(
plugin_id,
require_quiesced=True,
disable_events=False,
)
def _finalize(
self,
plugin_id: Optional[str],
*,
require_quiesced: bool,
disable_events: bool = True,
) -> bool:
"""按严格或兼容策略卸载插件,并在清理失败时保留实例所有权。"""
with self._lifecycle_lock:
plugins = self._select_running_plugins(plugin_id)
if require_quiesced and any(
not self._is_quiesced(current_id, plugin)
for current_id, plugin in plugins.items()
):
self._logger.warning("插件后台服务尚未全部收敛,拒绝卸载运行实例")
return False
try:
if disable_events:
for plugin in plugins.values():
self._disable_events(type(plugin))
self._clear_modules(plugin_id)
self._clear_tools()
except Exception as error: # noqa: BLE001 保留实例所有权供后续重试
self._logger.warning(f"卸载插件运行实例时发生错误: {error}")
return False
if plugin_id:
self._classes.pop(plugin_id, None)
self._running.pop(plugin_id, None)
self._quiesced_hooks.pop(plugin_id, None)
else:
self._classes.clear()
self._running.clear()
self._quiesced_hooks.clear()
self._logger.info("插件停止完成")
return True
def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]:
"""返回本阶段处理的稳定实例快照,并保持旧停机日志语义。"""
if plugin_id:
self._logger.info(f"正在停止插件 {plugin_id}...")
plugin = self._running.get(plugin_id)
plugins = {plugin_id: plugin} if plugin else {}
if not plugin:
self._logger.debug(f"插件 {plugin_id} 不存在或未加载")
else:
self._logger.info("正在停止所有插件...")
plugins = dict(self._running)
return plugins
self._logger.info("正在停止所有插件...")
return dict(self._running)
for current_id, plugin in plugins.items():
self._disable_events(type(plugin))
self._stop_plugin(plugin)
def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool:
"""判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。"""
required = {self._EVENT_HANDLERS_QUIESCED} | {
hook_name
for hook_name in ("close", "stop_service")
if callable(getattr(plugin, hook_name, None))
}
return required.issubset(self._quiesced_hooks.get(plugin_id, set()))
if plugin_id:
self._classes.pop(plugin_id, None)
self._running.pop(plugin_id, None)
self._clear_modules(plugin_id)
else:
self._classes.clear()
self._running.clear()
self._clear_modules(None)
self._clear_tools()
self._logger.info("插件停止完成")
def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool:
"""判断稳定快照中的全部插件是否已经停用事件入口。"""
return all(
self._EVENT_HANDLERS_QUIESCED
in self._quiesced_hooks.get(plugin_id, set())
for plugin_id in plugins
)
@observe_plugin_lifecycle("reload")
def reload(
@@ -194,14 +337,3 @@ class PluginLifecycle:
status = self.start(plugin_id)[plugin_id]
self._event_sender(reload_event, data={"plugin_id": plugin_id})
return status
def _stop_plugin(self, plugin: Any) -> None:
"""按插件旧 ABI 顺序关闭资源和服务。"""
try:
if hasattr(plugin, "close"):
plugin.close()
if hasattr(plugin, "stop_service"):
plugin.stop_service()
except Exception as error: # noqa: BLE001
name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__
self._logger.warning(f"停止插件 {name} 时发生错误: {error}")
+64 -21
View File
@@ -2,8 +2,8 @@
from __future__ import annotations
import time
import threading
import time
from collections.abc import Callable
from pathlib import Path
from typing import Any, Optional
@@ -27,6 +27,8 @@ class PluginMonitorController:
self._logger = log
self._thread: Optional[threading.Thread] = None
self._stop_event = threading.Event()
self._lifecycle_lock = threading.RLock()
self._closed = False
@property
def stop_event(self) -> threading.Event:
@@ -35,32 +37,73 @@ class PluginMonitorController:
def reload(self, enabled: bool) -> None:
"""按当前配置停止旧线程,并在启用时创建新线程。"""
self.stop()
if enabled:
stopped = self.stop()
if enabled and stopped:
self.start()
def start(self) -> None:
"""启动唯一的守护监控线程。"""
if self._thread and self._thread.is_alive():
self._logger.info("插件文件修改监测已经在运行中...")
return
self._logger.info("开始监测插件文件修改...")
self._stop_event.clear()
self._thread = threading.Thread(target=self._runner, daemon=True)
self._thread.start()
with self._lifecycle_lock:
if self._closed:
self._logger.info("插件文件修改监测已进入停机封口,跳过启动")
return
if self._thread and self._thread.is_alive():
self._logger.info("插件文件修改监测已经在运行中...")
return
self._logger.info("开始监测插件文件修改...")
self._stop_event.clear()
self._thread = threading.Thread(target=self._runner, daemon=True)
self._thread.start()
def stop(self) -> None:
"""请求监控线程退出,并在限定时间内等待其清理"""
if not self._thread or not self._thread.is_alive():
self._logger.info("未启用插件文件修改监测,无需停止")
return
self._logger.info("正在停止插件文件修改监测...")
def reopen(self) -> bool:
"""为新的应用生命周期解除封口,仍有旧线程时拒绝重开"""
with self._lifecycle_lock:
if self._thread and self._thread.is_alive():
self._logger.warning("旧插件文件监测线程仍在运行,无法开启新生命周期")
return False
self._thread = None
self._closed = False
return True
def stop(self, timeout: float = 5.0) -> bool:
"""临时停止监控线程,并返回其是否在预算内真正退出。"""
return self._stop_with_budget(timeout=timeout, close=False)
def close(self, timeout: float = 5.0) -> bool:
"""永久封口当前生命周期,并返回监控线程是否真正退出。"""
return self._stop_with_budget(timeout=timeout, close=True)
def _stop_with_budget(self, *, timeout: float, close: bool) -> bool:
"""在同一预算内取得生命周期锁、设置封口并等待线程退出。"""
timeout = max(0.0, timeout)
deadline = time.monotonic() + timeout
self._stop_event.set()
self._thread.join(timeout=5)
if self._thread.is_alive():
self._logger.warning("插件文件修改监测线程在5秒内未能正常停止")
self._thread = None
self._logger.info("插件文件修改监测停止完成")
if not self._lifecycle_lock.acquire(timeout=timeout):
self._logger.warning(
f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权"
)
return False
try:
if close:
self._closed = True
thread = self._thread
self._stop_event.set()
if not thread or not thread.is_alive():
self._thread = None
self._logger.info("未启用插件文件修改监测,无需停止")
return True
self._logger.info("正在停止插件文件修改监测...")
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if thread.is_alive():
self._logger.warning(
f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。"
)
return False
self._thread = None
self._logger.info("插件文件修改监测停止完成")
return True
finally:
self._lifecycle_lock.release()
class PluginChangeMonitor:
+293 -67
View File
@@ -1,9 +1,21 @@
import asyncio
import concurrent.futures
import inspect
import posixpath
import threading
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
from typing import (
Any,
Callable,
ContextManager,
Dict,
List,
Optional,
Tuple,
Type,
Union,
)
from watchfiles import watch
@@ -17,6 +29,7 @@ from app.runtime.execution import run_in_threadpool_to_completion
from app.runtime.log import logger
from app.runtime.observability import observe_compat_facade
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.thread import ThreadHelper
settings = RuntimeSettingsCompat()
from app.runtime.events import EventHandlerBinding, eventmanager
@@ -39,6 +52,7 @@ from app.runtime.extensions.plugin.sync import (
)
from app.runtime.extensions.plugin.clone import PluginCloneService
from app.runtime.extensions.plugin.access import PluginAccessPolicy
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
from app.runtime.extensions.plugin.catalog import PluginCatalogFacade
from app.runtime.extensions.plugin.paths import PluginPathResolver
from app.runtime.extensions.plugin.dependency import (
@@ -47,6 +61,7 @@ from app.runtime.extensions.plugin.dependency import (
PluginDependencyService,
)
from app.runtime.extensions.plugin.storage import PluginConfigStore, PluginInstanceStore
from app.schemas.exception import PluginMutationRejectedError
from app.schemas.types import EventType, SystemConfigKey
LegacyDiagnosticsConfigurator = Callable[..., None]
@@ -195,6 +210,15 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
runner=self._run_file_watcher,
log=logger,
)
self._plugin_quiesce_lock = threading.RLock()
self._plugin_quiesce_future: Optional[
concurrent.futures.Future[bool]
] = None
self._plugin_service_quiesce_future: Optional[
concurrent.futures.Future[bool]
] = None
self._plugin_mutation_admission = PluginMutationAdmission()
self._plugin_runtime_closed = False
self._plugin_dependencies = PluginDependencyService(
system=get_plugin_system,
log=logger,
@@ -315,12 +339,16 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
def init_config(self):
"""按最新系统配置完整重启插件。"""
# 停止已有插件
self.stop()
classification = self.classify_plugins()
self.apply_plugin_dependency_classification(classification)
for plugin_id in classification.ready:
self.start(plugin_id)
try:
with self.mutation("配置热重载"):
# 停止已有插件
self.stop()
classification = self.classify_plugins()
self.apply_plugin_dependency_classification(classification)
for plugin_id in classification.ready:
self.start(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def start(self, pid: Optional[str] = None) -> Dict[str, PluginRuntimeStatus]:
"""
@@ -328,8 +356,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID为空加载所有插件
"""
_legacy_diagnostics_configurator(enabled=settings.DEBUG, emitter=logger.warning)
return self._plugin_lifecycle.start(pid)
try:
with self.mutation("启动插件"):
with self._plugin_quiesce_lock:
_legacy_diagnostics_configurator(
enabled=settings.DEBUG,
emitter=logger.warning,
)
return self._plugin_lifecycle.start(pid)
except PluginMutationRejectedError as error:
logger.warning(str(error))
if pid:
return {pid: PluginRuntimeStatus.LOAD_FAILED}
return {}
def init_plugin(self, plugin_id: str, conf: dict):
"""
@@ -337,7 +376,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param plugin_id: 插件ID
:param conf: 插件配置
"""
self._plugin_lifecycle.initialize(plugin_id, conf)
try:
with self.mutation("初始化插件配置"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.initialize(plugin_id, conf)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def clear_plugin_agent_tools_cache(self) -> None:
"""
@@ -356,12 +400,127 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""兼容读取旧私有字段,实际版本由独立工具目录持有。"""
return self._plugin_tool_catalog.revision
def stop(self, pid: Optional[str] = None):
def stop(self, pid: Optional[str] = None) -> None:
"""
停止插件服务
:param pid: 插件ID为空停止所有插件
"""
self._plugin_lifecycle.stop(pid)
try:
with self.mutation("停止插件"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.stop(pid)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def mutation(self, operation: str) -> ContextManager[None]:
"""为一个完整插件可变事务取得可跨异步边界传播的准入 lease。"""
return self._plugin_mutation_admission.hold(operation)
def reopen_plugins(self) -> bool:
"""为新应用生命周期解除运行时封口,仍活跃的 quiesce owner 禁止复用。"""
with self._plugin_quiesce_lock:
futures = (
self._plugin_quiesce_future,
self._plugin_service_quiesce_future,
)
if any(future is not None and not future.done() for future in futures):
logger.warning("插件后台服务仍在停止,无法开启新的应用生命周期")
return False
if self._plugin_runtime_closed and self._running_plugins:
logger.warning("上一应用生命周期仍持有插件实例,拒绝解除运行时封口")
return False
if not self._plugin_mutation_admission.reopen():
logger.warning("上一应用生命周期仍有插件可变事务,拒绝解除运行时封口")
return False
self._plugin_runtime_closed = False
return True
async def quiesce_plugins(self, timeout: float = 240.0) -> bool:
"""封口变更事务并停用插件 handler,超时后保留 Future ownership。"""
if self._plugin_mutation_admission.is_held():
logger.warning("插件可变事务不能等待自身收敛,拒绝在事务内执行停机")
return False
with self._plugin_quiesce_lock:
self._plugin_runtime_closed = True
self._plugin_mutation_admission.seal()
future = self._plugin_quiesce_future
if future is None or future.done():
future = ThreadHelper().submit(self._quiesce_after_mutations)
self._plugin_quiesce_future = future
try:
result = await asyncio.wait_for(
asyncio.shield(asyncio.wrap_future(future)),
timeout=max(0.0, timeout),
)
return bool(result)
except asyncio.TimeoutError:
logger.error(f"插件后台服务未在 {timeout:g} 秒内收敛")
return False
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
logger.error(f"插件后台服务停止失败:{error}", exc_info=True)
return False
finally:
if future.done():
with self._plugin_quiesce_lock:
if self._plugin_quiesce_future is future:
self._plugin_quiesce_future = None
async def quiesce_plugin_services(self, timeout: float = 240.0) -> bool:
"""在事件结算后执行旧插件停机 hook,并有界等待同步 owner。"""
with self._plugin_quiesce_lock:
prepare_future = self._plugin_quiesce_future
if prepare_future is not None and not prepare_future.done():
logger.warning("插件事件入口仍在封口,拒绝提前关闭插件资源")
return False
if not self._plugin_runtime_closed:
logger.warning("插件运行时尚未封口,拒绝关闭插件资源")
return False
if self._plugin_mutation_admission.active_count:
logger.warning("插件可变事务仍在执行,拒绝关闭插件资源")
return False
future = self._plugin_service_quiesce_future
if future is None or future.done():
future = ThreadHelper().submit(
self._plugin_lifecycle.quiesce_services,
)
self._plugin_service_quiesce_future = future
try:
result = await asyncio.wait_for(
asyncio.shield(asyncio.wrap_future(future)),
timeout=max(0.0, timeout),
)
return bool(result)
except asyncio.TimeoutError:
logger.error(f"插件旧停机 hook 未在 {timeout:g} 秒内收敛")
return False
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
logger.error(f"插件旧停机 hook 执行失败:{error}", exc_info=True)
return False
finally:
if future.done():
with self._plugin_quiesce_lock:
if self._plugin_service_quiesce_future is future:
self._plugin_service_quiesce_future = None
def finalize_plugins(self) -> bool:
"""确认 quiesce owner 已结束后禁用 handler 并卸载插件实例。"""
with self._plugin_quiesce_lock:
futures = (
self._plugin_quiesce_future,
self._plugin_service_quiesce_future,
)
if any(future is not None and not future.done() for future in futures):
logger.warning("插件后台服务仍在停止,拒绝释放运行实例")
return False
if self._plugin_mutation_admission.active_count:
logger.warning("插件可变事务仍在执行,拒绝释放运行实例")
return False
return self._plugin_lifecycle.finalize()
def _quiesce_after_mutations(self) -> bool:
"""等待已获准变更自然结束后,再停用插件事件入口。"""
self._plugin_mutation_admission.wait_until_idle()
return self._plugin_lifecycle.quiesce_handlers()
@staticmethod
def _load_selective_plugins(pid: Optional[str], installed_plugins: List[str],
@@ -428,8 +587,10 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""返回配置重载日志使用的功能名称。"""
return "插件文件修改监测"
def start_monitor(self):
"""按当前配置启动插件文件修改监测"""
def start_monitor(self, *, reopen: bool = False) -> None:
"""按当前配置启动监控;新生命周期可显式解除既有封口"""
if reopen and not self._plugin_monitor.reopen():
return
if (
not self.is_plugin_settling()
and (settings.DEV or settings.PLUGIN_AUTO_RELOAD)
@@ -447,11 +608,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
)
)
def stop_monitor(self):
"""
停止监测插件文件修改监测
"""
self._plugin_monitor.stop()
def stop_monitor(self, timeout: float = 5.0) -> bool:
"""停止插件文件监控,并返回线程是否在预算内真正退出。"""
return self._plugin_monitor.stop(timeout=timeout)
def close_monitor(self, timeout: float = 5.0) -> bool:
"""封口当前生命周期的文件监控,并返回线程是否真正退出。"""
return self._plugin_monitor.close(timeout=timeout)
def _run_file_watcher(self):
"""
@@ -504,25 +667,31 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
"""
已安装本地插件源码变化时同步到运行目录
"""
return self._local_plugin_sync.sync(pid, candidate)
try:
with self.mutation("同步本地插件源码"):
return self._local_plugin_sync.sync(pid, candidate)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
@contextmanager
def suppress_plugin_monitor(self, plugin_id: str):
"""在插件目录原子更新期间阻止文件监控抢先重载半成品。"""
normalized_id = plugin_id.lower()
with self._monitor_suppression_lock:
self._suppressed_monitor_plugins[normalized_id] = (
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
)
try:
yield
finally:
with self.mutation("更新插件包"):
normalized_id = plugin_id.lower()
with self._monitor_suppression_lock:
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
if count <= 1:
self._suppressed_monitor_plugins.pop(normalized_id, None)
else:
self._suppressed_monitor_plugins[normalized_id] = count - 1
self._suppressed_monitor_plugins[normalized_id] = (
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
)
try:
yield
finally:
with self._monitor_suppression_lock:
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
if count <= 1:
self._suppressed_monitor_plugins.pop(normalized_id, None)
else:
self._suppressed_monitor_plugins[normalized_id] = count - 1
def is_plugin_monitor_suppressed(self, plugin_id: str) -> bool:
"""判断指定插件是否处于安装或替换写入阶段。"""
@@ -534,23 +703,43 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
从内存中移除一个插件
:param plugin_id: 插件ID
"""
self._plugin_lifecycle.stop(plugin_id)
self._plugin_registry.remove(plugin_id)
try:
with self.mutation("移除插件实例"):
with self._plugin_quiesce_lock:
self._plugin_lifecycle.stop(plugin_id)
self._plugin_registry.remove(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
def reload_plugin(self, plugin_id: str) -> PluginRuntimeStatus:
"""
将一个插件重新加载到内存
:param plugin_id: 插件ID
"""
return self._plugin_lifecycle.reload(plugin_id, EventType.PluginReload)
try:
with self.mutation("重新加载插件"):
with self._plugin_quiesce_lock:
return self._plugin_lifecycle.reload(
plugin_id,
EventType.PluginReload,
)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return PluginRuntimeStatus.LOAD_FAILED
def reload_plugin_tree(self, plugin_id: str) -> PluginRuntimeStatus:
"""重载源码插件,并同步刷新所有引用该源码的虚拟实例。"""
source_plugin_id = self.get_plugin_source_id(plugin_id)
status = self.reload_plugin(source_plugin_id)
for instance in self._plugin_instance_store.for_source(source_plugin_id):
self.reload_plugin(instance.instance_id)
return status
try:
with self.mutation("重载插件实例树"):
with self._plugin_quiesce_lock:
source_plugin_id = self.get_plugin_source_id(plugin_id)
status = self.reload_plugin(source_plugin_id)
for instance in self._plugin_instance_store.for_source(source_plugin_id):
self.reload_plugin(instance.instance_id)
return status
except PluginMutationRejectedError as error:
logger.warning(str(error))
return PluginRuntimeStatus.LOAD_FAILED
def get_plugin_reload_targets(self, plugin_id: str) -> List[str]:
"""返回源码更新后需要刷新注册信息的源插件及其实例 ID。"""
@@ -584,33 +773,40 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
安装本地不存在或需要更新的插件
"""
return self._plugin_sync.sync()
with self.mutation("同步插件包"):
return self._plugin_sync.sync()
@staticmethod
def install_plugin_missing_dependencies() -> List[str]:
"""
安装插件中缺失或不兼容的依赖项
"""
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing()
@staticmethod
def install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
"""安装插件缺失依赖并返回缺失项及安装成功状态。"""
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing_with_status()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return PluginDependencyService(
system=get_plugin_system,
log=logger,
).install_missing_with_status()
@staticmethod
async def async_install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
"""在异步启动链中恢复插件依赖并保留取消语义。"""
return await PluginDependencyService(
system=get_plugin_system,
log=logger,
).async_install_missing_with_status()
manager = PluginManager()
with manager.mutation("安装插件依赖"):
return await PluginDependencyService(
system=get_plugin_system,
log=logger,
).async_install_missing_with_status()
def classify_plugins(self) -> PluginDependencyClassification:
"""按源码依赖状态分类物理插件,并把结果映射到虚拟实例。"""
@@ -706,7 +902,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
def delete_plugin_instance(self, plugin_id: str) -> bool:
"""删除虚拟实例描述;调用方仍负责停止实例和清理业务数据。"""
return self._plugin_instance_store.delete(plugin_id)
try:
with self.mutation("删除插件实例描述"):
return self._plugin_instance_store.delete(plugin_id)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def save_plugin_config(self, pid: str, conf: dict, force: bool = False) -> bool:
"""
@@ -715,7 +916,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param conf: 配置
:param force: 强制保存
"""
return self._plugin_config_store.write(pid, conf, force)
try:
with self.mutation("保存插件配置"):
return self._plugin_config_store.write(pid, conf, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
async def async_save_plugin_config(
self, pid: str, conf: dict, force: bool = False
@@ -726,7 +932,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param conf: 配置
:param force: 强制保存
"""
return await self._plugin_config_store.async_write(pid, conf, force)
try:
with self.mutation("保存插件配置"):
return await self._plugin_config_store.async_write(pid, conf, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def delete_plugin_config(self, pid: str, force: bool = False) -> bool:
"""
@@ -734,7 +945,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID
:param force: 插件停止后仍允许按插件 ID 删除持久化配置
"""
return self._plugin_config_store.delete(pid, force)
try:
with self.mutation("删除插件配置"):
return self._plugin_config_store.delete(pid, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def delete_plugin_data(self, pid: str, force: bool = False) -> bool:
"""
@@ -742,7 +958,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param pid: 插件ID
:param force: 插件停止后仍允许按插件 ID 删除持久化数据
"""
return self._plugin_config_store.delete_data(pid, force)
try:
with self.mutation("删除插件数据"):
return self._plugin_config_store.delete_data(pid, force)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
def get_plugin_state(self, pid: str) -> bool:
"""
@@ -1130,14 +1351,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
:param icon: 自定义图标URL
:return: (是否成功, 错误信息)
"""
return self._plugin_clone.clone(
plugin_id=plugin_id,
suffix=suffix,
name=name,
description=description,
version=version,
icon=icon,
)
try:
with self.mutation("创建插件分身"):
return self._plugin_clone.clone(
plugin_id=plugin_id,
suffix=suffix,
name=name,
description=description,
version=version,
icon=icon,
)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False, str(error)
def _modify_plugin_files(self, plugin_dir: Path, original_id: str, suffix: str,
name: str, description: str, version: str = None,
+3 -2
View File
@@ -106,8 +106,8 @@ class TaskRegistry:
}
)
async def shutdown(self, *, timeout_seconds: float = 10.0) -> None:
"""停止接收并有限等待存量任务,超时任务保留登记并报告责任域"""
async def shutdown(self, *, timeout_seconds: float = 10.0) -> bool:
"""停止接收并有限等待存量任务,返回全部 owner 是否真实收敛"""
self._accepting = False
records = self.records
tasks = [record.task for record in records]
@@ -139,6 +139,7 @@ class TaskRegistry:
"timeout_seconds": timeout_seconds,
}
)
return all(record.task.done() for record in records)
_default_registry = TaskRegistry()
+10
View File
@@ -4,6 +4,7 @@ __all__ = (
"APIRateLimitException",
"RateLimitExceededException",
"OperationInterrupted",
"PluginMutationRejectedError",
"StorageQueryError",
"TMDbException",
)
@@ -49,6 +50,15 @@ class OperationInterrupted(KeyboardInterrupt):
pass
class PluginMutationRejectedError(RuntimeError):
"""表示插件运行时已封口,新的可变事务未获准执行。"""
def __init__(self, operation: str) -> None:
"""保存被拒绝的操作名称并生成稳定诊断消息。"""
self.operation = operation
super().__init__(f"插件运行时已进入停机阶段,拒绝{operation}")
class StorageQueryError(Exception):
"""
用于表示存储查询无法确认结果的异常类
+68 -24
View File
@@ -20,6 +20,9 @@ from app.runtime.log import logger
from app.schemas.types import EventType
AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0
def _get_skill_catalog() -> Any:
"""按需返回 Agent 技能目录实现,供消息应用层消费端口。"""
from app.agent.skills.registry import SkillHelper
@@ -106,6 +109,7 @@ class AgentInitializer:
self._initialized = False
self._manager: Any = None
self._compat_injected = False
self._shutdown_started = False
self._shutdown_complete = False
eventmanager.add_event_listener(
EventType.ConfigChanged,
@@ -117,6 +121,7 @@ class AgentInitializer:
初始化AI智能体管理器
"""
try:
self._shutdown_started = False
self._shutdown_complete = False
if agent_manager is not None:
if not settings.AI_AGENT_ENABLE:
@@ -142,7 +147,12 @@ class AgentInitializer:
async def handle_config_changed(self, event: Event) -> None:
"""仅在 manifest watch 命中时协调 service,关闭态保持 fail closed。"""
changed_keys = _event_changed_keys(event)
if not changed_keys or self._compat_injected or self._shutdown_complete:
if (
not changed_keys
or self._compat_injected
or self._shutdown_started
or self._shutdown_complete
):
return
try:
self._manager = await reconcile_agent_service(
@@ -156,24 +166,25 @@ class AgentInitializer:
self._initialized = False
logger.debug(f"配置变更协调AI智能体失败: {error}")
async def cleanup(self) -> None:
"""清理 initializer 引用;显式注入对象同时在此关闭"""
async def cleanup(self) -> bool:
"""清理 initializer 引用;未收敛的显式注入对象继续由本实例持有"""
try:
manager = self._manager
compat_injected = self._compat_injected
if manager is None:
return
try:
if compat_injected:
await manager.close()
logger.info("AI智能体管理器已关闭")
finally:
self._initialized = False
self._manager = None
self._compat_injected = False
return True
if compat_injected and await manager.close() is False:
logger.error("AI智能体管理器仍有会话 owner 未收敛")
return False
logger.info("AI智能体管理器已关闭")
self._initialized = False
self._manager = None
self._compat_injected = False
return True
except Exception as e:
logger.debug(f"关闭AI智能体管理器时发生错误: {e}")
return False
# 全局AI智能体初始化器实例
@@ -204,23 +215,56 @@ async def init_agent() -> bool:
return False
async def stop_agent():
async def stop_agent() -> bool:
"""
停止AI智能体异步版本用于在应用关闭时调用
停止AI智能体并在全部会话和工具资源释放后返回 True
"""
converged = True
close_blocking_executors = None
agent_initializer._shutdown_started = True
try:
if is_tool_factory_materialized():
from app.agent.tools.base import (
begin_blocking_executor_shutdown,
close_blocking_executors as close_executors,
)
# 必须在任何 manager await 之前封口,避免旧会话趁收尾窗口提交新同步调用。
begin_blocking_executor_shutdown(cancel_futures=True)
close_blocking_executors = close_executors
except Exception as e:
logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}")
converged = False
try:
if not agent_initializer._shutdown_complete:
if agent_initializer._compat_injected:
await agent_initializer.cleanup()
service_converged = await agent_initializer.cleanup()
else:
await begin_agent_shutdown()
await agent_initializer.cleanup()
agent_initializer._shutdown_complete = True
if is_tool_factory_materialized():
from app.agent.tools.base import shutdown_blocking_executors
shutdown_blocking_executors(wait=False, cancel_futures=True)
service_converged = await begin_agent_shutdown()
if service_converged is not False:
service_converged = await agent_initializer.cleanup()
converged = converged and service_converged is not False
except Exception as e:
logger.error(f"停止AI智能体时发生错误: {e}")
finally:
await close_materialized_terminal_sessions()
converged = False
if close_blocking_executors is not None:
try:
blocking_converged = await close_blocking_executors(
timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS,
cancel_futures=True,
)
converged = converged and blocking_converged
except Exception as e:
logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}")
converged = False
if converged:
try:
await close_materialized_terminal_sessions()
except Exception as e:
logger.error(f"关闭AI智能体终端会话时发生错误: {e}")
converged = False
agent_initializer._shutdown_complete = converged
return converged
+175 -31
View File
@@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.state import SystemHelper
from app.runtime.log import logger, LoggerManager
from app.startup.command_initializer import init_command, stop_command, restart_command
from app.startup.agent_initializer import stop_agent
from app.startup.domain_initializer import configure_domain_dependencies
from app.startup.modules_initializer import init_modules, stop_modules
from app.startup.modules_initializer import (
drain_events,
init_modules,
settle_events,
stop_modules,
)
from app.startup.monitor_initializer import stop_monitor, init_monitor
from app.startup.plugins_initializer import (
configure_plugin_services,
execute_task,
finalize_plugins,
init_plugins,
stop_plugins,
quiesce_plugin_services,
quiesce_plugins,
stop_plugin_monitor,
sync_plugins,
)
from app.startup.routers_initializer import init_routers
@@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import (
init_plugin_scheduler,
)
from app.db.engine import check_connection_budget, get_engine, get_global_async_engine
from app.startup.transfer_initializer import replay_pending_transfers
from app.startup.transfer_initializer import (
replay_pending_transfers,
stop_transfer_runtime,
)
from app.startup.workflow_initializer import init_workflow, stop_workflow
from app.startup.lifecycle.components import (
LifecycleComponent,
LifecycleFailurePolicy,
LifecycleMode,
lifecycle_manifest,
)
@@ -101,8 +114,8 @@ async def run_shutdown_step(
name: str,
callback: Callable[[], object],
timeout_seconds: float | None = None,
) -> None:
"""在有限预算内执行关闭阶段,并保留未收敛任务的资源所有权"""
) -> bool:
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛"""
try:
result = callback()
if inspect.isawaitable(result):
@@ -120,16 +133,22 @@ async def run_shutdown_step(
task.add_done_callback(_consume_shutdown_result)
if timeout_seconds:
try:
await asyncio.wait_for(
result = await asyncio.wait_for(
asyncio.shield(task), timeout=timeout_seconds
)
except asyncio.TimeoutError:
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
task.cancel()
return False
else:
await task
result = await task
if result is False:
logger.error("关闭%s未收敛,资源所有权保持不变", name)
return False
return True
except Exception as err:
logger.error(f"关闭{name}失败:{err}")
return False
async def run_startup_step(
@@ -152,6 +171,66 @@ async def run_startup_step(
logger.info("启动%s完成,耗时=%.2fms", name, elapsed_ms)
async def stop_lifecycle_components(
components: tuple[LifecycleComponent, ...],
) -> bool:
"""按声明顺序关闭组件,并在关键 owner 未收敛时停止释放依赖。"""
all_converged = True
for component in sorted(
(item for item in components if item.stop is not None),
key=lambda item: item.stop_order or 0,
):
completed = await run_shutdown_step(
component.name,
component.stop,
component.stop_timeout_seconds,
)
if completed:
continue
all_converged = False
if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST:
logger.error(
"关闭%s未收敛,停止释放其后续依赖",
component.name,
)
break
return all_converged
def select_startup_cleanup_components(
components: tuple[LifecycleComponent, ...],
*,
started_names: set[str],
active_component: LifecycleComponent | None,
) -> tuple[LifecycleComponent, ...]:
"""选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。"""
cleanup_names = set(started_names)
if active_component is not None:
cleanup_names.add(active_component.name)
# stop-only owner 没有启动回调,按已激活依赖递归纳入,避免清理时反向
# 实例化尚未触达的插件、模块或外部资源。
changed = True
while changed:
changed = False
for component in components:
if (
component.stop is None
or component.start is not None
or component.name in cleanup_names
or not set(component.dependencies).issubset(cleanup_names)
):
continue
cleanup_names.add(component.name)
changed = True
return tuple(
component
for component in components
if component.stop is not None and component.name in cleanup_names
)
async def initialize_modules_component(app: FastAPI) -> None:
"""启动模块并把其类型化运行时发布到当前 FastAPI AppState。"""
try:
@@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None:
configure_task_registry(task_registry)
async def stop_task_registry(app: FastAPI) -> None:
"""停止接收新后台任务,并取消、等待当前 lifespan 的存量任务"""
async def stop_task_registry(app: FastAPI) -> bool:
"""停止接收新后台任务,并把已封口登记器保留到下一次显式启动"""
task_registry = getattr(app.state, "task_registry", None)
try:
if isinstance(task_registry, TaskRegistry):
await task_registry.shutdown(timeout_seconds=30.0)
finally:
if not isinstance(task_registry, TaskRegistry):
configure_task_registry(None)
app.state.task_registry = None
return True
# 即使 owner 已收敛,也不能在后续插件/模块 stop hook 仍会运行时退回永久
# accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。
return await task_registry.shutdown(timeout_seconds=30.0)
def prepare_plugin_restore() -> None:
@@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=5,
start_timeout_seconds=30,
stop_timeout_seconds=60,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="数据库准备",
@@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
dependencies=("插件备份恢复",),
mode=LifecycleMode.NORMAL_ONLY,
start=init_plugins,
stop=stop_plugins,
stop=finalize_plugins,
start_order=90,
stop_order=60,
start_timeout_seconds=300,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件变更监控",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=stop_plugin_monitor,
stop_order=8,
stop_timeout_seconds=10,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="定时器",
@@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=50,
start_timeout_seconds=120,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="监控器",
@@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
stop_order=40,
start_timeout_seconds=120,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="整理后台服务",
dependencies=("模块服务",),
stop=stop_transfer_runtime,
stop_order=52,
stop_timeout_seconds=45,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="AI智能体会话",
dependencies=("模块服务",),
stop=stop_agent,
stop_order=51,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件事件入口",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=quiesce_plugins,
stop_order=53,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="事件尾任务结算",
dependencies=("模块服务", "插件"),
mode=LifecycleMode.NORMAL_ONLY,
stop=settle_events,
stop_order=54,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="插件后台服务",
dependencies=("插件",),
mode=LifecycleMode.NORMAL_ONLY,
stop=quiesce_plugin_services,
stop_order=55,
stop_timeout_seconds=300,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="事件投递屏障",
dependencies=("模块服务", "整理后台服务"),
stop=drain_events,
stop_order=58,
stop_timeout_seconds=120,
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
),
LifecycleComponent(
name="待处理整理回放",
dependencies=("监控器",),
dependencies=("监控器", "整理后台服务"),
mode=LifecycleMode.NORMAL_ONLY,
start=replay_pending_transfers,
start_order=120,
@@ -369,6 +513,9 @@ async def lifespan(app: FastAPI):
health = get_application_health(app)
health.begin_startup()
main_loop = asyncio.get_running_loop()
enabled_components: tuple[LifecycleComponent, ...] = ()
started_component_names: set[str] = set()
active_start_component: LifecycleComponent | None = None
try:
validate_process_topology(
workers=settings.API_WORKERS,
@@ -390,11 +537,14 @@ async def lifespan(app: FastAPI):
(item for item in enabled_components if item.start is not None),
key=lambda item: item.start_order or 0,
):
active_start_component = component
await run_startup_step(
component.name,
component.start,
component.start_timeout_seconds,
)
started_component_names.add(component.name)
active_start_component = None
if settings.MOVIEPILOT_SAFE_MODE:
print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.")
# 插件同步到本地
@@ -407,10 +557,15 @@ async def lifespan(app: FastAPI):
except BaseException:
# Uvicorn 在 lifespan 抛错时不会开始接流量;状态仍需供嵌入式入口和测试诊断。
health.mark_failed()
cleanup_components = select_startup_cleanup_components(
enabled_components,
started_names=started_component_names,
active_component=active_start_component,
)
try:
await stop_task_registry(app)
await stop_lifecycle_components(cleanup_components)
except Exception as cleanup_error:
logger.error(f"启动失败后的后台任务清理失败:{cleanup_error}")
logger.error(f"启动失败后的生命周期清理失败:{cleanup_error}")
finally:
global_vars.clear_loop(main_loop)
raise
@@ -421,21 +576,10 @@ async def lifespan(app: FastAPI):
health.mark_stopping()
print("Shutting down...")
global_vars.stop_system()
# 插件恢复会在线程池中修改源码与依赖,必须完成后再进入资源关闭阶段。
try:
await sync_plugins_task
except Exception as e:
print(str(e))
try:
for component in sorted(
(item for item in enabled_components if item.stop is not None),
key=lambda item: item.stop_order or 0,
):
await run_shutdown_step(
component.name,
component.stop,
component.stop_timeout_seconds,
)
# 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner
# 在统一预算内取消/等待,不能在屏障之前无界 await 绕过停机预算。
await stop_lifecycle_components(enabled_components)
finally:
try:
# 日志最后关闭,确保其他组件的收尾信息已写入文件
+17 -2
View File
@@ -129,7 +129,7 @@ from app.command import CommandChain
from app.schemas.message import Message
from app.schemas.message import MessageType
from app.schemas.types import EventType, SystemConfigKey
from app.startup.agent_initializer import init_agent, stop_agent
from app.startup.agent_initializer import init_agent
from app.startup.database import build_database_governance
from app.startup.managed_resources_initializer import (
init_managed_resources,
@@ -560,6 +560,22 @@ def close_browser_sessions() -> None:
BrowserSessionHelper.close_all_sessions()
async def drain_events() -> bool:
"""在插件卸载前等待已接收事件及其同步、异步处理器完成。"""
event_manager = EventManager.get_existing_instance()
if event_manager is None:
return True
return await event_manager.drain_async(seal=True)
async def settle_events() -> bool:
"""在插件 handler 停用后结算在途事件,但保留停机 hook 的尾事件入口。"""
event_manager = EventManager.get_existing_instance()
if event_manager is None:
return True
return await event_manager.drain_async(seal=False)
async def stop_modules():
"""
服务关闭
@@ -578,7 +594,6 @@ async def stop_modules():
logger.error(f"关闭{name}失败:{err}")
return True
await run_step("AI智能体", stop_agent)
await run_step("模块", lambda: ModuleManager().shutdown())
await run_step("事件消费", lambda: EventManager().stop_async())
await run_step("浏览器会话", close_browser_sessions)
+21 -10
View File
@@ -1,15 +1,26 @@
from app.monitor import Monitor
from app.runtime.execution import run_in_threadpool_to_completion
def init_monitor():
"""
初始化监控器
"""
Monitor()
def init_monitor() -> None:
"""初始化监控器;复用单例时必须显式开启新的应用 lifespan。"""
monitor = Monitor.get_existing_instance()
if monitor is None:
Monitor()
return
if not monitor.lifecycle_closed:
return
if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT):
raise RuntimeError("旧目录监控 owner 未收敛,无法开启新生命周期")
if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT):
raise RuntimeError("目录监控初始化失败")
def stop_monitor():
"""
停止监控器
"""
Monitor().stop()
async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool:
"""在线程池里永久关闭监控器,取消后仍等待同步 owner 收敛到终态。"""
monitor = Monitor.get_existing_instance()
if monitor is None:
return True
return bool(
await run_in_threadpool_to_completion(monitor.close, timeout=timeout)
)
+106 -52
View File
@@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import (
configure_plugin_resource_import_preparer,
configure_site_auth_level_provider,
)
from app.runtime.execution import run_in_threadpool_to_completion
from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult
from app.application.plugin.catalog import PluginCatalogService
from app.application.plugin.data import DeletePluginDataCommand
@@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork
from app.runtime.log import logger
from app.foundation.version import compare_version
from app.schemas.plugin import PluginRuntimeStatus
from app.schemas.exception import PluginMutationRejectedError
from app.schemas.types import SystemConfigKey
@@ -142,56 +144,64 @@ async def sync_plugins() -> bool:
"""
plugin_manager = None
try:
configure_plugin_services()
loop = global_vars.loop
plugin_manager = PluginManager()
plugin_manager.set_plugin_settling(True)
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
dependency_result = await (
plugin_manager.async_install_plugin_missing_dependencies_with_status()
)
if dependency_result is None:
return False
if not isinstance(dependency_result, PluginDependencyInstallResult):
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
return False
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
if not dependency_result.success:
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
changed_ids = await execute_task(
loop,
lambda: _activate_ready_plugins(
plugin_manager,
classification.ready,
sync_result or [],
previous_statuses,
),
"插件运行态激活",
)
if changed_ids is None:
return False
if not changed_ids:
logger.debug("没有新的插件进入可运行状态")
return False
for plugin_id in changed_ids:
register_plugin_api(plugin_id)
if dependency_result.success:
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
else:
logger.warning(
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
)
return True
with plugin_manager.mutation("启动后同步插件"):
configure_plugin_services()
plugin_manager.set_plugin_settling(True)
return await _sync_plugins_admitted(plugin_manager, loop)
except PluginMutationRejectedError as error:
logger.warning(str(error))
return False
except Exception as e:
logger.error(f"插件初始化过程中出现异常: {e}")
return False
async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool:
"""在一个 admission lease 内完成包、依赖、实例和动态路由同步。"""
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
dependency_result = await (
plugin_manager.async_install_plugin_missing_dependencies_with_status()
)
if dependency_result is None:
return False
if not isinstance(dependency_result, PluginDependencyInstallResult):
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
return False
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
if not dependency_result.success:
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
changed_ids = await execute_task(
loop,
lambda: _activate_ready_plugins(
plugin_manager,
classification.ready,
sync_result or [],
previous_statuses,
),
"插件运行态激活",
)
if changed_ids is None:
return False
if not changed_ids:
logger.debug("没有新的插件进入可运行状态")
return False
for plugin_id in changed_ids:
register_plugin_api(plugin_id)
if dependency_result.success:
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
else:
logger.warning(
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
)
return True
def _activate_ready_plugins(
plugin_manager: PluginManager,
ready_ids: tuple[str, ...],
@@ -217,12 +227,39 @@ def _activate_ready_plugins(
return changed_ids
async def quiesce_plugins(timeout: float = 240.0) -> bool:
"""封口插件变更并停用 handler,保留超时 Future 的运行所有权。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return await plugin_manager.quiesce_plugins(timeout=timeout)
async def quiesce_plugin_services(timeout: float = 240.0) -> bool:
"""在事件结算后有界执行旧插件 close、stop_service hook。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return await plugin_manager.quiesce_plugin_services(timeout=timeout)
def finalize_plugins() -> bool:
"""在事件屏障封口后卸载已停用 handler 的插件实例。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
return bool(plugin_manager.finalize_plugins())
async def execute_task(loop, task_func, task_name):
"""
执行后台任务
执行后台任务取消调用方时仍持有同步线程直到真实完成
"""
try:
result = await loop.run_in_executor(None, task_func)
# loop 参数属于既有调用 ABI;同步执行改由 completion-aware 适配器持有,
# 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。
del loop
result = await run_in_threadpool_to_completion(task_func)
if isinstance(result, PluginDependencyInstallResult):
processed_count = len(result.missing)
elif isinstance(result, list):
@@ -245,13 +282,15 @@ def init_plugins():
"""
configure_plugin_services()
plugin_manager = PluginManager()
if not plugin_manager.reopen_plugins():
raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛")
classification = plugin_manager.classify_plugins()
plugin_manager.apply_plugin_dependency_classification(classification)
plugin_manager.set_plugin_settling(True)
for plugin_id in classification.ready:
plugin_manager.start(plugin_id)
register_plugin_api()
plugin_manager.start_monitor()
plugin_manager.start_monitor(reopen=True)
logger.info(
"插件启动分类:立即加载=%s,等待依赖=%s,等待源码=%s",
len(classification.ready),
@@ -260,15 +299,30 @@ def init_plugins():
)
def stop_plugins():
"""
停止插件
"""
def stop_plugin_monitor(timeout: float = 5.0) -> bool:
"""封口已创建管理器的文件监控线程,并返回是否完成收口。"""
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
try:
plugin_manager = PluginManager()
return bool(plugin_manager.close_monitor(timeout=timeout))
except Exception as e:
logger.error(f"停止插件文件监控时发生错误:{e}", exc_info=True)
return False
def stop_plugins() -> bool:
"""停止已创建的插件监控和运行实例,不在停机阶段反向物化管理器。"""
try:
plugin_manager = PluginManager.get_existing_instance()
if plugin_manager is None:
return True
monitor_stopped = True
try:
plugin_manager.stop_monitor()
monitor_stopped = plugin_manager.stop_monitor()
finally:
plugin_manager.stop()
return bool(monitor_stopped)
except Exception as e:
logger.error(f"停止插件时发生错误:{e}", exc_info=True)
return False
+12
View File
@@ -11,3 +11,15 @@ def replay_pending_transfers():
回放本身在后台线程执行不阻塞启动流程
"""
TransferChain().replay_pending()
async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool:
"""关闭已存在的整理后台 owner,且不在关停阶段创建新的整理链实例。
:param timeout_seconds: worker pending 回放共享的最大等待秒数
:return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True
"""
transfer_chain = TransferChain.get_existing_instance()
if transfer_chain is None:
return True
return await transfer_chain.close(timeout_seconds=timeout_seconds)