mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 07:27:15 +08:00
refactor: own shutdown lifecycle boundaries
This commit is contained in:
@@ -155,7 +155,11 @@ class AgentServiceAdapter:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 必须返回 awaitable"
|
||||
)
|
||||
await result
|
||||
converged = await result
|
||||
if converged is False:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def cleanup(
|
||||
|
||||
@@ -638,6 +638,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
|
||||
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
|
||||
self._accepting_tasks = True
|
||||
self._close_cancel_requested: set[asyncio.Task] = set()
|
||||
self.tools = [
|
||||
StructuredTool.from_function(
|
||||
coroutine=self._control_task,
|
||||
@@ -811,6 +813,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
|
||||
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
|
||||
"""记录任务完成时间并取出异常避免未读取告警。"""
|
||||
cancel_requested = getattr(self, "_close_cancel_requested", None)
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.discard(task)
|
||||
record = self._tasks.get(task_id)
|
||||
if record:
|
||||
record.finished_at = datetime.now()
|
||||
@@ -895,6 +900,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
@staticmethod
|
||||
async def _cancel_records(
|
||||
records: list[_SubAgentRuntimeTask],
|
||||
*,
|
||||
cancel_requested: Optional[set[asyncio.Task]] = None,
|
||||
) -> list[_SubAgentRuntimeTask]:
|
||||
"""取消一组任务,并返回等待上限内仍未收敛的记录。"""
|
||||
cancellable_tasks = [
|
||||
@@ -903,7 +910,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
if cancellable_tasks:
|
||||
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
|
||||
for task in cancellable_tasks:
|
||||
if cancel_requested is not None and task in cancel_requested:
|
||||
continue
|
||||
task.cancel()
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.add(task)
|
||||
if not cancellable_tasks:
|
||||
return []
|
||||
|
||||
@@ -922,8 +933,15 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
|
||||
return [record for record in records if record.task in pending]
|
||||
|
||||
async def close(self) -> None:
|
||||
"""取消脱离当前 Agent 回合的子代理任务。"""
|
||||
def seal(self) -> None:
|
||||
"""封住新的 detached 子代理提交,既有任务继续由记录表持有。"""
|
||||
self._accepting_tasks = False
|
||||
|
||||
async def close(self) -> bool:
|
||||
"""有限等待 detached 子代理;超时保留记录并返回 False。"""
|
||||
self.seal()
|
||||
if not hasattr(self, "_close_cancel_requested"):
|
||||
self._close_cancel_requested = set()
|
||||
unfinished_records = [
|
||||
record for record in self._tasks.values() if not record.task.done()
|
||||
]
|
||||
@@ -931,8 +949,19 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(
|
||||
f"关闭子代理任务控制器,取消未完成任务: tasks={len(unfinished_records)}"
|
||||
)
|
||||
await self._cancel_records(unfinished_records)
|
||||
pending_records = await self._cancel_records(
|
||||
unfinished_records,
|
||||
cancel_requested=self._close_cancel_requested,
|
||||
)
|
||||
if pending_records:
|
||||
self._tasks = {
|
||||
record.task_id: record
|
||||
for record in pending_records
|
||||
}
|
||||
return False
|
||||
self._tasks.clear()
|
||||
self._close_cancel_requested.clear()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_description(
|
||||
@@ -1051,6 +1080,8 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
|
||||
timeout = normalized_timeout_ms / 1000
|
||||
for step_index, spec in enumerate(specs, start=1):
|
||||
if not self._accepting_tasks:
|
||||
return records, "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
record = self._create_pipeline_record(spec)
|
||||
records.append(record)
|
||||
pipeline_description = self._pipeline_description(
|
||||
@@ -1108,6 +1139,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
"""管理异步子代理任务。"""
|
||||
logger.info(f"收到子代理管控操作: action={action}")
|
||||
if action in {"start", "run", "pipeline"}:
|
||||
if not self._accepting_tasks:
|
||||
error = "子代理任务控制器正在关闭,不能再启动新任务。"
|
||||
return self._json_response({"success": False, "error": error})
|
||||
specs, error = self._normalize_specs(
|
||||
description=description,
|
||||
subagent_type=subagent_type,
|
||||
|
||||
+134
-33
@@ -436,6 +436,7 @@ class MoviePilotAgent:
|
||||
self._agent_started_at: Optional[datetime] = None
|
||||
self._compiled_agent_bundle: Optional[_CompiledAgentBundle] = None
|
||||
self._subagent_middlewares: tuple[Any, ...] = ()
|
||||
self._shutdown_started = False
|
||||
self._last_agent_cache_hit = False
|
||||
|
||||
# 流式token管理
|
||||
@@ -1751,10 +1752,25 @@ class MoviePilotAgent:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
def _seal_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> None:
|
||||
"""释放不再由 Agent 图持有的子代理控制器。"""
|
||||
"""同步封住子代理控制器的新任务入口,不等待既有任务退出。"""
|
||||
for middleware in middlewares:
|
||||
seal = getattr(middleware, "seal", None)
|
||||
if not callable(seal):
|
||||
continue
|
||||
try:
|
||||
seal()
|
||||
except Exception as error:
|
||||
logger.debug(f"封住子代理中间件失败: {error}")
|
||||
|
||||
@staticmethod
|
||||
async def _close_subagent_middleware_instances(
|
||||
middlewares: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""关闭子代理控制器,并返回仍持有未收敛任务的实例。"""
|
||||
pending_middlewares = []
|
||||
for middleware in middlewares:
|
||||
close = getattr(middleware, "close", None)
|
||||
if not callable(close):
|
||||
@@ -1762,9 +1778,30 @@ class MoviePilotAgent:
|
||||
try:
|
||||
result = close()
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
result = await result
|
||||
if result is False:
|
||||
pending_middlewares.append(middleware)
|
||||
except Exception as error:
|
||||
logger.debug(f"关闭子代理中间件失败: {error}")
|
||||
pending_middlewares.append(middleware)
|
||||
return tuple(pending_middlewares)
|
||||
|
||||
@staticmethod
|
||||
def _merge_subagent_middleware_owners(
|
||||
*groups: tuple[Any, ...],
|
||||
) -> tuple[Any, ...]:
|
||||
"""按对象身份合并当前图与延迟收敛控制器的 owner 集合。"""
|
||||
merged = []
|
||||
for group in groups:
|
||||
for middleware in group:
|
||||
if not any(middleware is existing for existing in merged):
|
||||
merged.append(middleware)
|
||||
return tuple(merged)
|
||||
|
||||
def begin_shutdown(self) -> None:
|
||||
"""在任何异步等待前封住当前 Agent 的 detached 子代理提交。"""
|
||||
self._shutdown_started = True
|
||||
self._seal_subagent_middleware_instances(self._subagent_middlewares)
|
||||
|
||||
async def _cache_agent(
|
||||
self,
|
||||
@@ -1786,7 +1823,11 @@ class MoviePilotAgent:
|
||||
for replacement in subagent_middlewares
|
||||
)
|
||||
)
|
||||
await self._close_subagent_middleware_instances(previous_middlewares)
|
||||
if self._shutdown_started:
|
||||
self._seal_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
previous_middlewares
|
||||
)
|
||||
self._compiled_agent_bundle = _CompiledAgentBundle(
|
||||
signature=signature,
|
||||
agent=agent,
|
||||
@@ -1798,15 +1839,21 @@ class MoviePilotAgent:
|
||||
mcp_config_signature=mcp_config_signature,
|
||||
catalog_checked_at=datetime.now(),
|
||||
)
|
||||
self._subagent_middlewares = subagent_middlewares
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
pending_middlewares,
|
||||
subagent_middlewares,
|
||||
)
|
||||
return agent
|
||||
|
||||
async def _invalidate_cached_agent(self) -> None:
|
||||
"""使当前图失效,并释放只属于该图的子代理控制器。"""
|
||||
async def _invalidate_cached_agent(self) -> bool:
|
||||
"""使当前图失效,未收敛的子代理控制器继续由 Agent 持有。"""
|
||||
subagent_middlewares = self._subagent_middlewares
|
||||
self._subagent_middlewares = ()
|
||||
self._compiled_agent_bundle = None
|
||||
await self._close_subagent_middleware_instances(subagent_middlewares)
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = pending_middlewares
|
||||
return not pending_middlewares
|
||||
|
||||
@staticmethod
|
||||
def _latest_turn_messages(messages: List[BaseMessage]) -> List[BaseMessage]:
|
||||
@@ -2003,9 +2050,13 @@ class MoviePilotAgent:
|
||||
if cached_agent:
|
||||
# 签名相同表示已编译图中的精确工具实例仍有效;新建快照仅用于复核。
|
||||
cached_bundle.catalog_checked_at = datetime.now()
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
temporary_subagent_middlewares = ()
|
||||
logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}")
|
||||
return cached_agent
|
||||
@@ -2123,14 +2174,22 @@ class MoviePilotAgent:
|
||||
temporary_subagent_middlewares = ()
|
||||
return cached_agent
|
||||
except asyncio.CancelledError:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
await self._close_subagent_middleware_instances(
|
||||
pending_middlewares = await self._close_subagent_middleware_instances(
|
||||
temporary_subagent_middlewares
|
||||
)
|
||||
self._subagent_middlewares = self._merge_subagent_middleware_owners(
|
||||
self._subagent_middlewares,
|
||||
pending_middlewares,
|
||||
)
|
||||
logger.error(f"创建 Agent 失败: {e}")
|
||||
raise
|
||||
|
||||
@@ -2539,14 +2598,20 @@ class MoviePilotAgent:
|
||||
)
|
||||
)
|
||||
|
||||
async def cleanup(self):
|
||||
async def cleanup(self) -> bool:
|
||||
"""
|
||||
清理智能体资源
|
||||
清理智能体资源;detached 子代理未收敛时保留 owner 并返回 False。
|
||||
"""
|
||||
await self._invalidate_cached_agent()
|
||||
self.begin_shutdown()
|
||||
if not await self._invalidate_cached_agent():
|
||||
logger.error(
|
||||
f"MoviePilot智能体仍有子代理 owner 未收敛: session_id={self.session_id}"
|
||||
)
|
||||
return False
|
||||
self._pending_secret_confirmation = None
|
||||
self.protected_output_callback = None
|
||||
logger.info(f"MoviePilot智能体已清理: session_id={self.session_id}")
|
||||
return True
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2626,6 +2691,7 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks: Dict[str, asyncio.Task] = {}
|
||||
self._session_cancel_requested: set[str] = set()
|
||||
self._close_finalizer_task: Optional[asyncio.Task] = None
|
||||
self._closed = False
|
||||
self._shutdown_timeout = AGENT_MANAGER_SHUTDOWN_TIMEOUT
|
||||
# 接收门禁与队列写入共用一把锁,确保关闭开始后不会再创建 worker。
|
||||
self._lifecycle_lock = asyncio.Lock()
|
||||
@@ -2693,22 +2759,33 @@ class AgentManager:
|
||||
async with self._lifecycle_lock:
|
||||
if self._accepting_tasks:
|
||||
return
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
raise AgentManagerUnavailableError("AgentManager 仍在完成上一代关闭")
|
||||
memory_manager.initialize()
|
||||
if not self._idle_cleanup_task or self._idle_cleanup_task.done():
|
||||
self._idle_cleanup_task = asyncio.create_task(
|
||||
self._cleanup_idle_sessions()
|
||||
)
|
||||
self._accepting_tasks = True
|
||||
self._closed = False
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> bool:
|
||||
"""
|
||||
关闭管理器
|
||||
关闭管理器,并诚实返回全部会话 owner 是否已经收敛。
|
||||
"""
|
||||
async with self._lifecycle_lock:
|
||||
if self._closed:
|
||||
return True
|
||||
if self._close_finalizer_task and not self._close_finalizer_task.done():
|
||||
return
|
||||
return False
|
||||
# 门禁必须先关闭;锁内完成清理可阻止等待中的请求在收口期间重新入队。
|
||||
self._accepting_tasks = False
|
||||
# 子代理提交门禁同样必须在第一次 await 前关闭,否则 detached task
|
||||
# 可趁 idle-cleanup 收尾窗口继续创建不属于新生命周期的任务。
|
||||
for agent in self.active_agents.values():
|
||||
begin_shutdown = getattr(agent, "begin_shutdown", None)
|
||||
if callable(begin_shutdown):
|
||||
begin_shutdown()
|
||||
if self._idle_cleanup_task:
|
||||
self._idle_cleanup_task.cancel()
|
||||
try:
|
||||
@@ -2751,8 +2828,8 @@ class AgentManager:
|
||||
self._session_workers.pop(session_id, None)
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if session_id not in timed_out_session_ids:
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
logger.error(
|
||||
"AgentManager 关闭时仍有 worker 未收敛,"
|
||||
f"保留 {len(timed_out_workers)} 个会话资源直到 worker 结束"
|
||||
@@ -2760,13 +2837,21 @@ class AgentManager:
|
||||
self._close_finalizer_task = asyncio.create_task(
|
||||
self._finish_deferred_close(timed_out_workers)
|
||||
)
|
||||
return
|
||||
return False
|
||||
|
||||
self._session_workers.clear()
|
||||
for agent in list(self.active_agents.values()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.clear()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 仍有 %d 个 detached 子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return False
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
return True
|
||||
|
||||
def _record_session_activity(self, session_id: str, user_id: str) -> None:
|
||||
"""
|
||||
@@ -3072,7 +3157,10 @@ class AgentManager:
|
||||
and isinstance(task.agent_factory, type)
|
||||
and not isinstance(existing_agent, task.agent_factory)
|
||||
):
|
||||
await existing_agent.cleanup()
|
||||
if await existing_agent.cleanup() is False:
|
||||
raise AgentManagerUnavailableError(
|
||||
f"Agent 会话 {session_id} 仍有子代理任务在停止"
|
||||
)
|
||||
self.active_agents.pop(session_id, None)
|
||||
|
||||
if session_id not in self.active_agents:
|
||||
@@ -3249,7 +3337,11 @@ class AgentManager:
|
||||
# 清理agent
|
||||
if session_id in self.active_agents:
|
||||
agent = self.active_agents[session_id]
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 仍有子代理 owner 未收敛,保留会话与记忆"
|
||||
)
|
||||
return
|
||||
del self.active_agents[session_id]
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
@@ -3290,9 +3382,14 @@ class AgentManager:
|
||||
self._session_deferred_cleanup_tasks.pop(session_id, None)
|
||||
self._session_queue_rejections.pop(session_id, None)
|
||||
self._session_last_queue_wait_ms.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
agent = self.active_agents.get(session_id)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
if await agent.cleanup() is False:
|
||||
logger.error(
|
||||
f"会话 {session_id} 的延迟清理仍有子代理 owner 未收敛"
|
||||
)
|
||||
return
|
||||
self.active_agents.pop(session_id, None)
|
||||
memory_manager.clear_memory(session_id, user_id)
|
||||
logger.info(f"会话 {session_id} 的记忆已清空")
|
||||
|
||||
@@ -3310,15 +3407,19 @@ class AgentManager:
|
||||
for session_id, worker in workers:
|
||||
if self._session_workers.get(session_id) is worker:
|
||||
self._session_workers.pop(session_id, None)
|
||||
agent = self.active_agents.pop(session_id, None)
|
||||
if agent:
|
||||
await agent.cleanup()
|
||||
for session_id, agent in list(self.active_agents.items()):
|
||||
await agent.cleanup()
|
||||
self.active_agents.pop(session_id, None)
|
||||
if await agent.cleanup() is not False:
|
||||
self.active_agents.pop(session_id, None)
|
||||
self._session_shutdown_pending.clear()
|
||||
self._session_cancel_requested.clear()
|
||||
if self.active_agents:
|
||||
logger.error(
|
||||
"AgentManager 延迟关闭后仍有 %d 个子代理 owner 未收敛",
|
||||
len(self.active_agents),
|
||||
)
|
||||
return
|
||||
await memory_manager.close()
|
||||
self._closed = True
|
||||
finally:
|
||||
self._close_finalizer_task = None
|
||||
|
||||
|
||||
@@ -20,7 +20,10 @@ from app.agent.capabilities.adapter import (
|
||||
build_agent_capability_registry,
|
||||
should_run_agent_service,
|
||||
)
|
||||
from app.runtime.capabilities.model import CapabilityMaterializationState
|
||||
from app.runtime.capabilities.model import (
|
||||
CapabilityLifecycleState,
|
||||
CapabilityMaterializationState,
|
||||
)
|
||||
from app.runtime.capabilities.runtime import CapabilityRuntime
|
||||
|
||||
|
||||
@@ -142,9 +145,11 @@ async def close_materialized_terminal_sessions() -> None:
|
||||
await close()
|
||||
|
||||
|
||||
async def begin_agent_shutdown() -> None:
|
||||
"""不可逆关闭首用闸门,并等待全部同步及异步能力释放。"""
|
||||
try:
|
||||
await _ensure_runtime().shutdown_async(reason="application_shutdown")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
async def begin_agent_shutdown() -> bool:
|
||||
"""不可逆关闭首用闸门,并返回 Agent service 是否真实收敛。"""
|
||||
runtime = _ensure_runtime()
|
||||
await runtime.shutdown_async(reason="application_shutdown")
|
||||
return (
|
||||
runtime.snapshot(AGENT_SERVICE_CAPABILITY_ID).lifecycle
|
||||
is CapabilityLifecycleState.STOPPED
|
||||
)
|
||||
|
||||
+117
-26
@@ -3,7 +3,7 @@ import inspect
|
||||
import json
|
||||
import threading
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
|
||||
from contextvars import Context, copy_context
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
@@ -168,33 +168,130 @@ _blocking_semaphores = {
|
||||
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
|
||||
}
|
||||
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.Lock()
|
||||
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
|
||||
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.RLock()
|
||||
_blocking_executor_accepting = True
|
||||
|
||||
|
||||
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
|
||||
"""按桶懒加载线程池,避免在导入阶段创建过多 worker。"""
|
||||
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
|
||||
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executor = _blocking_futures.pop(future, None)
|
||||
if executor is None or executor not in _blocking_retiring_executors:
|
||||
return
|
||||
if executor not in _blocking_futures.values():
|
||||
_blocking_retiring_executors.discard(executor)
|
||||
|
||||
|
||||
def _submit_blocking_call(
|
||||
bucket: str,
|
||||
bound_call: Callable[[], Any],
|
||||
) -> ConcurrentFuture[Any]:
|
||||
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
|
||||
context = copy_context()
|
||||
with _blocking_executor_lock:
|
||||
if not _blocking_executor_accepting:
|
||||
raise RuntimeError("Agent 工具阻塞执行器正在关闭,不能再提交新任务")
|
||||
executor = _blocking_executors.get(bucket)
|
||||
if executor:
|
||||
return executor
|
||||
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
return executor
|
||||
if executor is None:
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(executor.submit, context.run, bound_call)
|
||||
_blocking_futures[future] = executor
|
||||
future.add_done_callback(_discard_blocking_future)
|
||||
return future
|
||||
|
||||
|
||||
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
|
||||
"""关闭 Agent 工具阻塞线程池,释放长期运行进程或测试环境中的 worker。"""
|
||||
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
|
||||
"""撤销活动 executor 的提交资格,并保留其运行 Future 对应的 owner。"""
|
||||
with _blocking_executor_lock:
|
||||
executors = list(_blocking_executors.values())
|
||||
executors = tuple(_blocking_executors.values())
|
||||
_blocking_executors.clear()
|
||||
|
||||
_blocking_retiring_executors.update(executors)
|
||||
for executor in executors:
|
||||
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
executor.shutdown(wait=False, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
owned_executors = set(_blocking_futures.values())
|
||||
_blocking_retiring_executors.intersection_update(owned_executors)
|
||||
return executors
|
||||
|
||||
|
||||
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
|
||||
"""原子封住新阻塞工具提交,并请求取消尚未开始的同步调用。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
_blocking_executor_accepting = False
|
||||
_retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
|
||||
|
||||
def reopen_blocking_executors() -> bool:
|
||||
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
if _blocking_futures or _blocking_retiring_executors:
|
||||
return False
|
||||
_blocking_executor_accepting = True
|
||||
return True
|
||||
|
||||
|
||||
async def close_blocking_executors(
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
cancel_futures: bool = True,
|
||||
) -> bool:
|
||||
"""有限等待全部阻塞工具 Future,超时保留 Future 与 executor owner。"""
|
||||
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
futures = tuple(_blocking_futures)
|
||||
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
|
||||
if wrapped_futures:
|
||||
done, pending = await asyncio.wait(
|
||||
wrapped_futures,
|
||||
timeout=max(0.0, timeout_seconds),
|
||||
)
|
||||
if done:
|
||||
await asyncio.gather(*done, return_exceptions=True)
|
||||
for pending_future in pending:
|
||||
pending_future.add_done_callback(
|
||||
lambda completed: completed.exception()
|
||||
if not completed.cancelled()
|
||||
else None
|
||||
)
|
||||
|
||||
with _blocking_executor_lock:
|
||||
unfinished = tuple(
|
||||
future for future in _blocking_futures if not future.done()
|
||||
)
|
||||
retiring_count = len(_blocking_retiring_executors)
|
||||
if unfinished:
|
||||
logger.error(
|
||||
"Agent 阻塞工具未在 %.1f 秒内收敛:futures=%d,executors=%d",
|
||||
max(0.0, timeout_seconds),
|
||||
len(unfinished),
|
||||
retiring_count,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def shutdown_blocking_executors(
|
||||
*,
|
||||
wait: bool = True,
|
||||
cancel_futures: bool = False,
|
||||
) -> bool:
|
||||
"""同步清理测试 owner;非等待模式下保留尚未收敛的 executor 句柄。"""
|
||||
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
for executor in executors:
|
||||
if wait:
|
||||
executor.shutdown(wait=True, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
return not _blocking_futures and not _blocking_retiring_executors
|
||||
|
||||
|
||||
class ToolExecutionTimeoutError(TimeoutError):
|
||||
@@ -226,13 +323,7 @@ async def run_agent_blocking(
|
||||
|
||||
await semaphore.acquire()
|
||||
try:
|
||||
context = copy_context()
|
||||
# 长期 worker 保持空底层上下文,每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(
|
||||
_get_blocking_executor(bucket_name).submit,
|
||||
context.run,
|
||||
bound_call,
|
||||
)
|
||||
future = _submit_blocking_call(bucket_name, bound_call)
|
||||
except Exception:
|
||||
semaphore.release()
|
||||
raise
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from contextvars import copy_context
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
@@ -96,9 +97,11 @@ def refresh_plugin_registrations(plugin_id: str) -> None:
|
||||
|
||||
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载插件实例并重新注册其命令、定时任务和 API。"""
|
||||
runtime_status = get_plugin_manager().reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
plugin_manager = get_plugin_manager()
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
|
||||
|
||||
def summarize_plugin(plugin: Any) -> dict[str, Any]:
|
||||
@@ -351,8 +354,10 @@ async def install_plugin_runtime(
|
||||
|
||||
async def reload_runtime(target_id: str) -> object:
|
||||
"""通过 Agent 阻塞任务适配器重载源插件及其虚拟实例。"""
|
||||
mutation_context = copy_context()
|
||||
return await run_agent_blocking(
|
||||
"plugin",
|
||||
mutation_context.run,
|
||||
plugin_manager.reload_plugin_tree,
|
||||
target_id,
|
||||
)
|
||||
@@ -371,31 +376,32 @@ async def install_plugin_runtime(
|
||||
)
|
||||
return result
|
||||
|
||||
with plugin_manager.suppress_plugin_monitor(plugin_id):
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
result = await PluginInstallCommand(
|
||||
installed_plugins_reader=lambda: SystemConfigOper().get(
|
||||
SystemConfigKey.UserInstalledPlugins
|
||||
) or [],
|
||||
installed_plugins_writer=save_installed_plugins,
|
||||
plugin_ids_provider=plugin_manager.get_plugin_ids,
|
||||
compatibility_checker=skip_compatibility_check,
|
||||
package_installer=install_package,
|
||||
package_checkpointer=package_manager.async_checkpoint,
|
||||
package_committer=package_manager.async_commit,
|
||||
package_rollback=package_manager.async_rollback,
|
||||
install_reporter=lambda target_id, target_repo: (
|
||||
MoviePilotServerHelper.async_install_plugin_reg(
|
||||
plugin_id=target_id,
|
||||
repo_url=target_repo,
|
||||
)
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
mutation=plugin_manager.mutation,
|
||||
package_write_guard=plugin_manager.suppress_plugin_monitor,
|
||||
).execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force=force,
|
||||
)
|
||||
return result.success, result.message, result.refreshed_only
|
||||
|
||||
|
||||
@@ -409,48 +415,54 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
|
||||
from app.agent.tools.base import run_agent_blocking
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身:{instance_ids}")
|
||||
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
|
||||
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [
|
||||
plugin for plugin in install_plugins if plugin != plugin_id
|
||||
]
|
||||
await config_oper.async_set(
|
||||
SystemConfigKey.UserInstalledPlugins,
|
||||
install_plugins,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
|
||||
@@ -89,47 +89,51 @@ class UpdatePluginConfigTool(MoviePilotTool):
|
||||
)
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
with plugin_manager.mutation(f"更新插件 {plugin_id} 配置"):
|
||||
current_config = dict(plugin_manager.get_plugin_config(plugin_id) or {})
|
||||
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
# merge 模式以当前保存值为基准,replace 模式则从空配置开始重建。
|
||||
next_config = {} if replace else dict(current_config)
|
||||
if updates:
|
||||
next_config.update(updates)
|
||||
for key in remove_keys:
|
||||
next_config.pop(key, None)
|
||||
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
if not await plugin_manager.async_save_plugin_config(plugin_id, next_config):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
changed_keys = sorted(
|
||||
key
|
||||
for key in set(current_config.keys()) | set(next_config.keys())
|
||||
if current_config.get(key) != next_config.get(key)
|
||||
or (key in current_config) != (key in next_config)
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
if not await plugin_manager.async_save_plugin_config(
|
||||
plugin_id,
|
||||
next_config,
|
||||
):
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"保存插件 {plugin_id} 配置失败",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
"success": True,
|
||||
**plugin_info,
|
||||
"message": "插件配置已保存,请调用 reload_plugin 使最新配置生效",
|
||||
"replace": replace,
|
||||
"changed_keys": changed_keys,
|
||||
"removed_keys": remove_keys,
|
||||
"config_requires_reload": True,
|
||||
"previous_config": current_config,
|
||||
"saved_config": next_config,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
default=str,
|
||||
)
|
||||
|
||||
async def run(
|
||||
self,
|
||||
|
||||
@@ -40,4 +40,5 @@ def get_plugin_config_command() -> PluginConfigCommand:
|
||||
reload_runtime=manager.reload_plugin,
|
||||
publish_reset=publish_reset,
|
||||
refresh_registrations=refresh_registrations,
|
||||
mutation=manager.mutation,
|
||||
)
|
||||
|
||||
+153
-107
@@ -57,7 +57,10 @@ from app.api.dependencies.plugin import (
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.adapters.external.market import PluginHelper
|
||||
from app.adapters.system.plugin.package import PluginPackageManager
|
||||
from app.schemas.exception import PersistenceUnavailableError
|
||||
from app.schemas.exception import (
|
||||
PersistenceUnavailableError,
|
||||
PluginMutationRejectedError,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.api.context import get_background_task_registry, resolve_background_task_registry
|
||||
@@ -520,10 +523,15 @@ def reload_plugin(
|
||||
"""
|
||||
重新加载插件
|
||||
"""
|
||||
# 重新加载插件
|
||||
runtime_status = PluginManager().reload_plugin(plugin_id)
|
||||
# 注册插件服务
|
||||
register_plugin(plugin_id)
|
||||
plugin_manager = PluginManager()
|
||||
try:
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
# 重新加载插件
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
# 注册插件服务
|
||||
register_plugin(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
if runtime_status is _SchemaPluginRuntimeStatus.ACTIVE:
|
||||
return _SchemaResponse(success=True)
|
||||
return _SchemaResponse(
|
||||
@@ -600,14 +608,15 @@ async def install(
|
||||
),
|
||||
plugin_reloader=reload_runtime,
|
||||
registration_refresher=refresh_registrations,
|
||||
mutation=plugin_manager.mutation,
|
||||
package_write_guard=plugin_manager.suppress_plugin_monitor,
|
||||
)
|
||||
result = await command.execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=bool(force),
|
||||
)
|
||||
with plugin_manager.suppress_plugin_monitor(plugin_id):
|
||||
result = await command.execute(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=bool(force),
|
||||
)
|
||||
if not result.success:
|
||||
return _SchemaResponse(success=False, message=result.message)
|
||||
return _SchemaResponse(success=True)
|
||||
@@ -888,11 +897,12 @@ async def save_plugin_folders(
|
||||
保存插件文件夹分组配置
|
||||
"""
|
||||
try:
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(success=True)
|
||||
with PluginManager().mutation("保存插件文件夹配置"):
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(success=True)
|
||||
except PersistenceUnavailableError:
|
||||
raise
|
||||
except Exception as e:
|
||||
@@ -909,18 +919,26 @@ async def create_plugin_folder(
|
||||
"""
|
||||
创建新的插件文件夹
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
if folder_name not in folders:
|
||||
folders[folder_name] = []
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 创建成功"
|
||||
)
|
||||
else:
|
||||
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 已存在")
|
||||
try:
|
||||
with PluginManager().mutation(f"创建插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
if folder_name not in folders:
|
||||
folders[folder_name] = []
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 创建成功"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"文件夹 '{folder_name}' 已存在",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.delete(
|
||||
@@ -932,15 +950,26 @@ async def delete_plugin_folder(
|
||||
"""
|
||||
删除插件文件夹
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
if folder_name in folders:
|
||||
del folders[folder_name]
|
||||
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 删除成功"
|
||||
)
|
||||
else:
|
||||
return _SchemaResponse(success=False, message=f"文件夹 '{folder_name}' 不存在")
|
||||
try:
|
||||
with PluginManager().mutation(f"删除插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
if folder_name in folders:
|
||||
del folders[folder_name]
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 删除成功"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"文件夹 '{folder_name}' 不存在",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.put(
|
||||
@@ -956,12 +985,22 @@ async def update_folder_plugins(
|
||||
"""
|
||||
更新指定文件夹中的插件列表
|
||||
"""
|
||||
folders = get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
folders[folder_name] = plugin_ids
|
||||
await get_configured_system_config().async_set(SystemConfigKey.PluginFolders, folders)
|
||||
return _SchemaResponse(
|
||||
success=True, message=f"文件夹 '{folder_name}' 中的插件已更新"
|
||||
)
|
||||
try:
|
||||
with PluginManager().mutation(f"更新插件文件夹 {folder_name}"):
|
||||
folders = (
|
||||
get_configured_system_config().get(SystemConfigKey.PluginFolders) or {}
|
||||
)
|
||||
folders[folder_name] = plugin_ids
|
||||
await get_configured_system_config().async_set(
|
||||
SystemConfigKey.PluginFolders,
|
||||
folders,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
message=f"文件夹 '{folder_name}' 中的插件已更新",
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
@router.post(
|
||||
@@ -975,23 +1014,24 @@ def clone_plugin(
|
||||
"""
|
||||
创建插件分身
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
try:
|
||||
success, message = PluginManager().clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.suffix,
|
||||
name=clone_data.name,
|
||||
description=clone_data.description,
|
||||
version=clone_data.version,
|
||||
icon=clone_data.icon,
|
||||
)
|
||||
with plugin_manager.mutation(f"创建插件 {plugin_id} 分身"):
|
||||
success, message = plugin_manager.clone_plugin(
|
||||
plugin_id=plugin_id,
|
||||
suffix=clone_data.suffix,
|
||||
name=clone_data.name,
|
||||
description=clone_data.description,
|
||||
version=clone_data.version,
|
||||
icon=clone_data.icon,
|
||||
)
|
||||
|
||||
if success:
|
||||
# 分身服务已完成运行态加载,此处只补齐宿主注册。
|
||||
register_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return _SchemaResponse(success=True, message="插件分身创建成功")
|
||||
else:
|
||||
if success:
|
||||
# 分身服务已完成运行态加载,此处只补齐宿主注册。
|
||||
register_plugin(message)
|
||||
# 将分身插件添加到原插件所在的文件夹中
|
||||
_add_clone_to_plugin_folder(plugin_id, message)
|
||||
return _SchemaResponse(success=True, message="插件分身创建成功")
|
||||
return _SchemaResponse(success=False, message=message)
|
||||
except Exception as e:
|
||||
logger.error(f"创建插件分身失败:{str(e)}")
|
||||
@@ -1034,54 +1074,60 @@ def uninstall_plugin(
|
||||
卸载插件
|
||||
"""
|
||||
plugin_manager = PluginManager()
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"请先卸载该插件的分身:{instance_ids}",
|
||||
)
|
||||
config_oper = get_configured_system_config()
|
||||
# 删除已安装信息
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
for plugin in install_plugins:
|
||||
if plugin == plugin_id:
|
||||
install_plugins.remove(plugin)
|
||||
break
|
||||
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
# 移除插件API
|
||||
remove_plugin_api(plugin_id)
|
||||
# 移除插件服务
|
||||
remove_plugin_job(plugin_id)
|
||||
# 判断是否为分身
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif getattr(plugin_class, "is_clone", False):
|
||||
# 如果是分身插件,则删除分身数据和配置
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
# 删除分身文件
|
||||
plugin_base_dir = (
|
||||
get_api_runtime_config_snapshot().root_path
|
||||
/ "app"
|
||||
/ "plugins"
|
||||
/ plugin_id.lower()
|
||||
)
|
||||
if plugin_base_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(plugin_base_dir)
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception as e:
|
||||
logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}")
|
||||
# 从插件文件夹中移除该插件
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
# 移除插件
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
return _SchemaResponse(success=True)
|
||||
try:
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=f"请先卸载该插件的分身:{instance_ids}",
|
||||
)
|
||||
config_oper = get_configured_system_config()
|
||||
# 删除已安装信息
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
for plugin in install_plugins:
|
||||
if plugin == plugin_id:
|
||||
install_plugins.remove(plugin)
|
||||
break
|
||||
config_oper.set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
# 移除插件API
|
||||
remove_plugin_api(plugin_id)
|
||||
# 移除插件服务
|
||||
remove_plugin_job(plugin_id)
|
||||
# 判断是否为分身
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif getattr(plugin_class, "is_clone", False):
|
||||
# 如果是分身插件,则删除分身数据和配置
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
# 删除分身文件
|
||||
plugin_base_dir = (
|
||||
get_api_runtime_config_snapshot().root_path
|
||||
/ "app"
|
||||
/ "plugins"
|
||||
/ plugin_id.lower()
|
||||
)
|
||||
if plugin_base_dir.exists():
|
||||
try:
|
||||
shutil.rmtree(plugin_base_dir)
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}"
|
||||
)
|
||||
# 从插件文件夹中移除该插件
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
# 移除插件
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
return _SchemaResponse(success=True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return _SchemaResponse(success=False, message=str(error))
|
||||
|
||||
|
||||
def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str):
|
||||
|
||||
@@ -4,7 +4,9 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Any, ContextManager
|
||||
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -29,6 +31,7 @@ class PluginConfigCommand:
|
||||
reload_runtime: Callable[[str], Any],
|
||||
publish_reset: Callable[[str], Any],
|
||||
refresh_registrations: Callable[[str], Any],
|
||||
mutation: Callable[[str], ContextManager[None]],
|
||||
) -> None:
|
||||
"""保存插件管理 Facade 和运行时注册刷新端口。"""
|
||||
self._save_config = save_config
|
||||
@@ -39,21 +42,30 @@ class PluginConfigCommand:
|
||||
self._reload_runtime = reload_runtime
|
||||
self._publish_reset = publish_reset
|
||||
self._refresh_registrations = refresh_registrations
|
||||
self._mutation = mutation
|
||||
|
||||
def update(self, plugin_id: str, config: dict) -> PluginConfigResult:
|
||||
"""保存配置并按既有顺序重新初始化实例及运行时注册。"""
|
||||
if not self._save_config(plugin_id, config, False):
|
||||
return PluginConfigResult(False, "插件配置保存失败")
|
||||
self._initialize(plugin_id, config)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
try:
|
||||
with self._mutation(f"更新插件 {plugin_id} 配置"):
|
||||
if not self._save_config(plugin_id, config, False):
|
||||
return PluginConfigResult(False, "插件配置保存失败")
|
||||
self._initialize(plugin_id, config)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginConfigResult(False, str(error))
|
||||
|
||||
def reset(self, plugin_id: str) -> PluginConfigResult:
|
||||
"""通知插件补偿后停止实例、删除配置数据并重建运行态。"""
|
||||
self._publish_reset(plugin_id)
|
||||
self._stop(plugin_id)
|
||||
self._delete_config(plugin_id, True)
|
||||
self._delete_data(plugin_id, True)
|
||||
self._reload_runtime(plugin_id)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
try:
|
||||
with self._mutation(f"重置插件 {plugin_id} 配置和数据"):
|
||||
self._publish_reset(plugin_id)
|
||||
self._stop(plugin_id)
|
||||
self._delete_config(plugin_id, True)
|
||||
self._delete_data(plugin_id, True)
|
||||
self._reload_runtime(plugin_id)
|
||||
self._refresh_registrations(plugin_id)
|
||||
return PluginConfigResult(True)
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginConfigResult(False, str(error))
|
||||
|
||||
@@ -5,11 +5,12 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
from typing import Any, ContextManager, Optional
|
||||
|
||||
from app.schemas.exception import PersistenceUnavailableError
|
||||
from app.application.plugin.lifecycle import plugin_lifecycle
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
InstalledPluginsReader = Callable[[], list[str]]
|
||||
@@ -25,6 +26,8 @@ PackageCheckpointAction = Callable[[Any], Awaitable[object]]
|
||||
InstallReporter = Callable[[str, Optional[str]], Awaitable[object]]
|
||||
PluginReloader = Callable[[str], Awaitable[object]]
|
||||
PluginRegistrationRefresher = Callable[[str], Awaitable[object]]
|
||||
PluginMutationAdmission = Callable[[str], ContextManager[None]]
|
||||
PluginPackageWriteGuard = Callable[[str], ContextManager[None]]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -94,6 +97,8 @@ class PluginInstallCommand:
|
||||
install_reporter: InstallReporter,
|
||||
plugin_reloader: PluginReloader,
|
||||
registration_refresher: PluginRegistrationRefresher,
|
||||
mutation: PluginMutationAdmission,
|
||||
package_write_guard: PluginPackageWriteGuard,
|
||||
) -> None:
|
||||
"""保存安装用例所需端口,不绑定数据库、网络或运行时实现。"""
|
||||
self._installed_plugins_reader = installed_plugins_reader
|
||||
@@ -107,6 +112,8 @@ class PluginInstallCommand:
|
||||
self._install_reporter = install_reporter
|
||||
self._plugin_reloader = plugin_reloader
|
||||
self._registration_refresher = registration_refresher
|
||||
self._mutation = mutation
|
||||
self._package_write_guard = package_write_guard
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
@@ -120,20 +127,29 @@ class PluginInstallCommand:
|
||||
state = _InstallState()
|
||||
async with plugin_lifecycle.hold(plugin_id):
|
||||
try:
|
||||
return await self._execute_locked(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=force,
|
||||
state=state,
|
||||
with self._mutation(f"安装插件 {plugin_id}"):
|
||||
with self._package_write_guard(plugin_id):
|
||||
try:
|
||||
return await self._execute_locked(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url,
|
||||
release_version=release_version,
|
||||
force=force,
|
||||
state=state,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await self._rollback_cancelled(
|
||||
plugin_id=plugin_id,
|
||||
original_plugins=state.original_plugins,
|
||||
state=state,
|
||||
)
|
||||
raise
|
||||
except PluginMutationRejectedError as error:
|
||||
return PluginInstallResult(
|
||||
success=False,
|
||||
message=str(error),
|
||||
failure_stage="admission",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await self._rollback_cancelled(
|
||||
plugin_id=plugin_id,
|
||||
original_plugins=state.original_plugins,
|
||||
state=state,
|
||||
)
|
||||
raise
|
||||
|
||||
async def _execute_locked(
|
||||
self,
|
||||
|
||||
@@ -3,12 +3,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
|
||||
from typing import Any, ContextManager, Protocol
|
||||
|
||||
class PluginRuntime(Protocol):
|
||||
"""声明入口层消费的插件宿主能力。"""
|
||||
|
||||
def mutation(self, operation: str) -> ContextManager[None]:
|
||||
"""为完整插件可变事务取得停机准入 lease。"""
|
||||
...
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
"""允许兼容门面按既有 V3 方法名访问插件宿主能力。"""
|
||||
|
||||
|
||||
+170
-29
@@ -191,9 +191,16 @@ class TransferFailureNotificationAggregator:
|
||||
NOTIFICATION_DEBOUNCE_SECONDS = 30
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化分组缓冲和定时器。"""
|
||||
"""初始化分组缓冲、回调、定时器与关闭状态。"""
|
||||
self._buffers: dict[str, list[TransferFailureNotification]] = {}
|
||||
self._callbacks: dict[
|
||||
str,
|
||||
Callable[[list[TransferFailureNotification]], None],
|
||||
] = {}
|
||||
self._timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._generations: dict[str, int] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._closed = False
|
||||
|
||||
def schedule(
|
||||
self,
|
||||
@@ -204,48 +211,128 @@ class TransferFailureNotificationAggregator:
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
"""从整理线程安全地把失败快照加入事件循环中的聚合缓冲。"""
|
||||
loop.call_soon_threadsafe(
|
||||
self._schedule_on_loop,
|
||||
group_key,
|
||||
notification,
|
||||
callback,
|
||||
loop,
|
||||
)
|
||||
# 先在调用线程登记快照,关闭流程才能覆盖已接收但尚未进入事件循环的通知。
|
||||
with self._lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("整理失败通知聚合器正在关闭,不能再接收通知")
|
||||
self._buffers.setdefault(group_key, []).append(notification)
|
||||
self._callbacks[group_key] = callback
|
||||
generation = self._generations.get(group_key, 0) + 1
|
||||
self._generations[group_key] = generation
|
||||
try:
|
||||
loop.call_soon_threadsafe(
|
||||
self._schedule_on_loop,
|
||||
group_key,
|
||||
generation,
|
||||
callback,
|
||||
loop,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"创建整理失败通知聚合定时器失败,将立即发送 "
|
||||
f"(group={group_key}): {err}"
|
||||
)
|
||||
self.flush(group_key, generation, callback)
|
||||
|
||||
def _schedule_on_loop(
|
||||
self,
|
||||
group_key: str,
|
||||
notification: TransferFailureNotification,
|
||||
generation: int,
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
loop: asyncio.AbstractEventLoop,
|
||||
) -> None:
|
||||
"""在所属事件循环中更新缓冲并重置静默窗口。"""
|
||||
self._buffers.setdefault(group_key, []).append(notification)
|
||||
timer = self._timers.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
self._timers[group_key] = loop.call_later(
|
||||
self.NOTIFICATION_DEBOUNCE_SECONDS,
|
||||
self.flush,
|
||||
group_key,
|
||||
callback,
|
||||
)
|
||||
"""在所属事件循环中为已登记缓冲重置静默窗口。"""
|
||||
schedule_error: Optional[Exception] = None
|
||||
with self._lock:
|
||||
if (
|
||||
self._closed
|
||||
or group_key not in self._buffers
|
||||
or self._generations.get(group_key) != generation
|
||||
):
|
||||
return
|
||||
timer = self._timers.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
try:
|
||||
self._timers[group_key] = loop.call_later(
|
||||
self.NOTIFICATION_DEBOUNCE_SECONDS,
|
||||
self.flush,
|
||||
group_key,
|
||||
generation,
|
||||
callback,
|
||||
)
|
||||
except Exception as err:
|
||||
schedule_error = err
|
||||
if schedule_error is not None:
|
||||
logger.error(
|
||||
f"创建整理失败通知聚合定时器失败,将立即发送 "
|
||||
f"(group={group_key}): {schedule_error}"
|
||||
)
|
||||
self.flush(group_key, generation, callback)
|
||||
|
||||
def flush(
|
||||
self,
|
||||
group_key: str,
|
||||
generation: int,
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
) -> None:
|
||||
"""发送一个分组内的聚合结果并释放缓冲。"""
|
||||
notifications = self._buffers.pop(group_key, [])
|
||||
self._timers.pop(group_key, None)
|
||||
with self._lock:
|
||||
# 新通知已登记但 timer 重置回调尚未执行时,旧代不得提前发送新批次。
|
||||
if self._generations.get(group_key) != generation:
|
||||
return
|
||||
notifications = self._buffers.pop(group_key, [])
|
||||
self._callbacks.pop(group_key, None)
|
||||
timer = self._timers.pop(group_key, None)
|
||||
self._generations.pop(group_key, None)
|
||||
if timer:
|
||||
timer.cancel()
|
||||
if not notifications:
|
||||
return
|
||||
self._deliver(group_key, notifications, callback)
|
||||
|
||||
@staticmethod
|
||||
def _deliver(
|
||||
group_key: str,
|
||||
notifications: list[TransferFailureNotification],
|
||||
callback: Callable[[list[TransferFailureNotification]], None],
|
||||
) -> None:
|
||||
"""调用聚合通知回调,并统一观察发送异常。"""
|
||||
try:
|
||||
callback(notifications)
|
||||
except Exception as err:
|
||||
logger.error(f"发送整理失败聚合通知失败 (group={group_key}): {err}")
|
||||
|
||||
def close(self) -> None:
|
||||
"""停止接收新通知,取消定时器并同步发送全部已缓冲通知。"""
|
||||
with self._lock:
|
||||
if self._closed and not self._buffers and not self._timers:
|
||||
return
|
||||
self._closed = True
|
||||
timers = list(self._timers.values())
|
||||
pending = []
|
||||
orphaned = []
|
||||
for group_key, notifications in self._buffers.items():
|
||||
callback = self._callbacks.get(group_key)
|
||||
if callback is None:
|
||||
orphaned.append((group_key, len(notifications)))
|
||||
continue
|
||||
pending.append((group_key, notifications, callback))
|
||||
self._timers.clear()
|
||||
self._generations.clear()
|
||||
self._buffers.clear()
|
||||
self._callbacks.clear()
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
for group_key, notification_count in orphaned:
|
||||
logger.error(
|
||||
f"整理失败通知聚合缓冲缺少发送回调,无法刷新 "
|
||||
f"(group={group_key}, count={notification_count})"
|
||||
)
|
||||
for group_key, notifications, callback in pending:
|
||||
self._deliver(group_key, notifications, callback)
|
||||
|
||||
|
||||
# 作业锁:JobManager 与 TransferChain 共享,保护整理作业视图。
|
||||
job_lock = threading.Lock()
|
||||
@@ -267,7 +354,8 @@ class JobManager:
|
||||
# 记录仍由主程序整理线程直接执行的任务,避免把阻塞中的本地任务误判为失活
|
||||
_active_executions: set[Tuple[str, str]] = set()
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化当前进程内的整理作业状态。"""
|
||||
self._job_view = {}
|
||||
self._season_episodes = {}
|
||||
self._meta_to_media_ids = {}
|
||||
@@ -1011,25 +1099,68 @@ class JobManager:
|
||||
|
||||
class FailedRetryScheduler:
|
||||
"""
|
||||
负责失败整理记录的 debounce 聚合与 AI 重试调度。
|
||||
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。
|
||||
|
||||
缓冲不提供持久化保证;关闭时会取消尚未触发的记录,由上层 durable
|
||||
工作流在后续阶段承接需要跨进程保证的重试意图。
|
||||
"""
|
||||
|
||||
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
|
||||
super().__init__()
|
||||
self._retry_transfer_buffer: dict[str, list[int]] = {}
|
||||
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._retry_transfer_generations: dict[str, int] = {}
|
||||
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
|
||||
self._retry_transfer_lock = asyncio.Lock()
|
||||
self._closed = False
|
||||
|
||||
async def close(self):
|
||||
async def close(self) -> None:
|
||||
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
|
||||
async with self._retry_transfer_lock:
|
||||
self._closed = True
|
||||
timers = list(self._retry_transfer_timers.values())
|
||||
buffered_count = sum(
|
||||
len(history_ids)
|
||||
for history_ids in self._retry_transfer_buffer.values()
|
||||
)
|
||||
self._retry_transfer_timers.clear()
|
||||
self._retry_transfer_generations.clear()
|
||||
self._retry_transfer_buffer.clear()
|
||||
tasks = tuple(self._retry_transfer_tasks)
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
if buffered_count:
|
||||
logger.warning(
|
||||
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
|
||||
)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
|
||||
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
|
||||
if self._closed:
|
||||
return
|
||||
task = asyncio.create_task(
|
||||
self._flush_retry_transfer(group_key, generation),
|
||||
name="transfer.failed_retry.flush",
|
||||
)
|
||||
self._retry_transfer_tasks.add(task)
|
||||
task.add_done_callback(self._observe_retry_transfer_task)
|
||||
|
||||
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
|
||||
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
|
||||
self._retry_transfer_tasks.discard(task)
|
||||
if task.cancelled():
|
||||
return
|
||||
exception = task.exception()
|
||||
if exception is not None:
|
||||
logger.error(f"智能体重试整理后台任务异常: {exception}")
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_transfer_template_context(
|
||||
@@ -1054,7 +1185,7 @@ class FailedRetryScheduler:
|
||||
template_context=template_context,
|
||||
)
|
||||
|
||||
async def schedule_retry(self, history_id: int, group_key: str = ""):
|
||||
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
|
||||
"""
|
||||
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
|
||||
"""
|
||||
@@ -1062,6 +1193,8 @@ class FailedRetryScheduler:
|
||||
group_key = f"_default_{history_id}"
|
||||
|
||||
async with self._retry_transfer_lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
|
||||
if group_key not in self._retry_transfer_buffer:
|
||||
self._retry_transfer_buffer[group_key] = []
|
||||
if history_id not in self._retry_transfer_buffer[group_key]:
|
||||
@@ -1075,18 +1208,26 @@ class FailedRetryScheduler:
|
||||
self._retry_transfer_timers[group_key].cancel()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
generation = self._retry_transfer_generations.get(group_key, 0) + 1
|
||||
self._retry_transfer_generations[group_key] = generation
|
||||
self._retry_transfer_timers[group_key] = loop.call_later(
|
||||
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
|
||||
lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)),
|
||||
self._start_retry_transfer_task,
|
||||
group_key,
|
||||
generation,
|
||||
)
|
||||
|
||||
async def _flush_retry_transfer(self, group_key: str):
|
||||
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
|
||||
"""
|
||||
延迟定时器到期后,取出该分组的所有 history_id 并合并为一次 agent 调用。
|
||||
"""
|
||||
async with self._retry_transfer_lock:
|
||||
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
|
||||
if self._retry_transfer_generations.get(group_key) != generation:
|
||||
return
|
||||
history_ids = self._retry_transfer_buffer.pop(group_key, [])
|
||||
self._retry_transfer_timers.pop(group_key, None)
|
||||
self._retry_transfer_generations.pop(group_key, None)
|
||||
|
||||
if not history_ids:
|
||||
return
|
||||
|
||||
+335
-72
@@ -2,9 +2,12 @@ import asyncio
|
||||
import queue
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from concurrent.futures import CancelledError as FutureCancelledError
|
||||
from concurrent.futures import Future
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple, Union, Dict, Callable, Any
|
||||
@@ -87,10 +90,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
文件整理处理链
|
||||
"""
|
||||
|
||||
# worker 在构造期启动;若中途失败,单例仍需先发布给 lifespan 清理入口。
|
||||
_retain_failed_singleton = True
|
||||
|
||||
CONFIG_WATCH = {
|
||||
"TRANSFER_THREADS",
|
||||
}
|
||||
|
||||
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
|
||||
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
|
||||
_QUEUE_STOP_SENTINEL = object()
|
||||
|
||||
@staticmethod
|
||||
def _transfer_result_payload(
|
||||
task: TransferTask,
|
||||
@@ -108,7 +118,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
"transfer_history_id": history_id,
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化文件整理处理链。"""
|
||||
super().__init__()
|
||||
# 主要媒体文件后缀
|
||||
@@ -141,7 +151,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._progress = ProgressHelper(ProgressKey.FileTransfer)
|
||||
# 队列相关状态
|
||||
self._threads = []
|
||||
self._retiring_threads: List[threading.Thread] = []
|
||||
self._queue_active = False
|
||||
# 每一代 worker 使用独立停止信号,避免热更新启动新 worker 后旧线程重新取任务
|
||||
self._worker_stop_event = threading.Event()
|
||||
# 生命周期操作串行化;状态锁只保护短临界区,不覆盖同步文件 I/O 等待
|
||||
self._worker_lifecycle_lock = threading.RLock()
|
||||
self._worker_state_lock = threading.RLock()
|
||||
self._closing = False
|
||||
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
|
||||
self._replay_thread: Optional[threading.Thread] = None
|
||||
self._replay_stop_event = threading.Event()
|
||||
self._active_tasks = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
@@ -149,33 +169,210 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
# 启动整理任务
|
||||
self.__init()
|
||||
|
||||
def __init(self):
|
||||
"""
|
||||
启动文件整理线程
|
||||
"""
|
||||
self._queue_active = True
|
||||
for i in range(self.runtime_config.transfer_threads):
|
||||
logger.info(f"启动文件整理线程 {i + 1} ...")
|
||||
thread = threading.Thread(
|
||||
target=self.__start_transfer, name=f"transfer-{i}", daemon=True
|
||||
def __init(self) -> bool:
|
||||
"""启动一代文件整理线程,并返回是否成功取得 worker 所有权。"""
|
||||
with self._worker_lifecycle_lock:
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.warning("文件整理链已进入关闭状态,拒绝重新启动 worker")
|
||||
return False
|
||||
self._retiring_threads = [
|
||||
thread for thread in self._retiring_threads if thread.is_alive()
|
||||
]
|
||||
alive_threads = [thread for thread in self._threads if thread.is_alive()]
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"上一代文件整理线程尚未收敛,拒绝并行启动新 worker:%s",
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
self._threads = alive_threads
|
||||
return False
|
||||
stop_event = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=self.__start_transfer,
|
||||
args=(stop_event,),
|
||||
name=f"transfer-{index}",
|
||||
daemon=True,
|
||||
)
|
||||
for index in range(self.runtime_config.transfer_threads)
|
||||
]
|
||||
self._worker_stop_event = stop_event
|
||||
self._threads = threads
|
||||
self._queue_active = True
|
||||
for index, thread in enumerate(threads):
|
||||
logger.info(f"启动文件整理线程 {index + 1} ...")
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def __join_threads(
|
||||
threads: List[threading.Thread], deadline: float
|
||||
) -> List[threading.Thread]:
|
||||
"""在统一截止时间内等待线程,返回仍未收敛且继续由调用方持有的线程。"""
|
||||
current_thread = threading.current_thread()
|
||||
for thread in threads:
|
||||
if thread is current_thread or not thread.is_alive():
|
||||
continue
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
return [thread for thread in threads if thread.is_alive()]
|
||||
|
||||
def __acquire_worker_lifecycle_lock(self, deadline: float) -> bool:
|
||||
"""在统一截止时间内取得 worker 生命周期锁,并兼容同线程 RLock 重入。"""
|
||||
return self._worker_lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
)
|
||||
|
||||
def __request_worker_stop(self) -> List[threading.Thread]:
|
||||
"""发布当前 worker 代的停止信号,并用哨兵唤醒空闲线程。"""
|
||||
with self._worker_state_lock:
|
||||
self._queue_active = False
|
||||
self._worker_stop_event.set()
|
||||
current_threads = list(self._threads)
|
||||
threads = [*self._retiring_threads, *current_threads]
|
||||
for _ in current_threads:
|
||||
self._queue.put(self._QUEUE_STOP_SENTINEL)
|
||||
return threads
|
||||
|
||||
def __stop(self, timeout_seconds: float = _WORKER_RESTART_TIMEOUT_SECONDS) -> bool:
|
||||
"""在锁等待与线程 join 的共享预算内停止当前 worker 代。"""
|
||||
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
||||
if not self.__acquire_worker_lifecycle_lock(deadline):
|
||||
logger.error(
|
||||
"未在 %.1f 秒内取得文件整理 worker 生命周期锁",
|
||||
max(0.0, timeout_seconds),
|
||||
)
|
||||
self._threads.append(thread)
|
||||
thread.start()
|
||||
return False
|
||||
try:
|
||||
threads = self.__request_worker_stop()
|
||||
alive_threads = self.__join_threads(threads, deadline)
|
||||
with self._worker_state_lock:
|
||||
self._threads = []
|
||||
self._retiring_threads = alive_threads
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"文件整理线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
return False
|
||||
logger.info("文件整理线程已停止")
|
||||
return True
|
||||
finally:
|
||||
self._worker_lifecycle_lock.release()
|
||||
|
||||
def __stop(self):
|
||||
def close_workers(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
|
||||
"""
|
||||
停止文件整理进程
|
||||
"""
|
||||
self._queue_active = False
|
||||
for thread in self._threads:
|
||||
thread.join()
|
||||
self._threads = []
|
||||
logger.info("文件整理线程已停止")
|
||||
关闭整理 worker 与 pending 回放,并拒绝后续队列写入。
|
||||
|
||||
def on_config_changed(self):
|
||||
这是宿主生命周期使用的同步边界。停止信号只能阻止线程领取下一项工作,不能
|
||||
取消已经进入同步文件或数据库 I/O 的调用;超过预算时保留活线程句柄并返回
|
||||
False,调用方据此避免过早释放仍被使用的数据库等下游资源。
|
||||
:param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数
|
||||
:return: 全部后台线程均已收敛时返回 True,否则返回 False
|
||||
"""
|
||||
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
||||
if not self.__acquire_worker_lifecycle_lock(deadline):
|
||||
logger.error(
|
||||
"未在 %.1f 秒内取得整理后台生命周期锁,关闭未开始",
|
||||
max(0.0, timeout_seconds),
|
||||
)
|
||||
return False
|
||||
try:
|
||||
with self._worker_state_lock:
|
||||
self._closing = True
|
||||
worker_threads = self.__request_worker_stop()
|
||||
replay_thread = self._replay_thread
|
||||
self._replay_stop_event.set()
|
||||
|
||||
alive_workers = self.__join_threads(worker_threads, deadline)
|
||||
alive_replays = self.__join_threads(
|
||||
[replay_thread] if replay_thread else [], deadline
|
||||
)
|
||||
with self._worker_state_lock:
|
||||
self._threads = []
|
||||
self._retiring_threads = alive_workers
|
||||
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
|
||||
self._replay_thread = None
|
||||
|
||||
alive_threads = [*alive_workers, *alive_replays]
|
||||
if alive_threads:
|
||||
logger.error(
|
||||
"整理后台线程未在 %.1f 秒内收敛,仍由 TransferChain 持有:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
return False
|
||||
logger.info("文件整理 worker 与待处理回放线程已关闭")
|
||||
return True
|
||||
finally:
|
||||
self._worker_lifecycle_lock.release()
|
||||
|
||||
async def close(self, timeout_seconds: float = _WORKER_CLOSE_TIMEOUT_SECONDS) -> bool:
|
||||
"""
|
||||
收口整理线程、失败通知和 AI 重试,并返回依赖是否可以安全释放。
|
||||
|
||||
同步文件 I/O 在线程内无法被 asyncio 取消,因此先在线程池中执行有界
|
||||
``close_workers``。只有 worker 与 replay 全部退出后才关闭通知和重试;若
|
||||
超时则保留这些依赖,供仍在运行的整理回调继续使用。
|
||||
:param timeout_seconds: worker 与 replay 共享的最大等待秒数
|
||||
:return: 所有整理后台 owner 均已收敛时返回 True
|
||||
"""
|
||||
workers_closed = await asyncio.to_thread(
|
||||
self.close_workers,
|
||||
timeout_seconds,
|
||||
)
|
||||
if not workers_closed:
|
||||
return False
|
||||
self.failure_notification_aggregator.close()
|
||||
await self.retry_scheduler.close()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _observe_failed_retry_schedule(future: Future[None]) -> None:
|
||||
"""观察跨线程 AI 重试调度的完成结果,避免关闭竞态变成静默异常。"""
|
||||
try:
|
||||
future.result()
|
||||
except FutureCancelledError:
|
||||
return
|
||||
except Exception as err:
|
||||
logger.error(f"触发AI智能体重试整理失败: {err}")
|
||||
|
||||
def _schedule_failed_transfer_retry(
|
||||
self,
|
||||
history_id: int,
|
||||
group_key: str,
|
||||
) -> None:
|
||||
"""把失败历史提交到主事件循环,并持续持有和观察调度结果。"""
|
||||
retry_coroutine = self.retry_scheduler.schedule_retry(
|
||||
history_id,
|
||||
group_key=group_key,
|
||||
)
|
||||
try:
|
||||
future = asyncio.run_coroutine_threadsafe(
|
||||
retry_coroutine,
|
||||
global_vars.loop,
|
||||
)
|
||||
except Exception as err:
|
||||
retry_coroutine.close()
|
||||
logger.error(f"触发AI智能体重试整理失败: {err}")
|
||||
return
|
||||
future.add_done_callback(self._observe_failed_retry_schedule)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{history_id}")
|
||||
|
||||
def on_config_changed(self) -> None:
|
||||
"""配置变更时重启文件整理线程。"""
|
||||
self.__stop()
|
||||
self.__init()
|
||||
with self._worker_lifecycle_lock:
|
||||
if self._closing:
|
||||
logger.info("文件整理链正在关闭,忽略 worker 配置热更新")
|
||||
return
|
||||
if not self.__stop(
|
||||
timeout_seconds=self._WORKER_RESTART_TIMEOUT_SECONDS
|
||||
):
|
||||
logger.warning(
|
||||
"旧文件整理 worker 仍在收尾;其停止信号保持有效,新一代接管后续队列"
|
||||
)
|
||||
self.__init()
|
||||
|
||||
def __default_callback(
|
||||
self, task: TransferTask, transferinfo: TransferInfo, /
|
||||
@@ -344,19 +541,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
and self.runtime_config.ai_agent_enable
|
||||
and self.runtime_config.ai_agent_retry_transfer
|
||||
):
|
||||
try:
|
||||
# 使用 download_hash 或源文件父目录作为分组键,
|
||||
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
|
||||
group_key = build_transfer_failure_group_key(task)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.retry_scheduler.schedule_retry(
|
||||
history.id, group_key=group_key
|
||||
),
|
||||
global_vars.loop,
|
||||
)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{history.id}")
|
||||
except Exception as e:
|
||||
logger.error(f"触发AI智能体重试整理失败: {e}")
|
||||
# 使用 download_hash 或源文件父目录作为分组键,
|
||||
# 同一批次(如同一个种子)的失败记录会被合并为一次agent调用
|
||||
self._schedule_failed_transfer_retry(
|
||||
history.id,
|
||||
build_transfer_failure_group_key(task),
|
||||
)
|
||||
|
||||
# 返回失败
|
||||
ret_status = False
|
||||
@@ -653,7 +843,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
:param task: 任务信息
|
||||
:return: True表示任务已添加到队列,False表示任务无效或已存在(重复)
|
||||
"""
|
||||
return self._transfer_queue_service().put(task, self.__default_callback)
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.warning("文件整理链已关闭,拒绝新的队列任务")
|
||||
return False
|
||||
return self._transfer_queue_service().put(task, self.__default_callback)
|
||||
|
||||
def _transfer_queue_service(self) -> TransferQueueService:
|
||||
"""构建保持旧队列对象和私有兼容接缝的应用服务。"""
|
||||
@@ -667,26 +861,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
expire_tasks=self.__expire_stale_transfer_tasks,
|
||||
)
|
||||
|
||||
def replay_pending(self):
|
||||
def replay_pending(self) -> None:
|
||||
"""
|
||||
回放上次进程退出时仍未整理完的文件。
|
||||
|
||||
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
|
||||
挂死状态,同步执行会把整个启动流程堵住。
|
||||
"""
|
||||
threading.Thread(
|
||||
target=self.__replay_pending,
|
||||
name="MoviePilot-TransferReplay",
|
||||
daemon=True
|
||||
).start()
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.info("文件整理链正在关闭,跳过待处理文件回放")
|
||||
return
|
||||
if self._replay_thread and self._replay_thread.is_alive():
|
||||
logger.info("待处理文件回放已在运行,跳过重复启动")
|
||||
return
|
||||
stop_event = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self.__run_replay_pending,
|
||||
args=(stop_event,),
|
||||
name="MoviePilot-TransferReplay",
|
||||
daemon=True,
|
||||
)
|
||||
self._replay_stop_event = stop_event
|
||||
self._replay_thread = thread
|
||||
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
|
||||
thread.start()
|
||||
|
||||
def __replay_pending(self):
|
||||
def __run_replay_pending(self, stop_event: threading.Event) -> None:
|
||||
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
|
||||
try:
|
||||
self.__replay_pending(stop_event)
|
||||
finally:
|
||||
with self._worker_state_lock:
|
||||
if self._replay_thread is threading.current_thread():
|
||||
self._replay_thread = None
|
||||
|
||||
def __replay_pending(
|
||||
self, stop_event: Optional[threading.Event] = None
|
||||
) -> None:
|
||||
"""
|
||||
把落盘登记的待整理文件重新送回整理入口。
|
||||
|
||||
只回放「存储 + 源路径」这一最小事实,重新走完整的识别与整理流程,
|
||||
已经整理完成的由整理历史查重挡掉,因此不存在重复整理的问题。
|
||||
:param stop_event: 宿主关闭信号;只阻止处理下一条登记,不取消运行中的同步 I/O
|
||||
"""
|
||||
stop_event = stop_event or threading.Event()
|
||||
if stop_event.is_set():
|
||||
return
|
||||
try:
|
||||
pendings = self._pendingoper.list_all()
|
||||
except Exception as err:
|
||||
@@ -697,8 +919,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
|
||||
replayed = 0
|
||||
for storage, src_path in pendings:
|
||||
if stop_event.is_set():
|
||||
break
|
||||
try:
|
||||
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
|
||||
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
|
||||
if stop_event.is_set():
|
||||
break
|
||||
if not fileitem:
|
||||
if should_discard:
|
||||
# 源文件确认已消失,注销登记避免每次启动重复回放
|
||||
@@ -708,7 +935,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
replayed += 1
|
||||
except Exception as err:
|
||||
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
|
||||
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
|
||||
if stop_event.is_set():
|
||||
logger.info(
|
||||
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
|
||||
replayed,
|
||||
)
|
||||
else:
|
||||
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
|
||||
|
||||
@staticmethod
|
||||
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
|
||||
@@ -876,21 +1109,69 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self.jobview.try_remove_job(task)
|
||||
self._finish_scrape_batch_task(task)
|
||||
|
||||
def __start_transfer(self):
|
||||
def __settle_transfer_progress_if_idle(self) -> None:
|
||||
"""在没有 active 或未结算真实任务时结束进度并重置本批计数。"""
|
||||
with task_lock:
|
||||
# unfinished_tasks 同时覆盖队列内任务和已被其他 worker 取走、尚未来得及
|
||||
# 登记 active 的任务。持有 Queue 自身互斥锁完成判断和计数重置,使并发
|
||||
# enqueue 只能发生在旧批次归零之后;仍在 deque 中的停止哨兵不算真实任务。
|
||||
with self._queue.all_tasks_done:
|
||||
queued_stop_sentinels = sum(
|
||||
item is self._QUEUE_STOP_SENTINEL
|
||||
for item in self._queue.queue
|
||||
)
|
||||
has_unsettled_tasks = (
|
||||
self._queue.unfinished_tasks > queued_stop_sentinels
|
||||
)
|
||||
if (
|
||||
self._active_tasks != 0
|
||||
or self._processed_num <= 0
|
||||
or has_unsettled_tasks
|
||||
):
|
||||
return
|
||||
processed_num = self._processed_num
|
||||
fail_num = self._fail_num
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
__end_msg = (
|
||||
f"整理队列处理完成,共整理 {processed_num} 个文件,"
|
||||
f"失败 {fail_num} 个"
|
||||
)
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
|
||||
def __start_transfer(self, stop_event: threading.Event) -> None:
|
||||
"""
|
||||
处理队列
|
||||
处理当前 worker 代的队列,停止后不领取下一项任务。
|
||||
|
||||
:param stop_event: 当前 worker 代专属停止信号,热更新后不会被重新清除
|
||||
"""
|
||||
while not global_vars.is_system_stopped and self._queue_active:
|
||||
while not global_vars.is_system_stopped and not stop_event.is_set():
|
||||
try:
|
||||
item: TransferQueue = self._queue.get(
|
||||
block=True, timeout=self._transfer_interval
|
||||
)
|
||||
if item is self._QUEUE_STOP_SENTINEL:
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
if stop_event.is_set() or global_vars.is_system_stopped:
|
||||
break
|
||||
continue
|
||||
if stop_event.is_set() or global_vars.is_system_stopped:
|
||||
# 关闭信号与 queue.get 竞态时,把尚未处理的任务放回队列;其
|
||||
# TransferPending 登记保持不变,供同进程重启 worker 或下次启动回放。
|
||||
self._queue.put(item)
|
||||
self._queue.task_done()
|
||||
break
|
||||
if not item:
|
||||
continue
|
||||
|
||||
task = item.task
|
||||
if not task:
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
continue
|
||||
|
||||
# 文件信息
|
||||
@@ -966,18 +1247,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
with task_lock:
|
||||
# 减少运行中的任务数
|
||||
self._active_tasks -= 1
|
||||
# 检查是否所有任务都已完成且队列为空
|
||||
if self._active_tasks == 0 and self._queue.empty():
|
||||
# 结束进度
|
||||
__end_msg = f"整理队列处理完成,共整理 {self._processed_num} 个文件,失败 {self._fail_num} 个"
|
||||
logger.info(__end_msg)
|
||||
self._progress.update(value=100, text=__end_msg)
|
||||
self._progress.end()
|
||||
# 重置计数,_total_num 一并归零,否则会作为历史最大值一直
|
||||
# 累积,令后续批次的「当前共 N 个文件」与进度百分比失真
|
||||
self._total_num = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
|
||||
except queue.Empty:
|
||||
# 即使队列空了,如果还有任务在运行,也不应该结束进度
|
||||
@@ -1113,18 +1383,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
and self.runtime_config.ai_agent_enable
|
||||
and self.runtime_config.ai_agent_retry_transfer
|
||||
):
|
||||
try:
|
||||
# 使用 download_hash 或源文件父目录作为分组键
|
||||
group_key = build_transfer_failure_group_key(task)
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self.retry_scheduler.schedule_retry(
|
||||
his.id, group_key=group_key
|
||||
),
|
||||
global_vars.loop,
|
||||
)
|
||||
logger.info(f"已触发AI智能体重试整理历史记录 #{his.id}")
|
||||
except Exception as e:
|
||||
logger.error(f"触发AI智能体重试整理失败: {e}")
|
||||
# 使用 download_hash 或源文件父目录作为分组键
|
||||
self._schedule_failed_transfer_retry(
|
||||
his.id,
|
||||
build_transfer_failure_group_key(task),
|
||||
)
|
||||
|
||||
return False, "未识别到媒体信息"
|
||||
|
||||
|
||||
@@ -14,14 +14,24 @@ class Singleton(abc.ABCMeta, type):
|
||||
def get_existing_instance(cls, *args, **kwargs):
|
||||
"""按相同参数返回已创建实例,不触发初始化"""
|
||||
key = (cls, args, frozenset(kwargs.items()))
|
||||
return cls._instances.get(key)
|
||||
with cls._lock:
|
||||
return cls._instances.get(key)
|
||||
|
||||
def __call__(cls, *args, **kwargs):
|
||||
"""按类和构造参数创建或复用实例。"""
|
||||
key = (cls, args, frozenset(kwargs.items()))
|
||||
with cls._lock:
|
||||
if key not in cls._instances:
|
||||
cls._instances[key] = super().__call__(*args, **kwargs)
|
||||
if getattr(cls, "_retain_failed_singleton", False):
|
||||
# 启动线程的 lifecycle owner 必须先发布身份再执行 __init__;
|
||||
# 构造中途抛错时保留实例,启动失败清理才能找到已创建的 owner。
|
||||
instance = cls.__new__(cls, *args, **kwargs)
|
||||
if not isinstance(instance, cls):
|
||||
return instance
|
||||
cls._instances[key] = instance
|
||||
cls.__init__(instance, *args, **kwargs)
|
||||
else:
|
||||
cls._instances[key] = super().__call__(*args, **kwargs)
|
||||
return cls._instances[key]
|
||||
|
||||
|
||||
@@ -42,13 +52,23 @@ class SingletonClass(abc.ABCMeta, type):
|
||||
|
||||
def get_existing_instance(cls):
|
||||
"""返回已创建实例,不触发初始化"""
|
||||
return cls._instances.get(cls)
|
||||
with cls._lock:
|
||||
return cls._instances.get(cls)
|
||||
|
||||
def __call__(cls, *args, **kwargs):
|
||||
"""按类创建或复用唯一实例。"""
|
||||
with cls._lock:
|
||||
if cls not in cls._instances:
|
||||
cls._instances[cls] = super().__call__(*args, **kwargs)
|
||||
if getattr(cls, "_retain_failed_singleton", False):
|
||||
# 与参数化单例保持相同的 owner 发布顺序;锁会阻止其他线程
|
||||
# 在 __init__ 返回或抛错前读取半构造实例。
|
||||
instance = cls.__new__(cls, *args, **kwargs)
|
||||
if not isinstance(instance, cls):
|
||||
return instance
|
||||
cls._instances[cls] = instance
|
||||
cls.__init__(instance, *args, **kwargs)
|
||||
else:
|
||||
cls._instances[cls] = super().__call__(*args, **kwargs)
|
||||
return cls._instances[cls]
|
||||
|
||||
|
||||
|
||||
+399
-57
@@ -1,4 +1,5 @@
|
||||
import time
|
||||
import threading
|
||||
import traceback
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
@@ -27,6 +28,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
||||
"""
|
||||
# watcher/scheduler 在构造期启动;异常时保留实例供启动失败屏障收口。
|
||||
_retain_failed_singleton = True
|
||||
# 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
|
||||
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
|
||||
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
|
||||
@@ -54,11 +57,25 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
REBUILD_KEY_PREFIX = "rebuild:"
|
||||
PROBE_KEY = "probe"
|
||||
PENDING_KEY = "pending"
|
||||
RELOAD_STOP_TIMEOUT = 30.0
|
||||
LIFECYCLE_CLOSE_TIMEOUT = 90.0
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化目录监控依赖、owner 注册表与当前 lifespan 状态。"""
|
||||
super().__init__()
|
||||
# 生命周期操作串行化;永久关闭请求用 Event 提前封口,不能被阻塞 I/O 挡住。
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._owner_lock = threading.Lock()
|
||||
self._work_stop_event = threading.Event()
|
||||
self._shutdown_event = threading.Event()
|
||||
self._closed = False
|
||||
self._compensation_threads: Dict[int, threading.Thread] = {}
|
||||
self._scheduler_shutdown_thread: Optional[threading.Thread] = None
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
# 本地目录监控服务
|
||||
self._watchers = []
|
||||
# 已请求停止但尚未退出的 watcher,只保留 owner 身份,不再参与健康检查
|
||||
self._retired_watchers = []
|
||||
# 本地目录监控列表读写锁
|
||||
self._watcher_lock = Lock()
|
||||
# 启动失败待重试的本地监控配置
|
||||
@@ -87,9 +104,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self._poller = RemotePoller(store=self._store, dispatcher=self._dispatcher,
|
||||
alert_cb=self.__poller_alert)
|
||||
# 启动目录监控和文件整理
|
||||
self.init()
|
||||
if not self.init():
|
||||
raise RuntimeError("目录监控 owner 初始化未收敛")
|
||||
|
||||
def on_config_changed(self):
|
||||
def on_config_changed(self) -> None:
|
||||
"""配置变化时重建监控;lifespan 封口后拒绝重新启动。"""
|
||||
if self._shutdown_event.is_set():
|
||||
logger.info("目录监控已进入生命周期封口,忽略配置热更新")
|
||||
return
|
||||
self.init()
|
||||
|
||||
def get_reload_name(self):
|
||||
@@ -118,6 +140,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
强制全量扫描并处理所有文件(包括已存在的文件)。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
return self._poller.force_full_scan(storage=storage, mon_path=mon_path)
|
||||
|
||||
@staticmethod
|
||||
@@ -134,24 +158,61 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
return SnapshotStore.compare(old_snapshot, new_snapshot)
|
||||
|
||||
def init(self):
|
||||
def init(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""在旧 owner 收敛后启动一代监控,关闭中的生命周期拒绝重开。"""
|
||||
if self._shutdown_event.is_set():
|
||||
logger.info("目录监控生命周期已关闭,跳过启动")
|
||||
return False
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("目录监控未在 %.1f 秒内取得重载所有权", max(0.0, timeout))
|
||||
return False
|
||||
try:
|
||||
if self._shutdown_event.is_set():
|
||||
return False
|
||||
if not self.__stop_owned(deadline=deadline, close=False):
|
||||
logger.error("旧目录监控 owner 未收敛,取消本次配置热更新")
|
||||
return False
|
||||
if self._shutdown_event.is_set():
|
||||
return False
|
||||
if not self._recovery.reopen():
|
||||
logger.error("目录监控恢复线程仍未退出,取消本次配置热更新")
|
||||
return False
|
||||
if self._shutdown_event.is_set():
|
||||
self._recovery.request_stop()
|
||||
return False
|
||||
self._work_stop_event.clear()
|
||||
return self.__initialize_monitors()
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __initialize_monitors(self) -> bool:
|
||||
"""
|
||||
启动监控
|
||||
在已取得生命周期所有权后启动监控。
|
||||
|
||||
永久关闭请求可以在本方法阻塞于 FUSE 时从其他线程提前设置;每个可能产生
|
||||
新 owner 的边界都重新检查,确保解冻后只能退出,不能穿透旧生命周期。
|
||||
"""
|
||||
# 停止现有任务
|
||||
self.stop()
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
|
||||
# 读取目录配置
|
||||
monitor_dirs = DirectoryHelper().get_download_dirs()
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
if not monitor_dirs:
|
||||
logger.info("未找到任何目录监控配置")
|
||||
return
|
||||
return True
|
||||
|
||||
messagehelper = MessageHelper()
|
||||
|
||||
# 先筛出有效的监控配置,再按下载目录去重,避免非监控配置顶掉监控配置
|
||||
valid_dirs = []
|
||||
for mon_dir in monitor_dirs:
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
if not mon_dir.library_path:
|
||||
logger.warn(f"跳过监控配置 {mon_dir.download_path}:未设置媒体库目录")
|
||||
continue
|
||||
@@ -172,6 +233,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(f"找到 {len(monitor_dirs)} 个目录监控配置")
|
||||
|
||||
# 启动定时服务进程
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
self._scheduler = BackgroundScheduler(timezone=get_runtime_setting("TZ"))
|
||||
|
||||
mon_storages: Dict[str, List[Path]] = {}
|
||||
@@ -179,6 +242,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
local_started = 0
|
||||
local_failed = 0
|
||||
for mon_dir in monitor_dirs:
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
# 检查媒体库目录是不是下载目录的子目录
|
||||
mon_path = Path(mon_dir.download_path)
|
||||
target_path = Path(mon_dir.library_path)
|
||||
@@ -197,6 +263,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
mon_storages.setdefault(mon_dir.storage, []).append(mon_path)
|
||||
|
||||
for storage, paths in mon_storages.items():
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
# 远程目录监控 - 使用智能间隔
|
||||
# 先尝试加载已有快照获取文件数量
|
||||
snapshot_data = self._store.load(storage)
|
||||
@@ -238,6 +307,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(f"✓ 目录监控健康检查已启动: [间隔: {self.WATCHDOG_INTERVAL}秒]")
|
||||
|
||||
# 启动定时服务
|
||||
if not self.__accepting_work():
|
||||
self.__abort_interrupted_startup()
|
||||
return False
|
||||
if self._scheduler.get_jobs():
|
||||
self._scheduler.print_jobs()
|
||||
self._scheduler.start()
|
||||
@@ -253,6 +325,47 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(summary)
|
||||
else:
|
||||
logger.info(summary)
|
||||
return True
|
||||
|
||||
def __accepting_work(self) -> bool:
|
||||
"""判断当前 monitor 代是否仍允许创建 owner 和派发整理工作。"""
|
||||
return not self._work_stop_event.is_set() and not self._shutdown_event.is_set()
|
||||
|
||||
@property
|
||||
def lifecycle_closed(self) -> bool:
|
||||
"""返回当前 Monitor 是否已经被应用生命周期永久封口。"""
|
||||
return self._closed or self._shutdown_event.is_set()
|
||||
|
||||
def reopen(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""确认旧 owner 已收敛后,为新的应用 lifespan 显式解除永久封口。"""
|
||||
if not self.lifecycle_closed:
|
||||
return True
|
||||
deadline = time.monotonic() + max(0.0, timeout)
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("旧目录监控生命周期仍在收尾,无法重新开启")
|
||||
return False
|
||||
try:
|
||||
if not self.__stop_owned(deadline=deadline, close=True):
|
||||
logger.error("旧目录监控 owner 仍未收敛,无法重新开启")
|
||||
return False
|
||||
if not self._recovery.reopen():
|
||||
logger.error("旧目录监控恢复线程仍存活,无法重新开启")
|
||||
return False
|
||||
self._closed = False
|
||||
self._shutdown_event.clear()
|
||||
return True
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __abort_interrupted_startup(self) -> None:
|
||||
"""关闭请求穿透阻塞启动后,在当前生命周期锁内收拢已经创建的 owner。"""
|
||||
logger.info("目录监控启动期间收到关闭请求,正在回收本次已创建 owner")
|
||||
self.__stop_owned(
|
||||
deadline=time.monotonic() + self.RELOAD_STOP_TIMEOUT,
|
||||
close=self._shutdown_event.is_set(),
|
||||
)
|
||||
|
||||
def __start_local_monitor(self, mon_path: Path, monitor_mode: str) -> bool:
|
||||
"""
|
||||
@@ -261,6 +374,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:return: 是否启动成功
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return False
|
||||
logger.info(f"正在启动本地目录监控: {mon_path}")
|
||||
logger.info("*** 重要提示:目录监控只处理新增和修改的文件,不会处理监控启动前已存在的文件 ***")
|
||||
|
||||
@@ -298,6 +413,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 启动成功后再登记,避免失败的监控残留在列表中
|
||||
watcher.start()
|
||||
with self._watcher_lock:
|
||||
if not self.__accepting_work():
|
||||
watcher.stop()
|
||||
if watcher.is_alive():
|
||||
self._retired_watchers.append(watcher)
|
||||
return False
|
||||
self._watchers.append(watcher)
|
||||
self._pending_locals = [
|
||||
pending for pending in self._pending_locals
|
||||
@@ -312,13 +432,15 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self.__handle_start_failure(mon_path=mon_path, monitor_mode=monitor_mode, err=e)
|
||||
return False
|
||||
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception):
|
||||
def __handle_start_failure(self, mon_path: Path, monitor_mode: str, err: Exception) -> None:
|
||||
"""
|
||||
处理本地目录监控启动失败,登记待重试并按需告警。
|
||||
:param mon_path: 监控目录
|
||||
:param monitor_mode: 配置的监控模式
|
||||
:param err: 启动异常
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
err_msg = str(err)
|
||||
logger.error(f"启动本地目录监控失败: {mon_path}")
|
||||
logger.error(f"错误详情: {err_msg}")
|
||||
@@ -346,7 +468,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
f"启动本地目录监控失败: {mon_path}\n错误: {err_msg}\n"
|
||||
f"将自动退避重试")
|
||||
|
||||
def watchdog(self):
|
||||
def watchdog(self) -> None:
|
||||
"""
|
||||
目录监控健康检查:检测监控线程状态并驱动恢复。
|
||||
|
||||
@@ -356,8 +478,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
动作会把看门狗冻死在它自己要修复的挂载上,随后停滞检测、告警、重试驱动
|
||||
全部静默失效——这正是全进程雪崩的起点。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
try:
|
||||
broken = self.__check_watchers()
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self.__drive_recovery(broken)
|
||||
except Exception as e:
|
||||
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
||||
@@ -372,6 +498,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
with self._watcher_lock:
|
||||
watchers = list(self._watchers)
|
||||
self._retired_watchers = [
|
||||
watcher for watcher in self._retired_watchers if watcher.is_alive()
|
||||
]
|
||||
isolated = set(self._isolated)
|
||||
# 探测已确认挂载恢复的目录,本轮直接送去重建
|
||||
resumed = list(self._pending_rebuild.values())
|
||||
@@ -412,7 +541,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
broken.append(watcher)
|
||||
return broken
|
||||
|
||||
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
|
||||
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]) -> None:
|
||||
"""
|
||||
把所有会触碰挂载的恢复动作派发到一次性工作线程,并等待有限时间。
|
||||
|
||||
@@ -421,6 +550,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
可放弃的子进程探测来确认挂载何时恢复。
|
||||
:param broken: 需要重建的监控列表
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
actions: Dict[str, Callable[[], None]] = {}
|
||||
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
|
||||
for watcher in broken:
|
||||
@@ -432,6 +563,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
actions[self.PENDING_KEY] = self.__drive_pending
|
||||
|
||||
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
|
||||
for key, state in results.items():
|
||||
if state is RecoveryState.COMPLETED:
|
||||
@@ -445,11 +578,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
|
||||
f"({state.value}),将在后续健康检查周期重试: {key}")
|
||||
|
||||
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
|
||||
def __enter_isolation(self, watcher: LocalDirectoryWatcher) -> None:
|
||||
"""
|
||||
将一个监控目录转入挂载级故障隔离:停止对它的一切新访问,等待探测恢复。
|
||||
:param watcher: 重建未能返回的监控
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(watcher.watch_path)
|
||||
with self._watcher_lock:
|
||||
if key in self._isolated:
|
||||
@@ -466,7 +601,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
|
||||
stage="isolated")
|
||||
|
||||
def __probe_isolated(self):
|
||||
def __probe_isolated(self) -> None:
|
||||
"""
|
||||
对隔离中的监控目录做可放弃探测,挂载恢复应答后解除隔离并重建监控。
|
||||
|
||||
@@ -474,11 +609,17 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
超时可被 kill,因此本线程不会像内联 stat 那样永久冻死。探测通过后的重建
|
||||
仍有极小概率再次卡住,届时本线程会被下一轮的 BUSY 判定跳过,不再泄漏。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
keys = list(self._isolated)
|
||||
for key in keys:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
mon_path = Path(key)
|
||||
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
entry = self._isolated.get(key)
|
||||
if not entry:
|
||||
@@ -489,6 +630,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
|
||||
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
|
||||
continue
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
entry = self._isolated.pop(key, None)
|
||||
if not entry:
|
||||
@@ -502,19 +645,24 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
with self._watcher_lock:
|
||||
self._pending_rebuild[key] = entry["watcher"]
|
||||
|
||||
def __drive_pending(self):
|
||||
def __drive_pending(self) -> None:
|
||||
"""
|
||||
驱动两条待重试队列。两者都会访问挂载(启动重试走目录遍历与 exists,
|
||||
整理重试走 stat),必须在恢复工作线程里执行而不是看门狗线程里。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self.__retry_pending_locals()
|
||||
self._dispatcher.retry_pending()
|
||||
if self.__accepting_work():
|
||||
self._dispatcher.retry_pending()
|
||||
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher) -> None:
|
||||
"""
|
||||
重建一个本地目录监控线程。
|
||||
:param watcher: 需要重建的监控
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
# 卡死的线程阻塞在底层调用中无法强制回收,只能请求停止后由守护线程自然退出
|
||||
watcher.stop()
|
||||
new_watcher = LocalDirectoryWatcher(
|
||||
@@ -527,8 +675,13 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
new_watcher.start()
|
||||
except Exception as e:
|
||||
logger.error(f"重建目录监控失败: {watcher.watch_path} - {e}")
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
# 旧 watcher 可能仍卡在 FUSE 调用里;只有真实退出后才能移除句柄。
|
||||
self._watchers = [item for item in self._watchers if item is not watcher]
|
||||
if watcher.is_alive():
|
||||
self._retired_watchers.append(watcher)
|
||||
if all(pending["mon_path"] != watcher.watch_path for pending in self._pending_locals):
|
||||
self._pending_locals.append({
|
||||
"mon_path": watcher.watch_path,
|
||||
@@ -537,9 +690,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
})
|
||||
return
|
||||
with self._watcher_lock:
|
||||
if not self.__accepting_work():
|
||||
new_watcher.stop()
|
||||
if new_watcher.is_alive():
|
||||
self._retired_watchers.append(new_watcher)
|
||||
return
|
||||
registered = any(item is watcher for item in self._watchers)
|
||||
if registered:
|
||||
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
||||
if watcher.is_alive():
|
||||
# stop() 只能发信号,FUSE 上的旧线程可能永久不返回;继续持有它,
|
||||
# 让生命周期关闭如实失败,而不是把 daemon 泄漏伪装成已收敛。
|
||||
self._retired_watchers.append(watcher)
|
||||
if not registered:
|
||||
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
|
||||
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
|
||||
@@ -547,6 +709,9 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 派发事件的孤儿线程,必须就地停掉
|
||||
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
|
||||
new_watcher.stop()
|
||||
if new_watcher.is_alive():
|
||||
with self._watcher_lock:
|
||||
self._retired_watchers.append(new_watcher)
|
||||
return
|
||||
# 新监控的重启计数从零开始,同步重置告警基准
|
||||
self._restart_marks.pop(str(watcher.watch_path), None)
|
||||
@@ -557,7 +722,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
self.__start_compensation(mon_path=watcher.watch_path,
|
||||
since=watcher.last_activity_time)
|
||||
|
||||
def __start_compensation(self, mon_path: Path, since: float):
|
||||
def __start_compensation(self, mon_path: Path, since: float) -> None:
|
||||
"""
|
||||
在后台线程发起补偿扫描,避免遍历目录阻塞健康检查周期。
|
||||
:param mon_path: 监控目录
|
||||
@@ -567,14 +732,34 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
|
||||
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
|
||||
return
|
||||
Thread(
|
||||
target=self.__compensate_scan,
|
||||
thread = Thread(
|
||||
target=self.__run_compensation,
|
||||
kwargs={"mon_path": mon_path, "since": since},
|
||||
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
|
||||
daemon=True
|
||||
).start()
|
||||
)
|
||||
with self._owner_lock:
|
||||
# 与 close() 的 owner 快照共用同一把锁;封口先发生时绝不再启动,
|
||||
# 登记先发生时 close() 必然能看到并等待这条线程。
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
self._compensation_threads[id(thread)] = thread
|
||||
try:
|
||||
thread.start()
|
||||
except Exception:
|
||||
self._compensation_threads.pop(id(thread), None)
|
||||
raise
|
||||
|
||||
def __compensate_scan(self, mon_path: Path, since: float):
|
||||
def __run_compensation(self, mon_path: Path, since: float) -> None:
|
||||
"""执行补偿扫描,并在真实终态后释放 owner 句柄。"""
|
||||
current_thread = threading.current_thread()
|
||||
try:
|
||||
self.__compensate_scan(mon_path=mon_path, since=since)
|
||||
finally:
|
||||
with self._owner_lock:
|
||||
self._compensation_threads.pop(id(current_thread), None)
|
||||
|
||||
def __compensate_scan(self, mon_path: Path, since: float) -> None:
|
||||
"""
|
||||
补扫监控停摆期间落地的文件。
|
||||
|
||||
@@ -586,7 +771,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param mon_path: 监控目录
|
||||
:param since: 停摆起点(墙钟时间戳),仅用于统计与日志
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
candidates = self.__collect_compensation_files(mon_path)
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if candidates is None:
|
||||
return
|
||||
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
|
||||
@@ -599,6 +788,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
|
||||
handled = 0
|
||||
for file_path, file_modify_time, file_size in candidates:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if self._dispatcher.handle_file(
|
||||
storage="local",
|
||||
event_path=file_path,
|
||||
@@ -634,14 +825,18 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
return None
|
||||
return candidates
|
||||
|
||||
def __retry_pending_locals(self):
|
||||
def __retry_pending_locals(self) -> None:
|
||||
"""
|
||||
重试启动失败的本地目录监控,给网络存储/FUSE 挂载留出就绪时间。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
with self._watcher_lock:
|
||||
pending = list(self._pending_locals)
|
||||
isolated = set(self._isolated)
|
||||
for item in pending:
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
if str(item["mon_path"]) in isolated:
|
||||
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
|
||||
# 在「请求永不返回」的挂载上会再冻死一个线程
|
||||
@@ -655,7 +850,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
item["attempts"] = item.get("attempts", 0) + 1
|
||||
item["skip_cycles"] = min(item["attempts"], 10)
|
||||
|
||||
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
|
||||
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault") -> None:
|
||||
"""
|
||||
推送目录监控异常告警,同一目录在同一阶段仅推送一次。
|
||||
:param mon_path: 监控目录
|
||||
@@ -664,6 +859,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
(监控已暂停访问、等待挂载恢复),只按目录去重会把这条
|
||||
关键消息吞掉,因此阶段变化时重新推送
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if self._alerted_paths.get(key) == stage:
|
||||
@@ -672,7 +869,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
@staticmethod
|
||||
def __poller_alert(storage: str, message: str):
|
||||
def __poller_alert(storage: str, message: str) -> None:
|
||||
"""
|
||||
远程轮询监控告警回调,复用消息渠道推送。
|
||||
:param storage: 存储名称
|
||||
@@ -681,12 +878,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.warn(f"[{storage}] {message}")
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def __clear_alert(self, mon_path: Path, message: str):
|
||||
def __clear_alert(self, mon_path: Path, message: str) -> None:
|
||||
"""
|
||||
清除目录监控异常告警状态,并在此前告警过时推送恢复消息。
|
||||
:param mon_path: 监控目录
|
||||
:param message: 恢复内容
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
key = str(mon_path)
|
||||
with self._watcher_lock:
|
||||
if key not in self._alerted_paths:
|
||||
@@ -695,12 +894,14 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
logger.info(message)
|
||||
MessageHelper().put(message, title="目录监控")
|
||||
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]):
|
||||
def polling_observer(self, storage: str, mon_paths: List[Path]) -> None:
|
||||
"""
|
||||
轮询监控:执行一轮快照并按结果动态调整监控间隔。
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
file_count = self._poller.poll(storage=storage, mon_paths=mon_paths)
|
||||
if file_count is None or not self._scheduler:
|
||||
if not self.__accepting_work() or file_count is None or not self._scheduler:
|
||||
return
|
||||
# 动态调整监控间隔
|
||||
new_interval = SnapshotStore.adjust_interval(file_count)
|
||||
@@ -716,7 +917,10 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
except Exception as e:
|
||||
logger.error(f"调整监控间隔失败: {storage} - {e}")
|
||||
|
||||
def event_handler(self, event, text: str, event_path: str, file_size: float = None):
|
||||
def event_handler(
|
||||
self, event: Any, text: str, event_path: str,
|
||||
file_size: Optional[float] = None
|
||||
) -> None:
|
||||
"""
|
||||
处理文件变化。
|
||||
:param event: 事件
|
||||
@@ -724,7 +928,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
:param event_path: 事件文件路径
|
||||
:param file_size: 文件大小
|
||||
"""
|
||||
if event.is_directory:
|
||||
if not self.__accepting_work() or event.is_directory:
|
||||
return
|
||||
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
||||
return
|
||||
@@ -733,6 +937,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
file_modify_time = Path(event_path).stat().st_mtime
|
||||
except OSError as err:
|
||||
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
# 整理文件
|
||||
handle_kwargs = {
|
||||
"storage": "local",
|
||||
@@ -743,55 +949,191 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
handle_kwargs["file_modify_time"] = file_modify_time
|
||||
self._dispatcher.handle_file(**handle_kwargs)
|
||||
|
||||
def event_unreadable(self, event_path: Path):
|
||||
def event_unreadable(self, event_path: Path) -> None:
|
||||
"""
|
||||
处理读取失败的监控事件,登记待重试。
|
||||
:param event_path: 事件文件路径
|
||||
"""
|
||||
if not self.__accepting_work():
|
||||
return
|
||||
event_path = Path(event_path)
|
||||
if not self._dispatcher.is_transfer_candidate_path(event_path):
|
||||
return
|
||||
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
|
||||
|
||||
def stop(self):
|
||||
def stop(self, timeout: float = RELOAD_STOP_TIMEOUT) -> bool:
|
||||
"""在共享预算内临时停止全部监控 owner,供配置热重载使用。"""
|
||||
return self.__stop_with_budget(timeout=timeout, close=False)
|
||||
|
||||
def close(self, timeout: float = LIFECYCLE_CLOSE_TIMEOUT) -> bool:
|
||||
"""永久封口当前 lifespan,并在共享预算内等待全部监控 owner。"""
|
||||
return self.__stop_with_budget(timeout=timeout, close=True)
|
||||
|
||||
def __stop_with_budget(self, timeout: float, close: bool) -> bool:
|
||||
"""从等待生命周期锁开始计算一次停止操作的完整预算。"""
|
||||
timeout = max(0.0, timeout)
|
||||
deadline = time.monotonic() + timeout
|
||||
# 先封住工作入口再等待锁。配置重载若正阻塞在 FUSE 上,解冻后也只能清理,
|
||||
# 不能在外层生命周期已经超时返回后继续创建 watcher 或派发整理任务。
|
||||
self._work_stop_event.set()
|
||||
if close:
|
||||
self._shutdown_event.set()
|
||||
if not self._lifecycle_lock.acquire(
|
||||
timeout=max(0.0, deadline - time.monotonic())
|
||||
):
|
||||
logger.error("目录监控未在 %.1f 秒内取得停机所有权", timeout)
|
||||
return False
|
||||
try:
|
||||
return self.__stop_owned(deadline=deadline, close=close)
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
def __stop_owned(self, deadline: float, close: bool) -> bool:
|
||||
"""
|
||||
退出监控
|
||||
在已取得生命周期锁时等待全部 owner,共享调用方给出的绝对截止时间。
|
||||
|
||||
超时只报告未收敛并保留所有活句柄;只有 watcher、补偿扫描、恢复动作及
|
||||
scheduler shutdown 都到达终态后,才清空这一代的内存状态。
|
||||
"""
|
||||
# 先停定时服务,避免健康检查在停止过程中重建监控线程
|
||||
if self._scheduler:
|
||||
self._scheduler.remove_all_jobs()
|
||||
if self._scheduler.running:
|
||||
try:
|
||||
self._scheduler.shutdown()
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as e:
|
||||
logger.error(f"停止定时服务出现了错误:{e}")
|
||||
self._scheduler = None
|
||||
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
|
||||
self._work_stop_event.set()
|
||||
if close:
|
||||
self._closed = True
|
||||
self._shutdown_event.set()
|
||||
self._recovery.request_stop()
|
||||
self.__request_scheduler_stop()
|
||||
|
||||
with self._watcher_lock:
|
||||
watchers = tuple(self._watchers) + tuple(self._retired_watchers)
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
except Exception as err:
|
||||
logger.error(f"请求停止目录监控服务出现了错误:{err}")
|
||||
|
||||
recovery_converged = self._recovery.close(deadline=deadline)
|
||||
current_thread = threading.current_thread()
|
||||
# 恢复线程可能在封口后才从 FUSE 调用返回,并登记一个已经请求停止的新
|
||||
# watcher;RecoveryExecutor 收敛后重新取快照,确保它也使用剩余预算 join。
|
||||
with self._watcher_lock:
|
||||
registered_watchers = tuple(self._watchers) + tuple(self._retired_watchers)
|
||||
for watcher in registered_watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
except Exception as err:
|
||||
logger.error(f"等待目录监控服务停止出现了错误:{err}")
|
||||
|
||||
with self._owner_lock:
|
||||
compensation_threads = tuple(self._compensation_threads.values())
|
||||
scheduler_thread = self._scheduler_shutdown_thread
|
||||
for thread in compensation_threads:
|
||||
if thread is not current_thread and thread.is_alive():
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if (
|
||||
scheduler_thread is not None
|
||||
and scheduler_thread is not current_thread
|
||||
and scheduler_thread.is_alive()
|
||||
):
|
||||
scheduler_thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
|
||||
with self._watcher_lock:
|
||||
alive_watchers = tuple(
|
||||
watcher
|
||||
for watcher in (*self._watchers, *self._retired_watchers)
|
||||
if watcher.is_alive()
|
||||
)
|
||||
with self._owner_lock:
|
||||
alive_compensations = tuple(
|
||||
thread for thread in self._compensation_threads.values()
|
||||
if thread.is_alive()
|
||||
)
|
||||
scheduler_thread = self._scheduler_shutdown_thread
|
||||
scheduler_converged = (
|
||||
self._scheduler is None
|
||||
or (
|
||||
scheduler_thread is not None
|
||||
and not scheduler_thread.is_alive()
|
||||
and self._scheduler_shutdown_succeeded
|
||||
)
|
||||
)
|
||||
alive_recoveries = self._recovery.running_threads()
|
||||
if (
|
||||
alive_watchers
|
||||
or alive_compensations
|
||||
or alive_recoveries
|
||||
or not recovery_converged
|
||||
or not scheduler_converged
|
||||
):
|
||||
logger.error(
|
||||
"目录监控 owner 未在截止时间内收敛:watcher=%d,补偿=%d,恢复=%d,scheduler=%s",
|
||||
len(alive_watchers),
|
||||
len(alive_compensations),
|
||||
len(alive_recoveries),
|
||||
"未收敛" if not scheduler_converged else "已收敛",
|
||||
)
|
||||
return False
|
||||
|
||||
# 仅在所有 owner 真实终止后清理业务状态,确保失败重试仍能找到原句柄。
|
||||
self._dispatcher.clear_pending()
|
||||
with self._watcher_lock:
|
||||
watchers = self._watchers
|
||||
self._watchers = []
|
||||
self._retired_watchers = []
|
||||
self._pending_locals = []
|
||||
self._alerted_paths = {}
|
||||
self._restart_marks = {}
|
||||
self._stable_cycles = {}
|
||||
self._isolated = {}
|
||||
self._pending_rebuild = {}
|
||||
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
|
||||
# 被残留记录误判为 BUSY 而永远拿不到重建机会
|
||||
self._recovery.clear()
|
||||
with self._owner_lock:
|
||||
self._compensation_threads.clear()
|
||||
self._scheduler = None
|
||||
self._scheduler_shutdown_thread = None
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
if watchers:
|
||||
logger.info("正在停止本地目录监控服务...")
|
||||
for watcher in watchers:
|
||||
try:
|
||||
watcher.stop()
|
||||
watcher.join(timeout=5)
|
||||
if watcher.is_alive():
|
||||
logger.warning(f"本地目录监控线程在5秒内未能停止: {watcher.watch_path}")
|
||||
else:
|
||||
logger.debug(f"已停止本地目录监控服务: {watcher.watch_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"停止目录监控服务出现了错误:{e}")
|
||||
logger.info("本地目录监控服务已停止")
|
||||
return True
|
||||
|
||||
def __request_scheduler_stop(self) -> None:
|
||||
"""创建并登记唯一 scheduler shutdown 线程,避免阻塞生命周期调用线程。"""
|
||||
with self._owner_lock:
|
||||
running = self._scheduler_shutdown_thread
|
||||
if running is not None and running.is_alive():
|
||||
return
|
||||
if running is not None and self._scheduler_shutdown_succeeded:
|
||||
return
|
||||
scheduler = self._scheduler
|
||||
if scheduler is None:
|
||||
return
|
||||
self._scheduler_shutdown_succeeded = False
|
||||
thread = threading.Thread(
|
||||
target=self.__shutdown_scheduler,
|
||||
args=(scheduler,),
|
||||
name="MoviePilot-MonitorSchedulerShutdown",
|
||||
daemon=True,
|
||||
)
|
||||
self._scheduler_shutdown_thread = thread
|
||||
try:
|
||||
thread.start()
|
||||
except Exception as err:
|
||||
self._scheduler_shutdown_thread = None
|
||||
logger.error(f"启动定时监控停止线程失败:{err}")
|
||||
|
||||
def __shutdown_scheduler(self, scheduler: BackgroundScheduler) -> None:
|
||||
"""停止 scheduler 及其在途 job,并把真实终态写回 owner 注册表。"""
|
||||
succeeded = False
|
||||
try:
|
||||
scheduler.remove_all_jobs()
|
||||
if scheduler.running:
|
||||
scheduler.shutdown(wait=True)
|
||||
succeeded = True
|
||||
logger.info("定时监控服务已停止")
|
||||
except Exception as err:
|
||||
logger.error(f"停止定时服务出现了错误:{err}")
|
||||
finally:
|
||||
with self._owner_lock:
|
||||
if self._scheduler is scheduler:
|
||||
self._scheduler_shutdown_succeeded = succeeded
|
||||
# 缓存与快照存储是共享后端的代理,生命周期由应用全局管理,这里不再关闭
|
||||
|
||||
+70
-15
@@ -1,5 +1,5 @@
|
||||
"""
|
||||
监控恢复动作的可放弃执行单元。
|
||||
监控恢复动作的有限等待执行单元。
|
||||
|
||||
FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
|
||||
|
||||
@@ -11,8 +11,8 @@ FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
|
||||
本模块提供 block 型故障下唯一可行的两种自保手段:
|
||||
|
||||
1. RecoveryExecutor —— 把会触碰挂载的动作放进一次性守护线程执行,调用方只
|
||||
等待有限时间。超时即放弃该线程(它会作为守护线程悬挂到进程退出),换取
|
||||
调用方(健康检查这个全局自愈单点)永远活着。
|
||||
等待有限时间。超时后当前健康检查不再等待,但线程句柄仍由生命周期 owner
|
||||
持有到真实终态,换取看门狗可继续检测且停机屏障不会伪装收敛。
|
||||
2. probe_path —— 用子进程而非线程做挂载探测。子进程可以被 kill,因此探测
|
||||
本身是可放弃的,隔离期间可以无限次周期重试而不累积不可回收的资源。
|
||||
"""
|
||||
@@ -41,7 +41,7 @@ class RecoveryState(str, Enum):
|
||||
"""
|
||||
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
|
||||
COMPLETED = "completed"
|
||||
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
|
||||
# 超时仍未返回,判定为 block 型挂载故障,本轮不再等待但继续持有线程
|
||||
TIMEOUT = "timeout"
|
||||
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
|
||||
BUSY = "busy"
|
||||
@@ -55,10 +55,12 @@ class RecoveryExecutor:
|
||||
健康检查周期都会在同一个死挂载上多泄漏一个线程。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self) -> None:
|
||||
"""初始化在途线程注册表和当前生命周期的提交状态。"""
|
||||
# key -> 该 key 最近一次提交的执行线程
|
||||
self._running: Dict[str, threading.Thread] = {}
|
||||
self._lock = threading.Lock()
|
||||
self._accepting = True
|
||||
|
||||
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
|
||||
"""
|
||||
@@ -85,26 +87,70 @@ class RecoveryExecutor:
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if thread.is_alive():
|
||||
results[key] = RecoveryState.TIMEOUT
|
||||
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
|
||||
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应,本轮不再等待: {key}")
|
||||
else:
|
||||
results[key] = RecoveryState.COMPLETED
|
||||
return results
|
||||
|
||||
def discard(self, key: str):
|
||||
def discard(self, key: str) -> None:
|
||||
"""
|
||||
丢弃一个 key 的在途记录。监控停止或配置重载时调用,避免残留条目
|
||||
让重建后的同名目录被误判为 BUSY。
|
||||
清理一个已到终态的 key;活线程继续保留,供生命周期停机屏障追踪。
|
||||
:param key: 动作标识
|
||||
"""
|
||||
with self._lock:
|
||||
self._running.pop(key, None)
|
||||
thread = self._running.get(key)
|
||||
if thread is None or not thread.is_alive():
|
||||
self._running.pop(key, None)
|
||||
|
||||
def clear(self):
|
||||
def clear(self) -> None:
|
||||
"""
|
||||
清空全部在途记录。已经冻死的线程无法回收,这里只是不再跟踪它们。
|
||||
清理已经完成的记录,仍存活的线程继续由执行器持有。
|
||||
|
||||
兼容旧调用名,但不再丢弃挂死线程:遗失句柄会让宿主错误释放它仍可能使用
|
||||
的数据库和整理链资源。
|
||||
"""
|
||||
with self._lock:
|
||||
self._running = {
|
||||
key: thread
|
||||
for key, thread in self._running.items()
|
||||
if thread.is_alive()
|
||||
}
|
||||
|
||||
def request_stop(self) -> None:
|
||||
"""封住新恢复动作提交,既有线程只能自然完成。"""
|
||||
with self._lock:
|
||||
self._accepting = False
|
||||
|
||||
def close(self, deadline: float) -> bool:
|
||||
"""在绝对截止时间内等待全部恢复线程,超时继续保留活线程句柄。"""
|
||||
self.request_stop()
|
||||
with self._lock:
|
||||
threads = tuple(self._running.values())
|
||||
current_thread = threading.current_thread()
|
||||
for thread in threads:
|
||||
if thread is current_thread or not thread.is_alive():
|
||||
continue
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
self.clear()
|
||||
with self._lock:
|
||||
return not any(thread.is_alive() for thread in self._running.values())
|
||||
|
||||
def reopen(self) -> bool:
|
||||
"""在旧恢复线程全部结束后,为新的 Monitor 生命周期恢复提交。"""
|
||||
self.clear()
|
||||
with self._lock:
|
||||
if any(thread.is_alive() for thread in self._running.values()):
|
||||
return False
|
||||
self._running.clear()
|
||||
self._accepting = True
|
||||
return True
|
||||
|
||||
def running_threads(self) -> tuple[threading.Thread, ...]:
|
||||
"""返回仍存活且由执行器持有的恢复线程快照。"""
|
||||
with self._lock:
|
||||
return tuple(
|
||||
thread for thread in self._running.values() if thread.is_alive()
|
||||
)
|
||||
|
||||
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
|
||||
"""
|
||||
@@ -114,6 +160,8 @@ class RecoveryExecutor:
|
||||
:return: 执行线程,未启动时为 None
|
||||
"""
|
||||
with self._lock:
|
||||
if not self._accepting:
|
||||
return None
|
||||
running = self._running.get(key)
|
||||
if running is not None and running.is_alive():
|
||||
return None
|
||||
@@ -124,11 +172,18 @@ class RecoveryExecutor:
|
||||
daemon=True
|
||||
)
|
||||
self._running[key] = thread
|
||||
thread.start()
|
||||
return thread
|
||||
# 登记与 start 必须处于同一所有权临界区。否则 close() 可能把尚未
|
||||
# is_alive() 的句柄当成已结束并移除,随后该线程才真正启动。
|
||||
try:
|
||||
thread.start()
|
||||
except BaseException:
|
||||
if self._running.get(key) is thread:
|
||||
self._running.pop(key, None)
|
||||
raise
|
||||
return thread
|
||||
|
||||
@staticmethod
|
||||
def _execute(key: str, action: Callable[[], None]):
|
||||
def _execute(key: str, action: Callable[[], None]) -> None:
|
||||
"""
|
||||
执行一个恢复动作,异常只记录不外抛,避免一个目录的失败连累整批恢复。
|
||||
:param key: 动作标识
|
||||
|
||||
@@ -30,6 +30,9 @@ class EventDispatcher:
|
||||
event_factory: Callable[..., Any],
|
||||
error_handler: Callable[..., None],
|
||||
async_handle_sink: Callable[[Any], bool] | None = None,
|
||||
sync_handle_sink: (
|
||||
Callable[[Callable[..., Any], tuple[Any, ...]], bool] | None
|
||||
) = None,
|
||||
) -> None:
|
||||
"""注入注册表、绑定器、执行器和错误策略回调。"""
|
||||
self._registry = registry
|
||||
@@ -39,6 +42,7 @@ class EventDispatcher:
|
||||
self._event_factory = event_factory
|
||||
self._error_handler = error_handler
|
||||
self._async_handle_sink = async_handle_sink
|
||||
self._sync_handle_sink = sync_handle_sink
|
||||
|
||||
def dispatch_chain(self, event: Any) -> bool:
|
||||
"""同步按优先级顺序执行链式事件快照。"""
|
||||
@@ -134,6 +138,12 @@ class EventDispatcher:
|
||||
event.event_type,
|
||||
)
|
||||
else:
|
||||
if self._sync_handle_sink:
|
||||
self._sync_handle_sink(
|
||||
self.safe_invoke_sync,
|
||||
(handler, isolated),
|
||||
)
|
||||
continue
|
||||
self._executor().submit(
|
||||
self.safe_invoke_sync,
|
||||
handler,
|
||||
|
||||
+180
-26
@@ -4,6 +4,7 @@ import random
|
||||
import threading
|
||||
import traceback
|
||||
import uuid
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
from queue import Empty, PriorityQueue
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union, Any, Type
|
||||
@@ -32,6 +33,10 @@ MIN_EVENT_CONSUMER_THREADS = 1 # 最小事件消费者线程数
|
||||
INITIAL_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 1 # 事件队列空闲时的初始超时时间(秒)
|
||||
MAX_EVENT_QUEUE_IDLE_TIMEOUT_SECONDS = 5 # 事件队列空闲时的最大超时时间(秒)
|
||||
_EVENT_STOP_SENTINEL = object()
|
||||
_CURRENT_EVENT_HANDLER_OWNER: ContextVar[object | None] = ContextVar(
|
||||
"current_event_handler_owner",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -121,10 +126,11 @@ class EventManager(metaclass=Singleton):
|
||||
self.__lock = threading.Lock()
|
||||
# 退出事件
|
||||
self.__event = threading.Event()
|
||||
# 广播异步处理器的生命周期由事件总线自己持有,避免关闭后仍向主循环运行。
|
||||
# 广播处理器由事件总线统一持有,确保插件卸载前可以建立结算屏障。
|
||||
self.__lifecycle_lock = threading.RLock()
|
||||
self.__lifecycle_state = "new"
|
||||
self.__async_handles: Dict[int, _EventAsyncHandle] = {}
|
||||
self.__sync_handles: Dict[object, concurrent.futures.Future[Any]] = {}
|
||||
self.__async_handles: Dict[object, _EventAsyncHandle] = {}
|
||||
# 由上层管理器注册的处理器实例解析器
|
||||
self.__handler_instance_resolvers: Dict[str, HandlerInstanceResolver] = {}
|
||||
# 由启动组合层注入的错误通知回调
|
||||
@@ -155,6 +161,7 @@ class EventManager(metaclass=Singleton):
|
||||
event_factory=Event,
|
||||
error_handler=lambda **kwargs: self.__handle_event_error(**kwargs),
|
||||
async_handle_sink=self.__register_async_handle,
|
||||
sync_handle_sink=self.__register_sync_handle,
|
||||
)
|
||||
|
||||
def register_handler_instance_resolver(
|
||||
@@ -188,6 +195,9 @@ class EventManager(metaclass=Singleton):
|
||||
if self.__lifecycle_state == "stopping":
|
||||
logger.warning("事件处理仍在停止,忽略重复启动")
|
||||
return
|
||||
if self.__lifecycle_state == "sealed":
|
||||
logger.warning("事件处理已封口,忽略重复启动")
|
||||
return
|
||||
self.__lifecycle_state = "running"
|
||||
self.__event.set()
|
||||
self.__consumer_threads = []
|
||||
@@ -199,14 +209,19 @@ class EventManager(metaclass=Singleton):
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
停止广播事件处理线程
|
||||
兼容同步关闭入口,等待同步处理器并请求取消异步处理器。
|
||||
|
||||
调用线程无法安全等待主事件循环完成异步清理;需要完整异步收口时应使用
|
||||
stop_async()。
|
||||
"""
|
||||
logger.info("正在停止事件处理...")
|
||||
consumer_threads = self.__begin_stop()
|
||||
try:
|
||||
self.__join_consumer_threads(consumer_threads)
|
||||
self.__discard_stop_sentinels()
|
||||
self.__cancel_async_handles()
|
||||
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
|
||||
self.__cancel_async_handles(exclude_owner=current_owner)
|
||||
self.__wait_sync_handles(exclude_owner=current_owner)
|
||||
logger.info("事件处理停止完成")
|
||||
except Exception as e:
|
||||
logger.error(f"停止事件处理线程出错:{str(e)} - {traceback.format_exc()}")
|
||||
@@ -216,7 +231,7 @@ class EventManager(metaclass=Singleton):
|
||||
self.__consumer_threads = []
|
||||
|
||||
async def stop_async(self) -> None:
|
||||
"""停止广播消费者并等待已投递的异步处理器收口。"""
|
||||
"""停止广播消费者,等待同步处理器并取消收口异步处理器。"""
|
||||
logger.info("正在停止事件处理...")
|
||||
consumer_threads = self.__begin_stop()
|
||||
try:
|
||||
@@ -226,15 +241,29 @@ class EventManager(metaclass=Singleton):
|
||||
consumer_threads,
|
||||
)
|
||||
self.__discard_stop_sentinels()
|
||||
current_owner = _CURRENT_EVENT_HANDLER_OWNER.get()
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(self.__async_handles.values())
|
||||
for handle in handles:
|
||||
async_handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__async_handles.items()
|
||||
if owner is not current_owner
|
||||
)
|
||||
sync_handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__sync_handles.items()
|
||||
if owner is not current_owner
|
||||
)
|
||||
for handle in async_handles:
|
||||
handle.handle.cancel()
|
||||
if handles:
|
||||
if async_handles or sync_handles:
|
||||
await asyncio.gather(
|
||||
*(
|
||||
asyncio.shield(asyncio.wrap_future(handle.completion))
|
||||
for handle in handles
|
||||
for handle in async_handles
|
||||
),
|
||||
*(
|
||||
asyncio.shield(asyncio.wrap_future(handle))
|
||||
for handle in sync_handles
|
||||
),
|
||||
return_exceptions=True,
|
||||
)
|
||||
@@ -272,16 +301,81 @@ class EventManager(metaclass=Singleton):
|
||||
break
|
||||
if item[1] is not _EVENT_STOP_SENTINEL:
|
||||
pending.append(item)
|
||||
self.__event_queue.task_done()
|
||||
for item in pending:
|
||||
self.__event_queue.put(item)
|
||||
|
||||
def __cancel_async_handles(self) -> None:
|
||||
"""请求取消所有仍由事件总线持有的异步处理器。"""
|
||||
def __cancel_async_handles(
|
||||
self,
|
||||
*,
|
||||
exclude_owner: object | None = None,
|
||||
) -> None:
|
||||
"""请求取消异步处理器,并避免 handler 关闭事件总线时取消自身。"""
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(self.__async_handles.values())
|
||||
handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__async_handles.items()
|
||||
if owner is not exclude_owner
|
||||
)
|
||||
for handle in handles:
|
||||
handle.handle.cancel()
|
||||
|
||||
def __wait_sync_handles(
|
||||
self,
|
||||
*,
|
||||
exclude_owner: object | None = None,
|
||||
) -> None:
|
||||
"""等待同步处理器完成,并避免 handler 关闭事件总线时等待自身。"""
|
||||
with self.__lifecycle_lock:
|
||||
handles = tuple(
|
||||
handle
|
||||
for owner, handle in self.__sync_handles.items()
|
||||
if owner is not exclude_owner
|
||||
)
|
||||
if handles:
|
||||
concurrent.futures.wait(handles)
|
||||
|
||||
def __register_sync_handle(
|
||||
self,
|
||||
callback: Callable[..., Any],
|
||||
args: tuple[Any, ...],
|
||||
) -> bool:
|
||||
"""在同一生命周期临界区提交并登记同步广播处理器。"""
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state != "running":
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝同步广播处理器",
|
||||
self.__lifecycle_state,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
owner = object()
|
||||
|
||||
def _tracked_sync() -> Any:
|
||||
"""在同步 handler 调用栈中发布当前事件 owner。"""
|
||||
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
|
||||
try:
|
||||
return callback(*args)
|
||||
finally:
|
||||
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
|
||||
|
||||
handle = self.__executor.submit(_tracked_sync)
|
||||
except RuntimeError:
|
||||
logger.warning("同步事件处理器无法投递,线程池已停止")
|
||||
return False
|
||||
self.__sync_handles[owner] = handle
|
||||
handle.add_done_callback(
|
||||
lambda _completed, current_owner=owner: (
|
||||
self.__remove_sync_handle(current_owner)
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def __remove_sync_handle(self, owner: object) -> None:
|
||||
"""同步处理器完成后移除其 owner 句柄。"""
|
||||
with self.__lifecycle_lock:
|
||||
self.__sync_handles.pop(owner, None)
|
||||
|
||||
def __register_async_handle(
|
||||
self,
|
||||
coroutine: Any,
|
||||
@@ -290,12 +384,18 @@ class EventManager(metaclass=Singleton):
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state != "running":
|
||||
coroutine.close()
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝异步广播处理器",
|
||||
self.__lifecycle_state,
|
||||
)
|
||||
return False
|
||||
completion: concurrent.futures.Future[Any] = concurrent.futures.Future()
|
||||
started = threading.Event()
|
||||
owner = object()
|
||||
|
||||
async def _tracked() -> None:
|
||||
started.set()
|
||||
context_token = _CURRENT_EVENT_HANDLER_OWNER.set(owner)
|
||||
try:
|
||||
result = await coroutine
|
||||
except asyncio.CancelledError:
|
||||
@@ -307,6 +407,8 @@ class EventManager(metaclass=Singleton):
|
||||
else:
|
||||
if not completion.done():
|
||||
completion.set_result(result)
|
||||
finally:
|
||||
_CURRENT_EVENT_HANDLER_OWNER.reset(context_token)
|
||||
|
||||
tracked = _tracked()
|
||||
try:
|
||||
@@ -316,7 +418,7 @@ class EventManager(metaclass=Singleton):
|
||||
coroutine.close()
|
||||
logger.warning("异步事件处理器无法投递,事件循环已停止")
|
||||
return False
|
||||
self.__async_handles[id(completion)] = _EventAsyncHandle(
|
||||
self.__async_handles[owner] = _EventAsyncHandle(
|
||||
handle=handle,
|
||||
completion=completion,
|
||||
)
|
||||
@@ -329,13 +431,54 @@ class EventManager(metaclass=Singleton):
|
||||
completion.cancel()
|
||||
|
||||
handle.add_done_callback(_complete_unstarted_submission)
|
||||
completion.add_done_callback(self.__remove_async_handle)
|
||||
completion.add_done_callback(
|
||||
lambda _completed, current_owner=owner: (
|
||||
self.__remove_async_handle(current_owner)
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def __remove_async_handle(self, handle: concurrent.futures.Future[Any]) -> None:
|
||||
def __remove_async_handle(self, owner: object) -> None:
|
||||
"""异步处理器完成后移除其 owner 句柄。"""
|
||||
with self.__lifecycle_lock:
|
||||
self.__async_handles.pop(id(handle), None)
|
||||
self.__async_handles.pop(owner, None)
|
||||
|
||||
async def drain_async(
|
||||
self,
|
||||
timeout: Optional[float] = None,
|
||||
*,
|
||||
seal: bool = False,
|
||||
) -> bool:
|
||||
"""
|
||||
等待已接纳的广播事件及其派生事件自然结算。
|
||||
|
||||
seal=True 会在确认稳定空闲的同一临界区关闭后续广播提交,供插件卸载前
|
||||
建立不可穿透的投递屏障;超时或停止交错时保持原提交状态并返回 False。
|
||||
广播处理器调用栈内无法等待自身完成,因此会立即返回 False,且不执行封口。
|
||||
"""
|
||||
if _CURRENT_EVENT_HANDLER_OWNER.get() is not None:
|
||||
logger.warning("事件处理器内部不能建立事件投递屏障")
|
||||
return False
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = None if timeout is None else loop.time() + max(timeout, 0)
|
||||
while True:
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state not in {"running", "sealed"}:
|
||||
return False
|
||||
with self.__event_queue.all_tasks_done:
|
||||
queue_idle = self.__event_queue.unfinished_tasks == 0
|
||||
handlers_idle = not self.__sync_handles and not self.__async_handles
|
||||
if queue_idle and handlers_idle:
|
||||
if seal:
|
||||
self.__lifecycle_state = "sealed"
|
||||
return True
|
||||
if deadline is not None:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
return False
|
||||
await asyncio.sleep(min(0.01, remaining))
|
||||
else:
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
def check(self, etype: Union[EventType, ChainEventType]) -> bool:
|
||||
"""
|
||||
@@ -470,7 +613,15 @@ class EventManager(metaclass=Singleton):
|
||||
:param event: 要处理的事件对象
|
||||
"""
|
||||
logger.debug(f"Triggering broadcast event: {event}")
|
||||
self.__event_queue.put((event.priority, event))
|
||||
with self.__lifecycle_lock:
|
||||
if self.__lifecycle_state in {"sealed", "stopping", "stopped"}:
|
||||
logger.warning(
|
||||
"事件处理处于 %s 状态,拒绝广播事件 %s",
|
||||
self.__lifecycle_state,
|
||||
event.event_type,
|
||||
)
|
||||
return None
|
||||
self.__event_queue.put((event.priority, event))
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
@@ -580,15 +731,18 @@ class EventManager(metaclass=Singleton):
|
||||
while self.__event.is_set():
|
||||
try:
|
||||
priority, event = self.__event_queue.get(timeout=rate_limiter.current_wait)
|
||||
if event is _EVENT_STOP_SENTINEL:
|
||||
break
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
delivery="broadcast",
|
||||
)
|
||||
rate_limiter.reset()
|
||||
self.__dispatch_broadcast_event(event)
|
||||
try:
|
||||
if event is _EVENT_STOP_SENTINEL:
|
||||
break
|
||||
record_metric(
|
||||
"event.queue.depth",
|
||||
self.__event_queue.qsize(),
|
||||
delivery="broadcast",
|
||||
)
|
||||
rate_limiter.reset()
|
||||
self.__dispatch_broadcast_event(event)
|
||||
finally:
|
||||
self.__event_queue.task_done()
|
||||
except Empty:
|
||||
rate_limiter.current_wait = rate_limiter.current_wait * random.uniform(1, 1 + jitter_factor)
|
||||
rate_limiter.trigger_limit()
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""插件宿主可变事务的停机准入。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _MutationContext:
|
||||
"""记录一个可跨协程和受控线程传播的事务上下文。"""
|
||||
|
||||
admission: "PluginMutationAdmission"
|
||||
holders: int = 0
|
||||
open: bool = True
|
||||
|
||||
|
||||
class PluginMutationAdmission:
|
||||
"""在停机封口与插件可变事务之间维护真实 owner 计数。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化开放准入、活动 owner 计数和事务上下文。"""
|
||||
self._condition = threading.Condition()
|
||||
self._accepting = True
|
||||
self._active_count = 0
|
||||
self._current_context: ContextVar[_MutationContext | None] = ContextVar(
|
||||
"plugin_mutation_context",
|
||||
default=None,
|
||||
)
|
||||
|
||||
@property
|
||||
def active_count(self) -> int:
|
||||
"""返回尚未退出的 lease 数量,用于停机诊断和严格卸载判断。"""
|
||||
with self._condition:
|
||||
return self._active_count
|
||||
|
||||
@property
|
||||
def accepting(self) -> bool:
|
||||
"""返回当前生命周期是否仍接纳新的根事务。"""
|
||||
with self._condition:
|
||||
return self._accepting
|
||||
|
||||
def is_held(self) -> bool:
|
||||
"""判断当前执行上下文是否持有仍有效的本 admission lease。"""
|
||||
context = self._current_context.get()
|
||||
return bool(
|
||||
context
|
||||
and context.admission is self
|
||||
and context.open
|
||||
and context.holders > 0
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def hold(self, operation: str) -> Iterator[None]:
|
||||
"""取得可变事务 lease;封口后仅允许已获准事务的嵌套调用。"""
|
||||
context = self._current_context.get()
|
||||
nested = bool(
|
||||
context
|
||||
and context.admission is self
|
||||
and context.open
|
||||
and context.holders > 0
|
||||
)
|
||||
context_token = None
|
||||
if not nested:
|
||||
context = _MutationContext(admission=self)
|
||||
context_token = self._current_context.set(context)
|
||||
assert context is not None
|
||||
|
||||
acquired = False
|
||||
try:
|
||||
with self._condition:
|
||||
if not self._accepting and not nested:
|
||||
raise PluginMutationRejectedError(operation)
|
||||
self._active_count += 1
|
||||
context.holders += 1
|
||||
acquired = True
|
||||
yield
|
||||
finally:
|
||||
if acquired:
|
||||
with self._condition:
|
||||
self._active_count -= 1
|
||||
context.holders -= 1
|
||||
if context.holders == 0:
|
||||
context.open = False
|
||||
self._condition.notify_all()
|
||||
if context_token is not None:
|
||||
self._current_context.reset(context_token)
|
||||
|
||||
def seal(self) -> int:
|
||||
"""原子停止接纳根事务,并返回封口瞬间的活动 lease 数量。"""
|
||||
with self._condition:
|
||||
self._accepting = False
|
||||
return self._active_count
|
||||
|
||||
def wait_until_idle(self) -> None:
|
||||
"""自然等待全部已获准 lease 退出,不取消或遗失其 owner。"""
|
||||
with self._condition:
|
||||
while self._active_count:
|
||||
self._condition.wait()
|
||||
|
||||
def reopen(self) -> bool:
|
||||
"""仅在没有遗留 lease 时为新的应用生命周期重新开放准入。"""
|
||||
with self._condition:
|
||||
if self._active_count:
|
||||
return False
|
||||
self._accepting = True
|
||||
return True
|
||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Optional, ParamSpec, TypeVar, cast
|
||||
|
||||
@@ -30,7 +31,7 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
|
||||
try:
|
||||
result = func(*args, **kwargs)
|
||||
statuses = result.values() if isinstance(result, dict) else (result,)
|
||||
if PluginRuntimeStatus.LOAD_FAILED in statuses:
|
||||
if result is False or PluginRuntimeStatus.LOAD_FAILED in statuses:
|
||||
outcome = "error"
|
||||
return result
|
||||
except BaseException:
|
||||
@@ -52,6 +53,8 @@ def observe_plugin_lifecycle(operation: str) -> Callable[[Callable[P, R]], Calla
|
||||
class PluginLifecycle:
|
||||
"""管理插件发现、初始化、启停和热重载,不持有市场或 HTTP 路由职责。"""
|
||||
|
||||
_EVENT_HANDLERS_QUIESCED = "__event_handlers__"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
@@ -83,6 +86,8 @@ class PluginLifecycle:
|
||||
self._runtime_status_writer = runtime_status_writer
|
||||
self._logger = log
|
||||
self._event_sender = event_sender
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._quiesced_hooks: dict[str, set[str]] = {}
|
||||
|
||||
@observe_plugin_lifecycle("start")
|
||||
def start(
|
||||
@@ -116,6 +121,7 @@ class PluginLifecycle:
|
||||
self._classes[current_id] = plugin
|
||||
instance = plugin()
|
||||
instance.init_plugin(self._plugin_config(current_id))
|
||||
self._quiesced_hooks.pop(current_id, None)
|
||||
self._running[current_id] = instance
|
||||
self._logger.info(
|
||||
f"加载插件:{current_id} 版本:{instance.plugin_version}"
|
||||
@@ -156,31 +162,168 @@ class PluginLifecycle:
|
||||
|
||||
@observe_plugin_lifecycle("stop")
|
||||
def stop(self, plugin_id: Optional[str] = None) -> None:
|
||||
"""停止指定插件或全部插件,并清理模块缓存。"""
|
||||
"""按旧单阶段 ABI 先解绑 handler,再停止并强制卸载插件。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
self._quiesce_selected(plugins)
|
||||
self._finalize(
|
||||
plugin_id,
|
||||
require_quiesced=False,
|
||||
disable_events=not self._handlers_quiesced(plugins),
|
||||
)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce")
|
||||
def quiesce(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""先解绑事件 handler,再按旧 hook 顺序停止生产者并保留实例。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
return self._quiesce_selected(plugins)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce_handlers")
|
||||
def quiesce_handlers(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""禁止目标插件接收新事件,保留实例供在途 handler 和后续 hook 使用。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
return self._disable_selected_handlers(plugins)
|
||||
|
||||
@observe_plugin_lifecycle("quiesce_services")
|
||||
def quiesce_services(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""在事件结算屏障后执行旧 close、stop_service hook。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
if not self._handlers_quiesced(plugins):
|
||||
self._logger.warning("插件事件 handler 尚未全部停用,拒绝关闭插件资源")
|
||||
return False
|
||||
return self._quiesce_hooks(plugins)
|
||||
|
||||
def _quiesce_selected(self, plugins: dict[str, Any]) -> bool:
|
||||
"""兼容单阶段调用:先停用全部 handler,再执行稳定快照的旧 hooks。"""
|
||||
if not self._disable_selected_handlers(plugins):
|
||||
return False
|
||||
return self._quiesce_hooks(plugins)
|
||||
|
||||
def _disable_selected_handlers(self, plugins: dict[str, Any]) -> bool:
|
||||
"""先停用稳定快照的全部事件入口,任一失败时不执行破坏性 hook。"""
|
||||
all_converged = True
|
||||
for current_id, plugin in plugins.items():
|
||||
completed = self._quiesced_hooks.setdefault(current_id, set())
|
||||
if self._EVENT_HANDLERS_QUIESCED in completed:
|
||||
continue
|
||||
try:
|
||||
self._disable_events(type(plugin))
|
||||
except Exception as error: # noqa: BLE001 插件边界必须隔离
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停用插件 {current_id} 的事件 handler 时发生错误: {error}"
|
||||
)
|
||||
continue
|
||||
completed.add(self._EVENT_HANDLERS_QUIESCED)
|
||||
return all_converged
|
||||
|
||||
def _quiesce_hooks(self, plugins: dict[str, Any]) -> bool:
|
||||
"""执行旧 ABI hooks,并只重试尚未成功的步骤。"""
|
||||
all_converged = True
|
||||
for current_id, plugin in plugins.items():
|
||||
completed = self._quiesced_hooks.setdefault(current_id, set())
|
||||
for hook_name in ("close", "stop_service"):
|
||||
if hook_name in completed:
|
||||
continue
|
||||
hook = getattr(plugin, hook_name, None)
|
||||
if not callable(hook):
|
||||
completed.add(hook_name)
|
||||
continue
|
||||
try:
|
||||
result = hook()
|
||||
except Exception as error: # noqa: BLE001 插件边界必须隔离
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停止插件 {current_id} 的 {hook_name} 时发生错误: {error}"
|
||||
)
|
||||
continue
|
||||
if result is False:
|
||||
all_converged = False
|
||||
self._logger.warning(
|
||||
f"停止插件 {current_id} 的 {hook_name} 未收敛"
|
||||
)
|
||||
continue
|
||||
completed.add(hook_name)
|
||||
return all_converged
|
||||
|
||||
@observe_plugin_lifecycle("finalize")
|
||||
def finalize(self, plugin_id: Optional[str] = None) -> bool:
|
||||
"""在 handler、旧 hook 和事件屏障均收敛后卸载插件实例。"""
|
||||
return self._finalize(
|
||||
plugin_id,
|
||||
require_quiesced=True,
|
||||
disable_events=False,
|
||||
)
|
||||
|
||||
def _finalize(
|
||||
self,
|
||||
plugin_id: Optional[str],
|
||||
*,
|
||||
require_quiesced: bool,
|
||||
disable_events: bool = True,
|
||||
) -> bool:
|
||||
"""按严格或兼容策略卸载插件,并在清理失败时保留实例所有权。"""
|
||||
with self._lifecycle_lock:
|
||||
plugins = self._select_running_plugins(plugin_id)
|
||||
if require_quiesced and any(
|
||||
not self._is_quiesced(current_id, plugin)
|
||||
for current_id, plugin in plugins.items()
|
||||
):
|
||||
self._logger.warning("插件后台服务尚未全部收敛,拒绝卸载运行实例")
|
||||
return False
|
||||
|
||||
try:
|
||||
if disable_events:
|
||||
for plugin in plugins.values():
|
||||
self._disable_events(type(plugin))
|
||||
self._clear_modules(plugin_id)
|
||||
self._clear_tools()
|
||||
except Exception as error: # noqa: BLE001 保留实例所有权供后续重试
|
||||
self._logger.warning(f"卸载插件运行实例时发生错误: {error}")
|
||||
return False
|
||||
|
||||
if plugin_id:
|
||||
self._classes.pop(plugin_id, None)
|
||||
self._running.pop(plugin_id, None)
|
||||
self._quiesced_hooks.pop(plugin_id, None)
|
||||
else:
|
||||
self._classes.clear()
|
||||
self._running.clear()
|
||||
self._quiesced_hooks.clear()
|
||||
self._logger.info("插件停止完成")
|
||||
return True
|
||||
|
||||
def _select_running_plugins(self, plugin_id: Optional[str]) -> dict[str, Any]:
|
||||
"""返回本阶段处理的稳定实例快照,并保持旧停机日志语义。"""
|
||||
if plugin_id:
|
||||
self._logger.info(f"正在停止插件 {plugin_id}...")
|
||||
plugin = self._running.get(plugin_id)
|
||||
plugins = {plugin_id: plugin} if plugin else {}
|
||||
if not plugin:
|
||||
self._logger.debug(f"插件 {plugin_id} 不存在或未加载")
|
||||
else:
|
||||
self._logger.info("正在停止所有插件...")
|
||||
plugins = dict(self._running)
|
||||
return plugins
|
||||
self._logger.info("正在停止所有插件...")
|
||||
return dict(self._running)
|
||||
|
||||
for current_id, plugin in plugins.items():
|
||||
self._disable_events(type(plugin))
|
||||
self._stop_plugin(plugin)
|
||||
def _is_quiesced(self, plugin_id: str, plugin: Any) -> bool:
|
||||
"""判断 handler 及当前实例声明的旧 ABI hooks 是否均已成功收敛。"""
|
||||
required = {self._EVENT_HANDLERS_QUIESCED} | {
|
||||
hook_name
|
||||
for hook_name in ("close", "stop_service")
|
||||
if callable(getattr(plugin, hook_name, None))
|
||||
}
|
||||
return required.issubset(self._quiesced_hooks.get(plugin_id, set()))
|
||||
|
||||
if plugin_id:
|
||||
self._classes.pop(plugin_id, None)
|
||||
self._running.pop(plugin_id, None)
|
||||
self._clear_modules(plugin_id)
|
||||
else:
|
||||
self._classes.clear()
|
||||
self._running.clear()
|
||||
self._clear_modules(None)
|
||||
self._clear_tools()
|
||||
self._logger.info("插件停止完成")
|
||||
def _handlers_quiesced(self, plugins: dict[str, Any]) -> bool:
|
||||
"""判断稳定快照中的全部插件是否已经停用事件入口。"""
|
||||
return all(
|
||||
self._EVENT_HANDLERS_QUIESCED
|
||||
in self._quiesced_hooks.get(plugin_id, set())
|
||||
for plugin_id in plugins
|
||||
)
|
||||
|
||||
@observe_plugin_lifecycle("reload")
|
||||
def reload(
|
||||
@@ -194,14 +337,3 @@ class PluginLifecycle:
|
||||
status = self.start(plugin_id)[plugin_id]
|
||||
self._event_sender(reload_event, data={"plugin_id": plugin_id})
|
||||
return status
|
||||
|
||||
def _stop_plugin(self, plugin: Any) -> None:
|
||||
"""按插件旧 ABI 顺序关闭资源和服务。"""
|
||||
try:
|
||||
if hasattr(plugin, "close"):
|
||||
plugin.close()
|
||||
if hasattr(plugin, "stop_service"):
|
||||
plugin.stop_service()
|
||||
except Exception as error: # noqa: BLE001
|
||||
name = plugin.get_name() if hasattr(plugin, "get_name") else type(plugin).__name__
|
||||
self._logger.warning(f"停止插件 {name} 时发生错误: {error}")
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
@@ -27,6 +27,8 @@ class PluginMonitorController:
|
||||
self._logger = log
|
||||
self._thread: Optional[threading.Thread] = None
|
||||
self._stop_event = threading.Event()
|
||||
self._lifecycle_lock = threading.RLock()
|
||||
self._closed = False
|
||||
|
||||
@property
|
||||
def stop_event(self) -> threading.Event:
|
||||
@@ -35,32 +37,73 @@ class PluginMonitorController:
|
||||
|
||||
def reload(self, enabled: bool) -> None:
|
||||
"""按当前配置停止旧线程,并在启用时创建新线程。"""
|
||||
self.stop()
|
||||
if enabled:
|
||||
stopped = self.stop()
|
||||
if enabled and stopped:
|
||||
self.start()
|
||||
|
||||
def start(self) -> None:
|
||||
"""启动唯一的守护监控线程。"""
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.info("插件文件修改监测已经在运行中...")
|
||||
return
|
||||
self._logger.info("开始监测插件文件修改...")
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._runner, daemon=True)
|
||||
self._thread.start()
|
||||
with self._lifecycle_lock:
|
||||
if self._closed:
|
||||
self._logger.info("插件文件修改监测已进入停机封口,跳过启动")
|
||||
return
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.info("插件文件修改监测已经在运行中...")
|
||||
return
|
||||
self._logger.info("开始监测插件文件修改...")
|
||||
self._stop_event.clear()
|
||||
self._thread = threading.Thread(target=self._runner, daemon=True)
|
||||
self._thread.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""请求监控线程退出,并在限定时间内等待其清理。"""
|
||||
if not self._thread or not self._thread.is_alive():
|
||||
self._logger.info("未启用插件文件修改监测,无需停止")
|
||||
return
|
||||
self._logger.info("正在停止插件文件修改监测...")
|
||||
def reopen(self) -> bool:
|
||||
"""为新的应用生命周期解除封口,仍有旧线程时拒绝重开。"""
|
||||
with self._lifecycle_lock:
|
||||
if self._thread and self._thread.is_alive():
|
||||
self._logger.warning("旧插件文件监测线程仍在运行,无法开启新生命周期")
|
||||
return False
|
||||
self._thread = None
|
||||
self._closed = False
|
||||
return True
|
||||
|
||||
def stop(self, timeout: float = 5.0) -> bool:
|
||||
"""临时停止监控线程,并返回其是否在预算内真正退出。"""
|
||||
return self._stop_with_budget(timeout=timeout, close=False)
|
||||
|
||||
def close(self, timeout: float = 5.0) -> bool:
|
||||
"""永久封口当前生命周期,并返回监控线程是否真正退出。"""
|
||||
return self._stop_with_budget(timeout=timeout, close=True)
|
||||
|
||||
def _stop_with_budget(self, *, timeout: float, close: bool) -> bool:
|
||||
"""在同一预算内取得生命周期锁、设置封口并等待线程退出。"""
|
||||
timeout = max(0.0, timeout)
|
||||
deadline = time.monotonic() + timeout
|
||||
self._stop_event.set()
|
||||
self._thread.join(timeout=5)
|
||||
if self._thread.is_alive():
|
||||
self._logger.warning("插件文件修改监测线程在5秒内未能正常停止。")
|
||||
self._thread = None
|
||||
self._logger.info("插件文件修改监测停止完成")
|
||||
if not self._lifecycle_lock.acquire(timeout=timeout):
|
||||
self._logger.warning(
|
||||
f"插件文件修改监测线程在{timeout:g}秒内未能取得停机所有权。"
|
||||
)
|
||||
return False
|
||||
try:
|
||||
if close:
|
||||
self._closed = True
|
||||
thread = self._thread
|
||||
self._stop_event.set()
|
||||
if not thread or not thread.is_alive():
|
||||
self._thread = None
|
||||
self._logger.info("未启用插件文件修改监测,无需停止")
|
||||
return True
|
||||
self._logger.info("正在停止插件文件修改监测...")
|
||||
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||
if thread.is_alive():
|
||||
self._logger.warning(
|
||||
f"插件文件修改监测线程在{timeout:g}秒内未能正常停止。"
|
||||
)
|
||||
return False
|
||||
self._thread = None
|
||||
self._logger.info("插件文件修改监测停止完成")
|
||||
return True
|
||||
finally:
|
||||
self._lifecycle_lock.release()
|
||||
|
||||
|
||||
class PluginChangeMonitor:
|
||||
|
||||
@@ -1,9 +1,21 @@
|
||||
import asyncio
|
||||
import concurrent.futures
|
||||
import inspect
|
||||
import posixpath
|
||||
import threading
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Type, Union, Callable, Tuple
|
||||
from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ContextManager,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
from watchfiles import watch
|
||||
|
||||
@@ -17,6 +29,7 @@ from app.runtime.execution import run_in_threadpool_to_completion
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.observability import observe_compat_facade
|
||||
from app.runtime.settings import RuntimeSettingsCompat
|
||||
from app.runtime.thread import ThreadHelper
|
||||
|
||||
settings = RuntimeSettingsCompat()
|
||||
from app.runtime.events import EventHandlerBinding, eventmanager
|
||||
@@ -39,6 +52,7 @@ from app.runtime.extensions.plugin.sync import (
|
||||
)
|
||||
from app.runtime.extensions.plugin.clone import PluginCloneService
|
||||
from app.runtime.extensions.plugin.access import PluginAccessPolicy
|
||||
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
|
||||
from app.runtime.extensions.plugin.catalog import PluginCatalogFacade
|
||||
from app.runtime.extensions.plugin.paths import PluginPathResolver
|
||||
from app.runtime.extensions.plugin.dependency import (
|
||||
@@ -47,6 +61,7 @@ from app.runtime.extensions.plugin.dependency import (
|
||||
PluginDependencyService,
|
||||
)
|
||||
from app.runtime.extensions.plugin.storage import PluginConfigStore, PluginInstanceStore
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
|
||||
LegacyDiagnosticsConfigurator = Callable[..., None]
|
||||
@@ -195,6 +210,15 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
runner=self._run_file_watcher,
|
||||
log=logger,
|
||||
)
|
||||
self._plugin_quiesce_lock = threading.RLock()
|
||||
self._plugin_quiesce_future: Optional[
|
||||
concurrent.futures.Future[bool]
|
||||
] = None
|
||||
self._plugin_service_quiesce_future: Optional[
|
||||
concurrent.futures.Future[bool]
|
||||
] = None
|
||||
self._plugin_mutation_admission = PluginMutationAdmission()
|
||||
self._plugin_runtime_closed = False
|
||||
self._plugin_dependencies = PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
@@ -315,12 +339,16 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
|
||||
def init_config(self):
|
||||
"""按最新系统配置完整重启插件。"""
|
||||
# 停止已有插件
|
||||
self.stop()
|
||||
classification = self.classify_plugins()
|
||||
self.apply_plugin_dependency_classification(classification)
|
||||
for plugin_id in classification.ready:
|
||||
self.start(plugin_id)
|
||||
try:
|
||||
with self.mutation("配置热重载"):
|
||||
# 停止已有插件
|
||||
self.stop()
|
||||
classification = self.classify_plugins()
|
||||
self.apply_plugin_dependency_classification(classification)
|
||||
for plugin_id in classification.ready:
|
||||
self.start(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def start(self, pid: Optional[str] = None) -> Dict[str, PluginRuntimeStatus]:
|
||||
"""
|
||||
@@ -328,8 +356,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID,为空加载所有插件
|
||||
"""
|
||||
|
||||
_legacy_diagnostics_configurator(enabled=settings.DEBUG, emitter=logger.warning)
|
||||
return self._plugin_lifecycle.start(pid)
|
||||
try:
|
||||
with self.mutation("启动插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
_legacy_diagnostics_configurator(
|
||||
enabled=settings.DEBUG,
|
||||
emitter=logger.warning,
|
||||
)
|
||||
return self._plugin_lifecycle.start(pid)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
if pid:
|
||||
return {pid: PluginRuntimeStatus.LOAD_FAILED}
|
||||
return {}
|
||||
|
||||
def init_plugin(self, plugin_id: str, conf: dict):
|
||||
"""
|
||||
@@ -337,7 +376,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param plugin_id: 插件ID
|
||||
:param conf: 插件配置
|
||||
"""
|
||||
self._plugin_lifecycle.initialize(plugin_id, conf)
|
||||
try:
|
||||
with self.mutation("初始化插件配置"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.initialize(plugin_id, conf)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def clear_plugin_agent_tools_cache(self) -> None:
|
||||
"""
|
||||
@@ -356,12 +400,127 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""兼容读取旧私有字段,实际版本由独立工具目录持有。"""
|
||||
return self._plugin_tool_catalog.revision
|
||||
|
||||
def stop(self, pid: Optional[str] = None):
|
||||
def stop(self, pid: Optional[str] = None) -> None:
|
||||
"""
|
||||
停止插件服务
|
||||
:param pid: 插件ID,为空停止所有插件
|
||||
"""
|
||||
self._plugin_lifecycle.stop(pid)
|
||||
try:
|
||||
with self.mutation("停止插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.stop(pid)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def mutation(self, operation: str) -> ContextManager[None]:
|
||||
"""为一个完整插件可变事务取得可跨异步边界传播的准入 lease。"""
|
||||
return self._plugin_mutation_admission.hold(operation)
|
||||
|
||||
def reopen_plugins(self) -> bool:
|
||||
"""为新应用生命周期解除运行时封口,仍活跃的 quiesce owner 禁止复用。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
futures = (
|
||||
self._plugin_quiesce_future,
|
||||
self._plugin_service_quiesce_future,
|
||||
)
|
||||
if any(future is not None and not future.done() for future in futures):
|
||||
logger.warning("插件后台服务仍在停止,无法开启新的应用生命周期")
|
||||
return False
|
||||
if self._plugin_runtime_closed and self._running_plugins:
|
||||
logger.warning("上一应用生命周期仍持有插件实例,拒绝解除运行时封口")
|
||||
return False
|
||||
if not self._plugin_mutation_admission.reopen():
|
||||
logger.warning("上一应用生命周期仍有插件可变事务,拒绝解除运行时封口")
|
||||
return False
|
||||
self._plugin_runtime_closed = False
|
||||
return True
|
||||
|
||||
async def quiesce_plugins(self, timeout: float = 240.0) -> bool:
|
||||
"""封口变更事务并停用插件 handler,超时后保留 Future ownership。"""
|
||||
if self._plugin_mutation_admission.is_held():
|
||||
logger.warning("插件可变事务不能等待自身收敛,拒绝在事务内执行停机")
|
||||
return False
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_runtime_closed = True
|
||||
self._plugin_mutation_admission.seal()
|
||||
future = self._plugin_quiesce_future
|
||||
if future is None or future.done():
|
||||
future = ThreadHelper().submit(self._quiesce_after_mutations)
|
||||
self._plugin_quiesce_future = future
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(asyncio.wrap_future(future)),
|
||||
timeout=max(0.0, timeout),
|
||||
)
|
||||
return bool(result)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"插件后台服务未在 {timeout:g} 秒内收敛")
|
||||
return False
|
||||
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
|
||||
logger.error(f"插件后台服务停止失败:{error}", exc_info=True)
|
||||
return False
|
||||
finally:
|
||||
if future.done():
|
||||
with self._plugin_quiesce_lock:
|
||||
if self._plugin_quiesce_future is future:
|
||||
self._plugin_quiesce_future = None
|
||||
|
||||
async def quiesce_plugin_services(self, timeout: float = 240.0) -> bool:
|
||||
"""在事件结算后执行旧插件停机 hook,并有界等待同步 owner。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
prepare_future = self._plugin_quiesce_future
|
||||
if prepare_future is not None and not prepare_future.done():
|
||||
logger.warning("插件事件入口仍在封口,拒绝提前关闭插件资源")
|
||||
return False
|
||||
if not self._plugin_runtime_closed:
|
||||
logger.warning("插件运行时尚未封口,拒绝关闭插件资源")
|
||||
return False
|
||||
if self._plugin_mutation_admission.active_count:
|
||||
logger.warning("插件可变事务仍在执行,拒绝关闭插件资源")
|
||||
return False
|
||||
future = self._plugin_service_quiesce_future
|
||||
if future is None or future.done():
|
||||
future = ThreadHelper().submit(
|
||||
self._plugin_lifecycle.quiesce_services,
|
||||
)
|
||||
self._plugin_service_quiesce_future = future
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(asyncio.wrap_future(future)),
|
||||
timeout=max(0.0, timeout),
|
||||
)
|
||||
return bool(result)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"插件旧停机 hook 未在 {timeout:g} 秒内收敛")
|
||||
return False
|
||||
except Exception as error: # noqa: BLE001 Future 异常必须转为生命周期结果
|
||||
logger.error(f"插件旧停机 hook 执行失败:{error}", exc_info=True)
|
||||
return False
|
||||
finally:
|
||||
if future.done():
|
||||
with self._plugin_quiesce_lock:
|
||||
if self._plugin_service_quiesce_future is future:
|
||||
self._plugin_service_quiesce_future = None
|
||||
|
||||
def finalize_plugins(self) -> bool:
|
||||
"""确认 quiesce owner 已结束后禁用 handler 并卸载插件实例。"""
|
||||
with self._plugin_quiesce_lock:
|
||||
futures = (
|
||||
self._plugin_quiesce_future,
|
||||
self._plugin_service_quiesce_future,
|
||||
)
|
||||
if any(future is not None and not future.done() for future in futures):
|
||||
logger.warning("插件后台服务仍在停止,拒绝释放运行实例")
|
||||
return False
|
||||
if self._plugin_mutation_admission.active_count:
|
||||
logger.warning("插件可变事务仍在执行,拒绝释放运行实例")
|
||||
return False
|
||||
return self._plugin_lifecycle.finalize()
|
||||
|
||||
def _quiesce_after_mutations(self) -> bool:
|
||||
"""等待已获准变更自然结束后,再停用插件事件入口。"""
|
||||
self._plugin_mutation_admission.wait_until_idle()
|
||||
return self._plugin_lifecycle.quiesce_handlers()
|
||||
|
||||
@staticmethod
|
||||
def _load_selective_plugins(pid: Optional[str], installed_plugins: List[str],
|
||||
@@ -428,8 +587,10 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""返回配置重载日志使用的功能名称。"""
|
||||
return "插件文件修改监测"
|
||||
|
||||
def start_monitor(self):
|
||||
"""按当前配置启动插件文件修改监测。"""
|
||||
def start_monitor(self, *, reopen: bool = False) -> None:
|
||||
"""按当前配置启动监控;新生命周期可显式解除既有封口。"""
|
||||
if reopen and not self._plugin_monitor.reopen():
|
||||
return
|
||||
if (
|
||||
not self.is_plugin_settling()
|
||||
and (settings.DEV or settings.PLUGIN_AUTO_RELOAD)
|
||||
@@ -447,11 +608,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
)
|
||||
)
|
||||
|
||||
def stop_monitor(self):
|
||||
"""
|
||||
停止监测插件文件修改监测
|
||||
"""
|
||||
self._plugin_monitor.stop()
|
||||
def stop_monitor(self, timeout: float = 5.0) -> bool:
|
||||
"""停止插件文件监控,并返回线程是否在预算内真正退出。"""
|
||||
return self._plugin_monitor.stop(timeout=timeout)
|
||||
|
||||
def close_monitor(self, timeout: float = 5.0) -> bool:
|
||||
"""封口当前生命周期的文件监控,并返回线程是否真正退出。"""
|
||||
return self._plugin_monitor.close(timeout=timeout)
|
||||
|
||||
def _run_file_watcher(self):
|
||||
"""
|
||||
@@ -504,25 +667,31 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
已安装本地插件源码变化时,同步到运行目录
|
||||
"""
|
||||
return self._local_plugin_sync.sync(pid, candidate)
|
||||
try:
|
||||
with self.mutation("同步本地插件源码"):
|
||||
return self._local_plugin_sync.sync(pid, candidate)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
@contextmanager
|
||||
def suppress_plugin_monitor(self, plugin_id: str):
|
||||
"""在插件目录原子更新期间阻止文件监控抢先重载半成品。"""
|
||||
normalized_id = plugin_id.lower()
|
||||
with self._monitor_suppression_lock:
|
||||
self._suppressed_monitor_plugins[normalized_id] = (
|
||||
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self.mutation("更新插件包"):
|
||||
normalized_id = plugin_id.lower()
|
||||
with self._monitor_suppression_lock:
|
||||
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
|
||||
if count <= 1:
|
||||
self._suppressed_monitor_plugins.pop(normalized_id, None)
|
||||
else:
|
||||
self._suppressed_monitor_plugins[normalized_id] = count - 1
|
||||
self._suppressed_monitor_plugins[normalized_id] = (
|
||||
self._suppressed_monitor_plugins.get(normalized_id, 0) + 1
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
with self._monitor_suppression_lock:
|
||||
count = self._suppressed_monitor_plugins.get(normalized_id, 0)
|
||||
if count <= 1:
|
||||
self._suppressed_monitor_plugins.pop(normalized_id, None)
|
||||
else:
|
||||
self._suppressed_monitor_plugins[normalized_id] = count - 1
|
||||
|
||||
def is_plugin_monitor_suppressed(self, plugin_id: str) -> bool:
|
||||
"""判断指定插件是否处于安装或替换写入阶段。"""
|
||||
@@ -534,23 +703,43 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
从内存中移除一个插件
|
||||
:param plugin_id: 插件ID
|
||||
"""
|
||||
self._plugin_lifecycle.stop(plugin_id)
|
||||
self._plugin_registry.remove(plugin_id)
|
||||
try:
|
||||
with self.mutation("移除插件实例"):
|
||||
with self._plugin_quiesce_lock:
|
||||
self._plugin_lifecycle.stop(plugin_id)
|
||||
self._plugin_registry.remove(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
|
||||
def reload_plugin(self, plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""
|
||||
将一个插件重新加载到内存
|
||||
:param plugin_id: 插件ID
|
||||
"""
|
||||
return self._plugin_lifecycle.reload(plugin_id, EventType.PluginReload)
|
||||
try:
|
||||
with self.mutation("重新加载插件"):
|
||||
with self._plugin_quiesce_lock:
|
||||
return self._plugin_lifecycle.reload(
|
||||
plugin_id,
|
||||
EventType.PluginReload,
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return PluginRuntimeStatus.LOAD_FAILED
|
||||
|
||||
def reload_plugin_tree(self, plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载源码插件,并同步刷新所有引用该源码的虚拟实例。"""
|
||||
source_plugin_id = self.get_plugin_source_id(plugin_id)
|
||||
status = self.reload_plugin(source_plugin_id)
|
||||
for instance in self._plugin_instance_store.for_source(source_plugin_id):
|
||||
self.reload_plugin(instance.instance_id)
|
||||
return status
|
||||
try:
|
||||
with self.mutation("重载插件实例树"):
|
||||
with self._plugin_quiesce_lock:
|
||||
source_plugin_id = self.get_plugin_source_id(plugin_id)
|
||||
status = self.reload_plugin(source_plugin_id)
|
||||
for instance in self._plugin_instance_store.for_source(source_plugin_id):
|
||||
self.reload_plugin(instance.instance_id)
|
||||
return status
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return PluginRuntimeStatus.LOAD_FAILED
|
||||
|
||||
def get_plugin_reload_targets(self, plugin_id: str) -> List[str]:
|
||||
"""返回源码更新后需要刷新注册信息的源插件及其实例 ID。"""
|
||||
@@ -584,33 +773,40 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
安装本地不存在或需要更新的插件
|
||||
"""
|
||||
|
||||
return self._plugin_sync.sync()
|
||||
with self.mutation("同步插件包"):
|
||||
return self._plugin_sync.sync()
|
||||
|
||||
@staticmethod
|
||||
def install_plugin_missing_dependencies() -> List[str]:
|
||||
"""
|
||||
安装插件中缺失或不兼容的依赖项
|
||||
"""
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing()
|
||||
|
||||
@staticmethod
|
||||
def install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
|
||||
"""安装插件缺失依赖并返回缺失项及安装成功状态。"""
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing_with_status()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).install_missing_with_status()
|
||||
|
||||
@staticmethod
|
||||
async def async_install_plugin_missing_dependencies_with_status() -> PluginDependencyInstallResult:
|
||||
"""在异步启动链中恢复插件依赖并保留取消语义。"""
|
||||
return await PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).async_install_missing_with_status()
|
||||
manager = PluginManager()
|
||||
with manager.mutation("安装插件依赖"):
|
||||
return await PluginDependencyService(
|
||||
system=get_plugin_system,
|
||||
log=logger,
|
||||
).async_install_missing_with_status()
|
||||
|
||||
def classify_plugins(self) -> PluginDependencyClassification:
|
||||
"""按源码依赖状态分类物理插件,并把结果映射到虚拟实例。"""
|
||||
@@ -706,7 +902,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
|
||||
def delete_plugin_instance(self, plugin_id: str) -> bool:
|
||||
"""删除虚拟实例描述;调用方仍负责停止实例和清理业务数据。"""
|
||||
return self._plugin_instance_store.delete(plugin_id)
|
||||
try:
|
||||
with self.mutation("删除插件实例描述"):
|
||||
return self._plugin_instance_store.delete(plugin_id)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def save_plugin_config(self, pid: str, conf: dict, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -715,7 +916,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param conf: 配置
|
||||
:param force: 强制保存
|
||||
"""
|
||||
return self._plugin_config_store.write(pid, conf, force)
|
||||
try:
|
||||
with self.mutation("保存插件配置"):
|
||||
return self._plugin_config_store.write(pid, conf, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
async def async_save_plugin_config(
|
||||
self, pid: str, conf: dict, force: bool = False
|
||||
@@ -726,7 +932,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param conf: 配置
|
||||
:param force: 强制保存
|
||||
"""
|
||||
return await self._plugin_config_store.async_write(pid, conf, force)
|
||||
try:
|
||||
with self.mutation("保存插件配置"):
|
||||
return await self._plugin_config_store.async_write(pid, conf, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def delete_plugin_config(self, pid: str, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -734,7 +945,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID
|
||||
:param force: 插件停止后仍允许按插件 ID 删除持久化配置
|
||||
"""
|
||||
return self._plugin_config_store.delete(pid, force)
|
||||
try:
|
||||
with self.mutation("删除插件配置"):
|
||||
return self._plugin_config_store.delete(pid, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def delete_plugin_data(self, pid: str, force: bool = False) -> bool:
|
||||
"""
|
||||
@@ -742,7 +958,12 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param pid: 插件ID
|
||||
:param force: 插件停止后仍允许按插件 ID 删除持久化数据
|
||||
"""
|
||||
return self._plugin_config_store.delete_data(pid, force)
|
||||
try:
|
||||
with self.mutation("删除插件数据"):
|
||||
return self._plugin_config_store.delete_data(pid, force)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
|
||||
def get_plugin_state(self, pid: str) -> bool:
|
||||
"""
|
||||
@@ -1130,14 +1351,19 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton):
|
||||
:param icon: 自定义图标URL
|
||||
:return: (是否成功, 错误信息)
|
||||
"""
|
||||
return self._plugin_clone.clone(
|
||||
plugin_id=plugin_id,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
try:
|
||||
with self.mutation("创建插件分身"):
|
||||
return self._plugin_clone.clone(
|
||||
plugin_id=plugin_id,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False, str(error)
|
||||
|
||||
def _modify_plugin_files(self, plugin_dir: Path, original_id: str, suffix: str,
|
||||
name: str, description: str, version: str = None,
|
||||
|
||||
@@ -106,8 +106,8 @@ class TaskRegistry:
|
||||
}
|
||||
)
|
||||
|
||||
async def shutdown(self, *, timeout_seconds: float = 10.0) -> None:
|
||||
"""停止接收并有限等待存量任务,超时任务保留登记并报告责任域。"""
|
||||
async def shutdown(self, *, timeout_seconds: float = 10.0) -> bool:
|
||||
"""停止接收并有限等待存量任务,返回全部 owner 是否真实收敛。"""
|
||||
self._accepting = False
|
||||
records = self.records
|
||||
tasks = [record.task for record in records]
|
||||
@@ -139,6 +139,7 @@ class TaskRegistry:
|
||||
"timeout_seconds": timeout_seconds,
|
||||
}
|
||||
)
|
||||
return all(record.task.done() for record in records)
|
||||
|
||||
|
||||
_default_registry = TaskRegistry()
|
||||
|
||||
@@ -4,6 +4,7 @@ __all__ = (
|
||||
"APIRateLimitException",
|
||||
"RateLimitExceededException",
|
||||
"OperationInterrupted",
|
||||
"PluginMutationRejectedError",
|
||||
"StorageQueryError",
|
||||
"TMDbException",
|
||||
)
|
||||
@@ -49,6 +50,15 @@ class OperationInterrupted(KeyboardInterrupt):
|
||||
pass
|
||||
|
||||
|
||||
class PluginMutationRejectedError(RuntimeError):
|
||||
"""表示插件运行时已封口,新的可变事务未获准执行。"""
|
||||
|
||||
def __init__(self, operation: str) -> None:
|
||||
"""保存被拒绝的操作名称并生成稳定诊断消息。"""
|
||||
self.operation = operation
|
||||
super().__init__(f"插件运行时已进入停机阶段,拒绝{operation}")
|
||||
|
||||
|
||||
class StorageQueryError(Exception):
|
||||
"""
|
||||
用于表示存储查询无法确认结果的异常类。
|
||||
|
||||
@@ -20,6 +20,9 @@ from app.runtime.log import logger
|
||||
from app.schemas.types import EventType
|
||||
|
||||
|
||||
AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
|
||||
def _get_skill_catalog() -> Any:
|
||||
"""按需返回 Agent 技能目录实现,供消息应用层消费端口。"""
|
||||
from app.agent.skills.registry import SkillHelper
|
||||
@@ -106,6 +109,7 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
self._manager: Any = None
|
||||
self._compat_injected = False
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
eventmanager.add_event_listener(
|
||||
EventType.ConfigChanged,
|
||||
@@ -117,6 +121,7 @@ class AgentInitializer:
|
||||
初始化AI智能体管理器
|
||||
"""
|
||||
try:
|
||||
self._shutdown_started = False
|
||||
self._shutdown_complete = False
|
||||
if agent_manager is not None:
|
||||
if not settings.AI_AGENT_ENABLE:
|
||||
@@ -142,7 +147,12 @@ class AgentInitializer:
|
||||
async def handle_config_changed(self, event: Event) -> None:
|
||||
"""仅在 manifest watch 命中时协调 service,关闭态保持 fail closed。"""
|
||||
changed_keys = _event_changed_keys(event)
|
||||
if not changed_keys or self._compat_injected or self._shutdown_complete:
|
||||
if (
|
||||
not changed_keys
|
||||
or self._compat_injected
|
||||
or self._shutdown_started
|
||||
or self._shutdown_complete
|
||||
):
|
||||
return
|
||||
try:
|
||||
self._manager = await reconcile_agent_service(
|
||||
@@ -156,24 +166,25 @@ class AgentInitializer:
|
||||
self._initialized = False
|
||||
logger.debug(f"配置变更协调AI智能体失败: {error}")
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
"""清理 initializer 引用;显式注入对象同时在此关闭。"""
|
||||
async def cleanup(self) -> bool:
|
||||
"""清理 initializer 引用;未收敛的显式注入对象继续由本实例持有。"""
|
||||
try:
|
||||
manager = self._manager
|
||||
compat_injected = self._compat_injected
|
||||
if manager is None:
|
||||
return
|
||||
try:
|
||||
if compat_injected:
|
||||
await manager.close()
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
finally:
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
if compat_injected and await manager.close() is False:
|
||||
logger.error("AI智能体管理器仍有会话 owner 未收敛")
|
||||
return False
|
||||
logger.info("AI智能体管理器已关闭")
|
||||
self._initialized = False
|
||||
self._manager = None
|
||||
self._compat_injected = False
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.debug(f"关闭AI智能体管理器时发生错误: {e}")
|
||||
return False
|
||||
|
||||
|
||||
# 全局AI智能体初始化器实例
|
||||
@@ -204,23 +215,56 @@ async def init_agent() -> bool:
|
||||
return False
|
||||
|
||||
|
||||
async def stop_agent():
|
||||
async def stop_agent() -> bool:
|
||||
"""
|
||||
停止AI智能体(异步版本,用于在应用关闭时调用)
|
||||
停止AI智能体,并在全部会话和工具资源释放后返回 True。
|
||||
"""
|
||||
converged = True
|
||||
close_blocking_executors = None
|
||||
agent_initializer._shutdown_started = True
|
||||
try:
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import (
|
||||
begin_blocking_executor_shutdown,
|
||||
close_blocking_executors as close_executors,
|
||||
)
|
||||
|
||||
# 必须在任何 manager await 之前封口,避免旧会话趁收尾窗口提交新同步调用。
|
||||
begin_blocking_executor_shutdown(cancel_futures=True)
|
||||
close_blocking_executors = close_executors
|
||||
except Exception as e:
|
||||
logger.error(f"封住AI智能体阻塞工具提交时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
try:
|
||||
if not agent_initializer._shutdown_complete:
|
||||
if agent_initializer._compat_injected:
|
||||
await agent_initializer.cleanup()
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
else:
|
||||
await begin_agent_shutdown()
|
||||
await agent_initializer.cleanup()
|
||||
agent_initializer._shutdown_complete = True
|
||||
if is_tool_factory_materialized():
|
||||
from app.agent.tools.base import shutdown_blocking_executors
|
||||
|
||||
shutdown_blocking_executors(wait=False, cancel_futures=True)
|
||||
service_converged = await begin_agent_shutdown()
|
||||
if service_converged is not False:
|
||||
service_converged = await agent_initializer.cleanup()
|
||||
converged = converged and service_converged is not False
|
||||
except Exception as e:
|
||||
logger.error(f"停止AI智能体时发生错误: {e}")
|
||||
finally:
|
||||
await close_materialized_terminal_sessions()
|
||||
converged = False
|
||||
|
||||
if close_blocking_executors is not None:
|
||||
try:
|
||||
blocking_converged = await close_blocking_executors(
|
||||
timeout_seconds=AGENT_BLOCKING_EXECUTOR_SHUTDOWN_TIMEOUT_SECONDS,
|
||||
cancel_futures=True,
|
||||
)
|
||||
converged = converged and blocking_converged
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体阻塞工具线程池时发生错误: {e}")
|
||||
converged = False
|
||||
|
||||
if converged:
|
||||
try:
|
||||
await close_materialized_terminal_sessions()
|
||||
except Exception as e:
|
||||
logger.error(f"关闭AI智能体终端会话时发生错误: {e}")
|
||||
converged = False
|
||||
agent_initializer._shutdown_complete = converged
|
||||
return converged
|
||||
|
||||
@@ -38,14 +38,23 @@ from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.runtime.state import SystemHelper
|
||||
from app.runtime.log import logger, LoggerManager
|
||||
from app.startup.command_initializer import init_command, stop_command, restart_command
|
||||
from app.startup.agent_initializer import stop_agent
|
||||
from app.startup.domain_initializer import configure_domain_dependencies
|
||||
from app.startup.modules_initializer import init_modules, stop_modules
|
||||
from app.startup.modules_initializer import (
|
||||
drain_events,
|
||||
init_modules,
|
||||
settle_events,
|
||||
stop_modules,
|
||||
)
|
||||
from app.startup.monitor_initializer import stop_monitor, init_monitor
|
||||
from app.startup.plugins_initializer import (
|
||||
configure_plugin_services,
|
||||
execute_task,
|
||||
finalize_plugins,
|
||||
init_plugins,
|
||||
stop_plugins,
|
||||
quiesce_plugin_services,
|
||||
quiesce_plugins,
|
||||
stop_plugin_monitor,
|
||||
sync_plugins,
|
||||
)
|
||||
from app.startup.routers_initializer import init_routers
|
||||
@@ -55,10 +64,14 @@ from app.startup.scheduler_initializer import (
|
||||
init_plugin_scheduler,
|
||||
)
|
||||
from app.db.engine import check_connection_budget, get_engine, get_global_async_engine
|
||||
from app.startup.transfer_initializer import replay_pending_transfers
|
||||
from app.startup.transfer_initializer import (
|
||||
replay_pending_transfers,
|
||||
stop_transfer_runtime,
|
||||
)
|
||||
from app.startup.workflow_initializer import init_workflow, stop_workflow
|
||||
from app.startup.lifecycle.components import (
|
||||
LifecycleComponent,
|
||||
LifecycleFailurePolicy,
|
||||
LifecycleMode,
|
||||
lifecycle_manifest,
|
||||
)
|
||||
@@ -101,8 +114,8 @@ async def run_shutdown_step(
|
||||
name: str,
|
||||
callback: Callable[[], object],
|
||||
timeout_seconds: float | None = None,
|
||||
) -> None:
|
||||
"""在有限预算内执行关闭阶段,并保留未收敛任务的资源所有权。"""
|
||||
) -> bool:
|
||||
"""在有限预算内执行关闭阶段,并返回资源 owner 是否已经收敛。"""
|
||||
try:
|
||||
result = callback()
|
||||
if inspect.isawaitable(result):
|
||||
@@ -120,16 +133,22 @@ async def run_shutdown_step(
|
||||
task.add_done_callback(_consume_shutdown_result)
|
||||
if timeout_seconds:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
result = await asyncio.wait_for(
|
||||
asyncio.shield(task), timeout=timeout_seconds
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error("关闭%s超时,已请求取消并保留未收敛任务", name)
|
||||
task.cancel()
|
||||
return False
|
||||
else:
|
||||
await task
|
||||
result = await task
|
||||
if result is False:
|
||||
logger.error("关闭%s未收敛,资源所有权保持不变", name)
|
||||
return False
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return False
|
||||
|
||||
|
||||
async def run_startup_step(
|
||||
@@ -152,6 +171,66 @@ async def run_startup_step(
|
||||
logger.info("启动%s完成,耗时=%.2fms", name, elapsed_ms)
|
||||
|
||||
|
||||
async def stop_lifecycle_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
) -> bool:
|
||||
"""按声明顺序关闭组件,并在关键 owner 未收敛时停止释放依赖。"""
|
||||
all_converged = True
|
||||
for component in sorted(
|
||||
(item for item in components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
completed = await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
if completed:
|
||||
continue
|
||||
all_converged = False
|
||||
if component.stop_failure is LifecycleFailurePolicy.FAIL_FAST:
|
||||
logger.error(
|
||||
"关闭%s未收敛,停止释放其后续依赖",
|
||||
component.name,
|
||||
)
|
||||
break
|
||||
return all_converged
|
||||
|
||||
|
||||
def select_startup_cleanup_components(
|
||||
components: tuple[LifecycleComponent, ...],
|
||||
*,
|
||||
started_names: set[str],
|
||||
active_component: LifecycleComponent | None,
|
||||
) -> tuple[LifecycleComponent, ...]:
|
||||
"""选择启动失败时已启动、部分启动及其 stop-only owner 的清理集合。"""
|
||||
cleanup_names = set(started_names)
|
||||
if active_component is not None:
|
||||
cleanup_names.add(active_component.name)
|
||||
|
||||
# stop-only owner 没有启动回调,按已激活依赖递归纳入,避免清理时反向
|
||||
# 实例化尚未触达的插件、模块或外部资源。
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for component in components:
|
||||
if (
|
||||
component.stop is None
|
||||
or component.start is not None
|
||||
or component.name in cleanup_names
|
||||
or not set(component.dependencies).issubset(cleanup_names)
|
||||
):
|
||||
continue
|
||||
cleanup_names.add(component.name)
|
||||
changed = True
|
||||
|
||||
return tuple(
|
||||
component
|
||||
for component in components
|
||||
if component.stop is not None and component.name in cleanup_names
|
||||
)
|
||||
|
||||
|
||||
async def initialize_modules_component(app: FastAPI) -> None:
|
||||
"""启动模块并把其类型化运行时发布到当前 FastAPI AppState。"""
|
||||
try:
|
||||
@@ -175,15 +254,16 @@ def initialize_task_registry(app: FastAPI) -> None:
|
||||
configure_task_registry(task_registry)
|
||||
|
||||
|
||||
async def stop_task_registry(app: FastAPI) -> None:
|
||||
"""停止接收新后台任务,并取消、等待当前 lifespan 的存量任务。"""
|
||||
async def stop_task_registry(app: FastAPI) -> bool:
|
||||
"""停止接收新后台任务,并把已封口登记器保留到下一次显式启动。"""
|
||||
task_registry = getattr(app.state, "task_registry", None)
|
||||
try:
|
||||
if isinstance(task_registry, TaskRegistry):
|
||||
await task_registry.shutdown(timeout_seconds=30.0)
|
||||
finally:
|
||||
if not isinstance(task_registry, TaskRegistry):
|
||||
configure_task_registry(None)
|
||||
app.state.task_registry = None
|
||||
return True
|
||||
# 即使 owner 已收敛,也不能在后续插件/模块 stop hook 仍会运行时退回永久
|
||||
# accepting 的兼容默认登记器。下一次 initialize_task_registry 会显式替换它。
|
||||
return await task_registry.shutdown(timeout_seconds=30.0)
|
||||
|
||||
|
||||
def prepare_plugin_restore() -> None:
|
||||
@@ -217,6 +297,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=5,
|
||||
start_timeout_seconds=30,
|
||||
stop_timeout_seconds=60,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="数据库准备",
|
||||
@@ -284,11 +365,21 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
dependencies=("插件备份恢复",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=init_plugins,
|
||||
stop=stop_plugins,
|
||||
stop=finalize_plugins,
|
||||
start_order=90,
|
||||
stop_order=60,
|
||||
start_timeout_seconds=300,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件变更监控",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=stop_plugin_monitor,
|
||||
stop_order=8,
|
||||
stop_timeout_seconds=10,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="定时器",
|
||||
@@ -300,6 +391,7 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=50,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="监控器",
|
||||
@@ -311,10 +403,62 @@ def build_lifecycle_components(app: FastAPI) -> tuple[LifecycleComponent, ...]:
|
||||
stop_order=40,
|
||||
start_timeout_seconds=120,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="整理后台服务",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_transfer_runtime,
|
||||
stop_order=52,
|
||||
stop_timeout_seconds=45,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="AI智能体会话",
|
||||
dependencies=("模块服务",),
|
||||
stop=stop_agent,
|
||||
stop_order=51,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件事件入口",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugins,
|
||||
stop_order=53,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件尾任务结算",
|
||||
dependencies=("模块服务", "插件"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=settle_events,
|
||||
stop_order=54,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="插件后台服务",
|
||||
dependencies=("插件",),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
stop=quiesce_plugin_services,
|
||||
stop_order=55,
|
||||
stop_timeout_seconds=300,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="事件投递屏障",
|
||||
dependencies=("模块服务", "整理后台服务"),
|
||||
stop=drain_events,
|
||||
stop_order=58,
|
||||
stop_timeout_seconds=120,
|
||||
stop_failure=LifecycleFailurePolicy.FAIL_FAST,
|
||||
),
|
||||
LifecycleComponent(
|
||||
name="待处理整理回放",
|
||||
dependencies=("监控器",),
|
||||
dependencies=("监控器", "整理后台服务"),
|
||||
mode=LifecycleMode.NORMAL_ONLY,
|
||||
start=replay_pending_transfers,
|
||||
start_order=120,
|
||||
@@ -369,6 +513,9 @@ async def lifespan(app: FastAPI):
|
||||
health = get_application_health(app)
|
||||
health.begin_startup()
|
||||
main_loop = asyncio.get_running_loop()
|
||||
enabled_components: tuple[LifecycleComponent, ...] = ()
|
||||
started_component_names: set[str] = set()
|
||||
active_start_component: LifecycleComponent | None = None
|
||||
try:
|
||||
validate_process_topology(
|
||||
workers=settings.API_WORKERS,
|
||||
@@ -390,11 +537,14 @@ async def lifespan(app: FastAPI):
|
||||
(item for item in enabled_components if item.start is not None),
|
||||
key=lambda item: item.start_order or 0,
|
||||
):
|
||||
active_start_component = component
|
||||
await run_startup_step(
|
||||
component.name,
|
||||
component.start,
|
||||
component.start_timeout_seconds,
|
||||
)
|
||||
started_component_names.add(component.name)
|
||||
active_start_component = None
|
||||
if settings.MOVIEPILOT_SAFE_MODE:
|
||||
print("MoviePilot safe mode enabled: skip plugins, scheduler, monitor, commands and workflow.")
|
||||
# 插件同步到本地
|
||||
@@ -407,10 +557,15 @@ async def lifespan(app: FastAPI):
|
||||
except BaseException:
|
||||
# Uvicorn 在 lifespan 抛错时不会开始接流量;状态仍需供嵌入式入口和测试诊断。
|
||||
health.mark_failed()
|
||||
cleanup_components = select_startup_cleanup_components(
|
||||
enabled_components,
|
||||
started_names=started_component_names,
|
||||
active_component=active_start_component,
|
||||
)
|
||||
try:
|
||||
await stop_task_registry(app)
|
||||
await stop_lifecycle_components(cleanup_components)
|
||||
except Exception as cleanup_error:
|
||||
logger.error(f"启动失败后的后台任务清理失败:{cleanup_error}")
|
||||
logger.error(f"启动失败后的生命周期清理失败:{cleanup_error}")
|
||||
finally:
|
||||
global_vars.clear_loop(main_loop)
|
||||
raise
|
||||
@@ -421,21 +576,10 @@ async def lifespan(app: FastAPI):
|
||||
health.mark_stopping()
|
||||
print("Shutting down...")
|
||||
global_vars.stop_system()
|
||||
# 插件恢复会在线程池中修改源码与依赖,必须完成后再进入资源关闭阶段。
|
||||
try:
|
||||
await sync_plugins_task
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
try:
|
||||
for component in sorted(
|
||||
(item for item in enabled_components if item.stop is not None),
|
||||
key=lambda item: item.stop_order or 0,
|
||||
):
|
||||
await run_shutdown_step(
|
||||
component.name,
|
||||
component.stop,
|
||||
component.stop_timeout_seconds,
|
||||
)
|
||||
# 插件 settlement 已登记到最前置 TaskRegistry。由该 FAIL_FAST owner
|
||||
# 在统一预算内取消/等待,不能在屏障之前无界 await 绕过停机预算。
|
||||
await stop_lifecycle_components(enabled_components)
|
||||
finally:
|
||||
try:
|
||||
# 日志最后关闭,确保其他组件的收尾信息已写入文件
|
||||
|
||||
@@ -129,7 +129,7 @@ from app.command import CommandChain
|
||||
from app.schemas.message import Message
|
||||
from app.schemas.message import MessageType
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
from app.startup.agent_initializer import init_agent, stop_agent
|
||||
from app.startup.agent_initializer import init_agent
|
||||
from app.startup.database import build_database_governance
|
||||
from app.startup.managed_resources_initializer import (
|
||||
init_managed_resources,
|
||||
@@ -560,6 +560,22 @@ def close_browser_sessions() -> None:
|
||||
BrowserSessionHelper.close_all_sessions()
|
||||
|
||||
|
||||
async def drain_events() -> bool:
|
||||
"""在插件卸载前等待已接收事件及其同步、异步处理器完成。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=True)
|
||||
|
||||
|
||||
async def settle_events() -> bool:
|
||||
"""在插件 handler 停用后结算在途事件,但保留停机 hook 的尾事件入口。"""
|
||||
event_manager = EventManager.get_existing_instance()
|
||||
if event_manager is None:
|
||||
return True
|
||||
return await event_manager.drain_async(seal=False)
|
||||
|
||||
|
||||
async def stop_modules():
|
||||
"""
|
||||
服务关闭
|
||||
@@ -578,7 +594,6 @@ async def stop_modules():
|
||||
logger.error(f"关闭{name}失败:{err}")
|
||||
return True
|
||||
|
||||
await run_step("AI智能体", stop_agent)
|
||||
await run_step("模块", lambda: ModuleManager().shutdown())
|
||||
await run_step("事件消费", lambda: EventManager().stop_async())
|
||||
await run_step("浏览器会话", close_browser_sessions)
|
||||
|
||||
@@ -1,15 +1,26 @@
|
||||
from app.monitor import Monitor
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
|
||||
|
||||
def init_monitor():
|
||||
"""
|
||||
初始化监控器
|
||||
"""
|
||||
Monitor()
|
||||
def init_monitor() -> None:
|
||||
"""初始化监控器;复用单例时必须显式开启新的应用 lifespan。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
Monitor()
|
||||
return
|
||||
if not monitor.lifecycle_closed:
|
||||
return
|
||||
if not monitor.reopen(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("旧目录监控 owner 未收敛,无法开启新生命周期")
|
||||
if not monitor.init(timeout=Monitor.RELOAD_STOP_TIMEOUT):
|
||||
raise RuntimeError("目录监控初始化失败")
|
||||
|
||||
|
||||
def stop_monitor():
|
||||
"""
|
||||
停止监控器
|
||||
"""
|
||||
Monitor().stop()
|
||||
async def stop_monitor(timeout: float = Monitor.LIFECYCLE_CLOSE_TIMEOUT) -> bool:
|
||||
"""在线程池里永久关闭监控器,取消后仍等待同步 owner 收敛到终态。"""
|
||||
monitor = Monitor.get_existing_instance()
|
||||
if monitor is None:
|
||||
return True
|
||||
return bool(
|
||||
await run_in_threadpool_to_completion(monitor.close, timeout=timeout)
|
||||
)
|
||||
|
||||
@@ -18,6 +18,7 @@ from app.runtime.extensions.plugin_manager import (
|
||||
configure_plugin_resource_import_preparer,
|
||||
configure_site_auth_level_provider,
|
||||
)
|
||||
from app.runtime.execution import run_in_threadpool_to_completion
|
||||
from app.runtime.extensions.plugin.dependency import PluginDependencyInstallResult
|
||||
from app.application.plugin.catalog import PluginCatalogService
|
||||
from app.application.plugin.data import DeletePluginDataCommand
|
||||
@@ -49,6 +50,7 @@ from app.db.uow import SqlAlchemyUnitOfWork
|
||||
from app.runtime.log import logger
|
||||
from app.foundation.version import compare_version
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.schemas.exception import PluginMutationRejectedError
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
@@ -142,56 +144,64 @@ async def sync_plugins() -> bool:
|
||||
"""
|
||||
plugin_manager = None
|
||||
try:
|
||||
configure_plugin_services()
|
||||
loop = global_vars.loop
|
||||
plugin_manager = PluginManager()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
with plugin_manager.mutation("启动后同步插件"):
|
||||
configure_plugin_services()
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
return await _sync_plugins_admitted(plugin_manager, loop)
|
||||
except PluginMutationRejectedError as error:
|
||||
logger.warning(str(error))
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"插件初始化过程中出现异常: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _sync_plugins_admitted(plugin_manager: PluginManager, loop) -> bool:
|
||||
"""在一个 admission lease 内完成包、依赖、实例和动态路由同步。"""
|
||||
sync_result = await execute_task(loop, plugin_manager.sync, "插件同步到本地")
|
||||
dependency_result = await (
|
||||
plugin_manager.async_install_plugin_missing_dependencies_with_status()
|
||||
)
|
||||
if dependency_result is None:
|
||||
return False
|
||||
if not isinstance(dependency_result, PluginDependencyInstallResult):
|
||||
logger.error("缺失依赖项安装返回了无效结果,跳过插件重新初始化")
|
||||
return False
|
||||
previous_statuses = plugin_manager.get_plugin_runtime_statuses()
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
if not dependency_result.success:
|
||||
logger.error("缺失依赖项安装未完成,将继续激活当前已就绪插件")
|
||||
changed_ids = await execute_task(
|
||||
loop,
|
||||
lambda: _activate_ready_plugins(
|
||||
plugin_manager,
|
||||
classification.ready,
|
||||
sync_result or [],
|
||||
previous_statuses,
|
||||
),
|
||||
"插件运行态激活",
|
||||
)
|
||||
if changed_ids is None:
|
||||
return False
|
||||
|
||||
if not changed_ids:
|
||||
logger.debug("没有新的插件进入可运行状态")
|
||||
return False
|
||||
|
||||
for plugin_id in changed_ids:
|
||||
register_plugin_api(plugin_id)
|
||||
if dependency_result.success:
|
||||
logger.info(f"后台插件加载完成,共处理 {len(changed_ids)} 个插件")
|
||||
else:
|
||||
logger.warning(
|
||||
f"缺失依赖项仍未全部恢复,已激活 {len(changed_ids)} 个就绪插件"
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _activate_ready_plugins(
|
||||
plugin_manager: PluginManager,
|
||||
ready_ids: tuple[str, ...],
|
||||
@@ -217,12 +227,39 @@ def _activate_ready_plugins(
|
||||
return changed_ids
|
||||
|
||||
|
||||
async def quiesce_plugins(timeout: float = 240.0) -> bool:
|
||||
"""封口插件变更并停用 handler,保留超时 Future 的运行所有权。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugins(timeout=timeout)
|
||||
|
||||
|
||||
async def quiesce_plugin_services(timeout: float = 240.0) -> bool:
|
||||
"""在事件结算后有界执行旧插件 close、stop_service hook。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return await plugin_manager.quiesce_plugin_services(timeout=timeout)
|
||||
|
||||
|
||||
def finalize_plugins() -> bool:
|
||||
"""在事件屏障封口后卸载已停用 handler 的插件实例。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
return bool(plugin_manager.finalize_plugins())
|
||||
|
||||
|
||||
async def execute_task(loop, task_func, task_name):
|
||||
"""
|
||||
执行后台任务
|
||||
执行后台任务;取消调用方时仍持有同步线程直到真实完成。
|
||||
"""
|
||||
try:
|
||||
result = await loop.run_in_executor(None, task_func)
|
||||
# loop 参数属于既有调用 ABI;同步执行改由 completion-aware 适配器持有,
|
||||
# 避免外层 Task 被取消后把仍在修改插件源码/依赖的线程伪装成已结束。
|
||||
del loop
|
||||
result = await run_in_threadpool_to_completion(task_func)
|
||||
if isinstance(result, PluginDependencyInstallResult):
|
||||
processed_count = len(result.missing)
|
||||
elif isinstance(result, list):
|
||||
@@ -245,13 +282,15 @@ def init_plugins():
|
||||
"""
|
||||
configure_plugin_services()
|
||||
plugin_manager = PluginManager()
|
||||
if not plugin_manager.reopen_plugins():
|
||||
raise RuntimeError("上一应用生命周期的插件后台服务仍未收敛")
|
||||
classification = plugin_manager.classify_plugins()
|
||||
plugin_manager.apply_plugin_dependency_classification(classification)
|
||||
plugin_manager.set_plugin_settling(True)
|
||||
for plugin_id in classification.ready:
|
||||
plugin_manager.start(plugin_id)
|
||||
register_plugin_api()
|
||||
plugin_manager.start_monitor()
|
||||
plugin_manager.start_monitor(reopen=True)
|
||||
logger.info(
|
||||
"插件启动分类:立即加载=%s,等待依赖=%s,等待源码=%s",
|
||||
len(classification.ready),
|
||||
@@ -260,15 +299,30 @@ def init_plugins():
|
||||
)
|
||||
|
||||
|
||||
def stop_plugins():
|
||||
"""
|
||||
停止插件
|
||||
"""
|
||||
def stop_plugin_monitor(timeout: float = 5.0) -> bool:
|
||||
"""封口已创建管理器的文件监控线程,并返回是否完成收口。"""
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
try:
|
||||
plugin_manager = PluginManager()
|
||||
return bool(plugin_manager.close_monitor(timeout=timeout))
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件文件监控时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
|
||||
def stop_plugins() -> bool:
|
||||
"""停止已创建的插件监控和运行实例,不在停机阶段反向物化管理器。"""
|
||||
try:
|
||||
plugin_manager = PluginManager.get_existing_instance()
|
||||
if plugin_manager is None:
|
||||
return True
|
||||
monitor_stopped = True
|
||||
try:
|
||||
plugin_manager.stop_monitor()
|
||||
monitor_stopped = plugin_manager.stop_monitor()
|
||||
finally:
|
||||
plugin_manager.stop()
|
||||
return bool(monitor_stopped)
|
||||
except Exception as e:
|
||||
logger.error(f"停止插件时发生错误:{e}", exc_info=True)
|
||||
return False
|
||||
|
||||
@@ -11,3 +11,15 @@ def replay_pending_transfers():
|
||||
回放本身在后台线程执行,不阻塞启动流程。
|
||||
"""
|
||||
TransferChain().replay_pending()
|
||||
|
||||
|
||||
async def stop_transfer_runtime(timeout_seconds: float = 30.0) -> bool:
|
||||
"""关闭已存在的整理后台 owner,且不在关停阶段创建新的整理链实例。
|
||||
|
||||
:param timeout_seconds: worker 与 pending 回放共享的最大等待秒数
|
||||
:return: 没有已创建实例或所有整理后台 owner 均已收敛时返回 True
|
||||
"""
|
||||
transfer_chain = TransferChain.get_existing_instance()
|
||||
if transfer_chain is None:
|
||||
return True
|
||||
return await transfer_chain.close(timeout_seconds=timeout_seconds)
|
||||
|
||||
Reference in New Issue
Block a user