mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-06 16:07:01 +08:00
Merge remote-tracking branch 'origin/v3' into v3
# Conflicts: # app/api/endpoints/agent.py # app/api/endpoints/anthropic.py # app/api/endpoints/openai.py # app/chain/__init__.py # app/chain/message.py # app/chain/site.py # app/chain/subscribe.py # app/chain/transfer.py # app/modules/discord/__init__.py # app/modules/qqbot/__init__.py # app/modules/slack/__init__.py # app/modules/telegram/__init__.py # app/modules/wechat/__init__.py # app/runtime/extensions/module_manager.py # app/runtime/extensions/service_registry.py # tests/test_agent_interaction.py # tests/test_slash_command_interactions.py # tests/test_web_agent_stream.py
This commit is contained in:
jxxghp
@@ -0,0 +1,131 @@
|
||||
"""Agent 编排服务门面。
|
||||
|
||||
chain 层需要触发 Agent 后台任务、渲染提示词、查询模型能力时统一经本模块调用。
|
||||
具体实现由 startup 组合根注册，形成依赖倒置：
|
||||
|
||||
chain -> application.agent <- startup -> agent
|
||||
|
||||
门面保存 provider 而非重量级实现对象，注册本身不会物化 Agent、LLM 或工具树。
|
||||
本模块禁止静态或函数内导入 app.agent，否则会重新形成跨层循环依赖。
|
||||
"""
|
||||
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
Provider = Callable[[], Any]
|
||||
|
||||
# provider 注册表由 startup/agent_initializer 在组合根装配。
|
||||
_agent_manager_provider: Optional[Provider] = None
|
||||
_running_agent_manager_provider: Optional[Provider] = None
|
||||
_prompt_manager_provider: Optional[Provider] = None
|
||||
_agent_capability_manager_provider: Optional[Provider] = None
|
||||
_llm_helper_provider: Optional[Provider] = None
|
||||
_manual_redo_prompt_builder_provider: Optional[Provider] = None
|
||||
|
||||
|
||||
def register_agent_service_providers(
|
||||
*,
|
||||
agent_manager_provider: Provider,
|
||||
running_agent_manager_provider: Provider,
|
||||
prompt_manager_provider: Provider,
|
||||
capability_manager_provider: Provider,
|
||||
llm_helper_provider: Provider,
|
||||
manual_redo_prompt_builder_provider: Provider,
|
||||
) -> None:
|
||||
"""注册 Agent 服务 provider，保持组合根装配阶段零重量实现导入。"""
|
||||
global _agent_manager_provider, _running_agent_manager_provider
|
||||
global _prompt_manager_provider, _agent_capability_manager_provider
|
||||
global _llm_helper_provider, _manual_redo_prompt_builder_provider
|
||||
_agent_manager_provider = agent_manager_provider
|
||||
_running_agent_manager_provider = running_agent_manager_provider
|
||||
_prompt_manager_provider = prompt_manager_provider
|
||||
_agent_capability_manager_provider = capability_manager_provider
|
||||
_llm_helper_provider = llm_helper_provider
|
||||
_manual_redo_prompt_builder_provider = manual_redo_prompt_builder_provider
|
||||
|
||||
|
||||
def register_agent_services(
|
||||
agent_manager: Any,
|
||||
prompt_manager: Any,
|
||||
capability_manager: Any,
|
||||
llm_helper: Any,
|
||||
manual_redo_prompt_builder: Optional[Callable[[Any], str]] = None,
|
||||
) -> None:
|
||||
"""兼容直接对象注入；生产组合根应注册惰性 provider。"""
|
||||
register_agent_service_providers(
|
||||
agent_manager_provider=lambda: agent_manager,
|
||||
running_agent_manager_provider=lambda: agent_manager,
|
||||
prompt_manager_provider=lambda: prompt_manager,
|
||||
capability_manager_provider=lambda: capability_manager,
|
||||
llm_helper_provider=lambda: llm_helper,
|
||||
manual_redo_prompt_builder_provider=lambda: manual_redo_prompt_builder,
|
||||
)
|
||||
|
||||
|
||||
def _resolve(provider: Optional[Provider], service_name: str) -> Any:
|
||||
"""解析已注册服务；缺少组合根装配时给出稳定错误。"""
|
||||
if provider is None:
|
||||
raise RuntimeError(
|
||||
f"Agent 服务 {service_name} 未注册："
|
||||
"请先导入 app.startup.agent_initializer 完成组合根装配"
|
||||
)
|
||||
return provider()
|
||||
|
||||
|
||||
def get_agent_manager() -> Any:
|
||||
"""返回 canonical AgentManager；调用可能触发实现物化。"""
|
||||
return _resolve(_agent_manager_provider, "agent_manager")
|
||||
|
||||
|
||||
def get_running_agent_manager() -> Any | None:
|
||||
"""返回已进入 RUNNING 的 AgentManager，不触发实现物化。"""
|
||||
return _resolve(_running_agent_manager_provider, "running_agent_manager")
|
||||
|
||||
|
||||
def get_prompt_manager() -> Any:
|
||||
"""按需返回提示词管理器。"""
|
||||
return _resolve(_prompt_manager_provider, "prompt_manager")
|
||||
|
||||
|
||||
def supports_image_input(
|
||||
provider: Optional[str] = None,
|
||||
model: Optional[str] = None,
|
||||
base_url: Optional[str] = None,
|
||||
base_url_preset: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""判断当前模型是否启用了图片输入能力。"""
|
||||
llm_helper = _resolve(_llm_helper_provider, "llm_helper")
|
||||
return llm_helper.supports_image_input(
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
base_url_preset=base_url_preset,
|
||||
)
|
||||
|
||||
|
||||
def is_audio_input_available() -> bool:
|
||||
"""判断语音输入能力是否可用。"""
|
||||
capability_manager = _resolve(
|
||||
_agent_capability_manager_provider,
|
||||
"agent_capability_manager",
|
||||
)
|
||||
return capability_manager.is_audio_input_available()
|
||||
|
||||
|
||||
def transcribe_audio(content: bytes, filename: str = "input.ogg") -> Optional[str]:
|
||||
"""把音频内容转写为文本。"""
|
||||
capability_manager = _resolve(
|
||||
_agent_capability_manager_provider,
|
||||
"agent_capability_manager",
|
||||
)
|
||||
return capability_manager.transcribe_audio(content, filename=filename)
|
||||
|
||||
|
||||
def build_manual_redo_prompt(history: Any) -> str:
|
||||
"""构造整理记录 AI 重新整理提示词（builder 由 agent 层注册）。"""
|
||||
builder = _resolve(
|
||||
_manual_redo_prompt_builder_provider,
|
||||
"manual_redo_prompt_builder",
|
||||
)
|
||||
if builder is None:
|
||||
raise RuntimeError("整理记录重新整理提示词构建器未注册")
|
||||
return builder(history)
|
||||
@@ -0,0 +1,45 @@
|
||||
"""命令工具服务门面。
|
||||
|
||||
Agent 工具与 API 端点对命令注册表的操作统一经本模块调用，
|
||||
Command 实现由 startup 组合根在导入期注册，避免 application 层
|
||||
静态依赖顶层 command 模块。
|
||||
|
||||
依赖方向：
|
||||
|
||||
agent.tools / api.endpoints -> application.commands <- startup（注册 Command 类）
|
||||
"""
|
||||
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# Command 类：由 startup/command_initializer 在导入期注册。
|
||||
_command_class: Any = None
|
||||
|
||||
|
||||
def register_command_class(command_class: Any) -> None:
|
||||
"""注册 Command 类（组合根在导入期调用）。"""
|
||||
global _command_class
|
||||
_command_class = command_class
|
||||
|
||||
|
||||
def get_command_object() -> Any:
|
||||
"""返回命令注册表实例。"""
|
||||
if _command_class is None:
|
||||
raise RuntimeError(
|
||||
"命令服务未初始化：请先通过 register_command_class 注册 Command 类"
|
||||
)
|
||||
return _command_class()
|
||||
|
||||
|
||||
def get_commands() -> Dict[str, Any]:
|
||||
"""返回全部已注册命令。"""
|
||||
return get_command_object().get_commands()
|
||||
|
||||
|
||||
def get_command(name: str) -> Optional[Any]:
|
||||
"""按命令名查询注册表。"""
|
||||
return get_command_object().get(name)
|
||||
|
||||
|
||||
def init_commands(plugin_id: Optional[str] = None) -> None:
|
||||
"""初始化命令（可指定单个插件）。"""
|
||||
get_command_object().init_commands(plugin_id)
|
||||
@@ -1,66 +0,0 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.domain.context import MediaInfo
|
||||
from app.schemas import CustomRule, FilterRuleGroup
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class RuleHelper:
|
||||
"""读取用户过滤规则配置，并按媒体上下文选择适用规则组。"""
|
||||
|
||||
@staticmethod
|
||||
def get_rule_groups() -> List[FilterRuleGroup]:
|
||||
"""返回用户配置的全部过滤规则组。"""
|
||||
rule_groups: List[dict] = SystemConfigOper().get(
|
||||
SystemConfigKey.UserFilterRuleGroups
|
||||
)
|
||||
if not rule_groups:
|
||||
return []
|
||||
return [FilterRuleGroup(**group) for group in rule_groups]
|
||||
|
||||
def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]:
|
||||
"""按名称返回过滤规则组。"""
|
||||
return next(
|
||||
(group for group in self.get_rule_groups() if group.name == group_name),
|
||||
None,
|
||||
)
|
||||
|
||||
def get_rule_group_by_media(
|
||||
self,
|
||||
media: Optional[MediaInfo] = None,
|
||||
group_names: Optional[list] = None,
|
||||
) -> List[FilterRuleGroup]:
|
||||
"""按媒体类型、分类和候选名称筛选适用规则组。"""
|
||||
rule_groups = self.get_rule_groups()
|
||||
if group_names:
|
||||
rule_groups = [
|
||||
group for group in rule_groups if group.name in group_names
|
||||
]
|
||||
return [
|
||||
group
|
||||
for group in rule_groups
|
||||
if not group.media_type
|
||||
or (
|
||||
media
|
||||
and (
|
||||
(not group.category and group.media_type == media.type.value)
|
||||
or group.category == media.category
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_custom_rules() -> List[CustomRule]:
|
||||
"""返回用户配置的全部自定义过滤规则。"""
|
||||
rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules)
|
||||
if not rules:
|
||||
return []
|
||||
return [CustomRule(**rule) for rule in rules]
|
||||
|
||||
def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]:
|
||||
"""按 ID 返回一条自定义过滤规则。"""
|
||||
return next(
|
||||
(rule for rule in self.get_custom_rules() if rule.id == rule_id),
|
||||
None,
|
||||
)
|
||||
@@ -0,0 +1,190 @@
|
||||
"""插件 API 动态路由服务。
|
||||
|
||||
把插件 API 的动态注册/移除从 HTTP 端点层下沉到 application 层：
|
||||
FastAPI 实例由组合根（factory 创建应用后）注入，端点与 Agent 工具
|
||||
统一经本模块操作路由，消除 api.endpoints 对 factory 的反向依赖。
|
||||
|
||||
依赖方向：
|
||||
|
||||
api.endpoints.plugin / agent.tools -> application.plugins <- factory（注入实例）
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, FastAPI
|
||||
|
||||
from app.application.security.access import verify_apikey, verify_token
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
PROTECTED_ROUTES = {
|
||||
"/api/v1/openapi.json",
|
||||
"/docs",
|
||||
"/docs/oauth2-redirect",
|
||||
"/redoc",
|
||||
}
|
||||
PLUGIN_PREFIX = f"{settings.API_V1_STR}/plugin"
|
||||
|
||||
# FastAPI 应用实例：由 factory 在创建应用后调用 register_api_app 注入。
|
||||
_api_app: Optional[FastAPI] = None
|
||||
|
||||
|
||||
def register_api_app(api_app: FastAPI) -> None:
|
||||
"""注入 FastAPI 应用实例（组合根在创建应用后调用）。"""
|
||||
global _api_app
|
||||
_api_app = api_app
|
||||
|
||||
|
||||
def get_api_app() -> FastAPI:
|
||||
"""返回已注入的 FastAPI 应用实例。"""
|
||||
if _api_app is None:
|
||||
raise RuntimeError("插件路由服务未初始化：请先调用 register_api_app 注入应用实例")
|
||||
return _api_app
|
||||
|
||||
|
||||
def register_plugin_api(plugin_id: Optional[str] = None):
|
||||
"""
|
||||
动态注册插件 API
|
||||
:param plugin_id: 插件 ID，如果为 None，则注册所有插件
|
||||
"""
|
||||
_update_plugin_api_routes(plugin_id, action="add")
|
||||
|
||||
|
||||
def remove_plugin_api(plugin_id: str):
|
||||
"""
|
||||
动态移除单个插件的 API
|
||||
:param plugin_id: 插件 ID
|
||||
"""
|
||||
_update_plugin_api_routes(plugin_id, action="remove")
|
||||
|
||||
|
||||
def _update_plugin_api_routes(plugin_id: Optional[str], action: str):
|
||||
"""
|
||||
插件 API 路由注册和移除
|
||||
:param plugin_id: 插件 ID，如果 action 为 "add" 且 plugin_id 为 None，则处理所有插件
|
||||
如果 action 为 "remove"，plugin_id 必须是有效的插件 ID
|
||||
:param action: "add" 或 "remove"，决定是添加还是移除路由
|
||||
"""
|
||||
if action not in {"add", "remove"}:
|
||||
raise ValueError("Action must be 'add' or 'remove'")
|
||||
|
||||
app = get_api_app()
|
||||
is_modified = False
|
||||
existing_paths = {route.path: route for route in app.routes}
|
||||
|
||||
plugin_ids = [plugin_id] if plugin_id else PluginManager().get_running_plugin_ids()
|
||||
for plugin_id in plugin_ids:
|
||||
routes_removed = _remove_routes(plugin_id)
|
||||
if routes_removed:
|
||||
is_modified = True
|
||||
|
||||
if action != "add":
|
||||
continue
|
||||
# 获取插件的 API 路由信息
|
||||
plugin_apis = PluginManager().get_plugin_apis(plugin_id)
|
||||
for api in plugin_apis:
|
||||
api_path = f"{PLUGIN_PREFIX}{api.get('path', '')}"
|
||||
try:
|
||||
api["path"] = api_path
|
||||
allow_anonymous = api.pop("allow_anonymous", False)
|
||||
auth_mode = api.pop("auth", "apikey")
|
||||
dependencies = api.setdefault("dependencies", [])
|
||||
if not allow_anonymous:
|
||||
if (
|
||||
auth_mode == "bear"
|
||||
and Depends(verify_token) not in dependencies
|
||||
):
|
||||
dependencies.append(Depends(verify_token))
|
||||
elif Depends(verify_apikey) not in dependencies:
|
||||
dependencies.append(Depends(verify_apikey))
|
||||
app.add_api_route(**api, tags=["plugin"])
|
||||
is_modified = True
|
||||
logger.debug(f"Added plugin route: {api_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error adding plugin route {api_path}: {str(e)}")
|
||||
|
||||
if is_modified:
|
||||
_clean_protected_routes(existing_paths)
|
||||
app.openapi_schema = None
|
||||
app.setup()
|
||||
|
||||
|
||||
def _remove_routes(plugin_id: str) -> bool:
|
||||
"""
|
||||
移除与单个插件相关的路由
|
||||
:param plugin_id: 插件 ID
|
||||
:return: 是否有路由被移除
|
||||
"""
|
||||
if not plugin_id:
|
||||
return False
|
||||
app = get_api_app()
|
||||
prefix = f"{PLUGIN_PREFIX}/{plugin_id}/"
|
||||
routes_to_remove = [
|
||||
route for route in app.routes if route.path.startswith(prefix)
|
||||
]
|
||||
removed = False
|
||||
for route in routes_to_remove:
|
||||
try:
|
||||
app.routes.remove(route)
|
||||
removed = True
|
||||
logger.debug(f"Removed plugin route: {route.path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing plugin route {route.path}: {str(e)}")
|
||||
return removed
|
||||
|
||||
|
||||
def _clean_protected_routes(existing_paths: dict):
|
||||
"""
|
||||
清理受保护的路由，防止在插件操作中被删除或重复添加
|
||||
:param existing_paths: 当前应用的路由路径映射
|
||||
"""
|
||||
app = get_api_app()
|
||||
for protected_route in PROTECTED_ROUTES:
|
||||
try:
|
||||
existing_route = existing_paths.get(protected_route)
|
||||
if existing_route:
|
||||
app.routes.remove(existing_route)
|
||||
except Exception as e:
|
||||
logger.error(f"Error removing protected route {protected_route}: {str(e)}")
|
||||
|
||||
|
||||
def remove_plugin_from_folders(plugin_id: str):
|
||||
"""
|
||||
从所有文件夹中移除指定的插件
|
||||
:param plugin_id: 要移除的插件ID
|
||||
"""
|
||||
try:
|
||||
config_oper = SystemConfigOper()
|
||||
# 获取插件文件夹配置
|
||||
folders = config_oper.get(SystemConfigKey.PluginFolders) or {}
|
||||
|
||||
# 标记是否有修改
|
||||
modified = False
|
||||
|
||||
# 遍历所有文件夹，移除指定插件
|
||||
for folder_name, folder_data in folders.items():
|
||||
if isinstance(folder_data, dict) and "plugins" in folder_data:
|
||||
# 新格式：{"plugins": [...], "order": ..., "icon": ...}
|
||||
if plugin_id in folder_data["plugins"]:
|
||||
folder_data["plugins"].remove(plugin_id)
|
||||
logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}")
|
||||
modified = True
|
||||
elif isinstance(folder_data, list):
|
||||
# 旧格式：直接是插件列表
|
||||
if plugin_id in folder_data:
|
||||
folder_data.remove(plugin_id)
|
||||
logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}")
|
||||
modified = True
|
||||
|
||||
# 如果有修改，保存更新后的文件夹配置
|
||||
if modified:
|
||||
config_oper.set(SystemConfigKey.PluginFolders, folders)
|
||||
else:
|
||||
logger.debug(f"插件 {plugin_id} 不在任何文件夹中，无需移除")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"从文件夹中移除插件时出错：{str(e)}")
|
||||
# 文件夹处理失败不影响插件卸载的整体流程
|
||||
@@ -1,11 +1,79 @@
|
||||
"""过滤规则解析器与内置规则定义，过滤模块与 Agent 工具共享同一事实来源。"""
|
||||
"""
|
||||
规则域：用户规则组配置访问、内置规则定义与规则解析器，
|
||||
过滤模块与 Agent 工具共享同一事实来源。
|
||||
"""
|
||||
|
||||
import threading
|
||||
from typing import Dict
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from pyparsing import Forward, Literal, Word, alphas, infix_notation, opAssoc, alphanums, Combine, nums, ParseResults
|
||||
|
||||
from app.adapters.system import rust as rust_accel
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.domain.context import MediaInfo
|
||||
from app.schemas import CustomRule, FilterRuleGroup
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class RuleHelper:
|
||||
"""读取用户过滤规则配置，并按媒体上下文选择适用规则组。"""
|
||||
|
||||
@staticmethod
|
||||
def get_rule_groups() -> List[FilterRuleGroup]:
|
||||
"""返回用户配置的全部过滤规则组。"""
|
||||
rule_groups: List[dict] = SystemConfigOper().get(
|
||||
SystemConfigKey.UserFilterRuleGroups
|
||||
)
|
||||
if not rule_groups:
|
||||
return []
|
||||
return [FilterRuleGroup(**group) for group in rule_groups]
|
||||
|
||||
def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]:
|
||||
"""按名称返回过滤规则组。"""
|
||||
return next(
|
||||
(group for group in self.get_rule_groups() if group.name == group_name),
|
||||
None,
|
||||
)
|
||||
|
||||
def get_rule_group_by_media(
|
||||
self,
|
||||
media: Optional[MediaInfo] = None,
|
||||
group_names: Optional[list] = None,
|
||||
) -> List[FilterRuleGroup]:
|
||||
"""按媒体类型、分类和候选名称筛选适用规则组。"""
|
||||
rule_groups = self.get_rule_groups()
|
||||
if group_names:
|
||||
rule_groups = [
|
||||
group for group in rule_groups if group.name in group_names
|
||||
]
|
||||
return [
|
||||
group
|
||||
for group in rule_groups
|
||||
if not group.media_type
|
||||
or (
|
||||
media
|
||||
and (
|
||||
(not group.category and group.media_type == media.type.value)
|
||||
or group.category == media.category
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_custom_rules() -> List[CustomRule]:
|
||||
"""返回用户配置的全部自定义过滤规则。"""
|
||||
rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules)
|
||||
if not rules:
|
||||
return []
|
||||
return [CustomRule(**rule) for rule in rules]
|
||||
|
||||
def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]:
|
||||
"""按 ID 返回一条自定义过滤规则。"""
|
||||
return next(
|
||||
(rule for rule in self.get_custom_rules() if rule.id == rule_id),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
# 内置规则只在这里维护一份，便于过滤模块和 Agent 工具共享同一套事实来源。
|
||||
BUILTIN_RULE_SET: Dict[str, dict] = {
|
||||
@@ -0,0 +1,73 @@
|
||||
"""调度器工具服务门面。
|
||||
|
||||
Agent 工具与 API 端点对运行时调度器的操作统一经本模块调用，
|
||||
Scheduler 实现由 startup 组合根在导入期注册，避免 application 层
|
||||
静态依赖顶层 scheduler 模块（scheduler 反向依赖 chain，会成环）。
|
||||
|
||||
依赖方向：
|
||||
|
||||
agent.tools / api.endpoints -> application.scheduling <- startup（注册 Scheduler 类）
|
||||
"""
|
||||
|
||||
from typing import Any, List, Optional
|
||||
|
||||
# Agent 自主定时任务在运行时调度器中的任务 ID 前缀。
|
||||
AGENT_TASK_JOB_PREFIX = "agent-task"
|
||||
|
||||
# Scheduler 类：由 startup/scheduler_initializer 在导入期注册。
|
||||
_scheduler_class: Any = None
|
||||
|
||||
|
||||
def register_scheduler_class(scheduler_class: Any) -> None:
|
||||
"""注册 Scheduler 类（组合根在导入期调用）。"""
|
||||
global _scheduler_class
|
||||
_scheduler_class = scheduler_class
|
||||
|
||||
|
||||
def get_scheduler() -> Any:
|
||||
"""返回调度器实例。"""
|
||||
if _scheduler_class is None:
|
||||
raise RuntimeError(
|
||||
"调度器服务未初始化：请先通过 register_scheduler_class 注册 Scheduler 类"
|
||||
)
|
||||
return _scheduler_class()
|
||||
|
||||
|
||||
def list_scheduler_jobs() -> List[Any]:
|
||||
"""列出运行时调度器的全部任务。"""
|
||||
return get_scheduler().list()
|
||||
|
||||
|
||||
def start_scheduler_job(job_id: str) -> None:
|
||||
"""立即运行指定的运行时定时任务。"""
|
||||
get_scheduler().start(job_id)
|
||||
|
||||
|
||||
def update_plugin_job(plugin_id: str) -> None:
|
||||
"""更新插件的定时任务。"""
|
||||
get_scheduler().update_plugin_job(plugin_id)
|
||||
|
||||
|
||||
def remove_plugin_job(plugin_id: str) -> None:
|
||||
"""移除插件的定时任务。"""
|
||||
get_scheduler().remove_plugin_job(plugin_id)
|
||||
|
||||
|
||||
def start_agent_task(task_id: int) -> bool:
|
||||
"""立即执行 Agent 自主定时任务。"""
|
||||
return get_scheduler().start_agent_task(task_id)
|
||||
|
||||
|
||||
def get_agent_task_next_run(task_id: int) -> Optional[Any]:
|
||||
"""查询 Agent 自主定时任务的下一次运行时间。"""
|
||||
return get_scheduler().get_agent_task_next_run(task_id)
|
||||
|
||||
|
||||
def update_agent_task_job(task_id: int) -> Optional[Any]:
|
||||
"""更新 Agent 自主定时任务的注册信息，返回下一次运行时间。"""
|
||||
return get_scheduler().update_agent_task_job(task_id)
|
||||
|
||||
|
||||
def remove_agent_task_job(task_id: int) -> None:
|
||||
"""移除 Agent 自主定时任务的注册信息。"""
|
||||
get_scheduler().remove_agent_task_job(task_id)
|
||||
+891
-4
@@ -13,20 +13,38 @@ app.schemas -> app.schemas.transfer -> app.domain.* -> app.schemas.types -> app.
|
||||
TransferJob / TransferJobTask，那两个用 app.schemas 的同名 DTO——一个是工作项，一个是
|
||||
视图，分开表达之后两边都不必再迁就对方。
|
||||
"""
|
||||
import asyncio
|
||||
import threading
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Callable, List, Optional, Union
|
||||
from time import monotonic
|
||||
from typing import Callable, Dict, List, Optional, Tuple, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from app import schemas
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.application.agent import get_prompt_manager, get_running_agent_manager
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.media import normalize_music_type
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
from app.foundation import text as text_tools
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.agent import ReplyMode
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.history import DownloadHistory
|
||||
from app.schemas.media import OptionalMediaIdentityMixin
|
||||
from app.schemas.media import OptionalMediaIdentityMixin, resolve_media_identity
|
||||
from app.schemas.system import TransferDirectoryConf
|
||||
from app.schemas.tmdb import TmdbEpisode
|
||||
from app.schemas.transfer import TransferInfo
|
||||
from app.schemas.types import MediaSource, MediaType
|
||||
from app.schemas.transfer import TransferInfo, TransferJob, TransferJobTask
|
||||
from app.schemas.types import (
|
||||
MUSIC_ENTITY_ALBUM,
|
||||
MUSIC_ENTITY_RECORDING,
|
||||
MediaSource,
|
||||
MediaType,
|
||||
)
|
||||
|
||||
|
||||
|
||||
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
@@ -89,3 +107,872 @@ class TransferQueue(BaseModel):
|
||||
callback: Optional[Callable] = None
|
||||
# 整理结果
|
||||
result: Optional[TransferInfo] = None
|
||||
|
||||
|
||||
# 作业锁：JobManager 与 TransferChain 共享，保护整理作业视图。
|
||||
job_lock = threading.Lock()
|
||||
|
||||
class JobManager:
|
||||
"""
|
||||
作业管理器
|
||||
task任务负责一个文件的整理，job作业负责一个媒体的整理
|
||||
"""
|
||||
|
||||
# 整理中的作业
|
||||
_job_view: Dict[Tuple, TransferJob] = {}
|
||||
# 汇总季集清单
|
||||
_season_episodes: Dict[Tuple, List[int]] = {}
|
||||
# 记录从 meta 作业迁移到 media 作业的关系，用于清理提前失败后残留的 media 作业
|
||||
_meta_to_media_ids: Dict[Tuple, set[Tuple]] = {}
|
||||
# 记录任务最近一次状态心跳，供外部异步接管任务的失活检测使用
|
||||
_task_state_changed_at: Dict[Tuple[str, str], float] = {}
|
||||
# 记录仍由主程序整理线程直接执行的任务，避免把阻塞中的本地任务误判为失活
|
||||
_active_executions: set[Tuple[str, str]] = set()
|
||||
|
||||
def __init__(self):
|
||||
self._job_view = {}
|
||||
self._season_episodes = {}
|
||||
self._meta_to_media_ids = {}
|
||||
self._task_state_changed_at = {}
|
||||
self._active_executions = set()
|
||||
|
||||
@staticmethod
|
||||
def __get_meta_id(meta: MetaBase = None, season: Optional[int] = None) -> Tuple:
|
||||
"""
|
||||
获取元数据ID
|
||||
"""
|
||||
return meta.name, season
|
||||
|
||||
@staticmethod
|
||||
def __get_media_id(media: Optional[Union[MediaInfo, MusicInfo]] = None,
|
||||
season: Optional[int] = None) -> Tuple:
|
||||
"""
|
||||
获取媒体ID；音乐额外区分实体类型，并为无远端ID的曲目构造稳定身份。
|
||||
"""
|
||||
if not media:
|
||||
return None, season
|
||||
source, media_id = resolve_media_identity(media=media)
|
||||
if getattr(media, "type", None) == MediaType.MUSIC:
|
||||
music_type = normalize_music_type(
|
||||
getattr(media, "music_type", None),
|
||||
) or MUSIC_ENTITY_RECORDING
|
||||
if source and media_id:
|
||||
return "music", source, media_id, music_type
|
||||
|
||||
artists = tuple(
|
||||
text_tools.normalize_upper(artist)
|
||||
for artist in (getattr(media, "artists", None) or [])
|
||||
if text_tools.normalize_upper(artist)
|
||||
)
|
||||
if music_type == MUSIC_ENTITY_ALBUM:
|
||||
album_artist = text_tools.normalize_upper(
|
||||
getattr(media, "album_artist", None)
|
||||
or (artists[0] if artists else "")
|
||||
)
|
||||
album = text_tools.normalize_upper(
|
||||
getattr(media, "album", None) or getattr(media, "title", None) or ""
|
||||
)
|
||||
return "music", "local", music_type, album_artist, album, getattr(media, "year", None)
|
||||
|
||||
return (
|
||||
"music",
|
||||
"local",
|
||||
music_type,
|
||||
artists,
|
||||
text_tools.normalize_upper(getattr(media, "title", None) or ""),
|
||||
text_tools.normalize_upper(getattr(media, "album", None) or ""),
|
||||
getattr(media, "disc_number", None),
|
||||
getattr(media, "track_number", None),
|
||||
)
|
||||
return (source, media_id), season
|
||||
|
||||
@staticmethod
|
||||
def __get_file_key(fileitem: FileItem) -> Optional[Tuple[str, str]]:
|
||||
"""
|
||||
获取源文件唯一键，用于跨媒体作业识别同一个整理任务。
|
||||
"""
|
||||
if not fileitem or not fileitem.path:
|
||||
return None
|
||||
normalized_path = (
|
||||
Path(str(fileitem.path).replace("\\", "/")).as_posix().rstrip("/") or "/"
|
||||
)
|
||||
return fileitem.storage or "local", normalized_path
|
||||
|
||||
def __get_id(self, task: TransferTask = None) -> Tuple:
|
||||
"""
|
||||
获取作业ID
|
||||
"""
|
||||
if task.mediainfo:
|
||||
return self.__get_media_id(
|
||||
media=task.mediainfo, season=task.meta.begin_season
|
||||
)
|
||||
else:
|
||||
return self.__get_meta_id(meta=task.meta, season=task.meta.begin_season)
|
||||
|
||||
def get_job_id(self, task: TransferTask) -> Tuple:
|
||||
"""返回任务当前所属的稳定作业身份，供作业级附加状态隔离使用。"""
|
||||
return self.__get_id(task)
|
||||
|
||||
@staticmethod
|
||||
def __get_media(task: TransferTask) -> Union[schemas.MediaInfo, schemas.MusicInfo]:
|
||||
"""
|
||||
获取媒体信息
|
||||
"""
|
||||
if task.mediainfo:
|
||||
# 有媒体信息
|
||||
mediainfo = deepcopy(task.mediainfo)
|
||||
mediainfo.clear()
|
||||
if isinstance(mediainfo, MusicInfo):
|
||||
return schemas.MusicInfo(**mediainfo.to_dict())
|
||||
return schemas.MediaInfo(**mediainfo.to_dict())
|
||||
else:
|
||||
# 没有媒体信息
|
||||
meta: MetaBase = task.meta
|
||||
if isinstance(meta, MetaMusic):
|
||||
# 未识别的音乐按已解析元数据兜底展示；音乐年份为 int，
|
||||
# 不能复用 MediaInfo（year 为 str），否则触发 pydantic 校验异常
|
||||
return schemas.MusicInfo(
|
||||
title=meta.name,
|
||||
artists=list(meta.artists or []),
|
||||
artist=meta.artist,
|
||||
album=meta.album,
|
||||
album_artist=meta.album_artist,
|
||||
year=meta.year,
|
||||
title_year=f"{meta.name} ({meta.year})" if meta.year else meta.name,
|
||||
media_source=meta.media_source,
|
||||
media_id=meta.media_id,
|
||||
)
|
||||
return schemas.MediaInfo(
|
||||
title=meta.name,
|
||||
year=meta.year,
|
||||
title_year=f"{meta.name} ({meta.year})",
|
||||
type=meta.type.value if meta.type else None,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __get_meta(task: TransferTask) -> schemas.MetaInfo:
|
||||
"""
|
||||
获取元数据
|
||||
"""
|
||||
if isinstance(task.meta, MetaMusic):
|
||||
return schemas.MusicMeta(**task.meta.to_dict())
|
||||
return schemas.MetaInfo(**task.meta.to_dict())
|
||||
|
||||
def add_task(self, task: TransferTask, state: Optional[str] = "waiting") -> bool:
|
||||
"""
|
||||
添加整理任务，自动分组到对应的作业中
|
||||
:return: True表示任务已添加，False表示任务无效或已存在（重复）
|
||||
"""
|
||||
if not all([task, task.meta, task.fileitem]):
|
||||
return False
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if not file_key:
|
||||
return False
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_id(task)
|
||||
# 同一个源文件可能在识别前后落入不同作业，必须跨作业去重。
|
||||
if any(
|
||||
self.__get_file_key(t.fileitem) == file_key
|
||||
for job in self._job_view.values()
|
||||
for t in job.tasks
|
||||
):
|
||||
logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加")
|
||||
return False
|
||||
if __mediaid__ not in self._job_view:
|
||||
self._job_view[__mediaid__] = TransferJob(
|
||||
media=self.__get_media(task),
|
||||
season=task.meta.begin_season,
|
||||
tasks=[
|
||||
TransferJobTask(
|
||||
fileitem=task.fileitem,
|
||||
meta=self.__get_meta(task),
|
||||
downloader=task.downloader,
|
||||
download_hash=task.download_hash,
|
||||
state=state,
|
||||
)
|
||||
],
|
||||
)
|
||||
else:
|
||||
# 不重复添加任务
|
||||
if any(
|
||||
[
|
||||
self.__get_file_key(t.fileitem) == file_key
|
||||
for t in self._job_view[__mediaid__].tasks
|
||||
]
|
||||
):
|
||||
logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加")
|
||||
return False
|
||||
self._job_view[__mediaid__].tasks.append(
|
||||
TransferJobTask(
|
||||
fileitem=task.fileitem,
|
||||
meta=self.__get_meta(task),
|
||||
downloader=task.downloader,
|
||||
download_hash=task.download_hash,
|
||||
state=state,
|
||||
)
|
||||
)
|
||||
self._task_state_changed_at[file_key] = monotonic()
|
||||
# 添加季集信息
|
||||
if self._season_episodes.get(__mediaid__):
|
||||
self._season_episodes[__mediaid__].extend(task.meta.episode_list)
|
||||
self._season_episodes[__mediaid__] = list(
|
||||
set(self._season_episodes[__mediaid__])
|
||||
)
|
||||
else:
|
||||
self._season_episodes[__mediaid__] = task.meta.episode_list
|
||||
return True
|
||||
|
||||
def migrate_task(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
将任务从 meta 作业迁移到 media 作业
|
||||
"""
|
||||
curr_task, source_job_id = self.__remove_task_with_job_id(
|
||||
task.fileitem, preserve_execution=True
|
||||
)
|
||||
if not self.add_task(task, state=curr_task.state if curr_task else "waiting"):
|
||||
return False
|
||||
if curr_task and task.mediainfo:
|
||||
metaid = self.__get_meta_id(
|
||||
meta=task.meta, season=task.meta.begin_season
|
||||
)
|
||||
mediaid = self.__get_id(task)
|
||||
if source_job_id == metaid and mediaid != metaid:
|
||||
with job_lock:
|
||||
self._meta_to_media_ids.setdefault(metaid, set()).add(mediaid)
|
||||
return True
|
||||
|
||||
def __is_job_done(self, job_id: Tuple) -> bool:
|
||||
"""
|
||||
检查指定作业是否已完成
|
||||
"""
|
||||
if job_id not in self._job_view:
|
||||
return True
|
||||
return all(
|
||||
task.state in ["completed", "failed"]
|
||||
for task in self._job_view[job_id].tasks
|
||||
)
|
||||
|
||||
def __pop_job(self, job_id: Tuple):
|
||||
"""
|
||||
移除指定作业和对应季集缓存
|
||||
"""
|
||||
job = self._job_view.pop(job_id, None)
|
||||
self._season_episodes.pop(job_id, None)
|
||||
if not job:
|
||||
return
|
||||
for task in job.tasks:
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if file_key:
|
||||
self._task_state_changed_at.pop(file_key, None)
|
||||
self._active_executions.discard(file_key)
|
||||
|
||||
def __remove_done_job_groups(self, job_ids: set[Tuple]):
|
||||
"""
|
||||
清理已进入终态的独立作业或关联作业组。
|
||||
"""
|
||||
candidates = set(job_ids)
|
||||
for metaid, mediaids in list(self._meta_to_media_ids.items()):
|
||||
related_ids = {metaid, *mediaids}
|
||||
if not related_ids.intersection(candidates):
|
||||
continue
|
||||
if all(self.__is_job_done(job_id) for job_id in related_ids):
|
||||
for job_id in related_ids:
|
||||
self.__pop_job(job_id)
|
||||
self._meta_to_media_ids.pop(metaid, None)
|
||||
candidates.difference_update(related_ids)
|
||||
|
||||
referenced_ids = {
|
||||
job_id
|
||||
for metaid, mediaids in self._meta_to_media_ids.items()
|
||||
for job_id in {metaid, *mediaids}
|
||||
}
|
||||
for job_id in candidates - referenced_ids:
|
||||
if self.__is_job_done(job_id):
|
||||
self.__pop_job(job_id)
|
||||
|
||||
def start_execution(self, task: TransferTask):
|
||||
"""
|
||||
标记任务仍由主程序整理线程直接执行。
|
||||
|
||||
:param task: 整理任务
|
||||
"""
|
||||
if not task or not task.fileitem:
|
||||
return
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if not file_key:
|
||||
return
|
||||
with job_lock:
|
||||
self._active_executions.add(file_key)
|
||||
|
||||
def finish_execution(self, task: TransferTask):
|
||||
"""
|
||||
结束主程序整理线程对任务的直接执行标记。
|
||||
|
||||
:param task: 整理任务
|
||||
"""
|
||||
if not task or not task.fileitem:
|
||||
return
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if not file_key:
|
||||
return
|
||||
with job_lock:
|
||||
self._active_executions.discard(file_key)
|
||||
|
||||
def expire_stale_running_tasks(
|
||||
self, timeout_seconds: int
|
||||
) -> List[Tuple[FileItem, int]]:
|
||||
"""
|
||||
将外部接管后长期无心跳的运行中任务标记失败并清理作业视图。
|
||||
|
||||
主程序整理线程仍在直接执行的任务不会被清理，以免把阻塞中的真实任务
|
||||
误报为已终止。外部接管方可重复调用 ``running_task`` 刷新状态心跳。
|
||||
|
||||
:param timeout_seconds: 失活超时秒数，小于等于 0 时禁用
|
||||
:return: 已失活任务及其无心跳秒数
|
||||
"""
|
||||
if timeout_seconds <= 0:
|
||||
return []
|
||||
|
||||
current_time = monotonic()
|
||||
expired: List[Tuple[FileItem, int]] = []
|
||||
affected_job_ids: set[Tuple] = set()
|
||||
with job_lock:
|
||||
for mediaid, job in self._job_view.items():
|
||||
for task in job.tasks:
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if (
|
||||
not file_key
|
||||
or task.state != "running"
|
||||
or file_key in self._active_executions
|
||||
):
|
||||
continue
|
||||
updated_at = self._task_state_changed_at.get(file_key, current_time)
|
||||
inactive_seconds = current_time - updated_at
|
||||
if inactive_seconds < timeout_seconds:
|
||||
continue
|
||||
task.state = "failed"
|
||||
self._task_state_changed_at[file_key] = current_time
|
||||
episodes = getattr(task.meta, "episode_list", None) or []
|
||||
if mediaid in self._season_episodes:
|
||||
self._season_episodes[mediaid] = list(
|
||||
set(self._season_episodes[mediaid]) - set(episodes)
|
||||
)
|
||||
expired.append((task.fileitem, int(inactive_seconds)))
|
||||
affected_job_ids.add(mediaid)
|
||||
|
||||
self.__remove_done_job_groups(affected_job_ids)
|
||||
return expired
|
||||
|
||||
def running_task(self, task: TransferTask):
|
||||
"""
|
||||
设置任务为运行中，并刷新外部异步任务的状态心跳。
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_id(task)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return
|
||||
# 更新状态
|
||||
for t in self._job_view[__mediaid__].tasks:
|
||||
if t.fileitem == task.fileitem:
|
||||
t.state = "running"
|
||||
file_key = self.__get_file_key(t.fileitem)
|
||||
if file_key:
|
||||
self._task_state_changed_at[file_key] = monotonic()
|
||||
break
|
||||
|
||||
def finish_task(self, task: TransferTask):
|
||||
"""
|
||||
设置任务为完成/成功
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_id(task)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return
|
||||
# 更新状态
|
||||
for t in self._job_view[__mediaid__].tasks:
|
||||
if t.fileitem == task.fileitem:
|
||||
t.state = "completed"
|
||||
file_key = self.__get_file_key(t.fileitem)
|
||||
if file_key:
|
||||
self._task_state_changed_at[file_key] = monotonic()
|
||||
break
|
||||
|
||||
def fail_task(self, task: TransferTask):
|
||||
"""
|
||||
设置任务为失败
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_id(task)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return
|
||||
# 更新状态
|
||||
for t in self._job_view[__mediaid__].tasks:
|
||||
if t.fileitem == task.fileitem:
|
||||
t.state = "failed"
|
||||
file_key = self.__get_file_key(t.fileitem)
|
||||
if file_key:
|
||||
self._task_state_changed_at[file_key] = monotonic()
|
||||
break
|
||||
# 移除剧集信息
|
||||
if __mediaid__ in self._season_episodes:
|
||||
self._season_episodes[__mediaid__] = list(
|
||||
set(self._season_episodes[__mediaid__])
|
||||
- set(task.meta.episode_list)
|
||||
)
|
||||
|
||||
def fail_unfinished_task(self, task: TransferTask):
|
||||
"""
|
||||
将指定任务视图中的非终态任务标记为失败
|
||||
"""
|
||||
if not task or not task.fileitem:
|
||||
return
|
||||
file_key = self.__get_file_key(task.fileitem)
|
||||
if not file_key:
|
||||
return
|
||||
with job_lock:
|
||||
for mediaid, job in self._job_view.items():
|
||||
for job_task in job.tasks:
|
||||
if self.__get_file_key(job_task.fileitem) != file_key:
|
||||
continue
|
||||
if job_task.state not in ["completed", "failed"]:
|
||||
job_task.state = "failed"
|
||||
self._task_state_changed_at[file_key] = monotonic()
|
||||
if mediaid in self._season_episodes:
|
||||
self._season_episodes[mediaid] = list(
|
||||
set(self._season_episodes[mediaid])
|
||||
- set(task.meta.episode_list)
|
||||
)
|
||||
return
|
||||
|
||||
def remove_task(self, fileitem: FileItem) -> Optional[TransferJobTask]:
|
||||
"""
|
||||
根据文件项移除任务
|
||||
"""
|
||||
task, _ = self.__remove_task_with_job_id(fileitem)
|
||||
return task
|
||||
|
||||
def __remove_task_with_job_id(
|
||||
self,
|
||||
fileitem: FileItem,
|
||||
preserve_execution: bool = False,
|
||||
) -> Tuple[Optional[TransferJobTask], Optional[Tuple]]:
|
||||
"""
|
||||
根据文件项移除任务，并返回任务所在的作业ID
|
||||
"""
|
||||
file_key = self.__get_file_key(fileitem)
|
||||
if not file_key:
|
||||
return None, None
|
||||
with job_lock:
|
||||
for mediaid in list(self._job_view):
|
||||
job = self._job_view[mediaid]
|
||||
for task in job.tasks:
|
||||
if self.__get_file_key(task.fileitem) == file_key:
|
||||
job.tasks.remove(task)
|
||||
self._task_state_changed_at.pop(file_key, None)
|
||||
if not preserve_execution:
|
||||
self._active_executions.discard(file_key)
|
||||
# 如果没有作业了，则移除作业
|
||||
if not job.tasks:
|
||||
self._job_view.pop(mediaid)
|
||||
# 移除季集信息
|
||||
if mediaid in self._season_episodes:
|
||||
episodes = getattr(task.meta, "episode_list", None) or []
|
||||
self._season_episodes[mediaid] = list(
|
||||
set(self._season_episodes[mediaid])
|
||||
- set(episodes)
|
||||
)
|
||||
return task, mediaid
|
||||
return None, None
|
||||
|
||||
def remove_job(self, task: TransferTask) -> Optional[TransferJob]:
|
||||
"""
|
||||
移除任务对应的作业（强制，线程不安全）
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_id(task)
|
||||
if __mediaid__ in self._job_view:
|
||||
job = self._job_view[__mediaid__]
|
||||
self.__pop_job(__mediaid__)
|
||||
return job
|
||||
return None
|
||||
|
||||
def try_remove_job(self, task: TransferTask):
|
||||
"""
|
||||
尝试移除任务对应的作业（严格检查未完成作业，线程安全）
|
||||
"""
|
||||
with job_lock:
|
||||
__metaid__ = self.__get_meta_id(
|
||||
meta=task.meta, season=task.meta.begin_season
|
||||
)
|
||||
__mediaid__ = self.__get_media_id(
|
||||
media=task.mediainfo, season=task.meta.begin_season
|
||||
)
|
||||
|
||||
related_media_ids = set(self._meta_to_media_ids.get(__metaid__, set()))
|
||||
if task.mediainfo:
|
||||
related_media_ids.add(__mediaid__)
|
||||
|
||||
meta_done = self.__is_job_done(__metaid__)
|
||||
media_done = all(
|
||||
self.__is_job_done(mediaid) for mediaid in related_media_ids
|
||||
)
|
||||
|
||||
if meta_done and media_done:
|
||||
remove_ids = {__metaid__, self.__get_id(task), *related_media_ids}
|
||||
for job_id in remove_ids:
|
||||
self.__pop_job(job_id)
|
||||
self._meta_to_media_ids.pop(__metaid__, None)
|
||||
|
||||
def is_done(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
检查任务对应的作业是否整理完成（不管成功还是失败）
|
||||
"""
|
||||
with job_lock:
|
||||
__metaid__ = self.__get_meta_id(
|
||||
meta=task.meta, season=task.meta.begin_season
|
||||
)
|
||||
__mediaid__ = self.__get_media_id(
|
||||
media=task.mediainfo, season=task.meta.begin_season
|
||||
)
|
||||
if __metaid__ in self._job_view:
|
||||
meta_done = all(
|
||||
task.state in ["completed", "failed"]
|
||||
for task in self._job_view[__metaid__].tasks
|
||||
)
|
||||
else:
|
||||
meta_done = True
|
||||
if __mediaid__ in self._job_view:
|
||||
media_done = all(
|
||||
task.state in ["completed", "failed"]
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
)
|
||||
else:
|
||||
media_done = True
|
||||
return meta_done and media_done
|
||||
|
||||
def is_finished(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
检查任务对应的作业是否已完成且有成功的记录
|
||||
"""
|
||||
with job_lock:
|
||||
__metaid__ = self.__get_meta_id(
|
||||
meta=task.meta, season=task.meta.begin_season
|
||||
)
|
||||
__mediaid__ = self.__get_media_id(
|
||||
media=task.mediainfo, season=task.meta.begin_season
|
||||
)
|
||||
if __metaid__ in self._job_view:
|
||||
meta_finished = all(
|
||||
task.state in ["completed", "failed"]
|
||||
for task in self._job_view[__metaid__].tasks
|
||||
)
|
||||
else:
|
||||
meta_finished = True
|
||||
if __mediaid__ in self._job_view:
|
||||
tasks = self._job_view[__mediaid__].tasks
|
||||
media_finished = all(
|
||||
task.state in ["completed", "failed"] for task in tasks
|
||||
) and any(task.state == "completed" for task in tasks)
|
||||
else:
|
||||
media_finished = True
|
||||
return meta_finished and media_finished
|
||||
|
||||
def is_success(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
检查任务对应的作业是否全部成功
|
||||
"""
|
||||
with job_lock:
|
||||
__metaid__ = self.__get_meta_id(
|
||||
meta=task.meta, season=task.meta.begin_season
|
||||
)
|
||||
__mediaid__ = self.__get_media_id(
|
||||
media=task.mediainfo, season=task.meta.begin_season
|
||||
)
|
||||
if __metaid__ in self._job_view:
|
||||
meta_success = all(
|
||||
task.state in ["completed"]
|
||||
for task in self._job_view[__metaid__].tasks
|
||||
)
|
||||
else:
|
||||
meta_success = True
|
||||
if __mediaid__ in self._job_view:
|
||||
media_success = all(
|
||||
task.state in ["completed"]
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
)
|
||||
else:
|
||||
media_success = True
|
||||
return meta_success and media_success
|
||||
|
||||
def get_all_torrent_hashes(self) -> set[str]:
|
||||
"""
|
||||
获取所有种子的哈希值集合
|
||||
"""
|
||||
with job_lock:
|
||||
return {
|
||||
task.download_hash
|
||||
for job in self._job_view.values()
|
||||
for task in job.tasks
|
||||
}
|
||||
|
||||
def is_torrent_done(self, download_hash: str) -> bool:
|
||||
"""
|
||||
检查指定种子的所有任务是否都已完成
|
||||
"""
|
||||
with job_lock:
|
||||
if any(
|
||||
task.state not in {"completed", "failed"}
|
||||
for job in self._job_view.values()
|
||||
for task in job.tasks
|
||||
if task.download_hash == download_hash
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def is_torrent_success(self, download_hash: str) -> bool:
|
||||
"""
|
||||
检查指定种子的所有任务是否都已成功
|
||||
"""
|
||||
with job_lock:
|
||||
if any(
|
||||
task.state != "completed"
|
||||
for job in self._job_view.values()
|
||||
for task in job.tasks
|
||||
if task.download_hash == download_hash
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def has_tasks(
|
||||
self,
|
||||
meta: MetaBase,
|
||||
mediainfo: Optional[MediaInfo] = None,
|
||||
season: Optional[int] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判断作业是否还有任务正在处理
|
||||
"""
|
||||
with job_lock:
|
||||
if mediainfo:
|
||||
__mediaid__ = self.__get_media_id(media=mediainfo, season=season)
|
||||
if __mediaid__ in self._job_view:
|
||||
return True
|
||||
|
||||
__metaid__ = self.__get_meta_id(meta=meta, season=season)
|
||||
return (
|
||||
__metaid__ in self._job_view
|
||||
and len(self._job_view[__metaid__].tasks) > 0
|
||||
)
|
||||
|
||||
def success_tasks(
|
||||
self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
|
||||
) -> List[TransferJobTask]:
|
||||
"""
|
||||
获取作业中所有成功的任务
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return []
|
||||
return [
|
||||
task
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
if task.state == "completed"
|
||||
]
|
||||
|
||||
def all_tasks(
|
||||
self, media: MediaInfo, season: Optional[int] = None
|
||||
) -> List[TransferJobTask]:
|
||||
"""
|
||||
获取作业中全部任务
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return []
|
||||
return self._job_view[__mediaid__].tasks
|
||||
|
||||
def count(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
|
||||
"""
|
||||
获取作业中成功总数
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return 0
|
||||
return len(
|
||||
[
|
||||
task
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
if task.state == "completed"
|
||||
]
|
||||
)
|
||||
|
||||
def size(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
|
||||
"""
|
||||
获取作业中所有成功文件总大小
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
if __mediaid__ not in self._job_view:
|
||||
return 0
|
||||
return sum(
|
||||
[
|
||||
task.fileitem.size
|
||||
if task.fileitem.size is not None
|
||||
else (
|
||||
SystemUtils.get_directory_size(Path(task.fileitem.path))
|
||||
if task.fileitem.storage == "local"
|
||||
else 0
|
||||
)
|
||||
for task in self._job_view[__mediaid__].tasks
|
||||
if task.state == "completed"
|
||||
]
|
||||
)
|
||||
|
||||
def total(self) -> int:
|
||||
"""
|
||||
获取所有任务总数
|
||||
"""
|
||||
with job_lock:
|
||||
return sum([len(job.tasks) for job in self._job_view.values()])
|
||||
|
||||
def pending_total(self) -> int:
|
||||
"""
|
||||
获取未到终态的任务总数。
|
||||
|
||||
作业要等关联任务全部终态才整体移除,追更/分批场景下已完成任务会
|
||||
跨批次残留在视图中;批次统计若用全量 total() 会把历史任务计入
|
||||
「当前共 N 个文件」并压低进度百分比,因此只数未终态任务。
|
||||
"""
|
||||
with job_lock:
|
||||
return sum(
|
||||
1
|
||||
for job in self._job_view.values()
|
||||
for task in job.tasks
|
||||
if task.state not in ("completed", "failed")
|
||||
)
|
||||
|
||||
def list_jobs(self) -> List[TransferJob]:
|
||||
"""
|
||||
获取所有作业的任务列表
|
||||
"""
|
||||
with job_lock:
|
||||
return list(self._job_view.values())
|
||||
|
||||
def season_episodes(
|
||||
self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
|
||||
) -> List[int]:
|
||||
"""
|
||||
获取作业的季集清单
|
||||
"""
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
return self._season_episodes.get(__mediaid__) or []
|
||||
|
||||
|
||||
class FailedRetryScheduler:
|
||||
"""
|
||||
负责失败整理记录的 debounce 聚合与 AI 重试调度。
|
||||
"""
|
||||
|
||||
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._retry_transfer_buffer: dict[str, list[int]] = {}
|
||||
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._retry_transfer_lock = asyncio.Lock()
|
||||
|
||||
async def close(self):
|
||||
async with self._retry_transfer_lock:
|
||||
timers = list(self._retry_transfer_timers.values())
|
||||
self._retry_transfer_timers.clear()
|
||||
self._retry_transfer_buffer.clear()
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_transfer_template_context(
|
||||
history_ids: list[int],
|
||||
) -> tuple[str, dict[str, int | str]]:
|
||||
"""仅负责把失败重试任务的动态数据映射成模板变量。"""
|
||||
is_batch = len(history_ids) > 1
|
||||
task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry"
|
||||
template_context: dict[str, int | str] = {
|
||||
"history_ids_csv": ", ".join(str(item) for item in history_ids),
|
||||
"history_count": len(history_ids),
|
||||
}
|
||||
if not is_batch:
|
||||
template_context["history_id"] = history_ids[0]
|
||||
return task_type, template_context
|
||||
|
||||
def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str:
|
||||
"""根据失败记录数量构建统一的重试整理后台任务提示词。"""
|
||||
task_type, template_context = self._build_retry_transfer_template_context(history_ids)
|
||||
return get_prompt_manager().render_system_task_message(
|
||||
task_type,
|
||||
template_context=template_context,
|
||||
)
|
||||
|
||||
async def schedule_retry(self, history_id: int, group_key: str = ""):
|
||||
"""
|
||||
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
|
||||
"""
|
||||
if not group_key:
|
||||
group_key = f"_default_{history_id}"
|
||||
|
||||
async with self._retry_transfer_lock:
|
||||
if group_key not in self._retry_transfer_buffer:
|
||||
self._retry_transfer_buffer[group_key] = []
|
||||
if history_id not in self._retry_transfer_buffer[group_key]:
|
||||
self._retry_transfer_buffer[group_key].append(history_id)
|
||||
logger.info(
|
||||
f"智能体重试整理：记录 ID={history_id} 已加入缓冲区 "
|
||||
f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)"
|
||||
)
|
||||
|
||||
if group_key in self._retry_transfer_timers:
|
||||
self._retry_transfer_timers[group_key].cancel()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
self._retry_transfer_timers[group_key] = loop.call_later(
|
||||
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
|
||||
lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)),
|
||||
)
|
||||
|
||||
async def _flush_retry_transfer(self, group_key: str):
|
||||
"""
|
||||
延迟定时器到期后，取出该分组的所有 history_id 并合并为一次 agent 调用。
|
||||
"""
|
||||
async with self._retry_transfer_lock:
|
||||
history_ids = self._retry_transfer_buffer.pop(group_key, [])
|
||||
self._retry_transfer_timers.pop(group_key, None)
|
||||
|
||||
if not history_ids:
|
||||
return
|
||||
|
||||
ids_str = ", ".join(str(item) for item in history_ids)
|
||||
logger.info(
|
||||
f"智能体重试整理：开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})"
|
||||
)
|
||||
|
||||
try:
|
||||
manager = get_running_agent_manager()
|
||||
if manager is None:
|
||||
logger.warning("智能助手服务未运行，跳过整理失败自动重试")
|
||||
return
|
||||
await manager.run_background_prompt(
|
||||
message=self._build_retry_transfer_prompt(history_ids),
|
||||
session_prefix="__agent_retry_transfer_batch",
|
||||
reply_mode=ReplyMode.DISPATCH,
|
||||
)
|
||||
logger.info(
|
||||
f"智能体重试整理：批量处理完成 IDs=[{ids_str}] (group={group_key})"
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}"
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user