From 24671f8f186643ce65d2d86c31aa21dfb890264d Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:07:38 +0800 Subject: [PATCH 1/8] refactor(runtime): lazily activate host modules (#6331) --- app/api/endpoints/message.py | 15 +- app/api/endpoints/system.py | 5 +- app/chain/__init__.py | 4 +- app/foundation/singleton.py | 16 +- app/modules/__init__.py | 5 +- app/modules/acoustid/capability.toml | 19 + app/modules/anilist/capability.toml | 15 + app/modules/bangumi/capability.toml | 15 + app/modules/discord/capability.toml | 22 + app/modules/douban/capability.toml | 15 + app/modules/emby/capability.toml | 22 + app/modules/fanart/capability.toml | 19 + app/modules/feishu/capability.toml | 22 + app/modules/filemanager/capability.toml | 15 + app/modules/filter/capability.toml | 15 + app/modules/indexer/capability.toml | 15 + app/modules/jellyfin/capability.toml | 22 + app/modules/listenbrainz/capability.toml | 15 + app/modules/lrclib/capability.toml | 15 + app/modules/musicbrainz/capability.toml | 15 + app/modules/navidrome/capability.toml | 22 + app/modules/plex/capability.toml | 22 + app/modules/postgresql/capability.toml | 15 + app/modules/qbittorrent/capability.toml | 22 + app/modules/qqbot/capability.toml | 22 + app/modules/redis/capability.toml | 15 + app/modules/rtorrent/capability.toml | 22 + app/modules/slack/capability.toml | 22 + app/modules/subtitle/capability.toml | 15 + app/modules/synologychat/capability.toml | 22 + app/modules/telegram/capability.toml | 22 + app/modules/theaudiodb/capability.toml | 15 + app/modules/themoviedb/capability.toml | 15 + app/modules/thetvdb/capability.toml | 15 + app/modules/transmission/capability.toml | 22 + app/modules/trimemedia/capability.toml | 22 + app/modules/ugreen/capability.toml | 22 + app/modules/vocechat/capability.toml | 22 + app/modules/webpush/capability.toml | 22 + app/modules/wechat/capability.toml | 22 + app/modules/wechatclawbot/capability.toml | 22 + app/modules/zspace/capability.toml | 22 + app/runtime/capabilities/__init__.py | 47 + app/runtime/capabilities/errors.py | 32 + app/runtime/capabilities/model.py | 182 ++ app/runtime/capabilities/registry.py | 287 +++ app/runtime/capabilities/runtime.py | 1369 ++++++++++++++ app/runtime/events.py | 38 +- app/runtime/extensions/host_module_adapter.py | 260 +++ app/runtime/extensions/module_manager.py | 406 ++-- app/runtime/extensions/service_config.py | 76 + app/runtime/extensions/service_registry.py | 84 +- app/runtime/reload.py | 6 + app/startup/modules_initializer.py | 2 +- scripts/perf/README.md | 106 ++ scripts/perf/instrument/collect_proc.sh | 53 + scripts/perf/instrument/sitecustomize.py | 35 + scripts/perf/moviepilot_docker_ab.py | 1677 +++++++++++++++++ tests/test_capability_registry.py | 195 ++ tests/test_capability_runtime.py | 834 ++++++++ tests/test_config_reload_handler.py | 20 + tests/test_event_dispatch_snapshot.py | 160 ++ tests/test_lifecycle_shutdown.py | 2 +- .../test_module_manager_capability_adapter.py | 930 +++++++++ tests/test_navidrome_module.py | 9 +- tests/test_singleton_thread_safety.py | 41 + tests/test_system_i18n.py | 18 +- 67 files changed, 7364 insertions(+), 253 deletions(-) create mode 100644 app/modules/acoustid/capability.toml create mode 100644 app/modules/anilist/capability.toml create mode 100644 app/modules/bangumi/capability.toml create mode 100644 app/modules/discord/capability.toml create mode 100644 app/modules/douban/capability.toml create mode 100644 app/modules/emby/capability.toml create mode 100644 app/modules/fanart/capability.toml create mode 100644 app/modules/feishu/capability.toml create mode 100644 app/modules/filemanager/capability.toml create mode 100644 app/modules/filter/capability.toml create mode 100644 app/modules/indexer/capability.toml create mode 100644 app/modules/jellyfin/capability.toml create mode 100644 app/modules/listenbrainz/capability.toml create mode 100644 app/modules/lrclib/capability.toml create mode 100644 app/modules/musicbrainz/capability.toml create mode 100644 app/modules/navidrome/capability.toml create mode 100644 app/modules/plex/capability.toml create mode 100644 app/modules/postgresql/capability.toml create mode 100644 app/modules/qbittorrent/capability.toml create mode 100644 app/modules/qqbot/capability.toml create mode 100644 app/modules/redis/capability.toml create mode 100644 app/modules/rtorrent/capability.toml create mode 100644 app/modules/slack/capability.toml create mode 100644 app/modules/subtitle/capability.toml create mode 100644 app/modules/synologychat/capability.toml create mode 100644 app/modules/telegram/capability.toml create mode 100644 app/modules/theaudiodb/capability.toml create mode 100644 app/modules/themoviedb/capability.toml create mode 100644 app/modules/thetvdb/capability.toml create mode 100644 app/modules/transmission/capability.toml create mode 100644 app/modules/trimemedia/capability.toml create mode 100644 app/modules/ugreen/capability.toml create mode 100644 app/modules/vocechat/capability.toml create mode 100644 app/modules/webpush/capability.toml create mode 100644 app/modules/wechat/capability.toml create mode 100644 app/modules/wechatclawbot/capability.toml create mode 100644 app/modules/zspace/capability.toml create mode 100644 app/runtime/capabilities/__init__.py create mode 100644 app/runtime/capabilities/errors.py create mode 100644 app/runtime/capabilities/model.py create mode 100644 app/runtime/capabilities/registry.py create mode 100644 app/runtime/capabilities/runtime.py create mode 100644 app/runtime/extensions/host_module_adapter.py create mode 100644 app/runtime/extensions/service_config.py create mode 100644 scripts/perf/README.md create mode 100644 scripts/perf/instrument/collect_proc.sh create mode 100644 scripts/perf/instrument/sitecustomize.py create mode 100644 scripts/perf/moviepilot_docker_ab.py create mode 100644 tests/test_capability_registry.py create mode 100644 tests/test_capability_runtime.py create mode 100644 tests/test_event_dispatch_snapshot.py create mode 100644 tests/test_module_manager_capability_adapter.py create mode 100644 tests/test_singleton_thread_safety.py diff --git a/app/api/endpoints/message.py b/app/api/endpoints/message.py index d9c758a58..5b109f326 100644 --- a/app/api/endpoints/message.py +++ b/app/api/endpoints/message.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import json import time -from typing import Union, Any, List, Optional +from typing import Protocol, Union, Any, List, Optional from fastapi import BackgroundTasks, Depends, Request -from pywebpush import WebPushException, webpush from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import PlainTextResponse @@ -27,7 +28,13 @@ router = ResponseAPIRouter() _WNS_DEFAULT_TTL = 86400 -def is_webpush_subscription_gone(error: WebPushException) -> bool: +class WebPushError(Protocol): + """Web Push 订阅状态判断所需的最小异常协议。""" + + response: Any # 推送服务响应，状态码字段由具体 SDK 提供 + + +def is_webpush_subscription_gone(error: WebPushError) -> bool: """判断 Web Push 订阅是否已在浏览器或推送服务侧失效。""" response: Any = getattr(error, "response", None) status_code = getattr(response, "status_code", None) or getattr( @@ -359,6 +366,8 @@ def send_notification( """ 发送webpush通知 """ + from pywebpush import WebPushException, webpush + for sub in global_vars.get_subscriptions(): try: webpush( diff --git a/app/api/endpoints/system.py b/app/api/endpoints/system.py index 56645b66d..af033238e 100644 --- a/app/api/endpoints/system.py +++ b/app/api/endpoints/system.py @@ -1405,8 +1405,9 @@ def modulelist(_: schemas.TokenPayload = Depends(verify_token)): 查询已加载的模块ID列表 """ modules = [] - for module_id, module in ModuleManager().get_modules().items(): - name = module.get_name() + for spec in ModuleManager().list_specs(): + module_id = spec.id + name = str(spec.metadata["name"]) modules.append( { "id": module_id, diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 5cb69f08b..9d5e95234 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import copy import inspect import pickle import traceback from abc import ABCMeta -from collections.abc import Callable +from collections.abc import Callable, Sequence from datetime import datetime from pathlib import Path from typing import Optional, Any, Tuple, List, Set, Union, Dict diff --git a/app/foundation/singleton.py b/app/foundation/singleton.py index db5a088e8..a4d711fa2 100644 --- a/app/foundation/singleton.py +++ b/app/foundation/singleton.py @@ -9,6 +9,7 @@ class Singleton(abc.ABCMeta, type): """ _instances: dict = {} + _lock = threading.RLock() def get_existing_instance(cls, *args, **kwargs): """按相同参数返回已创建实例，不触发初始化""" @@ -18,9 +19,10 @@ class Singleton(abc.ABCMeta, type): def __call__(cls, *args, **kwargs): """按类和构造参数创建或复用实例。""" key = (cls, args, frozenset(kwargs.items())) - if key not in cls._instances: - cls._instances[key] = super().__call__(*args, **kwargs) - return cls._instances[key] + with cls._lock: + if key not in cls._instances: + cls._instances[key] = super().__call__(*args, **kwargs) + return cls._instances[key] class AbstractSingleton(abc.ABC, metaclass=Singleton): @@ -36,6 +38,7 @@ class SingletonClass(abc.ABCMeta, type): """ _instances: dict = {} + _lock = threading.RLock() def get_existing_instance(cls): """返回已创建实例，不触发初始化""" @@ -43,9 +46,10 @@ class SingletonClass(abc.ABCMeta, type): def __call__(cls, *args, **kwargs): """按类创建或复用唯一实例。""" - if cls not in cls._instances: - cls._instances[cls] = super().__call__(*args, **kwargs) - return cls._instances[cls] + with cls._lock: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] class AbstractSingletonClass(abc.ABC, metaclass=SingletonClass): diff --git a/app/modules/__init__.py b/app/modules/__init__.py index ee8edbee7..50eca16f5 100644 --- a/app/modules/__init__.py +++ b/app/modules/__init__.py @@ -3,7 +3,7 @@ from abc import abstractmethod, ABCMeta from typing import Generic, Tuple, Union, TypeVar, Type, Dict, Optional, Callable from pathlib import Path -from app.runtime.extensions.service_registry import ServiceConfigHelper +from app.runtime.extensions.service_config import ServiceConfigHelper from app.runtime.log import logger from app.schemas import Notification, NotificationConf, MediaServerConf, DownloaderConf from app.schemas.types import ModuleType, DownloaderType, MediaServerType, MessageChannel, StorageSchema, \ @@ -17,6 +17,9 @@ class _ModuleBase(ConfigReloadMixin, metaclass=ABCMeta): 输入参数与输出参数一致的，或没有输出的，可以被多个模块重复实现 """ + # Host Module 的配置事件由统一 Adapter 协调，避免同一 generation 被双重重载。 + CONFIG_RELOAD_MANAGED_EXTERNALLY = True + def __init__(self) -> None: """初始化模块生命周期锁""" super().__init__() diff --git a/app/modules/acoustid/capability.toml b/app/modules/acoustid/capability.toml new file mode 100644 index 000000000..0243c6d1c --- /dev/null +++ b/app/modules/acoustid/capability.toml @@ -0,0 +1,19 @@ +schema_version = 1 +id = "AcoustIdModule" +kind = "host_module" +entrypoint = "app.modules.acoustid:AcoustIdModule" +depends_on = [] + +[metadata] +name = "AcoustID" +type = "other" +subtype = "AcoustId" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["ACOUSTID_API_KEY"] + +[activation.selector] +kind = "setting_truthy" +key = "ACOUSTID_API_KEY" diff --git a/app/modules/anilist/capability.toml b/app/modules/anilist/capability.toml new file mode 100644 index 000000000..0d76d82f4 --- /dev/null +++ b/app/modules/anilist/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "AniListModule" +kind = "host_module" +entrypoint = "app.modules.anilist:AniListModule" +depends_on = [] + +[metadata] +name = "AniList" +type = "mediarecognize" +subtype = "AniList" +priority = 4 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST"] diff --git a/app/modules/bangumi/capability.toml b/app/modules/bangumi/capability.toml new file mode 100644 index 000000000..ad0668eab --- /dev/null +++ b/app/modules/bangumi/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "BangumiModule" +kind = "host_module" +entrypoint = "app.modules.bangumi:BangumiModule" +depends_on = [] + +[metadata] +name = "Bangumi" +type = "mediarecognize" +subtype = "Bangumi" +priority = 3 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST"] diff --git a/app/modules/discord/capability.toml b/app/modules/discord/capability.toml new file mode 100644 index 000000000..33d7008cb --- /dev/null +++ b/app/modules/discord/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "DiscordModule" +kind = "host_module" +entrypoint = "app.modules.discord:DiscordModule" +depends_on = [] + +[metadata] +name = "Discord" +type = "notification" +subtype = "Discord" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "discord" +enabled_field = "enabled" diff --git a/app/modules/douban/capability.toml b/app/modules/douban/capability.toml new file mode 100644 index 000000000..5acea8cee --- /dev/null +++ b/app/modules/douban/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "DoubanModule" +kind = "host_module" +entrypoint = "app.modules.douban:DoubanModule" +depends_on = [] + +[metadata] +name = "豆瓣" +type = "mediarecognize" +subtype = "Douban" +priority = 2 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/emby/capability.toml b/app/modules/emby/capability.toml new file mode 100644 index 000000000..e61047eaf --- /dev/null +++ b/app/modules/emby/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "EmbyModule" +kind = "host_module" +entrypoint = "app.modules.emby:EmbyModule" +depends_on = [] + +[metadata] +name = "Emby" +type = "mediaserver" +subtype = "Emby" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "emby" +enabled_field = "enabled" diff --git a/app/modules/fanart/capability.toml b/app/modules/fanart/capability.toml new file mode 100644 index 000000000..94439c182 --- /dev/null +++ b/app/modules/fanart/capability.toml @@ -0,0 +1,19 @@ +schema_version = 1 +id = "FanartModule" +kind = "host_module" +entrypoint = "app.modules.fanart:FanartModule" +depends_on = [] + +[metadata] +name = "Fanart" +type = "other" +subtype = "Fanart" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["FANART_API_KEY"] + +[activation.selector] +kind = "setting_truthy" +key = "FANART_API_KEY" diff --git a/app/modules/feishu/capability.toml b/app/modules/feishu/capability.toml new file mode 100644 index 000000000..67203bf20 --- /dev/null +++ b/app/modules/feishu/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "FeishuModule" +kind = "host_module" +entrypoint = "app.modules.feishu:FeishuModule" +depends_on = [] + +[metadata] +name = "飞书" +type = "notification" +subtype = "Feishu" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "feishu" +enabled_field = "enabled" diff --git a/app/modules/filemanager/capability.toml b/app/modules/filemanager/capability.toml new file mode 100644 index 000000000..ef6a278de --- /dev/null +++ b/app/modules/filemanager/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "FileManagerModule" +kind = "host_module" +entrypoint = "app.modules.filemanager:FileManagerModule" +depends_on = [] + +[metadata] +name = "文件整理" +type = "other" +subtype = "FileManager" +priority = 4 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/filter/capability.toml b/app/modules/filter/capability.toml new file mode 100644 index 000000000..3eadfcebd --- /dev/null +++ b/app/modules/filter/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "FilterModule" +kind = "host_module" +entrypoint = "app.modules.filter:FilterModule" +depends_on = [] + +[metadata] +name = "过滤器" +type = "other" +subtype = "Filter" +priority = 4 + +[activation] +policy = "bootstrap" +watch = ["CustomFilterRules", "CustomIdentifiers", "CustomReleaseGroups", "Customization"] diff --git a/app/modules/indexer/capability.toml b/app/modules/indexer/capability.toml new file mode 100644 index 000000000..f0e5b4f39 --- /dev/null +++ b/app/modules/indexer/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "IndexerModule" +kind = "host_module" +entrypoint = "app.modules.indexer:IndexerModule" +depends_on = [] + +[metadata] +name = "站点索引" +type = "indexer" +subtype = "Indexer" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/jellyfin/capability.toml b/app/modules/jellyfin/capability.toml new file mode 100644 index 000000000..2d420fb76 --- /dev/null +++ b/app/modules/jellyfin/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "JellyfinModule" +kind = "host_module" +entrypoint = "app.modules.jellyfin:JellyfinModule" +depends_on = [] + +[metadata] +name = "Jellyfin" +type = "mediaserver" +subtype = "Jellyfin" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "jellyfin" +enabled_field = "enabled" diff --git a/app/modules/listenbrainz/capability.toml b/app/modules/listenbrainz/capability.toml new file mode 100644 index 000000000..92466961b --- /dev/null +++ b/app/modules/listenbrainz/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "ListenBrainzModule" +kind = "host_module" +entrypoint = "app.modules.listenbrainz:ListenBrainzModule" +depends_on = [] + +[metadata] +name = "ListenBrainz" +type = "other" +subtype = "ListenBrainz" +priority = 5 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/lrclib/capability.toml b/app/modules/lrclib/capability.toml new file mode 100644 index 000000000..23ad94652 --- /dev/null +++ b/app/modules/lrclib/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "LrclibModule" +kind = "host_module" +entrypoint = "app.modules.lrclib:LrclibModule" +depends_on = [] + +[metadata] +name = "LRCLIB" +type = "other" +subtype = "Lrclib" +priority = 5 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/musicbrainz/capability.toml b/app/modules/musicbrainz/capability.toml new file mode 100644 index 000000000..55c7a1c0d --- /dev/null +++ b/app/modules/musicbrainz/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "MusicBrainzModule" +kind = "host_module" +entrypoint = "app.modules.musicbrainz:MusicBrainzModule" +depends_on = [] + +[metadata] +name = "MusicBrainz" +type = "mediarecognize" +subtype = "MusicBrainz" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/navidrome/capability.toml b/app/modules/navidrome/capability.toml new file mode 100644 index 000000000..9a41ebe03 --- /dev/null +++ b/app/modules/navidrome/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "NavidromeModule" +kind = "host_module" +entrypoint = "app.modules.navidrome:NavidromeModule" +depends_on = [] + +[metadata] +name = "Navidrome" +type = "mediaserver" +subtype = "Navidrome" +priority = 7 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "navidrome" +enabled_field = "enabled" diff --git a/app/modules/plex/capability.toml b/app/modules/plex/capability.toml new file mode 100644 index 000000000..0cf679309 --- /dev/null +++ b/app/modules/plex/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "PlexModule" +kind = "host_module" +entrypoint = "app.modules.plex:PlexModule" +depends_on = [] + +[metadata] +name = "Plex" +type = "mediaserver" +subtype = "Plex" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "plex" +enabled_field = "enabled" diff --git a/app/modules/postgresql/capability.toml b/app/modules/postgresql/capability.toml new file mode 100644 index 000000000..ef05508b4 --- /dev/null +++ b/app/modules/postgresql/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "PostgreSQLModule" +kind = "host_module" +entrypoint = "app.modules.postgresql:PostgreSQLModule" +depends_on = [] + +[metadata] +name = "PostgreSQL" +type = "other" +subtype = "PostgreSQL" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/qbittorrent/capability.toml b/app/modules/qbittorrent/capability.toml new file mode 100644 index 000000000..28d247aa7 --- /dev/null +++ b/app/modules/qbittorrent/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "QbittorrentModule" +kind = "host_module" +entrypoint = "app.modules.qbittorrent:QbittorrentModule" +depends_on = [] + +[metadata] +name = "Qbittorrent" +type = "downloader" +subtype = "Qbittorrent" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "qbittorrent" +enabled_field = "enabled" diff --git a/app/modules/qqbot/capability.toml b/app/modules/qqbot/capability.toml new file mode 100644 index 000000000..557b74a5b --- /dev/null +++ b/app/modules/qqbot/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "QQBotModule" +kind = "host_module" +entrypoint = "app.modules.qqbot:QQBotModule" +depends_on = [] + +[metadata] +name = "QQ" +type = "notification" +subtype = "QQ" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "qqbot" +enabled_field = "enabled" diff --git a/app/modules/redis/capability.toml b/app/modules/redis/capability.toml new file mode 100644 index 000000000..94d9b497a --- /dev/null +++ b/app/modules/redis/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "RedisModule" +kind = "host_module" +entrypoint = "app.modules.redis:RedisModule" +depends_on = [] + +[metadata] +name = "Redis缓存" +type = "other" +subtype = "Redis" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/rtorrent/capability.toml b/app/modules/rtorrent/capability.toml new file mode 100644 index 000000000..6206dd06d --- /dev/null +++ b/app/modules/rtorrent/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "RtorrentModule" +kind = "host_module" +entrypoint = "app.modules.rtorrent:RtorrentModule" +depends_on = [] + +[metadata] +name = "Rtorrent" +type = "downloader" +subtype = "Rtorrent" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "rtorrent" +enabled_field = "enabled" diff --git a/app/modules/slack/capability.toml b/app/modules/slack/capability.toml new file mode 100644 index 000000000..042e3a020 --- /dev/null +++ b/app/modules/slack/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "SlackModule" +kind = "host_module" +entrypoint = "app.modules.slack:SlackModule" +depends_on = [] + +[metadata] +name = "Slack" +type = "notification" +subtype = "Slack" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "slack" +enabled_field = "enabled" diff --git a/app/modules/subtitle/capability.toml b/app/modules/subtitle/capability.toml new file mode 100644 index 000000000..92d4d71f6 --- /dev/null +++ b/app/modules/subtitle/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "SubtitleModule" +kind = "host_module" +entrypoint = "app.modules.subtitle:SubtitleModule" +depends_on = [] + +[metadata] +name = "站点字幕" +type = "other" +subtype = "Subtitle" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/synologychat/capability.toml b/app/modules/synologychat/capability.toml new file mode 100644 index 000000000..89bafdd2f --- /dev/null +++ b/app/modules/synologychat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "SynologyChatModule" +kind = "host_module" +entrypoint = "app.modules.synologychat:SynologyChatModule" +depends_on = [] + +[metadata] +name = "Synology Chat" +type = "notification" +subtype = "SynologyChat" +priority = 5 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "synologychat" +enabled_field = "enabled" diff --git a/app/modules/telegram/capability.toml b/app/modules/telegram/capability.toml new file mode 100644 index 000000000..aae376504 --- /dev/null +++ b/app/modules/telegram/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TelegramModule" +kind = "host_module" +entrypoint = "app.modules.telegram:TelegramModule" +depends_on = [] + +[metadata] +name = "Telegram" +type = "notification" +subtype = "Telegram" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "telegram" +enabled_field = "enabled" diff --git a/app/modules/theaudiodb/capability.toml b/app/modules/theaudiodb/capability.toml new file mode 100644 index 000000000..5aa2fcc9a --- /dev/null +++ b/app/modules/theaudiodb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheAudioDbModule" +kind = "host_module" +entrypoint = "app.modules.theaudiodb:TheAudioDbModule" +depends_on = [] + +[metadata] +name = "TheAudioDB" +type = "mediarecognize" +subtype = "TheAudioDB" +priority = 1 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/themoviedb/capability.toml b/app/modules/themoviedb/capability.toml new file mode 100644 index 000000000..80cbe9815 --- /dev/null +++ b/app/modules/themoviedb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheMovieDbModule" +kind = "host_module" +entrypoint = "app.modules.themoviedb:TheMovieDbModule" +depends_on = [] + +[metadata] +name = "TheMovieDb" +type = "mediarecognize" +subtype = "TMDB" +priority = 1 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST", "TMDB_API_DOMAIN", "TMDB_API_KEY", "TMDB_LOCALE"] diff --git a/app/modules/thetvdb/capability.toml b/app/modules/thetvdb/capability.toml new file mode 100644 index 000000000..bba680f9f --- /dev/null +++ b/app/modules/thetvdb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheTvDbModule" +kind = "host_module" +entrypoint = "app.modules.thetvdb:TheTvDbModule" +depends_on = [] + +[metadata] +name = "TheTvDb" +type = "mediarecognize" +subtype = "TVDB" +priority = 4 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/transmission/capability.toml b/app/modules/transmission/capability.toml new file mode 100644 index 000000000..f63977037 --- /dev/null +++ b/app/modules/transmission/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TransmissionModule" +kind = "host_module" +entrypoint = "app.modules.transmission:TransmissionModule" +depends_on = [] + +[metadata] +name = "Transmission" +type = "downloader" +subtype = "Transmission" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "transmission" +enabled_field = "enabled" diff --git a/app/modules/trimemedia/capability.toml b/app/modules/trimemedia/capability.toml new file mode 100644 index 000000000..3ca94566e --- /dev/null +++ b/app/modules/trimemedia/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TrimeMediaModule" +kind = "host_module" +entrypoint = "app.modules.trimemedia:TrimeMediaModule" +depends_on = [] + +[metadata] +name = "飞牛影视" +type = "mediaserver" +subtype = "TrimeMedia" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "trimemedia" +enabled_field = "enabled" diff --git a/app/modules/ugreen/capability.toml b/app/modules/ugreen/capability.toml new file mode 100644 index 000000000..46288aa82 --- /dev/null +++ b/app/modules/ugreen/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "UgreenModule" +kind = "host_module" +entrypoint = "app.modules.ugreen:UgreenModule" +depends_on = [] + +[metadata] +name = "绿联影视" +type = "mediaserver" +subtype = "Ugreen" +priority = 5 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "ugreen" +enabled_field = "enabled" diff --git a/app/modules/vocechat/capability.toml b/app/modules/vocechat/capability.toml new file mode 100644 index 000000000..830f3e6a9 --- /dev/null +++ b/app/modules/vocechat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "VoceChatModule" +kind = "host_module" +entrypoint = "app.modules.vocechat:VoceChatModule" +depends_on = [] + +[metadata] +name = "VoceChat" +type = "notification" +subtype = "VoceChat" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "vocechat" +enabled_field = "enabled" diff --git a/app/modules/webpush/capability.toml b/app/modules/webpush/capability.toml new file mode 100644 index 000000000..ed9484185 --- /dev/null +++ b/app/modules/webpush/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WebPushModule" +kind = "host_module" +entrypoint = "app.modules.webpush:WebPushModule" +depends_on = [] + +[metadata] +name = "WebPush" +type = "notification" +subtype = "WebPush" +priority = 6 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "webpush" +enabled_field = "enabled" diff --git a/app/modules/wechat/capability.toml b/app/modules/wechat/capability.toml new file mode 100644 index 000000000..511625b01 --- /dev/null +++ b/app/modules/wechat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WechatModule" +kind = "host_module" +entrypoint = "app.modules.wechat:WechatModule" +depends_on = [] + +[metadata] +name = "企业微信" +type = "notification" +subtype = "Wechat" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "wechat" +enabled_field = "enabled" diff --git a/app/modules/wechatclawbot/capability.toml b/app/modules/wechatclawbot/capability.toml new file mode 100644 index 000000000..c64cf1975 --- /dev/null +++ b/app/modules/wechatclawbot/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WechatClawBotModule" +kind = "host_module" +entrypoint = "app.modules.wechatclawbot:WechatClawBotModule" +depends_on = [] + +[metadata] +name = "微信 ClawBot" +type = "notification" +subtype = "WechatClawBot" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "wechatclawbot" +enabled_field = "enabled" diff --git a/app/modules/zspace/capability.toml b/app/modules/zspace/capability.toml new file mode 100644 index 000000000..66e2fc95b --- /dev/null +++ b/app/modules/zspace/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "ZSpaceModule" +kind = "host_module" +entrypoint = "app.modules.zspace:ZSpaceModule" +depends_on = [] + +[metadata] +name = "极影视" +type = "mediaserver" +subtype = "ZSpace" +priority = 6 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "zspace" +enabled_field = "enabled" diff --git a/app/runtime/capabilities/__init__.py b/app/runtime/capabilities/__init__.py new file mode 100644 index 000000000..ee512115f --- /dev/null +++ b/app/runtime/capabilities/__init__.py @@ -0,0 +1,47 @@ +"""MoviePilot 内部能力运行时的惰性公共导出。""" + +from __future__ import annotations + +from importlib import import_module +from typing import Any + + +_EXPORT_MODULES = { + "ActivationPolicy": "app.runtime.capabilities.model", + "AdapterExecutionMode": "app.runtime.capabilities.model", + "AsyncCapabilityAdapter": "app.runtime.capabilities.model", + "CapabilityAdapterContractError": "app.runtime.capabilities.errors", + "CapabilityAdapterModeError": "app.runtime.capabilities.errors", + "CapabilityError": "app.runtime.capabilities.errors", + "CapabilityLifecycleState": "app.runtime.capabilities.model", + "CapabilityManifestError": "app.runtime.capabilities.errors", + "CapabilityMaterializationState": "app.runtime.capabilities.model", + "CapabilityObservation": "app.runtime.capabilities.model", + "CapabilityOperationError": "app.runtime.capabilities.errors", + "CapabilityRegistry": "app.runtime.capabilities.registry", + "CapabilityRuntime": "app.runtime.capabilities.runtime", + "CapabilityRuntimeClosedError": "app.runtime.capabilities.errors", + "CapabilitySnapshot": "app.runtime.capabilities.model", + "CapabilitySpec": "app.runtime.capabilities.model", + "SelectorSchema": "app.runtime.capabilities.model", + "SelectorSpec": "app.runtime.capabilities.model", + "SyncCapabilityAdapter": "app.runtime.capabilities.model", + "UnknownCapabilityError": "app.runtime.capabilities.errors", +} + +__all__ = sorted(_EXPORT_MODULES) + + +def __getattr__(name: str) -> Any: + """仅在显式访问公共符号时导入对应叶模块，并缓存 canonical 对象。""" + module_name = _EXPORT_MODULES.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(module_name), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """向交互式工具公开稳定导出名，而不触发叶模块导入。""" + return sorted(set(globals()) | set(__all__)) diff --git a/app/runtime/capabilities/errors.py b/app/runtime/capabilities/errors.py new file mode 100644 index 000000000..977409996 --- /dev/null +++ b/app/runtime/capabilities/errors.py @@ -0,0 +1,32 @@ +class CapabilityError(RuntimeError): + """Capability Runtime 错误基类。""" + + +class CapabilityManifestError(CapabilityError, ValueError): + """能力声明不完整、冲突或违反当前 schema。""" + + +class UnknownCapabilityError(CapabilityError, KeyError): + """请求了 Registry 中不存在的能力。""" + + +class CapabilityAdapterModeError(CapabilityError, TypeError): + """调用入口与适配器并发模型不匹配。""" + + +class CapabilityAdapterContractError(CapabilityError, TypeError): + """适配器回调返回值不符合已声明的并发模型。""" + + +class CapabilityOperationError(CapabilityError): + """能力物化或资源生命周期转换失败。""" + + def __init__(self, capability_id: str, operation: str, error: BaseException): + self.capability_id = capability_id + self.operation = operation + self.error = error + super().__init__(f"能力 {capability_id} 执行 {operation} 失败：{error}") + + +class CapabilityRuntimeClosedError(CapabilityError): + """Runtime 已进入关闭态，禁止启动或重新物化能力。""" diff --git a/app/runtime/capabilities/model.py b/app/runtime/capabilities/model.py new file mode 100644 index 000000000..2e9346b59 --- /dev/null +++ b/app/runtime/capabilities/model.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from types import MappingProxyType +from typing import Any, Awaitable, Callable, Mapping, Optional, Protocol + + +class ActivationPolicy(str, Enum): + """能力的启动触发策略。""" + + BOOTSTRAP = "bootstrap" + WHEN_CONFIGURED = "when_configured" + ON_FIRST_USE = "on_first_use" + + +class AdapterExecutionMode(str, Enum): + """领域适配器执行回调所使用的并发模型。""" + + SYNC = "sync" + ASYNC = "async" + + +class CapabilityMaterializationState(str, Enum): + """Python 实现对象的解析状态。""" + + UNRESOLVED = "unresolved" + RESOLVING = "resolving" + RESOLVED = "resolved" + FAILED = "failed" + + +class CapabilityLifecycleState(str, Enum): + """能力所拥有外部资源的生命周期状态。""" + + DISCOVERED = "discovered" + STARTING = "starting" + RUNNING = "running" + RELOADING = "reloading" + STOPPING = "stopping" + STOPPED = "stopped" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class SelectorSchema: + """声明一个 selector 可接受的精确参数集合。""" + + required_fields: frozenset[str] = frozenset() + optional_fields: frozenset[str] = frozenset() + validator: Optional[Callable[[Mapping[str, Any]], None]] = None + + +@dataclass(frozen=True, slots=True) +class SelectorSpec: + """由领域适配器解释的配置选择器。""" + + kind: str + config: Mapping[str, Any] + + +@dataclass(frozen=True, slots=True) +class CapabilitySpec: + """从 data-only manifest 构建的不可变能力声明。""" + + schema_version: int + id: str + kind: str + entrypoint: str + activation: ActivationPolicy + metadata: Mapping[str, Any] + selector: Optional[SelectorSpec] + watch: tuple[str, ...] + depends_on: tuple[str, ...] + source: Path + + +@dataclass(frozen=True, slots=True) +class CapabilitySnapshot: + """能力状态的只读快照。""" + + capability_id: str + materialization: CapabilityMaterializationState + lifecycle: CapabilityLifecycleState + generation: int + visible: bool + error: Optional[str] + + +@dataclass(frozen=True, slots=True) +class CapabilityObservation: + """单次状态转换的可复现观测记录。""" + + capability_id: str + generation: int + operation: str + outcome: str + reason: str + materialization: CapabilityMaterializationState + lifecycle: CapabilityLifecycleState + duration_ms: float + error: Optional[str] + + +class SyncCapabilityAdapter(Protocol): + """同步领域适配器合同。""" + + execution_mode: AdapterExecutionMode + + def materialize(self, spec: CapabilitySpec) -> Any: + """解析 manifest entrypoint 对应的 canonical 实现对象。""" + + def create( + self, + spec: CapabilitySpec, + implementation: Any, + generation: int, + previous: Any = None, + ) -> Any: + """创建尚未对外发布的候选资源。""" + + def start(self, spec: CapabilitySpec, candidate: Any, generation: int) -> None: + """启动候选资源；返回前候选不会对普通查询可见。""" + + def stop(self, spec: CapabilitySpec, instance: Any, generation: int) -> None: + """停止已经撤销运行态可见性的资源。""" + + def cleanup( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> None: + """清理由失败启动留下的候选资源。""" + + +class AsyncCapabilityAdapter(Protocol): + """异步领域适配器合同。""" + + execution_mode: AdapterExecutionMode + + def materialize(self, spec: CapabilitySpec) -> Awaitable[Any]: + """异步解析 manifest entrypoint 对应的 canonical 实现对象。""" + + def create( + self, + spec: CapabilitySpec, + implementation: Any, + generation: int, + previous: Any = None, + ) -> Awaitable[Any]: + """异步创建尚未对外发布的候选资源。""" + + def start( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + ) -> Awaitable[None]: + """异步启动候选资源。""" + + def stop( + self, + spec: CapabilitySpec, + instance: Any, + generation: int, + ) -> Awaitable[None]: + """异步停止已经撤销运行态可见性的资源。""" + + def cleanup( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> Awaitable[None]: + """异步清理由失败启动留下的候选资源。""" + + +EMPTY_MAPPING: Mapping[str, Any] = MappingProxyType({}) diff --git a/app/runtime/capabilities/registry.py b/app/runtime/capabilities/registry.py new file mode 100644 index 000000000..7a1a8ceca --- /dev/null +++ b/app/runtime/capabilities/registry.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import re +import tomllib +from pathlib import Path +from types import MappingProxyType +from typing import Any, Collection, Iterable, Mapping + +from app.runtime.capabilities.errors import ( + CapabilityManifestError, + UnknownCapabilityError, +) +from app.runtime.capabilities.model import ( + ActivationPolicy, + CapabilitySpec, + SelectorSchema, + SelectorSpec, +) + + +_SCHEMA_VERSION = 1 +_MANIFEST_NAME = "capability.toml" +_TOP_LEVEL_FIELDS = frozenset({ + "schema_version", + "id", + "kind", + "entrypoint", + "metadata", + "activation", + "depends_on", +}) +_REQUIRED_FIELDS = _TOP_LEVEL_FIELDS +_ACTIVATION_FIELDS = frozenset({"policy", "watch", "selector"}) +_ACTIVATION_REQUIRED_FIELDS = frozenset({"policy", "watch"}) +_IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,127}$") +_KIND_PATTERN = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$") +_ENTRYPOINT_PATTERN = re.compile( + r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*" + r":[A-Za-z_][A-Za-z0-9_]*$" +) + + +def _freeze(value: Any, *, field: str) -> Any: + """把 TOML 容器递归转换为不可变结构，并拒绝非配置标量。""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return tuple(_freeze(item, field=field) for item in value) + if isinstance(value, dict): + if not all(isinstance(key, str) and key for key in value): + raise CapabilityManifestError(f"{field} 包含非法键") + return MappingProxyType({ + key: _freeze(item, field=f"{field}.{key}") + for key, item in value.items() + }) + raise CapabilityManifestError(f"{field} 包含不支持的 TOML 值类型 {type(value).__name__}") + + +def _string_list(value: Any, *, field: str, path: Path) -> tuple[str, ...]: + """校验无重复的非空字符串列表。""" + if not isinstance(value, list): + raise CapabilityManifestError(f"{path}: {field} 必须是字符串数组") + if any(not isinstance(item, str) or not item.strip() for item in value): + raise CapabilityManifestError(f"{path}: {field} 只能包含非空字符串") + normalized = tuple(item.strip() for item in value) + if len(set(normalized)) != len(normalized): + raise CapabilityManifestError(f"{path}: {field} 不能包含重复值") + return normalized + + +class CapabilityRegistry: + """只读取 data-only manifest 的不可变能力注册表。""" + + def __init__( + self, + specs: Mapping[str, CapabilitySpec], + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> None: + self._specs = MappingProxyType(dict(specs)) + self._kinds = frozenset(kinds) + self._selector_schemas = MappingProxyType(dict(selector_schemas)) + + @classmethod + def discover( + cls, + roots: Iterable[Path | str], + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> "CapabilityRegistry": + """扫描全部声明根；任何根或 manifest 非法都会阻止 Registry 构建。""" + normalized_kinds = frozenset(kinds) + if not normalized_kinds: + raise CapabilityManifestError("至少需要注册一个 capability kind") + for kind in normalized_kinds: + if not isinstance(kind, str) or not _KIND_PATTERN.fullmatch(kind): + raise CapabilityManifestError(f"非法 capability kind：{kind!r}") + + normalized_selectors = dict(selector_schemas) + for selector_type, schema in normalized_selectors.items(): + if not isinstance(selector_type, str) or not _KIND_PATTERN.fullmatch(selector_type): + raise CapabilityManifestError(f"非法 selector type：{selector_type!r}") + if not isinstance(schema, SelectorSchema): + raise CapabilityManifestError(f"selector {selector_type} 未提供 SelectorSchema") + overlap = schema.required_fields & schema.optional_fields + if overlap: + raise CapabilityManifestError( + f"selector {selector_type} 字段同时声明为 required/optional：{sorted(overlap)}" + ) + + specs: dict[str, CapabilitySpec] = {} + normalized_roots = tuple(Path(root) for root in roots) + if not normalized_roots: + raise CapabilityManifestError("至少需要一个 capability 声明根") + for root in normalized_roots: + if not root.is_dir(): + raise CapabilityManifestError(f"声明根不存在或不是目录：{root}") + manifests = sorted(root.rglob(_MANIFEST_NAME)) + if not manifests: + raise CapabilityManifestError(f"声明根没有 {_MANIFEST_NAME}：{root}") + for manifest_path in manifests: + spec = cls._load_manifest( + manifest_path, + kinds=normalized_kinds, + selector_schemas=normalized_selectors, + ) + previous = specs.get(spec.id) + if previous: + raise CapabilityManifestError( + f"capability id 重复：{spec.id}，来源 {previous.source} 与 {spec.source}" + ) + specs[spec.id] = spec + return cls( + specs, + kinds=normalized_kinds, + selector_schemas=normalized_selectors, + ) + + @classmethod + def _load_manifest( + cls, + path: Path, + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> CapabilitySpec: + try: + with path.open("rb") as file: + data = tomllib.load(file) + except (OSError, tomllib.TOMLDecodeError) as error: + raise CapabilityManifestError(f"无法读取 {path}：{error}") from error + + unknown_fields = set(data) - _TOP_LEVEL_FIELDS + if unknown_fields: + raise CapabilityManifestError(f"{path}: 未知字段 {sorted(unknown_fields)}") + missing_fields = _REQUIRED_FIELDS - set(data) + if missing_fields: + raise CapabilityManifestError(f"{path}: 缺少字段 {sorted(missing_fields)}") + + schema_version = data["schema_version"] + if type(schema_version) is not int or schema_version != _SCHEMA_VERSION: + raise CapabilityManifestError( + f"{path}: 不支持 schema_version={schema_version!r}" + ) + + capability_id = data["id"] + if not isinstance(capability_id, str) or not _IDENTIFIER_PATTERN.fullmatch(capability_id): + raise CapabilityManifestError(f"{path}: 非法 capability id={capability_id!r}") + + kind = data["kind"] + if not isinstance(kind, str) or kind not in kinds: + raise CapabilityManifestError(f"{path}: 未注册 capability kind={kind!r}") + + entrypoint = data["entrypoint"] + if not isinstance(entrypoint, str) or not _ENTRYPOINT_PATTERN.fullmatch(entrypoint): + raise CapabilityManifestError(f"{path}: 非法 entrypoint={entrypoint!r}") + + metadata = data["metadata"] + if not isinstance(metadata, dict): + raise CapabilityManifestError(f"{path}: metadata 必须是 table") + name = metadata.get("name") + if not isinstance(name, str) or not name.strip(): + raise CapabilityManifestError(f"{path}: metadata.name 必须是非空字符串") + immutable_metadata = _freeze(metadata, field="metadata") + + activation_data = data["activation"] + if not isinstance(activation_data, dict): + raise CapabilityManifestError(f"{path}: activation 必须是 table") + unknown_activation_fields = set(activation_data) - _ACTIVATION_FIELDS + missing_activation_fields = _ACTIVATION_REQUIRED_FIELDS - set(activation_data) + if unknown_activation_fields or missing_activation_fields: + raise CapabilityManifestError( + f"{path}: activation 字段非法，missing={sorted(missing_activation_fields)} " + f"unknown={sorted(unknown_activation_fields)}" + ) + try: + activation = ActivationPolicy(activation_data["policy"]) + except (TypeError, ValueError) as error: + raise CapabilityManifestError( + f"{path}: 非法 activation.policy={activation_data['policy']!r}" + ) from error + + selector = cls._parse_selector( + path, + activation=activation, + data=activation_data.get("selector"), + selector_schemas=selector_schemas, + ) + watch = _string_list(activation_data["watch"], field="activation.watch", path=path) + depends_on = _string_list(data["depends_on"], field="depends_on", path=path) + if depends_on: + raise CapabilityManifestError( + f"{path}: 当前 schema 不支持非空 depends_on={list(depends_on)!r}" + ) + + return CapabilitySpec( + schema_version=schema_version, + id=capability_id, + kind=kind, + entrypoint=entrypoint, + activation=activation, + metadata=immutable_metadata, + selector=selector, + watch=watch, + depends_on=depends_on, + source=path, + ) + + @staticmethod + def _parse_selector( + path: Path, + *, + activation: ActivationPolicy, + data: Any, + selector_schemas: Mapping[str, SelectorSchema], + ) -> SelectorSpec | None: + if activation is not ActivationPolicy.WHEN_CONFIGURED: + if data is not None: + raise CapabilityManifestError( + f"{path}: activation={activation.value} 时不允许 selector" + ) + return None + if not isinstance(data, dict): + raise CapabilityManifestError(f"{path}: when_configured 必须提供 selector table") + selector_kind = data.get("kind") + if not isinstance(selector_kind, str) or selector_kind not in selector_schemas: + raise CapabilityManifestError(f"{path}: 未注册 selector kind={selector_kind!r}") + config = {key: value for key, value in data.items() if key != "kind"} + schema = selector_schemas[selector_kind] + missing = schema.required_fields - set(config) + unknown = set(config) - schema.required_fields - schema.optional_fields + if missing or unknown: + raise CapabilityManifestError( + f"{path}: selector {selector_kind} 字段非法，" + f"missing={sorted(missing)} unknown={sorted(unknown)}" + ) + immutable_config = _freeze(config, field="selector") + if schema.validator: + try: + schema.validator(immutable_config) + except Exception as error: + raise CapabilityManifestError( + f"{path}: selector {selector_kind} 校验失败：{error}" + ) from error + return SelectorSpec(kind=selector_kind, config=immutable_config) + + @property + def kinds(self) -> frozenset[str]: + """返回该 Registry 接受的 capability kind。""" + return self._kinds + + def get_spec(self, capability_id: str) -> CapabilitySpec | None: + """查询声明；不存在时返回 None。""" + return self._specs.get(capability_id) + + def require_spec(self, capability_id: str) -> CapabilitySpec: + """查询必需声明；不存在时给出稳定的领域错误。""" + spec = self.get_spec(capability_id) + if spec is None: + raise UnknownCapabilityError(f"未知 capability：{capability_id}") + return spec + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """按 ID 返回稳定排序的声明快照。""" + return tuple(self._specs[key] for key in sorted(self._specs)) diff --git a/app/runtime/capabilities/runtime.py b/app/runtime/capabilities/runtime.py new file mode 100644 index 000000000..59712ac0b --- /dev/null +++ b/app/runtime/capabilities/runtime.py @@ -0,0 +1,1369 @@ +from __future__ import annotations + +import asyncio +import inspect +import sys +import threading +import time +from collections import deque +from concurrent.futures import Future +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, Callable, Mapping, Optional + +from app.runtime.capabilities.errors import ( + CapabilityAdapterContractError, + CapabilityAdapterModeError, + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.model import ( + AdapterExecutionMode, + CapabilityLifecycleState, + CapabilityMaterializationState, + CapabilityObservation, + CapabilitySnapshot, + CapabilitySpec, +) +from app.runtime.capabilities.registry import CapabilityRegistry + + +@dataclass(slots=True) +class _CapabilityState: + """Runtime 私有状态；所有可变字段均由 lock 保护。""" + + spec: CapabilitySpec + lock: threading.RLock = field(default_factory=threading.RLock) + materialization: CapabilityMaterializationState = CapabilityMaterializationState.UNRESOLVED + lifecycle: CapabilityLifecycleState = CapabilityLifecycleState.DISCOVERED + generation: int = 0 + implementation: Any = None + instance: Any = None + # 撤销可见性后仍未确认释放的资源由 Runtime 持有，禁止并行创建第二实例。 + pending_stop: Any = None + pending_cleanup: Any = None + pending_cleanup_error: Optional[BaseException] = None + last_error: Optional[BaseException] = None + inflight: Optional[Future[Any]] = None + inflight_operation: Optional[str] = None + + +class CapabilityRuntime: + """协调能力实现物化和资源生命周期的通用运行时。""" + + def __init__( + self, + registry: CapabilityRegistry, + *, + adapters: Mapping[str, Any], + observer: Optional[Callable[[CapabilityObservation], None]] = None, + observation_limit: int = 1024, + ) -> None: + if observation_limit <= 0: + raise ValueError("observation_limit 必须大于 0") + missing_adapters = {spec.kind for spec in registry.list_specs()} - set(adapters) + if missing_adapters: + raise CapabilityAdapterContractError( + f"缺少 capability adapter：{sorted(missing_adapters)}" + ) + self._registry = registry + self._adapters = MappingProxyType(dict(adapters)) + self._states = { + spec.id: _CapabilityState(spec=spec) + for spec in registry.list_specs() + } + self._observer = observer + self._observations: deque[CapabilityObservation] = deque(maxlen=observation_limit) + self._observation_lock = threading.Lock() + self._shutdown_lock = threading.Lock() + self._shutdown = False + + @property + def is_shutdown(self) -> bool: + """返回 Runtime 是否已进入不可逆关闭态。""" + return self._shutdown + + def _state(self, capability_id: str) -> _CapabilityState: + self._registry.require_spec(capability_id) + return self._states[capability_id] + + def get_spec(self, capability_id: str) -> CapabilitySpec | None: + """返回保留在 Registry 中的声明，不受运行失败影响。""" + return self._registry.get_spec(capability_id) + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """返回全部声明；FAILED 能力不会从列表中消失。""" + return self._registry.list_specs() + + def _adapter(self, state: _CapabilityState, expected: AdapterExecutionMode) -> Any: + adapter = self._adapters[state.spec.kind] + actual = getattr(adapter, "execution_mode", None) + if actual is not expected: + raise CapabilityAdapterModeError( + f"能力 {state.spec.id} 的 adapter mode={actual!r}，" + f"不能通过 {expected.value} 入口调用" + ) + return adapter + + def _ensure_open(self) -> None: + if self._shutdown: + raise CapabilityRuntimeClosedError("Capability Runtime 已关闭") + + @staticmethod + def _sync_callback(adapter: Any, name: str, *args) -> Any: + callback = getattr(adapter, name, None) + if not callable(callback): + raise CapabilityAdapterContractError(f"同步 adapter 缺少 {name}()") + result = callback(*args) + if inspect.isawaitable(result): + close = getattr(result, "close", None) + if callable(close): + close() + raise CapabilityAdapterContractError( + f"同步 adapter 的 {name}() 不能返回 awaitable" + ) + return result + + @staticmethod + async def _async_callback(adapter: Any, name: str, *args) -> Any: + callback = getattr(adapter, name, None) + if not callable(callback): + raise CapabilityAdapterContractError(f"异步 adapter 缺少 {name}()") + result = callback(*args) + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"异步 adapter 的 {name}() 必须返回 awaitable" + ) + return await result + + @staticmethod + def _wait_sync(future: Future[Any]) -> Any: + return future.result() + + @staticmethod + async def _wait_async(future: Future[Any]) -> Any: + return await asyncio.shield(asyncio.wrap_future(future)) + + def _emit( + self, + state: _CapabilityState, + *, + generation: int, + operation: str, + outcome: str, + reason: str, + started_at: float, + error: Optional[BaseException] = None, + ) -> None: + with state.lock: + observation = CapabilityObservation( + capability_id=state.spec.id, + generation=generation, + operation=operation, + outcome=outcome, + reason=reason, + materialization=state.materialization, + lifecycle=state.lifecycle, + duration_ms=max(0.0, (time.monotonic() - started_at) * 1000), + error=str(error) if error else None, + ) + with self._observation_lock: + self._observations.append(observation) + if self._observer: + try: + self._observer(observation) + except Exception: + # 观测消费者不能改变能力生命周期的成功或失败语义。 + pass + + @staticmethod + def _finish_transition( + state: _CapabilityState, + future: Future[Any], + *, + result: Any = None, + error: Optional[BaseException] = None, + ) -> None: + with state.lock: + if state.inflight is future: + state.inflight = None + state.inflight_operation = None + if error is None: + future.set_result(result) + else: + future.set_exception(error) + + def _calibrate_consumer_materialization(self, state: _CapabilityState) -> None: + """从 sys.modules 校准显式 consumer import，不触发任何新导入。""" + module_name, symbol_name = state.spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict) or symbol_name not in namespace: + return + canonical = namespace[symbol_name] + with state.lock: + if state.inflight is not None: + return + if state.materialization is CapabilityMaterializationState.RESOLVED: + if state.implementation is not canonical: + state.last_error = CapabilityAdapterContractError( + f"能力 {state.spec.id} 的 canonical implementation identity 发生变化" + ) + return + state.implementation = canonical + state.materialization = CapabilityMaterializationState.RESOLVED + if state.lifecycle is CapabilityLifecycleState.DISCOVERED: + state.last_error = None + generation = state.generation + self._emit( + state, + generation=generation, + operation="consumer_materialize", + outcome="succeeded", + reason="sys_modules", + started_at=time.monotonic(), + ) + + def snapshot(self, capability_id: str) -> CapabilitySnapshot: + """返回状态快照，并校准外部显式导入产生的 canonical 对象。""" + state = self._state(capability_id) + self._calibrate_consumer_materialization(state) + with state.lock: + return CapabilitySnapshot( + capability_id=state.spec.id, + materialization=state.materialization, + lifecycle=state.lifecycle, + generation=state.generation, + visible=state.instance is not None, + error=str(state.last_error) if state.last_error else None, + ) + + def observations(self, capability_id: Optional[str] = None) -> tuple[CapabilityObservation, ...]: + """返回按发生顺序记录的不可变观测快照。""" + with self._observation_lock: + items = tuple(self._observations) + if capability_id is None: + return items + return tuple(item for item in items if item.capability_id == capability_id) + + def get_running(self, capability_id: str) -> Any: + """只查询已发布实例，不触发物化或启动。""" + state = self._state(capability_id) + with state.lock: + return state.instance + + def materialize(self, capability_id: str, *, reason: str, retry: bool = False) -> Any: + """通过同步 adapter 显式物化实现，不启动领域资源。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.materialization is CapabilityMaterializationState.RESOLVED: + return state.implementation + if state.materialization is CapabilityMaterializationState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + "materialize", + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "materialize" + state.materialization = CapabilityMaterializationState.RESOLVING + waiter = None + waiter_operation = None + owner = (generation, future) + if waiter is not None: + result = self._wait_sync(waiter) + if waiter_operation == "materialize": + return result + continue + break + + generation, future = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="materialize", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + implementation = self._sync_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.last_error = None + self._finish_transition(state, future, result=implementation) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return implementation + except BaseException as error: + with state.lock: + state.materialization = CapabilityMaterializationState.FAILED + state.last_error = error + operation_error = self._wrap_error(state.spec.id, "materialize", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + raise operation_error from error + + async def materialize_async( + self, + capability_id: str, + *, + reason: str, + retry: bool = False, + ) -> Any: + """通过异步 adapter 显式物化实现，不阻塞事件循环。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.materialization is CapabilityMaterializationState.RESOLVED: + return state.implementation + if state.materialization is CapabilityMaterializationState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + "materialize", + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "materialize" + state.materialization = CapabilityMaterializationState.RESOLVING + waiter = None + waiter_operation = None + owner = (generation, future) + if waiter is not None: + result = await self._wait_async(waiter) + if waiter_operation == "materialize": + return result + continue + break + + generation, future = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="materialize", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + implementation = await self._async_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.last_error = None + self._finish_transition(state, future, result=implementation) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return implementation + except BaseException as error: + with state.lock: + state.materialization = CapabilityMaterializationState.FAILED + state.last_error = error + operation_error = self._wrap_error(state.spec.id, "materialize", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + raise operation_error from error + + @staticmethod + def _canonical_implementation(spec: CapabilitySpec, implementation: Any) -> Any: + if implementation is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {spec.id} 的 implementation" + ) + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict) or symbol_name not in namespace: + return implementation + canonical = namespace[symbol_name] + if implementation is not canonical: + raise CapabilityAdapterContractError( + f"adapter 返回的 {spec.id} 实现不是 sys.modules 中的 canonical 对象" + ) + return canonical + + @staticmethod + def _wrap_error(capability_id: str, operation: str, error: BaseException) -> BaseException: + if isinstance(error, (CapabilityOperationError, CapabilityRuntimeClosedError)): + return error + return CapabilityOperationError(capability_id, operation, error) + + def activate(self, capability_id: str, *, reason: str, retry: bool = False) -> Any: + """同步启动能力，并在 start 成功后原子发布候选实例。""" + return self._activate_sync(capability_id, reason=reason, retry=retry, previous=None) + + async def activate_async( + self, + capability_id: str, + *, + reason: str, + retry: bool = False, + ) -> Any: + """异步启动能力，并以 Future 协调并发调用者。""" + return await self._activate_async(capability_id, reason=reason, retry=retry, previous=None) + + def _activate_sync( + self, + capability_id: str, + *, + reason: str, + retry: bool, + previous: Any, + operation: str = "activate", + ) -> Any: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if operation == "activate" and state.instance is not None: + return state.instance + if state.pending_stop is not None and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.lifecycle is CapabilityLifecycleState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = operation + pending_stop = state.pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPING + if pending_stop is not None + else CapabilityLifecycleState.STARTING + ) + if state.implementation is None: + state.materialization = CapabilityMaterializationState.RESOLVING + pending_cleanup = state.pending_cleanup + pending_error = state.pending_cleanup_error + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + implementation = state.implementation + waiter = None + waiter_operation = None + owner = ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) + if waiter is not None: + result = self._wait_sync(waiter) + if waiter_operation == operation: + return result + continue + break + + ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation=operation, + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + try: + if pending_stop is not None: + self._sync_callback( + adapter, + "stop", + state.spec, + pending_stop, + generation, + ) + pending_stop = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STARTING + if pending_cleanup is not None: + try: + self._sync_callback( + adapter, + "cleanup", + state.spec, + pending_cleanup, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + except BaseException: + with state.lock: + state.pending_cleanup = pending_cleanup + state.pending_cleanup_error = pending_error + raise + if implementation is None: + implementation = self._sync_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + candidate = self._sync_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + self._sync_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation=operation, + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = self._cleanup_failed_sync( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.implementation = implementation + state.materialization = ( + CapabilityMaterializationState.RESOLVED + if implementation is not None + else CapabilityMaterializationState.FAILED + ) + state.instance = None + if pending_stop is not None: + state.pending_stop = pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, operation, final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation=operation, + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + async def _activate_async( + self, + capability_id: str, + *, + reason: str, + retry: bool, + previous: Any, + operation: str = "activate", + ) -> Any: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if operation == "activate" and state.instance is not None: + return state.instance + if state.pending_stop is not None and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.lifecycle is CapabilityLifecycleState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = operation + pending_stop = state.pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPING + if pending_stop is not None + else CapabilityLifecycleState.STARTING + ) + if state.implementation is None: + state.materialization = CapabilityMaterializationState.RESOLVING + pending_cleanup = state.pending_cleanup + pending_error = state.pending_cleanup_error + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + implementation = state.implementation + waiter = None + waiter_operation = None + owner = ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) + if waiter is not None: + result = await self._wait_async(waiter) + if waiter_operation == operation: + return result + continue + break + + ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation=operation, + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + try: + if pending_stop is not None: + await self._async_callback( + adapter, + "stop", + state.spec, + pending_stop, + generation, + ) + pending_stop = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STARTING + if pending_cleanup is not None: + try: + await self._async_callback( + adapter, + "cleanup", + state.spec, + pending_cleanup, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + except BaseException: + with state.lock: + state.pending_cleanup = pending_cleanup + state.pending_cleanup_error = pending_error + raise + if implementation is None: + implementation = await self._async_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + candidate = await self._async_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + await self._async_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation=operation, + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = await self._cleanup_failed_async( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.implementation = implementation + state.materialization = ( + CapabilityMaterializationState.RESOLVED + if implementation is not None + else CapabilityMaterializationState.FAILED + ) + state.instance = None + if pending_stop is not None: + state.pending_stop = pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, operation, final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation=operation, + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + def _cleanup_failed_sync( + self, + state: _CapabilityState, + adapter: Any, + candidate: Any, + generation: int, + error: BaseException, + ) -> Optional[BaseException]: + if candidate is None: + return None + try: + self._sync_callback( + adapter, + "cleanup", + state.spec, + candidate, + generation, + error, + ) + return None + except BaseException as cleanup_error: + with state.lock: + state.pending_cleanup = candidate + state.pending_cleanup_error = cleanup_error + return RuntimeError(f"{error}; cleanup failed: {cleanup_error}") + + async def _cleanup_failed_async( + self, + state: _CapabilityState, + adapter: Any, + candidate: Any, + generation: int, + error: BaseException, + ) -> Optional[BaseException]: + if candidate is None: + return None + try: + await self._async_callback( + adapter, + "cleanup", + state.spec, + candidate, + generation, + error, + ) + return None + except BaseException as cleanup_error: + with state.lock: + state.pending_cleanup = candidate + state.pending_cleanup_error = cleanup_error + return RuntimeError(f"{error}; cleanup failed: {cleanup_error}") + + def reload(self, capability_id: str, *, reason: str) -> Any: + """撤销当前实例后，通过同步 adapter 完成新 generation 的资源切换。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + else: + waiter = None + waiter_operation = None + if state.pending_stop is not None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.instance is None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("只有 RUNNING 能力可以 reload"), + ) + previous = state.instance + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "reload" + state.instance = None + state.lifecycle = CapabilityLifecycleState.RELOADING + implementation = state.implementation + break + result = self._wait_sync(waiter) + if waiter_operation == "reload": + return result + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="reload", + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + previous_released = False + try: + self._sync_callback(adapter, "stop", state.spec, previous, generation) + previous_released = True + candidate = self._sync_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + self._sync_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation="reload", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = self._cleanup_failed_sync( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.instance = None + if not previous_released: + state.pending_stop = previous + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, "reload", final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="reload", + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + async def reload_async(self, capability_id: str, *, reason: str) -> Any: + """撤销当前实例后，通过异步 adapter 完成新 generation 的资源切换。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + else: + waiter = None + waiter_operation = None + if state.pending_stop is not None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.instance is None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("只有 RUNNING 能力可以 reload"), + ) + previous = state.instance + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "reload" + state.instance = None + state.lifecycle = CapabilityLifecycleState.RELOADING + implementation = state.implementation + break + result = await self._wait_async(waiter) + if waiter_operation == "reload": + return result + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="reload", + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + previous_released = False + try: + await self._async_callback(adapter, "stop", state.spec, previous, generation) + previous_released = True + candidate = await self._async_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + await self._async_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation="reload", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = await self._cleanup_failed_async( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.instance = None + if not previous_released: + state.pending_stop = previous + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, "reload", final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="reload", + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + def stop(self, capability_id: str, *, reason: str) -> None: + """同步撤销并停止能力资源；物化实现保留供后续显式重启。""" + self._stop_sync(capability_id, reason=reason, shutdown=False) + + def _stop_sync(self, capability_id: str, *, reason: str, shutdown: bool) -> None: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + while True: + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + stop_owner = ( + state.instance + if state.instance is not None + else state.pending_stop + ) + pending = state.pending_cleanup + pending_error = state.pending_cleanup_error + if stop_owner is None and pending is None: + if state.lifecycle is not CapabilityLifecycleState.FAILED: + state.lifecycle = CapabilityLifecycleState.STOPPED + return + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "stop" + state.instance = None + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + state.lifecycle = CapabilityLifecycleState.STOPPING + waiter = None + waiter_operation = None + owner = (generation, future, stop_owner, pending, pending_error) + if waiter is not None: + try: + self._wait_sync(waiter) + except BaseException: + if not shutdown: + raise + if waiter_operation == "stop": + return + continue + break + + generation, future, stop_owner, pending, pending_error = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="stop", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + if stop_owner is not None: + self._sync_callback(adapter, "stop", state.spec, stop_owner, generation) + stop_owner = None + if pending is not None: + self._sync_callback( + adapter, + "cleanup", + state.spec, + pending, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + pending = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STOPPED + state.last_error = None + self._finish_transition(state, future, result=None) + self._emit( + state, + generation=generation, + operation="stop", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + except BaseException as error: + with state.lock: + state.lifecycle = CapabilityLifecycleState.FAILED + state.last_error = error + if stop_owner is not None: + state.pending_stop = stop_owner + if pending is not None: + state.pending_cleanup = pending + state.pending_cleanup_error = error + operation_error = self._wrap_error(state.spec.id, "stop", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="stop", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + if not shutdown: + raise operation_error from error + + async def stop_async(self, capability_id: str, *, reason: str) -> None: + """异步撤销并停止能力资源。""" + await self._stop_async(capability_id, reason=reason, shutdown=False) + + async def _stop_async(self, capability_id: str, *, reason: str, shutdown: bool) -> None: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + while True: + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + stop_owner = ( + state.instance + if state.instance is not None + else state.pending_stop + ) + pending = state.pending_cleanup + pending_error = state.pending_cleanup_error + if stop_owner is None and pending is None: + if state.lifecycle is not CapabilityLifecycleState.FAILED: + state.lifecycle = CapabilityLifecycleState.STOPPED + return + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "stop" + state.instance = None + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + state.lifecycle = CapabilityLifecycleState.STOPPING + waiter = None + waiter_operation = None + owner = (generation, future, stop_owner, pending, pending_error) + if waiter is not None: + try: + await self._wait_async(waiter) + except BaseException: + if not shutdown: + raise + if waiter_operation == "stop": + return + continue + break + + generation, future, stop_owner, pending, pending_error = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="stop", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + if stop_owner is not None: + await self._async_callback( + adapter, + "stop", + state.spec, + stop_owner, + generation, + ) + stop_owner = None + if pending is not None: + await self._async_callback( + adapter, + "cleanup", + state.spec, + pending, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + pending = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STOPPED + state.last_error = None + self._finish_transition(state, future, result=None) + self._emit( + state, + generation=generation, + operation="stop", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + except BaseException as error: + with state.lock: + state.lifecycle = CapabilityLifecycleState.FAILED + state.last_error = error + if stop_owner is not None: + state.pending_stop = stop_owner + if pending is not None: + state.pending_cleanup = pending + state.pending_cleanup_error = error + operation_error = self._wrap_error(state.spec.id, "stop", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="stop", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + if not shutdown: + raise operation_error from error + + def shutdown(self, *, reason: str) -> None: + """不可逆关闭仅含同步 adapter 的 Runtime，并阻止并发首启重新发布。""" + async_kinds = { + kind + for kind, adapter in self._adapters.items() + if getattr(adapter, "execution_mode", None) is AdapterExecutionMode.ASYNC + } + if async_kinds: + raise CapabilityAdapterModeError( + f"同步 shutdown 不能处理异步 adapter：{sorted(async_kinds)}" + ) + with self._shutdown_lock: + self._shutdown = True + for spec in self._registry.list_specs(): + self._stop_sync(spec.id, reason=reason, shutdown=True) + + async def shutdown_async(self, *, reason: str) -> None: + """不可逆关闭混合同步/异步 adapter 的 Runtime。""" + with self._shutdown_lock: + self._shutdown = True + for spec in self._registry.list_specs(): + adapter = self._adapters[spec.kind] + if getattr(adapter, "execution_mode", None) is AdapterExecutionMode.ASYNC: + await self._stop_async(spec.id, reason=reason, shutdown=True) + else: + await asyncio.to_thread( + self._stop_sync, + spec.id, + reason=reason, + shutdown=True, + ) diff --git a/app/runtime/events.py b/app/runtime/events.py index aea2174d6..983ca1ec4 100644 --- a/app/runtime/events.py +++ b/app/runtime/events.py @@ -415,21 +415,28 @@ class EventManager(metaclass=Singleton): 同步方式调度链式事件，按优先级顺序逐个调用事件处理器，并记录每个处理器的处理时间 :param event: 要调度的事件对象 """ - handlers = self.__chain_subscribers.get(event.event_type, {}) + # 运行期可以动态注册或移除处理器；当前事件始终使用调度开始时的快照。 + with self.__lock: + handlers = tuple( + self.__chain_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for chain event: {event}") return False # 过滤出启用的处理器 - enabled_handlers = {handler_id: (priority, handler) for handler_id, (priority, handler) in handlers.items() - if self.__is_handler_enabled(handler)} + enabled_handlers = tuple( + (handler_id, priority, handler) + for handler_id, (priority, handler) in handlers + if self.__is_handler_enabled(handler) + ) if not enabled_handlers: logger.debug(f"No enabled handlers found for chain event: {event}. Skipping execution.") return False self.__log_event_lifecycle(event, "Started") - for handler_id, (priority, handler) in enabled_handlers.items(): + for handler_id, priority, handler in enabled_handlers: start_time = time.time() self.__safe_invoke_handler(handler, event) logger.debug( @@ -444,21 +451,28 @@ class EventManager(metaclass=Singleton): 异步方式调度链式事件，按优先级顺序逐个调用事件处理器，并记录每个处理器的处理时间 :param event: 要调度的事件对象 """ - handlers = self.__chain_subscribers.get(event.event_type, {}) + # 快照在锁内建立、在锁外执行，处理器可以安全地修改后续订阅。 + with self.__lock: + handlers = tuple( + self.__chain_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for chain event: {event}") return False # 过滤出启用的处理器 - enabled_handlers = {handler_id: (priority, handler) for handler_id, (priority, handler) in handlers.items() - if self.__is_handler_enabled(handler)} + enabled_handlers = tuple( + (handler_id, priority, handler) + for handler_id, (priority, handler) in handlers + if self.__is_handler_enabled(handler) + ) if not enabled_handlers: logger.debug(f"No enabled handlers found for chain event: {event}. Skipping execution.") return False self.__log_event_lifecycle(event, "Started") - for handler_id, (priority, handler) in enabled_handlers.items(): + for handler_id, priority, handler in enabled_handlers: start_time = time.time() await self.__safe_invoke_handler_async(handler, event) logger.debug( @@ -473,7 +487,11 @@ class EventManager(metaclass=Singleton): 异步方式调度广播事件，通过线程池逐个调用事件处理器 :param event: 要调度的事件对象 """ - handlers = self.__broadcast_subscribers.get(event.event_type, {}) + # 快照隔离当前调度与运行期订阅变更；变更从下一个事件开始生效。 + with self.__lock: + handlers = tuple( + self.__broadcast_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for broadcast event: {event}") return @@ -481,7 +499,7 @@ class EventManager(metaclass=Singleton): if event.event_type == EventType.MessageAction and isinstance(event.event_data, dict): target_plugin_id = event.event_data.get("__mp_target_plugin_id") # 为每个处理器提供独立的事件实例，防止某个处理器对 event_data 的修改影响其他处理器 - for handler_id, handler in handlers.items(): + for handler_id, handler in handlers: if target_plugin_id and not self.__should_dispatch_to_target_plugin( handler, handler_id, str(target_plugin_id) ): diff --git a/app/runtime/extensions/host_module_adapter.py b/app/runtime/extensions/host_module_adapter.py new file mode 100644 index 000000000..633dd4bdf --- /dev/null +++ b/app/runtime/extensions/host_module_adapter.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping + +from app.runtime.capabilities.model import ( + ActivationPolicy, + AdapterExecutionMode, + CapabilitySpec, + SelectorSchema, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.config import settings +from app.runtime.extensions.service_config import ServiceConfigHelper +from app.schemas.types import ( + DownloaderType, + MediaRecognizeType, + MediaServerType, + MessageChannel, + ModuleType, + OtherModulesType, + StorageSchema, + SystemConfigKey, +) + + +HOST_MODULE_KIND = "host_module" +_SETTING_SELECTOR = "setting_truthy" +_SERVICE_SELECTOR = "system_config_item" +_MODULE_ROOT = Path(__file__).resolve().parents[2] / "modules" +_SERVICE_CONFIG_GETTERS = MappingProxyType({ + SystemConfigKey.Downloaders.value: ServiceConfigHelper.get_downloader_configs, + SystemConfigKey.MediaServers.value: ServiceConfigHelper.get_mediaserver_configs, + SystemConfigKey.Notifications.value: ServiceConfigHelper.get_notification_configs, +}) +_SUBTYPE_NAMES = frozenset( + item.name + for enum_type in ( + DownloaderType, + MediaServerType, + MessageChannel, + StorageSchema, + OtherModulesType, + MediaRecognizeType, + ) + for item in enum_type +) + + +@dataclass(frozen=True, slots=True) +class HostModuleConfigSnapshot: + """一次 reconcile 使用的不可变配置视图，避免每个能力重复查询配置。""" + + settings: Mapping[str, Any] + services: Mapping[str, tuple[Any, ...]] + + +def _validate_setting_selector(config: Mapping[str, Any]) -> None: + """限制 setting selector 只能读取已声明的应用设置。""" + key = config["key"] + if not isinstance(key, str) or not key or not hasattr(settings, key): + raise ValueError(f"未知应用设置：{key!r}") + + +def _validate_service_selector(config: Mapping[str, Any]) -> None: + """限制服务 selector 使用经过 Schema 校验的三个宿主服务配置。""" + key = config["key"] + if key not in _SERVICE_CONFIG_GETTERS: + raise ValueError(f"不支持的服务配置：{key!r}") + if config["match_field"] != "type": + raise ValueError("服务 selector 的 match_field 必须是 type") + if config["enabled_field"] != "enabled": + raise ValueError("服务 selector 的 enabled_field 必须是 enabled") + match_value = config["match_value"] + if not isinstance(match_value, str) or not match_value: + raise ValueError("服务 selector 的 match_value 必须是非空字符串") + + +HOST_MODULE_SELECTOR_SCHEMAS = MappingProxyType({ + _SETTING_SELECTOR: SelectorSchema( + required_fields=frozenset({"key"}), + validator=_validate_setting_selector, + ), + _SERVICE_SELECTOR: SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + validator=_validate_service_selector, + ), +}) + + +def _validate_manifest_inventory(registry: CapabilityRegistry) -> None: + """校验一级模块包与 manifest 一一对应，并固定宿主声明合同。""" + module_packages = { + child.name + for child in _MODULE_ROOT.iterdir() + if child.is_dir() + and not child.name.startswith("_") + and (child / "__init__.py").is_file() + } + specs = registry.list_specs() + manifest_packages = {spec.source.parent.name for spec in specs} + if module_packages != manifest_packages: + missing = sorted(module_packages - manifest_packages) + unknown = sorted(manifest_packages - module_packages) + raise ValueError( + f"Host Module manifest inventory 不一致：missing={missing} unknown={unknown}" + ) + + allowed_metadata = {"name", "type", "subtype", "priority"} + module_type_values = {item.value for item in ModuleType} + for spec in specs: + if spec.source.parent.parent != _MODULE_ROOT: + raise ValueError(f"Host Module manifest 必须位于一级模块包：{spec.source}") + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + expected_module = f"app.modules.{spec.source.parent.name}" + if module_name != expected_module or symbol_name != spec.id: + raise ValueError( + f"{spec.source}: entrypoint 必须指向同包且类名等于 capability id" + ) + if set(spec.metadata) != allowed_metadata: + raise ValueError( + f"{spec.source}: metadata 字段必须是 {sorted(allowed_metadata)}" + ) + if spec.metadata["type"] not in module_type_values: + raise ValueError(f"{spec.source}: 非法 metadata.type={spec.metadata['type']!r}") + if spec.metadata["subtype"] not in _SUBTYPE_NAMES: + raise ValueError( + f"{spec.source}: 非法 metadata.subtype={spec.metadata['subtype']!r}" + ) + priority = spec.metadata["priority"] + if isinstance(priority, bool) or not isinstance(priority, int): + raise ValueError(f"{spec.source}: metadata.priority 必须是整数") + if spec.activation is ActivationPolicy.WHEN_CONFIGURED: + selector_key = str(spec.selector.config["key"]) + if selector_key not in spec.watch: + raise ValueError( + f"{spec.source}: activation.watch 必须包含 selector 配置键" + ) + + +def build_host_module_registry() -> CapabilityRegistry: + """从现有物理模块包构建 import-free Host Module Registry。""" + registry = CapabilityRegistry.discover( + (_MODULE_ROOT,), + kinds={HOST_MODULE_KIND}, + selector_schemas=HOST_MODULE_SELECTOR_SCHEMAS, + ) + _validate_manifest_inventory(registry) + return registry + + +def capture_host_module_config( + specs: tuple[CapabilitySpec, ...], +) -> HostModuleConfigSnapshot: + """对本轮涉及的设置和服务配置各读取一次并冻结容器。""" + setting_keys: set[str] = set() + service_keys: set[str] = set() + for spec in specs: + selector = spec.selector + if selector is None: + continue + key = str(selector.config["key"]) + if selector.kind == _SETTING_SELECTOR: + setting_keys.add(key) + elif selector.kind == _SERVICE_SELECTOR: + service_keys.add(key) + + setting_values = { + key: getattr(settings, key) + for key in sorted(setting_keys) + } + service_values = { + key: tuple(_SERVICE_CONFIG_GETTERS[key]()) + for key in sorted(service_keys) + } + return HostModuleConfigSnapshot( + settings=MappingProxyType(setting_values), + services=MappingProxyType(service_values), + ) + + +def should_run_host_module( + spec: CapabilitySpec, + snapshot: HostModuleConfigSnapshot, +) -> bool: + """依据有限 selector 语法判断能力是否应拥有运行资源。""" + if spec.activation is ActivationPolicy.BOOTSTRAP: + return True + if spec.activation is ActivationPolicy.ON_FIRST_USE: + return False + selector = spec.selector + if selector is None: + return False + if selector.kind == _SETTING_SELECTOR: + return bool(snapshot.settings[selector.config["key"]]) + if selector.kind == _SERVICE_SELECTOR: + config = selector.config + return any( + getattr(item, config["match_field"]) == config["match_value"] + and bool(getattr(item, config["enabled_field"])) + for item in snapshot.services[config["key"]] + ) + raise ValueError(f"未支持的 Host Module selector：{selector.kind}") + + +class HostModuleAdapter: + """把现有模块类接入 Capability Runtime，保持类路径与对象 identity 不变。""" + + execution_mode = AdapterExecutionMode.SYNC + + @staticmethod + def materialize(spec: CapabilitySpec) -> type: + """按 manifest entrypoint 导入并返回原始模块类。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + implementation = getattr(importlib.import_module(module_name), symbol_name) + if not isinstance(implementation, type): + raise TypeError(f"{spec.entrypoint} 不是模块类") + return implementation + + @staticmethod + def create( + spec: CapabilitySpec, + implementation: type, + generation: int, + previous: Any = None, + ) -> Any: + """首次创建实例；配置重载继续使用原实例以保留既有模块语义。""" + del spec, generation + return previous if previous is not None else implementation() + + @staticmethod + def start(spec: CapabilitySpec, candidate: Any, generation: int) -> None: + """初始化候选实例拥有的连接、线程或客户端资源。""" + del spec, generation + candidate.init_module() + + @staticmethod + def stop(spec: CapabilitySpec, instance: Any, generation: int) -> None: + """停止实例拥有的资源；Runtime 会先撤销其运行态可见性。""" + del spec, generation + instance.stop() + + @staticmethod + def cleanup( + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> None: + """启动失败后尽力回收候选实例已创建的部分资源。""" + del spec, generation, error + candidate.stop() diff --git a/app/runtime/extensions/module_manager.py b/app/runtime/extensions/module_manager.py index dd189cecc..88efff72d 100644 --- a/app/runtime/extensions/module_manager.py +++ b/app/runtime/extensions/module_manager.py @@ -1,22 +1,42 @@ -import traceback -from typing import Generator, Optional, Tuple, Any, Union, List +from __future__ import annotations + +import sys +import threading +from typing import Any, Generator, List, Optional, Tuple, Union -from app.runtime.config import settings -from app.runtime.events import EventHandlerBinding, eventmanager -from app.foundation.reflection import ModuleHelper -from app.runtime.log import logger -from app.schemas.types import EventType, ModuleType, DownloaderType, MediaServerType, MessageChannel, StorageSchema, \ - OtherModulesType, MediaRecognizeType from app.foundation.reflection import ObjectUtils from app.foundation.singleton import Singleton +from app.runtime.capabilities.model import ( + CapabilityLifecycleState, + CapabilityObservation, + CapabilitySpec, +) +from app.runtime.capabilities.runtime import CapabilityRuntime +from app.runtime.config import settings +from app.runtime.events import Event, EventHandlerBinding, eventmanager +from app.runtime.extensions.host_module_adapter import ( + HOST_MODULE_KIND, + HostModuleAdapter, + build_host_module_registry, + capture_host_module_config, + should_run_host_module, +) +from app.runtime.log import logger +from app.schemas.types import ( + DownloaderType, + EventType, + MediaRecognizeType, + MediaServerType, + MessageChannel, + ModuleType, + OtherModulesType, + StorageSchema, +) class ModuleManager(metaclass=Singleton): - """ - 模块管理器 - """ + """以 Capability Runtime 管理宿主模块，并保留旧插件同步查询合同。""" - # 子模块类型集合 SubType = Union[ DownloaderType, MediaServerType, @@ -26,176 +46,286 @@ class ModuleManager(metaclass=Singleton): MediaRecognizeType, ] - def __init__(self): - """初始化模块注册表并装载当前启用的运行模块。""" - # 模块列表 - self._modules: dict = {} - # 运行态模块列表 - self._running_modules: dict = {} - # 事件总线通过该解析器绑定已启用的模块实例。 + def __init__(self) -> None: + """发现 data-only manifest，并按当前配置激活所需宿主模块。""" + self._lock = threading.RLock() + self._lifecycle_lock = threading.RLock() + self._modules: dict[str, type] = {} + self._running_modules: dict[str, Any] = {} + registry = build_host_module_registry() + self._runtime = CapabilityRuntime( + registry, + adapters={HOST_MODULE_KIND: HostModuleAdapter()}, + observer=self._observe_transition, + ) + # pkgutil 的既有发现顺序按一级包名稳定排列，兼容视图继续保持该顺序。 + self._specs = tuple( + sorted(self._runtime.list_specs(), key=lambda item: item.source.parent.name) + ) eventmanager.register_handler_instance_resolver( "modules", self.resolve_event_handler_instance, ) + eventmanager.add_event_listener( + EventType.ConfigChanged, + self.handle_config_changed, + ) self.load_modules() + @staticmethod + def _observe_transition(observation: CapabilityObservation) -> None: + """把 Runtime 的稳定转换结果接入现有日志面。""" + if observation.outcome == "failed": + logger.error( + "Host Module %s %s 失败：%s", + observation.capability_id, + observation.operation, + observation.error, + ) + elif observation.outcome == "succeeded": + logger.debug( + "Host Module %s %s 完成，generation=%s，耗时=%.2fms", + observation.capability_id, + observation.operation, + observation.generation, + observation.duration_ms, + ) + + @staticmethod + def _event_changed_keys(event: Optional[Event]) -> set[str]: + """兼容对象和 dict 两种配置事件载荷。""" + if not event: + return set() + event_data = event.event_data + if isinstance(event_data, dict): + keys = event_data.get("key", set()) + else: + keys = getattr(event_data, "key", set()) + if isinstance(keys, str): + return {keys} + return {str(key) for key in (keys or set())} + + def _remember_materialized(self, module_id: str, implementation: type) -> type: + """更新旧 `_modules` 视图，但不改变能力资源生命周期。""" + with self._lock: + self._modules[module_id] = implementation + return implementation + + def _consumer_materialized_class(self, spec: CapabilitySpec) -> Optional[type]: + """识别插件显式旧导入产生的真实类，不触发新的 Python import。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict): + return None + implementation = namespace.get(symbol_name) + return implementation if isinstance(implementation, type) else None + + def _refresh_running_projection(self) -> None: + """从 Runtime 已发布实例重建插件可见的运行模块字典。""" + running = { + spec.id: instance + for spec in self._specs + if (instance := self._runtime.get_running(spec.id)) is not None + } + with self._lock: + self._running_modules = running + def resolve_event_handler_instance( - self, - owner_class: type, + self, + owner_class: type, ) -> Optional[EventHandlerBinding]: - """为模块声明的事件方法解析当前运行实例。""" - module_id = owner_class.__name__ - if module_id not in self._modules: - return None - module = self._running_modules.get(module_id) - owner_name = module_id - if module and callable(getattr(module, "get_name", None)): - owner_name = module.get_name() - return EventHandlerBinding( - instance=module, - owner_name=owner_name, + """按 canonical class identity 绑定当前 generation，停止态阻断 fallback 构造。""" + for spec in self._specs: + with self._lock: + implementation = self._modules.get(spec.id) + if implementation is None: + implementation = self._consumer_materialized_class(spec) + if implementation is not None: + self._remember_materialized(spec.id, implementation) + # 同步 Runtime 的物化观测，但不创建或启动实例。 + self._runtime.snapshot(spec.id) + if implementation is not owner_class: + continue + return EventHandlerBinding( + instance=self._runtime.get_running(spec.id), + owner_name=str(spec.metadata["name"]), + ) + return None + + def _reconcile( + self, + *, + reason: str, + changed_keys: Optional[set[str]] = None, + reload_running: bool = False, + ) -> None: + """以一次配置快照串行协调需要启动、重载或停止的能力。""" + with self._lifecycle_lock: + selected = tuple( + spec + for spec in self._specs + if changed_keys is None or changed_keys.intersection(spec.watch) + ) + snapshot = capture_host_module_config(selected) + for spec in selected: + desired = should_run_host_module(spec, snapshot) + running = self._runtime.get_running(spec.id) + try: + if desired and running is None: + instance = self._runtime.activate( + spec.id, + reason=reason, + retry=True, + ) + self._remember_materialized(spec.id, type(instance)) + elif desired and running is not None and reload_running: + instance = self._runtime.reload(spec.id, reason=reason) + self._remember_materialized(spec.id, type(instance)) + elif not desired and running is not None: + self._runtime.stop(spec.id, reason=reason) + except Exception: + # 单能力失败由 Runtime 完整记录；其它无依赖能力继续 reconcile。 + continue + self._refresh_running_projection() + + def load_modules(self) -> None: + """按当前配置启动未运行模块；已运行模块保持当前 generation。""" + self._reconcile(reason="module_manager_load") + + def handle_config_changed(self, event: Event) -> None: + """配置变更时仅协调 watch 命中的能力，并保证单一生命周期 writer。""" + changed_keys = self._event_changed_keys(event) + if not changed_keys: + return + self._reconcile( + reason="config_changed", + changed_keys=changed_keys, + reload_running=True, ) - def load_modules(self): - """ - 加载所有模块 - """ - # 扫描模块目录 - modules = ModuleHelper.load( - "app.modules", - filter_func=lambda _, obj: hasattr(obj, 'init_module') and hasattr(obj, 'init_setting') - ) - self._running_modules = {} - self._modules = {} - for module in modules: - module_id = module.__name__ - self._modules[module_id] = module - try: - # 生成实例 - _module = module() - # 初始化模块 - if self.check_setting(_module.init_setting()): - # 通过模板开关控制加载 - _module.init_module() - self._running_modules[module_id] = _module - logger.debug(f"Moudle Loaded：{module_id}") - except Exception as err: - logger.error(f"Load Moudle Error：{module_id}，{str(err)} - {traceback.format_exc()}", exc_info=True) - - def stop(self): - """ - 停止所有模块 - """ + def stop(self) -> None: + """停止全部运行模块但保留 Runtime，使旧插件可随后再次 load。""" logger.info("正在停止所有模块...") - for module_id, module in self._running_modules.items(): - try: - module.stop() - logger.debug(f"Moudle Stoped：{module_id}") - except Exception as err: - logger.error(f"Stop Moudle Error：{module_id}，{str(err)} - {traceback.format_exc()}", exc_info=True) + with self._lifecycle_lock: + for spec in reversed(self._specs): + snapshot = self._runtime.snapshot(spec.id) + if ( + self._runtime.get_running(spec.id) is None + and snapshot.lifecycle is not CapabilityLifecycleState.FAILED + ): + continue + try: + self._runtime.stop(spec.id, reason="module_manager_stop") + except Exception: + continue + self._refresh_running_projection() logger.info("所有模块停止完成") - def reload(self): - """ - 重新加载所有模块 - """ - self.stop() - self.load_modules() - eventmanager.send_event(etype=EventType.ModuleReload, data={}) + def shutdown(self) -> None: + """进程关闭时不可逆停止 Runtime，阻止并发能力重新发布。""" + logger.info("正在关闭模块运行时...") + with self._lifecycle_lock: + self._runtime.shutdown(reason="application_shutdown") + self._refresh_running_projection() + logger.info("模块运行时关闭完成") + + def reload(self) -> None: + """保留旧插件可观察的 stop、load、ModuleReload 同步顺序。""" + with self._lifecycle_lock: + self.stop() + self.load_modules() + eventmanager.send_event(etype=EventType.ModuleReload, data={}) def test(self, modleid: str) -> Tuple[bool, str]: - """ - 测试模块 - """ - if modleid not in self._running_modules: + """测试已运行模块；未启用模块保持旧合同返回 `(False, "")`。""" + module = self.get_running_module(modleid) + if module is None: return False, "" - module = self._running_modules[modleid] - if hasattr(module, "test") \ - and ObjectUtils.check_method(getattr(module, "test")): + if hasattr(module, "test") and ObjectUtils.check_method(module.test): result = module.test() - if not result: - return False, "" - return result + return result if result else (False, "") return True, "模块不支持测试" @staticmethod def check_setting(setting: Optional[tuple]) -> bool: - """ - 检查开关是否己打开，开关使用,分隔多个值，符合其中即代表开启 - """ + """保留旧模块开关的 truthy 与 membership 判定语义。""" if not setting: return True switch, value = setting option = getattr(settings, switch) if not option: return False - if option and value is True: + if value is True: return True - if value in option: - return True - return False + return value in option def get_running_module(self, module_id: str) -> Any: - """ - 根据模块id获取模块运行实例 - """ - if not module_id: + """根据模块 ID 返回已发布的运行实例，不触发物化。""" + if not module_id or self._runtime.get_spec(module_id) is None: return None - if not self._running_modules: - return None - return self._running_modules.get(module_id) + return self._runtime.get_running(module_id) + + def _running_snapshot(self) -> tuple[Any, ...]: + """直接读取 Runtime 发布视图，转换期间不暴露旧或候选实例。""" + return tuple( + instance + for spec in self._specs + if (instance := self._runtime.get_running(spec.id)) is not None + ) def get_running_modules(self, method: str) -> Generator: - """ - 获取实现了同一方法的模块列表 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, method) \ - and ObjectUtils.check_method(getattr(module, method)): + """返回实现了指定方法的运行模块快照。""" + for module in self._running_snapshot(): + candidate = getattr(module, method, None) + if callable(candidate) and ObjectUtils.check_method(candidate): yield module def get_running_type_modules(self, module_type: ModuleType) -> Generator: - """ - 获取指定类型的模块列表 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, 'get_type') \ - and module.get_type() == module_type: + """返回指定类型的运行模块快照。""" + for module in self._running_snapshot(): + if module.get_type() == module_type: yield module def get_running_subtype_module(self, module_subtype: SubType) -> Generator: - """ - 获取指定子类型的模块 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, 'get_subtype') \ - and module.get_subtype() == module_subtype: + """返回指定子类型的运行模块快照。""" + for module in self._running_snapshot(): + if module.get_subtype() == module_subtype: yield module def get_module(self, module_id: str) -> Any: - """ - 根据模块id获取模块 - """ - if not module_id: + """显式物化并返回 canonical 模块类；失败保持旧合同返回 None。""" + if not module_id or self._runtime.get_spec(module_id) is None: return None - if not self._modules: + with self._lock: + implementation = self._modules.get(module_id) + if implementation is not None: + return implementation + try: + implementation = self._runtime.materialize( + module_id, + reason="compat_get_module", + retry=True, + ) + except Exception: return None - return self._modules.get(module_id) + return self._remember_materialized(module_id, implementation) - def get_modules(self) -> dict: - """ - 获取模块列表 - """ - return self._modules + def get_modules(self) -> dict[str, type]: + """兼容性显式物化全部真实类；单个失败不阻断其它模块。""" + for spec in self._specs: + self.get_module(spec.id) + with self._lock: + return dict(self._modules) def get_module_ids(self) -> List[str]: - """ - 获取模块id列表 - """ - return list(self._modules.keys()) + """从 manifest 返回全部模块 ID，不物化实现。""" + return [spec.id for spec in self._specs] + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """返回全部轻量模块声明，包含物化或启动失败的能力。""" + return self._specs + + def get_specs(self) -> tuple[CapabilitySpec, ...]: + """兼容内部调用命名，返回与 `list_specs` 相同的声明快照。""" + return self.list_specs() diff --git a/app/runtime/extensions/service_config.py b/app/runtime/extensions/service_config.py new file mode 100644 index 000000000..39a083fda --- /dev/null +++ b/app/runtime/extensions/service_config.py @@ -0,0 +1,76 @@ +from typing import List, Optional, Type + +from pydantic import ValidationError + +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.log import logger +from app.schemas import ( + DownloaderConf, + MediaServerConf, + NotificationConf, + NotificationSwitchConf, +) +from app.schemas.types import NotificationType, SystemConfigKey + + +class ServiceConfigHelper: + """读取并校验通知、下载器和媒体服务器的宿主配置。""" + + @staticmethod + def get_configs(config_key: SystemConfigKey, conf_type: Type) -> List: + """按指定 Schema 过滤单条非法配置，避免影响同组其它服务。""" + config_data = SystemConfigOper().get(config_key) + if not config_data: + return [] + configs = [] + for conf in config_data: + if not isinstance(conf, dict): + logger.warning(f"{config_key.value} 配置格式不正确，已跳过：{conf}") + continue + try: + configs.append(conf_type(**conf)) + except ValidationError as err: + logger.error( + f"{config_key.value} 配置 {conf.get('name')} 校验失败，已跳过：{err}" + ) + return configs + + @staticmethod + def get_downloader_configs() -> List[DownloaderConf]: + """返回已通过结构校验的下载器配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.Downloaders, + DownloaderConf, + ) + + @staticmethod + def get_mediaserver_configs() -> List[MediaServerConf]: + """返回已通过结构校验的媒体服务器配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.MediaServers, + MediaServerConf, + ) + + @staticmethod + def get_notification_configs() -> List[NotificationConf]: + """返回已通过结构校验的通知配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.Notifications, + NotificationConf, + ) + + @staticmethod + def get_notification_switches() -> List[NotificationSwitchConf]: + """返回已通过结构校验的通知场景开关。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.NotificationSwitchs, + NotificationSwitchConf, + ) + + @staticmethod + def get_notification_switch(mtype: NotificationType) -> Optional[str]: + """返回指定通知场景的目标范围。""" + for switch in ServiceConfigHelper.get_notification_switches(): + if switch.type == mtype.value: + return switch.action + return None diff --git a/app/runtime/extensions/service_registry.py b/app/runtime/extensions/service_registry.py index 38f559270..610953501 100644 --- a/app/runtime/extensions/service_registry.py +++ b/app/runtime/extensions/service_registry.py @@ -1,84 +1,18 @@ from typing import Dict, List, Optional, Type, TypeVar, Generic, Iterator -from pydantic import ValidationError - -from app.runtime.extensions.module_manager import ModuleManager from app.db.oper.systemconfig import SystemConfigOper -from app.runtime.log import logger -from app.schemas import DownloaderConf, MediaServerConf, NotificationConf, NotificationSwitchConf, ServiceInfo -from app.schemas.types import NotificationType, SystemConfigKey, ModuleType +from app.runtime.extensions.module_manager import ModuleManager +from app.runtime.extensions.service_config import ServiceConfigHelper +from app.schemas import ServiceInfo +from app.schemas.types import SystemConfigKey, ModuleType TConf = TypeVar("TConf") - -class ServiceConfigHelper: - """ - 配置帮助类，获取不同类型的服务配置 - """ - - @staticmethod - def get_configs(config_key: SystemConfigKey, conf_type: Type) -> List: - """ - 通用获取配置的方法，根据 config_key 获取相应的配置并返回指定类型的配置列表 - - :param config_key: 系统配置的 key - :param conf_type: 用于实例化配置对象的类类型 - :return: 配置对象列表 - """ - config_data = SystemConfigOper().get(config_key) - if not config_data: - return [] - configs = [] - for conf in config_data: - if not isinstance(conf, dict): - logger.warn(f"{config_key.value} 配置格式不正确，已跳过：{conf}") - continue - try: - # 直接使用 conf_type 来实例化配置对象 - configs.append(conf_type(**conf)) - except ValidationError as e: - # 单条配置存在非法值时跳过，避免影响其它服务的初始化 - logger.error(f"{config_key.value} 配置 {conf.get('name')} 校验失败，已跳过：{e}") - return configs - - @staticmethod - def get_downloader_configs() -> List[DownloaderConf]: - """ - 获取下载器的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.Downloaders, DownloaderConf) - - @staticmethod - def get_mediaserver_configs() -> List[MediaServerConf]: - """ - 获取媒体服务器的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.MediaServers, MediaServerConf) - - @staticmethod - def get_notification_configs() -> List[NotificationConf]: - """ - 获取消息通知渠道的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.Notifications, NotificationConf) - - @staticmethod - def get_notification_switches() -> List[NotificationSwitchConf]: - """ - 获取消息通知场景的开关 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.NotificationSwitchs, NotificationSwitchConf) - - @staticmethod - def get_notification_switch(mtype: NotificationType) -> Optional[str]: - """ - 获取指定类型的消息通知场景的开关 - """ - switchs = ServiceConfigHelper.get_notification_switches() - for switch in switchs: - if switch.type == mtype.value: - return switch.action - return None +__all__ = [ + "ServiceBaseHelper", + "ServiceConfigHelper", + "SystemConfigOper", +] class ServiceBaseHelper(Generic[TConf]): diff --git a/app/runtime/reload.py b/app/runtime/reload.py index 8d4bc6948..dac3e52a1 100644 --- a/app/runtime/reload.py +++ b/app/runtime/reload.py @@ -14,10 +14,16 @@ class ConfigReloadMixin: 可选地重写 get_reload_name 方法提供模块名称（用于日志显示） """ + # 统一生命周期管理器可以继承此 Mixin 的重载方法，但由外部唯一负责事件绑定。 + CONFIG_RELOAD_MANAGED_EXTERNALLY: bool = False + def __init_subclass__(cls, **kwargs): """为声明了 CONFIG_WATCH 的子类生成配置变更处理器。""" super().__init_subclass__(**kwargs) + if getattr(cls, "CONFIG_RELOAD_MANAGED_EXTERNALLY", False): + return + config_watch = getattr(cls, "CONFIG_WATCH", None) if not config_watch: return diff --git a/app/startup/modules_initializer.py b/app/startup/modules_initializer.py index 0fdedcf14..a54b29ef9 100644 --- a/app/startup/modules_initializer.py +++ b/app/startup/modules_initializer.py @@ -184,7 +184,7 @@ async def stop_modules(): logger.error(f"关闭{name}失败：{err}") await run_step("AI智能体", stop_agent) - await run_step("模块", lambda: ModuleManager().stop()) + await run_step("模块", lambda: ModuleManager().shutdown()) await run_step("事件消费", lambda: EventManager().stop()) await run_step("虚拟显示", lambda: DisplayHelper().stop()) await run_step("DoH服务", lambda: DohHelper().shutdown()) diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 000000000..42597b6ac --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,106 @@ +# MoviePilot Docker A/B Harness + +该工具用同一个冻结 Docker substrate 对两个 Git commit 做源码级 A/B。派生镜像会先清空 +`/app`，复制目标 commit 的完整 `git archive`，再从 substrate 注入镜像构建阶段生成的插件目录、 +`sites.*.so` 和 `user.sites.v3.bin`。如果依赖或 Docker substrate 输入发生变化，工具会拒绝继续， +避免把外部构建输入漂移误算成性能收益。 + +工具不会读取工作区或容器内的 `app.env`，不挂载真实配置、媒体目录或 Docker socket。固定配置只使用 +内置实验室占位凭据，样本不发布主机端口，并运行在无外网的 internal network。 + +## 一次性预热浏览器 + +默认从固定命名 volume `mp-perf-v3-browser-seed` 复制 CloakBrowser 缓存。该 volume 只需联网预热 +一次： + +```bash +docker volume create mp-perf-v3-browser-seed +docker run --rm \ + --mount source=mp-perf-v3-browser-seed,target=/moviepilot/.cloakbrowser \ + --entrypoint python3 \ + jxxghp/moviepilot-v3@sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720 \ + -m cloakbrowser install +``` + +也可以在 `seed` 或 `run` 时显式传入 `--allow-browser-download`，但该选项会让 seed 阶段联网； +正式 Before/After 样本仍然只克隆预热结果并使用 internal network。 + +## 分阶段执行 + +所有全局参数必须放在子命令前。结果默认写到系统临时目录下的 +`moviepilot-perf-results//`。 + +```bash +PYTHON=../.venv/bin/python +CAMPAIGN=v3-perf-001 + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + build --before-ref upstream/v3 --after-ref HEAD + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + seed + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + sample --variant before --index 1 --points 1,5,10,30 +``` + +开发 harness 时可以用小数分钟做短冒烟，例如 `--points 0,0.02`。正式数据必须保持 +`1,5,10,30`。 + +## 完整三组 A/B + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-001 \ + run \ + --before-ref upstream/v3 \ + --after-ref HEAD \ + --points 1,5,10,30 +``` + +执行顺序固定为： + +```text +Before-1 → After-1 → After-2 → Before-2 → Before-3 → After-3 +``` + +每个样本都从 SQLite 和浏览器 seed 克隆新的命名 volume。样本结束后立即移除容器和样本卷； +完整 `run` 结束后还会移除 campaign seed 与 internal network。派生镜像和本地结果保留，便于复核。 + +## 输出 + +```text +// +├── build.json +├── seed.json +├── results.json +├── report.md +└── samples/ + ├── before-1/ + │ ├── result.json + │ ├── container.log + │ └── modules/ + └── ... +``` + +- `results.json`：Engine stats、进程 PSS/USS/RSS/线程、网络累计值和模块前缀计数； +- `report.md`：三次原值与中位数 Before/After 汇总； +- `modules/`：由目标 MoviePilot Python 进程自身写出的完整 `sys.modules` 名称清单； +- `container.log`：已移除实验室凭据值和本地实例 UUID。 + +容器 working set 统一按 Docker Engine API 的 +`memory_stats.usage - memory_stats.stats.inactive_file` 计算。进程 PSS 仅用于归因，不能替代容器指标。 + +## 清理 + +清理严格限定到 campaign 标签；不会执行 Docker prune： + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-001 cleanup --images +``` + +本地 JSON、Markdown 和日志不会被 `cleanup` 删除。 diff --git a/scripts/perf/instrument/collect_proc.sh b/scripts/perf/instrument/collect_proc.sh new file mode 100644 index 000000000..8cabc6333 --- /dev/null +++ b/scripts/perf/instrument/collect_proc.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# 读取采样开始时已经存在的容器进程，避免把采样器自身计入结果。 + +set -uo pipefail + +self_pid="$$" +process_dirs=(/proc/[0-9]*) + +printf 'pid\tppid\tthreads\trss_kib\tpss_kib\tuss_kib\tcomm\texe\tcmdline\n' + +for process_dir in "${process_dirs[@]}"; do + pid="${process_dir##*/}" + if [[ "${pid}" == "${self_pid}" ]] || [[ ! -r "${process_dir}/status" ]]; then + continue + fi + + status_values="$(awk ' + /^PPid:/ { ppid=$2 } + /^Threads:/ { threads=$2 } + END { printf "%d %d", ppid + 0, threads + 0 } + ' "${process_dir}/status" 2>/dev/null || true)" + read -r ppid threads <<< "${status_values:-0 0}" + + rss_kib=0 + pss_kib=0 + uss_kib=0 + if [[ -r "${process_dir}/smaps_rollup" ]]; then + memory_values="$(awk ' + /^Rss:/ { rss=$2 } + /^Pss:/ { pss=$2 } + /^Private_Clean:/ { uss += $2 } + /^Private_Dirty:/ { uss += $2 } + /^Private_Hugetlb:/ { uss += $2 } + END { printf "%d %d %d", rss + 0, pss + 0, uss + 0 } + ' "${process_dir}/smaps_rollup" 2>/dev/null || true)" + read -r rss_kib pss_kib uss_kib <<< "${memory_values:-0 0 0}" + fi + + comm="" + if [[ -r "${process_dir}/comm" ]]; then + IFS= read -r comm < "${process_dir}/comm" || true + fi + executable="$(readlink "${process_dir}/exe" 2>/dev/null || true)" + command_line="$(tr '\000\t' ' ' < "${process_dir}/cmdline" 2>/dev/null || true)" + comm="${comm//$'\t'/ }" + executable="${executable//$'\t'/ }" + command_line="${command_line//$'\t'/ }" + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${pid}" "${ppid:-0}" "${threads:-0}" \ + "${rss_kib:-0}" "${pss_kib:-0}" "${uss_kib:-0}" \ + "${comm}" "${executable}" "${command_line}" +done diff --git a/scripts/perf/instrument/sitecustomize.py b/scripts/perf/instrument/sitecustomize.py new file mode 100644 index 000000000..086e242dc --- /dev/null +++ b/scripts/perf/instrument/sitecustomize.py @@ -0,0 +1,35 @@ +"""MoviePilot Docker A/B 测量时使用的最小 ``sys.modules`` 快照探针。""" + +import os +import signal +import sys + + +_OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR") +_snapshot_index = 0 + + +def _dump_modules(_signum, _frame) -> None: + """收到 SIGUSR1 时原子写出当前解释器已经导入的模块名称。""" + global _snapshot_index + if not _OUTPUT_DIR: + return + + _snapshot_index += 1 + os.makedirs(_OUTPUT_DIR, exist_ok=True) + final_path = os.path.join( + _OUTPUT_DIR, + f"modules-{os.getpid()}-{_snapshot_index}.txt", + ) + temporary_path = f"{final_path}.tmp" + with open(temporary_path, "w", encoding="utf-8") as output: + for module_name in sorted(sys.modules): + output.write(module_name) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary_path, final_path) + + +if _OUTPUT_DIR and hasattr(signal, "SIGUSR1"): + signal.signal(signal.SIGUSR1, _dump_modules) diff --git a/scripts/perf/moviepilot_docker_ab.py b/scripts/perf/moviepilot_docker_ab.py new file mode 100644 index 000000000..797085cda --- /dev/null +++ b/scripts/perf/moviepilot_docker_ab.py @@ -0,0 +1,1677 @@ +"""对两个 MoviePilot Git commit 执行可复现的 Docker 内存 A/B 测量。""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import statistics +import subprocess +import sys +import tarfile +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Optional + +try: + import docker +except ImportError: # pragma: no cover - 仅用于让 --help 在缺依赖环境仍可使用 + docker = None + + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[1] +INSTRUMENT_DIR = SCRIPT_DIR / "instrument" +DEFAULT_SUBSTRATE = ( + "jxxghp/moviepilot-v3@" + "sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720" +) +DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed" +CAMPAIGN_LABEL = "org.moviepilot.perf.campaign" +ROLE_LABEL = "org.moviepilot.perf.role" +SOURCE_LABEL = "org.moviepilot.perf.source-commit" +SUBSTRATE_LABEL = "org.moviepilot.perf.substrate" +CRITICAL_SUBSTRATE_PATHS = ( + "requirements.in", + "docker/Dockerfile", + "scripts/uv-pip-compat.sh", +) +SEED_COMPATIBILITY_PATHS = ("database/versions",) +MODULE_PREFIXES = ( + "lark_oapi", + "slack_bolt", + "slack_sdk", + "discord", + "plexapi", + "telebot", + "langgraph", + "langchain", + "app.agent", + "app.agent.orchestrator", + "app.agent.tools", + "app.modules", +) +BALANCED_RUN_ORDER = ( + ("before", 1), + ("after", 1), + ("after", 2), + ("before", 2), + ("before", 3), + ("after", 3), +) +LAB_API_TOKEN = "moviepilot-perf-lab-token-00000001" +LAB_PASSWORD = "MoviePilot-Perf-Lab-Only-00000001!" +LAB_SECRET_KEY = "moviepilot-perf-secret-key-lab-only-00000001" +LAB_RESOURCE_SECRET_KEY = "moviepilot-perf-resource-key-lab-only-00000001" + + +OVERLAY_DOCKERFILE = r""" +ARG MP_SUBSTRATE +FROM ${MP_SUBSTRATE} AS frozen + +RUN set -eux; \ + mkdir -p /frozen/plugins /frozen/site; \ + cp -a /app/app/plugins/. /frozen/plugins/; \ + rm -f /frozen/plugins/__init__.py; \ + rm -rf /frozen/plugins/__pycache__; \ + find /app/app/application/site -maxdepth 1 -type f \ + \( -name 'sites.*.so' -o -name 'user.sites.v3.bin' \) \ + -exec cp -a '{}' /frozen/site/ \; + +FROM ${MP_SUBSTRATE} +ARG MP_SOURCE_COMMIT +ARG MP_CAMPAIGN + +USER root +RUN rm -rf /app && mkdir -p /app/app/plugins /app/app/application/site +COPY source/ /app/ +COPY --from=frozen /frozen/plugins/ /app/app/plugins/ +COPY --from=frozen /frozen/site/ /app/app/application/site/ + +RUN cp -f /app/docker/nginx.common.conf /etc/nginx/common.conf \ + && cp -f /app/docker/nginx.template.conf /etc/nginx/nginx.template.conf \ + && cp -f /app/docker/update.sh /usr/local/bin/mp_update.sh \ + && cp -f /app/docker/entrypoint.sh /entrypoint.sh \ + && cp -f /app/docker/docker_http_proxy.conf /etc/nginx/docker_http_proxy.conf \ + && chmod +x /entrypoint.sh /usr/local/bin/mp_update.sh + +LABEL org.moviepilot.perf.campaign="${MP_CAMPAIGN}" \ + org.moviepilot.perf.source-commit="${MP_SOURCE_COMMIT}" \ + org.moviepilot.perf.substrate="${MP_SUBSTRATE}" +""".lstrip() + + +IMAGE_FINGERPRINT_SCRIPT = r""" +import hashlib +import importlib.metadata +import json +import platform +from pathlib import Path + + +def tree_fingerprint(root, selected_names=None): + root_path = Path(root) + digest = hashlib.sha256() + count = 0 + total = 0 + if not root_path.exists(): + return {"files": 0, "bytes": 0, "sha256": digest.hexdigest()} + for path in sorted(item for item in root_path.rglob("*") if item.is_file()): + if selected_names and not any(path.match(pattern) for pattern in selected_names): + continue + relative = path.relative_to(root_path).as_posix() + size = path.stat().st_size + count += 1 + total += size + digest.update(relative.encode("utf-8", errors="surrogateescape")) + digest.update(b"\0") + digest.update(str(size).encode("ascii")) + digest.update(b"\0") + with path.open("rb") as input_file: + for chunk in iter(lambda: input_file.read(1024 * 1024), b""): + digest.update(chunk) + return {"files": count, "bytes": total, "sha256": digest.hexdigest()} + + +packages = sorted( + f"{distribution.metadata.get('Name', '')}=={distribution.version}" + for distribution in importlib.metadata.distributions() +) +package_digest = hashlib.sha256("\n".join(packages).encode("utf-8")).hexdigest() +print(json.dumps({ + "python": platform.python_version(), + "packages": {"count": len(packages), "sha256": package_digest}, + "public": tree_fingerprint("/public"), + "plugins": tree_fingerprint("/app/app/plugins"), + "site_resources": tree_fingerprint( + "/app/app/application/site", + ("sites.*.so", "user.sites.v3.bin"), + ), +}, sort_keys=True)) +""" + + +VOLUME_FINGERPRINT_SCRIPT = r""" +import hashlib +import json +from pathlib import Path + +root = Path("/volume") +digest = hashlib.sha256() +count = 0 +total = 0 +if root.exists(): + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.relative_to(root).as_posix() + size = path.stat().st_size + count += 1 + total += size + digest.update(relative.encode("utf-8", errors="surrogateescape")) + digest.update(b"\0") + digest.update(str(size).encode("ascii")) + digest.update(b"\n") +print(json.dumps({"files": count, "bytes": total, "layout_sha256": digest.hexdigest()})) +""" + + +class HarnessError(RuntimeError): + """表示测量合同无法继续成立。""" + + +def utc_now() -> str: + """返回适合写入 JSON 的 UTC 时间。""" + return datetime.now(timezone.utc).isoformat() + + +def atomic_write_json(path: Path, payload: Any) -> None: + """原子写入 JSON，避免长时间测量中断后留下半个结果文件。""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_suffix(f"{path.suffix}.tmp") + temporary_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary_path.replace(path) + + +def write_text(path: Path, content: str) -> None: + """创建父目录并写入 UTF-8 文本。""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def run_command( + command: list[str], + *, + cwd: Optional[Path] = None, + log_path: Optional[Path] = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + """不经过 shell 执行命令，并在需要时保存完整输出。""" + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if log_path: + write_text(log_path, result.stdout + result.stderr) + if check and result.returncode != 0: + detail = (result.stderr or result.stdout).strip().splitlines() + tail = "\n".join(detail[-20:]) + raise HarnessError( + f"命令失败（exit={result.returncode}）：{' '.join(command)}\n{tail}" + ) + return result + + +def require_docker_client(): + """连接 Docker Engine，并在依赖或 daemon 不可用时给出明确错误。""" + if docker is None: + raise HarnessError( + "缺少 docker Python SDK，请使用 MoviePilot 工作区运行环境执行" + ) + try: + client = docker.from_env() + client.ping() + return client + except Exception as default_error: + context = run_command( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + check=False, + ) + context_host = context.stdout.strip() + if not context_host: + raise HarnessError( + f"无法连接 Docker Engine：{default_error}" + ) from default_error + try: + client = docker.DockerClient(base_url=context_host) + client.ping() + return client + except Exception as context_error: + raise HarnessError( + f"无法通过当前 Docker context 连接 Engine：{context_error}" + ) from context_error + + +def normalize_campaign(value: str) -> str: + """限制 campaign 名称，确保 Docker 资源名和标签可安全复用。""" + normalized = value.strip().lower() + if not re.fullmatch(r"[a-z0-9][a-z0-9_.-]{0,39}", normalized): + raise argparse.ArgumentTypeError( + "campaign 只能包含小写字母、数字、点、下划线和短横线，最长 40 字符" + ) + return normalized + + +def parse_points(value: str) -> list[float]: + """解析以分钟为单位的升序采样点。""" + try: + points = sorted({float(item.strip()) for item in value.split(",")}) + except ValueError as error: + raise argparse.ArgumentTypeError("采样点必须是逗号分隔的分钟数") from error + if not points or any(point < 0 for point in points): + raise argparse.ArgumentTypeError("采样点不得为空或小于 0") + return points + + +def campaign_directory(args: argparse.Namespace) -> Path: + """返回当前 campaign 的本地结果目录。""" + return args.output_dir.expanduser().resolve() / args.campaign + + +def resource_prefix(args: argparse.Namespace) -> str: + """生成本工具拥有的 Docker 资源名前缀。""" + return f"mpperf-{args.campaign}" + + +def image_tag(args: argparse.Namespace, variant: str) -> str: + """返回 Before 或 After 派生镜像标签。""" + return f"moviepilot-perf:{args.campaign}-{variant}" + + +def labels(args: argparse.Namespace, role: str) -> dict[str, str]: + """给 Docker 资源附加可审计的精确所有权标签。""" + return {CAMPAIGN_LABEL: args.campaign, ROLE_LABEL: role} + + +def resolve_platform(client, requested: str) -> str: + """将 auto 解析为 Docker daemon 的原生 Linux 架构。""" + if requested != "auto": + return requested + architecture = str(client.info().get("Architecture") or "").lower() + aliases = { + "aarch64": "arm64", + "arm64": "arm64", + "x86_64": "amd64", + "amd64": "amd64", + } + if architecture not in aliases: + raise HarnessError(f"无法把 Docker 架构 {architecture!r} 映射为目标平台") + return f"linux/{aliases[architecture]}" + + +def git_output(repo: Path, *arguments: str) -> str: + """执行只读 Git 命令并返回去除尾部换行的输出。""" + result = run_command(["git", *arguments], cwd=repo) + return result.stdout.strip() + + +def resolve_git_ref(repo: Path, ref: str) -> str: + """把用户给出的 ref 固定为 commit SHA。""" + return git_output(repo, "rev-parse", "--verify", f"{ref}^{{commit}}") + + +def git_path_changed(repo: Path, before: str, after: str, paths: Iterable[str]) -> bool: + """判断两个 commit 在指定运行时 substrate 输入上是否有差异。""" + result = run_command( + ["git", "diff", "--quiet", f"{before}..{after}", "--", *paths], + cwd=repo, + check=False, + ) + if result.returncode not in (0, 1): + raise HarnessError(result.stderr.strip() or "git diff 执行失败") + return result.returncode == 1 + + +def assert_commit_order(repo: Path, before: str, after: str) -> None: + """要求 After 位于 Before 之后，避免比较两条无关历史。""" + result = run_command( + ["git", "merge-base", "--is-ancestor", before, after], + cwd=repo, + check=False, + ) + if result.returncode != 0: + raise HarnessError("After commit 必须是 Before commit 的后代") + + +def ensure_substrate(client, args: argparse.Namespace, pull: bool): + """取得冻结 substrate，只有显式要求时才访问镜像仓库。""" + if pull: + run_command( + ["docker", "pull", "--platform", args.platform, args.substrate], + log_path=campaign_directory(args) / "substrate-pull.log", + ) + try: + return client.images.get(args.substrate) + except Exception as error: + raise HarnessError( + f"本地不存在 substrate {args.substrate}；如需下载请添加 --pull-substrate" + ) from error + + +def image_fingerprint(client, image: str) -> dict[str, Any]: + """在不启动主程序和网络的临时容器中计算运行时资产指纹。""" + try: + output = client.containers.run( + image, + command=["-c", IMAGE_FINGERPRINT_SCRIPT], + entrypoint="python3", + network_disabled=True, + remove=True, + stdout=True, + stderr=True, + ) + return json.loads(output.decode("utf-8")) + except Exception as error: + raise HarnessError(f"无法计算镜像 {image} 的资产指纹：{error}") from error + + +def build_overlay_image( + args: argparse.Namespace, + variant: str, + commit: str, +) -> dict[str, Any]: + """从冻结 substrate 构建仅替换指定 Git commit 源码的派生镜像。""" + campaign_dir = campaign_directory(args) + with tempfile.TemporaryDirectory( + prefix=f"mpperf-{args.campaign}-{variant}-" + ) as temp: + context = Path(temp) + source_dir = context / "source" + source_dir.mkdir() + archive_path = context / "source.tar" + run_command( + ["git", "archive", "--format=tar", "--output", str(archive_path), commit], + cwd=args.repo, + ) + with tarfile.open(archive_path, "r") as archive: + try: + archive.extractall(source_dir, filter="data") + except TypeError: # pragma: no cover - Python 3.11 早期补丁版本兼容 + archive.extractall(source_dir) + archive_path.unlink() + write_text(context / "Dockerfile", OVERLAY_DOCKERFILE) + + tag = image_tag(args, variant) + build_log = campaign_dir / f"build-{variant}.log" + run_command( + [ + "docker", + "build", + "--pull=false", + "--platform", + args.platform, + "--build-arg", + f"MP_SUBSTRATE={args.substrate}", + "--build-arg", + f"MP_SOURCE_COMMIT={commit}", + "--build-arg", + f"MP_CAMPAIGN={args.campaign}", + "--tag", + tag, + "--file", + str(context / "Dockerfile"), + str(context), + ], + log_path=build_log, + ) + + client = require_docker_client() + image = client.images.get(tag) + image_labels = image.attrs.get("Config", {}).get("Labels") or {} + if image_labels.get(SOURCE_LABEL) != commit: + raise HarnessError(f"镜像 {tag} 的 source commit 标签校验失败") + return { + "variant": variant, + "tag": tag, + "image_id": image.id, + "source_commit": commit, + "fingerprint": image_fingerprint(client, tag), + } + + +def command_build(args: argparse.Namespace) -> dict[str, Any]: + """构建 Before/After 派生镜像并记录冻结输入。""" + client = require_docker_client() + args.platform = resolve_platform(client, args.platform) + args.repo = args.repo.expanduser().resolve() + if not (args.repo / ".git").exists(): + raise HarnessError(f"不是 MoviePilot Git 仓库：{args.repo}") + + campaign_dir = campaign_directory(args) + campaign_dir.mkdir(parents=True, exist_ok=True) + substrate = ensure_substrate(client, args, args.pull_substrate) + substrate_labels = substrate.attrs.get("Config", {}).get("Labels") or {} + substrate_revision = substrate_labels.get("org.opencontainers.image.revision") + if not substrate_revision: + raise HarnessError("substrate 缺少 org.opencontainers.image.revision 标签") + + before_commit = resolve_git_ref(args.repo, args.before_ref) + after_commit = resolve_git_ref(args.repo, args.after_ref) + resolve_git_ref(args.repo, substrate_revision) + assert_commit_order(args.repo, before_commit, after_commit) + + if git_path_changed( + args.repo, + substrate_revision, + before_commit, + CRITICAL_SUBSTRATE_PATHS, + ): + raise HarnessError( + "substrate revision 到 Before commit 的依赖或 Docker substrate 输入已变化，" + "禁止使用源码 overlay A/B" + ) + if git_path_changed( + args.repo, + before_commit, + after_commit, + CRITICAL_SUBSTRATE_PATHS, + ): + raise HarnessError( + "Before/After 的依赖或 Docker substrate 输入已变化，禁止使用源码 overlay A/B" + ) + if git_path_changed( + args.repo, + before_commit, + after_commit, + SEED_COMPATIBILITY_PATHS, + ): + raise HarnessError( + "Before/After 的数据库迁移集合不同，禁止复用同一个迁移后 SQLite seed" + ) + + substrate_fingerprint = image_fingerprint(client, args.substrate) + before_image = build_overlay_image(args, "before", before_commit) + after_image = build_overlay_image(args, "after", after_commit) + for key in ("python", "packages", "public", "plugins", "site_resources"): + if before_image["fingerprint"].get(key) != after_image["fingerprint"].get(key): + raise HarnessError(f"Before/After 冻结资产指纹不一致：{key}") + + manifest = { + "schema_version": 1, + "campaign": args.campaign, + "generated_at": utc_now(), + "platform": args.platform, + "before_ref": args.before_ref, + "after_ref": args.after_ref, + "before_commit": before_commit, + "after_commit": after_commit, + "critical_substrate_paths": list(CRITICAL_SUBSTRATE_PATHS), + "seed_compatibility_paths": list(SEED_COMPATIBILITY_PATHS), + "substrate": { + "reference": args.substrate, + "image_id": substrate.id, + "source_revision": substrate_revision, + "repo_digests": substrate.attrs.get("RepoDigests") or [], + "fingerprint": substrate_fingerprint, + }, + "images": {"before": before_image, "after": after_image}, + } + atomic_write_json(campaign_dir / "build.json", manifest) + print(f"Build manifest: {campaign_dir / 'build.json'}") + return manifest + + +def load_build_manifest(args: argparse.Namespace) -> dict[str, Any]: + """读取并校验当前 campaign 的构建结果。""" + path = campaign_directory(args) / "build.json" + if not path.exists(): + raise HarnessError("缺少 build.json，请先执行 build") + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("campaign") != args.campaign: + raise HarnessError("build.json 的 campaign 不匹配") + return payload + + +def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, str]: + """返回不依赖用户 app.env、外部服务或真实凭据的固定空载配置。""" + environment = { + "TZ": "Asia/Shanghai", + "PUID": "0", + "PGID": "0", + "UMASK": "000", + "PORT": "3001", + "NGINX_PORT": "3000", + "API_WORKERS": "1", + "DB_TYPE": "sqlite", + "CACHE_BACKEND_TYPE": "cachetools", + "AI_AGENT_ENABLE": "false", + "DEV": "false", + "DEBUG": "false", + "MOVIEPILOT_SAFE_MODE": "false", + "MOVIEPILOT_AUTO_UPDATE": "false", + "MOVIEPILOT_DOCKER_KEEPALIVE_ON_FAILURE": "false", + "MOVIEPILOT_BACKEND_READY_TIMEOUT": str(args.ready_timeout), + "AUTO_UPDATE_RESOURCE": "false", + "PLUGIN_MARKET": "", + "PLUGIN_AUTO_RELOAD": "false", + "PLUGIN_LOCAL_REPO_PATHS": "", + "PLUGIN_STATISTIC_SHARE": "false", + "SUBSCRIBE_STATISTIC_SHARE": "false", + "USAGE_STATISTIC_SHARE": "false", + "WORKFLOW_STATISTIC_SHARE": "false", + "MEDIA_RECOGNIZE_SHARE": "false", + "MP_SERVER_HOST": "", + "GITHUB_TOKEN": "", + "REPO_GITHUB_TOKEN": "", + "AUTH_SITE": "", + "SKILL_MARKET": "", + "BROWSER_EMULATION": "cloakbrowser", + "FANART_ENABLE": "false", + "API_TOKEN": LAB_API_TOKEN, + "SUPERUSER": "admin", + "SUPERUSER_PASSWORD": LAB_PASSWORD, + "SECRET_KEY": LAB_SECRET_KEY, + "RESOURCE_SECRET_KEY": LAB_RESOURCE_SECRET_KEY, + "LOG_LEVEL": "INFO", + } + if instrument: + environment.update( + { + "PYTHONPATH": "/opt/moviepilot-perf/instrument", + "MP_PERF_OUTPUT_DIR": "/opt/moviepilot-perf/out/modules", + } + ) + return environment + + +def get_volume(client, name: str): + """返回命名 volume；不存在时返回 None。""" + try: + return client.volumes.get(name) + except docker.errors.NotFound: + return None + + +def assert_owned(resource, args: argparse.Namespace, kind: str) -> None: + """删除或复用资源前验证 campaign 标签，避免误伤用户资源。""" + resource_labels = resource.attrs.get("Labels") or {} + if resource_labels.get(CAMPAIGN_LABEL) != args.campaign: + raise HarnessError(f"拒绝操作非本 campaign 的 {kind}：{resource.name}") + + +def prepare_volume( + client, + args: argparse.Namespace, + name: str, + role: str, + replace: bool, +): + """创建工具拥有的命名 volume，并按显式 replace 处理同名旧资源。""" + existing = get_volume(client, name) + if existing: + assert_owned(existing, args, "volume") + if not replace: + raise HarnessError(f"volume 已存在：{name}；如需重建请使用 --replace") + existing.remove(force=True) + return client.volumes.create(name=name, labels=labels(args, role)) + + +def clone_volume(client, image: str, source: str, target: str) -> None: + """在无网络、无主程序的短容器中复制命名 volume。""" + try: + client.containers.run( + image, + command=["-c", "cp -a /source/. /target/"], + entrypoint="/bin/sh", + network_disabled=True, + remove=True, + volumes={ + source: {"bind": "/source", "mode": "ro"}, + target: {"bind": "/target", "mode": "rw"}, + }, + ) + except Exception as error: + raise HarnessError(f"复制 volume {source} → {target} 失败：{error}") from error + + +def volume_fingerprint(client, image: str, volume_name: str) -> dict[str, Any]: + """记录 volume 文件数量、总大小和路径布局哈希，不读取配置内容。""" + try: + output = client.containers.run( + image, + command=["-c", VOLUME_FINGERPRINT_SCRIPT], + entrypoint="python3", + network_disabled=True, + remove=True, + volumes={volume_name: {"bind": "/volume", "mode": "ro"}}, + ) + return json.loads(output.decode("utf-8")) + except Exception as error: + raise HarnessError(f"无法计算 volume {volume_name} 指纹：{error}") from error + + +def ensure_internal_network(client, args: argparse.Namespace): + """创建或复用当前 campaign 的无外网 Docker network。""" + name = f"{resource_prefix(args)}-internal" + try: + network = client.networks.get(name) + assert_owned(network, args, "network") + if not network.attrs.get("Internal"): + raise HarnessError(f"network {name} 不是 internal network") + return network + except docker.errors.NotFound: + return client.networks.create( + name, + driver="bridge", + internal=True, + labels=labels(args, "measurement-network"), + ) + + +def remove_owned_container(client, args: argparse.Namespace, name: str) -> None: + """移除同 campaign 遗留容器，绝不按模糊前缀删除。""" + try: + container = client.containers.get(name) + except docker.errors.NotFound: + return + assert_owned(container, args, "container") + container.remove(force=True, v=False) + + +def create_app_container( + client, + args: argparse.Namespace, + *, + image: str, + name: str, + role: str, + config_volume: str, + browser_volume: str, + network_name: str, + output_dir: Optional[Path], +): + """按固定资源和挂载合同创建 MoviePilot 容器。""" + remove_owned_container(client, args, name) + volume_mounts: dict[str, dict[str, str]] = { + config_volume: {"bind": "/config", "mode": "rw"}, + browser_volume: {"bind": "/moviepilot/.cloakbrowser", "mode": "rw"}, + } + instrument = output_dir is not None + if output_dir: + output_dir.mkdir(parents=True, exist_ok=True) + volume_mounts[str(INSTRUMENT_DIR)] = { + "bind": "/opt/moviepilot-perf/instrument", + "mode": "ro", + } + volume_mounts[str(output_dir.resolve())] = { + "bind": "/opt/moviepilot-perf/out", + "mode": "rw", + } + return client.containers.create( + image, + name=name, + detach=True, + environment=fixed_environment(args, instrument=instrument), + volumes=volume_mounts, + network=network_name, + nano_cpus=int(args.cpus * 1_000_000_000), + mem_limit=args.memory, + memswap_limit=args.memory, + pids_limit=2048, + shm_size="256m", + labels=labels(args, role), + ) + + +def container_running(container) -> bool: + """刷新并返回容器是否仍在运行。""" + container.reload() + return container.status == "running" + + +def wait_for_exec_success( + container, + command: list[str], + timeout: float, + description: str, +) -> float: + """轮询容器内的无副作用探针并返回耗时秒数。""" + started = time.monotonic() + deadline = started + timeout + while time.monotonic() < deadline: + if not container_running(container): + raise HarnessError(f"等待{description}时容器提前退出") + result = container.exec_run(command) + if result.exit_code == 0: + return time.monotonic() - started + time.sleep(0.5) + raise HarnessError(f"等待{description}超时（{timeout:.0f}s）") + + +def wait_for_ready(container, started_at: float, timeout: float) -> float: + """等待公开 health endpoint 完成完整同步启动阶段。""" + deadline = started_at + timeout + command = [ + "curl", + "-fsS", + "--max-time", + "2", + "http://127.0.0.1:3001/api/v1/system/global?token=moviepilot", + ] + while time.monotonic() < deadline: + if not container_running(container): + raise HarnessError("等待 health ready 时容器提前退出") + result = container.exec_run(command) + if result.exit_code == 0: + return time.monotonic() - started_at + time.sleep(0.5) + raise HarnessError(f"等待 health ready 超时（{timeout:.0f}s）") + + +def assert_no_app_env(container) -> None: + """只检查 app.env 不存在；绝不读取文件内容。""" + result = container.exec_run(["/bin/sh", "-c", "test ! -e /config/app.env"]) + if result.exit_code != 0: + raise HarnessError("测量 config volume 出现 app.env，已停止以避免读取用户配置") + + +def capture_engine_stats(container) -> dict[str, Any]: + """从 Docker Engine API 读取原始 cgroup 和网络累计值。""" + try: + stats = container.stats(stream=False, one_shot=True) + except TypeError: # pragma: no cover - 旧 Docker SDK 兼容 + stats = container.stats(stream=False) + memory_stats = stats.get("memory_stats") or {} + memory_detail = memory_stats.get("stats") or {} + memory_current = int(memory_stats.get("usage") or 0) + inactive_file = int( + memory_detail.get("inactive_file") + or memory_detail.get("total_inactive_file") + or 0 + ) + networks = stats.get("networks") or {} + rx_bytes = sum(int(item.get("rx_bytes") or 0) for item in networks.values()) + tx_bytes = sum(int(item.get("tx_bytes") or 0) for item in networks.values()) + return { + "memory_current_bytes": memory_current, + "inactive_file_bytes": inactive_file, + "working_set_bytes": max(memory_current - inactive_file, 0), + "network_rx_bytes": rx_bytes, + "network_tx_bytes": tx_bytes, + } + + +def capture_processes(container) -> dict[str, Any]: + """读取采样开始时已有进程的 PSS/USS/RSS 与线程数。""" + result = container.exec_run( + ["/bin/bash", "/opt/moviepilot-perf/instrument/collect_proc.sh"] + ) + if result.exit_code != 0: + raise HarnessError( + "进程采样失败：" + result.output.decode("utf-8", errors="replace") + ) + lines = result.output.decode("utf-8", errors="replace").splitlines() + processes: list[dict[str, Any]] = [] + for line in lines[1:]: + fields = line.split("\t", 8) + if len(fields) != 9: + continue + pid, ppid, threads, rss, pss, uss, comm, executable, command_line = fields + processes.append( + { + "pid": int(pid), + "ppid": int(ppid), + "threads": int(threads), + "rss_kib": int(rss), + "pss_kib": int(pss), + "uss_kib": int(uss), + "comm": comm, + "executable": executable, + "cmdline": command_line, + } + ) + if not processes: + raise HarnessError("进程采样结果为空") + python_processes = [ + process + for process in processes + if "python" in Path(process["executable"]).name.lower() + ] + main_python = max(python_processes, key=lambda item: item["pss_kib"], default=None) + xvfb_processes = [ + process + for process in processes + if "xvfb" in f"{process['comm']} {process['cmdline']}".lower() + ] + return { + "items": sorted(processes, key=lambda item: item["pid"]), + "totals": { + "rss_kib": sum(item["rss_kib"] for item in processes), + "pss_kib": sum(item["pss_kib"] for item in processes), + "uss_kib": sum(item["uss_kib"] for item in processes), + "threads": sum(item["threads"] for item in processes), + }, + "main_python": main_python, + "xvfb": { + "count": len(xvfb_processes), + "pss_kib": sum(item["pss_kib"] for item in xvfb_processes), + }, + } + + +def capture_modules( + container, + output_dir: Path, + main_python: Optional[dict[str, Any]], +) -> dict[str, Any]: + """通过已注入的 SIGUSR1 handler 获取目标进程自身的 sys.modules。""" + if not main_python: + raise HarnessError("未找到主 Python 进程，无法采集 sys.modules") + modules_dir = output_dir / "modules" + modules_dir.mkdir(parents=True, exist_ok=True) + existing = set(modules_dir.glob("modules-*.txt")) + result = container.exec_run(["kill", "-USR1", str(main_python["pid"])]) + if result.exit_code != 0: + raise HarnessError("向主 Python 进程发送 SIGUSR1 失败") + deadline = time.monotonic() + 5 + snapshot_path: Optional[Path] = None + while time.monotonic() < deadline: + candidates = set(modules_dir.glob("modules-*.txt")) - existing + if candidates: + snapshot_path = max(candidates, key=lambda path: path.stat().st_mtime_ns) + break + time.sleep(0.05) + if snapshot_path is None: + raise HarnessError("主 Python 进程没有写出 sys.modules 快照") + content = snapshot_path.read_bytes() + names = [line for line in content.decode("utf-8").splitlines() if line] + prefix_counts = { + prefix: sum( + 1 for name in names if name == prefix or name.startswith(f"{prefix}.") + ) + for prefix in MODULE_PREFIXES + } + return { + "count": len(names), + "sha256": hashlib.sha256(content).hexdigest(), + "prefix_counts": prefix_counts, + "raw_file": snapshot_path.relative_to(output_dir).as_posix(), + } + + +def capture_measurement( + container, + output_dir: Path, + minute: float, + settled_at: float, +) -> dict[str, Any]: + """按低干扰顺序采集 Engine、进程和模块三层数据。""" + captured_at = time.monotonic() + engine = capture_engine_stats(container) + processes = capture_processes(container) + modules = capture_modules(container, output_dir, processes["main_python"]) + return { + "target_minute": minute, + "elapsed_seconds": captured_at - settled_at, + "captured_at": utc_now(), + "engine": engine, + "processes": processes, + "modules": modules, + } + + +def redact_logs(content: str) -> str: + """移除实验室凭据值和启动生成的本地实例标识。""" + redacted = content + for secret_value in ( + LAB_API_TOKEN, + LAB_PASSWORD, + LAB_SECRET_KEY, + LAB_RESOURCE_SECRET_KEY, + ): + redacted = redacted.replace(secret_value, "") + redacted = re.sub( + r"(当前用户UUID[:：]\s*)[^\s]+", + r"\1", + redacted, + ) + return redacted + + +def save_container_logs(container, path: Path) -> None: + """保存脱敏后的完整容器日志。""" + try: + raw = container.logs(stdout=True, stderr=True, timestamps=True) + write_text(path, redact_logs(raw.decode("utf-8", errors="replace"))) + except Exception as error: + write_text(path, f"无法读取容器日志：{error}\n") + + +def stop_and_remove_container(container, timeout: int) -> dict[str, Any]: + """优雅停止工具拥有的容器，并在超时后限定到该容器强制清理。""" + outcome: dict[str, Any] = {"requested_at": utc_now()} + try: + if container_running(container): + started = time.monotonic() + container.stop(timeout=timeout) + outcome["elapsed_seconds"] = time.monotonic() - started + container.reload() + outcome["exit_code"] = container.attrs.get("State", {}).get("ExitCode") + except Exception as error: + outcome["error"] = str(error) + finally: + try: + container.remove(force=True, v=False) + except docker.errors.NotFound: + pass + return outcome + + +def seed_volume_names(args: argparse.Namespace) -> tuple[str, str]: + """返回迁移后 SQLite 和预热浏览器两个 seed volume 名称。""" + prefix = resource_prefix(args) + return f"{prefix}-config-seed", f"{prefix}-browser-seed" + + +def command_seed(args: argparse.Namespace) -> dict[str, Any]: + """生成不含 app.env 的迁移后 SQLite 和预热浏览器 seed。""" + client = require_docker_client() + build = load_build_manifest(args) + before_image = build["images"]["before"]["tag"] + config_seed_name, browser_seed_name = seed_volume_names(args) + if not args.browser_source_volume and not args.allow_browser_download: + raise HarnessError( + "seed 需要 --browser-source-volume，或显式 --allow-browser-download 执行一次预热" + ) + if args.browser_source_volume: + if get_volume(client, args.browser_source_volume) is None: + raise HarnessError( + f"浏览器来源 volume 不存在：{args.browser_source_volume}" + ) + config_seed = prepare_volume( + client, args, config_seed_name, "config-seed", args.replace + ) + browser_seed = prepare_volume( + client, args, browser_seed_name, "browser-seed", args.replace + ) + if args.browser_source_volume: + clone_volume( + client, + before_image, + args.browser_source_volume, + browser_seed.name, + ) + before_browser = volume_fingerprint(client, before_image, browser_seed.name) + if before_browser["files"] == 0 and not args.allow_browser_download: + raise HarnessError("浏览器 seed 为空，且未允许一次性下载") + + network = ensure_internal_network(client, args) + network_name = "bridge" if args.allow_browser_download else network.name + container_name = f"{resource_prefix(args)}-seed" + seed_dir = campaign_directory(args) / "seed" + seed_dir.mkdir(parents=True, exist_ok=True) + container = create_app_container( + client, + args, + image=before_image, + name=container_name, + role="seed", + config_volume=config_seed.name, + browser_volume=browser_seed.name, + network_name=network_name, + output_dir=None, + ) + started_at = time.monotonic() + result: dict[str, Any] = { + "schema_version": 1, + "campaign": args.campaign, + "generated_at": utc_now(), + "image": before_image, + "browser_source": ( + "named-volume" if args.browser_source_volume else "one-time-download" + ), + "browser_before": before_browser, + } + success = False + try: + container.start() + ready_seconds = wait_for_ready(container, started_at, args.ready_timeout) + settled_wait = wait_for_exec_success( + container, + ["/bin/sh", "-c", "test -e /var/log/nginx/__moviepilot__"], + args.settle_timeout, + "后台初始化完成标志", + ) + assert_no_app_env(container) + result.update( + { + "http_ready_seconds": ready_seconds, + "settled_wait_seconds_after_ready": settled_wait, + "engine_at_settled": capture_engine_stats(container), + } + ) + success = True + except Exception as error: + result["error"] = str(error) + raise + finally: + save_container_logs(container, seed_dir / "container.log") + result["shutdown"] = stop_and_remove_container(container, args.stop_timeout) + if success: + result["browser_after"] = volume_fingerprint( + client, before_image, browser_seed.name + ) + atomic_write_json(campaign_directory(args) / "seed.json", result) + if not success: + for volume in (config_seed, browser_seed): + try: + volume.remove(force=True) + except Exception: + pass + print(f"Seed manifest: {campaign_directory(args) / 'seed.json'}") + return result + + +def require_seed_volumes(client, args: argparse.Namespace) -> tuple[str, str]: + """确认 seed 已完成且两个命名 volume 仍然存在。""" + seed_path = campaign_directory(args) / "seed.json" + if not seed_path.exists(): + raise HarnessError("缺少 seed.json，请先执行 seed") + seed = json.loads(seed_path.read_text(encoding="utf-8")) + if seed.get("error"): + raise HarnessError("seed.json 记录了失败，必须重新生成 seed") + names = seed_volume_names(args) + for name in names: + volume = get_volume(client, name) + if volume is None: + raise HarnessError(f"seed volume 不存在：{name}") + assert_owned(volume, args, "seed volume") + return names + + +def sample_volume_names( + args: argparse.Namespace, + variant: str, + index: int, +) -> tuple[str, str]: + """返回单个样本的隔离配置和浏览器卷名称。""" + prefix = f"{resource_prefix(args)}-{variant}-{index}" + return f"{prefix}-config", f"{prefix}-browser" + + +def sample_result_directory( + args: argparse.Namespace, + variant: str, + index: int, +) -> Path: + """返回单个样本的原始结果目录。""" + return campaign_directory(args) / "samples" / f"{variant}-{index}" + + +def command_sample(args: argparse.Namespace) -> dict[str, Any]: + """执行一个隔离样本并在约定时间点采集完整指标。""" + client = require_docker_client() + build = load_build_manifest(args) + config_seed, browser_seed = require_seed_volumes(client, args) + image = build["images"][args.variant]["tag"] + output_dir = sample_result_directory(args, args.variant, args.index) + if output_dir.exists(): + if not args.replace: + raise HarnessError( + f"样本结果已存在：{output_dir}；如需重测请使用 --replace" + ) + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) + + config_volume_name, browser_volume_name = sample_volume_names( + args, args.variant, args.index + ) + config_volume = prepare_volume( + client, args, config_volume_name, "sample-config", replace=True + ) + browser_volume = prepare_volume( + client, args, browser_volume_name, "sample-browser", replace=True + ) + clone_volume(client, image, config_seed, config_volume.name) + clone_volume(client, image, browser_seed, browser_volume.name) + browser_before = volume_fingerprint(client, image, browser_volume.name) + network = ensure_internal_network(client, args) + container_name = f"{resource_prefix(args)}-{args.variant}-{args.index}" + container = create_app_container( + client, + args, + image=image, + name=container_name, + role=f"sample-{args.variant}-{args.index}", + config_volume=config_volume.name, + browser_volume=browser_volume.name, + network_name=network.name, + output_dir=output_dir, + ) + result: dict[str, Any] = { + "schema_version": 1, + "campaign": args.campaign, + "variant": args.variant, + "sample_index": args.index, + "source_commit": build[f"{args.variant}_commit"], + "image": image, + "started_at": utc_now(), + "points_minutes": args.points, + "resources": { + "cpus": args.cpus, + "memory": args.memory, + "network": "internal", + "database": "sqlite-seed-clone", + "browser": "prewarmed-seed-clone", + }, + "browser_before": browser_before, + "measurements": [], + } + started_at = time.monotonic() + try: + container.start() + ready_seconds = wait_for_ready(container, started_at, args.ready_timeout) + ready_at = time.monotonic() + wait_for_exec_success( + container, + ["/bin/sh", "-c", "test -e /var/log/nginx/__moviepilot__"], + args.settle_timeout, + "后台初始化完成标志", + ) + settled_at = time.monotonic() + assert_no_app_env(container) + result["http_ready_seconds"] = ready_seconds + result["settled_seconds"] = settled_at - started_at + result["settled_wait_seconds_after_ready"] = settled_at - ready_at + + for point in args.points: + deadline = settled_at + point * 60 + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(remaining) + if not container_running(container): + raise HarnessError(f"容器在 {point:g}m 采样前退出") + print(f"[{args.variant}-{args.index}] sampling {point:g}m") + result["measurements"].append( + capture_measurement(container, output_dir, point, settled_at) + ) + atomic_write_json(output_dir / "result.partial.json", result) + assert_no_app_env(container) + except Exception as error: + result["error"] = str(error) + raise + finally: + save_container_logs(container, output_dir / "container.log") + result["shutdown"] = stop_and_remove_container(container, args.stop_timeout) + try: + result["browser_after"] = volume_fingerprint( + client, image, browser_volume.name + ) + except Exception as error: + result["browser_after_error"] = str(error) + result["completed_at"] = utc_now() + atomic_write_json(output_dir / "result.json", result) + partial = output_dir / "result.partial.json" + partial.unlink(missing_ok=True) + for volume in (config_volume, browser_volume): + try: + volume.remove(force=True) + except Exception: + pass + update_aggregate_results(args) + return result + + +def load_sample_results(args: argparse.Namespace) -> list[dict[str, Any]]: + """读取当前 campaign 已完成或失败的所有样本结果。""" + sample_root = campaign_directory(args) / "samples" + results = [] + if not sample_root.exists(): + return results + for path in sorted(sample_root.glob("*/result.json")): + results.append(json.loads(path.read_text(encoding="utf-8"))) + return results + + +def median(values: Iterable[float]) -> Optional[float]: + """空序列返回 None，否则返回浮点中位数。""" + items = list(values) + return statistics.median(items) if items else None + + +def measurement_at(result: dict[str, Any], minute: float) -> Optional[dict[str, Any]]: + """按浮点容差返回目标采样点。""" + for measurement in result.get("measurements") or []: + if abs(float(measurement["target_minute"]) - minute) < 1e-9: + return measurement + return None + + +def format_mib(value: Optional[float]) -> str: + """把字节数格式化为 MiB。""" + if value is None: + return "—" + return f"{value / 1024 / 1024:.1f}" + + +def format_kib_as_mib(value: Optional[float]) -> str: + """把 KiB 数格式化为 MiB。""" + if value is None: + return "—" + return f"{value / 1024:.1f}" + + +def format_bytes_as_kib(value: Optional[float]) -> str: + """把字节数格式化为 KiB。""" + if value is None: + return "—" + return f"{value / 1024:.1f}" + + +def build_markdown_report( + build: dict[str, Any], + seed: Optional[dict[str, Any]], + samples: list[dict[str, Any]], +) -> str: + """生成不含本机路径和凭据的 Markdown 汇总。""" + points = sorted( + { + float(measurement["target_minute"]) + for sample in samples + for measurement in sample.get("measurements") or [] + } + ) + lines = [ + "# MoviePilot Docker A/B 测量结果", + "", + f"- Campaign：`{build['campaign']}`", + f"- Platform：`{build['platform']}`", + f"- Before：`{build['before_commit']}`", + f"- After：`{build['after_commit']}`", + f"- Substrate：`{build['substrate']['reference']}`", + "- working set：`memory.current - inactive_file`", + "- 配置：迁移后 SQLite seed、空插件配置、Agent 关闭、浏览器缓存预热、internal network", + "", + ] + if seed: + lines.extend( + [ + "## Seed", + "", + f"- 浏览器来源：`{seed.get('browser_source', 'unknown')}`", + f"- HTTP ready：{seed.get('http_ready_seconds', 0):.2f}s", + f"- 浏览器文件数：{seed.get('browser_after', {}).get('files', 0)}", + "", + ] + ) + + headers = ( + ["版本", "样本", "HTTP ready(s)"] + + [f"{point:g}m WS(MiB)" for point in points] + + [ + "末次 Python PSS(MiB)", + "末次 Python USS(MiB)", + "Python Threads", + "末次 Xvfb PSS(MiB)", + "RX(KiB)", + "TX(KiB)", + "sys.modules", + "状态", + ] + ) + lines.extend(["## 原始样本", "", "| " + " | ".join(headers) + " |"]) + lines.append("| " + " | ".join(["---"] * len(headers)) + " |") + variant_order = {"before": 0, "after": 1} + for sample in sorted( + samples, + key=lambda item: ( + variant_order.get(item["variant"], 99), + item["sample_index"], + ), + ): + row = [ + sample["variant"], + str(sample["sample_index"]), + f"{sample.get('http_ready_seconds', 0):.2f}" + if "http_ready_seconds" in sample + else "—", + ] + for point in points: + target = measurement_at(sample, point) + row.append( + format_mib(target["engine"]["working_set_bytes"] if target else None) + ) + final = ( + sample.get("measurements", [])[-1] if sample.get("measurements") else None + ) + python_pss = ( + final.get("processes", {}).get("main_python", {}).get("pss_kib") + if final and final.get("processes", {}).get("main_python") + else None + ) + python_uss = ( + final.get("processes", {}).get("main_python", {}).get("uss_kib") + if final and final.get("processes", {}).get("main_python") + else None + ) + python_threads = ( + final.get("processes", {}).get("main_python", {}).get("threads") + if final and final.get("processes", {}).get("main_python") + else None + ) + xvfb_pss = ( + final.get("processes", {}).get("xvfb", {}).get("pss_kib") if final else None + ) + row.extend( + [ + format_kib_as_mib(python_pss), + format_kib_as_mib(python_uss), + str(python_threads) if python_threads is not None else "—", + format_kib_as_mib(xvfb_pss), + format_bytes_as_kib( + final.get("engine", {}).get("network_rx_bytes") if final else None + ), + format_bytes_as_kib( + final.get("engine", {}).get("network_tx_bytes") if final else None + ), + str(final.get("modules", {}).get("count")) if final else "—", + "失败" if sample.get("error") else "完成", + ] + ) + lines.append("| " + " | ".join(row) + " |") + + lines.extend(["", "## 中位数对照", ""]) + if points: + lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |") + lines.append("| --- | ---: | ---: | ---: | ---: |") + for point in points: + before_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in samples + if sample["variant"] == "before" and measurement_at(sample, point) + ] + after_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in samples + if sample["variant"] == "after" and measurement_at(sample, point) + ] + before_median = median(before_values) + after_median = median(after_values) + if before_median is None or after_median is None: + lines.append(f"| {point:g}m | — | — | — | — |") + continue + delta = after_median - before_median + percent = delta / before_median * 100 if before_median else 0 + lines.append( + f"| {point:g}m | {format_mib(before_median)} | " + f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | {percent:.1f}% |" + ) + ready_before = median( + sample["http_ready_seconds"] + for sample in samples + if sample["variant"] == "before" and "http_ready_seconds" in sample + ) + ready_after = median( + sample["http_ready_seconds"] + for sample in samples + if sample["variant"] == "after" and "http_ready_seconds" in sample + ) + lines.extend(["", "## 启动时间", ""]) + if ready_before is not None and ready_after is not None: + startup_change = ( + (ready_after - ready_before) / ready_before * 100 if ready_before else 0 + ) + lines.append( + f"Before 中位数 {ready_before:.2f}s，After 中位数 {ready_after:.2f}s，" + f"变化 {startup_change:.1f}%。" + ) + else: + lines.append("样本尚不完整。") + lines.extend( + [ + "", + "## 说明", + "", + "- 完整 Engine、进程、线程、网络和模块前缀数据见 `results.json`。", + "- 每个 `sys.modules` 完整名称清单位于对应样本的 `modules/` 目录。", + "- 报告不包含 app.env、真实 token、密码或本机挂载路径。", + "", + ] + ) + return "\n".join(lines) + + +def update_aggregate_results(args: argparse.Namespace) -> None: + """汇总当前 campaign 的 build、seed 和所有样本。""" + campaign_dir = campaign_directory(args) + build_path = campaign_dir / "build.json" + if not build_path.exists(): + return + build = json.loads(build_path.read_text(encoding="utf-8")) + seed_path = campaign_dir / "seed.json" + seed = ( + json.loads(seed_path.read_text(encoding="utf-8")) + if seed_path.exists() + else None + ) + samples = load_sample_results(args) + aggregate = { + "schema_version": 1, + "generated_at": utc_now(), + "build": build, + "seed": seed, + "samples": samples, + } + atomic_write_json(campaign_dir / "results.json", aggregate) + write_text( + campaign_dir / "report.md", + build_markdown_report(build, seed, samples), + ) + + +def cleanup_runtime_resources(client, args: argparse.Namespace) -> dict[str, list[str]]: + """仅按 campaign 标签清理容器、volume 和 internal network。""" + removed: dict[str, list[str]] = {"containers": [], "volumes": [], "networks": []} + label_filter = f"{CAMPAIGN_LABEL}={args.campaign}" + for container in client.containers.list(all=True, filters={"label": label_filter}): + name = container.name + container.remove(force=True, v=False) + removed["containers"].append(name) + for volume in client.volumes.list(filters={"label": label_filter}): + name = volume.name + volume.remove(force=True) + removed["volumes"].append(name) + for network in client.networks.list(filters={"label": label_filter}): + name = network.name + try: + network.remove() + removed["networks"].append(name) + except Exception as error: + raise HarnessError(f"清理 network {name} 失败：{error}") from error + return removed + + +def command_cleanup(args: argparse.Namespace) -> dict[str, Any]: + """清理当前 campaign 的 Docker 资源，保留本地结果文件。""" + client = require_docker_client() + removed: dict[str, Any] = cleanup_runtime_resources(client, args) + removed["images"] = [] + if args.images: + build_path = campaign_directory(args) / "build.json" + if build_path.exists(): + build = json.loads(build_path.read_text(encoding="utf-8")) + for variant in ("before", "after"): + tag = build.get("images", {}).get(variant, {}).get("tag") + if not tag: + continue + try: + image = client.images.get(tag) + except docker.errors.ImageNotFound: + continue + image_labels = image.attrs.get("Config", {}).get("Labels") or {} + if image_labels.get(CAMPAIGN_LABEL) != args.campaign: + raise HarnessError(f"拒绝删除非本 campaign 镜像：{tag}") + client.images.remove(tag, force=False, noprune=True) + removed["images"].append(tag) + atomic_write_json(campaign_directory(args) / "cleanup.json", removed) + print(json.dumps(removed, ensure_ascii=False, indent=2)) + return removed + + +def command_run(args: argparse.Namespace) -> None: + """构建、生成 seed，并按平衡顺序串行执行三组 Before/After。""" + command_build(args) + command_seed(args) + completed = False + try: + for variant, index in BALANCED_RUN_ORDER: + sample_args = argparse.Namespace(**vars(args)) + sample_args.variant = variant + sample_args.index = index + sample_args.replace = args.replace + command_sample(sample_args) + completed = True + finally: + update_aggregate_results(args) + if not args.keep_resources: + client = require_docker_client() + cleanup_runtime_resources(client, args) + if completed: + print(f"Results: {campaign_directory(args) / 'results.json'}") + print(f"Report: {campaign_directory(args) / 'report.md'}") + + +def add_common_build_arguments(parser: argparse.ArgumentParser) -> None: + """添加 build 与 run 共用的 ref 和 substrate 参数。""" + parser.add_argument("--before-ref", default="upstream/v3", help="Before Git ref") + parser.add_argument("--after-ref", default="HEAD", help="After Git ref") + parser.add_argument( + "--pull-substrate", + action="store_true", + help="显式从镜像仓库拉取冻结 substrate", + ) + + +def add_seed_arguments(parser: argparse.ArgumentParser) -> None: + """添加 seed 与 run 共用的浏览器缓存参数。""" + browser_group = parser.add_mutually_exclusive_group(required=False) + browser_group.add_argument( + "--browser-source-volume", + help=( + "从已有命名 volume 复制 CloakBrowser 缓存，不访问外网；" + f"默认 {DEFAULT_BROWSER_SOURCE_VOLUME}" + ), + ) + browser_group.add_argument( + "--allow-browser-download", + action="store_true", + help="允许 seed 阶段一次性联网下载 CloakBrowser；样本仍使用 internal network", + ) + parser.add_argument( + "--replace", + action="store_true", + help="替换同 campaign 的 seed 或样本结果", + ) + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行解析器。""" + parser = argparse.ArgumentParser( + description="MoviePilot V3 reproducible Docker A/B measurement harness" + ) + parser.add_argument("--campaign", type=normalize_campaign, required=True) + parser.add_argument("--repo", type=Path, default=PROJECT_ROOT) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(tempfile.gettempdir()) / "moviepilot-perf-results", + help="结果根目录，默认位于系统临时目录", + ) + parser.add_argument("--substrate", default=DEFAULT_SUBSTRATE) + parser.add_argument( + "--platform", + default="auto", + help="Docker 平台，默认使用 daemon 原生架构；正式数据禁止 QEMU 跨架构", + ) + parser.add_argument("--cpus", type=float, default=4.0) + parser.add_argument("--memory", default="2g") + parser.add_argument("--ready-timeout", type=int, default=300) + parser.add_argument("--settle-timeout", type=int, default=300) + parser.add_argument("--stop-timeout", type=int, default=120) + + subparsers = parser.add_subparsers(dest="command", required=True) + build = subparsers.add_parser( + "build", help="构建冻结 substrate 的 Before/After overlay 镜像" + ) + add_common_build_arguments(build) + + seed = subparsers.add_parser( + "seed", help="创建迁移后 SQLite 和预热浏览器 seed volume" + ) + add_seed_arguments(seed) + + sample = subparsers.add_parser("sample", help="执行一个 Before 或 After 样本") + sample.add_argument("--variant", choices=("before", "after"), required=True) + sample.add_argument("--index", type=int, choices=(1, 2, 3), required=True) + sample.add_argument( + "--points", type=parse_points, default=parse_points("1,5,10,30") + ) + sample.add_argument("--replace", action="store_true") + + run = subparsers.add_parser("run", help="完整执行 build、seed 和三组平衡 A/B") + add_common_build_arguments(run) + add_seed_arguments(run) + run.add_argument("--points", type=parse_points, default=parse_points("1,5,10,30")) + run.add_argument( + "--keep-resources", + action="store_true", + help="完成或失败后保留 seed volume 和 internal network 供诊断", + ) + + cleanup = subparsers.add_parser("cleanup", help="清理当前 campaign 的 Docker 资源") + cleanup.add_argument( + "--images", action="store_true", help="同时移除 Before/After 派生镜像" + ) + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + """解析参数并执行选定阶段。""" + parser = build_parser() + args = parser.parse_args(argv) + if args.command in {"seed", "run"}: + if not args.browser_source_volume and not args.allow_browser_download: + args.browser_source_volume = DEFAULT_BROWSER_SOURCE_VOLUME + if args.cpus <= 0: + parser.error("--cpus 必须大于 0") + if args.ready_timeout <= 0 or args.settle_timeout <= 0 or args.stop_timeout <= 0: + parser.error("timeout 必须大于 0") + try: + if args.command == "build": + command_build(args) + elif args.command == "seed": + command_seed(args) + elif args.command == "sample": + command_sample(args) + elif args.command == "run": + command_run(args) + elif args.command == "cleanup": + command_cleanup(args) + else: # pragma: no cover - argparse 已保证不可达 + parser.error(f"未知命令：{args.command}") + except HarnessError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_capability_registry.py b/tests/test_capability_registry.py new file mode 100644 index 000000000..82adc2586 --- /dev/null +++ b/tests/test_capability_registry.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from app.runtime.capabilities.errors import CapabilityManifestError +from app.runtime.capabilities.model import ( + ActivationPolicy, + SelectorSchema, +) +from app.runtime.capabilities.registry import CapabilityRegistry + + +_BASE_MANIFEST = """ +schema_version = 1 +id = "sample.capability" +kind = "sample" +entrypoint = "sample_implementation:SampleCapability" +depends_on = [] + +[metadata] +name = "Sample capability" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["sample.config"] + +[activation.selector] +kind = "configured" +key = "sample.config" +enabled = true +""" + + +def _write_manifest(root: Path, content: str = _BASE_MANIFEST, name: str = "sample") -> Path: + manifest_dir = root / name + manifest_dir.mkdir(parents=True) + manifest_path = manifest_dir / "capability.toml" + manifest_path.write_text(content.strip() + "\n", encoding="utf-8") + return manifest_path + + +def _discover(root: Path) -> CapabilityRegistry: + return CapabilityRegistry.discover( + roots=[root], + kinds={"sample"}, + selector_schemas={ + "configured": SelectorSchema( + required_fields=frozenset({"key", "enabled"}), + ) + }, + ) + + +def test_discovery_reads_toml_without_importing_entrypoint(tmp_path: Path) -> None: + """能力发现只能读取声明，不能执行 entrypoint 对应的 Python 模块。""" + _write_manifest(tmp_path) + (tmp_path / "sample_implementation.py").write_text( + "raise AssertionError('entrypoint must not be imported during discovery')\n", + encoding="utf-8", + ) + sys.modules.pop("sample_implementation", None) + + registry = _discover(tmp_path) + + spec = registry.get_spec("sample.capability") + assert spec is not None + assert spec.activation is ActivationPolicy.WHEN_CONFIGURED + assert spec.selector is not None + assert spec.selector.kind == "configured" + assert spec.selector.config == {"key": "sample.config", "enabled": True} + assert spec.watch == ("sample.config",) + assert spec.depends_on == () + assert "sample_implementation" not in sys.modules + + +def test_discovered_specs_are_recursively_immutable(tmp_path: Path) -> None: + """Registry 暴露的声明及嵌套 metadata/selector 都不能被调用方改写。""" + _write_manifest(tmp_path) + spec = _discover(tmp_path).require_spec("sample.capability") + + with pytest.raises(TypeError): + spec.metadata["name"] = "changed" + with pytest.raises(TypeError): + spec.selector.config["key"] = "changed" # type: ignore[union-attr] + with pytest.raises(AttributeError): + spec.watch.append("changed") # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + ("replacement", "match"), + [ + ("schema_version = 1", "缺少字段"), + (_BASE_MANIFEST.replace("schema_version = 1", "schema_version = 2"), "schema_version"), + (_BASE_MANIFEST.replace("schema_version = 1", "schema_version = 1.0"), "schema_version"), + (_BASE_MANIFEST.replace('id = "sample.capability"', 'id = "bad id"'), "id"), + (_BASE_MANIFEST.replace('kind = "sample"', 'kind = "unknown"'), "kind"), + ( + _BASE_MANIFEST.replace( + 'entrypoint = "sample_implementation:SampleCapability"', + 'entrypoint = "sample_implementation.SampleCapability"', + ), + "entrypoint", + ), + (_BASE_MANIFEST.replace('kind = "configured"', 'kind = "unknown"'), "selector"), + (_BASE_MANIFEST.replace('enabled = true', 'extra = true'), "selector"), + (_BASE_MANIFEST.replace("depends_on = []", 'depends_on = ["other"]'), "depends_on"), + (_BASE_MANIFEST.replace("watch =", "unknown_field = true\nwatch ="), "activation"), + ], +) +def test_registry_fails_closed_for_invalid_manifest( + tmp_path: Path, + replacement: str, + match: str, +) -> None: + """未知或不完整声明必须阻止 Registry 构建，不能静默丢失能力。""" + _write_manifest(tmp_path, replacement) + + with pytest.raises(CapabilityManifestError, match=match): + _discover(tmp_path) + + +def test_selector_presence_must_match_activation_policy(tmp_path: Path) -> None: + """只有 when_configured 声明可以且必须携带配置 selector。""" + bootstrap = _BASE_MANIFEST.replace( + 'policy = "when_configured"', 'policy = "bootstrap"' + ) + _write_manifest(tmp_path, bootstrap) + + with pytest.raises(CapabilityManifestError, match="selector"): + _discover(tmp_path) + + +def test_registry_rejects_duplicate_ids_across_roots(tmp_path: Path) -> None: + """多个声明根出现相同 capability ID 时必须 fail closed。""" + first_root = tmp_path / "first" + second_root = tmp_path / "second" + _write_manifest(first_root) + _write_manifest(second_root) + + with pytest.raises(CapabilityManifestError, match="重复"): + CapabilityRegistry.discover( + roots=[first_root, second_root], + kinds={"sample"}, + selector_schemas={ + "configured": SelectorSchema( + required_fields=frozenset({"key", "enabled"}), + ) + }, + ) + + +def test_registry_rejects_root_without_manifest(tmp_path: Path) -> None: + """注册了声明根却没有任何 manifest 时应直接失败。""" + with pytest.raises(CapabilityManifestError, match="capability.toml"): + _discover(tmp_path) + + +def test_current_host_module_manifests_follow_the_strict_nested_schema() -> None: + """仓内 Host Module 声明必须全部通过同一套嵌套 schema。""" + modules_root = Path(__file__).parents[1] / "app" / "modules" + imported_before = set(sys.modules) + registry = CapabilityRegistry.discover( + roots=[modules_root], + kinds={"host_module"}, + selector_schemas={ + "system_config_item": SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + ), + "setting_truthy": SelectorSchema( + required_fields=frozenset({"key"}), + ), + }, + ) + + specs = registry.list_specs() + declared_directories = {spec.source.parent for spec in specs} + module_directories = { + path + for path in modules_root.iterdir() + if path.is_dir() and (path / "__init__.py").is_file() + } + entrypoint_modules = {spec.entrypoint.split(":", maxsplit=1)[0] for spec in specs} + + assert declared_directories == module_directories + assert all(spec.kind == "host_module" for spec in specs) + assert not ((set(sys.modules) - imported_before) & entrypoint_modules) diff --git a/tests/test_capability_runtime.py b/tests/test_capability_runtime.py new file mode 100644 index 000000000..3d0641141 --- /dev/null +++ b/tests/test_capability_runtime.py @@ -0,0 +1,834 @@ +from __future__ import annotations + +import asyncio +import sys +import threading +import types +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from unittest.mock import patch + +import pytest + +from app.runtime.capabilities.errors import ( + CapabilityAdapterModeError, + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.model import ( + AdapterExecutionMode, + CapabilityLifecycleState, + CapabilityMaterializationState, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.capabilities.runtime import CapabilityRuntime + + +_MANIFEST = """ +schema_version = 1 +id = "sample.capability" +kind = "sample" +entrypoint = "sample_implementation:SampleCapability" +depends_on = [] + +[metadata] +name = "Sample capability" + +[activation] +policy = "on_first_use" +watch = [] +""" + + +def _registry(tmp_path: Path) -> CapabilityRegistry: + manifest_dir = tmp_path / "sample" + manifest_dir.mkdir(parents=True) + (manifest_dir / "capability.toml").write_text( + _MANIFEST.strip() + "\n", + encoding="utf-8", + ) + return CapabilityRegistry.discover( + roots=[tmp_path], + kinds={"sample"}, + selector_schemas={}, + ) + + +@dataclass +class _Candidate: + generation: int + started: bool = False + stopped: bool = False + + +class _SyncAdapter: + execution_mode = AdapterExecutionMode.SYNC + + def __init__(self) -> None: + self.materialize_calls = 0 + self.create_calls = 0 + self.start_calls = 0 + self.stop_calls = 0 + self.stop_instances = [] + self.cleanup_calls = 0 + self.fail_materialize = False + self.fail_start = False + self.fail_stop = False + self.start_entered: Optional[threading.Event] = None + self.start_release: Optional[threading.Event] = None + self.stop_entered: Optional[threading.Event] = None + self.stop_release: Optional[threading.Event] = None + + def materialize(self, spec) -> object: + self.materialize_calls += 1 + if self.fail_materialize: + raise RuntimeError("materialize failed") + return object() + + def create(self, spec, implementation: object, generation: int, previous: Any = None) -> _Candidate: + self.create_calls += 1 + return _Candidate(generation=generation) + + def start(self, spec, candidate: _Candidate, generation: int) -> None: + self.start_calls += 1 + if self.start_entered: + self.start_entered.set() + if self.start_release: + assert self.start_release.wait(timeout=5) + candidate.started = True + if self.fail_start: + raise RuntimeError("start failed") + + def stop(self, spec, instance: _Candidate, generation: int) -> None: + self.stop_calls += 1 + self.stop_instances.append(instance) + if self.stop_entered: + self.stop_entered.set() + if self.stop_release: + assert self.stop_release.wait(timeout=5) + instance.stopped = True + if self.fail_stop: + raise RuntimeError("stop failed") + + def cleanup(self, spec, candidate: _Candidate, generation: int, error: BaseException) -> None: + self.cleanup_calls += 1 + candidate.stopped = True + + +class _AsyncAdapter: + execution_mode = AdapterExecutionMode.ASYNC + + def __init__(self) -> None: + self.materialize_calls = 0 + self.create_calls = 0 + self.start_calls = 0 + self.stop_calls = 0 + self.stop_instances = [] + self.cleanup_calls = 0 + self.fail_materialize = False + self.fail_start = False + self.fail_stop = False + self.start_entered = asyncio.Event() + self.start_release = asyncio.Event() + self.stop_entered: Optional[asyncio.Event] = None + self.stop_release: Optional[asyncio.Event] = None + + async def materialize(self, spec) -> object: + self.materialize_calls += 1 + await asyncio.sleep(0) + if self.fail_materialize: + raise RuntimeError("async materialize failed") + return object() + + async def create(self, spec, implementation: object, generation: int, previous: Any = None) -> _Candidate: + self.create_calls += 1 + await asyncio.sleep(0) + return _Candidate(generation=generation) + + async def start(self, spec, candidate: _Candidate, generation: int) -> None: + self.start_calls += 1 + self.start_entered.set() + await self.start_release.wait() + candidate.started = True + if self.fail_start: + raise RuntimeError("async start failed") + + async def stop(self, spec, instance: _Candidate, generation: int) -> None: + self.stop_calls += 1 + self.stop_instances.append(instance) + if self.stop_entered: + self.stop_entered.set() + if self.stop_release: + await self.stop_release.wait() + await asyncio.sleep(0) + instance.stopped = True + if self.fail_stop: + raise RuntimeError("async stop failed") + + async def cleanup(self, spec, candidate: _Candidate, generation: int, error: BaseException) -> None: + self.cleanup_calls += 1 + await asyncio.sleep(0) + candidate.stopped = True + + +def test_materialize_and_start_have_independent_state_axes(tmp_path: Path) -> None: + """兼容查询只物化代码，资源必须等显式 activate 成功后才对外可见。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + implementation = runtime.materialize("sample.capability", reason="compat_lookup") + materialized = runtime.snapshot("sample.capability") + + assert implementation is not None + assert materialized.materialization is CapabilityMaterializationState.RESOLVED + assert materialized.lifecycle is CapabilityLifecycleState.DISCOVERED + assert materialized.visible is False + assert adapter.start_calls == 0 + + instance = runtime.activate("sample.capability", reason="first_use") + running = runtime.snapshot("sample.capability") + + assert runtime.get_running("sample.capability") is instance + assert running.lifecycle is CapabilityLifecycleState.RUNNING + assert running.visible is True + assert running.generation == 2 + + +def test_materialize_failure_does_not_claim_resource_lifecycle_failure(tmp_path: Path) -> None: + """仅解析代码失败时，资源轴尚未启动，必须保持 DISCOVERED。""" + adapter = _SyncAdapter() + adapter.fail_materialize = True + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityOperationError, match="materialize failed"): + runtime.materialize("sample.capability", reason="compat_lookup") + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.materialization is CapabilityMaterializationState.FAILED + assert snapshot.lifecycle is CapabilityLifecycleState.DISCOVERED + assert snapshot.visible is False + + +def test_state_read_calibrates_consumer_import_without_importing_new_module(tmp_path: Path) -> None: + """显式旧导入存在时应复用 sys.modules 中的 canonical symbol。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + module = types.ModuleType("sample_implementation") + canonical = type("SampleCapability", (), {}) + module.SampleCapability = canonical + + with patch.dict(sys.modules, {"sample_implementation": module}): + snapshot = runtime.snapshot("sample.capability") + implementation = runtime.materialize( + "sample.capability", + reason="compat_lookup", + ) + + assert snapshot.materialization is CapabilityMaterializationState.RESOLVED + assert snapshot.lifecycle is CapabilityLifecycleState.DISCOVERED + assert snapshot.generation == 0 + assert implementation is canonical + assert adapter.materialize_calls == 0 + + +def test_state_read_does_not_invoke_module_level_lazy_export(tmp_path: Path) -> None: + """sys.modules 校准只能读模块字典，不能触发模块级 __getattr__。""" + runtime = CapabilityRuntime( + _registry(tmp_path), + adapters={"sample": _SyncAdapter()}, + ) + module = types.ModuleType("sample_implementation") + lazy_reads = [] + + def resolve(name: str) -> object: + lazy_reads.append(name) + raise AssertionError("state read must not resolve lazy exports") + + module.__getattr__ = resolve + with patch.dict(sys.modules, {"sample_implementation": module}): + snapshot = runtime.snapshot("sample.capability") + + assert snapshot.materialization is CapabilityMaterializationState.UNRESOLVED + assert lazy_reads == [] + + +def test_sync_activate_is_single_flight_and_publishes_only_after_start(tmp_path: Path) -> None: + """并发首启只能创建一个候选实例，start 返回前普通查询不可见。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + results = [] + errors = [] + + def activate() -> None: + try: + results.append(runtime.activate("sample.capability", reason="concurrent")) + except BaseException as error: # pragma: no cover - diagnostic collection + errors.append(error) + + first = threading.Thread(target=activate) + second = threading.Thread(target=activate) + first.start() + assert adapter.start_entered.wait(timeout=5) + second.start() + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STARTING + adapter.start_release.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not errors + assert len(results) == 2 + assert results[0] is results[1] + assert adapter.materialize_calls == 1 + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + assert runtime.snapshot("sample.capability").generation == 1 + + +def test_failed_start_cleans_candidate_and_requires_explicit_retry(tmp_path: Path) -> None: + """半初始化候选必须清理；FAILED 不得被普通 activate 隐式重试。""" + adapter = _SyncAdapter() + adapter.fail_start = True + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityOperationError, match="start failed"): + runtime.activate("sample.capability", reason="first_attempt") + + failed = runtime.snapshot("sample.capability") + assert failed.materialization is CapabilityMaterializationState.RESOLVED + assert failed.lifecycle is CapabilityLifecycleState.FAILED + assert failed.visible is False + assert adapter.cleanup_calls == 1 + + with pytest.raises(CapabilityOperationError, match="显式 retry"): + runtime.activate("sample.capability", reason="implicit_retry") + assert adapter.start_calls == 1 + + adapter.fail_start = False + instance = runtime.activate("sample.capability", reason="explicit_retry", retry=True) + + assert instance.started is True + assert runtime.snapshot("sample.capability").generation == 2 + assert adapter.start_calls == 2 + + +def test_adapter_must_return_candidate_before_start(tmp_path: Path) -> None: + """create 没有候选对象时不得进入 start 或伪造 RUNNING 可见性。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with patch.object(adapter, "create", return_value=None), pytest.raises( + CapabilityOperationError, + match="candidate", + ): + runtime.activate("sample.capability", reason="invalid_candidate") + + assert adapter.start_calls == 0 + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert runtime.get_running("sample.capability") is None + + +def test_stop_withdraws_visibility_before_adapter_callback(tmp_path: Path) -> None: + """释放外部资源可能阻塞，但运行实例必须在 stop 回调前撤销发布。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + instance = runtime.activate("sample.capability", reason="start") + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + + stopper = threading.Thread( + target=lambda: runtime.stop("sample.capability", reason="configuration_removed") + ) + stopper.start() + assert adapter.stop_entered.wait(timeout=5) + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + adapter.stop_release.set() + stopper.join(timeout=5) + + assert instance.stopped is True + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + + +def test_stop_failure_retains_ownership_until_same_instance_stops(tmp_path: Path) -> None: + """stop 失败后的隐藏资源必须保留所有权，禁止用 retry 绕过清理。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + instance = runtime.activate("sample.capability", reason="start") + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.stop("sample.capability", reason="configuration_removed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.activate("sample.capability", reason="unsafe_retry", retry=True) + assert adapter.create_calls == 1 + + adapter.fail_stop = False + runtime.stop("sample.capability", reason="stop_retry") + + assert adapter.stop_instances == [instance, instance, instance] + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + replacement = runtime.activate("sample.capability", reason="after_release") + assert replacement is not instance + assert adapter.create_calls == 2 + + +def test_reload_withdraws_old_instance_and_publishes_one_new_generation(tmp_path: Path) -> None: + """同步 reload 在 stop/start 回调期间不暴露旧实例或半初始化候选。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + old_instance = runtime.activate("sample.capability", reason="initial") + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + results = [] + + reloader = threading.Thread( + target=lambda: results.append(runtime.reload("sample.capability", reason="config_changed")) + ) + reloader.start() + assert adapter.stop_entered.wait(timeout=5) + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.RELOADING + adapter.stop_release.set() + reloader.join(timeout=5) + + assert len(results) == 1 + assert results[0] is not old_instance + assert runtime.get_running("sample.capability") is results[0] + assert runtime.snapshot("sample.capability").generation == 2 + + +def test_failed_reload_cleans_candidate_and_keeps_instance_invisible(tmp_path: Path) -> None: + """reload 新 generation 启动失败时不得恢复旧实例或发布候选。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + runtime.activate("sample.capability", reason="initial") + adapter.fail_start = True + + with pytest.raises(CapabilityOperationError, match="start failed"): + runtime.reload("sample.capability", reason="config_changed") + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.lifecycle is CapabilityLifecycleState.FAILED + assert snapshot.visible is False + assert adapter.cleanup_calls == 1 + + +def test_reload_stop_failure_does_not_create_or_reuse_live_previous(tmp_path: Path) -> None: + """reload 未释放旧资源时必须失败关闭，不能创建或复用同一活对象。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + old_instance = runtime.activate("sample.capability", reason="initial") + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.reload("sample.capability", reason="config_changed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert adapter.stop_instances == [old_instance] + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + with pytest.raises(CapabilityOperationError, match="重试 stop"): + runtime.activate("sample.capability", reason="implicit_retry") + + adapter.fail_stop = False + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + recovered = [] + recovery = threading.Thread( + target=lambda: recovered.append( + runtime.activate( + "sample.capability", + reason="recover_after_reload", + retry=True, + ) + ) + ) + recovery.start() + assert adapter.stop_entered.wait(timeout=5) + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + assert runtime.get_running("sample.capability") is None + assert adapter.create_calls == 1 + adapter.stop_release.set() + recovery.join(timeout=5) + + assert len(recovered) == 1 + replacement = recovered[0] + assert adapter.stop_instances == [old_instance, old_instance] + assert replacement is not old_instance + assert adapter.create_calls == 2 + + +def test_shutdown_prevents_inflight_start_from_resurrecting_instance(tmp_path: Path) -> None: + """shutdown 与首启竞争时，候选只能清理，不能在关闭开始后重新发布。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + activate_errors = [] + + def activate() -> None: + try: + runtime.activate("sample.capability", reason="racing_start") + except BaseException as error: + activate_errors.append(error) + + starter = threading.Thread(target=activate) + starter.start() + assert adapter.start_entered.wait(timeout=5) + closer = threading.Thread(target=lambda: runtime.shutdown(reason="application_shutdown")) + closer.start() + adapter.start_release.set() + starter.join(timeout=5) + closer.join(timeout=5) + + assert len(activate_errors) == 1 + assert isinstance(activate_errors[0], CapabilityRuntimeClosedError) + assert adapter.cleanup_calls == 1 + assert runtime.get_running("sample.capability") is None + assert runtime.is_shutdown is True + with pytest.raises(CapabilityRuntimeClosedError): + runtime.activate("sample.capability", reason="late_start") + + +def test_shutdown_cannot_return_between_open_check_and_sync_claim(tmp_path: Path) -> None: + """open check 与 inflight claim 必须共享 barrier，关闭扫描不能漏过首启。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + check_entered = threading.Event() + check_release = threading.Event() + shutdown_returned = threading.Event() + activate_errors = [] + original_ensure_open = runtime._ensure_open + first_check = True + check_lock = threading.Lock() + + def gated_ensure_open() -> None: + nonlocal first_check + original_ensure_open() + with check_lock: + should_wait = first_check + first_check = False + if should_wait: + check_entered.set() + assert check_release.wait(timeout=5) + + def activate() -> None: + try: + runtime.activate("sample.capability", reason="preclaim_race") + except BaseException as error: + activate_errors.append(error) + + def shutdown() -> None: + runtime.shutdown(reason="application_shutdown") + shutdown_returned.set() + + with patch.object(runtime, "_ensure_open", side_effect=gated_ensure_open): + starter = threading.Thread(target=activate) + starter.start() + assert check_entered.wait(timeout=5) + closer = threading.Thread(target=shutdown) + closer.start() + + assert not shutdown_returned.wait(timeout=0.1) + check_release.set() + assert adapter.start_entered.wait(timeout=5) + assert not shutdown_returned.is_set() + adapter.start_release.set() + starter.join(timeout=5) + closer.join(timeout=5) + + assert shutdown_returned.is_set() + assert len(activate_errors) <= 1 + assert not activate_errors or isinstance( + activate_errors[0], + CapabilityRuntimeClosedError, + ) + assert runtime.get_running("sample.capability") is None + + +@pytest.mark.asyncio +async def test_shutdown_cannot_return_between_open_check_and_async_claim( + tmp_path: Path, +) -> None: + """异步 activate 的同步 claim 区间也必须受同一关闭 barrier 保护。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + check_entered = threading.Event() + check_release = threading.Event() + shutdown_returned = threading.Event() + returned_before_release = [] + closer_threads = [] + original_ensure_open = runtime._ensure_open + first_check = True + check_lock = threading.Lock() + + def gated_ensure_open() -> None: + nonlocal first_check + original_ensure_open() + with check_lock: + should_wait = first_check + first_check = False + if should_wait: + check_entered.set() + assert check_release.wait(timeout=5) + + def shutdown() -> None: + asyncio.run(runtime.shutdown_async(reason="application_shutdown")) + shutdown_returned.set() + + def coordinate_shutdown() -> None: + assert check_entered.wait(timeout=5) + closer = threading.Thread(target=shutdown) + closer_threads.append(closer) + closer.start() + returned_before_release.append(shutdown_returned.wait(timeout=0.1)) + check_release.set() + + coordinator = threading.Thread(target=coordinate_shutdown) + coordinator.start() + with patch.object(runtime, "_ensure_open", side_effect=gated_ensure_open): + activate_task = asyncio.create_task( + runtime.activate_async("sample.capability", reason="preclaim_race") + ) + await adapter.start_entered.wait() + await asyncio.to_thread(coordinator.join, 5) + assert returned_before_release == [False] + assert not shutdown_returned.is_set() + adapter.start_release.set() + try: + await activate_task + except CapabilityRuntimeClosedError: + pass + await asyncio.to_thread(closer_threads[0].join, 5) + + assert shutdown_returned.is_set() + assert runtime.get_running("sample.capability") is None + + +@pytest.mark.asyncio +async def test_async_adapter_uses_same_single_flight_state_machine(tmp_path: Path) -> None: + """异步回调等待不能阻塞事件循环，并发调用共享同一 generation。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + first = asyncio.create_task(runtime.activate_async("sample.capability", reason="first")) + await adapter.start_entered.wait() + second = asyncio.create_task(runtime.activate_async("sample.capability", reason="second")) + await asyncio.sleep(0) + + assert runtime.get_running("sample.capability") is None + adapter.start_release.set() + first_instance, second_instance = await asyncio.gather(first, second) + + assert first_instance is second_instance + assert adapter.materialize_calls == 1 + assert adapter.start_calls == 1 + assert runtime.snapshot("sample.capability").generation == 1 + + +@pytest.mark.asyncio +async def test_stop_async_failure_retains_ownership_for_explicit_retry(tmp_path: Path) -> None: + """异步 stop 失败后只能重试释放同一实例，不能直接启动新实例。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + instance = await initial + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.stop_async("sample.capability", reason="configuration_removed") + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.activate_async("sample.capability", reason="unsafe_retry", retry=True) + assert adapter.create_calls == 1 + + adapter.fail_stop = False + await runtime.stop_async("sample.capability", reason="stop_retry") + + assert adapter.stop_instances == [instance, instance, instance] + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + + +@pytest.mark.asyncio +async def test_async_reload_uses_reloading_state_and_hides_candidate(tmp_path: Path) -> None: + """异步 reload 与同步入口遵守相同状态和发布边界。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + old_instance = await initial + + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + reload_task = asyncio.create_task( + runtime.reload_async("sample.capability", reason="config_changed") + ) + await adapter.start_entered.wait() + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.RELOADING + adapter.start_release.set() + new_instance = await reload_task + + assert new_instance is not old_instance + assert runtime.get_running("sample.capability") is new_instance + assert runtime.snapshot("sample.capability").generation == 2 + + +@pytest.mark.asyncio +async def test_failed_async_reload_cleans_candidate_and_enters_failed(tmp_path: Path) -> None: + """异步 reload 失败与同步入口一致，不发布半初始化候选。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + await initial + + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + adapter.fail_start = True + reload_task = asyncio.create_task( + runtime.reload_async("sample.capability", reason="config_changed") + ) + await adapter.start_entered.wait() + adapter.start_release.set() + + with pytest.raises(CapabilityOperationError, match="async start failed"): + await reload_task + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.lifecycle is CapabilityLifecycleState.FAILED + assert snapshot.visible is False + assert adapter.cleanup_calls == 1 + + +@pytest.mark.asyncio +async def test_async_reload_stop_failure_retains_previous_without_new_create( + tmp_path: Path, +) -> None: + """异步 reload 也必须保留未释放旧实例并禁止创建第二份资源。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + old_instance = await initial + + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.reload_async("sample.capability", reason="config_changed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert adapter.stop_instances == [old_instance] + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + with pytest.raises(CapabilityOperationError, match="重试 stop"): + await runtime.activate_async("sample.capability", reason="implicit_retry") + adapter.fail_stop = False + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + adapter.stop_entered = asyncio.Event() + adapter.stop_release = asyncio.Event() + recovery = asyncio.create_task( + runtime.activate_async( + "sample.capability", + reason="recover_after_reload", + retry=True, + ) + ) + await adapter.stop_entered.wait() + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + assert runtime.get_running("sample.capability") is None + assert adapter.create_calls == 1 + adapter.stop_release.set() + await adapter.start_entered.wait() + adapter.start_release.set() + replacement = await recovery + + assert adapter.stop_instances == [old_instance, old_instance] + assert replacement is not old_instance + assert adapter.create_calls == 2 + + +def test_one_failed_capability_does_not_remove_specs_or_block_other_capabilities( + tmp_path: Path, +) -> None: + """单项失败只改变自身状态，Registry 中的其它声明仍可继续运行。""" + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + (first_dir / "capability.toml").write_text(_MANIFEST.strip() + "\n", encoding="utf-8") + (second_dir / "capability.toml").write_text( + _MANIFEST.replace("sample.capability", "other.capability") + .replace("sample_implementation", "other_implementation") + .strip() + + "\n", + encoding="utf-8", + ) + registry = CapabilityRegistry.discover( + roots=[tmp_path], + kinds={"sample"}, + selector_schemas={}, + ) + adapter = _SyncAdapter() + runtime = CapabilityRuntime(registry, adapters={"sample": adapter}) + adapter.fail_start = True + + with pytest.raises(CapabilityOperationError): + runtime.activate("sample.capability", reason="fail") + adapter.fail_start = False + other = runtime.activate("other.capability", reason="continue") + + assert other.started is True + assert {spec.id for spec in runtime.list_specs()} == { + "sample.capability", + "other.capability", + } + assert runtime.snapshot("sample.capability").error == "start failed" + + +@pytest.mark.asyncio +async def test_sync_and_async_entrypoints_reject_wrong_adapter_mode(tmp_path: Path) -> None: + """入口与 adapter 执行模型不匹配时应在执行回调前失败。""" + async_runtime = CapabilityRuntime( + _registry(tmp_path / "async"), + adapters={"sample": _AsyncAdapter()}, + ) + with pytest.raises(CapabilityAdapterModeError): + async_runtime.activate("sample.capability", reason="wrong_mode") + + sync_runtime = CapabilityRuntime( + _registry(tmp_path / "sync"), + adapters={"sample": _SyncAdapter()}, + ) + with pytest.raises(CapabilityAdapterModeError): + await sync_runtime.activate_async("sample.capability", reason="wrong_mode") + + +def test_adapter_mode_requires_declared_enum_member(tmp_path: Path) -> None: + """并发模型必须显式声明 enum，不能依赖字符串相等的偶然兼容。""" + adapter = _SyncAdapter() + adapter.execution_mode = "sync" + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityAdapterModeError): + runtime.activate("sample.capability", reason="invalid_mode") diff --git a/tests/test_config_reload_handler.py b/tests/test_config_reload_handler.py index 3d1367a34..9789b0cd0 100644 --- a/tests/test_config_reload_handler.py +++ b/tests/test_config_reload_handler.py @@ -126,3 +126,23 @@ async def test_resolve_falls_back_to_qualname_when_name_mismatched(monkeypatch): assert wrapper.__name__ == "wrapper" assert instance.reload_count == 1 + + +def test_externally_managed_reload_class_does_not_register_listener(monkeypatch): + """外部统一管理配置生命周期时，Mixin 保留重载能力但不重复绑定事件。""" + registrations = [] + monkeypatch.setattr( + eventmanager, + "add_event_listener", + lambda *args, **kwargs: registrations.append((args, kwargs)), + ) + + class _ExternallyManagedReloadRecorder(ConfigReloadMixin): + CONFIG_RELOAD_MANAGED_EXTERNALLY = True + CONFIG_WATCH = {"TEST_RELOAD_KEY"} + + def on_config_changed(self): + pass + + assert registrations == [] + assert "handle_config_changed" not in _ExternallyManagedReloadRecorder.__dict__ diff --git a/tests/test_event_dispatch_snapshot.py b/tests/test_event_dispatch_snapshot.py new file mode 100644 index 000000000..ee425919f --- /dev/null +++ b/tests/test_event_dispatch_snapshot.py @@ -0,0 +1,160 @@ +"""事件调度订阅快照的并发回归测试。""" + +import pytest + +from app.runtime.events import Event, eventmanager +from app.schemas.types import ChainEventType, EventType + + +class _ImmediateExecutor: + """在当前线程执行广播 handler，使订阅变更精确发生在调度迭代期间。""" + + @staticmethod + def submit(func, *args, **kwargs): + return func(*args, **kwargs) + + +@pytest.fixture +def isolated_eventmanager(monkeypatch): + """隔离全局事件总线的订阅表和广播执行器。""" + monkeypatch.setattr( + eventmanager, + "_EventManager__broadcast_subscribers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__chain_subscribers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__handler_instance_resolvers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__executor", + _ImmediateExecutor(), + ) + return eventmanager + + +def test_broadcast_dispatch_uses_subscription_snapshot(isolated_eventmanager): + """广播事件中新增或移除的 handler 从下一个事件开始生效。""" + calls = [] + + def late_handler(_event): + calls.append("late") + + def removed_handler(_event): + calls.append("removed") + + def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + EventType.ConfigChanged, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_broadcast_event + dispatch(Event(EventType.ConfigChanged, {})) + assert calls == ["mutating", "removed"] + + calls.clear() + dispatch(Event(EventType.ConfigChanged, {})) + assert calls == ["mutating", "late"] + + +def test_sync_chain_dispatch_uses_subscription_snapshot(isolated_eventmanager): + """同步链式事件中的订阅变更不影响当前处理器序列。""" + calls = [] + + def late_handler(_event): + calls.append("late") + + def removed_handler(_event): + calls.append("removed") + + def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_chain_event + assert dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "removed"] + + calls.clear() + assert dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "late"] + + +@pytest.mark.asyncio +async def test_async_chain_dispatch_uses_subscription_snapshot( + isolated_eventmanager, +): + """异步链式事件中的订阅变更不影响当前处理器序列。""" + calls = [] + + async def late_handler(_event): + calls.append("late") + + async def removed_handler(_event): + calls.append("removed") + + async def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_chain_event_async + assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "removed"] + + calls.clear() + assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "late"] diff --git a/tests/test_lifecycle_shutdown.py b/tests/test_lifecycle_shutdown.py index 309d6d520..6feb4d568 100644 --- a/tests/test_lifecycle_shutdown.py +++ b/tests/test_lifecycle_shutdown.py @@ -364,7 +364,7 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: """替换 stop_modules 的资源所有者，避免测试启动真实后台服务""" dependencies = {} for name, method_name in ( - ("ModuleManager", "stop"), + ("ModuleManager", "shutdown"), ("EventManager", "stop"), ("DisplayHelper", "stop"), ("DohHelper", "shutdown"), diff --git a/tests/test_module_manager_capability_adapter.py b/tests/test_module_manager_capability_adapter.py new file mode 100644 index 000000000..b9e78418a --- /dev/null +++ b/tests/test_module_manager_capability_adapter.py @@ -0,0 +1,930 @@ +"""Host Module Adapter 对 Capability Runtime 的兼容合同测试。""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Iterator +from unittest.mock import Mock + +import pytest + +from app.db.oper.systemconfig import SystemConfigOper +from app.foundation.singleton import Singleton +from app.runtime.capabilities.errors import CapabilityRuntimeClosedError +from app.runtime.capabilities.model import SelectorSchema +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.events import Event, EventHandlerBinding, eventmanager +from app.runtime.extensions import module_manager as module_manager_extension +from app.runtime.extensions.module_manager import ModuleManager +from app.schemas import ConfigChangeEventData +from app.schemas.types import EventType + + +_SAMPLE_MANIFEST = """ +schema_version = 1 +id = "SampleModule" +kind = "host_module" +entrypoint = "fixture_sample_module:SampleModule" +depends_on = [] + +[metadata] +name = "Sample" +type = "notification" +subtype = "Telegram" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "sample" +enabled_field = "enabled" +""" + +_OTHER_MANIFEST = """ +schema_version = 1 +id = "OtherModule" +kind = "host_module" +entrypoint = "fixture_other_module:OtherModule" +depends_on = [] + +[metadata] +name = "Other" +type = "notification" +subtype = "Telegram" +priority = 20 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "other" +enabled_field = "enabled" +""" + +_MODULE_SOURCE = """ +class {class_name}: + instances = [] + + def __init__(self): + self.events = ["create"] + type(self).instances.append(self) + + def init_module(self): + self.events.append("start") + + def stop(self): + self.events.append("stop") + + def test(self): + return True, "ok" + + def capability_method(self): + return "handled" + + @staticmethod + def get_name(): + return "{name}" + + @staticmethod + def get_type(): + return "notification" + + @staticmethod + def get_subtype(): + return "Telegram" + + @staticmethod + def get_priority(): + return {priority} +""" + + +def _write_capability(root: Path, directory: str, manifest: str) -> None: + """写入一个合成 Host Module 声明。""" + capability_dir = root / directory + capability_dir.mkdir(parents=True) + (capability_dir / "capability.toml").write_text( + manifest.strip() + "\n", + encoding="utf-8", + ) + + +def _build_registry(root: Path) -> CapabilityRegistry: + """用生产 schema 构造只包含两个合成模块的 Registry。""" + _write_capability(root, "sample", _SAMPLE_MANIFEST) + _write_capability(root, "other", _OTHER_MANIFEST) + return CapabilityRegistry.discover( + roots=[root], + kinds={"host_module"}, + selector_schemas={ + "system_config_item": SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + ), + "setting_truthy": SelectorSchema( + required_fields=frozenset({"key"}), + ), + }, + ) + + +def _config_changed_listeners() -> dict: + """读取 ConfigChanged 监听快照，用于验证全局测试状态完整恢复。""" + subscribers = getattr(eventmanager, "_EventManager__broadcast_subscribers") + return dict(subscribers.get(EventType.ConfigChanged, {})) + + +def _run_real_host_module_check(tmp_path: Path, body: str) -> None: + """在隔离后端和进程内网络守卫下执行真实 Host Module 合同检查。""" + project_root = Path(__file__).parents[1] + prelude = r""" +import ipaddress +import socket +import sys + +network_attempts = [] +allowed_hosts = {"127.0.0.1", "::1", "localhost", "0.0.0.0", "::", ""} +real_getaddrinfo = socket.getaddrinfo +real_connect = socket.socket.connect + +def is_allowed_host(host): + normalized = host.decode() if isinstance(host, (bytes, bytearray)) else host + if normalized is None or normalized in allowed_hosts: + return True + try: + address = ipaddress.ip_address(str(normalized).split("%", 1)[0]) + return address.is_loopback or address.is_unspecified + except ValueError: + return False + +def block_network(operation, host): + network_attempts.append((operation, host)) + raise AssertionError(f"Host Module 合同测试禁止真实出站：{operation} {host!r}") + +def guarded_getaddrinfo(host, *args, **kwargs): + if not is_allowed_host(host): + block_network("DNS", host) + return real_getaddrinfo(host, *args, **kwargs) + +def guarded_connect(sock, address): + if isinstance(address, tuple) and address and not is_allowed_host(address[0]): + block_network("socket", address[0]) + return real_connect(sock, address) + +socket.getaddrinfo = guarded_getaddrinfo +socket.socket.connect = guarded_connect + +from app.testing.bootstrap import prepare_backend +prepare_backend() +""" + code = f"{prelude}\n{body}\nassert network_attempts == [], network_attempts\n" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"真实 Host Module 合同检查失败：\nstdout:\n{result.stdout[-4000:]}\n" + f"stderr:\n{result.stderr[-8000:]}" + ) + + +@pytest.fixture +def module_manager_harness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[SimpleNamespace]: + """用合成声明和内存配置隔离 ModuleManager 单例。""" + source_root = tmp_path / "source" + source_root.mkdir() + (source_root / "fixture_sample_module.py").write_text( + _MODULE_SOURCE.format(class_name="SampleModule", name="Sample", priority=10), + encoding="utf-8", + ) + (source_root / "fixture_other_module.py").write_text( + _MODULE_SOURCE.format(class_name="OtherModule", name="Other", priority=20), + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(source_root)) + + registry = _build_registry(tmp_path / "capabilities") + monkeypatch.setattr( + module_manager_extension, + "build_host_module_registry", + lambda: registry, + ) + + config_values = {"Notifications": []} + + def get_config(_self, key=None): + key_value = getattr(key, "value", key) + if key_value is None: + return dict(config_values) + return config_values.get(key_value) + + monkeypatch.setattr(SystemConfigOper, "get", get_config) + + singleton_key = (ModuleManager, (), frozenset()) + previous_manager = Singleton._instances.pop(singleton_key, None) + resolver_attr = "_EventManager__handler_instance_resolvers" + previous_resolvers = dict(getattr(eventmanager, resolver_attr)) + previous_config_changed_listeners = _config_changed_listeners() + for module_name in ("fixture_sample_module", "fixture_other_module"): + sys.modules.pop(module_name, None) + + manager = ModuleManager() + restored = False + + def restore() -> None: + """撤销 Manager 构造写入的单例、resolver 和事件监听器。""" + nonlocal restored + if restored: + return + try: + manager.shutdown() + except (AttributeError, CapabilityRuntimeClosedError): + pass + Singleton._instances.pop(singleton_key, None) + if previous_manager is not None: + Singleton._instances[singleton_key] = previous_manager + setattr(eventmanager, resolver_attr, previous_resolvers) + subscribers = getattr(eventmanager, "_EventManager__broadcast_subscribers") + if previous_config_changed_listeners: + subscribers[EventType.ConfigChanged] = dict( + previous_config_changed_listeners + ) + else: + subscribers.pop(EventType.ConfigChanged, None) + for module_name in ("fixture_sample_module", "fixture_other_module"): + sys.modules.pop(module_name, None) + restored = True + + try: + yield SimpleNamespace( + manager=manager, + config_values=config_values, + previous_config_changed_listeners=previous_config_changed_listeners, + restore=restore, + ) + finally: + restore() + + +def _enable_sample(config_values: dict) -> None: + """写入可通过 sample selector 的最小合法通知配置。""" + config_values["Notifications"] = [ + { + "name": "sample", + "type": "sample", + "config": {}, + "switchs": [], + "enabled": True, + } + ] + + +def test_harness_restores_module_manager_config_listener( + module_manager_harness, +) -> None: + """Fixture teardown 不能把临时 Manager 的 bound listener 留在全局事件总线。""" + manager = module_manager_harness.manager + current_listeners = _config_changed_listeners() + + assert current_listeners != ( + module_manager_harness.previous_config_changed_listeners + ) + assert any( + getattr(listener, "__self__", None) is manager + for listener in current_listeners.values() + ) + + module_manager_harness.restore() + + assert _config_changed_listeners() == ( + module_manager_harness.previous_config_changed_listeners + ) + + +def test_specs_are_lightweight_and_do_not_materialize_modules( + module_manager_harness, +) -> None: + """ModuleManager 的声明视图不能解析任何 Host Module 实现。""" + manager = module_manager_harness.manager + + specs = manager.list_specs() + + assert manager.get_specs() == specs + assert [spec.id for spec in specs] == ["OtherModule", "SampleModule"] + assert [spec.metadata["name"] for spec in specs] == ["Other", "Sample"] + assert "fixture_sample_module" not in sys.modules + assert "fixture_other_module" not in sys.modules + assert manager.get_running_module("SampleModule") is None + + +def test_get_module_materializes_one_canonical_class_without_starting_it( + module_manager_harness, +) -> None: + """兼容查询返回 canonical class，但不创建或启动资源。""" + manager = module_manager_harness.manager + + module_class = manager.get_module("SampleModule") + canonical_class = importlib.import_module( + "fixture_sample_module" + ).SampleModule + + assert module_class is canonical_class + assert manager.get_module("SampleModule") is canonical_class + assert canonical_class.instances == [] + assert manager.get_running_module("SampleModule") is None + assert "fixture_other_module" not in sys.modules + + +def test_get_modules_materializes_all_real_classes_without_starting_them( + module_manager_harness, +) -> None: + """旧 get_modules 合同保留真实 class 字典，不返回代理或隐式激活。""" + manager = module_manager_harness.manager + + modules = manager.get_modules() + + sample_module = importlib.import_module("fixture_sample_module") + other_module = importlib.import_module("fixture_other_module") + assert modules == { + "OtherModule": other_module.OtherModule, + "SampleModule": sample_module.SampleModule, + } + assert sample_module.SampleModule.instances == [] + assert other_module.OtherModule.instances == [] + assert manager.get_running_module("SampleModule") is None + assert manager.get_running_module("OtherModule") is None + + +def test_config_reconcile_reload_and_stop_preserve_manager_contract( + module_manager_harness, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """配置激活、全量 reload 与可重启 stop 保持同步可观察顺序。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + + manager.load_modules() + first = manager.get_running_module("SampleModule") + assert first is not None + assert first.events == ["create", "start"] + + send_event = Mock() + monkeypatch.setattr(eventmanager, "send_event", send_event) + manager.reload() + second = manager.get_running_module("SampleModule") + + assert second is not None + assert second is not first + assert first.events == ["create", "start", "stop"] + assert second.events == ["create", "start"] + send_event.assert_called_once_with(etype=EventType.ModuleReload, data={}) + + manager.stop() + assert manager.get_running_module("SampleModule") is None + assert second.events == ["create", "start", "stop"] + + manager.load_modules() + restarted = manager.get_running_module("SampleModule") + assert restarted is not None + assert restarted is not second + assert restarted.events == ["create", "start"] + + module_manager_harness.config_values["Notifications"] = [] + manager.load_modules() + assert manager.get_running_module("SampleModule") is None + assert restarted.events == ["create", "start", "stop"] + + +def test_config_event_reloads_same_instance_and_tracks_selector_changes( + module_manager_harness, +) -> None: + """配置事件由 Host Adapter 唯一协调，并保留模块实例内的重载状态。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + running = manager.get_running_module("SampleModule") + + manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key="Notifications"), + ) + ) + + assert manager.get_running_module("SampleModule") is running + assert running.events == ["create", "start", "stop", "start"] + + module_manager_harness.config_values["Notifications"] = [] + manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key="Notifications"), + ) + ) + assert manager.get_running_module("SampleModule") is None + assert running.events == ["create", "start", "stop", "start", "stop"] + + +def test_shutdown_is_irreversible(module_manager_harness) -> None: + """shutdown 撤销全部可见实例，并拒绝通过 load_modules 再次启动。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + running = manager.get_running_module("SampleModule") + assert running is not None + + manager.shutdown() + + assert manager.get_running_module("SampleModule") is None + assert running.events == ["create", "start", "stop"] + manager.load_modules() + assert manager.get_running_module("SampleModule") is None + assert type(running).instances == [running] + + +def test_all_real_host_modules_zero_arg_construct_without_starting_resources( + tmp_path: Path, +) -> None: + """每份真实 manifest 都必须能解析 canonical class 并零参数构造且不启动资源。""" + body = r""" +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 + +adapter = HostModuleAdapter() +lifecycle_events = [] +instances = {} + +def make_recorder(operation, capability_id): + def record(instance): + lifecycle_events.append((operation, capability_id, id(instance))) + return record + +for spec in specs: + implementation = adapter.materialize(spec) + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + assert implementation is getattr(sys.modules[module_name], symbol_name) + implementation.init_module = make_recorder("start", spec.id) + implementation.stop = make_recorder("stop", spec.id) + + instance = adapter.create(spec, implementation, generation=1) + assert type(instance) is implementation + instances[spec.id] = instance + +assert set(instances) == {spec.id for spec in specs} +assert lifecycle_events == [] +""" + _run_real_host_module_check(tmp_path, body) + + +def test_real_manifest_inventory_drives_full_module_manager_lifecycle( + tmp_path: Path, +) -> None: + """真实声明自动驱动全量激活、原实例重载、禁用停止和不可逆关闭门禁。""" + body = r""" +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.capabilities.model import ActivationPolicy +from app.runtime.config import settings +from app.runtime.events import Event +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) +from app.schemas import ConfigChangeEventData +from app.schemas.types import EventType + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 +spec_by_id = {spec.id: spec for spec in specs} + +events = {spec.id: [] for spec in specs} +adapter = HostModuleAdapter() + +def make_recorder(operation, capability_id): + def record(instance): + events[capability_id].append((operation, id(instance))) + return record + +for spec in specs: + implementation = adapter.materialize(spec) + implementation.init_module = make_recorder("start", spec.id) + implementation.stop = make_recorder("stop", spec.id) + +config_values = {} +enabled_service_values = {} +selector_keys = set() +configured_ids = set() +for spec in specs: + if spec.activation is not ActivationPolicy.WHEN_CONFIGURED: + continue + configured_ids.add(spec.id) + selector = spec.selector + assert selector is not None + key = str(selector.config["key"]) + selector_keys.add(key) + if selector.kind == "setting_truthy": + setattr(settings, key, f"enabled:{spec.id}") + elif selector.kind == "system_config_item": + enabled_service_values.setdefault(key, []).append({ + "name": f"contract-{spec.id}", + "type": selector.config["match_value"], + "config": {}, + "enabled": True, + }) + else: + raise AssertionError(f"未覆盖的 Host Module selector：{selector.kind}") +config_values.update({key: list(value) for key, value in enabled_service_values.items()}) + +def get_config(_self, key=None): + key_value = getattr(key, "value", key) + if key_value is None: + return dict(config_values) + return config_values.get(key_value) + +SystemConfigOper.get = get_config + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +bootstrap_ids = { + spec.id + for spec in specs + if spec.activation is ActivationPolicy.BOOTSTRAP +} +initial_ids = bootstrap_ids | configured_ids +assert bootstrap_ids +assert configured_ids +assert initial_ids == {spec.id for spec in specs} +initial_instances = { + capability_id: manager.get_running_module(capability_id) + for capability_id in initial_ids +} +assert all(initial_instances.values()) +assert { + capability_id: [operation for operation, _instance_id in events[capability_id]] + for capability_id in initial_ids +} == {capability_id: ["start"] for capability_id in initial_ids} + +watch_keys = {key for spec in specs for key in spec.watch} +watched_ids = { + spec.id + for spec in specs + if spec.id in initial_ids and watch_keys.intersection(spec.watch) +} +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=watch_keys), + ) +) + +for capability_id, initial_instance in initial_instances.items(): + assert manager.get_running_module(capability_id) is initial_instance + operations = [operation for operation, _instance_id in events[capability_id]] + expected = ["start", "stop", "start"] if capability_id in watched_ids else ["start"] + assert operations == expected, (capability_id, operations) + assert { + instance_id for _operation, instance_id in events[capability_id] + } == {id(initial_instance)} + +for spec in specs: + if spec.id not in configured_ids: + continue + selector = spec.selector + key = str(selector.config["key"]) + if selector.kind == "setting_truthy": + setattr(settings, key, False) +for key in enabled_service_values: + config_values[key] = [] + +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=selector_keys), + ) +) +for capability_id in configured_ids: + assert manager.get_running_module(capability_id) is None + assert [operation for operation, _instance_id in events[capability_id]] == [ + "start", + "stop", + "start", + "stop", + ] +for capability_id in bootstrap_ids: + assert manager.get_running_module(capability_id) is initial_instances[capability_id] + +manager.shutdown() +assert all(manager.get_running_module(spec.id) is None for spec in specs) +events_after_shutdown = { + capability_id: list(capability_events) + for capability_id, capability_events in events.items() +} + +for spec in specs: + if spec.id not in configured_ids: + continue + selector = spec.selector + key = str(selector.config["key"]) + if selector.kind == "setting_truthy": + setattr(settings, key, f"re-enabled:{spec.id}") +for key, value in enabled_service_values.items(): + config_values[key] = list(value) + +manager.load_modules() +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=watch_keys), + ) +) +assert all(manager.get_running_module(spec.id) is None for spec in specs) +assert events == events_after_shutdown +assert set(spec_by_id) == set(events) +""" + _run_real_host_module_check(tmp_path, body) + + +def test_default_config_keeps_every_manifest_configured_entrypoint_unimported( + tmp_path: Path, +) -> None: + """默认配置惰性边界由全部 when-configured manifest 自动生成。""" + body = r""" +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.capabilities.model import ActivationPolicy +from app.runtime.config import settings +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 +configured_specs = tuple( + spec for spec in specs + if spec.activation is ActivationPolicy.WHEN_CONFIGURED +) +configured_modules = { + spec.entrypoint.split(":", maxsplit=1)[0] + for spec in configured_specs +} +assert configured_modules +assert configured_modules.isdisjoint(sys.modules) + +for spec in configured_specs: + selector = spec.selector + assert selector is not None + if selector.kind == "setting_truthy": + setattr(settings, str(selector.config["key"]), False) + +SystemConfigOper.get = lambda _self, key=None: {} if key is None else [] + +adapter = HostModuleAdapter() +for spec in specs: + if spec.activation is not ActivationPolicy.BOOTSTRAP: + continue + implementation = adapter.materialize(spec) + implementation.init_module = lambda _self: None + implementation.stop = lambda _self: None + +assert configured_modules.isdisjoint(sys.modules) + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +assert manager.get_specs() == manager.list_specs() +assert {spec.id for spec in manager.list_specs()} == {spec.id for spec in specs} +assert all(manager.get_running_module(spec.id) is None for spec in configured_specs) +assert configured_modules.isdisjoint(sys.modules) +manager.shutdown() +assert configured_modules.isdisjoint(sys.modules) +""" + _run_real_host_module_check(tmp_path, body) + + +def test_event_resolver_uses_exact_class_and_blocks_stopped_owner_fallback( + module_manager_harness, +) -> None: + """同名 class 不能冒充 owner；已停止 owner 必须返回 Binding(None)。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + module_class = manager.get_module("SampleModule") + running = manager.get_running_module("SampleModule") + + active_binding = manager.resolve_event_handler_instance(module_class) + assert active_binding == EventHandlerBinding( + instance=running, + owner_name="Sample", + ) + + impostor = type("SampleModule", (), {}) + impostor.__module__ = module_class.__module__ + assert manager.resolve_event_handler_instance(impostor) is None + + manager.stop() + stopped_binding = manager.resolve_event_handler_instance(module_class) + assert stopped_binding == EventHandlerBinding( + instance=None, + owner_name="Sample", + ) + + +def test_default_modulelist_does_not_import_unconfigured_provider_sdks( + tmp_path: Path, +) -> None: + """默认配置下构造 Manager 和查询模块列表都不能拉起重量 provider SDK。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.config import settings + +def empty_config(self, key=None): + return {} if key is None else [] + +SystemConfigOper.get = empty_config +settings.ACOUSTID_API_KEY = None +settings.FANART_API_KEY = None + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +assert len(manager.list_specs()) == 37 +assert manager.get_specs() == manager.list_specs() + +from app.api.endpoints.system import modulelist +response = modulelist(None) +assert len(response.data["modules"]) == 37 + +heavy_prefixes = ( + "lark_oapi", + "slack_bolt", + "slack_sdk", + "discord", + "plexapi", + "telebot", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in heavy_prefixes) +) +assert loaded == [], loaded +manager.shutdown() +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", "import sys\n" + code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"子进程模块发现失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) + + +def test_lazy_boundary_annotations_are_reflectable_without_provider_sdks( + tmp_path: Path, +) -> None: + """宿主公共注解可被反射，且反射过程不加载可选 provider SDK。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +import sys +from typing import Any, Optional, get_type_hints + +provider_prefixes = ("qbittorrentapi", "transmission_rpc", "pywebpush") + +def loaded_provider_modules(): + return sorted( + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in provider_prefixes + ) + ) + +assert loaded_provider_modules() == [] + +from app.chain import ChainBase +from app.api.endpoints.message import WebPushError, is_webpush_subscription_gone + +assert get_type_hints(ChainBase.torrent_files)["return"] == Optional[Any] +assert get_type_hints(is_webpush_subscription_gone)["error"] is WebPushError +assert loaded_provider_modules() == [] +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"轻量注解反射失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) + + +def test_manifest_metadata_matches_legacy_module_class_contract(tmp_path: Path) -> None: + """manifest 投影必须与插件仍可调用的模块类 metadata 完全一致。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +from app.db.oper.systemconfig import SystemConfigOper + +SystemConfigOper.get = lambda self, key=None: {} if key is None else [] + +from app.runtime.config import settings +settings.ACOUSTID_API_KEY = None +settings.FANART_API_KEY = None + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +modules = manager.get_modules() +assert len(modules) == len(manager.list_specs()) == 37 +for spec in manager.list_specs(): + implementation = modules[spec.id] + assert implementation.get_name() == spec.metadata["name"] + assert implementation.get_type().value == spec.metadata["type"] + assert implementation.get_subtype().name == spec.metadata["subtype"] + assert implementation.get_priority() == spec.metadata["priority"] +manager.shutdown() +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", "import sys\n" + code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"模块 metadata 兼容检查失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) diff --git a/tests/test_navidrome_module.py b/tests/test_navidrome_module.py index 6147e2f71..e6c0ea572 100644 --- a/tests/test_navidrome_module.py +++ b/tests/test_navidrome_module.py @@ -21,9 +21,12 @@ def test_navidrome_module_has_no_system_switch(): assert NavidromeModule().init_setting() is None -def test_navidrome_module_is_loaded_by_module_manager(): - """模块管理器应能加载 Navidrome，否则媒体服务器列表里不会出现该类型。""" - assert "NavidromeModule" in ModuleManager()._running_modules +def test_navidrome_module_is_discovered_without_unconfigured_activation(): + """Navidrome 始终可发现，但没有启用配置时不应创建服务资源。""" + manager = ModuleManager() + + assert "NavidromeModule" in manager.get_module_ids() + assert manager.get_running_module("NavidromeModule") is None def test_navidrome_module_ignores_non_music_media(): diff --git a/tests/test_singleton_thread_safety.py b/tests/test_singleton_thread_safety.py new file mode 100644 index 000000000..1d5ef2074 --- /dev/null +++ b/tests/test_singleton_thread_safety.py @@ -0,0 +1,41 @@ +import threading + +from app.foundation.singleton import Singleton, SingletonClass + + +def _construct_concurrently(singleton_type, count: int = 16): + """让多个线程同时越过起跑线，放大首次构造竞态。""" + barrier = threading.Barrier(count) + instances = [] + + def construct() -> None: + barrier.wait() + instances.append(singleton_type()) + + threads = [threading.Thread(target=construct) for _ in range(count)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + assert not thread.is_alive() + return instances + + +def test_parameterized_singleton_first_construction_is_single_flight(): + """同一参数的并发首次构造只能发布一个完整实例。""" + class _ParameterizedSingleton(metaclass=Singleton): + pass + + instances = _construct_concurrently(_ParameterizedSingleton) + + assert len({id(instance) for instance in instances}) == 1 + + +def test_class_singleton_first_construction_is_single_flight(): + """按类单例的并发首次构造只能发布一个完整实例。""" + class _ClassSingleton(metaclass=SingletonClass): + pass + + instances = _construct_concurrently(_ClassSingleton) + + assert len({id(instance) for instance in instances}) == 1 diff --git a/tests/test_system_i18n.py b/tests/test_system_i18n.py index b3b7db8fc..2f4237356 100644 --- a/tests/test_system_i18n.py +++ b/tests/test_system_i18n.py @@ -1,24 +1,18 @@ from unittest.mock import patch +from types import SimpleNamespace from app.api.endpoints import system as system_endpoint from app.runtime.localization import LocaleHelper -class _FakeDoubanModule: - """构造带中文名称的模块类，模拟真实 DoubanModule。""" - - @staticmethod - def get_name() -> str: - """获取模块中文名称""" - return "豆瓣" - - class _FakeModuleManager: """提供 system 模块接口测试所需的最小模块管理器。""" - def get_modules(self) -> dict: - """返回模块字典""" - return {"DoubanModule": _FakeDoubanModule} + def list_specs(self) -> tuple: + """返回 manifest 元数据视图。""" + return ( + SimpleNamespace(id="DoubanModule", metadata={"name": "豆瓣"}), + ) def test(self, moduleid: str) -> tuple[bool, str]: """返回模块测试结果""" From 7e851dbfa769b0d13c4af1f269e6ac3f18e5f779 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 16 Aug 2026 16:30:16 +0800 Subject: [PATCH 2/8] =?UTF-8?q?refactor(chain):=20=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=93=BE=E5=8A=9F=E8=83=BD=E5=9F=9F=20mixin=20=E5=8C=96?= =?UTF-8?q?=EF=BC=8C=E6=B8=85=E7=90=86=E6=9C=AA=E4=BD=BF=E7=94=A8=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E5=B9=B6=E6=A0=B9=E6=B2=BB=E5=85=BC=E5=AE=B9=E5=B1=82?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=E5=AF=BC=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ChainBase 拆分为 RecognitionMixin/MessageProcessingMixin/NotificationMixin - TransferChain 拆分为 7 个功能 mixin（_mixins.py），SubscribeChain 音乐订阅域拆出 _music.py - 斜杠命令交互四件套收敛为 InteractionChainMixin 委托，会话管理器移至 application 层，chain 层不再 re-export - 模块基础类收敛到 app/modules/_base（notification/mediaserver 语义重命名） - 清理 app/chain/__init__.py 24 个未使用导入，修正 49 处测试 patch 目标到实际命名空间 - 兼容层 legacy 符号不再并入 __all__，根治 schemas 初始化反向拉起 application.transfer 的循环导入 - 修复 bangumi 集数为字符串时 set_bangumi_info 抛 TypeError - 新增重复代码等架构门禁测试；capability 清单校验排除下划线内部目录 --- .gitignore | 4 + app/agent/orchestrator.py | 19 +- app/agent/tools/impl/_plugin_tool_utils.py | 21 +- app/agent/tools/impl/create_agent_task.py | 5 +- app/agent/tools/impl/delete_agent_task.py | 4 +- app/agent/tools/impl/list_slash_commands.py | 6 +- app/agent/tools/impl/query_agent_tasks.py | 5 +- app/agent/tools/impl/query_schedulers.py | 8 +- app/agent/tools/impl/run_agent_task.py | 4 +- app/agent/tools/impl/run_scheduler.py | 12 +- app/agent/tools/impl/run_slash_command.py | 9 +- app/agent/tools/impl/update_agent_task.py | 5 +- app/api/endpoints/plugin.py | 168 +- app/application/agent.py | 100 + app/application/commands.py | 45 + app/application/plugins.py | 190 ++ app/application/scheduling.py | 73 + app/application/transfer.py | 892 +++++- app/chain/__init__.py | 991 +------ app/chain/_interaction.py | 86 + app/chain/_messaging.py | 486 ++++ app/chain/_mixins.py | 1559 ++++++++++ app/chain/_music.py | 420 +++ app/chain/_recognition.py | 518 ++++ app/chain/agent.py | 14 + app/chain/message.py | 26 +- app/chain/search.py | 8 +- app/chain/site.py | 73 +- app/chain/subscribe.py | 459 +-- app/chain/transfer.py | 2509 +---------------- app/domain/context.py | 5 + app/factory.py | 5 + app/modules/_base/__init__.py | 15 + app/modules/_base/downloader.py | 109 + app/modules/_base/mediaserver.py | 192 ++ app/modules/_base/notification.py | 149 + app/modules/discord/__init__.py | 102 +- app/modules/emby/__init__.py | 146 +- app/modules/feishu/__init__.py | 13 +- app/modules/jellyfin/__init__.py | 147 +- app/modules/plex/__init__.py | 36 +- app/modules/qbittorrent/__init__.py | 96 +- app/modules/qqbot/__init__.py | 44 +- app/modules/rtorrent/__init__.py | 96 +- app/modules/slack/__init__.py | 102 +- app/modules/synologychat/__init__.py | 48 +- app/modules/telegram/__init__.py | 106 +- app/modules/transmission/__init__.py | 96 +- app/modules/trimemedia/__init__.py | 176 +- app/modules/ugreen/__init__.py | 159 +- app/modules/vocechat/__init__.py | 48 +- app/modules/wechat/__init__.py | 117 +- app/modules/wechatclawbot/__init__.py | 16 +- app/modules/zspace/__init__.py | 147 +- app/runtime/compat/imports.py | 5 +- app/runtime/compat/manifest.py | 12 + app/scheduler.py | 3 +- app/schemas/agent.py | 8 + app/startup/agent_initializer.py | 14 + app/startup/command_initializer.py | 4 + app/startup/scheduler_initializer.py | 4 + docs/rules/05-architecture.md | 34 + tests/test_agent_image_capability.py | 17 +- tests/test_agent_image_support.py | 19 +- tests/test_agent_interaction.py | 19 +- tests/test_agent_message_routing.py | 25 +- tests/test_agent_scheduled_tasks.py | 28 + tests/test_agent_session_status.py | 17 +- tests/test_api_response.py | 10 +- tests/test_architecture_dependencies.py | 118 +- tests/test_capability_registry.py | 2 + tests/test_chain_layering.py | 7 +- tests/test_discord_command_registration.py | 4 +- tests/test_downloader_path_mapping.py | 36 + tests/test_duplicate_code.py | 143 + tests/test_episode_format_helper.py | 14 +- tests/test_episode_group_recognition.py | 4 +- tests/test_manual_transfer_history.py | 19 +- tests/test_media_recognize_share.py | 66 +- .../test_media_recognize_share_statistics.py | 9 +- tests/test_media_source_routing.py | 4 +- tests/test_message_notifications.py | 43 +- tests/test_music_plugin_recognize.py | 2 +- tests/test_music_subscribe.py | 19 +- tests/test_music_transfer.py | 11 +- tests/test_music_workflows.py | 6 +- tests/test_qbittorrent_compat.py | 38 + tests/test_recognize_source_selection.py | 8 +- tests/test_slack_command_registration.py | 2 +- tests/test_slash_command_interactions.py | 6 +- tests/test_subscribe_chain.py | 13 + tests/test_system_notification_dispatch.py | 2 +- tests/test_telegram_typing_lifecycle.py | 17 +- tests/test_transfer_custom_words.py | 8 +- tests/test_transfer_failed_retry_buttons.py | 25 +- tests/test_transfer_job_manager.py | 26 +- tests/test_transfer_mounted_disk_cleanup.py | 20 +- tests/test_transfer_movie_collection.py | 16 +- tests/test_transfer_overwrite_declined.py | 10 +- tests/test_transfer_stale_tasks.py | 8 +- tests/test_transfer_sync_extra_files.py | 111 +- tests/test_transfer_tmdb_category.py | 4 + 102 files changed, 6041 insertions(+), 5888 deletions(-) create mode 100644 app/application/agent.py create mode 100644 app/application/commands.py create mode 100644 app/application/plugins.py create mode 100644 app/application/scheduling.py create mode 100644 app/chain/_interaction.py create mode 100644 app/chain/_messaging.py create mode 100644 app/chain/_mixins.py create mode 100644 app/chain/_music.py create mode 100644 app/chain/_recognition.py create mode 100644 app/chain/agent.py create mode 100644 app/modules/_base/__init__.py create mode 100644 app/modules/_base/downloader.py create mode 100644 app/modules/_base/mediaserver.py create mode 100644 app/modules/_base/notification.py create mode 100644 tests/test_duplicate_code.py diff --git a/.gitignore b/.gitignore index 5b11aa738..855d45635 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ app/plugins/** config/cookies/ config/app.env config/user.db* +config/systemconfig.db* config/sites/** config/agent/ config/logs/ @@ -26,6 +27,9 @@ config/plugins/ config/temp/ config/cache/ config/.cache/ +# 运行期设置持久化目录（settings 写回 app.env 的落点）与本地验证产物 +app/config/ +.verify_tmp/ .runtime/ public/ .moviepilot.env diff --git a/app/agent/orchestrator.py b/app/agent/orchestrator.py index f4d3b6581..8bf21d07a 100644 --- a/app/agent/orchestrator.py +++ b/app/agent/orchestrator.py @@ -6,7 +6,6 @@ import traceback import uuid from dataclasses import dataclass from datetime import datetime, timedelta -from enum import Enum from typing import Any, Callable, Dict, List, Optional from fastapi.concurrency import run_in_threadpool @@ -68,7 +67,7 @@ from app.agent.tools.impl.mcp import ( select_legacy_mcp_tools, ) from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool -from app.chain import ChainBase +from app.chain.agent import AgentChain from app.runtime.config import settings from app.runtime.events import eventmanager from app.runtime.extensions.plugin_manager import PluginManager @@ -77,17 +76,12 @@ from app.db.oper.agenttask import AgentTaskOper from app.db.oper.user import UserOper from app.runtime.log import logger from app.schemas import AgentLLMProviderEventData, AgentTokensUsageEventData, Notification, NotificationType +from app.schemas.agent import ReplyMode from app.schemas.message import ChannelCapabilityManager, ChannelCapability from app.schemas.types import ChainEventType, EventType, MessageChannel from app.foundation.identity import SYSTEM_INTERNAL_USER_ID -class AgentChain(ChainBase): - """Agent 业务处理链。""" - - pass - - def _finish_processing_status(status: Optional[dict], user_id: Optional[str] = None) -> None: """结束入站消息的渠道处理状态。""" if not status: @@ -321,15 +315,6 @@ class _ThinkTagStripper: self.buffer = "" -class ReplyMode(str, Enum): - """ - Agent 最终回复处理模式。 - """ - - DISPATCH = "dispatch" - CAPTURE_ONLY = "capture_only" - - HEARTBEAT_SESSION_PREFIX = "__agent_heartbeat_" UNSUPPORTED_IMAGE_INPUT_MESSAGE = "当前模型不支持图片输入，请更换支持图片输入的模型，或在系统设置中关闭图片输入支持后重试。" AGENT_EXECUTION_ERROR_PREFIX = "智能助手执行失败" diff --git a/app/agent/tools/impl/_plugin_tool_utils.py b/app/agent/tools/impl/_plugin_tool_utils.py index 5070f8354..049f7cbf9 100644 --- a/app/agent/tools/impl/_plugin_tool_utils.py +++ b/app/agent/tools/impl/_plugin_tool_utils.py @@ -70,14 +70,14 @@ def reload_plugin_runtime(plugin_id: str) -> None: 重载插件并重新注册其命令、定时任务和 API。 """ # 这些依赖只在真正执行重载时才导入，避免普通查询工具引入不必要的初始化开销。 - from app.api.endpoints.plugin import register_plugin_api - from app.command import Command - from app.scheduler import Scheduler + from app.application.plugins import register_plugin_api + from app.application.commands import init_commands + from app.application.scheduling import update_plugin_job plugin_manager = PluginManager() plugin_manager.reload_plugin(plugin_id) - Scheduler().update_plugin_job(plugin_id) - Command().init_commands(plugin_id) + update_plugin_job(plugin_id) + init_commands(plugin_id) register_plugin_api(plugin_id) @@ -333,8 +333,11 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]: """ 按现有卸载逻辑移除插件，并清理运行态注册与分组信息。 """ - from app.api.endpoints.plugin import _remove_plugin_from_folders, remove_plugin_api - from app.scheduler import Scheduler + from app.application.plugins import ( + remove_plugin_api, + remove_plugin_from_folders, + ) + from app.application.scheduling import remove_plugin_job config_oper = SystemConfigOper() install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or [] @@ -343,7 +346,7 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]: await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins) remove_plugin_api(plugin_id) - Scheduler().remove_plugin_job(plugin_id) + remove_plugin_job(plugin_id) plugin_manager = PluginManager() plugin_class = plugin_manager.plugins.get(plugin_id) @@ -362,7 +365,7 @@ async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]: except Exception: clone_files_removed = False - _remove_plugin_from_folders(plugin_id) + remove_plugin_from_folders(plugin_id) plugin_manager.remove_plugin(plugin_id) return { diff --git a/app/agent/tools/impl/create_agent_task.py b/app/agent/tools/impl/create_agent_task.py index d4383cc40..686a336f2 100644 --- a/app/agent/tools/impl/create_agent_task.py +++ b/app/agent/tools/impl/create_agent_task.py @@ -99,7 +99,7 @@ class CreateAgentTaskTool(MoviePilotTool): def _create_task(self, payload: CreateAgentTaskInput) -> dict: """持久化任务并立即注册到运行时调度器。""" - from app.scheduler import Scheduler + from app.application.scheduling import update_agent_task_job trigger_value = payload.trigger if payload.trigger_type == "date" and payload.delay_minutes is not None: @@ -130,8 +130,7 @@ class CreateAgentTaskTool(MoviePilotTool): source=self._source or (chat.source if chat else None), original_chat_id=chat.original_chat_id if chat else None, ) - scheduler = Scheduler() - next_run_at = scheduler.update_agent_task_job(task.id) + next_run_at = update_agent_task_job(task.id) return AgentTaskOper.to_dict( task, next_run_at=next_run_at, diff --git a/app/agent/tools/impl/delete_agent_task.py b/app/agent/tools/impl/delete_agent_task.py index 642f0d758..0f12a54ed 100644 --- a/app/agent/tools/impl/delete_agent_task.py +++ b/app/agent/tools/impl/delete_agent_task.py @@ -31,14 +31,14 @@ class DeleteAgentTaskTool(MoviePilotTool): def _delete_task(self, task_id: int) -> bool: """删除当前用户的任务并移除运行时调度。""" - from app.scheduler import Scheduler + from app.application.scheduling import remove_agent_task_job deleted = AgentTaskOper().delete( task_id=task_id, user_id=str(self._user_id), ) if deleted: - Scheduler().remove_agent_task_job(task_id) + remove_agent_task_job(task_id) return deleted async def run(self, task_id: int, **kwargs: object) -> str: diff --git a/app/agent/tools/impl/list_slash_commands.py b/app/agent/tools/impl/list_slash_commands.py index 4d2b65d47..89984c211 100644 --- a/app/agent/tools/impl/list_slash_commands.py +++ b/app/agent/tools/impl/list_slash_commands.py @@ -14,7 +14,6 @@ class ListSlashCommandsInput(BaseModel): """查询所有可用斜杠命令工具的输入参数模型""" - class ListSlashCommandsTool(MoviePilotTool): name: str = "list_slash_commands" tags: list[str] = [ @@ -41,10 +40,9 @@ class ListSlashCommandsTool(MoviePilotTool): logger.info(f"执行工具: {self.name}") try: - from app.command import Command + from app.application.commands import get_commands - command_obj = Command() - all_commands = command_obj.get_commands() + all_commands = get_commands() if not all_commands: return "当前没有可用的命令" diff --git a/app/agent/tools/impl/query_agent_tasks.py b/app/agent/tools/impl/query_agent_tasks.py index 9559e024a..e43780bb6 100644 --- a/app/agent/tools/impl/query_agent_tasks.py +++ b/app/agent/tools/impl/query_agent_tasks.py @@ -48,7 +48,7 @@ class QueryAgentTasksTool(MoviePilotTool): enabled: Optional[bool], ) -> list[dict]: """读取当前用户的任务及运行时下一次触发时间。""" - from app.scheduler import Scheduler + from app.application.scheduling import get_agent_task_next_run oper = AgentTaskOper() if task_id: @@ -56,12 +56,11 @@ class QueryAgentTasksTool(MoviePilotTool): tasks = [task] if task else [] else: tasks = oper.list(user_id=str(self._user_id), enabled=enabled) - scheduler = Scheduler() result = [] for task in tasks: data = oper.to_dict( task, - next_run_at=scheduler.get_agent_task_next_run(task.id), + next_run_at=get_agent_task_next_run(task.id), timezone=settings.TZ, ) if task_id: diff --git a/app/agent/tools/impl/query_schedulers.py b/app/agent/tools/impl/query_schedulers.py index bb787fed7..21a6eab11 100644 --- a/app/agent/tools/impl/query_schedulers.py +++ b/app/agent/tools/impl/query_schedulers.py @@ -39,13 +39,15 @@ class QuerySchedulersTool(MoviePilotTool): """查询非 Agent 自主任务的运行时定时服务。""" logger.info(f"执行工具: {self.name}") try: - from app.scheduler import AGENT_TASK_JOB_PREFIX, Scheduler + from app.application.scheduling import ( + AGENT_TASK_JOB_PREFIX, + list_scheduler_jobs, + ) - scheduler = Scheduler() agent_task_prefix = f"{AGENT_TASK_JOB_PREFIX}-" schedulers = [ scheduler_item - for scheduler_item in scheduler.list() + for scheduler_item in list_scheduler_jobs() if not str(scheduler_item.id or "").startswith(agent_task_prefix) ] if schedulers: diff --git a/app/agent/tools/impl/run_agent_task.py b/app/agent/tools/impl/run_agent_task.py index 00009064a..10ebc4966 100644 --- a/app/agent/tools/impl/run_agent_task.py +++ b/app/agent/tools/impl/run_agent_task.py @@ -56,7 +56,7 @@ class RunAgentTaskTool(MoviePilotTool): async def run(self, task_id: int, **kwargs: object) -> str: """立即执行当前用户拥有且已启用的 Agent 自主定时任务。""" - from app.scheduler import Scheduler + from app.application.scheduling import start_agent_task payload = RunAgentTaskInput(task_id=task_id) status, task_name = await self.run_blocking( @@ -70,7 +70,7 @@ class RunAgentTaskTool(MoviePilotTool): return f"Agent 定时任务 {task_id} 已暂停，请先恢复后再执行" if status == "running": return f"Agent 定时任务 {task_id} 正在执行，请勿重复触发" - if not Scheduler().start_agent_task(payload.task_id): + if not start_agent_task(payload.task_id): return f"Agent 定时任务 {task_id} 尚未注册到运行时调度器，无法立即执行" return ( f"Agent 定时任务 {task_id} 已提交立即执行：{task_name}。" diff --git a/app/agent/tools/impl/run_scheduler.py b/app/agent/tools/impl/run_scheduler.py index fb40b5435..bec87b5b1 100644 --- a/app/agent/tools/impl/run_scheduler.py +++ b/app/agent/tools/impl/run_scheduler.py @@ -46,12 +46,14 @@ class RunSchedulerTool(MoviePilotTool): @staticmethod def _run_scheduler_sync(job_id: str) -> tuple[bool, str]: """同步触发定时服务，避免调度器扫描阻塞事件循环。""" - from app.scheduler import Scheduler + from app.application.scheduling import ( + list_scheduler_jobs, + start_scheduler_job, + ) - scheduler = Scheduler() - for scheduler_item in scheduler.list(): + for scheduler_item in list_scheduler_jobs(): if scheduler_item.id == job_id: - scheduler.start(job_id) + start_scheduler_job(job_id) return True, scheduler_item.name return False, "" @@ -60,7 +62,7 @@ class RunSchedulerTool(MoviePilotTool): logger.info(f"执行工具: {self.name}, 参数: job_id={job_id}") try: - from app.scheduler import AGENT_TASK_JOB_PREFIX + from app.application.scheduling import AGENT_TASK_JOB_PREFIX if job_id.startswith(f"{AGENT_TASK_JOB_PREFIX}-"): return ( diff --git a/app/agent/tools/impl/run_slash_command.py b/app/agent/tools/impl/run_slash_command.py index 565144bd9..5286cc08e 100644 --- a/app/agent/tools/impl/run_slash_command.py +++ b/app/agent/tools/impl/run_slash_command.py @@ -57,16 +57,15 @@ class RunSlashCommandTool(MoviePilotTool): if not command.startswith("/"): command = f"/{command}" - # 从全局 Command 单例中验证命令是否存在（包含系统预设命令 + 插件命令 + 其他命令） - from app.command import Command + # 从命令注册表中验证命令是否存在（包含系统预设命令 + 插件命令 + 其他命令） + from app.application.commands import get_command, get_commands cmd_name = command.split()[0] - command_obj = Command() - matched_command = command_obj.get(cmd_name) + matched_command = get_command(cmd_name) if not matched_command: # 列出所有可用命令帮助用户 - all_commands = command_obj.get_commands() + all_commands = get_commands() available_cmds = [ f"{cmd} - {info.get('description', '无描述')}" for cmd, info in all_commands.items() diff --git a/app/agent/tools/impl/update_agent_task.py b/app/agent/tools/impl/update_agent_task.py index 9f7afa213..a9c2689bc 100644 --- a/app/agent/tools/impl/update_agent_task.py +++ b/app/agent/tools/impl/update_agent_task.py @@ -100,7 +100,7 @@ class UpdateAgentTaskTool(MoviePilotTool): def _update_task(self, payload: UpdateAgentTaskInput) -> Optional[dict]: """更新当前用户的任务并刷新运行时调度。""" - from app.scheduler import Scheduler + from app.application.scheduling import update_agent_task_job oper = AgentTaskOper() task = oper.get(task_id=payload.task_id, user_id=str(self._user_id)) @@ -174,8 +174,7 @@ class UpdateAgentTaskTool(MoviePilotTool): if current and current.last_status == "running": return {"error": f"Agent 定时任务 {payload.task_id} 正在执行，请稍后再修改"} return None - scheduler = Scheduler() - next_run_at = scheduler.update_agent_task_job(payload.task_id) + next_run_at = update_agent_task_job(payload.task_id) updated_task = oper.get(task_id=payload.task_id, user_id=str(self._user_id)) return oper.to_dict( updated_task, diff --git a/app/api/endpoints/plugin.py b/app/api/endpoints/plugin.py index 4ba7c565a..38784d409 100644 --- a/app/api/endpoints/plugin.py +++ b/app/api/endpoints/plugin.py @@ -12,7 +12,13 @@ from starlette.responses import StreamingResponse from app import schemas from app.api.response import ResponseAPIRouter -from app.command import Command +from app.application.plugins import ( + register_plugin_api, + remove_plugin_api, + remove_plugin_from_folders, +) +from app.application.commands import init_commands +from app.application.scheduling import remove_plugin_job, update_plugin_job from app.runtime.cache import async_fresh from app.runtime.config import settings from app.runtime.events import eventmanager @@ -26,22 +32,12 @@ from app.application.security.access import ( from app.db.models import User from app.db.oper.systemconfig import SystemConfigOper from app.api.deps import get_current_active_superuser, get_current_active_superuser_async -from app.factory import app from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.market import PluginHelper from app.runtime.log import logger -from app.scheduler import Scheduler from app.schemas.event import PluginDataResetEventData from app.schemas.types import ChainEventType, SystemConfigKey -PROTECTED_ROUTES = { - "/api/v1/openapi.json", - "/docs", - "/docs/oauth2-redirect", - "/redoc", -} -PLUGIN_PREFIX = f"{settings.API_V1_STR}/plugin" - router = ResponseAPIRouter() _plugin_release_refresh_tasks: set[asyncio.Task] = set() @@ -106,117 +102,14 @@ def _schedule_plugin_release_refresh(plugin_id: str, repo_url: str) -> None: task.add_done_callback(_discard_task) -def register_plugin_api(plugin_id: Optional[str] = None): - """ - 动态注册插件 API - :param plugin_id: 插件 ID，如果为 None，则注册所有插件 - """ - _update_plugin_api_routes(plugin_id, action="add") - - -def remove_plugin_api(plugin_id: str): - """ - 动态移除单个插件的 API - :param plugin_id: 插件 ID - """ - _update_plugin_api_routes(plugin_id, action="remove") - - -def _update_plugin_api_routes(plugin_id: Optional[str], action: str): - """ - 插件 API 路由注册和移除 - :param plugin_id: 插件 ID，如果 action 为 "add" 且 plugin_id 为 None，则处理所有插件 - 如果 action 为 "remove"，plugin_id 必须是有效的插件 ID - :param action: "add" 或 "remove"，决定是添加还是移除路由 - """ - if action not in {"add", "remove"}: - raise ValueError("Action must be 'add' or 'remove'") - - is_modified = False - existing_paths = {route.path: route for route in app.routes} - - plugin_ids = [plugin_id] if plugin_id else PluginManager().get_running_plugin_ids() - for plugin_id in plugin_ids: - routes_removed = _remove_routes(plugin_id) - if routes_removed: - is_modified = True - - if action != "add": - continue - # 获取插件的 API 路由信息 - plugin_apis = PluginManager().get_plugin_apis(plugin_id) - for api in plugin_apis: - api_path = f"{PLUGIN_PREFIX}{api.get('path', '')}" - try: - api["path"] = api_path - allow_anonymous = api.pop("allow_anonymous", False) - auth_mode = api.pop("auth", "apikey") - dependencies = api.setdefault("dependencies", []) - if not allow_anonymous: - if ( - auth_mode == "bear" - and Depends(verify_token) not in dependencies - ): - dependencies.append(Depends(verify_token)) - elif Depends(verify_apikey) not in dependencies: - dependencies.append(Depends(verify_apikey)) - app.add_api_route(**api, tags=["plugin"]) - is_modified = True - logger.debug(f"Added plugin route: {api_path}") - except Exception as e: - logger.error(f"Error adding plugin route {api_path}: {str(e)}") - - if is_modified: - _clean_protected_routes(existing_paths) - app.openapi_schema = None - app.setup() - - -def _remove_routes(plugin_id: str) -> bool: - """ - 移除与单个插件相关的路由 - :param plugin_id: 插件 ID - :return: 是否有路由被移除 - """ - if not plugin_id: - return False - prefix = f"{PLUGIN_PREFIX}/{plugin_id}/" - routes_to_remove = [ - route for route in app.routes if route.path.startswith(prefix) - ] - removed = False - for route in routes_to_remove: - try: - app.routes.remove(route) - removed = True - logger.debug(f"Removed plugin route: {route.path}") - except Exception as e: - logger.error(f"Error removing plugin route {route.path}: {str(e)}") - return removed - - -def _clean_protected_routes(existing_paths: dict): - """ - 清理受保护的路由，防止在插件操作中被删除或重复添加 - :param existing_paths: 当前应用的路由路径映射 - """ - for protected_route in PROTECTED_ROUTES: - try: - existing_route = existing_paths.get(protected_route) - if existing_route: - app.routes.remove(existing_route) - except Exception as e: - logger.error(f"Error removing protected route {protected_route}: {str(e)}") - - def register_plugin(plugin_id: str): """ 注册一个插件相关的服务 """ # 注册插件服务 - Scheduler().update_plugin_job(plugin_id) + update_plugin_job(plugin_id) # 注册菜单命令 - Command().init_commands(plugin_id) + init_commands(plugin_id) # 注册插件API register_plugin_api(plugin_id) @@ -1045,7 +938,7 @@ def uninstall_plugin( # 移除插件API remove_plugin_api(plugin_id) # 移除插件服务 - Scheduler().remove_plugin_job(plugin_id) + remove_plugin_job(plugin_id) # 判断是否为分身 plugin_manager = PluginManager() plugin_class = plugin_manager.plugins.get(plugin_id) @@ -1062,7 +955,7 @@ def uninstall_plugin( except Exception as e: logger.error(f"删除插件分身目录 {plugin_base_dir} 失败: {str(e)}") # 从插件文件夹中移除该插件 - _remove_plugin_from_folders(plugin_id) + remove_plugin_from_folders(plugin_id) # 移除插件 plugin_manager.remove_plugin(plugin_id) return schemas.Response(success=True) @@ -1121,42 +1014,3 @@ def _add_clone_to_plugin_folder(original_plugin_id: str, clone_plugin_id: str): except Exception as e: logger.error(f"处理插件文件夹时出错：{str(e)}") # 文件夹处理失败不影响插件分身创建的整体流程 - - -def _remove_plugin_from_folders(plugin_id: str): - """ - 从所有文件夹中移除指定的插件 - :param plugin_id: 要移除的插件ID - """ - try: - config_oper = SystemConfigOper() - # 获取插件文件夹配置 - folders = config_oper.get(SystemConfigKey.PluginFolders) or {} - - # 标记是否有修改 - modified = False - - # 遍历所有文件夹，移除指定插件 - for folder_name, folder_data in folders.items(): - if isinstance(folder_data, dict) and "plugins" in folder_data: - # 新格式：{"plugins": [...], "order": ..., "icon": ...} - if plugin_id in folder_data["plugins"]: - folder_data["plugins"].remove(plugin_id) - logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}") - modified = True - elif isinstance(folder_data, list): - # 旧格式：直接是插件列表 - if plugin_id in folder_data: - folder_data.remove(plugin_id) - logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}") - modified = True - - # 如果有修改，保存更新后的文件夹配置 - if modified: - config_oper.set(SystemConfigKey.PluginFolders, folders) - else: - logger.debug(f"插件 {plugin_id} 不在任何文件夹中，无需移除") - - except Exception as e: - logger.error(f"从文件夹中移除插件时出错：{str(e)}") - # 文件夹处理失败不影响插件卸载的整体流程 diff --git a/app/application/agent.py b/app/application/agent.py new file mode 100644 index 000000000..cab4fd58f --- /dev/null +++ b/app/application/agent.py @@ -0,0 +1,100 @@ +"""Agent 编排服务门面。 + +chain 层需要触发 Agent 后台任务、渲染提示词、查询模型能力时统一经本模块调用。 +具体实现由 app.agent 在启动时注册，形成依赖倒置： + + chain -> application.agent <- agent（startup 在导入期注册） + +静态依赖图上 application 不依赖 agent，agent 作为入口层向 application +注册实现，从而拆除 chain <-> agent 的互指环。 + +注意：本模块禁止静态导入 app.agent 下的任何模块（含函数内导入）， +否则会形成 agent -> chain -> application -> agent 的新环。 +未注册时的兜底注册由 startup/agent_initializer 在导入期完成。 +""" + +from typing import Any, Callable, Optional + +# 注册表：启动期由 startup/agent_initializer 填充。 +_agent_manager: Any = None +_prompt_manager: Any = None +_agent_capability_manager: Any = None +_llm_helper: Any = None +_manual_redo_prompt_builder: Optional[Callable[[Any], str]] = None + + +def register_agent_services( + agent_manager: Any, + prompt_manager: Any, + capability_manager: Any, + llm_helper: Any, + manual_redo_prompt_builder: Optional[Callable[[Any], str]] = None, +) -> None: + """注册 Agent 服务实现（由 startup 组合根在导入期调用）。""" + global _agent_manager, _prompt_manager, _agent_capability_manager, _llm_helper + global _manual_redo_prompt_builder + _agent_manager = agent_manager + _prompt_manager = prompt_manager + _agent_capability_manager = capability_manager + _llm_helper = llm_helper + _manual_redo_prompt_builder = manual_redo_prompt_builder + + +def _ensure_registered() -> None: + """校验 Agent 服务已注册。 + + 正常启动路径由 startup/agent_initializer 在导入期注册；未注册时 + 直接抛出带指引的错误，避免在此处静态导入 app.agent 破坏依赖方向。 + """ + if _agent_manager is None: + raise RuntimeError( + "Agent 服务未注册：请先导入 app.startup.agent_initializer 完成组合根装配" + ) + + +def get_agent_manager() -> Any: + """返回 AgentManager 单例。""" + _ensure_registered() + return _agent_manager + + +def get_prompt_manager() -> Any: + """返回提示词管理器。""" + _ensure_registered() + return _prompt_manager + + +def supports_image_input( + provider: Optional[str] = None, + model: Optional[str] = None, + base_url: Optional[str] = None, + base_url_preset: Optional[str] = None, +) -> bool: + """判断当前模型是否启用了图片输入能力。""" + _ensure_registered() + return _llm_helper.supports_image_input( + provider=provider, + model=model, + base_url=base_url, + base_url_preset=base_url_preset, + ) + + +def is_audio_input_available() -> bool: + """判断语音输入能力是否可用。""" + _ensure_registered() + return _agent_capability_manager.is_audio_input_available() + + +def transcribe_audio(content: bytes, filename: str = "input.ogg") -> Optional[str]: + """把音频内容转写为文本。""" + _ensure_registered() + return _agent_capability_manager.transcribe_audio(content, filename=filename) + + +def build_manual_redo_prompt(history: Any) -> str: + """构造整理记录 AI 重新整理提示词（builder 由 agent 层注册）。""" + _ensure_registered() + if _manual_redo_prompt_builder is None: + raise RuntimeError("整理记录重新整理提示词构建器未注册") + return _manual_redo_prompt_builder(history) diff --git a/app/application/commands.py b/app/application/commands.py new file mode 100644 index 000000000..7aebbe229 --- /dev/null +++ b/app/application/commands.py @@ -0,0 +1,45 @@ +"""命令工具服务门面。 + +Agent 工具与 API 端点对命令注册表的操作统一经本模块调用， +Command 实现由 startup 组合根在导入期注册，避免 application 层 +静态依赖顶层 command 模块。 + +依赖方向： + + agent.tools / api.endpoints -> application.commands <- startup（注册 Command 类） +""" + +from typing import Any, Dict, Optional + +# Command 类：由 startup/command_initializer 在导入期注册。 +_command_class: Any = None + + +def register_command_class(command_class: Any) -> None: + """注册 Command 类（组合根在导入期调用）。""" + global _command_class + _command_class = command_class + + +def get_command_object() -> Any: + """返回命令注册表实例。""" + if _command_class is None: + raise RuntimeError( + "命令服务未初始化：请先通过 register_command_class 注册 Command 类" + ) + return _command_class() + + +def get_commands() -> Dict[str, Any]: + """返回全部已注册命令。""" + return get_command_object().get_commands() + + +def get_command(name: str) -> Optional[Any]: + """按命令名查询注册表。""" + return get_command_object().get(name) + + +def init_commands(plugin_id: Optional[str] = None) -> None: + """初始化命令（可指定单个插件）。""" + get_command_object().init_commands(plugin_id) diff --git a/app/application/plugins.py b/app/application/plugins.py new file mode 100644 index 000000000..178118e21 --- /dev/null +++ b/app/application/plugins.py @@ -0,0 +1,190 @@ +"""插件 API 动态路由服务。 + +把插件 API 的动态注册/移除从 HTTP 端点层下沉到 application 层： +FastAPI 实例由组合根（factory 创建应用后）注入，端点与 Agent 工具 +统一经本模块操作路由，消除 api.endpoints 对 factory 的反向依赖。 + +依赖方向： + + api.endpoints.plugin / agent.tools -> application.plugins <- factory（注入实例） +""" + +from typing import Optional + +from fastapi import Depends, FastAPI + +from app.application.security.access import verify_apikey, verify_token +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.config import settings +from app.runtime.extensions.plugin_manager import PluginManager +from app.runtime.log import logger +from app.schemas.types import SystemConfigKey + +PROTECTED_ROUTES = { + "/api/v1/openapi.json", + "/docs", + "/docs/oauth2-redirect", + "/redoc", +} +PLUGIN_PREFIX = f"{settings.API_V1_STR}/plugin" + +# FastAPI 应用实例：由 factory 在创建应用后调用 register_api_app 注入。 +_api_app: Optional[FastAPI] = None + + +def register_api_app(api_app: FastAPI) -> None: + """注入 FastAPI 应用实例（组合根在创建应用后调用）。""" + global _api_app + _api_app = api_app + + +def get_api_app() -> FastAPI: + """返回已注入的 FastAPI 应用实例。""" + if _api_app is None: + raise RuntimeError("插件路由服务未初始化：请先调用 register_api_app 注入应用实例") + return _api_app + + +def register_plugin_api(plugin_id: Optional[str] = None): + """ + 动态注册插件 API + :param plugin_id: 插件 ID，如果为 None，则注册所有插件 + """ + _update_plugin_api_routes(plugin_id, action="add") + + +def remove_plugin_api(plugin_id: str): + """ + 动态移除单个插件的 API + :param plugin_id: 插件 ID + """ + _update_plugin_api_routes(plugin_id, action="remove") + + +def _update_plugin_api_routes(plugin_id: Optional[str], action: str): + """ + 插件 API 路由注册和移除 + :param plugin_id: 插件 ID，如果 action 为 "add" 且 plugin_id 为 None，则处理所有插件 + 如果 action 为 "remove"，plugin_id 必须是有效的插件 ID + :param action: "add" 或 "remove"，决定是添加还是移除路由 + """ + if action not in {"add", "remove"}: + raise ValueError("Action must be 'add' or 'remove'") + + app = get_api_app() + is_modified = False + existing_paths = {route.path: route for route in app.routes} + + plugin_ids = [plugin_id] if plugin_id else PluginManager().get_running_plugin_ids() + for plugin_id in plugin_ids: + routes_removed = _remove_routes(plugin_id) + if routes_removed: + is_modified = True + + if action != "add": + continue + # 获取插件的 API 路由信息 + plugin_apis = PluginManager().get_plugin_apis(plugin_id) + for api in plugin_apis: + api_path = f"{PLUGIN_PREFIX}{api.get('path', '')}" + try: + api["path"] = api_path + allow_anonymous = api.pop("allow_anonymous", False) + auth_mode = api.pop("auth", "apikey") + dependencies = api.setdefault("dependencies", []) + if not allow_anonymous: + if ( + auth_mode == "bear" + and Depends(verify_token) not in dependencies + ): + dependencies.append(Depends(verify_token)) + elif Depends(verify_apikey) not in dependencies: + dependencies.append(Depends(verify_apikey)) + app.add_api_route(**api, tags=["plugin"]) + is_modified = True + logger.debug(f"Added plugin route: {api_path}") + except Exception as e: + logger.error(f"Error adding plugin route {api_path}: {str(e)}") + + if is_modified: + _clean_protected_routes(existing_paths) + app.openapi_schema = None + app.setup() + + +def _remove_routes(plugin_id: str) -> bool: + """ + 移除与单个插件相关的路由 + :param plugin_id: 插件 ID + :return: 是否有路由被移除 + """ + if not plugin_id: + return False + app = get_api_app() + prefix = f"{PLUGIN_PREFIX}/{plugin_id}/" + routes_to_remove = [ + route for route in app.routes if route.path.startswith(prefix) + ] + removed = False + for route in routes_to_remove: + try: + app.routes.remove(route) + removed = True + logger.debug(f"Removed plugin route: {route.path}") + except Exception as e: + logger.error(f"Error removing plugin route {route.path}: {str(e)}") + return removed + + +def _clean_protected_routes(existing_paths: dict): + """ + 清理受保护的路由，防止在插件操作中被删除或重复添加 + :param existing_paths: 当前应用的路由路径映射 + """ + app = get_api_app() + for protected_route in PROTECTED_ROUTES: + try: + existing_route = existing_paths.get(protected_route) + if existing_route: + app.routes.remove(existing_route) + except Exception as e: + logger.error(f"Error removing protected route {protected_route}: {str(e)}") + + +def remove_plugin_from_folders(plugin_id: str): + """ + 从所有文件夹中移除指定的插件 + :param plugin_id: 要移除的插件ID + """ + try: + config_oper = SystemConfigOper() + # 获取插件文件夹配置 + folders = config_oper.get(SystemConfigKey.PluginFolders) or {} + + # 标记是否有修改 + modified = False + + # 遍历所有文件夹，移除指定插件 + for folder_name, folder_data in folders.items(): + if isinstance(folder_data, dict) and "plugins" in folder_data: + # 新格式：{"plugins": [...], "order": ..., "icon": ...} + if plugin_id in folder_data["plugins"]: + folder_data["plugins"].remove(plugin_id) + logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}") + modified = True + elif isinstance(folder_data, list): + # 旧格式：直接是插件列表 + if plugin_id in folder_data: + folder_data.remove(plugin_id) + logger.info(f"已从文件夹 '{folder_name}' 中移除插件 {plugin_id}") + modified = True + + # 如果有修改，保存更新后的文件夹配置 + if modified: + config_oper.set(SystemConfigKey.PluginFolders, folders) + else: + logger.debug(f"插件 {plugin_id} 不在任何文件夹中，无需移除") + + except Exception as e: + logger.error(f"从文件夹中移除插件时出错：{str(e)}") + # 文件夹处理失败不影响插件卸载的整体流程 diff --git a/app/application/scheduling.py b/app/application/scheduling.py new file mode 100644 index 000000000..4d8e2dfec --- /dev/null +++ b/app/application/scheduling.py @@ -0,0 +1,73 @@ +"""调度器工具服务门面。 + +Agent 工具与 API 端点对运行时调度器的操作统一经本模块调用， +Scheduler 实现由 startup 组合根在导入期注册，避免 application 层 +静态依赖顶层 scheduler 模块（scheduler 反向依赖 chain，会成环）。 + +依赖方向： + + agent.tools / api.endpoints -> application.scheduling <- startup（注册 Scheduler 类） +""" + +from typing import Any, List, Optional + +# Agent 自主定时任务在运行时调度器中的任务 ID 前缀。 +AGENT_TASK_JOB_PREFIX = "agent-task" + +# Scheduler 类：由 startup/scheduler_initializer 在导入期注册。 +_scheduler_class: Any = None + + +def register_scheduler_class(scheduler_class: Any) -> None: + """注册 Scheduler 类（组合根在导入期调用）。""" + global _scheduler_class + _scheduler_class = scheduler_class + + +def get_scheduler() -> Any: + """返回调度器实例。""" + if _scheduler_class is None: + raise RuntimeError( + "调度器服务未初始化：请先通过 register_scheduler_class 注册 Scheduler 类" + ) + return _scheduler_class() + + +def list_scheduler_jobs() -> List[Any]: + """列出运行时调度器的全部任务。""" + return get_scheduler().list() + + +def start_scheduler_job(job_id: str) -> None: + """立即运行指定的运行时定时任务。""" + get_scheduler().start(job_id) + + +def update_plugin_job(plugin_id: str) -> None: + """更新插件的定时任务。""" + get_scheduler().update_plugin_job(plugin_id) + + +def remove_plugin_job(plugin_id: str) -> None: + """移除插件的定时任务。""" + get_scheduler().remove_plugin_job(plugin_id) + + +def start_agent_task(task_id: int) -> bool: + """立即执行 Agent 自主定时任务。""" + return get_scheduler().start_agent_task(task_id) + + +def get_agent_task_next_run(task_id: int) -> Optional[Any]: + """查询 Agent 自主定时任务的下一次运行时间。""" + return get_scheduler().get_agent_task_next_run(task_id) + + +def update_agent_task_job(task_id: int) -> Optional[Any]: + """更新 Agent 自主定时任务的注册信息，返回下一次运行时间。""" + return get_scheduler().update_agent_task_job(task_id) + + +def remove_agent_task_job(task_id: int) -> None: + """移除 Agent 自主定时任务的注册信息。""" + get_scheduler().remove_agent_task_job(task_id) diff --git a/app/application/transfer.py b/app/application/transfer.py index d7b2ac888..6bdbdf50f 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -13,20 +13,38 @@ app.schemas -> app.schemas.transfer -> app.domain.* -> app.schemas.types -> app. TransferJob / TransferJobTask，那两个用 app.schemas 的同名 DTO——一个是工作项，一个是 视图，分开表达之后两边都不必再迁就对方。 """ +import asyncio +import threading +from copy import deepcopy from pathlib import Path -from typing import Callable, List, Optional, Union +from time import monotonic +from typing import Callable, Dict, List, Optional, Tuple, Union from pydantic import BaseModel, ConfigDict +from app import schemas +from app.adapters.system.host import SystemUtils +from app.application.agent import get_agent_manager, get_prompt_manager from app.domain.context import MediaInfo, MusicInfo +from app.domain.media import normalize_music_type from app.domain.meta.metabase import MetaBase +from app.domain.meta.metamusic import MetaMusic +from app.foundation import text as text_tools +from app.runtime.log import logger +from app.schemas.agent import ReplyMode from app.schemas.file import FileItem from app.schemas.history import DownloadHistory -from app.schemas.media import OptionalMediaIdentityMixin +from app.schemas.media import OptionalMediaIdentityMixin, resolve_media_identity from app.schemas.system import TransferDirectoryConf from app.schemas.tmdb import TmdbEpisode -from app.schemas.transfer import TransferInfo -from app.schemas.types import MediaSource, MediaType +from app.schemas.transfer import TransferInfo, TransferJob, TransferJobTask +from app.schemas.types import ( + MUSIC_ENTITY_ALBUM, + MUSIC_ENTITY_RECORDING, + MediaSource, + MediaType, +) + class TransferTask(OptionalMediaIdentityMixin, BaseModel): @@ -89,3 +107,869 @@ class TransferQueue(BaseModel): callback: Optional[Callable] = None # 整理结果 result: Optional[TransferInfo] = None + + +# 作业锁：JobManager 与 TransferChain 共享，保护整理作业视图。 +job_lock = threading.Lock() + +class JobManager: + """ + 作业管理器 + task任务负责一个文件的整理，job作业负责一个媒体的整理 + """ + + # 整理中的作业 + _job_view: Dict[Tuple, TransferJob] = {} + # 汇总季集清单 + _season_episodes: Dict[Tuple, List[int]] = {} + # 记录从 meta 作业迁移到 media 作业的关系，用于清理提前失败后残留的 media 作业 + _meta_to_media_ids: Dict[Tuple, set[Tuple]] = {} + # 记录任务最近一次状态心跳，供外部异步接管任务的失活检测使用 + _task_state_changed_at: Dict[Tuple[str, str], float] = {} + # 记录仍由主程序整理线程直接执行的任务，避免把阻塞中的本地任务误判为失活 + _active_executions: set[Tuple[str, str]] = set() + + def __init__(self): + self._job_view = {} + self._season_episodes = {} + self._meta_to_media_ids = {} + self._task_state_changed_at = {} + self._active_executions = set() + + @staticmethod + def __get_meta_id(meta: MetaBase = None, season: Optional[int] = None) -> Tuple: + """ + 获取元数据ID + """ + return meta.name, season + + @staticmethod + def __get_media_id(media: Optional[Union[MediaInfo, MusicInfo]] = None, + season: Optional[int] = None) -> Tuple: + """ + 获取媒体ID；音乐额外区分实体类型，并为无远端ID的曲目构造稳定身份。 + """ + if not media: + return None, season + source, media_id = resolve_media_identity(media=media) + if getattr(media, "type", None) == MediaType.MUSIC: + music_type = normalize_music_type( + getattr(media, "music_type", None), + ) or MUSIC_ENTITY_RECORDING + if source and media_id: + return "music", source, media_id, music_type + + artists = tuple( + text_tools.normalize_upper(artist) + for artist in (getattr(media, "artists", None) or []) + if text_tools.normalize_upper(artist) + ) + if music_type == MUSIC_ENTITY_ALBUM: + album_artist = text_tools.normalize_upper( + getattr(media, "album_artist", None) + or (artists[0] if artists else "") + ) + album = text_tools.normalize_upper( + getattr(media, "album", None) or getattr(media, "title", None) or "" + ) + return "music", "local", music_type, album_artist, album, getattr(media, "year", None) + + return ( + "music", + "local", + music_type, + artists, + text_tools.normalize_upper(getattr(media, "title", None) or ""), + text_tools.normalize_upper(getattr(media, "album", None) or ""), + getattr(media, "disc_number", None), + getattr(media, "track_number", None), + ) + return (source, media_id), season + + @staticmethod + def __get_file_key(fileitem: FileItem) -> Optional[Tuple[str, str]]: + """ + 获取源文件唯一键，用于跨媒体作业识别同一个整理任务。 + """ + if not fileitem or not fileitem.path: + return None + normalized_path = ( + Path(str(fileitem.path).replace("\\", "/")).as_posix().rstrip("/") or "/" + ) + return fileitem.storage or "local", normalized_path + + def __get_id(self, task: TransferTask = None) -> Tuple: + """ + 获取作业ID + """ + if task.mediainfo: + return self.__get_media_id( + media=task.mediainfo, season=task.meta.begin_season + ) + else: + return self.__get_meta_id(meta=task.meta, season=task.meta.begin_season) + + def get_job_id(self, task: TransferTask) -> Tuple: + """返回任务当前所属的稳定作业身份，供作业级附加状态隔离使用。""" + return self.__get_id(task) + + @staticmethod + def __get_media(task: TransferTask) -> Union[schemas.MediaInfo, schemas.MusicInfo]: + """ + 获取媒体信息 + """ + if task.mediainfo: + # 有媒体信息 + mediainfo = deepcopy(task.mediainfo) + mediainfo.clear() + if isinstance(mediainfo, MusicInfo): + return schemas.MusicInfo(**mediainfo.to_dict()) + return schemas.MediaInfo(**mediainfo.to_dict()) + else: + # 没有媒体信息 + meta: MetaBase = task.meta + if isinstance(meta, MetaMusic): + # 未识别的音乐按已解析元数据兜底展示；音乐年份为 int， + # 不能复用 MediaInfo（year 为 str），否则触发 pydantic 校验异常 + return schemas.MusicInfo( + title=meta.name, + artists=list(meta.artists or []), + artist=meta.artist, + album=meta.album, + album_artist=meta.album_artist, + year=meta.year, + title_year=f"{meta.name} ({meta.year})" if meta.year else meta.name, + media_source=meta.media_source, + media_id=meta.media_id, + ) + return schemas.MediaInfo( + title=meta.name, + year=meta.year, + title_year=f"{meta.name} ({meta.year})", + type=meta.type.value if meta.type else None, + ) + + @staticmethod + def __get_meta(task: TransferTask) -> schemas.MetaInfo: + """ + 获取元数据 + """ + if isinstance(task.meta, MetaMusic): + return schemas.MusicMeta(**task.meta.to_dict()) + return schemas.MetaInfo(**task.meta.to_dict()) + + def add_task(self, task: TransferTask, state: Optional[str] = "waiting") -> bool: + """ + 添加整理任务，自动分组到对应的作业中 + :return: True表示任务已添加，False表示任务无效或已存在（重复） + """ + if not all([task, task.meta, task.fileitem]): + return False + file_key = self.__get_file_key(task.fileitem) + if not file_key: + return False + with job_lock: + __mediaid__ = self.__get_id(task) + # 同一个源文件可能在识别前后落入不同作业，必须跨作业去重。 + if any( + self.__get_file_key(t.fileitem) == file_key + for job in self._job_view.values() + for t in job.tasks + ): + logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加") + return False + if __mediaid__ not in self._job_view: + self._job_view[__mediaid__] = TransferJob( + media=self.__get_media(task), + season=task.meta.begin_season, + tasks=[ + TransferJobTask( + fileitem=task.fileitem, + meta=self.__get_meta(task), + downloader=task.downloader, + download_hash=task.download_hash, + state=state, + ) + ], + ) + else: + # 不重复添加任务 + if any( + [ + self.__get_file_key(t.fileitem) == file_key + for t in self._job_view[__mediaid__].tasks + ] + ): + logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加") + return False + self._job_view[__mediaid__].tasks.append( + TransferJobTask( + fileitem=task.fileitem, + meta=self.__get_meta(task), + downloader=task.downloader, + download_hash=task.download_hash, + state=state, + ) + ) + self._task_state_changed_at[file_key] = monotonic() + # 添加季集信息 + if self._season_episodes.get(__mediaid__): + self._season_episodes[__mediaid__].extend(task.meta.episode_list) + self._season_episodes[__mediaid__] = list( + set(self._season_episodes[__mediaid__]) + ) + else: + self._season_episodes[__mediaid__] = task.meta.episode_list + return True + + def migrate_task(self, task: TransferTask) -> bool: + """ + 将任务从 meta 作业迁移到 media 作业 + """ + curr_task, source_job_id = self.__remove_task_with_job_id( + task.fileitem, preserve_execution=True + ) + if not self.add_task(task, state=curr_task.state if curr_task else "waiting"): + return False + if curr_task and task.mediainfo: + metaid = self.__get_meta_id( + meta=task.meta, season=task.meta.begin_season + ) + mediaid = self.__get_id(task) + if source_job_id == metaid and mediaid != metaid: + with job_lock: + self._meta_to_media_ids.setdefault(metaid, set()).add(mediaid) + return True + + def __is_job_done(self, job_id: Tuple) -> bool: + """ + 检查指定作业是否已完成 + """ + if job_id not in self._job_view: + return True + return all( + task.state in ["completed", "failed"] + for task in self._job_view[job_id].tasks + ) + + def __pop_job(self, job_id: Tuple): + """ + 移除指定作业和对应季集缓存 + """ + job = self._job_view.pop(job_id, None) + self._season_episodes.pop(job_id, None) + if not job: + return + for task in job.tasks: + file_key = self.__get_file_key(task.fileitem) + if file_key: + self._task_state_changed_at.pop(file_key, None) + self._active_executions.discard(file_key) + + def __remove_done_job_groups(self, job_ids: set[Tuple]): + """ + 清理已进入终态的独立作业或关联作业组。 + """ + candidates = set(job_ids) + for metaid, mediaids in list(self._meta_to_media_ids.items()): + related_ids = {metaid, *mediaids} + if not related_ids.intersection(candidates): + continue + if all(self.__is_job_done(job_id) for job_id in related_ids): + for job_id in related_ids: + self.__pop_job(job_id) + self._meta_to_media_ids.pop(metaid, None) + candidates.difference_update(related_ids) + + referenced_ids = { + job_id + for metaid, mediaids in self._meta_to_media_ids.items() + for job_id in {metaid, *mediaids} + } + for job_id in candidates - referenced_ids: + if self.__is_job_done(job_id): + self.__pop_job(job_id) + + def start_execution(self, task: TransferTask): + """ + 标记任务仍由主程序整理线程直接执行。 + + :param task: 整理任务 + """ + if not task or not task.fileitem: + return + file_key = self.__get_file_key(task.fileitem) + if not file_key: + return + with job_lock: + self._active_executions.add(file_key) + + def finish_execution(self, task: TransferTask): + """ + 结束主程序整理线程对任务的直接执行标记。 + + :param task: 整理任务 + """ + if not task or not task.fileitem: + return + file_key = self.__get_file_key(task.fileitem) + if not file_key: + return + with job_lock: + self._active_executions.discard(file_key) + + def expire_stale_running_tasks( + self, timeout_seconds: int + ) -> List[Tuple[FileItem, int]]: + """ + 将外部接管后长期无心跳的运行中任务标记失败并清理作业视图。 + + 主程序整理线程仍在直接执行的任务不会被清理，以免把阻塞中的真实任务 + 误报为已终止。外部接管方可重复调用 ``running_task`` 刷新状态心跳。 + + :param timeout_seconds: 失活超时秒数，小于等于 0 时禁用 + :return: 已失活任务及其无心跳秒数 + """ + if timeout_seconds <= 0: + return [] + + current_time = monotonic() + expired: List[Tuple[FileItem, int]] = [] + affected_job_ids: set[Tuple] = set() + with job_lock: + for mediaid, job in self._job_view.items(): + for task in job.tasks: + file_key = self.__get_file_key(task.fileitem) + if ( + not file_key + or task.state != "running" + or file_key in self._active_executions + ): + continue + updated_at = self._task_state_changed_at.get(file_key, current_time) + inactive_seconds = current_time - updated_at + if inactive_seconds < timeout_seconds: + continue + task.state = "failed" + self._task_state_changed_at[file_key] = current_time + episodes = getattr(task.meta, "episode_list", None) or [] + if mediaid in self._season_episodes: + self._season_episodes[mediaid] = list( + set(self._season_episodes[mediaid]) - set(episodes) + ) + expired.append((task.fileitem, int(inactive_seconds))) + affected_job_ids.add(mediaid) + + self.__remove_done_job_groups(affected_job_ids) + return expired + + def running_task(self, task: TransferTask): + """ + 设置任务为运行中，并刷新外部异步任务的状态心跳。 + """ + with job_lock: + __mediaid__ = self.__get_id(task) + if __mediaid__ not in self._job_view: + return + # 更新状态 + for t in self._job_view[__mediaid__].tasks: + if t.fileitem == task.fileitem: + t.state = "running" + file_key = self.__get_file_key(t.fileitem) + if file_key: + self._task_state_changed_at[file_key] = monotonic() + break + + def finish_task(self, task: TransferTask): + """ + 设置任务为完成/成功 + """ + with job_lock: + __mediaid__ = self.__get_id(task) + if __mediaid__ not in self._job_view: + return + # 更新状态 + for t in self._job_view[__mediaid__].tasks: + if t.fileitem == task.fileitem: + t.state = "completed" + file_key = self.__get_file_key(t.fileitem) + if file_key: + self._task_state_changed_at[file_key] = monotonic() + break + + def fail_task(self, task: TransferTask): + """ + 设置任务为失败 + """ + with job_lock: + __mediaid__ = self.__get_id(task) + if __mediaid__ not in self._job_view: + return + # 更新状态 + for t in self._job_view[__mediaid__].tasks: + if t.fileitem == task.fileitem: + t.state = "failed" + file_key = self.__get_file_key(t.fileitem) + if file_key: + self._task_state_changed_at[file_key] = monotonic() + break + # 移除剧集信息 + if __mediaid__ in self._season_episodes: + self._season_episodes[__mediaid__] = list( + set(self._season_episodes[__mediaid__]) + - set(task.meta.episode_list) + ) + + def fail_unfinished_task(self, task: TransferTask): + """ + 将指定任务视图中的非终态任务标记为失败 + """ + if not task or not task.fileitem: + return + file_key = self.__get_file_key(task.fileitem) + if not file_key: + return + with job_lock: + for mediaid, job in self._job_view.items(): + for job_task in job.tasks: + if self.__get_file_key(job_task.fileitem) != file_key: + continue + if job_task.state not in ["completed", "failed"]: + job_task.state = "failed" + self._task_state_changed_at[file_key] = monotonic() + if mediaid in self._season_episodes: + self._season_episodes[mediaid] = list( + set(self._season_episodes[mediaid]) + - set(task.meta.episode_list) + ) + return + + def remove_task(self, fileitem: FileItem) -> Optional[TransferJobTask]: + """ + 根据文件项移除任务 + """ + task, _ = self.__remove_task_with_job_id(fileitem) + return task + + def __remove_task_with_job_id( + self, + fileitem: FileItem, + preserve_execution: bool = False, + ) -> Tuple[Optional[TransferJobTask], Optional[Tuple]]: + """ + 根据文件项移除任务，并返回任务所在的作业ID + """ + file_key = self.__get_file_key(fileitem) + if not file_key: + return None, None + with job_lock: + for mediaid in list(self._job_view): + job = self._job_view[mediaid] + for task in job.tasks: + if self.__get_file_key(task.fileitem) == file_key: + job.tasks.remove(task) + self._task_state_changed_at.pop(file_key, None) + if not preserve_execution: + self._active_executions.discard(file_key) + # 如果没有作业了，则移除作业 + if not job.tasks: + self._job_view.pop(mediaid) + # 移除季集信息 + if mediaid in self._season_episodes: + episodes = getattr(task.meta, "episode_list", None) or [] + self._season_episodes[mediaid] = list( + set(self._season_episodes[mediaid]) + - set(episodes) + ) + return task, mediaid + return None, None + + def remove_job(self, task: TransferTask) -> Optional[TransferJob]: + """ + 移除任务对应的作业（强制，线程不安全） + """ + with job_lock: + __mediaid__ = self.__get_id(task) + if __mediaid__ in self._job_view: + job = self._job_view[__mediaid__] + self.__pop_job(__mediaid__) + return job + return None + + def try_remove_job(self, task: TransferTask): + """ + 尝试移除任务对应的作业（严格检查未完成作业，线程安全） + """ + with job_lock: + __metaid__ = self.__get_meta_id( + meta=task.meta, season=task.meta.begin_season + ) + __mediaid__ = self.__get_media_id( + media=task.mediainfo, season=task.meta.begin_season + ) + + related_media_ids = set(self._meta_to_media_ids.get(__metaid__, set())) + if task.mediainfo: + related_media_ids.add(__mediaid__) + + meta_done = self.__is_job_done(__metaid__) + media_done = all( + self.__is_job_done(mediaid) for mediaid in related_media_ids + ) + + if meta_done and media_done: + remove_ids = {__metaid__, self.__get_id(task), *related_media_ids} + for job_id in remove_ids: + self.__pop_job(job_id) + self._meta_to_media_ids.pop(__metaid__, None) + + def is_done(self, task: TransferTask) -> bool: + """ + 检查任务对应的作业是否整理完成（不管成功还是失败） + """ + with job_lock: + __metaid__ = self.__get_meta_id( + meta=task.meta, season=task.meta.begin_season + ) + __mediaid__ = self.__get_media_id( + media=task.mediainfo, season=task.meta.begin_season + ) + if __metaid__ in self._job_view: + meta_done = all( + task.state in ["completed", "failed"] + for task in self._job_view[__metaid__].tasks + ) + else: + meta_done = True + if __mediaid__ in self._job_view: + media_done = all( + task.state in ["completed", "failed"] + for task in self._job_view[__mediaid__].tasks + ) + else: + media_done = True + return meta_done and media_done + + def is_finished(self, task: TransferTask) -> bool: + """ + 检查任务对应的作业是否已完成且有成功的记录 + """ + with job_lock: + __metaid__ = self.__get_meta_id( + meta=task.meta, season=task.meta.begin_season + ) + __mediaid__ = self.__get_media_id( + media=task.mediainfo, season=task.meta.begin_season + ) + if __metaid__ in self._job_view: + meta_finished = all( + task.state in ["completed", "failed"] + for task in self._job_view[__metaid__].tasks + ) + else: + meta_finished = True + if __mediaid__ in self._job_view: + tasks = self._job_view[__mediaid__].tasks + media_finished = all( + task.state in ["completed", "failed"] for task in tasks + ) and any(task.state == "completed" for task in tasks) + else: + media_finished = True + return meta_finished and media_finished + + def is_success(self, task: TransferTask) -> bool: + """ + 检查任务对应的作业是否全部成功 + """ + with job_lock: + __metaid__ = self.__get_meta_id( + meta=task.meta, season=task.meta.begin_season + ) + __mediaid__ = self.__get_media_id( + media=task.mediainfo, season=task.meta.begin_season + ) + if __metaid__ in self._job_view: + meta_success = all( + task.state in ["completed"] + for task in self._job_view[__metaid__].tasks + ) + else: + meta_success = True + if __mediaid__ in self._job_view: + media_success = all( + task.state in ["completed"] + for task in self._job_view[__mediaid__].tasks + ) + else: + media_success = True + return meta_success and media_success + + def get_all_torrent_hashes(self) -> set[str]: + """ + 获取所有种子的哈希值集合 + """ + with job_lock: + return { + task.download_hash + for job in self._job_view.values() + for task in job.tasks + } + + def is_torrent_done(self, download_hash: str) -> bool: + """ + 检查指定种子的所有任务是否都已完成 + """ + with job_lock: + if any( + task.state not in {"completed", "failed"} + for job in self._job_view.values() + for task in job.tasks + if task.download_hash == download_hash + ): + return False + return True + + def is_torrent_success(self, download_hash: str) -> bool: + """ + 检查指定种子的所有任务是否都已成功 + """ + with job_lock: + if any( + task.state != "completed" + for job in self._job_view.values() + for task in job.tasks + if task.download_hash == download_hash + ): + return False + return True + + def has_tasks( + self, + meta: MetaBase, + mediainfo: Optional[MediaInfo] = None, + season: Optional[int] = None, + ) -> bool: + """ + 判断作业是否还有任务正在处理 + """ + with job_lock: + if mediainfo: + __mediaid__ = self.__get_media_id(media=mediainfo, season=season) + if __mediaid__ in self._job_view: + return True + + __metaid__ = self.__get_meta_id(meta=meta, season=season) + return ( + __metaid__ in self._job_view + and len(self._job_view[__metaid__].tasks) > 0 + ) + + def success_tasks( + self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None + ) -> List[TransferJobTask]: + """ + 获取作业中所有成功的任务 + """ + with job_lock: + __mediaid__ = self.__get_media_id(media=media, season=season) + if __mediaid__ not in self._job_view: + return [] + return [ + task + for task in self._job_view[__mediaid__].tasks + if task.state == "completed" + ] + + def all_tasks( + self, media: MediaInfo, season: Optional[int] = None + ) -> List[TransferJobTask]: + """ + 获取作业中全部任务 + """ + with job_lock: + __mediaid__ = self.__get_media_id(media=media, season=season) + if __mediaid__ not in self._job_view: + return [] + return self._job_view[__mediaid__].tasks + + def count(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int: + """ + 获取作业中成功总数 + """ + with job_lock: + __mediaid__ = self.__get_media_id(media=media, season=season) + if __mediaid__ not in self._job_view: + return 0 + return len( + [ + task + for task in self._job_view[__mediaid__].tasks + if task.state == "completed" + ] + ) + + def size(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int: + """ + 获取作业中所有成功文件总大小 + """ + with job_lock: + __mediaid__ = self.__get_media_id(media=media, season=season) + if __mediaid__ not in self._job_view: + return 0 + return sum( + [ + task.fileitem.size + if task.fileitem.size is not None + else ( + SystemUtils.get_directory_size(Path(task.fileitem.path)) + if task.fileitem.storage == "local" + else 0 + ) + for task in self._job_view[__mediaid__].tasks + if task.state == "completed" + ] + ) + + def total(self) -> int: + """ + 获取所有任务总数 + """ + with job_lock: + return sum([len(job.tasks) for job in self._job_view.values()]) + + def pending_total(self) -> int: + """ + 获取未到终态的任务总数。 + + 作业要等关联任务全部终态才整体移除,追更/分批场景下已完成任务会 + 跨批次残留在视图中;批次统计若用全量 total() 会把历史任务计入 + 「当前共 N 个文件」并压低进度百分比,因此只数未终态任务。 + """ + with job_lock: + return sum( + 1 + for job in self._job_view.values() + for task in job.tasks + if task.state not in ("completed", "failed") + ) + + def list_jobs(self) -> List[TransferJob]: + """ + 获取所有作业的任务列表 + """ + with job_lock: + return list(self._job_view.values()) + + def season_episodes( + self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None + ) -> List[int]: + """ + 获取作业的季集清单 + """ + with job_lock: + __mediaid__ = self.__get_media_id(media=media, season=season) + return self._season_episodes.get(__mediaid__) or [] + + +class FailedRetryScheduler: + """ + 负责失败整理记录的 debounce 聚合与 AI 重试调度。 + """ + + RETRY_TRANSFER_DEBOUNCE_SECONDS = 300 + + def __init__(self): + super().__init__() + self._retry_transfer_buffer: dict[str, list[int]] = {} + self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {} + self._retry_transfer_lock = asyncio.Lock() + + async def close(self): + async with self._retry_transfer_lock: + timers = list(self._retry_transfer_timers.values()) + self._retry_transfer_timers.clear() + self._retry_transfer_buffer.clear() + + for timer in timers: + timer.cancel() + + @staticmethod + def _build_retry_transfer_template_context( + history_ids: list[int], + ) -> tuple[str, dict[str, int | str]]: + """仅负责把失败重试任务的动态数据映射成模板变量。""" + is_batch = len(history_ids) > 1 + task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry" + template_context: dict[str, int | str] = { + "history_ids_csv": ", ".join(str(item) for item in history_ids), + "history_count": len(history_ids), + } + if not is_batch: + template_context["history_id"] = history_ids[0] + return task_type, template_context + + def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str: + """根据失败记录数量构建统一的重试整理后台任务提示词。""" + task_type, template_context = self._build_retry_transfer_template_context(history_ids) + return get_prompt_manager().render_system_task_message( + task_type, + template_context=template_context, + ) + + async def schedule_retry(self, history_id: int, group_key: str = ""): + """ + 同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。 + """ + if not group_key: + group_key = f"_default_{history_id}" + + async with self._retry_transfer_lock: + if group_key not in self._retry_transfer_buffer: + self._retry_transfer_buffer[group_key] = [] + if history_id not in self._retry_transfer_buffer[group_key]: + self._retry_transfer_buffer[group_key].append(history_id) + logger.info( + f"智能体重试整理：记录 ID={history_id} 已加入缓冲区 " + f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)" + ) + + if group_key in self._retry_transfer_timers: + self._retry_transfer_timers[group_key].cancel() + + loop = asyncio.get_running_loop() + self._retry_transfer_timers[group_key] = loop.call_later( + self.RETRY_TRANSFER_DEBOUNCE_SECONDS, + lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)), + ) + + async def _flush_retry_transfer(self, group_key: str): + """ + 延迟定时器到期后，取出该分组的所有 history_id 并合并为一次 agent 调用。 + """ + async with self._retry_transfer_lock: + history_ids = self._retry_transfer_buffer.pop(group_key, []) + self._retry_transfer_timers.pop(group_key, None) + + if not history_ids: + return + + ids_str = ", ".join(str(item) for item in history_ids) + logger.info( + f"智能体重试整理：开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})" + ) + + try: + await get_agent_manager().run_background_prompt( + message=self._build_retry_transfer_prompt(history_ids), + session_prefix="__agent_retry_transfer_batch", + reply_mode=ReplyMode.DISPATCH, + ) + logger.info( + f"智能体重试整理：批量处理完成 IDs=[{ids_str}] (group={group_key})" + ) + except Exception as err: + logger.error( + f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}" + ) + + diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 9d5e95234..b6f9b4749 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -5,27 +5,21 @@ import inspect import pickle import traceback from abc import ABCMeta -from collections.abc import Callable, Sequence +from collections.abc import Callable from datetime import datetime from pathlib import Path from typing import Optional, Any, Tuple, List, Set, Union, Dict from fastapi.concurrency import run_in_threadpool -from app.runtime.cache import FileCache, AsyncFileCache, fresh, async_fresh -from app.runtime.config import settings -from app.domain.context import Context, MediaInfo, MusicInfo, SubtitleInfo, TorrentInfo -from app.runtime.events import Event, EventManager +from app.runtime.cache import FileCache, AsyncFileCache +from app.domain.context import Context, MediaInfo, SubtitleInfo, TorrentInfo +from app.runtime.events import EventManager from app.domain.meta.metabase import MetaBase -from app.domain.meta.metamusic import MetaMusic from app.runtime.extensions.module_manager import ModuleManager from app.runtime.extensions.plugin_manager import PluginManager from app.db.oper.message import MessageOper -from app.db.oper.systemconfig import SystemConfigOper -from app.db.oper.user import UserOper -from app.application.messaging.message import MessageHelper, MessageQueueManager, MessageTemplateHelper -from app.adapters.external.server import MoviePilotServerHelper -from app.runtime.extensions.service_registry import ServiceConfigHelper +from app.application.messaging.message import MessageHelper, MessageQueueManager from app.runtime.log import logger from app.schemas import ( RateLimitExceededException, @@ -33,17 +27,12 @@ from app.schemas import ( ExistMediaInfo, DownloaderTorrent, CommingMessage, - Notification, WebhookEventInfo, TmdbEpisode, MediaPerson, FileItem, TransferDirectoryConf, - MessageResponse, ) -from app.foundation.identity import normalize_internal_user_id -from app.schemas.media import normalize_media_source, resolve_media_identity -from app.schemas.message import ChannelCapability, ChannelCapabilityManager from app.schemas.category import CategoryConfig from app.schemas.types import ( TorrentStatus, @@ -51,15 +40,14 @@ from app.schemas.types import ( MediaSourceSelection, MediaImageType, EventType, - ChainEventType, - MessageChannel, - MediaSource, - SystemConfigKey, ) from app.foundation.reflection import ObjectUtils +from app.chain._messaging import MessageProcessingMixin, NotificationMixin +from app.chain._recognition import RecognitionMixin -class ChainBase(metaclass=ABCMeta): +class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, + metaclass=ABCMeta): """ 处理链基类 """ @@ -129,96 +117,6 @@ class ChainBase(metaclass=ABCMeta): """ self.filecache.delete(filename) - def start_message_processing_status( - self, - channel: MessageChannel, - source: Optional[str], - userid: Optional[Union[str, int]] = None, - message_id: Optional[Union[str, int]] = None, - chat_id: Optional[Union[str, int]] = None, - text: Optional[str] = None, - ) -> Optional[dict]: - """ - 启动渠道侧消息输入/处理状态。 - 具体表现由消息模块实现，例如 typing 保活或消息 reaction。 - """ - if not channel or not ChannelCapabilityManager.supports_capability( - channel, ChannelCapability.PROCESSING_STATUS - ): - return None - try: - status = self.run_module( - "mark_message_processing_started", - channel=channel, - source=source, - userid=userid, - message_id=message_id, - chat_id=chat_id, - text=text, - ) - except Exception as err: - logger.debug(f"启动消息处理状态失败: {err}") - return None - return status if isinstance(status, dict) else None - - def finish_message_processing_status( - self, - status: Optional[dict] = None, - channel: Optional[MessageChannel] = None, - source: Optional[str] = None, - userid: Optional[Union[str, int]] = None, - message_id: Optional[Union[str, int]] = None, - chat_id: Optional[Union[str, int]] = None, - ) -> None: - """ - 结束渠道侧消息输入/处理状态。 - 优先使用 start 返回的 status，缺失时使用显式渠道和消息定位参数。 - """ - target_channel = channel - if status: - try: - target_channel = MessageChannel(status.get("channel")) - except Exception: - target_channel = channel - if not target_channel or not ChannelCapabilityManager.supports_capability( - target_channel, ChannelCapability.PROCESSING_STATUS - ): - return - try: - self.run_module( - "mark_message_processing_finished", - channel=target_channel, - source=(status or {}).get("source") or source, - userid=(status or {}).get("userid") or userid, - message_id=(status or {}).get("message_id") or message_id, - chat_id=(status or {}).get("chat_id") or chat_id, - status=status, - ) - except Exception as err: - logger.debug(f"结束消息处理状态失败: {err}") - - @staticmethod - def _normalize_notification_for_dispatch( - message: Notification - ) -> Notification: - """ - 规范化待发送的通知消息。 - 后台任务会复用内部占位用户ID作为会话身份，这里在真正发送前清空， - 让消息重新走默认通知路由或基于 targets 的目标解析。 - """ - dispatch_message = copy.deepcopy(message) - dispatch_message.userid = normalize_internal_user_id( - dispatch_message.userid - ) - return dispatch_message - - @staticmethod - def _build_notice_message_data(message: Notification) -> dict: - """ - 构造消息通知事件数据。 - """ - return {**message.model_dump(exclude={"save_history"}), "type": message.mtype} - async def async_remove_cache(self, filename: str) -> None: """ 异步删除缓存，同时删除Redis和本地缓存 @@ -520,498 +418,6 @@ class ChainBase(metaclass=ABCMeta): method, result, *args, **kwargs ) - @staticmethod - def _can_use_media_recognize_share( - meta: Optional[MetaBase], - media_source: Optional[MediaSource], - media_id: Optional[str], - ) -> bool: - """ - 仅在名称识别场景下使用共享识别，显式ID识别不再重复回查 - """ - return bool( - settings.MEDIA_RECOGNIZE_SHARE - and meta - and not media_source - and not media_id - ) - - @staticmethod - def _snapshot_recognize_cache_meta(meta: Optional[MetaBase]) -> Optional[MetaBase]: - """ - 保存共享识别前的本地缓存关键元数据，用于共享成功后回填正缓存覆盖负缓存。 - """ - if not meta: - return None - return copy.deepcopy(meta) - - def _update_local_recognize_cache( - self, - meta: Optional[MetaBase], - mediainfo: Optional[MediaInfo], - ) -> None: - """ - 共享识别成功后回填本地识别缓存，避免名称负缓存导致后续重复回查共享。 - """ - if not meta or not mediainfo: - return - self.run_module( - "update_recognize_cache", - meta=meta, - mediainfo=mediainfo, - ) - - async def _async_update_local_recognize_cache( - self, - meta: Optional[MetaBase], - mediainfo: Optional[MediaInfo], - ) -> None: - """ - 异步回填本地识别缓存。 - """ - if not meta or not mediainfo: - return - await self.async_run_module( - "async_update_recognize_cache", - meta=meta, - mediainfo=mediainfo, - ) - - @staticmethod - def _record_media_recognize_share_hit() -> None: - """记录一次共享媒体识别成功命中，统计失败不影响识别结果。""" - try: - SystemConfigOper().increment(SystemConfigKey.MediaRecognizeShareCount) - except Exception as err: - logger.error(f"记录共享媒体识别命中次数失败：{str(err)}") - - def _run_native_media_recognize( - self, - module_kwargs: dict, - cache: bool, - ) -> Optional[MediaInfo]: - """执行同步原生媒体模块识别，具体媒体领域可覆写该路由钩子。""" - with fresh(not cache): - return self.run_module("recognize_media", **module_kwargs) - - async def _async_run_native_media_recognize( - self, - module_kwargs: dict, - cache: bool, - ) -> Optional[MediaInfo]: - """执行异步原生媒体模块识别，具体媒体领域可覆写该路由钩子。""" - async with async_fresh(not cache): - return await self.async_run_module( - "async_recognize_media", **module_kwargs - ) - - def recognize_media( - self, - meta: MetaBase = None, - mtype: Optional[MediaType] = None, - media_source: Optional[MediaSource] = None, - media_id: Optional[str] = None, - episode_group: Optional[str] = None, - cache: bool = True, - share_meta: MetaBase = None, - music_type: Optional[str] = None, - ) -> Optional[MediaInfo]: - """ - 识别媒体信息，不含Fanart图片 - :param meta: 识别的元数据 - :param share_meta: 共享识别查询/上报使用的原始元数据 - :param mtype: 识别的媒体类型 - :param media_source: 请求级识别数据源 - :param media_id: 数据源原生ID，必须与media_source成对提供 - :param episode_group: 剧集组 - :param cache: 是否使用缓存 - :param music_type: 音乐实体类型，显式音乐 ID 必须据此区分单曲与专辑 - :return: 识别的媒体信息，包括剧集信息 - """ - # 仅传数据源是请求级识别源约束（按名称识别限定数据源），显式 media_id 才要求来源成对 - explicit_identity = media_id is not None - requested_source = normalize_media_source(media_source) or media_source - media_source, media_id = resolve_media_identity( - media=meta, - media_source=media_source, - media_id=media_id, - ) - if explicit_identity and (not media_source or not media_id): - logger.warning("媒体识别需要同时提供有效的 media_source 和 media_id") - return None - if not media_id and requested_source is not None: - media_source = requested_source - # meta 自带同源身份（如 {tmdbid=} 标题）时直接按身份识别，避免退化为名称搜索 - meta_source, meta_id = resolve_media_identity(media=meta) - if meta_id and meta_source == requested_source: - media_source, media_id = meta_source, meta_id - if not episode_group and hasattr(meta, "episode_group"): - episode_group = meta.episode_group - if not mtype and not (media_source and media_id) and meta and meta.type in [ - MediaType.TV, MediaType.MOVIE, MediaType.MUSIC - ]: - mtype = meta.type - share_query_meta = share_meta or meta - module_kwargs = { - "meta": meta, - "mtype": mtype, - "media_source": media_source, - "media_id": media_id, - "episode_group": episode_group, - "cache": cache, - } - if music_type is not None: - module_kwargs["music_type"] = music_type - mediainfo = self._run_native_media_recognize(module_kwargs, cache) - # 原生识别未取得远端身份时，允许插件按已知要素补充匹配媒体信息（影视与音乐统一） - mediainfo = self._supplement_media_recognize( - meta=meta, mtype=mtype, media_source=media_source, - media_id=media_id, mediainfo=mediainfo, - music_type=music_type, - ) - fallback_mediainfo = ( - mediainfo - if mediainfo and not self._media_info_has_identity(mediainfo) - else None - ) - if mediainfo and self._media_info_has_identity(mediainfo): - # 电影、电视剧、音乐统一上报；音乐的 tmdb 等字段恒为 None，身份取数据源原生 ID - if not getattr(mediainfo, "recognize_cache_hit", False): - MoviePilotServerHelper.report_recognize_share( - meta=meta, - mediainfo=mediainfo, - keyword_meta=share_query_meta, - ) - return mediainfo - - if self._can_use_media_recognize_share( - share_query_meta, media_source, media_id - ): - shared_cache_meta = self._snapshot_recognize_cache_meta(meta) - share_query_kwargs = { - "meta": meta, - "mtype": mtype, - "keyword_meta": share_query_meta, - } - if music_type is not None: - share_query_kwargs["music_type"] = music_type - shared_item = MoviePilotServerHelper.query_recognize_share( - **share_query_kwargs, - ) - shared_params = MoviePilotServerHelper.to_recognize_params(shared_item) - if shared_params: - shared_module_kwargs = { - "meta": meta, - "mtype": shared_params.get("mtype") or mtype, - "media_source": shared_params.get("media_source"), - "media_id": shared_params.get("media_id"), - "episode_group": episode_group, - "cache": cache, - } - shared_music_type = shared_params.get("music_type") or music_type - if shared_music_type is not None: - shared_module_kwargs["music_type"] = shared_music_type - mediainfo = self._run_native_media_recognize( - shared_module_kwargs, - cache, - ) - if mediainfo and self._media_info_has_identity(mediainfo): - self._update_local_recognize_cache(shared_cache_meta, mediainfo) - self._record_media_recognize_share_hit() - return mediainfo - if mediainfo and not fallback_mediainfo: - fallback_mediainfo = mediainfo - return fallback_mediainfo - - async def async_recognize_media( - self, - meta: MetaBase = None, - mtype: Optional[MediaType] = None, - media_source: Optional[MediaSource] = None, - media_id: Optional[str] = None, - episode_group: Optional[str] = None, - cache: bool = True, - share_meta: MetaBase = None, - music_type: Optional[str] = None, - ) -> Optional[MediaInfo]: - """ - 识别媒体信息，不含Fanart图片（异步版本） - :param meta: 识别的元数据 - :param share_meta: 共享识别查询/上报使用的原始元数据 - :param mtype: 识别的媒体类型 - :param media_source: 请求级识别数据源 - :param media_id: 数据源原生ID，必须与media_source成对提供 - :param episode_group: 剧集组 - :param cache: 是否使用缓存 - :param music_type: 音乐实体类型，显式音乐 ID 必须据此区分单曲与专辑 - :return: 识别的媒体信息，包括剧集信息 - """ - # 仅传数据源是请求级识别源约束（按名称识别限定数据源），显式 media_id 才要求来源成对 - explicit_identity = media_id is not None - requested_source = normalize_media_source(media_source) or media_source - media_source, media_id = resolve_media_identity( - media=meta, - media_source=media_source, - media_id=media_id, - ) - if explicit_identity and (not media_source or not media_id): - logger.warning("媒体识别需要同时提供有效的 media_source 和 media_id") - return None - if not media_id and requested_source is not None: - media_source = requested_source - # meta 自带同源身份（如 {tmdbid=} 标题）时直接按身份识别，避免退化为名称搜索 - meta_source, meta_id = resolve_media_identity(media=meta) - if meta_id and meta_source == requested_source: - media_source, media_id = meta_source, meta_id - if not episode_group and hasattr(meta, "episode_group"): - episode_group = meta.episode_group - if not mtype and not (media_source and media_id) and meta and meta.type in [ - MediaType.TV, MediaType.MOVIE, MediaType.MUSIC - ]: - mtype = meta.type - share_query_meta = share_meta or meta - module_kwargs = { - "meta": meta, - "mtype": mtype, - "media_source": media_source, - "media_id": media_id, - "episode_group": episode_group, - "cache": cache, - } - if music_type is not None: - module_kwargs["music_type"] = music_type - mediainfo = await self._async_run_native_media_recognize(module_kwargs, cache) - # 原生识别未取得远端身份时，允许插件按已知要素补充匹配媒体信息（影视与音乐统一） - mediainfo = await self._async_supplement_media_recognize( - meta=meta, mtype=mtype, media_source=media_source, - media_id=media_id, mediainfo=mediainfo, - music_type=music_type, - ) - fallback_mediainfo = ( - mediainfo - if mediainfo and not self._media_info_has_identity(mediainfo) - else None - ) - if mediainfo and self._media_info_has_identity(mediainfo): - # 电影、电视剧、音乐统一上报；音乐的 tmdb 等字段恒为 None，身份取数据源原生 ID - if not getattr(mediainfo, "recognize_cache_hit", False): - await MoviePilotServerHelper.async_report_recognize_share( - meta=meta, - mediainfo=mediainfo, - keyword_meta=share_query_meta, - ) - return mediainfo - - if self._can_use_media_recognize_share( - share_query_meta, media_source, media_id - ): - shared_cache_meta = self._snapshot_recognize_cache_meta(meta) - share_query_kwargs = { - "meta": meta, - "mtype": mtype, - "keyword_meta": share_query_meta, - } - if music_type is not None: - share_query_kwargs["music_type"] = music_type - shared_item = await MoviePilotServerHelper.async_query_recognize_share( - **share_query_kwargs, - ) - shared_params = MoviePilotServerHelper.to_recognize_params(shared_item) - if shared_params: - shared_module_kwargs = { - "meta": meta, - "mtype": shared_params.get("mtype") or mtype, - "media_source": shared_params.get("media_source"), - "media_id": shared_params.get("media_id"), - "episode_group": episode_group, - "cache": cache, - } - shared_music_type = shared_params.get("music_type") or music_type - if shared_music_type is not None: - shared_module_kwargs["music_type"] = shared_music_type - mediainfo = await self._async_run_native_media_recognize( - shared_module_kwargs, - cache, - ) - if mediainfo and self._media_info_has_identity(mediainfo): - await self._async_update_local_recognize_cache(shared_cache_meta, mediainfo) - await run_in_threadpool(self._record_media_recognize_share_hit) - return mediainfo - if mediainfo and not fallback_mediainfo: - fallback_mediainfo = mediainfo - return fallback_mediainfo - - @staticmethod - def _media_recognize_plugin_payload( - meta: Optional[MetaBase], - mtype: Optional[MediaType], - media_source: Optional[MediaSource], - media_id: Optional[str], - is_music: bool, - music_type: Optional[str] = None, - ) -> dict: - """ - 构造媒体识别链式事件的已知要素载荷，供插件匹配媒体信息；影视与音乐统一协议， - 仅要素字段随媒体类型不同 - """ - if is_music: - return { - "title": getattr(meta, "title", None), - "artists": list(getattr(meta, "artists", None) or []), - "album": getattr(meta, "album", None), - "year": getattr(meta, "year", None), - "isrc": getattr(meta, "isrc", None), - "media_source": media_source, - "media_id": media_id, - "music_type": music_type, - } - return { - "title": getattr(meta, "title", None) or getattr(meta, "name", None), - "year": getattr(meta, "year", None), - "season": getattr(meta, "begin_season", None), - "type": mtype.value if isinstance(mtype, MediaType) else None, - "media_source": media_source, - "media_id": media_id, - } - - @classmethod - def _media_info_from_plugin( - cls, - event_data: dict, - is_music: bool, - mtype: Optional[MediaType] = None, - music_type: Optional[str] = None, - ) -> Optional[MediaInfo]: - """ - 解析插件返回的媒体信息，缺少数据源或身份字段的结果不采信； - 音乐构造 MusicInfo，影视构造 MediaInfo - """ - if not isinstance(event_data, dict): - return None - plugin_info = event_data.get("mediainfo") - if not isinstance(plugin_info, dict): - return None - if not plugin_info.get("media_source"): - logger.warn("插件返回的媒体信息缺少数据源，忽略 ...") - return None - try: - if is_music: - if not plugin_info.get("media_id"): - logger.warn("插件返回的音乐媒体信息缺少媒体ID，忽略 ...") - return None - info: MediaInfo = MusicInfo.from_dict(plugin_info) - if not info.media_source or not info.media_id: - return None - if music_type and info.music_type != music_type: - logger.warn( - f"插件返回的音乐实体类型为 {info.music_type}，" - f"与请求的 {music_type} 不一致，忽略 ..." - ) - return None - return info - # 影视：插件未提供类型时使用请求推断的类型 - if not plugin_info.get("type") and mtype: - plugin_info = {**plugin_info, "type": mtype} - info = MediaInfo() - info.from_dict(plugin_info) - except Exception as err: - logger.warn(f"插件返回的媒体信息格式错误：{err}") - return None - # 影视与音乐统一要求远端身份，无身份的结果不采信，避免未验证结果进入识别管线 - if not info.media_source or not cls._media_info_has_identity(info): - logger.warn("插件返回的媒体信息缺少远端身份，忽略 ...") - return None - return info - - @staticmethod - def _media_info_has_identity(mediainfo) -> bool: - """判断媒体信息是否具备完整的规范媒体身份。""" - media_source, media_id = resolve_media_identity(media=mediainfo) - return bool(media_source and media_id) - - def _supplement_media_recognize( - self, - meta: Optional[MetaBase], - mtype: Optional[MediaType], - media_source: Optional[MediaSource], - media_id: Optional[str], - mediainfo, - music_type: Optional[str] = None, - ): - """ - 媒体识别插件补充（影视与音乐统一）：原生模块未给出带远端身份的结果时， - 广播媒体识别链式事件，允许插件（如第三方媒体源）按已知要素匹配并返回标准信息 - """ - is_music = ( - isinstance(meta, MetaMusic) - or mtype == MediaType.MUSIC - or isinstance(mediainfo, MusicInfo) - ) - # 已有远端身份时无需插件介入 - if mediainfo and self._media_info_has_identity(mediainfo): - return mediainfo - etype = ChainEventType.MusicMediaRecognize if is_music else ChainEventType.MediaRecognize - if not self.eventmanager.check(etype): - return mediainfo - result: Event = self.eventmanager.send_event( - etype, - self._media_recognize_plugin_payload( - meta, mtype, media_source, media_id, is_music, music_type - ), - ) - if not result: - return mediainfo - plugin_info = self._media_info_from_plugin( - result.event_data or {}, is_music, mtype, music_type - ) - if not plugin_info: - return mediainfo - logger.info( - f"插件补充媒体识别成功：{plugin_info.title}" - f"（{plugin_info.media_source}:{plugin_info.media_id}）" - ) - return plugin_info - - async def _async_supplement_media_recognize( - self, - meta: Optional[MetaBase], - mtype: Optional[MediaType], - media_source: Optional[MediaSource], - media_id: Optional[str], - mediainfo, - music_type: Optional[str] = None, - ): - """媒体识别插件补充的异步版本，影视与音乐统一流程""" - is_music = ( - isinstance(meta, MetaMusic) - or mtype == MediaType.MUSIC - or isinstance(mediainfo, MusicInfo) - ) - # 已有远端身份时无需插件介入 - if mediainfo and self._media_info_has_identity(mediainfo): - return mediainfo - etype = ChainEventType.MusicMediaRecognize if is_music else ChainEventType.MediaRecognize - if not self.eventmanager.check(etype): - return mediainfo - result: Event = await self.eventmanager.async_send_event( - etype, - self._media_recognize_plugin_payload( - meta, mtype, media_source, media_id, is_music, music_type - ), - ) - if not result: - return mediainfo - plugin_info = self._media_info_from_plugin( - result.event_data or {}, is_music, mtype, music_type - ) - if not plugin_info: - return mediainfo - logger.info( - f"插件补充媒体识别成功：{plugin_info.title}" - f"（{plugin_info.media_source}:{plugin_info.media_id}）" - ) - return plugin_info - def match_doubaninfo( self, name: str, @@ -1281,7 +687,7 @@ class ChainBase(metaclass=ABCMeta): return self.run_module("webhook_parser", body=body, form=form, args=args) def search_medias( - self, meta: MetaBase, media_source: Optional[MediaSourceSelection] = None + self, meta: MetaBase, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaInfo]]: """ 搜索媒体信息 @@ -1294,7 +700,7 @@ class ChainBase(metaclass=ABCMeta): ) async def async_search_medias( - self, meta: MetaBase, media_source: Optional[MediaSourceSelection] = None + self, meta: MetaBase, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaInfo]]: """ 搜索媒体信息（异步版本） @@ -1307,7 +713,7 @@ class ChainBase(metaclass=ABCMeta): ) def search_persons( - self, name: str, media_source: Optional[MediaSourceSelection] = None + self, name: str, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaPerson]]: """ 搜索人物信息 @@ -1320,7 +726,7 @@ class ChainBase(metaclass=ABCMeta): ) async def async_search_persons( - self, name: str, media_source: Optional[MediaSourceSelection] = None + self, name: str, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaPerson]]: """ 搜索人物信息（异步版本） @@ -1333,7 +739,7 @@ class ChainBase(metaclass=ABCMeta): ) def search_collections( - self, name: str, media_source: Optional[MediaSourceSelection] = None + self, name: str, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaInfo]]: """ 搜索集合信息 @@ -1346,7 +752,7 @@ class ChainBase(metaclass=ABCMeta): ) async def async_search_collections( - self, name: str, media_source: Optional[MediaSourceSelection] = None + self, name: str, media_source: Optional[MediaSourceSelection] = None ) -> Optional[List[MediaInfo]]: """ 搜索集合信息（异步版本） @@ -1785,373 +1191,6 @@ class ChainBase(metaclass=ABCMeta): """ return self.run_module("media_files", mediainfo=mediainfo) - def post_message( - self, - message: Optional[Notification] = None, - meta: Optional[MetaBase] = None, - mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None, - torrentinfo: Optional[TorrentInfo] = None, - transferinfo: Optional[TransferInfo] = None, - **kwargs, - ) -> None: - """ - 发送消息 - :param message: Notification实例 - :param meta: 元数据 - :param mediainfo: 媒体信息 - :param torrentinfo: 种子信息 - :param transferinfo: 文件整理信息 - :param kwargs: 其他参数(覆盖业务对象属性值) - :return: 成功或失败 - """ - # 添加格式化的时间参数 - kwargs.setdefault("current_time", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - # 渲染消息 - message = MessageTemplateHelper.render( - message=message, - meta=meta, - mediainfo=mediainfo, - torrentinfo=torrentinfo, - transferinfo=transferinfo, - **kwargs, - ) - # 检查消息是否有效 - if not message: - logger.warning("消息为空，跳过发送") - return - if message.save_history: - self.messageoper.add(**message.model_dump()) - dispatch_message = self._normalize_notification_for_dispatch(message) - # 发送消息按设置隔离 - if not dispatch_message.userid and dispatch_message.mtype: - # 消息隔离设置 - notify_action = ServiceConfigHelper.get_notification_switch( - dispatch_message.mtype - ) - if notify_action: - # 'admin' 'user,admin' 'user' 'all' - actions = notify_action.split(",") - # 是否已发送管理员标志 - admin_sended = False - send_orignal = False - useroper = UserOper() - for action in actions: - send_message = copy.deepcopy(dispatch_message) - if action == "admin" and not admin_sended: - # 仅发送管理员 - logger.info(f"{send_message.mtype} 的消息已设置发送给管理员") - # 读取管理员消息IDS - send_message.targets = useroper.get_settings(settings.SUPERUSER) - admin_sended = True - elif action == "user" and send_message.username: - # 发送对应用户 - logger.info( - f"{send_message.mtype} 的消息已设置发送给用户 {send_message.username}" - ) - # 读取用户消息IDS - send_message.targets = useroper.get_settings( - send_message.username - ) - if send_message.targets is None: - # 没有找到用户 - if not admin_sended: - # 回滚发送管理员 - logger.info( - f"用户 {send_message.username} 不存在，消息将发送给管理员" - ) - # 读取管理员消息IDS - send_message.targets = useroper.get_settings( - settings.SUPERUSER - ) - admin_sended = True - else: - # 管理员发过了，此消息不发了 - logger.info( - f"用户 {send_message.username} 不存在，消息无法发送到对应用户" - ) - continue - elif send_message.username == settings.SUPERUSER: - # 管理员同名已发送 - admin_sended = True - else: - # 按原消息发送全体 - if not admin_sended: - send_orignal = True - break - # 按设定发送 - self.eventmanager.send_event( - etype=EventType.NoticeMessage, - data=self._build_notice_message_data(send_message), - ) - self.messagequeue.send_message( - "post_message", message=send_message, **kwargs - ) - if not send_orignal: - return - # 发送消息事件 - self.eventmanager.send_event( - etype=EventType.NoticeMessage, - data=self._build_notice_message_data(dispatch_message), - ) - # 按原消息发送 - self.messagequeue.send_message( - "post_message", - message=dispatch_message, - immediately=True if dispatch_message.userid else False, - **kwargs, - ) - - async def async_post_message( - self, - message: Optional[Notification] = None, - meta: Optional[MetaBase] = None, - mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None, - torrentinfo: Optional[TorrentInfo] = None, - transferinfo: Optional[TransferInfo] = None, - **kwargs, - ) -> None: - """ - 异步发送消息 - :param message: Notification实例 - :param meta: 元数据 - :param mediainfo: 媒体信息 - :param torrentinfo: 种子信息 - :param transferinfo: 文件整理信息 - :param kwargs: 其他参数(覆盖业务对象属性值) - :return: 成功或失败 - """ - # 添加格式化的时间参数 - kwargs.setdefault("current_time", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) - # 渲染消息 - message = MessageTemplateHelper.render( - message=message, - meta=meta, - mediainfo=mediainfo, - torrentinfo=torrentinfo, - transferinfo=transferinfo, - **kwargs, - ) - # 检查消息是否有效 - if not message: - logger.warning("消息为空，跳过发送") - return - if message.save_history: - await self.messageoper.async_add(**message.model_dump()) - dispatch_message = self._normalize_notification_for_dispatch(message) - # 发送消息按设置隔离 - if not dispatch_message.userid and dispatch_message.mtype: - # 消息隔离设置 - notify_action = ServiceConfigHelper.get_notification_switch( - dispatch_message.mtype - ) - if notify_action: - # 'admin' 'user,admin' 'user' 'all' - actions = notify_action.split(",") - # 是否已发送管理员标志 - admin_sended = False - send_orignal = False - useroper = UserOper() - for action in actions: - send_message = copy.deepcopy(dispatch_message) - if action == "admin" and not admin_sended: - # 仅发送管理员 - logger.info(f"{send_message.mtype} 的消息已设置发送给管理员") - # 读取管理员消息IDS - send_message.targets = useroper.get_settings(settings.SUPERUSER) - admin_sended = True - elif action == "user" and send_message.username: - # 发送对应用户 - logger.info( - f"{send_message.mtype} 的消息已设置发送给用户 {send_message.username}" - ) - # 读取用户消息IDS - send_message.targets = useroper.get_settings( - send_message.username - ) - if send_message.targets is None: - # 没有找到用户 - if not admin_sended: - # 回滚发送管理员 - logger.info( - f"用户 {send_message.username} 不存在，消息将发送给管理员" - ) - # 读取管理员消息IDS - send_message.targets = useroper.get_settings( - settings.SUPERUSER - ) - admin_sended = True - else: - # 管理员发过了，此消息不发了 - logger.info( - f"用户 {send_message.username} 不存在，消息无法发送到对应用户" - ) - continue - elif send_message.username == settings.SUPERUSER: - # 管理员同名已发送 - admin_sended = True - else: - # 按原消息发送全体 - if not admin_sended: - send_orignal = True - break - # 按设定发送 - await self.eventmanager.async_send_event( - etype=EventType.NoticeMessage, - data=self._build_notice_message_data(send_message), - ) - await self.messagequeue.async_send_message( - "post_message", message=send_message, **kwargs - ) - if not send_orignal: - return - # 发送消息事件 - await self.eventmanager.async_send_event( - etype=EventType.NoticeMessage, - data=self._build_notice_message_data(dispatch_message), - ) - # 按原消息发送 - await self.messagequeue.async_send_message( - "post_message", - message=dispatch_message, - immediately=True if dispatch_message.userid else False, - **kwargs, - ) - - def post_medias_message( - self, message: Notification, medias: List[MediaInfo] - ) -> None: - """ - 发送媒体信息选择列表 - :param message: 消息体 - :param medias: 媒体列表 - :return: 成功或失败 - """ - note_list = [media.to_dict() for media in medias] - if message.save_history: - self.messageoper.add(**message.model_dump(), note=note_list) - dispatch_message = self._normalize_notification_for_dispatch(message) - return self.messagequeue.send_message( - "post_medias_message", - message=dispatch_message, - medias=medias, - immediately=True if dispatch_message.userid else False, - ) - - def post_torrents_message( - self, message: Notification, torrents: List[Context] - ) -> None: - """ - 发送种子信息选择列表 - :param message: 消息体 - :param torrents: 种子列表 - :return: 成功或失败 - """ - note_list = [torrent.torrent_info.to_dict() for torrent in torrents] - if message.save_history: - self.messageoper.add(**message.model_dump(), note=note_list) - dispatch_message = self._normalize_notification_for_dispatch(message) - return self.messagequeue.send_message( - "post_torrents_message", - message=dispatch_message, - torrents=torrents, - immediately=True if dispatch_message.userid else False, - ) - - def delete_message( - self, - channel: MessageChannel, - source: str, - message_id: Union[str, int], - chat_id: Optional[Union[str, int]] = None, - ) -> bool: - """ - 删除消息 - :param channel: 消息渠道 - :param source: 消息源（指定特定的消息模块） - :param message_id: 消息ID - :param chat_id: 聊天ID（如群组ID） - :return: 删除是否成功 - """ - return self.run_module( - "delete_message", - channel=channel, - source=source, - message_id=message_id, - chat_id=chat_id, - ) - - def edit_message( - self, - channel: MessageChannel, - source: str, - message_id: Union[str, int], - chat_id: Union[str, int], - text: str, - title: Optional[str] = None, - buttons: Optional[List[List[dict]]] = None, - metadata: Optional[Dict[str, Any]] = None, - ) -> bool: - """ - 编辑已发送的消息 - :param channel: 消息渠道 - :param source: 消息源（指定特定的消息模块） - :param message_id: 消息ID - :param chat_id: 聊天ID - :param text: 新的消息内容 - :param title: 消息标题 - :param buttons: 更新后的按钮列表 - :param metadata: 其他消息元数据 - :return: 编辑是否成功 - """ - if channel == MessageChannel.WebAgent: - try: - from app.application.messaging.agent import edit_web_agent_message - - return edit_web_agent_message( - user_id=str((metadata or {}).get("userid") or ""), - message_id=message_id, - title=title, - text=text, - buttons=buttons, - ) - except Exception as err: - logger.debug(f"编辑 WebAgent 消息失败: {err}") - return False - - return self.run_module( - "edit_message", - channel=channel, - source=source, - message_id=message_id, - chat_id=chat_id, - text=text, - title=title, - buttons=buttons, - metadata=metadata, - ) - - def send_direct_message(self, message: Notification) -> Optional[MessageResponse]: - """ - 直接发送消息并返回消息ID等信息（用于后续编辑消息的场景） - 不经过消息队列、不保存消息历史 - :param message: 消息体 - :return: 消息响应（包含message_id, chat_id等） - """ - return self.run_module( - "send_direct_message", - message=self._normalize_notification_for_dispatch(message), - ) - - def finalize_message( - self, - response: MessageResponse, - ) -> bool: - """ - 对已发送消息执行渠道收尾动作。 - 例如关闭流式卡片状态；无特殊收尾的渠道直接返回 False。 - """ - return self.run_module("finalize_message", response=response) - def metadata_img( self, mediainfo: MediaInfo, diff --git a/app/chain/_interaction.py b/app/chain/_interaction.py new file mode 100644 index 000000000..3a55bf738 --- /dev/null +++ b/app/chain/_interaction.py @@ -0,0 +1,86 @@ +from typing import Optional, Tuple, Union + +from app.schemas.types import MessageChannel + + +class InteractionChainMixin: + """ + 斜杠命令交互四件套委托：remote_list / parse_callback / + handle_callback_interaction / handle_text_interaction。 + + subscribe、site 等业务链的交互入口完全同构，唯一差异是各自的 + 交互处理器构造参数。本 mixin 将四件套委托提取为公共实现， + 子类只需注入处理器类并实现 _interaction_handler 构造器。 + + 子类注入约定： + - `_interaction_handler_type`：交互处理器类，提供静态 parse_callback； + - `_interaction_handler()`：按各链业务动作构造处理器实例。 + """ + + # 交互处理器类，子类注入（如 SubscribeInteractionHandler / SiteInteractionHandler） + _interaction_handler_type: type = None + + def _interaction_handler(self): + """ + 构造交互处理器实例，由子类按各自业务动作注入实现。 + """ + raise NotImplementedError + + def remote_list( + self, + arg_str: str = "", + channel: MessageChannel = None, + userid: Union[str, int] = None, + source: Optional[str] = None, + ): + """ + 斜杠命令统一入口，委托交互处理器。 + """ + return self._interaction_handler().remote_list( + arg_str=arg_str, channel=channel, userid=userid, source=source + ) + + @classmethod + def parse_callback(cls, callback_data: str) -> Optional[Tuple[str, str]]: + """ + 解析斜杠命令按钮回调。 + """ + return cls._interaction_handler_type.parse_callback(callback_data) + + def handle_callback_interaction( + self, + callback_data: str, + channel: MessageChannel, + source: str, + userid: Union[str, int], + username: str, + original_message_id: Optional[Union[str, int]] = None, + original_chat_id: Optional[str] = None, + ) -> bool: + """委托交互处理器处理按钮回调。""" + return self._interaction_handler().handle_callback_interaction( + callback_data=callback_data, + channel=channel, + source=source, + userid=userid, + username=username, + original_message_id=original_message_id, + original_chat_id=original_chat_id, + ) + + def handle_text_interaction( + self, + channel: MessageChannel, + source: str, + userid: Union[str, int], + username: str, + text: str, + ) -> bool: + """委托交互处理器处理文本输入。""" + return self._interaction_handler().handle_text_interaction( + channel=channel, + source=source, + userid=userid, + username=username, + text=text, + ) diff --git a/app/chain/_messaging.py b/app/chain/_messaging.py new file mode 100644 index 000000000..92ff989c6 --- /dev/null +++ b/app/chain/_messaging.py @@ -0,0 +1,486 @@ +"""消息处理与通知发送 mixin。 + +从 ChainBase 拆出的消息域：渠道输入状态机、通知派发规范化、消息渲染、 +隔离路由与队列发送。方法经 MRO 解析，依赖 ChainBase 实例的 run_module、 +eventmanager、messageoper、messagequeue 等协作对象。 +""" +import copy +from datetime import datetime +from typing import Any, Dict, List, Optional, Union + +from app.db.oper.user import UserOper +from app.domain.context import Context, MediaInfo, MusicInfo, TorrentInfo +from app.domain.meta.metabase import MetaBase +from app.foundation.identity import normalize_internal_user_id +from app.application.messaging.message import MessageTemplateHelper +from app.runtime.config import settings +from app.runtime.extensions.service_registry import ServiceConfigHelper +from app.runtime.log import logger +from app.schemas import MessageResponse, Notification, TransferInfo +from app.schemas.message import ChannelCapability, ChannelCapabilityManager +from app.schemas.types import EventType, MessageChannel + + +class MessageProcessingMixin: + """消息输入/处理状态机与通知派发规范化。""" + + def start_message_processing_status( + self, + channel: MessageChannel, + source: Optional[str], + userid: Optional[Union[str, int]] = None, + message_id: Optional[Union[str, int]] = None, + chat_id: Optional[Union[str, int]] = None, + text: Optional[str] = None, + ) -> Optional[dict]: + """ + 启动渠道侧消息输入/处理状态。 + 具体表现由消息模块实现，例如 typing 保活或消息 reaction。 + """ + if not channel or not ChannelCapabilityManager.supports_capability( + channel, ChannelCapability.PROCESSING_STATUS + ): + return None + try: + status = self.run_module( + "mark_message_processing_started", + channel=channel, + source=source, + userid=userid, + message_id=message_id, + chat_id=chat_id, + text=text, + ) + except Exception as err: + logger.debug(f"启动消息处理状态失败: {err}") + return None + return status if isinstance(status, dict) else None + + def finish_message_processing_status( + self, + status: Optional[dict] = None, + channel: Optional[MessageChannel] = None, + source: Optional[str] = None, + userid: Optional[Union[str, int]] = None, + message_id: Optional[Union[str, int]] = None, + chat_id: Optional[Union[str, int]] = None, + ) -> None: + """ + 结束渠道侧消息输入/处理状态。 + 优先使用 start 返回的 status，缺失时使用显式渠道和消息定位参数。 + """ + target_channel = channel + if status: + try: + target_channel = MessageChannel(status.get("channel")) + except Exception: + target_channel = channel + if not target_channel or not ChannelCapabilityManager.supports_capability( + target_channel, ChannelCapability.PROCESSING_STATUS + ): + return + try: + self.run_module( + "mark_message_processing_finished", + channel=target_channel, + source=(status or {}).get("source") or source, + userid=(status or {}).get("userid") or userid, + message_id=(status or {}).get("message_id") or message_id, + chat_id=(status or {}).get("chat_id") or chat_id, + status=status, + ) + except Exception as err: + logger.debug(f"结束消息处理状态失败: {err}") + + @staticmethod + def _normalize_notification_for_dispatch( + message: Notification + ) -> Notification: + """ + 规范化待发送的通知消息。 + 后台任务会复用内部占位用户ID作为会话身份，这里在真正发送前清空， + 让消息重新走默认通知路由或基于 targets 的目标解析。 + """ + dispatch_message = copy.deepcopy(message) + dispatch_message.userid = normalize_internal_user_id( + dispatch_message.userid + ) + return dispatch_message + + @staticmethod + def _build_notice_message_data(message: Notification) -> dict: + """ + 构造消息通知事件数据。 + """ + return {**message.model_dump(exclude={"save_history"}), "type": message.mtype} + + +class NotificationMixin: + """通知消息发送域：渲染、隔离路由、队列发送与消息编辑。""" + + def post_message( + self, + message: Optional[Notification] = None, + meta: Optional[MetaBase] = None, + mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None, + torrentinfo: Optional[TorrentInfo] = None, + transferinfo: Optional[TransferInfo] = None, + **kwargs, + ) -> None: + """ + 发送消息 + :param message: Notification实例 + :param meta: 元数据 + :param mediainfo: 媒体信息 + :param torrentinfo: 种子信息 + :param transferinfo: 文件整理信息 + :param kwargs: 其他参数(覆盖业务对象属性值) + :return: 成功或失败 + """ + # 添加格式化的时间参数 + kwargs.setdefault("current_time", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + # 渲染消息 + message = MessageTemplateHelper.render( + message=message, + meta=meta, + mediainfo=mediainfo, + torrentinfo=torrentinfo, + transferinfo=transferinfo, + **kwargs, + ) + # 检查消息是否有效 + if not message: + logger.warning("消息为空，跳过发送") + return + if message.save_history: + self.messageoper.add(**message.model_dump()) + dispatch_message = self._normalize_notification_for_dispatch(message) + # 发送消息按设置隔离 + if not dispatch_message.userid and dispatch_message.mtype: + # 消息隔离设置 + notify_action = ServiceConfigHelper.get_notification_switch( + dispatch_message.mtype + ) + if notify_action: + # 'admin' 'user,admin' 'user' 'all' + actions = notify_action.split(",") + # 是否已发送管理员标志 + admin_sended = False + send_orignal = False + useroper = UserOper() + for action in actions: + send_message = copy.deepcopy(dispatch_message) + if action == "admin" and not admin_sended: + # 仅发送管理员 + logger.info(f"{send_message.mtype} 的消息已设置发送给管理员") + # 读取管理员消息IDS + send_message.targets = useroper.get_settings(settings.SUPERUSER) + admin_sended = True + elif action == "user" and send_message.username: + # 发送对应用户 + logger.info( + f"{send_message.mtype} 的消息已设置发送给用户 {send_message.username}" + ) + # 读取用户消息IDS + send_message.targets = useroper.get_settings( + send_message.username + ) + if send_message.targets is None: + # 没有找到用户 + if not admin_sended: + # 回滚发送管理员 + logger.info( + f"用户 {send_message.username} 不存在，消息将发送给管理员" + ) + # 读取管理员消息IDS + send_message.targets = useroper.get_settings( + settings.SUPERUSER + ) + admin_sended = True + else: + # 管理员发过了，此消息不发了 + logger.info( + f"用户 {send_message.username} 不存在，消息无法发送到对应用户" + ) + continue + elif send_message.username == settings.SUPERUSER: + # 管理员同名已发送 + admin_sended = True + else: + # 按原消息发送全体 + if not admin_sended: + send_orignal = True + break + # 按设定发送 + self.eventmanager.send_event( + etype=EventType.NoticeMessage, + data=self._build_notice_message_data(send_message), + ) + self.messagequeue.send_message( + "post_message", message=send_message, **kwargs + ) + if not send_orignal: + return + # 发送消息事件 + self.eventmanager.send_event( + etype=EventType.NoticeMessage, + data=self._build_notice_message_data(dispatch_message), + ) + # 按原消息发送 + self.messagequeue.send_message( + "post_message", + message=dispatch_message, + immediately=True if dispatch_message.userid else False, + **kwargs, + ) + + async def async_post_message( + self, + message: Optional[Notification] = None, + meta: Optional[MetaBase] = None, + mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None, + torrentinfo: Optional[TorrentInfo] = None, + transferinfo: Optional[TransferInfo] = None, + **kwargs, + ) -> None: + """ + 异步发送消息 + :param message: Notification实例 + :param meta: 元数据 + :param mediainfo: 媒体信息 + :param torrentinfo: 种子信息 + :param transferinfo: 文件整理信息 + :param kwargs: 其他参数(覆盖业务对象属性值) + :return: 成功或失败 + """ + # 添加格式化的时间参数 + kwargs.setdefault("current_time", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + # 渲染消息 + message = MessageTemplateHelper.render( + message=message, + meta=meta, + mediainfo=mediainfo, + torrentinfo=torrentinfo, + transferinfo=transferinfo, + **kwargs, + ) + # 检查消息是否有效 + if not message: + logger.warning("消息为空，跳过发送") + return + if message.save_history: + await self.messageoper.async_add(**message.model_dump()) + dispatch_message = self._normalize_notification_for_dispatch(message) + # 发送消息按设置隔离 + if not dispatch_message.userid and dispatch_message.mtype: + # 消息隔离设置 + notify_action = ServiceConfigHelper.get_notification_switch( + dispatch_message.mtype + ) + if notify_action: + # 'admin' 'user,admin' 'user' 'all' + actions = notify_action.split(",") + # 是否已发送管理员标志 + admin_sended = False + send_orignal = False + useroper = UserOper() + for action in actions: + send_message = copy.deepcopy(dispatch_message) + if action == "admin" and not admin_sended: + # 仅发送管理员 + logger.info(f"{send_message.mtype} 的消息已设置发送给管理员") + # 读取管理员消息IDS + send_message.targets = useroper.get_settings(settings.SUPERUSER) + admin_sended = True + elif action == "user" and send_message.username: + # 发送对应用户 + logger.info( + f"{send_message.mtype} 的消息已设置发送给用户 {send_message.username}" + ) + # 读取用户消息IDS + send_message.targets = useroper.get_settings( + send_message.username + ) + if send_message.targets is None: + # 没有找到用户 + if not admin_sended: + # 回滚发送管理员 + logger.info( + f"用户 {send_message.username} 不存在，消息将发送给管理员" + ) + # 读取管理员消息IDS + send_message.targets = useroper.get_settings( + settings.SUPERUSER + ) + admin_sended = True + else: + # 管理员发过了，此消息不发了 + logger.info( + f"用户 {send_message.username} 不存在，消息无法发送到对应用户" + ) + continue + elif send_message.username == settings.SUPERUSER: + # 管理员同名已发送 + admin_sended = True + else: + # 按原消息发送全体 + if not admin_sended: + send_orignal = True + break + # 按设定发送 + await self.eventmanager.async_send_event( + etype=EventType.NoticeMessage, + data=self._build_notice_message_data(send_message), + ) + await self.messagequeue.async_send_message( + "post_message", message=send_message, **kwargs + ) + if not send_orignal: + return + # 发送消息事件 + await self.eventmanager.async_send_event( + etype=EventType.NoticeMessage, + data=self._build_notice_message_data(dispatch_message), + ) + # 按原消息发送 + await self.messagequeue.async_send_message( + "post_message", + message=dispatch_message, + immediately=True if dispatch_message.userid else False, + **kwargs, + ) + + def post_medias_message( + self, message: Notification, medias: List[MediaInfo] + ) -> None: + """ + 发送媒体信息选择列表 + :param message: 消息体 + :param medias: 媒体列表 + :return: 成功或失败 + """ + note_list = [media.to_dict() for media in medias] + if message.save_history: + self.messageoper.add(**message.model_dump(), note=note_list) + dispatch_message = self._normalize_notification_for_dispatch(message) + return self.messagequeue.send_message( + "post_medias_message", + message=dispatch_message, + medias=medias, + immediately=True if dispatch_message.userid else False, + ) + + def post_torrents_message( + self, message: Notification, torrents: List[Context] + ) -> None: + """ + 发送种子信息选择列表 + :param message: 消息体 + :param torrents: 种子列表 + :return: 成功或失败 + """ + note_list = [torrent.torrent_info.to_dict() for torrent in torrents] + if message.save_history: + self.messageoper.add(**message.model_dump(), note=note_list) + dispatch_message = self._normalize_notification_for_dispatch(message) + return self.messagequeue.send_message( + "post_torrents_message", + message=dispatch_message, + torrents=torrents, + immediately=True if dispatch_message.userid else False, + ) + + def delete_message( + self, + channel: MessageChannel, + source: str, + message_id: Union[str, int], + chat_id: Optional[Union[str, int]] = None, + ) -> bool: + """ + 删除消息 + :param channel: 消息渠道 + :param source: 消息源（指定特定的消息模块） + :param message_id: 消息ID + :param chat_id: 聊天ID（如群组ID） + :return: 删除是否成功 + """ + return self.run_module( + "delete_message", + channel=channel, + source=source, + message_id=message_id, + chat_id=chat_id, + ) + + def edit_message( + self, + channel: MessageChannel, + source: str, + message_id: Union[str, int], + chat_id: Union[str, int], + text: str, + title: Optional[str] = None, + buttons: Optional[List[List[dict]]] = None, + metadata: Optional[Dict[str, Any]] = None, + ) -> bool: + """ + 编辑已发送的消息 + :param channel: 消息渠道 + :param source: 消息源（指定特定的消息模块） + :param message_id: 消息ID + :param chat_id: 聊天ID + :param text: 新的消息内容 + :param title: 消息标题 + :param buttons: 更新后的按钮列表 + :param metadata: 其他消息元数据 + :return: 编辑是否成功 + """ + if channel == MessageChannel.WebAgent: + try: + from app.application.messaging.agent import edit_web_agent_message + + return edit_web_agent_message( + user_id=str((metadata or {}).get("userid") or ""), + message_id=message_id, + title=title, + text=text, + buttons=buttons, + ) + except Exception as err: + logger.debug(f"编辑 WebAgent 消息失败: {err}") + return False + + return self.run_module( + "edit_message", + channel=channel, + source=source, + message_id=message_id, + chat_id=chat_id, + text=text, + title=title, + buttons=buttons, + metadata=metadata, + ) + + def send_direct_message(self, message: Notification) -> Optional[MessageResponse]: + """ + 直接发送消息并返回消息ID等信息（用于后续编辑消息的场景） + 不经过消息队列、不保存消息历史 + :param message: 消息体 + :return: 消息响应（包含message_id, chat_id等） + """ + return self.run_module( + "send_direct_message", + message=self._normalize_notification_for_dispatch(message), + ) + + def finalize_message( + self, + response: MessageResponse, + ) -> bool: + """ + 对已发送消息执行渠道收尾动作。 + 例如关闭流式卡片状态；无特殊收尾的渠道直接返回 False。 + """ + return self.run_module("finalize_message", response=response) diff --git a/app/chain/_mixins.py b/app/chain/_mixins.py new file mode 100644 index 000000000..d780b4124 --- /dev/null +++ b/app/chain/_mixins.py @@ -0,0 +1,1559 @@ +"""整理链功能域 mixin。 + +TransferChain 从 5000+ 行的单体拆出这些内聚功能域，每个 mixin 只承载一类 +整理辅助逻辑；主流程（do_transfer / manual_transfer / remote_transfer）仍留在 +TransferChain 中。mixin 方法运行时经 MRO 解析，共享 TransferChain 实例状态。 + +注意：这里的方法均已去掉私有名前缀双下划线（__ -> _），因为 Python 的名字 +改编按定义类生效，方法迁到 mixin 后 __ 前缀会改变改编目标，导致跨类调用失败。 +""" +import asyncio +from copy import deepcopy +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +from app import schemas +from app.adapters.system.host import SystemUtils +from app.application.agent import build_manual_redo_prompt, get_agent_manager +from app.application.formatting import EpisodeFormatRuleHelper +from app.application.history import clear_transfer_failures, resolve_history +from app.application.transfer import TransferTask, job_lock +from app.chain.media import MediaChain +from app.chain.storage import StorageChain +from app.chain.subscribe import SubscribeChain +from app.db.models.downloadhistory import DownloadFiles, DownloadHistory +from app.db.models.transferhistory import TransferHistory +from app.db.oper.downloadhistory import DownloadHistoryOper +from app.db.oper.systemconfig import SystemConfigOper +from app.db.oper.transferhistory import TransferHistoryOper +from app.domain.context import MediaInfo, MusicInfo +from app.domain.media import normalize_music_type +from app.domain.meta.metabase import MetaBase +from app.domain.meta.metamusic import MetaMusic +from app.foundation import text as text_tools +from app.runtime.config import global_vars, settings +from app.runtime.log import logger +from app.schemas import ( + FileItem, + Notification, + TmdbEpisode, + TransferInfo, +) +from app.schemas.agent import ReplyMode +from app.schemas.types import ( + MUSIC_ENTITY_ALBUM, + EventType, + MediaSource, + MediaType, + MessageChannel, + SystemConfigKey, +) + +# 字幕文件常见的语言/默认/强制标记，整理同名字幕时只允许剥离这些字幕专属尾缀。 +SUBTITLE_STEM_TAGS = { + "cc", + "chi", + "chs", + "cht", + "cn", + "default", + "en", + "eng", + "english", + "forced", + "gb", + "gb2312", + "hk", + "ja", + "jap", + "japanese", + "jp", + "jpn", + "sc", + "sdh", + "tc", + "zh", + "zh-cn", + "zh-hans", + "zh-hant", + "zh-tw", + "zh_cn", + "zh_hans", + "zh_hant", + "zh_tw", + "zho", + "中英", + "中字", + "双语", + "简中", + "简体", + "繁中", + "繁体", +} + + +class FileFilterMixin: + @staticmethod + def _requires_automatic_category(task: TransferTask) -> bool: + """ + 判断当前整理任务是否需要根据媒体识别结果自动创建类别目录。 + + :param task: 整理任务 + :return: 是否必须具备自动分类结果 + """ + target_directory = task.target_directory + if target_directory and target_directory.media_category: + return False + if task.library_category_folder is not None: + return bool(task.library_category_folder) + return bool( + target_directory and target_directory.library_category_folder + ) + + def _is_subtitle_file(self, fileitem: FileItem) -> bool: + """ + 判断是否为字幕文件 + """ + if not fileitem.extension: + return False + return ( + True if f".{fileitem.extension.lower()}" in self._subtitle_exts else False + ) + + def _is_audio_file(self, fileitem: FileItem) -> bool: + """ + 判断是否为音频文件 + """ + if not fileitem.extension: + return False + return True if f".{fileitem.extension.lower()}" in self._audio_exts else False + + def _is_media_file( + self, + fileitem: FileItem, + mtype: Optional[MediaType] = None, + ) -> bool: + """ + 判断是否为主要媒体文件 + """ + if mtype == MediaType.MUSIC: + if fileitem.type != "file" or not fileitem.extension: + return False + return f".{fileitem.extension.lower()}" in self._audio_exts + if fileitem.type == "dir": + # 蓝光原盘判断 + return StorageChain().is_bluray_folder(fileitem) + if not fileitem.extension: + return False + extension = f".{fileitem.extension.lower()}" + return extension in self._media_exts + + def _is_primary_media_file( + self, + fileitem: FileItem, + mediainfo: Optional[MediaInfo | MusicInfo], + ) -> bool: + """判断文件在当前媒体上下文中是否属于主要媒体文件。""" + return self._is_media_file( + fileitem, + getattr(mediainfo, "type", None), + ) + + @staticmethod + def _music_info_from_meta(meta: MetaMusic) -> MusicInfo: + """将音频文件标签解析结果转换为可整理的最小音乐信息。""" + return MusicInfo.from_meta(meta) + + @classmethod + def _match_music_album_context( + cls, + file_item: FileItem, + file_path: Path, + file_meta: MetaMusic, + ) -> tuple[MetaMusic, Optional[MusicInfo]]: + """为缺少远端身份的本地音频尝试目录级专辑匹配，命中后回填文件元数据。 + + WAV 等无标签文件只能依靠目录结构和曲目特征识别；匹配结果由 MediaChain + 按目录缓存，同一专辑目录内的后续文件不会重复请求远端。 + """ + # 目录级匹配需要读取本地音频时长，远端存储文件无法参与 + if file_meta.media_id or getattr(file_item, "storage", "local") != "local": + return file_meta, None + try: + matched = MediaChain().recognize_music_album_directory(file_path.parent) + except Exception as err: + logger.debug(f"音乐专辑目录匹配失败：{file_path} - {err}") + return file_meta, None + info = matched.get(str(file_path.resolve())) + if not info or not info.media_id: + return file_meta, None + logger.info(f"{file_path.name} 通过专辑目录匹配识别为：{info.artist} - {info.title}") + merged_meta = deepcopy(file_meta) + # 保留本地音频的实际技术参数，仅回填身份和名称字段 + if info.title: + merged_meta.title = info.title + if info.artists: + merged_meta.artists = list(info.artists) + if info.album: + merged_meta.album = info.album + if info.album_artist: + merged_meta.album_artist = info.album_artist + if info.year: + merged_meta.year = info.year + if info.disc_number: + merged_meta.disc_number = info.disc_number + if info.track_number: + merged_meta.track_number = info.track_number + if info.total_tracks: + merged_meta.total_tracks = info.total_tracks + merged_meta.media_source = info.media_source + merged_meta.media_id = info.media_id + merged_info = cls._music_info_from_meta(merged_meta) + # 补齐曲目级远端信息，供后续刮削和展示使用 + merged_info.music_type = info.music_type + merged_info.artist_ids = list(info.artist_ids) + merged_info.album_id = info.album_id + merged_info.album_type = info.album_type + merged_info.release_date = info.release_date + merged_info.cover_url = info.cover_url + merged_info.category = info.category + merged_info.genres = list(info.genres) + merged_info.detail_link = info.detail_link + return merged_meta, merged_info + + @staticmethod + def _download_history_music_type( + download_history: Optional[DownloadHistory], + ) -> Optional[str]: + """从下载历史字段或旧版音乐备注中恢复音乐实体类型。""" + music_type = normalize_music_type( + getattr(download_history, "music_type", None), + allow_artist=False, + ) + if music_type: + return music_type + note = getattr(download_history, "note", None) + music_note = note.get("music") if isinstance(note, dict) else None + media_payload = music_note.get("media") if isinstance(music_note, dict) else None + if not isinstance(media_payload, dict): + return None + return normalize_music_type( + media_payload.get("music_type"), + allow_artist=False, + ) + + @classmethod + def _restore_music_download_context( + cls, + download_history: Optional[DownloadHistory], + file_path: Path, + ) -> tuple[Optional[MetaMusic], Optional[MusicInfo]]: + """从下载历史恢复音乐上下文，并用当前音频标签覆盖曲目级字段。""" + note = getattr(download_history, "note", None) + music_note = note.get("music") if isinstance(note, dict) else None + if not isinstance(music_note, dict) or music_note.get("version") != 1: + return None, None + try: + saved_meta = MetaMusic.from_dict(music_note.get("meta") or {}) + saved_info = MusicInfo.from_dict(music_note.get("media") or {}) + except (TypeError, ValueError): + return None, None + + file_tags = MediaChain.read_path_meta(file_path) + file_meta = deepcopy(saved_meta) + file_meta.org_string = file_path.name + # 曲目标题始终优先使用当前文件自身的标签（缺失时回退为文件名）， + # 防止整包目录继续沿用订阅/下载标题（单曲名、专辑名等）导致所有文件重名。 + if file_tags.title: + file_meta.title = file_tags.title + is_album_context = saved_info.music_type == MUSIC_ENTITY_ALBUM + for field_name in ( + "artists", + "disc_number", + "track_number", + "total_discs", + "version", + "isrc", + ): + if getattr(file_tags, field_name, None): + setattr(file_meta, field_name, deepcopy(getattr(file_tags, field_name))) + for field_name in ("album", "album_artist", "year", "total_tracks"): + file_value = getattr(file_tags, field_name, None) + # 整专下载以订阅选中的专辑字段为准，避免单个错误标签把曲目拆到其它专辑目录。 + if file_value and (not is_album_context or not getattr(file_meta, field_name, None)): + setattr(file_meta, field_name, deepcopy(file_value)) + for field_name in ( + "audio_format", + "bit_depth", + "sample_rate", + "bitrate", + "duration", + ): + if getattr(file_tags, field_name, None): + setattr(file_meta, field_name, getattr(file_tags, field_name)) + file_meta.media_source = saved_info.media_source or saved_meta.media_source + file_meta.media_id = saved_info.media_id or saved_meta.media_id + + file_info = cls._music_info_from_meta(file_meta) + file_info.media_source = saved_info.media_source + file_info.media_id = saved_info.media_id + file_info.music_type = saved_info.music_type + file_info.artist_ids = list(saved_info.artist_ids) + file_info.album_id = saved_info.album_id + file_info.album_type = saved_info.album_type + file_info.release_date = saved_info.release_date + file_info.cover_url = saved_info.cover_url + file_info.lyrics = saved_info.lyrics + file_info.category = saved_info.category + file_info.genres = list(saved_info.genres) + file_info.detail_link = saved_info.detail_link + file_info.listen_count = saved_info.listen_count + return file_meta, file_info + + @staticmethod + def _is_music_retry_source(history: TransferHistory, src_path: Path) -> bool: + """ + 判断重新整理来源是否应走音乐链路：历史类型为音乐，或源路径为音频文件。 + """ + if history.type == MediaType.MUSIC.value: + return True + return src_path.suffix.lower() in settings.RMT_AUDIOEXT + + def _recognize_music_retry_media( + self, + history: TransferHistory, + src_path: Path, + ) -> Optional[Union[MusicInfo, MediaInfo]]: + """ + 重新整理重试时恢复音乐信息。 + + 优先按历史记录中的 MusicBrainz 身份恢复；单音频文件回退按音频标签与文件名识别； + 音乐专辑目录返回 None，交由整理链按音频后缀逐文件解析识别。 + """ + if history.media_source and history.media_id: + retry_info = MediaChain().recognize_media( + mtype=MediaType.MUSIC, + media_source=history.media_source, + media_id=history.media_id, + music_type=getattr(history, "music_type", None), + ) + if retry_info: + return retry_info + if src_path.is_file(): + # 音频走统一路径识别入口，自动路由到音乐识别链 + recognize_context = MediaChain().recognize_by_path(str(src_path)) + return recognize_context.media_info if recognize_context else None + return None + + def _is_allowed_file(self, fileitem: FileItem) -> bool: + """ + 判断是否允许的扩展名 + """ + if not fileitem.extension: + return False + return True if f".{fileitem.extension.lower()}" in self._allowed_exts else False + + @staticmethod + def _is_allow_filesize(fileitem: FileItem, min_filesize: int) -> bool: + """ + 判断是否满足最小文件大小 + """ + return ( + True + if not min_filesize or (fileitem.size or 0) > min_filesize * 1024 * 1024 + else False + ) + + @staticmethod + def _is_hidden_or_recycle_path(file_path: Optional[str]) -> bool: + """ + 判断是否隐藏或回收站路径 + """ + if not file_path: + return False + normalized_path = file_path.replace("\\", "/") + return ( + "/@Recycle/" in normalized_path + or "/#recycle/" in normalized_path + or "/." in normalized_path + or "/@eaDir" in normalized_path + ) + + @staticmethod + def _should_delete_empty_source_directories( + task: TransferTask, + delete_mounted_local_disk_empty_dirs: bool, + mounted_filesystem_cache: Dict[Path, bool], + ) -> bool: + """ + 判断移动整理后是否应删除源空目录。 + + 仅在关闭挂载盘空目录清理且源存储为本地时检测文件系统， + 避免默认流程产生额外系统调用。 + """ + if delete_mounted_local_disk_empty_dirs: + return True + if task.fileitem.storage != "local": + return True + + source_directory = ( + Path(task.target_directory.download_path) + if task.target_directory and task.target_directory.download_path + else Path(task.fileitem.path).parent + ) + if source_directory not in mounted_filesystem_cache: + mounted_filesystem_cache[source_directory] = ( + SystemUtils.is_network_filesystem( + source_directory, include_local_fuse=True + ) + ) + return not mounted_filesystem_cache[source_directory] + + @staticmethod + def _is_overwrite_declined(task: TransferTask, transferinfo: TransferInfo, + transferhis: TransferHistoryOper) -> bool: + """ + 判断本次未入库是否为「同路径已有成功记录 + 覆盖模式裁定不覆盖」。 + + 只有同路径此前已成功整理过才需要保护：这类文件是查重闸放行的同路径新版本， + 媒体库中的原有版本仍然在位，不应因一次不覆盖裁决把成功记录改写成失败记录。 + 没有成功记录时（如目标同名文件来自其他源路径）保持原有失败语义， + 用户仍能在历史与通知中看到裁决结果。 + :param task: 整理任务 + :param transferinfo: 整理结果 + :param transferhis: 历史操作对象 + :return: True 表示应保留原成功记录 + """ + if not transferinfo.overwrite_skipped or not task.fileitem: + return False + try: + history = resolve_history( + task.fileitem.path, + storage=task.fileitem.storage, + transfer_history_oper=transferhis, + ) + except Exception as err: + logger.error(f"查询整理历史失败: {task.fileitem.path} - {err}") + return False + return bool(history and history.status) + + +class ScrapeBatchMixin: + + def _send_metadata_scrape_event( + self, task: TransferTask, transferinfo: TransferInfo + ): + """ + 发送元数据刮削事件，保持对外事件载荷兼容。 + """ + if ( + not task + or not transferinfo + or not transferinfo.need_scrape + or not self._is_primary_media_file(task.fileitem, task.mediainfo) + ): + return + + target_diritem = transferinfo.target_diritem + if not target_diritem: + return + + self.eventmanager.send_event( + EventType.MetadataScrape, + self._build_metadata_scrape_payload( + task=task, + fileitem=target_diritem, + file_list=transferinfo.file_list_new, + overwrite=False, + ), + ) + + @staticmethod + def _build_metadata_scrape_payload( + task: TransferTask, + fileitem: FileItem, + file_list: Optional[list[str]], + overwrite: bool, + ) -> dict[str, Any]: + """构造刮削事件载荷，并为音乐批次保留逐文件身份上下文。""" + paths = list(dict.fromkeys(file_list or [])) + payload: dict[str, Any] = { + "meta": task.meta, + "mediainfo": task.mediainfo, + "fileitem": fileitem, + "file_list": paths, + "overwrite": overwrite, + } + if isinstance(task.mediainfo, MusicInfo): + payload["file_contexts"] = [ + { + "path": path, + "meta": task.meta, + "mediainfo": task.mediainfo, + } + for path in paths + ] + return payload + + def _register_scrape_batch_task(self, task: TransferTask): + """ + 登记批次任务。刮削事件只在批次关闭且任务全部完成后统一发送。 + """ + if not task or not task.transfer_batch_id: + return + with job_lock: + batch = self._scrape_batches.setdefault( + task.transfer_batch_id, + { + "pending": set(), + "targets": {}, + "closed": False, + }, + ) + batch["pending"].add(task.fileitem.path) + + def _close_scrape_batch(self, batch_id: Optional[str]): + """ + 标记批次不再接收新任务，并尝试发送已聚合的刮削事件。 + """ + if not batch_id: + return + with job_lock: + batch = self._scrape_batches.setdefault( + batch_id, + { + "pending": set(), + "targets": {}, + "closed": False, + }, + ) + batch["closed"] = True + self._flush_scrape_batch_if_ready(batch_id) + + def _record_scrape_target(self, task: TransferTask, transferinfo: TransferInfo): + """ + 记录批次内需要刮削的目标文件，按目标媒体根目录聚合。 + """ + if ( + not task + or not task.transfer_batch_id + or not transferinfo + or not transferinfo.need_scrape + or not self._is_primary_media_file(task.fileitem, task.mediainfo) + ): + return + + target_diritem = transferinfo.target_diritem + if not target_diritem: + return + + target_files = transferinfo.file_list_new or [] + target_key = (target_diritem.storage, target_diritem.path) + with job_lock: + batch = self._scrape_batches.setdefault( + task.transfer_batch_id, + { + "pending": set(), + "targets": {}, + "closed": False, + }, + ) + target = batch["targets"].setdefault( + target_key, + { + "fileitem": target_diritem, + "meta": task.meta, + "mediainfo": task.mediainfo, + "files": [], + "file_contexts": {}, + "overwrite": False, + }, + ) + if not target.get("meta"): + target["meta"] = task.meta + if not target.get("mediainfo"): + target["mediainfo"] = task.mediainfo + for target_file in target_files: + if target_file and target_file not in target["files"]: + target["files"].append(target_file) + if target_file and isinstance(task.mediainfo, MusicInfo): + target["file_contexts"][target_file] = { + "path": target_file, + "meta": task.meta, + "mediainfo": task.mediainfo, + } + + def _finish_scrape_batch_task(self, task: TransferTask): + """ + 标记批次内单个任务已结束。 + """ + if not task or not task.transfer_batch_id: + return + with job_lock: + batch = self._scrape_batches.get(task.transfer_batch_id) + if not batch: + return + batch["pending"].discard(task.fileitem.path) + self._flush_scrape_batch_if_ready(task.transfer_batch_id) + + def _flush_scrape_batch_if_ready(self, batch_id: Optional[str]): + """ + 批次任务全部结束后发送聚合后的刮削事件。 + """ + if not batch_id: + return + + with job_lock: + batch = self._scrape_batches.get(batch_id) + if ( + not batch + or not batch.get("closed") + or batch.get("pending") + ): + return + targets = list(batch.get("targets", {}).values()) + self._scrape_batches.pop(batch_id, None) + + for target in targets: + fileitem = target.get("fileitem") + if not fileitem: + continue + file_list = list(dict.fromkeys(target.get("files") or [])) + file_contexts = target.get("file_contexts") or {} + payload = { + "meta": target.get("meta"), + "mediainfo": target.get("mediainfo"), + "fileitem": fileitem, + "file_list": file_list, + "overwrite": target.get("overwrite", False), + } + if file_contexts: + payload["file_contexts"] = [ + file_contexts[path] + for path in file_list + if path in file_contexts + ] + self.eventmanager.send_event( + EventType.MetadataScrape, + payload, + ) + + +class EpisodeFormatMixin: + + def recommend_name(self, meta: MetaBase, mediainfo: MediaInfo) -> Optional[str]: + """ + 获取重命名后的名称 + :param meta: 元数据 + :param mediainfo: 媒体信息 + :return: 重命名后的名称（含目录） + """ + # 获取集信息，供重命名模块使用 + episodes_info: Optional[List[TmdbEpisode]] = None + if mediainfo.type == MediaType.TV: + # 判断注意season为0的情况 + season_num = mediainfo.season + if season_num is None and meta.season_seq: + if meta.season_seq.isdigit(): + season_num = int(meta.season_seq) + # 默认值1 + if season_num is None: + season_num = 1 + episodes_info = self.run_module( + "tmdb_episodes", + tmdbid=mediainfo.tmdb_id, + season=season_num, + episode_group=mediainfo.episode_group, + ) + if episodes_info: + return self.run_module( + "recommend_name", + meta=meta, + mediainfo=mediainfo, + episodes_info=episodes_info, + ) + # 电影或无集信息时保持原有参数集，避免影响旧签名的模块实现 + return self.run_module("recommend_name", meta=meta, mediainfo=mediainfo) + + def recommend_episode_format( + self, + fileitem: FileItem, + fileitems: Optional[List[FileItem]] = None, + ) -> Tuple[bool, str, Optional[dict]]: + """ + 根据目录样本推荐集数定位模板 + """ + if not fileitem and not fileitems: + logger.warn("推荐集数定位模板失败：缺少目录参数") + return False, "缺少目录参数", None + + rules = self._get_episode_format_rules() + if fileitems: + state, errmsg, sample_files = self._get_selected_episode_format_sample_files( + fileitems + ) + if not state: + logger.warn(f"推荐集数定位模板失败：{errmsg}") + return False, errmsg, None + target_path = sample_files[0].path if sample_files else None + else: + if not fileitem or not fileitem.path: + logger.warn("推荐集数定位模板失败：缺少目录参数") + return False, "缺少目录参数", None + directory = self._resolve_episode_format_directory(fileitem) + if not directory or directory.type != "dir": + logger.warn(f"推荐集数定位模板失败：目录不存在 - {fileitem.path}") + return False, "目录不存在", None + sample_files = self._get_episode_format_sample_files(directory) + target_path = directory.path + logger.info( + f"开始匹配集数定位规则：{target_path}，规则数 {len(rules)}，样本数 {len(sample_files)}" + ) + state, errmsg, data = EpisodeFormatRuleHelper().recommend( + rules=rules, + sample_files=sample_files, + ) + if not state: + logger.warn(f"集数定位模板推荐失败：{target_path} - {errmsg}") + return state, errmsg, data + logger.info( + f"集数定位模板推荐成功：{target_path} - 规则 {data.get('rule_name') if data else None}" + ) + return state, errmsg, data + + @staticmethod + def _get_episode_format_rules() -> List[schemas.EpisodeFormatRule]: + """ + 获取启用的集数定位规则 + """ + rule_items = SystemConfigOper().get(SystemConfigKey.EpisodeFormatRuleTable) or [] + rules: List[schemas.EpisodeFormatRule] = [] + for item in rule_items: + if not isinstance(item, dict): + continue + try: + rule = schemas.EpisodeFormatRule(**item) + except Exception as err: + logger.warn(f"忽略无效的集数定位规则：{err}") + continue + if rule.enabled: + rules.append(rule) + return sorted(rules, key=lambda item: item.order) + + def _resolve_episode_format_directory( + self, fileitem: FileItem + ) -> Optional[FileItem]: + """ + 将文件或目录入参归一化为目录对象 + """ + storage_chain = StorageChain() + if fileitem.type == "dir": + return storage_chain.get_item(fileitem) + source_path = Path(fileitem.path) + parent_item = FileItem( + storage=fileitem.storage, + path=source_path.parent.as_posix(), + type="dir", + name=source_path.parent.name, + ) + return storage_chain.get_item(parent_item) + + def _get_selected_episode_format_sample_files( + self, fileitems: List[FileItem] + ) -> Tuple[bool, str, List[FileItem]]: + """ + 获取当前选择文件中可参与模板推荐的样本文件。 + """ + if not fileitems: + return False, "没有可用于识别的样本文件", [] + + expected_dir_key: Optional[Tuple[str, str]] = None + selected_files: List[FileItem] = [] + seen_files = set() + for item in fileitems: + if not item or not item.path or item.type != "file": + return False, "当前选择不满足智能识别条件", [] + + dir_key = ( + item.storage or "local", + Path(item.path).parent.as_posix(), + ) + if expected_dir_key is None: + expected_dir_key = dir_key + elif dir_key != expected_dir_key: + return False, "当前选择不满足智能识别条件", [] + + file_key = (item.storage or "local", item.path) + if file_key in seen_files: + continue + seen_files.add(file_key) + + if not ( + self._is_media_file(item) + or self._is_subtitle_file(item) + or self._is_audio_file(item) + ): + continue + if self._is_hidden_or_recycle_path(item.path): + continue + selected_files.append(item) + + if not selected_files: + return False, "没有可用于识别的样本文件", [] + return True, "", selected_files + + def _get_episode_format_sample_files( + self, directory: FileItem + ) -> List[FileItem]: + """ + 获取目录下可参与模板推荐的样本文件。 + + 推荐结果最终会在手动整理链路中作为 `episode_format` + 交由 `FormatParser` 过滤主视频、字幕和外挂音频，因此这里需要把 + 同目录下的主视频、字幕和外挂音频一起纳入推荐流程。 + """ + file_items = StorageChain().list_files(directory, recursion=False) or [] + sample_files: List[FileItem] = [] + for item in file_items: + if not item or item.type != "file": + continue + if not ( + self._is_media_file(item) + or self._is_subtitle_file(item) + or self._is_audio_file(item) + ): + continue + if self._is_hidden_or_recycle_path(item.path): + continue + sample_files.append(item) + return sample_files + + +class HistoryMatchMixin: + @staticmethod + def _match_download_file( + download_file: DownloadFiles, + file_path: Path, + save_path: Path, + ) -> bool: + """ + 判断下载文件记录是否明确对应当前文件。 + """ + if download_file.fullpath == file_path.as_posix(): + return True + + filepath = download_file.filepath + if not filepath: + return False + + try: + return (save_path / Path(filepath)).as_posix() == file_path.as_posix() + except (TypeError, ValueError): + return False + + def _resolve_history_from_download_files( + self, + downloadhis: DownloadHistoryOper, + download_files: List[DownloadFiles], + file_path: Optional[Path] = None, + save_path: Optional[Path] = None, + ) -> Optional[DownloadHistory]: + """ + 从下载文件记录中解析唯一的下载历史。 + """ + if file_path and save_path: + download_files = [ + download_file + for download_file in download_files + if self._match_download_file( + download_file=download_file, + file_path=file_path, + save_path=save_path, + ) + ] + + download_hashes = { + download_file.download_hash + for download_file in download_files + if download_file.download_hash + } + if len(download_hashes) == 1: + return downloadhis.get_by_hash(next(iter(download_hashes))) + return None + + def _resolve_download_history( + self, + downloadhis: DownloadHistoryOper, + file_path: Path, + bluray_dir: bool = False, + download_hash: Optional[str] = None, + ) -> Optional[DownloadHistory]: + """ + 根据显式 hash、文件路径或种子根目录回查下载历史。 + """ + if download_hash: + return downloadhis.get_by_hash(download_hash) + + if bluray_dir: + return downloadhis.get_by_path(file_path.as_posix()) + + download_file = downloadhis.get_file_by_fullpath(file_path.as_posix()) + if download_file: + return downloadhis.get_by_hash(download_file.download_hash) + + # 多文件种子里的字幕/附加文件可能没有稳定的 fullpath 记录， + # 退回到父目录和 savepath 继续查找，尽量补齐同一种子的关联信息。 + shared_download_roots = self._get_shared_download_roots(file_path) + + for parent_path in file_path.parents: + parent_posix = parent_path.as_posix() + download_files = downloadhis.get_files_by_savepath(parent_posix) or [] + + if parent_posix in shared_download_roots: + # 共享下载根目录只能接受有明确文件记录的匹配， + # 避免单文件/磁力任务把整个根目录污染成同一媒体。 + history = self._resolve_history_from_download_files( + downloadhis=downloadhis, + download_files=download_files, + file_path=file_path, + save_path=parent_path, + ) + if history: + return history + break + + download_history = downloadhis.get_by_path(parent_posix) + if download_history: + return download_history + + history = self._resolve_history_from_download_files( + downloadhis=downloadhis, + download_files=download_files, + ) + if history: + return history + + return None + + @staticmethod + def _is_movie_year_conflict( + file_meta: MetaBase, + # 两种 DownloadHistory 都会进来：库模型（本文件按 ORM 行查历史）与 + # schemas DTO（TransferTask.download_history）。本函数只按 getattr 取 + # year 与 type，对两者一视同仁 + media: Union[DownloadHistory, schemas.DownloadHistory, MediaInfo, MusicInfo] + ) -> bool: + """ + 判断文件名年份是否与已识别电影年份冲突。 + + 多电影合集只保存一条下载历史，不能把合集首部电影的媒体 ID 套用到其它年份的文件； + 电视剧季包仍应继续复用同一条下载历史。 + """ + file_year = getattr(file_meta, "year", None) + media_year = getattr(media, "year", None) + if not file_meta or not media or not file_year or not media_year: + return False + media_type = getattr(media, "type", None) + if not isinstance(media_type, MediaType): + try: + media_type = MediaType(media_type) + except (TypeError, ValueError): + return False + return ( + media_type == MediaType.MOVIE + and str(file_year) != str(media_year) + ) + + @staticmethod + def _optional_attr_equal( + source: MetaBase, + target: MetaBase, + attr: str, + normalizer: Callable = None, + ) -> bool: + """ + 比较可选识别字段。 + + 字段两边都没有识别到时不参与判断；只要任意一边识别到了，就要求两边值一致， + 避免把同名不同年份或不同季集的附加文件误归到当前主视频。 + """ + source_value = getattr(source, attr, None) + target_value = getattr(target, attr, None) + if source_value is None and target_value is None: + return True + if source_value is None or target_value is None: + return False + if normalizer: + source_value = normalizer(source_value) + target_value = normalizer(target_value) + return source_value == target_value + + def _is_same_media_meta( + self, source_meta: MetaBase, target_meta: MetaBase + ) -> bool: + """ + 判断两个文件识别出的媒体身份是否一致。 + """ + if not source_meta or not target_meta: + return False + if source_meta.type != target_meta.type: + return False + if text_tools.normalize_upper(source_meta.name) != text_tools.normalize_upper( + target_meta.name + ): + return False + if not self._optional_attr_equal(source_meta, target_meta, "year", str): + return False + for attr in ( + "begin_season", + "end_season", + "begin_episode", + "end_episode", + ): + if not self._optional_attr_equal(source_meta, target_meta, attr, int): + return False + return True + + +class FileKeyMixin: + @staticmethod + def _get_file_key(fileitem: FileItem) -> Tuple[str, str]: + """ + 获取文件缓存键。 + """ + normalized_path = Path(str(fileitem.path).replace("\\", "/")).as_posix() + return fileitem.storage or "local", normalized_path + + @staticmethod + def _get_file_stem(fileitem: FileItem) -> str: + """ + 获取文件主干名，用于判断同名附加文件。 + """ + file_name = fileitem.name or Path(fileitem.path).name + return Path(file_name).stem.lower() + + @classmethod + def _get_subtitle_media_stem(cls, subtitle_fileitem: FileItem) -> str: + """ + 获取字幕对应主视频的候选主干名。 + """ + current_stem = cls._get_file_stem(subtitle_fileitem) + while current_stem: + media_stem, separator, suffix = current_stem.rpartition(".") + if not separator or suffix not in SUBTITLE_STEM_TAGS: + return current_stem + current_stem = media_stem + return current_stem + + def _get_extra_media_stem(self, extra_fileitem: FileItem) -> str: + """ + 获取附加文件对应主视频的候选主干名。 + """ + if self._is_subtitle_file(extra_fileitem): + return self._get_subtitle_media_stem(extra_fileitem) + return self._get_file_stem(extra_fileitem) + + def _get_related_main_file_key( + self, + extra_fileitem: FileItem, + main_fileitems: List[FileItem], + ) -> Optional[Tuple[str, str]]: + """ + 获取与附加文件名完全匹配的主视频键。 + """ + if not ( + self._is_subtitle_file(extra_fileitem) + or self._is_audio_file(extra_fileitem) + ): + return None + + extra_media_stem = self._get_extra_media_stem(extra_fileitem) + matched_items: List[FileItem] = [] + for main_fileitem in main_fileitems: + main_stem = self._get_file_stem(main_fileitem) + if main_stem and main_stem == extra_media_stem: + matched_items.append(main_fileitem) + + if len(matched_items) != 1: + return None + return self._get_file_key(matched_items[0]) + + @staticmethod + def _normalize_dir_path(dir_path: Union[str, Path]) -> str: + """ + 归一化目录路径，用于同一父目录候选缓存。 + """ + normalized = Path(dir_path).as_posix().rstrip("/") + return normalized or "/" + + def _get_dir_key(self, dir_item: FileItem) -> Tuple[str, str]: + """ + 获取目录缓存键。 + """ + return dir_item.storage, self._normalize_dir_path(dir_item.path) + + def _get_file_parent_key(self, current_item: FileItem) -> Tuple[str, str]: + """ + 获取文件父目录缓存键。 + """ + return ( + current_item.storage, + self._normalize_dir_path(Path(current_item.path).parent), + ) + + +class ManualHistoryMixin: + @staticmethod + def _get_subscribe_custom_words( + history_record: Optional[DownloadHistory], + ) -> Optional[List[str]]: + """ + 获取整理用自定义识别词：优先使用下载时保存的快照，无快照（历史旧记录）时再按来源实时反查订阅。 + + 快照优先可避免整理阶段因订阅季号漂移、来源解析失败或订阅完成被删导致识别词丢失，从而原样入库到偏移前的季集。 + """ + if not history_record: + return None + # 下载时保存的完整订阅识别词快照优先 + if history_record.custom_words: + return history_record.custom_words.split("\n") + # 兜底：历史旧记录无快照时，按下载来源实时反查订阅 + if not isinstance(history_record.note, dict): + return None + subscribe = SubscribeChain().get_subscribe_by_source( + history_record.note.get("source") + ) + return ( + subscribe.custom_words.split("\n") + if subscribe and subscribe.custom_words + else None + ) + + @staticmethod + def _is_successful_move_history(history: Optional[TransferHistory]) -> bool: + """判断历史记录是否为已成功完成的移动类整理。""" + return bool( + history + and history.status + and history.mode + and "move" in history.mode + ) + + def _get_manual_transfer_history( + self, + fileitem: FileItem, + transfer_history_oper: TransferHistoryOper, + include_move_dest: bool = False, + ) -> Optional[TransferHistory]: + """查询文件源路径历史，并兼容从成功移动后的目标现址重新整理。""" + # resolve_history 在命中失败记录时会再确认一次有无成功记录， + # 避免 get_by_src 无排序导致同源多行时返回哪条不确定 + history = resolve_history( + fileitem.path, + storage=fileitem.storage, + transfer_history_oper=transfer_history_oper, + ) + if history or not include_move_dest: + return history + + history = transfer_history_oper.get_by_dest( + fileitem.path, + storage=fileitem.storage, + ) + return history if self._is_successful_move_history(history) else None + + def get_manual_transfer_histories( + self, + fileitems: List[FileItem], + ) -> List[TransferHistory]: + """ + 查询文件或目录命中的成功整理记录，供手动整理界面显示重整状态。 + + :param fileitems: 待查询的文件或目录项 + :return: 去重后的成功整理记录 + """ + transfer_history_oper = TransferHistoryOper() + histories: Dict[int, TransferHistory] = {} + for fileitem in fileitems or []: + if not fileitem or not fileitem.path: + continue + storage = fileitem.storage or "local" + if fileitem.type == "dir": + matched_histories = transfer_history_oper.list_success_by_src( + fileitem.path, + storage=storage, + recursive=True, + ) + matched_histories.extend( + transfer_history_oper.list_success_move_by_dest( + fileitem.path, + storage=storage, + recursive=True, + ) + ) + else: + history = self._get_manual_transfer_history( + fileitem=fileitem, + transfer_history_oper=transfer_history_oper, + include_move_dest=True, + ) + matched_histories = [history] if history and history.status else [] + + for history in matched_histories: + histories[history.id] = history + return list(histories.values()) + + @staticmethod + def _delete_manual_transfer_history( + history: TransferHistory, + transfer_history_oper: TransferHistoryOper, + ) -> Tuple[bool, str]: + """删除手动重整历史；非成功移动记录同时清理可能存在的旧目标。""" + if ( + history.dest_fileitem + and not ManualHistoryMixin._is_successful_move_history(history) + ): + dest_fileitem = FileItem(**history.dest_fileitem) + storage_chain = StorageChain() + if ( + storage_chain.exists(dest_fileitem) + and not storage_chain.delete_media_file(dest_fileitem) + ): + return False, f"{dest_fileitem.path} 删除失败" + transfer_history_oper.delete(history.id) + # 删除记录是用户显式要求重来，失败计数一并清零，否则重整仍会受上一轮次数限制 + clear_transfer_failures(history.src, history.src_storage) + return True, "" + + +class FailedRetryMixin: + @staticmethod + def build_failed_transfer_buttons( + history_id: Optional[int], + ) -> Optional[List[List[dict]]]: + """ + 构建整理失败通知的操作按钮。 + """ + if not history_id: + return None + return [ + [ + {"text": "重试", "callback_data": f"transfer_retry_{history_id}"}, + { + "text": "智能助手接管", + "callback_data": f"transfer_ai_retry_{history_id}", + }, + ] + ] + + def redo_transfer_history(self, history_id: int) -> Tuple[bool, str]: + """ + 按历史记录直接重新整理，自动重新识别媒体信息。 + """ + return self._re_transfer(logid=history_id) + + @staticmethod + def parse_failed_transfer_callback( + callback_data: str, + ) -> Optional[tuple[str, int]]: + """ + 解析整理失败通知按钮回调。 + """ + for prefix, action in ( + ("transfer_retry_", "retry"), + ("transfer_ai_retry_", "ai_retry"), + ): + if callback_data.startswith(prefix): + history_id = callback_data.replace(prefix, "", 1) + if history_id.isdigit(): + return action, int(history_id) + return None + + def handle_failed_transfer_callback( + self, + *, + callback_data: str, + channel: MessageChannel, + source: str, + userid: Union[str, int], + username: str, + ) -> bool: + """ + 处理整理失败通知中的重试类按钮。 + """ + callback = self.parse_failed_transfer_callback(callback_data) + if not callback: + return False + + action, history_id = callback + if action == "retry": + self._retry_transfer_history( + history_id=history_id, + channel=channel, + source=source, + userid=userid, + username=username, + ) + else: + self._take_over_transfer_history_by_ai( + history_id=history_id, + channel=channel, + source=source, + userid=userid, + username=username, + ) + return True + + def _retry_transfer_history( + self, + history_id: int, + channel: MessageChannel, + source: str, + userid: Union[str, int], + username: str, + ) -> None: + """ + 立即重新整理一条失败的整理记录。 + """ + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title=f"开始重新整理记录 #{history_id} ...", + save_history=False, + ) + ) + + state, errmsg = self.redo_transfer_history(history_id) + if state: + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title=f"整理记录 #{history_id} 已重新整理", + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + return + + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="重新整理失败", + text=errmsg, + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + + def _take_over_transfer_history_by_ai( + self, + history_id: int, + channel: MessageChannel, + source: str, + userid: Union[str, int], + username: str, + ) -> None: + """ + 由智能助手接管一条失败的整理记录。 + """ + + if not settings.AI_AGENT_ENABLE: + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="MoviePilot智能助手未启用，请在系统设置中启用", + save_history=False, + ) + ) + return + + history = TransferHistoryOper().get(history_id) + if not history: + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="重新整理失败", + text=f"整理记录 #{history_id} 不存在", + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + return + + redo_prompt = build_manual_redo_prompt(history) + + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title=f"已将整理记录 #{history_id} 交给智能助手处理", + text="处理完成后会在这里回复结果。", + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + + async def _run_ai_takeover(): + final_output = "" + + def _capture_output(text_output: str): + nonlocal final_output + final_output = text_output or "" + + try: + await get_agent_manager().run_background_prompt( + message=redo_prompt, + session_prefix=f"__agent_manual_redo_{history_id}", + output_callback=_capture_output, + reply_mode=ReplyMode.CAPTURE_ONLY, + allow_message_tools=False, + ) + await self.async_post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="智能助手整理完成", + text=final_output.strip() + or f"整理记录 #{history_id} 已由智能助手处理完成。", + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + except Exception as e: + await self.async_post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="智能助手整理失败", + text=str(e), + link=settings.MP_DOMAIN("#/history"), + save_history=False, + ) + ) + + asyncio.run_coroutine_threadsafe(_run_ai_takeover(), global_vars.loop) + + def _re_transfer( + self, + logid: int, + mtype: MediaType = None, + media_source: Optional[MediaSource] = None, + media_id: Optional[str] = None, + ) -> Tuple[bool, str]: + """ + 根据历史记录，重新识别整理，只支持简单条件 + :param logid: 历史记录ID + :param mtype: 媒体类型 + :param media_source: 媒体数据源 + :param media_id: 数据源原生 ID，必须与 media_source 成对提供 + """ + # 查询历史记录 + history: TransferHistory = TransferHistoryOper().get(logid) + if not history: + logger.error(f"整理记录不存在，ID：{logid}") + return False, "整理记录不存在" + # 按源目录路径重新整理 + src_path = Path(history.src) + if not src_path.exists(): + return False, f"源目录不存在：{src_path}" + # 查询媒体信息 + explicit_identity = media_source is not None or media_id is not None + if explicit_identity and (not media_source or not media_id): + return False, "媒体重新识别需要同时提供 media_source 和 media_id" + if mtype and media_source and media_id: + mediainfo = MediaChain().recognize_media( + mtype=mtype, + media_source=media_source, + media_id=media_id, + music_type=( + getattr(history, "music_type", None) + if mtype == MediaType.MUSIC + else None + ), + episode_group=history.episode_group, + ) + if mediainfo and not isinstance(mediainfo, MusicInfo): + # 更新媒体图片 + self.obtain_images(mediainfo=mediainfo) + elif history.media_source and history.media_id: + try: + history_type = mtype or MediaType(history.type) + except ValueError: + history_type = mtype + mediainfo = MediaChain().recognize_media( + mtype=history_type, + media_source=history.media_source, + media_id=history.media_id, + music_type=( + getattr(history, "music_type", None) + if history_type == MediaType.MUSIC + else None + ), + episode_group=history.episode_group, + ) + mtype = history_type + if mediainfo and not isinstance(mediainfo, MusicInfo): + self.obtain_images(mediainfo=mediainfo) + elif mtype == MediaType.MUSIC or self._is_music_retry_source(history, src_path): + # 音乐重新整理走音乐识别链，避免默认影视识别误入 TMDB + mtype = MediaType.MUSIC + mediainfo = self._recognize_music_retry_media(history, src_path) + else: + recognize_context = MediaChain().recognize_by_path( + str(src_path), + episode_group=history.episode_group, + obtain_images=True, + ) + mediainfo = recognize_context.media_info if recognize_context else None + # 音乐专辑目录允许无预识别信息，由整理链按音频后缀逐文件解析识别 + if not mediainfo and not (mtype == MediaType.MUSIC and src_path.is_dir()): + return False, ( + f"未识别到媒体信息，类型：{mtype.value if mtype else None}，" + f"media_source：{media_source}，media_id：{media_id}" + ) + # 重新执行整理 + if mediainfo: + logger.info(f"{src_path.name} 识别为：{mediainfo.title_year}") + + # 删除旧的已整理文件 + if history.dest_fileitem: + # 解析目标文件对象 + dest_fileitem = FileItem(**history.dest_fileitem) + StorageChain().delete_file(dest_fileitem) + + # 强制整理 + if history.src_fileitem: + state, errmsg = self.do_transfer( + fileitem=FileItem(**history.src_fileitem), + mediainfo=mediainfo, + mtype=mtype, + download_hash=history.download_hash, + force=True, + background=False, + manual=True, + ) + if not state: + return False, errmsg + + return True, "" diff --git a/app/chain/_music.py b/app/chain/_music.py new file mode 100644 index 000000000..1fd4be24f --- /dev/null +++ b/app/chain/_music.py @@ -0,0 +1,420 @@ +import copy +from typing import Any, List, Optional, Tuple + +from app.application.torrent import TorrentHelper +from app.chain.download import DownloadChain +from app.chain.media import MediaChain +from app.chain.search import SearchChain +from app.db.models.subscribe import Subscribe +from app.db.oper.subscribe import SubscribeOper +from app.db.oper.systemconfig import SystemConfigOper +from app.domain.context import Context, MediaInfo, MusicInfo +from app.domain.media import MUSIC_SUBSCRIBABLE_TYPES +from app.domain.meta.metamusic import MetaMusic +from app.runtime.log import logger +from app.schemas.types import ( + MUSIC_ENTITY_ALBUM, + MUSIC_ENTITY_RECORDING, + MediaType, + SystemConfigKey, +) + + +def _normalize_music_total_tracks(value: Any) -> Optional[int]: + """将专辑曲目总数归一为正整数，无效或未知值返回 None。""" + try: + total_tracks = int(value or 0) + except (TypeError, ValueError): + return None + return total_tracks if total_tracks > 0 else None + + +class MusicSubscribeMixin: + """ + 音乐订阅功能域 mixin：单曲/专辑目标识别、实体快照同步、候选筛选、 + 择优下载与完成推进。 + + 该域方法通过 self 复用 SubscribeChain 主体的 get_sub_sites / get_params / + filter_torrents / check_and_handle_existing_media / finish_subscribe_or_not / + get_subscribe_source_keyword 等编排能力，因此仅作为 mixin 混入 SubscribeChain， + 不独立成链。build_subscribe_meta / _subscribe_media_key 等订阅通用辅助仍保留在 + subscribe.py，方法内延迟导入以避免 _music ↔ subscribe 的模块级循环。 + """ + + @staticmethod + def _validate_music_subscribe_target( + mediainfo: MediaInfo, + requested_music_type: Optional[str] = None, + ) -> Optional[str]: + """校验音乐订阅实体一致性，并确保专辑具备可验证的曲目总数。""" + if mediainfo.type != MediaType.MUSIC: + return "识别结果不是音乐" + music_type = getattr(mediainfo, "music_type", None) + if requested_music_type and requested_music_type not in MUSIC_SUBSCRIBABLE_TYPES: + return "音乐订阅仅支持单曲或专辑" + if music_type not in MUSIC_SUBSCRIBABLE_TYPES: + return "音乐订阅仅支持单曲或专辑" + if requested_music_type and requested_music_type != music_type: + return f"音乐订阅类型不匹配：请求 {requested_music_type}，识别为 {music_type}" + if music_type == MUSIC_ENTITY_ALBUM \ + and _normalize_music_total_tracks(getattr(mediainfo, "total_tracks", None)) is None: + return "专辑总曲目数未知，无法校验整张专辑资源" + return None + + @staticmethod + def _ensure_music_subscribe_entity( + subscribe: Subscribe, + mediainfo: Optional[MusicInfo], + ) -> Optional[MusicInfo]: + """保持已持久化的单曲/专辑实体边界，拒绝远端详情把订阅类型改写。""" + if not mediainfo: + return None + expected_type = getattr(subscribe, "music_type", None) + actual_type = getattr(mediainfo, "music_type", None) + if expected_type and expected_type not in MUSIC_SUBSCRIBABLE_TYPES: + logger.warning(f"音乐订阅 {subscribe.name} 的实体类型无效：{expected_type}") + return None + if actual_type not in MUSIC_SUBSCRIBABLE_TYPES: + logger.warning( + f"音乐订阅 {subscribe.name} 识别为不可订阅实体：{actual_type}" + ) + if expected_type in MUSIC_SUBSCRIBABLE_TYPES: + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + return None + if expected_type and actual_type != expected_type: + logger.warning( + f"音乐订阅 {subscribe.name} 实体不匹配：" + f"订阅为 {expected_type}，远端识别为 {actual_type}，使用订阅快照" + ) + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + if actual_type == MUSIC_ENTITY_ALBUM: + remote_total = _normalize_music_total_tracks(getattr(mediainfo, "total_tracks", None)) + stored_total = _normalize_music_total_tracks(getattr(subscribe, "total_tracks", None)) + resolved_total = remote_total or stored_total + if resolved_total is not None and mediainfo.total_tracks != resolved_total: + # 识别模块结果可能来自共享缓存，补齐订阅快照时不得原地修改。 + mediainfo = copy.copy(mediainfo) + mediainfo.total_tracks = resolved_total + return mediainfo + + @staticmethod + def _recognize_music_subscribe(subscribe: Subscribe) -> Optional[MusicInfo]: + """按订阅身份恢复音乐目标，远端暂不可用时使用已持久化的稳定快照。""" + # 延迟导入订阅通用辅助，避免 _music ↔ subscribe 模块级循环 + from app.chain.subscribe import build_subscribe_meta + if subscribe.media_source and subscribe.media_id: + # 与影视共用统一识别入口，按媒体源和原生 ID 恢复音乐详情 + mediainfo = MediaChain().recognize_media( + media_source=subscribe.media_source, + media_id=str(subscribe.media_id), + mtype=MediaType.MUSIC, + music_type=getattr(subscribe, "music_type", None), + ) + if mediainfo: + return MusicSubscribeMixin._ensure_music_subscribe_entity(subscribe, mediainfo) + if getattr(subscribe, "music_type", None) in {MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_ALBUM}: + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + # 旧订阅没有保存实体类型时不能猜测为单曲，否则可能误把专辑按单曲完成。 + return None + if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: + # 缺少远端 ID 的专辑不能退化为单曲识别，使用已保存专辑快照更可靠。 + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + # 旧订阅没有实体类型时只允许走 Recording 识别，不能从全局混合搜索中猜成专辑或艺术家。 + mediainfo = MediaChain().recognize_media( + meta=build_subscribe_meta(subscribe), + mtype=MediaType.MUSIC, + media_source=subscribe.media_source, + music_type=MUSIC_ENTITY_RECORDING, + ) + return MusicSubscribeMixin._ensure_music_subscribe_entity(subscribe, mediainfo) + + @staticmethod + async def _async_recognize_music_subscribe(subscribe: Subscribe) -> Optional[MusicInfo]: + """异步按订阅身份恢复音乐目标，远端暂不可用时使用已持久化的稳定快照。""" + # 延迟导入订阅通用辅助，避免 _music ↔ subscribe 模块级循环 + from app.chain.subscribe import build_subscribe_meta + if subscribe.media_source and subscribe.media_id: + # 与影视共用统一识别入口，按媒体源和原生 ID 恢复音乐详情 + mediainfo = await MediaChain().async_recognize_media( + media_source=subscribe.media_source, + media_id=str(subscribe.media_id), + mtype=MediaType.MUSIC, + music_type=getattr(subscribe, "music_type", None), + ) + if mediainfo: + return MusicSubscribeMixin._ensure_music_subscribe_entity(subscribe, mediainfo) + if getattr(subscribe, "music_type", None) in {MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_ALBUM}: + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + return None + if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: + return MusicSubscribeMixin._music_info_from_subscribe(subscribe) + mediainfo = await MediaChain().async_recognize_media( + meta=build_subscribe_meta(subscribe), + mtype=MediaType.MUSIC, + media_source=subscribe.media_source, + music_type=MUSIC_ENTITY_RECORDING, + ) + return MusicSubscribeMixin._ensure_music_subscribe_entity(subscribe, mediainfo) + + @staticmethod + def _music_info_from_subscribe(subscribe: Subscribe) -> MusicInfo: + """从订阅行恢复不依赖远端请求的最小音乐目标，保留专辑完成判断所需字段。""" + year_text = str(subscribe.year or "")[:4] + music_type = getattr(subscribe, "music_type", None) + # 音乐订阅的 description 由标准 MusicInfo.overview 生成，首段固定为艺术家。 + artist_text = str(getattr(subscribe, "description", None) or "") \ + .split(" · ", maxsplit=1)[0].strip() + artists = [ + artist.strip() for artist in artist_text.split(" / ") if artist.strip() + ] + return MusicInfo( + media_source=subscribe.media_source, + media_id=str(subscribe.media_id) if subscribe.media_id is not None else None, + music_type=music_type, + title=subscribe.name, + artists=artists, + album=subscribe.name if music_type == MUSIC_ENTITY_ALBUM else None, + year=int(year_text) if year_text.isdigit() else None, + total_tracks=getattr(subscribe, "total_tracks", None) + if music_type == MUSIC_ENTITY_ALBUM else None, + cover_url=getattr(subscribe, "poster", None) or getattr(subscribe, "backdrop", None), + ) + + @staticmethod + def _sync_music_subscribe_target(subscribe: Subscribe, mediainfo: MusicInfo) -> None: + """把远端识别得到的专辑类型和总曲目数同步到订阅，供搜索失败与完成历史复用。""" + update_data = {} + if mediainfo.music_type and getattr(subscribe, "music_type", None) != mediainfo.music_type: + update_data["music_type"] = mediainfo.music_type + if mediainfo.music_type == MUSIC_ENTITY_ALBUM: + # 远端详情可能暂时不返回曲目数；已确认的订阅快照不能因此被清空。 + total_tracks = _normalize_music_total_tracks(mediainfo.total_tracks) \ + or _normalize_music_total_tracks(getattr(subscribe, "total_tracks", None)) + else: + total_tracks = None + if getattr(subscribe, "total_tracks", None) != total_tracks: + update_data["total_tracks"] = total_tracks + if not update_data: + return + SubscribeOper().update(subscribe.id, update_data) + for key, value in update_data.items(): + setattr(subscribe, key, value) + + @staticmethod + def _is_music_download_complete( + subscribe: Subscribe, + mediainfo: MusicInfo, + downloads: Optional[List[Context]], + ) -> bool: + """判断音乐下载是否满足订阅完成条件；专辑必须由下载层确认整专曲目覆盖。""" + if not downloads: + return False + music_type = getattr(subscribe, "music_type", None) or mediainfo.music_type + if music_type != MUSIC_ENTITY_ALBUM: + return True + return any(context.confirmed_full_coverage for context in downloads) + + def _prepare_music_subscribe( + self, + subscribe: Subscribe, + ) -> Optional[Tuple[MusicInfo, MetaMusic]]: + """识别音乐订阅目标、同步实体快照，并在搜索前处理已完整入库的目标。""" + # 延迟导入订阅通用辅助，避免 _music ↔ subscribe 模块级循环 + from app.chain.subscribe import _subscribe_media_key + mediainfo = self._recognize_music_subscribe(subscribe) + if not mediainfo: + logger.warning( + f"未识别到音乐订阅目标：{subscribe.name}，" + f"媒体源：{subscribe.media_source}，媒体ID：{subscribe.media_id}" + ) + return None + validation_error = self._validate_music_subscribe_target( + mediainfo, + getattr(subscribe, "music_type", None), + ) + if validation_error: + logger.warning(f"音乐订阅 {subscribe.name} 无法继续：{validation_error}") + return None + self._sync_music_subscribe_target(subscribe, mediainfo) + meta = MetaMusic.from_music_info(mediainfo) + exists, _ = self.check_and_handle_existing_media( + subscribe=subscribe, + meta=meta, + mediainfo=mediainfo, + mediakey=_subscribe_media_key(subscribe), + ) + if exists: + return None + return mediainfo, meta + + def _filter_music_subscribe_contexts( + self, + subscribe: Subscribe, + mediainfo: MusicInfo, + contexts: List[Context], + ) -> List[Context]: + """按站点、音乐实体、订阅参数和优先级规则筛选并绑定下载上下文。""" + sites = self.get_sub_sites(subscribe) + default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \ + if subscribe.best_version else SystemConfigKey.SubscribeFilterRuleGroups + rule_groups = subscribe.filter_groups or SystemConfigOper().get(default_rule_key) or [] + torrent_helper = TorrentHelper() + matched: List[Context] = [] + for source_context in contexts or []: + source_torrent = source_context.torrent_info + if not source_torrent or source_torrent.category not in (MediaType.MUSIC, MediaType.MUSIC.value): + continue + # 过滤模块会就地写入 pri_order；RSS 缓存会被多个订阅复用，必须隔离候选副本。 + torrent = copy.copy(source_torrent) + if sites and torrent.site not in sites: + continue + if not SearchChain.matches_music_resource( + mediainfo, + torrent.title, + torrent.description, + ): + continue + if not torrent_helper.filter_torrent(torrent, self.get_params(subscribe)): + continue + filtered = self.filter_torrents( + rule_groups=rule_groups, + torrent_list=[torrent], + mediainfo=mediainfo, + ) + if filtered is not None: + if not filtered: + continue + torrent = filtered[0] + + context = copy.copy(source_context) + context.torrent_info = torrent + meta = MetaMusic.from_music_info(mediainfo) + meta.org_string = torrent.title + meta.apply_audio_quality(f"{torrent.title} {torrent.description or ''}", overwrite=True) + if subscribe.best_version: + # 用户规则组可用格式、码率等内置规则定义洗版顺序；未命中规则 + # 优先级时再回退到规范化音质分数，确保零配置也能自动升级。 + music_priority = torrent.pri_order or meta.audio_quality_score + if music_priority <= (subscribe.current_priority or 0): + logger.info( + f"{torrent.title} 音质优先级 {music_priority} " + f"未高于当前版本 {subscribe.current_priority or 0}" + ) + continue + torrent.pri_order = music_priority + context.meta_info = meta + context.media_info = mediainfo + context.match_source = str(mediainfo.media_source or "title") + context.candidate_recognized = False + context.media_info_is_target = True + if subscribe.media_category: + context.media_info.category = subscribe.media_category + matched.append(context) + return matched + + def _download_music_subscribe( + self, + subscribe: Subscribe, + mediainfo: MusicInfo, + contexts: List[Context], + ) -> None: + """批量择优下载音乐候选，并按单曲或整专完成语义推进订阅。""" + if not contexts: + return + downloads, _ = DownloadChain().batch_download( + contexts=contexts, + username=subscribe.username, + save_path=subscribe.save_path, + downloader=subscribe.downloader, + source=self.get_subscribe_source_keyword(subscribe), + custom_words=subscribe.custom_words, + ) + successful = [ + context for context in downloads or [] + if context and context.meta_info and context.torrent_info + ] + quality_downloads = successful + if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: + quality_downloads = [ + context for context in successful + if context.confirmed_full_coverage + ] + if subscribe.best_version and quality_downloads: + best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order) + best_meta = best_context.meta_info + quality_data = { + "current_priority": best_context.torrent_info.pri_order, + "current_audio_format": best_meta.audio_format, + "current_bitrate": best_meta.bitrate, + "current_bit_depth": best_meta.bit_depth, + "current_sample_rate": best_meta.sample_rate, + } + SubscribeOper().update(subscribe.id, quality_data) + for key, value in quality_data.items(): + setattr(subscribe, key, value) + current_subscribe = SubscribeOper().get(subscribe.id) + if current_subscribe: + self.finish_subscribe_or_not( + subscribe=current_subscribe, + meta=MetaMusic.from_music_info(mediainfo), + mediainfo=mediainfo, + downloads=downloads, + ) + + def _search_music_subscribe(self, subscribe: Subscribe) -> None: + """复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。""" + target = self._prepare_music_subscribe(subscribe) + if not target: + return + mediainfo, _ = target + + sites = self.get_sub_sites(subscribe) + default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \ + if subscribe.best_version else SystemConfigKey.SubscribeFilterRuleGroups + rule_groups = subscribe.filter_groups or SystemConfigOper().get(default_rule_key) or [] + keywords = [subscribe.keyword] if subscribe.keyword else SearchChain.music_site_keywords(mediainfo) + if not keywords: + keywords = [subscribe.name] + + searchchain = SearchChain() + contexts: List[Context] = [] + for keyword in keywords: + contexts = searchchain.search_by_title( + title=keyword, + sites=sites, + mtype=MediaType.MUSIC, + rule_groups=rule_groups, + ) + contexts = self._filter_music_subscribe_contexts( + subscribe=subscribe, + mediainfo=mediainfo, + contexts=contexts, + ) + if contexts: + break + + if not contexts: + logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源") + return + + self._download_music_subscribe(subscribe, mediainfo, contexts) + + def _match_music_subscribe( + self, + subscribe: Subscribe, + contexts: List[Context], + ) -> None: + """直接匹配本轮 RSS 缓存中的音乐资源，避免再次调用站点搜索接口。""" + target = self._prepare_music_subscribe(subscribe) + if not target: + return + mediainfo, _ = target + matched = self._filter_music_subscribe_contexts( + subscribe=subscribe, + mediainfo=mediainfo, + contexts=contexts, + ) + if not matched: + logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源") + return + self._download_music_subscribe(subscribe, mediainfo, matched) diff --git a/app/chain/_recognition.py b/app/chain/_recognition.py new file mode 100644 index 000000000..656e9ef71 --- /dev/null +++ b/app/chain/_recognition.py @@ -0,0 +1,518 @@ +"""媒体识别管线 mixin。 + +从 ChainBase 拆出的识别域：原生模块识别路由、识别缓存回填、共享识别、 +插件补充识别。方法经 MRO 解析，依赖 ChainBase 实例的 run_module/eventmanager +等协作对象。 +""" +import copy +from typing import Optional + +from fastapi.concurrency import run_in_threadpool + +from app.adapters.external.server import MoviePilotServerHelper +from app.db.oper.systemconfig import SystemConfigOper +from app.domain.context import MediaInfo, MusicInfo +from app.domain.meta.metabase import MetaBase +from app.domain.meta.metamusic import MetaMusic +from app.runtime.cache import fresh, async_fresh +from app.runtime.config import settings +from app.runtime.events import Event +from app.runtime.log import logger +from app.schemas.media import normalize_media_source, resolve_media_identity +from app.schemas.types import ChainEventType, MediaSource, MediaType, SystemConfigKey + + +class RecognitionMixin: + + @staticmethod + def _can_use_media_recognize_share( + meta: Optional[MetaBase], + media_source: Optional[MediaSource], + media_id: Optional[str], + ) -> bool: + """ + 仅在名称识别场景下使用共享识别，显式ID识别不再重复回查 + """ + return bool( + settings.MEDIA_RECOGNIZE_SHARE + and meta + and not media_source + and not media_id + ) + + @staticmethod + def _snapshot_recognize_cache_meta(meta: Optional[MetaBase]) -> Optional[MetaBase]: + """ + 保存共享识别前的本地缓存关键元数据，用于共享成功后回填正缓存覆盖负缓存。 + """ + if not meta: + return None + return copy.deepcopy(meta) + + def _update_local_recognize_cache( + self, + meta: Optional[MetaBase], + mediainfo: Optional[MediaInfo], + ) -> None: + """ + 共享识别成功后回填本地识别缓存，避免名称负缓存导致后续重复回查共享。 + """ + if not meta or not mediainfo: + return + self.run_module( + "update_recognize_cache", + meta=meta, + mediainfo=mediainfo, + ) + + async def _async_update_local_recognize_cache( + self, + meta: Optional[MetaBase], + mediainfo: Optional[MediaInfo], + ) -> None: + """ + 异步回填本地识别缓存。 + """ + if not meta or not mediainfo: + return + await self.async_run_module( + "async_update_recognize_cache", + meta=meta, + mediainfo=mediainfo, + ) + + @staticmethod + def _record_media_recognize_share_hit() -> None: + """记录一次共享媒体识别成功命中，统计失败不影响识别结果。""" + try: + SystemConfigOper().increment(SystemConfigKey.MediaRecognizeShareCount) + except Exception as err: + logger.error(f"记录共享媒体识别命中次数失败：{str(err)}") + + def _run_native_media_recognize( + self, + module_kwargs: dict, + cache: bool, + ) -> Optional[MediaInfo]: + """执行同步原生媒体模块识别，具体媒体领域可覆写该路由钩子。""" + with fresh(not cache): + return self.run_module("recognize_media", **module_kwargs) + + async def _async_run_native_media_recognize( + self, + module_kwargs: dict, + cache: bool, + ) -> Optional[MediaInfo]: + """执行异步原生媒体模块识别，具体媒体领域可覆写该路由钩子。""" + async with async_fresh(not cache): + return await self.async_run_module( + "async_recognize_media", **module_kwargs + ) + + def recognize_media( + self, + meta: MetaBase = None, + mtype: Optional[MediaType] = None, + media_source: Optional[MediaSource] = None, + media_id: Optional[str] = None, + episode_group: Optional[str] = None, + cache: bool = True, + share_meta: MetaBase = None, + music_type: Optional[str] = None, + ) -> Optional[MediaInfo]: + """ + 识别媒体信息，不含Fanart图片 + :param meta: 识别的元数据 + :param share_meta: 共享识别查询/上报使用的原始元数据 + :param mtype: 识别的媒体类型 + :param media_source: 请求级识别数据源 + :param media_id: 数据源原生ID，必须与media_source成对提供 + :param episode_group: 剧集组 + :param cache: 是否使用缓存 + :param music_type: 音乐实体类型，显式音乐 ID 必须据此区分单曲与专辑 + :return: 识别的媒体信息，包括剧集信息 + """ + # 仅传数据源是请求级识别源约束（按名称识别限定数据源），显式 media_id 才要求来源成对 + explicit_identity = media_id is not None + requested_source = normalize_media_source(media_source) or media_source + media_source, media_id = resolve_media_identity( + media=meta, + media_source=media_source, + media_id=media_id, + ) + if explicit_identity and (not media_source or not media_id): + logger.warning("媒体识别需要同时提供有效的 media_source 和 media_id") + return None + if not media_id and requested_source is not None: + media_source = requested_source + # meta 自带同源身份（如 {tmdbid=} 标题）时直接按身份识别，避免退化为名称搜索 + meta_source, meta_id = resolve_media_identity(media=meta) + if meta_id and meta_source == requested_source: + media_source, media_id = meta_source, meta_id + if not episode_group and hasattr(meta, "episode_group"): + episode_group = meta.episode_group + if not mtype and not (media_source and media_id) and meta and meta.type in [ + MediaType.TV, MediaType.MOVIE, MediaType.MUSIC + ]: + mtype = meta.type + share_query_meta = share_meta or meta + module_kwargs = { + "meta": meta, + "mtype": mtype, + "media_source": media_source, + "media_id": media_id, + "episode_group": episode_group, + "cache": cache, + } + if music_type is not None: + module_kwargs["music_type"] = music_type + mediainfo = self._run_native_media_recognize(module_kwargs, cache) + # 原生识别未取得远端身份时，允许插件按已知要素补充匹配媒体信息（影视与音乐统一） + mediainfo = self._supplement_media_recognize( + meta=meta, mtype=mtype, media_source=media_source, + media_id=media_id, mediainfo=mediainfo, + music_type=music_type, + ) + fallback_mediainfo = ( + mediainfo + if mediainfo and not self._media_info_has_identity(mediainfo) + else None + ) + if mediainfo and self._media_info_has_identity(mediainfo): + # 电影、电视剧、音乐统一上报；音乐的 tmdb 等字段恒为 None，身份取数据源原生 ID + if not getattr(mediainfo, "recognize_cache_hit", False): + MoviePilotServerHelper.report_recognize_share( + meta=meta, + mediainfo=mediainfo, + keyword_meta=share_query_meta, + ) + return mediainfo + + if self._can_use_media_recognize_share( + share_query_meta, media_source, media_id + ): + shared_cache_meta = self._snapshot_recognize_cache_meta(meta) + share_query_kwargs = { + "meta": meta, + "mtype": mtype, + "keyword_meta": share_query_meta, + } + if music_type is not None: + share_query_kwargs["music_type"] = music_type + shared_item = MoviePilotServerHelper.query_recognize_share( + **share_query_kwargs, + ) + shared_params = MoviePilotServerHelper.to_recognize_params(shared_item) + if shared_params: + shared_module_kwargs = { + "meta": meta, + "mtype": shared_params.get("mtype") or mtype, + "media_source": shared_params.get("media_source"), + "media_id": shared_params.get("media_id"), + "episode_group": episode_group, + "cache": cache, + } + shared_music_type = shared_params.get("music_type") or music_type + if shared_music_type is not None: + shared_module_kwargs["music_type"] = shared_music_type + mediainfo = self._run_native_media_recognize( + shared_module_kwargs, + cache, + ) + if mediainfo and self._media_info_has_identity(mediainfo): + self._update_local_recognize_cache(shared_cache_meta, mediainfo) + self._record_media_recognize_share_hit() + return mediainfo + if mediainfo and not fallback_mediainfo: + fallback_mediainfo = mediainfo + return fallback_mediainfo + + async def async_recognize_media( + self, + meta: MetaBase = None, + mtype: Optional[MediaType] = None, + media_source: Optional[MediaSource] = None, + media_id: Optional[str] = None, + episode_group: Optional[str] = None, + cache: bool = True, + share_meta: MetaBase = None, + music_type: Optional[str] = None, + ) -> Optional[MediaInfo]: + """ + 识别媒体信息，不含Fanart图片（异步版本） + :param meta: 识别的元数据 + :param share_meta: 共享识别查询/上报使用的原始元数据 + :param mtype: 识别的媒体类型 + :param media_source: 请求级识别数据源 + :param media_id: 数据源原生ID，必须与media_source成对提供 + :param episode_group: 剧集组 + :param cache: 是否使用缓存 + :param music_type: 音乐实体类型，显式音乐 ID 必须据此区分单曲与专辑 + :return: 识别的媒体信息，包括剧集信息 + """ + # 仅传数据源是请求级识别源约束（按名称识别限定数据源），显式 media_id 才要求来源成对 + explicit_identity = media_id is not None + requested_source = normalize_media_source(media_source) or media_source + media_source, media_id = resolve_media_identity( + media=meta, + media_source=media_source, + media_id=media_id, + ) + if explicit_identity and (not media_source or not media_id): + logger.warning("媒体识别需要同时提供有效的 media_source 和 media_id") + return None + if not media_id and requested_source is not None: + media_source = requested_source + # meta 自带同源身份（如 {tmdbid=} 标题）时直接按身份识别，避免退化为名称搜索 + meta_source, meta_id = resolve_media_identity(media=meta) + if meta_id and meta_source == requested_source: + media_source, media_id = meta_source, meta_id + if not episode_group and hasattr(meta, "episode_group"): + episode_group = meta.episode_group + if not mtype and not (media_source and media_id) and meta and meta.type in [ + MediaType.TV, MediaType.MOVIE, MediaType.MUSIC + ]: + mtype = meta.type + share_query_meta = share_meta or meta + module_kwargs = { + "meta": meta, + "mtype": mtype, + "media_source": media_source, + "media_id": media_id, + "episode_group": episode_group, + "cache": cache, + } + if music_type is not None: + module_kwargs["music_type"] = music_type + mediainfo = await self._async_run_native_media_recognize(module_kwargs, cache) + # 原生识别未取得远端身份时，允许插件按已知要素补充匹配媒体信息（影视与音乐统一） + mediainfo = await self._async_supplement_media_recognize( + meta=meta, mtype=mtype, media_source=media_source, + media_id=media_id, mediainfo=mediainfo, + music_type=music_type, + ) + fallback_mediainfo = ( + mediainfo + if mediainfo and not self._media_info_has_identity(mediainfo) + else None + ) + if mediainfo and self._media_info_has_identity(mediainfo): + # 电影、电视剧、音乐统一上报；音乐的 tmdb 等字段恒为 None，身份取数据源原生 ID + if not getattr(mediainfo, "recognize_cache_hit", False): + await MoviePilotServerHelper.async_report_recognize_share( + meta=meta, + mediainfo=mediainfo, + keyword_meta=share_query_meta, + ) + return mediainfo + + if self._can_use_media_recognize_share( + share_query_meta, media_source, media_id + ): + shared_cache_meta = self._snapshot_recognize_cache_meta(meta) + share_query_kwargs = { + "meta": meta, + "mtype": mtype, + "keyword_meta": share_query_meta, + } + if music_type is not None: + share_query_kwargs["music_type"] = music_type + shared_item = await MoviePilotServerHelper.async_query_recognize_share( + **share_query_kwargs, + ) + shared_params = MoviePilotServerHelper.to_recognize_params(shared_item) + if shared_params: + shared_module_kwargs = { + "meta": meta, + "mtype": shared_params.get("mtype") or mtype, + "media_source": shared_params.get("media_source"), + "media_id": shared_params.get("media_id"), + "episode_group": episode_group, + "cache": cache, + } + shared_music_type = shared_params.get("music_type") or music_type + if shared_music_type is not None: + shared_module_kwargs["music_type"] = shared_music_type + mediainfo = await self._async_run_native_media_recognize( + shared_module_kwargs, + cache, + ) + if mediainfo and self._media_info_has_identity(mediainfo): + await self._async_update_local_recognize_cache(shared_cache_meta, mediainfo) + await run_in_threadpool(self._record_media_recognize_share_hit) + return mediainfo + if mediainfo and not fallback_mediainfo: + fallback_mediainfo = mediainfo + return fallback_mediainfo + + @staticmethod + def _media_recognize_plugin_payload( + meta: Optional[MetaBase], + mtype: Optional[MediaType], + media_source: Optional[MediaSource], + media_id: Optional[str], + is_music: bool, + music_type: Optional[str] = None, + ) -> dict: + """ + 构造媒体识别链式事件的已知要素载荷，供插件匹配媒体信息；影视与音乐统一协议， + 仅要素字段随媒体类型不同 + """ + if is_music: + return { + "title": getattr(meta, "title", None), + "artists": list(getattr(meta, "artists", None) or []), + "album": getattr(meta, "album", None), + "year": getattr(meta, "year", None), + "isrc": getattr(meta, "isrc", None), + "media_source": media_source, + "media_id": media_id, + "music_type": music_type, + } + return { + "title": getattr(meta, "title", None) or getattr(meta, "name", None), + "year": getattr(meta, "year", None), + "season": getattr(meta, "begin_season", None), + "type": mtype.value if isinstance(mtype, MediaType) else None, + "media_source": media_source, + "media_id": media_id, + } + + @classmethod + def _media_info_from_plugin( + cls, + event_data: dict, + is_music: bool, + mtype: Optional[MediaType] = None, + music_type: Optional[str] = None, + ) -> Optional[MediaInfo]: + """ + 解析插件返回的媒体信息，缺少数据源或身份字段的结果不采信； + 音乐构造 MusicInfo，影视构造 MediaInfo + """ + if not isinstance(event_data, dict): + return None + plugin_info = event_data.get("mediainfo") + if not isinstance(plugin_info, dict): + return None + if not plugin_info.get("media_source"): + logger.warn("插件返回的媒体信息缺少数据源，忽略 ...") + return None + try: + if is_music: + if not plugin_info.get("media_id"): + logger.warn("插件返回的音乐媒体信息缺少媒体ID，忽略 ...") + return None + info: MediaInfo = MusicInfo.from_dict(plugin_info) + if not info.media_source or not info.media_id: + return None + if music_type and info.music_type != music_type: + logger.warn( + f"插件返回的音乐实体类型为 {info.music_type}，" + f"与请求的 {music_type} 不一致，忽略 ..." + ) + return None + return info + # 影视：插件未提供类型时使用请求推断的类型 + if not plugin_info.get("type") and mtype: + plugin_info = {**plugin_info, "type": mtype} + info = MediaInfo() + info.from_dict(plugin_info) + except Exception as err: + logger.warn(f"插件返回的媒体信息格式错误：{err}") + return None + # 影视与音乐统一要求远端身份，无身份的结果不采信，避免未验证结果进入识别管线 + if not info.media_source or not cls._media_info_has_identity(info): + logger.warn("插件返回的媒体信息缺少远端身份，忽略 ...") + return None + return info + + @staticmethod + def _media_info_has_identity(mediainfo) -> bool: + """判断媒体信息是否具备完整的规范媒体身份。""" + media_source, media_id = resolve_media_identity(media=mediainfo) + return bool(media_source and media_id) + + def _supplement_media_recognize( + self, + meta: Optional[MetaBase], + mtype: Optional[MediaType], + media_source: Optional[MediaSource], + media_id: Optional[str], + mediainfo, + music_type: Optional[str] = None, + ): + """ + 媒体识别插件补充（影视与音乐统一）：原生模块未给出带远端身份的结果时， + 广播媒体识别链式事件，允许插件（如第三方媒体源）按已知要素匹配并返回标准信息 + """ + is_music = ( + isinstance(meta, MetaMusic) + or mtype == MediaType.MUSIC + or isinstance(mediainfo, MusicInfo) + ) + # 已有远端身份时无需插件介入 + if mediainfo and self._media_info_has_identity(mediainfo): + return mediainfo + etype = ChainEventType.MusicMediaRecognize if is_music else ChainEventType.MediaRecognize + if not self.eventmanager.check(etype): + return mediainfo + result: Event = self.eventmanager.send_event( + etype, + self._media_recognize_plugin_payload( + meta, mtype, media_source, media_id, is_music, music_type + ), + ) + if not result: + return mediainfo + plugin_info = self._media_info_from_plugin( + result.event_data or {}, is_music, mtype, music_type + ) + if not plugin_info: + return mediainfo + logger.info( + f"插件补充媒体识别成功：{plugin_info.title}" + f"（{plugin_info.media_source}:{plugin_info.media_id}）" + ) + return plugin_info + + async def _async_supplement_media_recognize( + self, + meta: Optional[MetaBase], + mtype: Optional[MediaType], + media_source: Optional[MediaSource], + media_id: Optional[str], + mediainfo, + music_type: Optional[str] = None, + ): + """媒体识别插件补充的异步版本，影视与音乐统一流程""" + is_music = ( + isinstance(meta, MetaMusic) + or mtype == MediaType.MUSIC + or isinstance(mediainfo, MusicInfo) + ) + # 已有远端身份时无需插件介入 + if mediainfo and self._media_info_has_identity(mediainfo): + return mediainfo + etype = ChainEventType.MusicMediaRecognize if is_music else ChainEventType.MediaRecognize + if not self.eventmanager.check(etype): + return mediainfo + result: Event = await self.eventmanager.async_send_event( + etype, + self._media_recognize_plugin_payload( + meta, mtype, media_source, media_id, is_music, music_type + ), + ) + if not result: + return mediainfo + plugin_info = self._media_info_from_plugin( + result.event_data or {}, is_music, mtype, music_type + ) + if not plugin_info: + return mediainfo + logger.info( + f"插件补充媒体识别成功：{plugin_info.title}" + f"（{plugin_info.media_source}:{plugin_info.media_id}）" + ) + return plugin_info + diff --git a/app/chain/agent.py b/app/chain/agent.py new file mode 100644 index 000000000..11cddaa32 --- /dev/null +++ b/app/chain/agent.py @@ -0,0 +1,14 @@ +"""Agent 业务处理链。 + +AgentChain 是 agent 编排在链层的入口：Agent 运行时会话需要复用 +ChainBase 提供的消息处理状态机（渠道处理状态、直发消息等）， +因此继承关系归属链层；具体 Agent 运行时（MoviePilotAgent 等）留在 app.agent。 +""" + +from app.chain import ChainBase + + +class AgentChain(ChainBase): + """Agent 业务处理链。""" + + pass diff --git a/app/chain/message.py b/app/chain/message.py index d665e9f27..15ee57d73 100644 --- a/app/chain/message.py +++ b/app/chain/message.py @@ -11,8 +11,12 @@ from pathlib import Path from typing import Any, Optional, Dict, Union, List, Tuple from urllib.parse import unquote, urlparse -from app.agent.orchestrator import agent_manager -from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.application.agent import ( + get_agent_manager, + is_audio_input_available, + supports_image_input, + transcribe_audio, +) from app.chain import ChainBase from app.chain.download import DownloadChain from app.chain.media import MediaChain @@ -68,7 +72,7 @@ class MessageChain(ChainBase): return clear_task = None try: - clear_task = agent_manager.clear_session(session_id=session_id, user_id=str(userid)) + clear_task = get_agent_manager().clear_session(session_id=session_id, user_id=str(userid)) asyncio.run_coroutine_threadsafe( clear_task, global_vars.loop, @@ -346,7 +350,7 @@ class MessageChain(ChainBase): if not session_info: return False session_id, _ = session_info - if not agent_manager.matches_secret_confirmation( + if not get_agent_manager().matches_secret_confirmation( session_id, str(userid), channel=channel.value, @@ -966,7 +970,7 @@ class MessageChain(ChainBase): if session_id: clear_task = None try: - clear_task = agent_manager.clear_session( + clear_task = get_agent_manager().clear_session( session_id=session_id, user_id=str(userid) ) asyncio.run_coroutine_threadsafe( @@ -1015,7 +1019,7 @@ class MessageChain(ChainBase): session_id, _ = session_info try: future = asyncio.run_coroutine_threadsafe( - agent_manager.stop_current_task(session_id=session_id), + get_agent_manager().stop_current_task(session_id=session_id), global_vars.loop, ) stopped = future.result(timeout=10) @@ -1180,7 +1184,7 @@ class MessageChain(ChainBase): return session_id, _ = session_info - status = agent_manager.get_session_status(session_id=session_id) + status = get_agent_manager().get_session_status(session_id=session_id) self.post_message( Notification( channel=channel, @@ -1254,7 +1258,7 @@ class MessageChain(ChainBase): # 将可直接输入给 LLM 的附件统一转换为 data URL original_images = images all_files = list(files or []) - if images and LLMHelper.supports_image_input( + if images and supports_image_input( provider=settings.LLM_PROVIDER, model=settings.LLM_MODEL, ): @@ -1333,7 +1337,7 @@ class MessageChain(ChainBase): process_kwargs["has_audio_input"] = True # 在事件循环中处理 asyncio.run_coroutine_threadsafe( - agent_manager.process_message(**process_kwargs), + get_agent_manager().process_message(**process_kwargs), global_vars.loop, ) return True @@ -1353,7 +1357,7 @@ class MessageChain(ChainBase): """ if not audio_refs: return None - if not AgentCapabilityManager.is_audio_input_available(): + if not is_audio_input_available(): logger.warning("音频输入能力未配置或未启用，跳过语音识别") return None @@ -1460,7 +1464,7 @@ class MessageChain(ChainBase): ) continue - transcript = AgentCapabilityManager.transcribe_audio( + transcript = transcribe_audio( content=content, filename=filename ) if transcript: diff --git a/app/chain/search.py b/app/chain/search.py index 53f7c2965..9e87768bd 100644 --- a/app/chain/search.py +++ b/app/chain/search.py @@ -509,10 +509,10 @@ class SearchChain(ChainBase): """ 通过统一后台提示词机制执行资源推荐。 """ - from app.agent.orchestrator import ReplyMode, agent_manager - from app.agent.prompt import prompt_manager + from app.application.agent import get_agent_manager, get_prompt_manager + from app.schemas.agent import ReplyMode - prompt = prompt_manager.render_system_task_message( + prompt = get_prompt_manager().render_system_task_message( "search_recommend", template_context={"search_results": search_results_text}, ) @@ -521,7 +521,7 @@ class SearchChain(ChainBase): def on_output(text: str): full_output[0] = text - await agent_manager.run_background_prompt( + await get_agent_manager().run_background_prompt( message=prompt, session_prefix="__agent_search_recommend", output_callback=on_output, diff --git a/app/chain/site.py b/app/chain/site.py index 5c0246a72..f19b8af9a 100644 --- a/app/chain/site.py +++ b/app/chain/site.py @@ -1,13 +1,14 @@ import base64 import re from datetime import datetime -from typing import Callable, List, Optional, Tuple, Union, Dict +from typing import Callable, Optional, Tuple, Union, Dict from urllib.parse import urljoin from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from lxml import etree from app.chain import ChainBase +from app.chain._interaction import InteractionChainMixin from app.runtime.config import global_vars, settings from app.runtime.events import Event, eventmanager from app.db.models.site import Site @@ -17,10 +18,7 @@ from app.adapters.network.browser import PlaywrightHelper from app.adapters.network.cloudflare import under_challenge from app.application.security.cookie import CookieHelper from app.adapters.external.cookiecloud import CookieCloudHelper -from app.application.messaging.site import ( - SiteInteractionHandler, - site_interaction_manager, -) +from app.application.messaging.site import SiteInteractionHandler from app.application.rss import RssHelper from app.runtime.log import logger from app.schemas import MessageChannel, Notification, SiteUserData @@ -33,12 +31,13 @@ from app.foundation import url as url_tools from app.foundation.dom import DomUtils - -class SiteChain(ChainBase): +class SiteChain(InteractionChainMixin, ChainBase): """ 站点管理处理链 """ + # 交互处理器类注入，供 InteractionChainMixin 的 parse_callback 委托 + _interaction_handler_type = SiteInteractionHandler def __init__(self): """初始化站点管理处理链及特殊站点测试器""" @@ -752,66 +751,6 @@ class SiteChain(ChainBase): """构造 /sites 交互处理器，Cookie 更新动作由本链提供。""" return SiteInteractionHandler(messenger=self, cookie_updater=self.update_cookie) - def remote_list( - self, - arg_str: str = "", - channel: MessageChannel = None, - userid: Union[str, int] = None, - source: Optional[str] = None, - ): - """ - /sites 统一入口，委托交互处理器。 - """ - return self._interaction_handler().remote_list( - arg_str=arg_str, channel=channel, userid=userid, source=source - ) - - @staticmethod - def parse_callback(callback_data: str) -> Optional[Tuple[str, str]]: - """ - 解析 /sites 按钮回调。 - """ - return SiteInteractionHandler.parse_callback(callback_data) - - def handle_callback_interaction( - self, - callback_data: str, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - original_message_id: Optional[Union[str, int]] = None, - original_chat_id: Optional[str] = None, - ) -> bool: - """委托交互处理器处理按钮回调。""" - return self._interaction_handler().handle_callback_interaction( - callback_data=callback_data, - channel=channel, - source=source, - userid=userid, - username=username, - original_message_id=original_message_id, - original_chat_id=original_chat_id, - ) - - def handle_text_interaction( - self, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - text: str, - ) -> bool: - """委托交互处理器处理文本输入。""" - return self._interaction_handler().handle_text_interaction( - channel=channel, - source=source, - userid=userid, - username=username, - text=text, - ) - - def remote_disable(self, arg_str: str, channel: MessageChannel, userid: Union[str, int] = None, source: Optional[str] = None): """ diff --git a/app/chain/subscribe.py b/app/chain/subscribe.py index 60cb3f923..578b739b7 100644 --- a/app/chain/subscribe.py +++ b/app/chain/subscribe.py @@ -1,7 +1,6 @@ import copy import json import random -import re import threading import time from datetime import datetime @@ -9,6 +8,8 @@ from typing import Any, Callable, Dict, List, Optional, Union, Tuple from app import schemas from app.chain import ChainBase +from app.chain._interaction import InteractionChainMixin +from app.chain._music import MusicSubscribeMixin from app.chain.download import DownloadChain from app.chain.media import MediaChain from app.chain.mediaserver import MediaServerChain @@ -19,7 +20,6 @@ from app.runtime.config import settings, global_vars from app.domain.context import ( Context, MediaInfo, - MusicInfo, TorrentInfo, ) from app.runtime.events import eventmanager, Event @@ -32,10 +32,7 @@ from app.db.models.subscribe import Subscribe from app.db.oper.site import SiteOper from app.db.oper.subscribe import SubscribeOper from app.db.oper.systemconfig import SystemConfigOper -from app.application.messaging.subscribe import ( - SubscribeInteractionHandler, - subscribe_interaction_manager, -) +from app.application.messaging.subscribe import SubscribeInteractionHandler from app.application.mediaserver import MediaServerHelper from app.application.subscribe import add_subscribe, async_add_subscribe from app.adapters.external.server import MoviePilotServerHelper @@ -43,22 +40,11 @@ from app.application.torrent import TorrentHelper from app.runtime.log import logger from app.schemas import (SubscribeEpisodesRefreshEventData, SubscribeCompletionCheckEventData) -from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaSource, MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType, ChainEventType, \ +from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType, ChainEventType, \ ContentType -from app.domain.media import MUSIC_SUBSCRIBABLE_TYPES from app.schemas.media import build_media_key, normalize_media_source, resolve_media_identity - -def _normalize_music_total_tracks(value: Any) -> Optional[int]: - """将专辑曲目总数归一为正整数，无效或未知值返回 None。""" - try: - total_tracks = int(value or 0) - except (TypeError, ValueError): - return None - return total_tracks if total_tracks > 0 else None - - def build_subscribe_meta(subscribe: Subscribe) -> MetaBase: """ 按订阅对象构造主程序链路共用的媒体元数据。 @@ -116,7 +102,7 @@ def _subscribe_media_keys(subscribe: Subscribe) -> List[Union[str, int]]: return [candidate for candidate in candidates if candidate not in (None, "")] -class SubscribeChain(ChainBase): +class SubscribeChain(MusicSubscribeMixin, InteractionChainMixin, ChainBase): """ 订阅管理处理链。 @@ -133,6 +119,9 @@ class SubscribeChain(ChainBase): 电影下载优先级 writer 单独维护。 """ + # 交互处理器类注入，供 InteractionChainMixin 的 parse_callback 委托 + _interaction_handler_type = SubscribeInteractionHandler + _rlock = threading.RLock() # 避免莫名原因导致长时间持有锁 _LOCK_TIMOUT = 3600 * 2 @@ -1261,378 +1250,6 @@ class SubscribeChain(ChainBase): return True return False - @staticmethod - def _validate_music_subscribe_target( - mediainfo: MediaInfo, - requested_music_type: Optional[str] = None, - ) -> Optional[str]: - """校验音乐订阅实体一致性，并确保专辑具备可验证的曲目总数。""" - if mediainfo.type != MediaType.MUSIC: - return "识别结果不是音乐" - music_type = getattr(mediainfo, "music_type", None) - if requested_music_type and requested_music_type not in MUSIC_SUBSCRIBABLE_TYPES: - return "音乐订阅仅支持单曲或专辑" - if music_type not in MUSIC_SUBSCRIBABLE_TYPES: - return "音乐订阅仅支持单曲或专辑" - if requested_music_type and requested_music_type != music_type: - return f"音乐订阅类型不匹配：请求 {requested_music_type}，识别为 {music_type}" - if music_type == MUSIC_ENTITY_ALBUM \ - and _normalize_music_total_tracks(getattr(mediainfo, "total_tracks", None)) is None: - return "专辑总曲目数未知，无法校验整张专辑资源" - return None - - @staticmethod - def _ensure_music_subscribe_entity( - subscribe: Subscribe, - mediainfo: Optional[MusicInfo], - ) -> Optional[MusicInfo]: - """保持已持久化的单曲/专辑实体边界，拒绝远端详情把订阅类型改写。""" - if not mediainfo: - return None - expected_type = getattr(subscribe, "music_type", None) - actual_type = getattr(mediainfo, "music_type", None) - if expected_type and expected_type not in MUSIC_SUBSCRIBABLE_TYPES: - logger.warning(f"音乐订阅 {subscribe.name} 的实体类型无效：{expected_type}") - return None - if actual_type not in MUSIC_SUBSCRIBABLE_TYPES: - logger.warning( - f"音乐订阅 {subscribe.name} 识别为不可订阅实体：{actual_type}" - ) - if expected_type in MUSIC_SUBSCRIBABLE_TYPES: - return SubscribeChain._music_info_from_subscribe(subscribe) - return None - if expected_type and actual_type != expected_type: - logger.warning( - f"音乐订阅 {subscribe.name} 实体不匹配：" - f"订阅为 {expected_type}，远端识别为 {actual_type}，使用订阅快照" - ) - return SubscribeChain._music_info_from_subscribe(subscribe) - if actual_type == MUSIC_ENTITY_ALBUM: - remote_total = _normalize_music_total_tracks(getattr(mediainfo, "total_tracks", None)) - stored_total = _normalize_music_total_tracks(getattr(subscribe, "total_tracks", None)) - resolved_total = remote_total or stored_total - if resolved_total is not None and mediainfo.total_tracks != resolved_total: - # 识别模块结果可能来自共享缓存，补齐订阅快照时不得原地修改。 - mediainfo = copy.copy(mediainfo) - mediainfo.total_tracks = resolved_total - return mediainfo - - @staticmethod - def _recognize_music_subscribe(subscribe: Subscribe) -> Optional[MusicInfo]: - """按订阅身份恢复音乐目标，远端暂不可用时使用已持久化的稳定快照。""" - if subscribe.media_source and subscribe.media_id: - # 与影视共用统一识别入口，按媒体源和原生 ID 恢复音乐详情 - mediainfo = MediaChain().recognize_media( - media_source=subscribe.media_source, - media_id=str(subscribe.media_id), - mtype=MediaType.MUSIC, - music_type=getattr(subscribe, "music_type", None), - ) - if mediainfo: - return SubscribeChain._ensure_music_subscribe_entity(subscribe, mediainfo) - if getattr(subscribe, "music_type", None) in {MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_ALBUM}: - return SubscribeChain._music_info_from_subscribe(subscribe) - # 旧订阅没有保存实体类型时不能猜测为单曲，否则可能误把专辑按单曲完成。 - return None - if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: - # 缺少远端 ID 的专辑不能退化为单曲识别，使用已保存专辑快照更可靠。 - return SubscribeChain._music_info_from_subscribe(subscribe) - # 旧订阅没有实体类型时只允许走 Recording 识别，不能从全局混合搜索中猜成专辑或艺术家。 - mediainfo = MediaChain().recognize_media( - meta=build_subscribe_meta(subscribe), - mtype=MediaType.MUSIC, - media_source=subscribe.media_source, - music_type=MUSIC_ENTITY_RECORDING, - ) - return SubscribeChain._ensure_music_subscribe_entity(subscribe, mediainfo) - - @staticmethod - async def _async_recognize_music_subscribe(subscribe: Subscribe) -> Optional[MusicInfo]: - """异步按订阅身份恢复音乐目标，远端暂不可用时使用已持久化的稳定快照。""" - if subscribe.media_source and subscribe.media_id: - # 与影视共用统一识别入口，按媒体源和原生 ID 恢复音乐详情 - mediainfo = await MediaChain().async_recognize_media( - media_source=subscribe.media_source, - media_id=str(subscribe.media_id), - mtype=MediaType.MUSIC, - music_type=getattr(subscribe, "music_type", None), - ) - if mediainfo: - return SubscribeChain._ensure_music_subscribe_entity(subscribe, mediainfo) - if getattr(subscribe, "music_type", None) in {MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_ALBUM}: - return SubscribeChain._music_info_from_subscribe(subscribe) - return None - if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: - return SubscribeChain._music_info_from_subscribe(subscribe) - mediainfo = await MediaChain().async_recognize_media( - meta=build_subscribe_meta(subscribe), - mtype=MediaType.MUSIC, - media_source=subscribe.media_source, - music_type=MUSIC_ENTITY_RECORDING, - ) - return SubscribeChain._ensure_music_subscribe_entity(subscribe, mediainfo) - - @staticmethod - def _music_info_from_subscribe(subscribe: Subscribe) -> MusicInfo: - """从订阅行恢复不依赖远端请求的最小音乐目标，保留专辑完成判断所需字段。""" - year_text = str(subscribe.year or "")[:4] - music_type = getattr(subscribe, "music_type", None) - # 音乐订阅的 description 由标准 MusicInfo.overview 生成，首段固定为艺术家。 - artist_text = str(getattr(subscribe, "description", None) or "") \ - .split(" · ", maxsplit=1)[0].strip() - artists = [ - artist.strip() for artist in artist_text.split(" / ") if artist.strip() - ] - return MusicInfo( - media_source=subscribe.media_source, - media_id=str(subscribe.media_id) if subscribe.media_id is not None else None, - music_type=music_type, - title=subscribe.name, - artists=artists, - album=subscribe.name if music_type == MUSIC_ENTITY_ALBUM else None, - year=int(year_text) if year_text.isdigit() else None, - total_tracks=getattr(subscribe, "total_tracks", None) - if music_type == MUSIC_ENTITY_ALBUM else None, - cover_url=getattr(subscribe, "poster", None) or getattr(subscribe, "backdrop", None), - ) - - @staticmethod - def _sync_music_subscribe_target(subscribe: Subscribe, mediainfo: MusicInfo) -> None: - """把远端识别得到的专辑类型和总曲目数同步到订阅，供搜索失败与完成历史复用。""" - update_data = {} - if mediainfo.music_type and getattr(subscribe, "music_type", None) != mediainfo.music_type: - update_data["music_type"] = mediainfo.music_type - if mediainfo.music_type == MUSIC_ENTITY_ALBUM: - # 远端详情可能暂时不返回曲目数；已确认的订阅快照不能因此被清空。 - total_tracks = _normalize_music_total_tracks(mediainfo.total_tracks) \ - or _normalize_music_total_tracks(getattr(subscribe, "total_tracks", None)) - else: - total_tracks = None - if getattr(subscribe, "total_tracks", None) != total_tracks: - update_data["total_tracks"] = total_tracks - if not update_data: - return - SubscribeOper().update(subscribe.id, update_data) - for key, value in update_data.items(): - setattr(subscribe, key, value) - - @staticmethod - def _is_music_download_complete( - subscribe: Subscribe, - mediainfo: MusicInfo, - downloads: Optional[List[Context]], - ) -> bool: - """判断音乐下载是否满足订阅完成条件；专辑必须由下载层确认整专曲目覆盖。""" - if not downloads: - return False - music_type = getattr(subscribe, "music_type", None) or mediainfo.music_type - if music_type != MUSIC_ENTITY_ALBUM: - return True - return any(context.confirmed_full_coverage for context in downloads) - - def _prepare_music_subscribe( - self, - subscribe: Subscribe, - ) -> Optional[Tuple[MusicInfo, MetaMusic]]: - """识别音乐订阅目标、同步实体快照，并在搜索前处理已完整入库的目标。""" - mediainfo = self._recognize_music_subscribe(subscribe) - if not mediainfo: - logger.warning( - f"未识别到音乐订阅目标：{subscribe.name}，" - f"媒体源：{subscribe.media_source}，媒体ID：{subscribe.media_id}" - ) - return None - validation_error = self._validate_music_subscribe_target( - mediainfo, - getattr(subscribe, "music_type", None), - ) - if validation_error: - logger.warning(f"音乐订阅 {subscribe.name} 无法继续：{validation_error}") - return None - self._sync_music_subscribe_target(subscribe, mediainfo) - meta = MetaMusic.from_music_info(mediainfo) - exists, _ = self.check_and_handle_existing_media( - subscribe=subscribe, - meta=meta, - mediainfo=mediainfo, - mediakey=_subscribe_media_key(subscribe), - ) - if exists: - return None - return mediainfo, meta - - def _filter_music_subscribe_contexts( - self, - subscribe: Subscribe, - mediainfo: MusicInfo, - contexts: List[Context], - ) -> List[Context]: - """按站点、音乐实体、订阅参数和优先级规则筛选并绑定下载上下文。""" - sites = self.get_sub_sites(subscribe) - default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \ - if subscribe.best_version else SystemConfigKey.SubscribeFilterRuleGroups - rule_groups = subscribe.filter_groups or SystemConfigOper().get(default_rule_key) or [] - torrent_helper = TorrentHelper() - matched: List[Context] = [] - for source_context in contexts or []: - source_torrent = source_context.torrent_info - if not source_torrent or source_torrent.category not in (MediaType.MUSIC, MediaType.MUSIC.value): - continue - # 过滤模块会就地写入 pri_order；RSS 缓存会被多个订阅复用，必须隔离候选副本。 - torrent = copy.copy(source_torrent) - if sites and torrent.site not in sites: - continue - if not SearchChain.matches_music_resource( - mediainfo, - torrent.title, - torrent.description, - ): - continue - if not torrent_helper.filter_torrent(torrent, self.get_params(subscribe)): - continue - filtered = self.filter_torrents( - rule_groups=rule_groups, - torrent_list=[torrent], - mediainfo=mediainfo, - ) - if filtered is not None: - if not filtered: - continue - torrent = filtered[0] - - context = copy.copy(source_context) - context.torrent_info = torrent - meta = MetaMusic.from_music_info(mediainfo) - meta.org_string = torrent.title - meta.apply_audio_quality(f"{torrent.title} {torrent.description or ''}", overwrite=True) - if subscribe.best_version: - # 用户规则组可用格式、码率等内置规则定义洗版顺序；未命中规则 - # 优先级时再回退到规范化音质分数，确保零配置也能自动升级。 - music_priority = torrent.pri_order or meta.audio_quality_score - if music_priority <= (subscribe.current_priority or 0): - logger.info( - f"{torrent.title} 音质优先级 {music_priority} " - f"未高于当前版本 {subscribe.current_priority or 0}" - ) - continue - torrent.pri_order = music_priority - context.meta_info = meta - context.media_info = mediainfo - context.match_source = str(mediainfo.media_source or "title") - context.candidate_recognized = False - context.media_info_is_target = True - if subscribe.media_category: - context.media_info.category = subscribe.media_category - matched.append(context) - return matched - - def _download_music_subscribe( - self, - subscribe: Subscribe, - mediainfo: MusicInfo, - contexts: List[Context], - ) -> None: - """批量择优下载音乐候选，并按单曲或整专完成语义推进订阅。""" - if not contexts: - return - downloads, _ = DownloadChain().batch_download( - contexts=contexts, - username=subscribe.username, - save_path=subscribe.save_path, - downloader=subscribe.downloader, - source=self.get_subscribe_source_keyword(subscribe), - custom_words=subscribe.custom_words, - ) - successful = [ - context for context in downloads or [] - if context and context.meta_info and context.torrent_info - ] - quality_downloads = successful - if getattr(subscribe, "music_type", None) == MUSIC_ENTITY_ALBUM: - quality_downloads = [ - context for context in successful - if context.confirmed_full_coverage - ] - if subscribe.best_version and quality_downloads: - best_context = max(quality_downloads, key=lambda item: item.torrent_info.pri_order) - best_meta = best_context.meta_info - quality_data = { - "current_priority": best_context.torrent_info.pri_order, - "current_audio_format": best_meta.audio_format, - "current_bitrate": best_meta.bitrate, - "current_bit_depth": best_meta.bit_depth, - "current_sample_rate": best_meta.sample_rate, - } - SubscribeOper().update(subscribe.id, quality_data) - for key, value in quality_data.items(): - setattr(subscribe, key, value) - current_subscribe = SubscribeOper().get(subscribe.id) - if current_subscribe: - self.finish_subscribe_or_not( - subscribe=current_subscribe, - meta=MetaMusic.from_music_info(mediainfo), - mediainfo=mediainfo, - downloads=downloads, - ) - - def _search_music_subscribe(self, subscribe: Subscribe) -> None: - """复用站点标题搜索、订阅过滤和批量下载完成单个音乐订阅。""" - target = self._prepare_music_subscribe(subscribe) - if not target: - return - mediainfo, _ = target - - sites = self.get_sub_sites(subscribe) - default_rule_key = SystemConfigKey.BestVersionFilterRuleGroups \ - if subscribe.best_version else SystemConfigKey.SubscribeFilterRuleGroups - rule_groups = subscribe.filter_groups or SystemConfigOper().get(default_rule_key) or [] - keywords = [subscribe.keyword] if subscribe.keyword else SearchChain.music_site_keywords(mediainfo) - if not keywords: - keywords = [subscribe.name] - - searchchain = SearchChain() - contexts: List[Context] = [] - for keyword in keywords: - contexts = searchchain.search_by_title( - title=keyword, - sites=sites, - mtype=MediaType.MUSIC, - rule_groups=rule_groups, - ) - contexts = self._filter_music_subscribe_contexts( - subscribe=subscribe, - mediainfo=mediainfo, - contexts=contexts, - ) - if contexts: - break - - if not contexts: - logger.warning(f"音乐订阅 {subscribe.keyword or subscribe.name} 未搜索到符合条件的资源") - return - - self._download_music_subscribe(subscribe, mediainfo, contexts) - - def _match_music_subscribe( - self, - subscribe: Subscribe, - contexts: List[Context], - ) -> None: - """直接匹配本轮 RSS 缓存中的音乐资源，避免再次调用站点搜索接口。""" - target = self._prepare_music_subscribe(subscribe) - if not target: - return - mediainfo, _ = target - matched = self._filter_music_subscribe_contexts( - subscribe=subscribe, - mediainfo=mediainfo, - contexts=contexts, - ) - if not matched: - logger.info(f"音乐订阅 {subscribe.name} 未匹配到符合条件的 RSS 资源") - return - self._download_music_subscribe(subscribe, mediainfo, matched) - def search( self, sid: Optional[int] = None, @@ -3247,66 +2864,6 @@ class SubscribeChain(ChainBase): """构造 /subscribes 交互处理器，业务动作由本链提供。""" return SubscribeInteractionHandler(messenger=self, actions=self) - def remote_list( - self, - arg_str: str = "", - channel: MessageChannel = None, - userid: Union[str, int] = None, - source: Optional[str] = None, - ): - """ - /subscribes 统一入口，委托交互处理器。 - """ - return self._interaction_handler().remote_list( - arg_str=arg_str, channel=channel, userid=userid, source=source - ) - - @staticmethod - def parse_callback(callback_data: str) -> Optional[Tuple[str, str]]: - """ - 解析 /subscribes 按钮回调。 - """ - return SubscribeInteractionHandler.parse_callback(callback_data) - - def handle_callback_interaction( - self, - callback_data: str, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - original_message_id: Optional[Union[str, int]] = None, - original_chat_id: Optional[str] = None, - ) -> bool: - """委托交互处理器处理按钮回调。""" - return self._interaction_handler().handle_callback_interaction( - callback_data=callback_data, - channel=channel, - source=source, - userid=userid, - username=username, - original_message_id=original_message_id, - original_chat_id=original_chat_id, - ) - - def handle_text_interaction( - self, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - text: str, - ) -> bool: - """委托交互处理器处理文本输入。""" - return self._interaction_handler().handle_text_interaction( - channel=channel, - source=source, - userid=userid, - username=username, - text=text, - ) - - def remote_delete(self, arg_str: str, channel: MessageChannel, userid: Union[str, int] = None, source: Optional[str] = None): """ diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 2c4df67e1..2c3c86b76 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -6,16 +6,11 @@ import traceback import uuid from copy import deepcopy from pathlib import Path -from time import monotonic from typing import List, Optional, Tuple, Union, Dict, Callable, Any -from app import schemas -from app.agent.orchestrator import ReplyMode, agent_manager, prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt from app.chain import ChainBase from app.chain.media import MediaChain from app.chain.storage import StorageChain -from app.chain.subscribe import SubscribeChain from app.chain.tmdb import TmdbChain from app.runtime.config import settings, global_vars from app.domain.context import MediaInfo, MusicInfo @@ -24,19 +19,17 @@ from app.domain.meta.metabase import MetaBase from app.domain.meta.metamusic import MetaMusic from app.domain.metainfo import MetaInfoPath from app.db.oper.downloadhistory import DownloadHistoryOper -from app.db.models.downloadhistory import DownloadHistory, DownloadFiles -from app.db.models.transferhistory import TransferHistory +from app.db.models.downloadhistory import DownloadHistory from app.db.oper.systemconfig import SystemConfigOper from app.db.oper.transferpending import TransferPendingOper from app.db.oper.transferhistory import TransferHistoryOper from app.application.directory import DirectoryHelper -from app.application.audio import AudioMetadataHelper -from app.application.formatting import EpisodeFormatRuleHelper, FormatParser +from app.application.formatting import FormatParser from app.runtime.progress import ProgressHelper from app.application.history import (add_transfer_fail, add_transfer_success, - clear_transfer_failures, describe_history_gate, - evaluate_history_gate, is_skip_action, - record_transfer_failure, resolve_history) + clear_transfer_failures, describe_history_gate, + evaluate_history_gate, is_skip_action, + record_transfer_failure) from app.runtime.log import logger from app.schemas import StorageOperSelectionEventData from app.schemas import ( @@ -46,7 +39,6 @@ from app.schemas import ( FileItem, TransferDirectoryConf, TransferJob, - TransferJobTask, TmdbEpisode, ) from app.schemas.exception import OperationInterrupted @@ -60,931 +52,27 @@ from app.schemas.types import ( SystemConfigKey, ChainEventType, ContentType, - MUSIC_ENTITY_ALBUM, - MUSIC_ENTITY_RECORDING, MediaSource, ) from app.runtime.reload import ConfigReloadMixin -from app.application.transfer import TransferQueue, TransferTask -from app.domain.media import normalize_music_type -from app.schemas.media import normalize_media_source, resolve_media_identity +from app.application.transfer import (FailedRetryScheduler, JobManager, + TransferQueue, TransferTask, job_lock) +from app.chain._mixins import (EpisodeFormatMixin, FailedRetryMixin, + FileFilterMixin, FileKeyMixin, + HistoryMatchMixin, ManualHistoryMixin, + ScrapeBatchMixin) +from app.schemas.media import resolve_media_identity from app.foundation.singleton import Singleton from app.domain import episode as episode_rules -from app.foundation import text as text_tools -from app.adapters.system.host import SystemUtils # 下载器锁 downloader_lock = threading.Lock() -# 作业锁 -job_lock = threading.Lock() # 任务锁 task_lock = threading.Lock() -# 字幕文件常见的语言/默认/强制标记，整理同名字幕时只允许剥离这些字幕专属尾缀。 -SUBTITLE_STEM_TAGS = { - "cc", - "chi", - "chs", - "cht", - "cn", - "default", - "en", - "eng", - "english", - "forced", - "gb", - "gb2312", - "hk", - "ja", - "jap", - "japanese", - "jp", - "jpn", - "sc", - "sdh", - "tc", - "zh", - "zh-cn", - "zh-hans", - "zh-hant", - "zh-tw", - "zh_cn", - "zh_hans", - "zh_hant", - "zh_tw", - "zho", - "中英", - "中字", - "双语", - "简中", - "简体", - "繁中", - "繁体", -} - -class JobManager: - """ - 作业管理器 - task任务负责一个文件的整理，job作业负责一个媒体的整理 - """ - - # 整理中的作业 - _job_view: Dict[Tuple, TransferJob] = {} - # 汇总季集清单 - _season_episodes: Dict[Tuple, List[int]] = {} - # 记录从 meta 作业迁移到 media 作业的关系，用于清理提前失败后残留的 media 作业 - _meta_to_media_ids: Dict[Tuple, set[Tuple]] = {} - # 记录任务最近一次状态心跳，供外部异步接管任务的失活检测使用 - _task_state_changed_at: Dict[Tuple[str, str], float] = {} - # 记录仍由主程序整理线程直接执行的任务，避免把阻塞中的本地任务误判为失活 - _active_executions: set[Tuple[str, str]] = set() - - def __init__(self): - self._job_view = {} - self._season_episodes = {} - self._meta_to_media_ids = {} - self._task_state_changed_at = {} - self._active_executions = set() - - @staticmethod - def __get_meta_id(meta: MetaBase = None, season: Optional[int] = None) -> Tuple: - """ - 获取元数据ID - """ - return meta.name, season - - @staticmethod - def __get_media_id(media: Optional[Union[MediaInfo, MusicInfo]] = None, - season: Optional[int] = None) -> Tuple: - """ - 获取媒体ID；音乐额外区分实体类型，并为无远端ID的曲目构造稳定身份。 - """ - if not media: - return None, season - source, media_id = resolve_media_identity(media=media) - if getattr(media, "type", None) == MediaType.MUSIC: - music_type = normalize_music_type( - getattr(media, "music_type", None), - ) or MUSIC_ENTITY_RECORDING - if source and media_id: - return "music", source, media_id, music_type - - artists = tuple( - text_tools.normalize_upper(artist) - for artist in (getattr(media, "artists", None) or []) - if text_tools.normalize_upper(artist) - ) - if music_type == MUSIC_ENTITY_ALBUM: - album_artist = text_tools.normalize_upper( - getattr(media, "album_artist", None) - or (artists[0] if artists else "") - ) - album = text_tools.normalize_upper( - getattr(media, "album", None) or getattr(media, "title", None) or "" - ) - return "music", "local", music_type, album_artist, album, getattr(media, "year", None) - - return ( - "music", - "local", - music_type, - artists, - text_tools.normalize_upper(getattr(media, "title", None) or ""), - text_tools.normalize_upper(getattr(media, "album", None) or ""), - getattr(media, "disc_number", None), - getattr(media, "track_number", None), - ) - return (source, media_id), season - - @staticmethod - def __get_file_key(fileitem: FileItem) -> Optional[Tuple[str, str]]: - """ - 获取源文件唯一键，用于跨媒体作业识别同一个整理任务。 - """ - if not fileitem or not fileitem.path: - return None - normalized_path = ( - Path(str(fileitem.path).replace("\\", "/")).as_posix().rstrip("/") or "/" - ) - return fileitem.storage or "local", normalized_path - - def __get_id(self, task: TransferTask = None) -> Tuple: - """ - 获取作业ID - """ - if task.mediainfo: - return self.__get_media_id( - media=task.mediainfo, season=task.meta.begin_season - ) - else: - return self.__get_meta_id(meta=task.meta, season=task.meta.begin_season) - - def get_job_id(self, task: TransferTask) -> Tuple: - """返回任务当前所属的稳定作业身份，供作业级附加状态隔离使用。""" - return self.__get_id(task) - - @staticmethod - def __get_media(task: TransferTask) -> Union[schemas.MediaInfo, schemas.MusicInfo]: - """ - 获取媒体信息 - """ - if task.mediainfo: - # 有媒体信息 - mediainfo = deepcopy(task.mediainfo) - mediainfo.clear() - if isinstance(mediainfo, MusicInfo): - return schemas.MusicInfo(**mediainfo.to_dict()) - return schemas.MediaInfo(**mediainfo.to_dict()) - else: - # 没有媒体信息 - meta: MetaBase = task.meta - if isinstance(meta, MetaMusic): - # 未识别的音乐按已解析元数据兜底展示；音乐年份为 int， - # 不能复用 MediaInfo（year 为 str），否则触发 pydantic 校验异常 - return schemas.MusicInfo( - title=meta.name, - artists=list(meta.artists or []), - artist=meta.artist, - album=meta.album, - album_artist=meta.album_artist, - year=meta.year, - title_year=f"{meta.name} ({meta.year})" if meta.year else meta.name, - media_source=meta.media_source, - media_id=meta.media_id, - ) - return schemas.MediaInfo( - title=meta.name, - year=meta.year, - title_year=f"{meta.name} ({meta.year})", - type=meta.type.value if meta.type else None, - ) - - @staticmethod - def __get_meta(task: TransferTask) -> schemas.MetaInfo: - """ - 获取元数据 - """ - if isinstance(task.meta, MetaMusic): - return schemas.MusicMeta(**task.meta.to_dict()) - return schemas.MetaInfo(**task.meta.to_dict()) - - def add_task(self, task: TransferTask, state: Optional[str] = "waiting") -> bool: - """ - 添加整理任务，自动分组到对应的作业中 - :return: True表示任务已添加，False表示任务无效或已存在（重复） - """ - if not all([task, task.meta, task.fileitem]): - return False - file_key = self.__get_file_key(task.fileitem) - if not file_key: - return False - with job_lock: - __mediaid__ = self.__get_id(task) - # 同一个源文件可能在识别前后落入不同作业，必须跨作业去重。 - if any( - self.__get_file_key(t.fileitem) == file_key - for job in self._job_view.values() - for t in job.tasks - ): - logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加") - return False - if __mediaid__ not in self._job_view: - self._job_view[__mediaid__] = TransferJob( - media=self.__get_media(task), - season=task.meta.begin_season, - tasks=[ - TransferJobTask( - fileitem=task.fileitem, - meta=self.__get_meta(task), - downloader=task.downloader, - download_hash=task.download_hash, - state=state, - ) - ], - ) - else: - # 不重复添加任务 - if any( - [ - self.__get_file_key(t.fileitem) == file_key - for t in self._job_view[__mediaid__].tasks - ] - ): - logger.debug(f"任务 {task.fileitem.name} 已存在，跳过重复添加") - return False - self._job_view[__mediaid__].tasks.append( - TransferJobTask( - fileitem=task.fileitem, - meta=self.__get_meta(task), - downloader=task.downloader, - download_hash=task.download_hash, - state=state, - ) - ) - self._task_state_changed_at[file_key] = monotonic() - # 添加季集信息 - if self._season_episodes.get(__mediaid__): - self._season_episodes[__mediaid__].extend(task.meta.episode_list) - self._season_episodes[__mediaid__] = list( - set(self._season_episodes[__mediaid__]) - ) - else: - self._season_episodes[__mediaid__] = task.meta.episode_list - return True - - def migrate_task(self, task: TransferTask) -> bool: - """ - 将任务从 meta 作业迁移到 media 作业 - """ - curr_task, source_job_id = self.__remove_task_with_job_id( - task.fileitem, preserve_execution=True - ) - if not self.add_task(task, state=curr_task.state if curr_task else "waiting"): - return False - if curr_task and task.mediainfo: - metaid = self.__get_meta_id( - meta=task.meta, season=task.meta.begin_season - ) - mediaid = self.__get_id(task) - if source_job_id == metaid and mediaid != metaid: - with job_lock: - self._meta_to_media_ids.setdefault(metaid, set()).add(mediaid) - return True - - def __is_job_done(self, job_id: Tuple) -> bool: - """ - 检查指定作业是否已完成 - """ - if job_id not in self._job_view: - return True - return all( - task.state in ["completed", "failed"] - for task in self._job_view[job_id].tasks - ) - - def __pop_job(self, job_id: Tuple): - """ - 移除指定作业和对应季集缓存 - """ - job = self._job_view.pop(job_id, None) - self._season_episodes.pop(job_id, None) - if not job: - return - for task in job.tasks: - file_key = self.__get_file_key(task.fileitem) - if file_key: - self._task_state_changed_at.pop(file_key, None) - self._active_executions.discard(file_key) - - def __remove_done_job_groups(self, job_ids: set[Tuple]): - """ - 清理已进入终态的独立作业或关联作业组。 - """ - candidates = set(job_ids) - for metaid, mediaids in list(self._meta_to_media_ids.items()): - related_ids = {metaid, *mediaids} - if not related_ids.intersection(candidates): - continue - if all(self.__is_job_done(job_id) for job_id in related_ids): - for job_id in related_ids: - self.__pop_job(job_id) - self._meta_to_media_ids.pop(metaid, None) - candidates.difference_update(related_ids) - - referenced_ids = { - job_id - for metaid, mediaids in self._meta_to_media_ids.items() - for job_id in {metaid, *mediaids} - } - for job_id in candidates - referenced_ids: - if self.__is_job_done(job_id): - self.__pop_job(job_id) - - def start_execution(self, task: TransferTask): - """ - 标记任务仍由主程序整理线程直接执行。 - - :param task: 整理任务 - """ - if not task or not task.fileitem: - return - file_key = self.__get_file_key(task.fileitem) - if not file_key: - return - with job_lock: - self._active_executions.add(file_key) - - def finish_execution(self, task: TransferTask): - """ - 结束主程序整理线程对任务的直接执行标记。 - - :param task: 整理任务 - """ - if not task or not task.fileitem: - return - file_key = self.__get_file_key(task.fileitem) - if not file_key: - return - with job_lock: - self._active_executions.discard(file_key) - - def expire_stale_running_tasks( - self, timeout_seconds: int - ) -> List[Tuple[FileItem, int]]: - """ - 将外部接管后长期无心跳的运行中任务标记失败并清理作业视图。 - - 主程序整理线程仍在直接执行的任务不会被清理，以免把阻塞中的真实任务 - 误报为已终止。外部接管方可重复调用 ``running_task`` 刷新状态心跳。 - - :param timeout_seconds: 失活超时秒数，小于等于 0 时禁用 - :return: 已失活任务及其无心跳秒数 - """ - if timeout_seconds <= 0: - return [] - - current_time = monotonic() - expired: List[Tuple[FileItem, int]] = [] - affected_job_ids: set[Tuple] = set() - with job_lock: - for mediaid, job in self._job_view.items(): - for task in job.tasks: - file_key = self.__get_file_key(task.fileitem) - if ( - not file_key - or task.state != "running" - or file_key in self._active_executions - ): - continue - updated_at = self._task_state_changed_at.get(file_key, current_time) - inactive_seconds = current_time - updated_at - if inactive_seconds < timeout_seconds: - continue - task.state = "failed" - self._task_state_changed_at[file_key] = current_time - episodes = getattr(task.meta, "episode_list", None) or [] - if mediaid in self._season_episodes: - self._season_episodes[mediaid] = list( - set(self._season_episodes[mediaid]) - set(episodes) - ) - expired.append((task.fileitem, int(inactive_seconds))) - affected_job_ids.add(mediaid) - - self.__remove_done_job_groups(affected_job_ids) - return expired - - def running_task(self, task: TransferTask): - """ - 设置任务为运行中，并刷新外部异步任务的状态心跳。 - """ - with job_lock: - __mediaid__ = self.__get_id(task) - if __mediaid__ not in self._job_view: - return - # 更新状态 - for t in self._job_view[__mediaid__].tasks: - if t.fileitem == task.fileitem: - t.state = "running" - file_key = self.__get_file_key(t.fileitem) - if file_key: - self._task_state_changed_at[file_key] = monotonic() - break - - def finish_task(self, task: TransferTask): - """ - 设置任务为完成/成功 - """ - with job_lock: - __mediaid__ = self.__get_id(task) - if __mediaid__ not in self._job_view: - return - # 更新状态 - for t in self._job_view[__mediaid__].tasks: - if t.fileitem == task.fileitem: - t.state = "completed" - file_key = self.__get_file_key(t.fileitem) - if file_key: - self._task_state_changed_at[file_key] = monotonic() - break - - def fail_task(self, task: TransferTask): - """ - 设置任务为失败 - """ - with job_lock: - __mediaid__ = self.__get_id(task) - if __mediaid__ not in self._job_view: - return - # 更新状态 - for t in self._job_view[__mediaid__].tasks: - if t.fileitem == task.fileitem: - t.state = "failed" - file_key = self.__get_file_key(t.fileitem) - if file_key: - self._task_state_changed_at[file_key] = monotonic() - break - # 移除剧集信息 - if __mediaid__ in self._season_episodes: - self._season_episodes[__mediaid__] = list( - set(self._season_episodes[__mediaid__]) - - set(task.meta.episode_list) - ) - - def fail_unfinished_task(self, task: TransferTask): - """ - 将指定任务视图中的非终态任务标记为失败 - """ - if not task or not task.fileitem: - return - file_key = self.__get_file_key(task.fileitem) - if not file_key: - return - with job_lock: - for mediaid, job in self._job_view.items(): - for job_task in job.tasks: - if self.__get_file_key(job_task.fileitem) != file_key: - continue - if job_task.state not in ["completed", "failed"]: - job_task.state = "failed" - self._task_state_changed_at[file_key] = monotonic() - if mediaid in self._season_episodes: - self._season_episodes[mediaid] = list( - set(self._season_episodes[mediaid]) - - set(task.meta.episode_list) - ) - return - - def remove_task(self, fileitem: FileItem) -> Optional[TransferJobTask]: - """ - 根据文件项移除任务 - """ - task, _ = self.__remove_task_with_job_id(fileitem) - return task - - def __remove_task_with_job_id( - self, - fileitem: FileItem, - preserve_execution: bool = False, - ) -> Tuple[Optional[TransferJobTask], Optional[Tuple]]: - """ - 根据文件项移除任务，并返回任务所在的作业ID - """ - file_key = self.__get_file_key(fileitem) - if not file_key: - return None, None - with job_lock: - for mediaid in list(self._job_view): - job = self._job_view[mediaid] - for task in job.tasks: - if self.__get_file_key(task.fileitem) == file_key: - job.tasks.remove(task) - self._task_state_changed_at.pop(file_key, None) - if not preserve_execution: - self._active_executions.discard(file_key) - # 如果没有作业了，则移除作业 - if not job.tasks: - self._job_view.pop(mediaid) - # 移除季集信息 - if mediaid in self._season_episodes: - episodes = getattr(task.meta, "episode_list", None) or [] - self._season_episodes[mediaid] = list( - set(self._season_episodes[mediaid]) - - set(episodes) - ) - return task, mediaid - return None, None - - def remove_job(self, task: TransferTask) -> Optional[TransferJob]: - """ - 移除任务对应的作业（强制，线程不安全） - """ - with job_lock: - __mediaid__ = self.__get_id(task) - if __mediaid__ in self._job_view: - job = self._job_view[__mediaid__] - self.__pop_job(__mediaid__) - return job - return None - - def try_remove_job(self, task: TransferTask): - """ - 尝试移除任务对应的作业（严格检查未完成作业，线程安全） - """ - with job_lock: - __metaid__ = self.__get_meta_id( - meta=task.meta, season=task.meta.begin_season - ) - __mediaid__ = self.__get_media_id( - media=task.mediainfo, season=task.meta.begin_season - ) - - related_media_ids = set(self._meta_to_media_ids.get(__metaid__, set())) - if task.mediainfo: - related_media_ids.add(__mediaid__) - - meta_done = self.__is_job_done(__metaid__) - media_done = all( - self.__is_job_done(mediaid) for mediaid in related_media_ids - ) - - if meta_done and media_done: - remove_ids = {__metaid__, self.__get_id(task), *related_media_ids} - for job_id in remove_ids: - self.__pop_job(job_id) - self._meta_to_media_ids.pop(__metaid__, None) - - def is_done(self, task: TransferTask) -> bool: - """ - 检查任务对应的作业是否整理完成（不管成功还是失败） - """ - with job_lock: - __metaid__ = self.__get_meta_id( - meta=task.meta, season=task.meta.begin_season - ) - __mediaid__ = self.__get_media_id( - media=task.mediainfo, season=task.meta.begin_season - ) - if __metaid__ in self._job_view: - meta_done = all( - task.state in ["completed", "failed"] - for task in self._job_view[__metaid__].tasks - ) - else: - meta_done = True - if __mediaid__ in self._job_view: - media_done = all( - task.state in ["completed", "failed"] - for task in self._job_view[__mediaid__].tasks - ) - else: - media_done = True - return meta_done and media_done - - def is_finished(self, task: TransferTask) -> bool: - """ - 检查任务对应的作业是否已完成且有成功的记录 - """ - with job_lock: - __metaid__ = self.__get_meta_id( - meta=task.meta, season=task.meta.begin_season - ) - __mediaid__ = self.__get_media_id( - media=task.mediainfo, season=task.meta.begin_season - ) - if __metaid__ in self._job_view: - meta_finished = all( - task.state in ["completed", "failed"] - for task in self._job_view[__metaid__].tasks - ) - else: - meta_finished = True - if __mediaid__ in self._job_view: - tasks = self._job_view[__mediaid__].tasks - media_finished = all( - task.state in ["completed", "failed"] for task in tasks - ) and any(task.state == "completed" for task in tasks) - else: - media_finished = True - return meta_finished and media_finished - - def is_success(self, task: TransferTask) -> bool: - """ - 检查任务对应的作业是否全部成功 - """ - with job_lock: - __metaid__ = self.__get_meta_id( - meta=task.meta, season=task.meta.begin_season - ) - __mediaid__ = self.__get_media_id( - media=task.mediainfo, season=task.meta.begin_season - ) - if __metaid__ in self._job_view: - meta_success = all( - task.state in ["completed"] - for task in self._job_view[__metaid__].tasks - ) - else: - meta_success = True - if __mediaid__ in self._job_view: - media_success = all( - task.state in ["completed"] - for task in self._job_view[__mediaid__].tasks - ) - else: - media_success = True - return meta_success and media_success - - def get_all_torrent_hashes(self) -> set[str]: - """ - 获取所有种子的哈希值集合 - """ - with job_lock: - return { - task.download_hash - for job in self._job_view.values() - for task in job.tasks - } - - def is_torrent_done(self, download_hash: str) -> bool: - """ - 检查指定种子的所有任务是否都已完成 - """ - with job_lock: - if any( - task.state not in {"completed", "failed"} - for job in self._job_view.values() - for task in job.tasks - if task.download_hash == download_hash - ): - return False - return True - - def is_torrent_success(self, download_hash: str) -> bool: - """ - 检查指定种子的所有任务是否都已成功 - """ - with job_lock: - if any( - task.state != "completed" - for job in self._job_view.values() - for task in job.tasks - if task.download_hash == download_hash - ): - return False - return True - - def has_tasks( - self, - meta: MetaBase, - mediainfo: Optional[MediaInfo] = None, - season: Optional[int] = None, - ) -> bool: - """ - 判断作业是否还有任务正在处理 - """ - with job_lock: - if mediainfo: - __mediaid__ = self.__get_media_id(media=mediainfo, season=season) - if __mediaid__ in self._job_view: - return True - - __metaid__ = self.__get_meta_id(meta=meta, season=season) - return ( - __metaid__ in self._job_view - and len(self._job_view[__metaid__].tasks) > 0 - ) - - def success_tasks( - self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None - ) -> List[TransferJobTask]: - """ - 获取作业中所有成功的任务 - """ - with job_lock: - __mediaid__ = self.__get_media_id(media=media, season=season) - if __mediaid__ not in self._job_view: - return [] - return [ - task - for task in self._job_view[__mediaid__].tasks - if task.state == "completed" - ] - - def all_tasks( - self, media: MediaInfo, season: Optional[int] = None - ) -> List[TransferJobTask]: - """ - 获取作业中全部任务 - """ - with job_lock: - __mediaid__ = self.__get_media_id(media=media, season=season) - if __mediaid__ not in self._job_view: - return [] - return self._job_view[__mediaid__].tasks - - def count(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int: - """ - 获取作业中成功总数 - """ - with job_lock: - __mediaid__ = self.__get_media_id(media=media, season=season) - if __mediaid__ not in self._job_view: - return 0 - return len( - [ - task - for task in self._job_view[__mediaid__].tasks - if task.state == "completed" - ] - ) - - def size(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int: - """ - 获取作业中所有成功文件总大小 - """ - with job_lock: - __mediaid__ = self.__get_media_id(media=media, season=season) - if __mediaid__ not in self._job_view: - return 0 - return sum( - [ - task.fileitem.size - if task.fileitem.size is not None - else ( - SystemUtils.get_directory_size(Path(task.fileitem.path)) - if task.fileitem.storage == "local" - else 0 - ) - for task in self._job_view[__mediaid__].tasks - if task.state == "completed" - ] - ) - - def total(self) -> int: - """ - 获取所有任务总数 - """ - with job_lock: - return sum([len(job.tasks) for job in self._job_view.values()]) - - def pending_total(self) -> int: - """ - 获取未到终态的任务总数。 - - 作业要等关联任务全部终态才整体移除,追更/分批场景下已完成任务会 - 跨批次残留在视图中;批次统计若用全量 total() 会把历史任务计入 - 「当前共 N 个文件」并压低进度百分比,因此只数未终态任务。 - """ - with job_lock: - return sum( - 1 - for job in self._job_view.values() - for task in job.tasks - if task.state not in ("completed", "failed") - ) - - def list_jobs(self) -> List[TransferJob]: - """ - 获取所有作业的任务列表 - """ - with job_lock: - return list(self._job_view.values()) - - def season_episodes( - self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None - ) -> List[int]: - """ - 获取作业的季集清单 - """ - with job_lock: - __mediaid__ = self.__get_media_id(media=media, season=season) - return self._season_episodes.get(__mediaid__) or [] - - -class FailedRetryScheduler: - """ - 负责失败整理记录的 debounce 聚合与 AI 重试调度。 - """ - - RETRY_TRANSFER_DEBOUNCE_SECONDS = 300 - - def __init__(self): - super().__init__() - self._retry_transfer_buffer: dict[str, list[int]] = {} - self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {} - self._retry_transfer_lock = asyncio.Lock() - - async def close(self): - async with self._retry_transfer_lock: - timers = list(self._retry_transfer_timers.values()) - self._retry_transfer_timers.clear() - self._retry_transfer_buffer.clear() - - for timer in timers: - timer.cancel() - - @staticmethod - def _build_retry_transfer_template_context( - history_ids: list[int], - ) -> tuple[str, dict[str, int | str]]: - """仅负责把失败重试任务的动态数据映射成模板变量。""" - is_batch = len(history_ids) > 1 - task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry" - template_context: dict[str, int | str] = { - "history_ids_csv": ", ".join(str(item) for item in history_ids), - "history_count": len(history_ids), - } - if not is_batch: - template_context["history_id"] = history_ids[0] - return task_type, template_context - - def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str: - """根据失败记录数量构建统一的重试整理后台任务提示词。""" - task_type, template_context = self._build_retry_transfer_template_context(history_ids) - return prompt_manager.render_system_task_message( - task_type, - template_context=template_context, - ) - - async def schedule_retry(self, history_id: int, group_key: str = ""): - """ - 同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。 - """ - if not group_key: - group_key = f"_default_{history_id}" - - async with self._retry_transfer_lock: - if group_key not in self._retry_transfer_buffer: - self._retry_transfer_buffer[group_key] = [] - if history_id not in self._retry_transfer_buffer[group_key]: - self._retry_transfer_buffer[group_key].append(history_id) - logger.info( - f"智能体重试整理：记录 ID={history_id} 已加入缓冲区 " - f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)" - ) - - if group_key in self._retry_transfer_timers: - self._retry_transfer_timers[group_key].cancel() - - loop = asyncio.get_running_loop() - self._retry_transfer_timers[group_key] = loop.call_later( - self.RETRY_TRANSFER_DEBOUNCE_SECONDS, - lambda gk=group_key: asyncio.create_task(self._flush_retry_transfer(gk)), - ) - - async def _flush_retry_transfer(self, group_key: str): - """ - 延迟定时器到期后，取出该分组的所有 history_id 并合并为一次 agent 调用。 - """ - async with self._retry_transfer_lock: - history_ids = self._retry_transfer_buffer.pop(group_key, []) - self._retry_transfer_timers.pop(group_key, None) - - if not history_ids: - return - - ids_str = ", ".join(str(item) for item in history_ids) - logger.info( - f"智能体重试整理：开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})" - ) - - try: - await agent_manager.run_background_prompt( - message=self._build_retry_transfer_prompt(history_ids), - session_prefix="__agent_retry_transfer_batch", - reply_mode=ReplyMode.DISPATCH, - ) - logger.info( - f"智能体重试整理：批量处理完成 IDs=[{ids_str}] (group={group_key})" - ) - except Exception as err: - logger.error( - f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}" - ) - - -class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): +class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, HistoryMatchMixin, FileKeyMixin, + ManualHistoryMixin, FailedRetryMixin, ChainBase, ConfigReloadMixin, metaclass=Singleton): """ 文件整理处理链 """ @@ -993,23 +81,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): "TRANSFER_THREADS", } - @staticmethod - def _requires_automatic_category(task: TransferTask) -> bool: - """ - 判断当前整理任务是否需要根据媒体识别结果自动创建类别目录。 - - :param task: 整理任务 - :return: 是否必须具备自动分类结果 - """ - target_directory = task.target_directory - if target_directory and target_directory.media_category: - return False - if task.library_category_folder is not None: - return bool(task.library_category_folder) - return bool( - target_directory and target_directory.library_category_folder - ) - def __init__(self): """初始化文件整理处理链。""" super().__init__() @@ -1077,333 +148,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): self.__stop() self.__init() - def __is_subtitle_file(self, fileitem: FileItem) -> bool: - """ - 判断是否为字幕文件 - """ - if not fileitem.extension: - return False - return ( - True if f".{fileitem.extension.lower()}" in self._subtitle_exts else False - ) - - def __is_audio_file(self, fileitem: FileItem) -> bool: - """ - 判断是否为音频文件 - """ - if not fileitem.extension: - return False - return True if f".{fileitem.extension.lower()}" in self._audio_exts else False - - def __is_media_file( - self, - fileitem: FileItem, - mtype: Optional[MediaType] = None, - ) -> bool: - """ - 判断是否为主要媒体文件 - """ - if mtype == MediaType.MUSIC: - if fileitem.type != "file" or not fileitem.extension: - return False - return f".{fileitem.extension.lower()}" in self._audio_exts - if fileitem.type == "dir": - # 蓝光原盘判断 - return StorageChain().is_bluray_folder(fileitem) - if not fileitem.extension: - return False - extension = f".{fileitem.extension.lower()}" - return extension in self._media_exts - - def _is_primary_media_file( - self, - fileitem: FileItem, - mediainfo: Optional[MediaInfo | MusicInfo], - ) -> bool: - """判断文件在当前媒体上下文中是否属于主要媒体文件。""" - return self.__is_media_file( - fileitem, - getattr(mediainfo, "type", None), - ) - - @staticmethod - def _music_info_from_meta(meta: MetaMusic) -> MusicInfo: - """将音频文件标签解析结果转换为可整理的最小音乐信息。""" - return MusicInfo.from_meta(meta) - - @classmethod - def _match_music_album_context( - cls, - file_item: FileItem, - file_path: Path, - file_meta: MetaMusic, - ) -> tuple[MetaMusic, Optional[MusicInfo]]: - """为缺少远端身份的本地音频尝试目录级专辑匹配，命中后回填文件元数据。 - - WAV 等无标签文件只能依靠目录结构和曲目特征识别；匹配结果由 MediaChain - 按目录缓存，同一专辑目录内的后续文件不会重复请求远端。 - """ - # 目录级匹配需要读取本地音频时长，远端存储文件无法参与 - if file_meta.media_id or getattr(file_item, "storage", "local") != "local": - return file_meta, None - try: - matched = MediaChain().recognize_music_album_directory(file_path.parent) - except Exception as err: - logger.debug(f"音乐专辑目录匹配失败：{file_path} - {err}") - return file_meta, None - info = matched.get(str(file_path.resolve())) - if not info or not info.media_id: - return file_meta, None - logger.info(f"{file_path.name} 通过专辑目录匹配识别为：{info.artist} - {info.title}") - merged_meta = deepcopy(file_meta) - # 保留本地音频的实际技术参数，仅回填身份和名称字段 - if info.title: - merged_meta.title = info.title - if info.artists: - merged_meta.artists = list(info.artists) - if info.album: - merged_meta.album = info.album - if info.album_artist: - merged_meta.album_artist = info.album_artist - if info.year: - merged_meta.year = info.year - if info.disc_number: - merged_meta.disc_number = info.disc_number - if info.track_number: - merged_meta.track_number = info.track_number - if info.total_tracks: - merged_meta.total_tracks = info.total_tracks - merged_meta.media_source = info.media_source - merged_meta.media_id = info.media_id - merged_info = cls._music_info_from_meta(merged_meta) - # 补齐曲目级远端信息，供后续刮削和展示使用 - merged_info.music_type = info.music_type - merged_info.artist_ids = list(info.artist_ids) - merged_info.album_id = info.album_id - merged_info.album_type = info.album_type - merged_info.release_date = info.release_date - merged_info.cover_url = info.cover_url - merged_info.category = info.category - merged_info.genres = list(info.genres) - merged_info.detail_link = info.detail_link - return merged_meta, merged_info - - @staticmethod - def _download_history_music_type( - download_history: Optional[DownloadHistory], - ) -> Optional[str]: - """从下载历史字段或旧版音乐备注中恢复音乐实体类型。""" - music_type = normalize_music_type( - getattr(download_history, "music_type", None), - allow_artist=False, - ) - if music_type: - return music_type - note = getattr(download_history, "note", None) - music_note = note.get("music") if isinstance(note, dict) else None - media_payload = music_note.get("media") if isinstance(music_note, dict) else None - if not isinstance(media_payload, dict): - return None - return normalize_music_type( - media_payload.get("music_type"), - allow_artist=False, - ) - - @classmethod - def _restore_music_download_context( - cls, - download_history: Optional[DownloadHistory], - file_path: Path, - ) -> tuple[Optional[MetaMusic], Optional[MusicInfo]]: - """从下载历史恢复音乐上下文，并用当前音频标签覆盖曲目级字段。""" - note = getattr(download_history, "note", None) - music_note = note.get("music") if isinstance(note, dict) else None - if not isinstance(music_note, dict) or music_note.get("version") != 1: - return None, None - try: - saved_meta = MetaMusic.from_dict(music_note.get("meta") or {}) - saved_info = MusicInfo.from_dict(music_note.get("media") or {}) - except (TypeError, ValueError): - return None, None - - file_tags = MediaChain.read_path_meta(file_path) - file_meta = deepcopy(saved_meta) - file_meta.org_string = file_path.name - # 曲目标题始终优先使用当前文件自身的标签（缺失时回退为文件名）， - # 防止整包目录继续沿用订阅/下载标题（单曲名、专辑名等）导致所有文件重名。 - if file_tags.title: - file_meta.title = file_tags.title - is_album_context = saved_info.music_type == MUSIC_ENTITY_ALBUM - for field_name in ( - "artists", - "disc_number", - "track_number", - "total_discs", - "version", - "isrc", - ): - if getattr(file_tags, field_name, None): - setattr(file_meta, field_name, deepcopy(getattr(file_tags, field_name))) - for field_name in ("album", "album_artist", "year", "total_tracks"): - file_value = getattr(file_tags, field_name, None) - # 整专下载以订阅选中的专辑字段为准，避免单个错误标签把曲目拆到其它专辑目录。 - if file_value and (not is_album_context or not getattr(file_meta, field_name, None)): - setattr(file_meta, field_name, deepcopy(file_value)) - for field_name in ( - "audio_format", - "bit_depth", - "sample_rate", - "bitrate", - "duration", - ): - if getattr(file_tags, field_name, None): - setattr(file_meta, field_name, getattr(file_tags, field_name)) - file_meta.media_source = saved_info.media_source or saved_meta.media_source - file_meta.media_id = saved_info.media_id or saved_meta.media_id - - file_info = cls._music_info_from_meta(file_meta) - file_info.media_source = saved_info.media_source - file_info.media_id = saved_info.media_id - file_info.music_type = saved_info.music_type - file_info.artist_ids = list(saved_info.artist_ids) - file_info.album_id = saved_info.album_id - file_info.album_type = saved_info.album_type - file_info.release_date = saved_info.release_date - file_info.cover_url = saved_info.cover_url - file_info.lyrics = saved_info.lyrics - file_info.category = saved_info.category - file_info.genres = list(saved_info.genres) - file_info.detail_link = saved_info.detail_link - file_info.listen_count = saved_info.listen_count - return file_meta, file_info - - @staticmethod - def _is_music_retry_source(history: TransferHistory, src_path: Path) -> bool: - """ - 判断重新整理来源是否应走音乐链路：历史类型为音乐，或源路径为音频文件。 - """ - if history.type == MediaType.MUSIC.value: - return True - return src_path.suffix.lower() in settings.RMT_AUDIOEXT - - def _recognize_music_retry_media( - self, - history: TransferHistory, - src_path: Path, - ) -> Optional[Union[MusicInfo, MediaInfo]]: - """ - 重新整理重试时恢复音乐信息。 - - 优先按历史记录中的 MusicBrainz 身份恢复；单音频文件回退按音频标签与文件名识别； - 音乐专辑目录返回 None，交由整理链按音频后缀逐文件解析识别。 - """ - if history.media_source and history.media_id: - retry_info = MediaChain().recognize_media( - mtype=MediaType.MUSIC, - media_source=history.media_source, - media_id=history.media_id, - music_type=getattr(history, "music_type", None), - ) - if retry_info: - return retry_info - if src_path.is_file(): - # 音频走统一路径识别入口，自动路由到音乐识别链 - recognize_context = MediaChain().recognize_by_path(str(src_path)) - return recognize_context.media_info if recognize_context else None - return None - - def __is_allowed_file(self, fileitem: FileItem) -> bool: - """ - 判断是否允许的扩展名 - """ - if not fileitem.extension: - return False - return True if f".{fileitem.extension.lower()}" in self._allowed_exts else False - - @staticmethod - def __is_allow_filesize(fileitem: FileItem, min_filesize: int) -> bool: - """ - 判断是否满足最小文件大小 - """ - return ( - True - if not min_filesize or (fileitem.size or 0) > min_filesize * 1024 * 1024 - else False - ) - - @staticmethod - def __is_hidden_or_recycle_path(file_path: Optional[str]) -> bool: - """ - 判断是否隐藏或回收站路径 - """ - if not file_path: - return False - normalized_path = file_path.replace("\\", "/") - return ( - "/@Recycle/" in normalized_path - or "/#recycle/" in normalized_path - or "/." in normalized_path - or "/@eaDir" in normalized_path - ) - - @staticmethod - def __should_delete_empty_source_directories( - task: TransferTask, - delete_mounted_local_disk_empty_dirs: bool, - mounted_filesystem_cache: Dict[Path, bool], - ) -> bool: - """ - 判断移动整理后是否应删除源空目录。 - - 仅在关闭挂载盘空目录清理且源存储为本地时检测文件系统， - 避免默认流程产生额外系统调用。 - """ - if delete_mounted_local_disk_empty_dirs: - return True - if task.fileitem.storage != "local": - return True - - source_directory = ( - Path(task.target_directory.download_path) - if task.target_directory and task.target_directory.download_path - else Path(task.fileitem.path).parent - ) - if source_directory not in mounted_filesystem_cache: - mounted_filesystem_cache[source_directory] = ( - SystemUtils.is_network_filesystem( - source_directory, include_local_fuse=True - ) - ) - return not mounted_filesystem_cache[source_directory] - - @staticmethod - def __is_overwrite_declined(task: TransferTask, transferinfo: TransferInfo, - transferhis: TransferHistoryOper) -> bool: - """ - 判断本次未入库是否为「同路径已有成功记录 + 覆盖模式裁定不覆盖」。 - - 只有同路径此前已成功整理过才需要保护：这类文件是查重闸放行的同路径新版本， - 媒体库中的原有版本仍然在位，不应因一次不覆盖裁决把成功记录改写成失败记录。 - 没有成功记录时（如目标同名文件来自其他源路径）保持原有失败语义， - 用户仍能在历史与通知中看到裁决结果。 - :param task: 整理任务 - :param transferinfo: 整理结果 - :param transferhis: 历史操作对象 - :return: True 表示应保留原成功记录 - """ - if not transferinfo.overwrite_skipped or not task.fileitem: - return False - try: - history = resolve_history( - task.fileitem.path, - storage=task.fileitem.storage, - transfer_history_oper=transferhis, - ) - except Exception as err: - logger.error(f"查询整理历史失败: {task.fileitem.path} - {err}") - return False - return bool(history and history.status) - def __default_callback( self, task: TransferTask, transferinfo: TransferInfo, / ) -> Tuple[bool, str]: @@ -1459,7 +203,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): # 媒体库里原有版本仍然在位，写失败记录会用 add_force 顶掉原成功记录，此后该路径 # 永远处于失败态，每个新事件都会重试并重推失败通知。此时保留原记录、不写历史、 # 不发事件与通知、不触发重试，仅把任务置为未入库 - overwrite_declined = self.__is_overwrite_declined( + overwrite_declined = self._is_overwrite_declined( task, transferinfo, transferhis ) history = None @@ -1492,7 +236,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): ) # 整理失败事件 - if self.__is_media_file(task.fileitem): + if self._is_media_file(task.fileitem): # 主要媒体文件整理失败事件 self.eventmanager.send_event( EventType.TransferFailed, @@ -1506,7 +250,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): "transfer_history_id": history.id if history else None, }, ) - elif self.__is_subtitle_file(task.fileitem): + elif self._is_subtitle_file(task.fileitem): # 字幕整理失败事件 self.eventmanager.send_event( EventType.SubtitleTransferFailed, @@ -1520,7 +264,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): "transfer_history_id": history.id if history else None, }, ) - elif self.__is_audio_file(task.fileitem): + elif self._is_audio_file(task.fileitem): # 音频文件整理失败事件 self.eventmanager.send_event( EventType.AudioTransferFailed, @@ -1627,7 +371,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): "transfer_history_id": history.id if history else None, }, ) - elif self.__is_subtitle_file(task.fileitem): + elif self._is_subtitle_file(task.fileitem): # 字幕整理完成事件 self.eventmanager.send_event( EventType.SubtitleTransferComplete, @@ -1641,7 +385,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): "transfer_history_id": history.id if history else None, }, ) - elif self.__is_audio_file(task.fileitem): + elif self._is_audio_file(task.fileitem): # 音频文件整理完成事件 self.eventmanager.send_event( EventType.AudioTransferComplete, @@ -1669,7 +413,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): self.jobview.finish_task(task) # 登记批次级刮削目标 - self.__record_scrape_target(task, transferinfo) + self._record_scrape_target(task, transferinfo) # 全部整理完成且有成功的任务时，发送消息和事件 if self.jobview.is_finished(task): @@ -1682,7 +426,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): )) __notify() if not task.transfer_batch_id: - self.__send_metadata_scrape_event(task, transferinfo) + self._send_metadata_scrape_event(task, transferinfo) # 只要该种子的所有任务都已整理完成，则设置种子状态为已整理 self.__mark_torrent_completed_if_done(task.download_hash, task.downloader) @@ -1724,11 +468,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): if ( not t.download_hash and t.fileitem - and self.__should_delete_empty_source_directories( - t, - delete_mounted_local_disk_empty_dirs, - mounted_filesystem_cache, - ) + and self._should_delete_empty_source_directories( + t, + delete_mounted_local_disk_empty_dirs, + mounted_filesystem_cache, + ) ): # 删除剩余空目录 StorageChain().delete_media_file(t.fileitem, delete_self=False) @@ -1762,7 +506,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): # 维护整理任务视图，如果任务已存在则不添加到队列 if not self.__put_to_jobview(task): return False - self.__register_scrape_batch_task(task) + self._register_scrape_batch_task(task) # 添加到队列 self._queue.put(TransferQueue(task=task, callback=self.__default_callback)) # 落盘登记：队列是纯内存的，进程重启（挂载挂死后的人工重启、升级、OOM） @@ -1938,204 +682,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): logger.error(f"检查种子 {download_hash} 下载进度失败：{e}") return False - def __send_metadata_scrape_event( - self, task: TransferTask, transferinfo: TransferInfo - ): - """ - 发送元数据刮削事件，保持对外事件载荷兼容。 - """ - if ( - not task - or not transferinfo - or not transferinfo.need_scrape - or not self._is_primary_media_file(task.fileitem, task.mediainfo) - ): - return - - target_diritem = transferinfo.target_diritem - if not target_diritem: - return - - self.eventmanager.send_event( - EventType.MetadataScrape, - self.__build_metadata_scrape_payload( - task=task, - fileitem=target_diritem, - file_list=transferinfo.file_list_new, - overwrite=False, - ), - ) - - @staticmethod - def __build_metadata_scrape_payload( - task: TransferTask, - fileitem: FileItem, - file_list: Optional[list[str]], - overwrite: bool, - ) -> dict[str, Any]: - """构造刮削事件载荷，并为音乐批次保留逐文件身份上下文。""" - paths = list(dict.fromkeys(file_list or [])) - payload: dict[str, Any] = { - "meta": task.meta, - "mediainfo": task.mediainfo, - "fileitem": fileitem, - "file_list": paths, - "overwrite": overwrite, - } - if isinstance(task.mediainfo, MusicInfo): - payload["file_contexts"] = [ - { - "path": path, - "meta": task.meta, - "mediainfo": task.mediainfo, - } - for path in paths - ] - return payload - - def __register_scrape_batch_task(self, task: TransferTask): - """ - 登记批次任务。刮削事件只在批次关闭且任务全部完成后统一发送。 - """ - if not task or not task.transfer_batch_id: - return - with job_lock: - batch = self._scrape_batches.setdefault( - task.transfer_batch_id, - { - "pending": set(), - "targets": {}, - "closed": False, - }, - ) - batch["pending"].add(task.fileitem.path) - - def __close_scrape_batch(self, batch_id: Optional[str]): - """ - 标记批次不再接收新任务，并尝试发送已聚合的刮削事件。 - """ - if not batch_id: - return - with job_lock: - batch = self._scrape_batches.setdefault( - batch_id, - { - "pending": set(), - "targets": {}, - "closed": False, - }, - ) - batch["closed"] = True - self.__flush_scrape_batch_if_ready(batch_id) - - def __record_scrape_target(self, task: TransferTask, transferinfo: TransferInfo): - """ - 记录批次内需要刮削的目标文件，按目标媒体根目录聚合。 - """ - if ( - not task - or not task.transfer_batch_id - or not transferinfo - or not transferinfo.need_scrape - or not self._is_primary_media_file(task.fileitem, task.mediainfo) - ): - return - - target_diritem = transferinfo.target_diritem - if not target_diritem: - return - - target_files = transferinfo.file_list_new or [] - target_key = (target_diritem.storage, target_diritem.path) - with job_lock: - batch = self._scrape_batches.setdefault( - task.transfer_batch_id, - { - "pending": set(), - "targets": {}, - "closed": False, - }, - ) - target = batch["targets"].setdefault( - target_key, - { - "fileitem": target_diritem, - "meta": task.meta, - "mediainfo": task.mediainfo, - "files": [], - "file_contexts": {}, - "overwrite": False, - }, - ) - if not target.get("meta"): - target["meta"] = task.meta - if not target.get("mediainfo"): - target["mediainfo"] = task.mediainfo - for target_file in target_files: - if target_file and target_file not in target["files"]: - target["files"].append(target_file) - if target_file and isinstance(task.mediainfo, MusicInfo): - target["file_contexts"][target_file] = { - "path": target_file, - "meta": task.meta, - "mediainfo": task.mediainfo, - } - - def __finish_scrape_batch_task(self, task: TransferTask): - """ - 标记批次内单个任务已结束。 - """ - if not task or not task.transfer_batch_id: - return - with job_lock: - batch = self._scrape_batches.get(task.transfer_batch_id) - if not batch: - return - batch["pending"].discard(task.fileitem.path) - self.__flush_scrape_batch_if_ready(task.transfer_batch_id) - - def __flush_scrape_batch_if_ready(self, batch_id: Optional[str]): - """ - 批次任务全部结束后发送聚合后的刮削事件。 - """ - if not batch_id: - return - - with job_lock: - batch = self._scrape_batches.get(batch_id) - if ( - not batch - or not batch.get("closed") - or batch.get("pending") - ): - return - targets = list(batch.get("targets", {}).values()) - self._scrape_batches.pop(batch_id, None) - - for target in targets: - fileitem = target.get("fileitem") - if not fileitem: - continue - file_list = list(dict.fromkeys(target.get("files") or [])) - file_contexts = target.get("file_contexts") or {} - payload = { - "meta": target.get("meta"), - "mediainfo": target.get("mediainfo"), - "fileitem": fileitem, - "file_list": file_list, - "overwrite": target.get("overwrite", False), - } - if file_contexts: - payload["file_contexts"] = [ - file_contexts[path] - for path in file_list - if path in file_contexts - ] - self.eventmanager.send_event( - EventType.MetadataScrape, - payload, - ) - def remove_from_queue(self, fileitem: FileItem): """ 从待整理队列移除 @@ -2179,7 +725,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): """ self.jobview.fail_unfinished_task(task) self.jobview.try_remove_job(task) - self.__finish_scrape_batch_task(task) + self._finish_scrape_batch_task(task) def __start_transfer(self): """ @@ -2607,7 +1153,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): finally: # 移除已完成的任务 self.jobview.try_remove_job(task) - self.__finish_scrape_batch_task(task) + self._finish_scrape_batch_task(task) def get_queue_tasks(self) -> List[TransferJob]: """ @@ -2616,193 +1162,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): self.__expire_stale_transfer_tasks() return self.jobview.list_jobs() - def recommend_name(self, meta: MetaBase, mediainfo: MediaInfo) -> Optional[str]: - """ - 获取重命名后的名称 - :param meta: 元数据 - :param mediainfo: 媒体信息 - :return: 重命名后的名称（含目录） - """ - # 获取集信息，供重命名模块使用 - episodes_info: Optional[List[TmdbEpisode]] = None - if mediainfo.type == MediaType.TV: - # 判断注意season为0的情况 - season_num = mediainfo.season - if season_num is None and meta.season_seq: - if meta.season_seq.isdigit(): - season_num = int(meta.season_seq) - # 默认值1 - if season_num is None: - season_num = 1 - episodes_info = self.run_module( - "tmdb_episodes", - tmdbid=mediainfo.tmdb_id, - season=season_num, - episode_group=mediainfo.episode_group, - ) - if episodes_info: - return self.run_module( - "recommend_name", - meta=meta, - mediainfo=mediainfo, - episodes_info=episodes_info, - ) - # 电影或无集信息时保持原有参数集，避免影响旧签名的模块实现 - return self.run_module("recommend_name", meta=meta, mediainfo=mediainfo) - - def recommend_episode_format( - self, - fileitem: FileItem, - fileitems: Optional[List[FileItem]] = None, - ) -> Tuple[bool, str, Optional[dict]]: - """ - 根据目录样本推荐集数定位模板 - """ - if not fileitem and not fileitems: - logger.warn("推荐集数定位模板失败：缺少目录参数") - return False, "缺少目录参数", None - - rules = self.__get_episode_format_rules() - if fileitems: - state, errmsg, sample_files = self.__get_selected_episode_format_sample_files( - fileitems - ) - if not state: - logger.warn(f"推荐集数定位模板失败：{errmsg}") - return False, errmsg, None - target_path = sample_files[0].path if sample_files else None - else: - if not fileitem or not fileitem.path: - logger.warn("推荐集数定位模板失败：缺少目录参数") - return False, "缺少目录参数", None - directory = self.__resolve_episode_format_directory(fileitem) - if not directory or directory.type != "dir": - logger.warn(f"推荐集数定位模板失败：目录不存在 - {fileitem.path}") - return False, "目录不存在", None - sample_files = self.__get_episode_format_sample_files(directory) - target_path = directory.path - logger.info( - f"开始匹配集数定位规则：{target_path}，规则数 {len(rules)}，样本数 {len(sample_files)}" - ) - state, errmsg, data = EpisodeFormatRuleHelper().recommend( - rules=rules, - sample_files=sample_files, - ) - if not state: - logger.warn(f"集数定位模板推荐失败：{target_path} - {errmsg}") - return state, errmsg, data - logger.info( - f"集数定位模板推荐成功：{target_path} - 规则 {data.get('rule_name') if data else None}" - ) - return state, errmsg, data - - @staticmethod - def __get_episode_format_rules() -> List[schemas.EpisodeFormatRule]: - """ - 获取启用的集数定位规则 - """ - rule_items = SystemConfigOper().get(SystemConfigKey.EpisodeFormatRuleTable) or [] - rules: List[schemas.EpisodeFormatRule] = [] - for item in rule_items: - if not isinstance(item, dict): - continue - try: - rule = schemas.EpisodeFormatRule(**item) - except Exception as err: - logger.warn(f"忽略无效的集数定位规则：{err}") - continue - if rule.enabled: - rules.append(rule) - return sorted(rules, key=lambda item: item.order) - - def __resolve_episode_format_directory( - self, fileitem: FileItem - ) -> Optional[FileItem]: - """ - 将文件或目录入参归一化为目录对象 - """ - storage_chain = StorageChain() - if fileitem.type == "dir": - return storage_chain.get_item(fileitem) - source_path = Path(fileitem.path) - parent_item = FileItem( - storage=fileitem.storage, - path=source_path.parent.as_posix(), - type="dir", - name=source_path.parent.name, - ) - return storage_chain.get_item(parent_item) - - def __get_selected_episode_format_sample_files( - self, fileitems: List[FileItem] - ) -> Tuple[bool, str, List[FileItem]]: - """ - 获取当前选择文件中可参与模板推荐的样本文件。 - """ - if not fileitems: - return False, "没有可用于识别的样本文件", [] - - expected_dir_key: Optional[Tuple[str, str]] = None - selected_files: List[FileItem] = [] - seen_files = set() - for item in fileitems: - if not item or not item.path or item.type != "file": - return False, "当前选择不满足智能识别条件", [] - - dir_key = ( - item.storage or "local", - Path(item.path).parent.as_posix(), - ) - if expected_dir_key is None: - expected_dir_key = dir_key - elif dir_key != expected_dir_key: - return False, "当前选择不满足智能识别条件", [] - - file_key = (item.storage or "local", item.path) - if file_key in seen_files: - continue - seen_files.add(file_key) - - if not ( - self.__is_media_file(item) - or self.__is_subtitle_file(item) - or self.__is_audio_file(item) - ): - continue - if self.__is_hidden_or_recycle_path(item.path): - continue - selected_files.append(item) - - if not selected_files: - return False, "没有可用于识别的样本文件", [] - return True, "", selected_files - - def __get_episode_format_sample_files( - self, directory: FileItem - ) -> List[FileItem]: - """ - 获取目录下可参与模板推荐的样本文件。 - - 推荐结果最终会在手动整理链路中作为 `episode_format` - 交由 `FormatParser` 过滤主视频、字幕和外挂音频，因此这里需要把 - 同目录下的主视频、字幕和外挂音频一起纳入推荐流程。 - """ - file_items = StorageChain().list_files(directory, recursion=False) or [] - sample_files: List[FileItem] = [] - for item in file_items: - if not item or item.type != "file": - continue - if not ( - self.__is_media_file(item) - or self.__is_subtitle_file(item) - or self.__is_audio_file(item) - ): - continue - if self.__is_hidden_or_recycle_path(item.path): - continue - sample_files.append(item) - return sample_files - def process(self, progress_callback: Optional[Callable[..., None]] = None) -> bool: """ 获取下载器中的种子列表，并执行整理 @@ -2862,9 +1221,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): break if progress_callback: torrent_name = ( - getattr(torrent, "title", None) - or getattr(torrent, "name", None) - or torrent.hash + getattr(torrent, "title", None) + or getattr(torrent, "name", None) + or torrent.hash ) progress_callback( value=(index - 1) / total_num * 100, @@ -3116,397 +1475,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): return shared_roots - @staticmethod - def _match_download_file( - download_file: DownloadFiles, - file_path: Path, - save_path: Path, - ) -> bool: - """ - 判断下载文件记录是否明确对应当前文件。 - """ - if download_file.fullpath == file_path.as_posix(): - return True - - filepath = download_file.filepath - if not filepath: - return False - - try: - return (save_path / Path(filepath)).as_posix() == file_path.as_posix() - except (TypeError, ValueError): - return False - - def _resolve_history_from_download_files( - self, - downloadhis: DownloadHistoryOper, - download_files: List[DownloadFiles], - file_path: Optional[Path] = None, - save_path: Optional[Path] = None, - ) -> Optional[DownloadHistory]: - """ - 从下载文件记录中解析唯一的下载历史。 - """ - if file_path and save_path: - download_files = [ - download_file - for download_file in download_files - if self._match_download_file( - download_file=download_file, - file_path=file_path, - save_path=save_path, - ) - ] - - download_hashes = { - download_file.download_hash - for download_file in download_files - if download_file.download_hash - } - if len(download_hashes) == 1: - return downloadhis.get_by_hash(next(iter(download_hashes))) - return None - - def _resolve_download_history( - self, - downloadhis: DownloadHistoryOper, - file_path: Path, - bluray_dir: bool = False, - download_hash: Optional[str] = None, - ) -> Optional[DownloadHistory]: - """ - 根据显式 hash、文件路径或种子根目录回查下载历史。 - """ - if download_hash: - return downloadhis.get_by_hash(download_hash) - - if bluray_dir: - return downloadhis.get_by_path(file_path.as_posix()) - - download_file = downloadhis.get_file_by_fullpath(file_path.as_posix()) - if download_file: - return downloadhis.get_by_hash(download_file.download_hash) - - # 多文件种子里的字幕/附加文件可能没有稳定的 fullpath 记录， - # 退回到父目录和 savepath 继续查找，尽量补齐同一种子的关联信息。 - shared_download_roots = self._get_shared_download_roots(file_path) - - for parent_path in file_path.parents: - parent_posix = parent_path.as_posix() - download_files = downloadhis.get_files_by_savepath(parent_posix) or [] - - if parent_posix in shared_download_roots: - # 共享下载根目录只能接受有明确文件记录的匹配， - # 避免单文件/磁力任务把整个根目录污染成同一媒体。 - history = self._resolve_history_from_download_files( - downloadhis=downloadhis, - download_files=download_files, - file_path=file_path, - save_path=parent_path, - ) - if history: - return history - break - - download_history = downloadhis.get_by_path(parent_posix) - if download_history: - return download_history - - history = self._resolve_history_from_download_files( - downloadhis=downloadhis, - download_files=download_files, - ) - if history: - return history - - return None - - @staticmethod - def _is_movie_year_conflict( - file_meta: MetaBase, - # 两种 DownloadHistory 都会进来：库模型（本文件按 ORM 行查历史）与 - # schemas DTO（TransferTask.download_history）。本函数只按 getattr 取 - # year 与 type，对两者一视同仁 - media: Union[DownloadHistory, schemas.DownloadHistory, MediaInfo, MusicInfo] - ) -> bool: - """ - 判断文件名年份是否与已识别电影年份冲突。 - - 多电影合集只保存一条下载历史，不能把合集首部电影的媒体 ID 套用到其它年份的文件； - 电视剧季包仍应继续复用同一条下载历史。 - """ - file_year = getattr(file_meta, "year", None) - media_year = getattr(media, "year", None) - if not file_meta or not media or not file_year or not media_year: - return False - media_type = getattr(media, "type", None) - if not isinstance(media_type, MediaType): - try: - media_type = MediaType(media_type) - except (TypeError, ValueError): - return False - return ( - media_type == MediaType.MOVIE - and str(file_year) != str(media_year) - ) - - @staticmethod - def __optional_attr_equal( - source: MetaBase, - target: MetaBase, - attr: str, - normalizer: Callable = None, - ) -> bool: - """ - 比较可选识别字段。 - - 字段两边都没有识别到时不参与判断；只要任意一边识别到了，就要求两边值一致， - 避免把同名不同年份或不同季集的附加文件误归到当前主视频。 - """ - source_value = getattr(source, attr, None) - target_value = getattr(target, attr, None) - if source_value is None and target_value is None: - return True - if source_value is None or target_value is None: - return False - if normalizer: - source_value = normalizer(source_value) - target_value = normalizer(target_value) - return source_value == target_value - - def __is_same_media_meta( - self, source_meta: MetaBase, target_meta: MetaBase - ) -> bool: - """ - 判断两个文件识别出的媒体身份是否一致。 - """ - if not source_meta or not target_meta: - return False - if source_meta.type != target_meta.type: - return False - if text_tools.normalize_upper(source_meta.name) != text_tools.normalize_upper( - target_meta.name - ): - return False - if not self.__optional_attr_equal(source_meta, target_meta, "year", str): - return False - for attr in ( - "begin_season", - "end_season", - "begin_episode", - "end_episode", - ): - if not self.__optional_attr_equal(source_meta, target_meta, attr, int): - return False - return True - - @staticmethod - def __get_file_key(fileitem: FileItem) -> Tuple[str, str]: - """ - 获取文件缓存键。 - """ - normalized_path = Path(str(fileitem.path).replace("\\", "/")).as_posix() - return fileitem.storage or "local", normalized_path - - @staticmethod - def __get_file_stem(fileitem: FileItem) -> str: - """ - 获取文件主干名，用于判断同名附加文件。 - """ - file_name = fileitem.name or Path(fileitem.path).name - return Path(file_name).stem.lower() - - @classmethod - def __get_subtitle_media_stem(cls, subtitle_fileitem: FileItem) -> str: - """ - 获取字幕对应主视频的候选主干名。 - """ - current_stem = cls.__get_file_stem(subtitle_fileitem) - while current_stem: - media_stem, separator, suffix = current_stem.rpartition(".") - if not separator or suffix not in SUBTITLE_STEM_TAGS: - return current_stem - current_stem = media_stem - return current_stem - - def __get_extra_media_stem(self, extra_fileitem: FileItem) -> str: - """ - 获取附加文件对应主视频的候选主干名。 - """ - if self.__is_subtitle_file(extra_fileitem): - return self.__get_subtitle_media_stem(extra_fileitem) - return self.__get_file_stem(extra_fileitem) - - def __get_related_main_file_key( - self, - extra_fileitem: FileItem, - main_fileitems: List[FileItem], - ) -> Optional[Tuple[str, str]]: - """ - 获取与附加文件名完全匹配的主视频键。 - """ - if not ( - self.__is_subtitle_file(extra_fileitem) - or self.__is_audio_file(extra_fileitem) - ): - return None - - extra_media_stem = self.__get_extra_media_stem(extra_fileitem) - matched_items: List[FileItem] = [] - for main_fileitem in main_fileitems: - main_stem = self.__get_file_stem(main_fileitem) - if main_stem and main_stem == extra_media_stem: - matched_items.append(main_fileitem) - - if len(matched_items) != 1: - return None - return self.__get_file_key(matched_items[0]) - - @staticmethod - def __normalize_dir_path(dir_path: Union[str, Path]) -> str: - """ - 归一化目录路径，用于同一父目录候选缓存。 - """ - normalized = Path(dir_path).as_posix().rstrip("/") - return normalized or "/" - - def __get_dir_key(self, dir_item: FileItem) -> Tuple[str, str]: - """ - 获取目录缓存键。 - """ - return dir_item.storage, self.__normalize_dir_path(dir_item.path) - - def __get_file_parent_key(self, current_item: FileItem) -> Tuple[str, str]: - """ - 获取文件父目录缓存键。 - """ - return ( - current_item.storage, - self.__normalize_dir_path(Path(current_item.path).parent), - ) - - @staticmethod - def _get_subscribe_custom_words( - history_record: Optional[DownloadHistory], - ) -> Optional[List[str]]: - """ - 获取整理用自定义识别词：优先使用下载时保存的快照，无快照（历史旧记录）时再按来源实时反查订阅。 - - 快照优先可避免整理阶段因订阅季号漂移、来源解析失败或订阅完成被删导致识别词丢失，从而原样入库到偏移前的季集。 - """ - if not history_record: - return None - # 下载时保存的完整订阅识别词快照优先 - if history_record.custom_words: - return history_record.custom_words.split("\n") - # 兜底：历史旧记录无快照时，按下载来源实时反查订阅 - if not isinstance(history_record.note, dict): - return None - subscribe = SubscribeChain().get_subscribe_by_source( - history_record.note.get("source") - ) - return ( - subscribe.custom_words.split("\n") - if subscribe and subscribe.custom_words - else None - ) - - @staticmethod - def _is_successful_move_history(history: Optional[TransferHistory]) -> bool: - """判断历史记录是否为已成功完成的移动类整理。""" - return bool( - history - and history.status - and history.mode - and "move" in history.mode - ) - - def _get_manual_transfer_history( - self, - fileitem: FileItem, - transfer_history_oper: TransferHistoryOper, - include_move_dest: bool = False, - ) -> Optional[TransferHistory]: - """查询文件源路径历史，并兼容从成功移动后的目标现址重新整理。""" - # resolve_history 在命中失败记录时会再确认一次有无成功记录， - # 避免 get_by_src 无排序导致同源多行时返回哪条不确定 - history = resolve_history( - fileitem.path, - storage=fileitem.storage, - transfer_history_oper=transfer_history_oper, - ) - if history or not include_move_dest: - return history - - history = transfer_history_oper.get_by_dest( - fileitem.path, - storage=fileitem.storage, - ) - return history if self._is_successful_move_history(history) else None - - def get_manual_transfer_histories( - self, - fileitems: List[FileItem], - ) -> List[TransferHistory]: - """ - 查询文件或目录命中的成功整理记录，供手动整理界面显示重整状态。 - - :param fileitems: 待查询的文件或目录项 - :return: 去重后的成功整理记录 - """ - transfer_history_oper = TransferHistoryOper() - histories: Dict[int, TransferHistory] = {} - for fileitem in fileitems or []: - if not fileitem or not fileitem.path: - continue - storage = fileitem.storage or "local" - if fileitem.type == "dir": - matched_histories = transfer_history_oper.list_success_by_src( - fileitem.path, - storage=storage, - recursive=True, - ) - matched_histories.extend( - transfer_history_oper.list_success_move_by_dest( - fileitem.path, - storage=storage, - recursive=True, - ) - ) - else: - history = self._get_manual_transfer_history( - fileitem=fileitem, - transfer_history_oper=transfer_history_oper, - include_move_dest=True, - ) - matched_histories = [history] if history and history.status else [] - - for history in matched_histories: - histories[history.id] = history - return list(histories.values()) - - @staticmethod - def _delete_manual_transfer_history( - history: TransferHistory, - transfer_history_oper: TransferHistoryOper, - ) -> Tuple[bool, str]: - """删除手动重整历史；非成功移动记录同时清理可能存在的旧目标。""" - if ( - history.dest_fileitem - and not TransferChain._is_successful_move_history(history) - ): - dest_fileitem = FileItem(**history.dest_fileitem) - storage_chain = StorageChain() - if ( - storage_chain.exists(dest_fileitem) - and not storage_chain.delete_media_file(dest_fileitem) - ): - return False, f"{dest_fileitem.path} 删除失败" - transfer_history_oper.delete(history.id) - # 删除记录是用户显式要求重来，失败计数一并清零，否则重整仍会受上一轮次数限制 - clear_transfer_failures(history.src, history.src_storage) - return True, "" - def do_transfer( self, fileitem: FileItem, @@ -3719,19 +1687,19 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): if batch_mtype == MediaType.MUSIC: # 明确的音乐批次只接收音频主文件，避免混合下载目录中的视频或字幕 # 被音乐身份和命名模板整理进音乐库。 - if not self.__is_media_file(item, batch_mtype): + if not self._is_media_file(item, batch_mtype): return False - if not self.__is_allow_filesize(item, min_filesize): + if not self._is_allow_filesize(item, min_filesize): return False # 过滤后缀和大小（蓝光目录、附加文件不过滤） elif ( not is_bluray_dir - and not self.__is_subtitle_file(item) - and not self.__is_audio_file(item) + and not self._is_subtitle_file(item) + and not self._is_audio_file(item) ): - if not self.__is_media_file(item, batch_mtype): + if not self._is_media_file(item, batch_mtype): return False - if not self.__is_allow_filesize(item, min_filesize): + if not self._is_allow_filesize(item, min_filesize): return False # 回收站及隐藏的文件不处理 if ( @@ -3804,7 +1772,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): """ 添加待整理文件项并去重。 """ - file_key = self.__get_file_key(item) + file_key = self._get_file_key(item) if file_key in seen_file_keys: return False planned_items.append((item, is_bluray_dir)) @@ -3825,10 +1793,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): for item, is_bluray_dir in items: if not item or item.type != "file": continue - dir_key = self.__get_file_parent_key(item) - if not is_bluray_dir and self.__is_media_file(item, batch_mtype): + dir_key = self._get_file_parent_key(item) + if not is_bluray_dir and self._is_media_file(item, batch_mtype): main_items_by_dir.setdefault(dir_key, []).append(item) - elif self.__is_subtitle_file(item) or self.__is_audio_file(item): + elif self._is_subtitle_file(item) or self._is_audio_file(item): extra_items_by_dir.setdefault(dir_key, []).append((item, is_bluray_dir)) return main_items_by_dir, extra_items_by_dir @@ -3851,10 +1819,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): for item in storagechain.list_files(parent_item, recursion=False) or []: if not item or item.type != "file": continue - if self.__is_media_file(item, batch_mtype): + if self._is_media_file(item, batch_mtype): main_fileitems.append(item) continue - if not (self.__is_subtitle_file(item) or self.__is_audio_file(item)): + if not (self._is_subtitle_file(item) or self._is_audio_file(item)): continue if not _is_allowed_transfer_item(item, False): continue @@ -3877,13 +1845,13 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): (item, is_bluray_dir) for item, is_bluray_dir in items if item - and ( - is_bluray_dir - or ( - item.type == "file" - and self.__is_media_file(item, batch_mtype) - ) - ) + and ( + is_bluray_dir + or ( + item.type == "file" + and self._is_media_file(item, batch_mtype) + ) + ) ] single_file_mode = len(items) == 1 and fileitem.type == "file" @@ -3893,15 +1861,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): sibling_main_items, sibling_extra_items = _get_single_file_sibling_items( current_item ) - current_dir_key = self.__get_file_parent_key(current_item) - if not current_bluray_dir and self.__is_media_file( + current_dir_key = self._get_file_parent_key(current_item) + if not current_bluray_dir and self._is_media_file( current_item, batch_mtype ): main_items = [(current_item, current_bluray_dir)] main_items_by_dir[current_dir_key] = [current_item] extra_items_by_dir[current_dir_key] = sibling_extra_items - elif self.__is_subtitle_file(current_item) or self.__is_audio_file(current_item): - related_main_file_key = self.__get_related_main_file_key( + elif self._is_subtitle_file(current_item) or self._is_audio_file(current_item): + related_main_file_key = self._get_related_main_file_key( extra_fileitem=current_item, main_fileitems=sibling_main_items, ) @@ -3909,7 +1877,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): ( main_item for main_item in sibling_main_items - if self.__get_file_key(main_item) == related_main_file_key + if self._get_file_key(main_item) == related_main_file_key ), None, ) @@ -3920,7 +1888,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): download_history_oper, ) if main_meta: - inherited_map[self.__get_file_key(current_item)] = deepcopy(main_meta) + inherited_map[self._get_file_key(current_item)] = deepcopy(main_meta) return list(items), inherited_map if not main_items: @@ -3951,7 +1919,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): for main_item, main_bluray_dir in main_items: _append_item(planned_items, seen_file_keys, main_item, main_bluray_dir) - if main_bluray_dir or not self.__is_media_file( + if main_bluray_dir or not self._is_media_file( main_item, batch_mtype ): continue @@ -3973,13 +1941,13 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): if not main_meta: continue - dir_key = self.__get_file_parent_key(main_item) + dir_key = self._get_file_parent_key(main_item) main_fileitems = main_items_by_dir.get(dir_key) or [main_item] - main_file_key = self.__get_file_key(main_item) + main_file_key = self._get_file_key(main_item) for extra_item, extra_bluray_dir in extra_items_by_dir.get(dir_key, []): - if self.__get_file_key(extra_item) in seen_file_keys: + if self._get_file_key(extra_item) in seen_file_keys: continue - related_main_file_key = self.__get_related_main_file_key( + related_main_file_key = self._get_related_main_file_key( extra_fileitem=extra_item, main_fileitems=main_fileitems, ) @@ -3991,7 +1959,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): extra_item, extra_bluray_dir, ): - inherited_map[self.__get_file_key(extra_item)] = deepcopy(main_meta) + inherited_map[self._get_file_key(extra_item)] = deepcopy(main_meta) continue if single_file_mode or not sync_extra_files: @@ -4001,7 +1969,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): Path(extra_item.path), subscribe_custom_words, ) - if not self.__is_same_media_meta(main_meta, extra_meta): + if not self._is_same_media_meta(main_meta, extra_meta): continue if _append_item( planned_items, @@ -4009,7 +1977,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): extra_item, extra_bluray_dir, ): - inherited_map[self.__get_file_key(extra_item)] = deepcopy(extra_meta) + inherited_map[self._get_file_key(extra_item)] = deepcopy(extra_meta) for item, is_bluray_dir in items: _append_item(planned_items, seen_file_keys, item, is_bluray_dir) @@ -4075,7 +2043,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): ) if transferd: should_reorganize = manual and ( - reorganize or not transferd.status + reorganize or not transferd.status ) if should_reorganize: state, message = self._delete_manual_transfer_history( @@ -4153,7 +2121,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): if not meta: # 文件元数据(优先使用订阅识别词) inherited_meta = inherited_meta_map.get( - self.__get_file_key(file_item) + self._get_file_key(file_item) ) if history_music_meta: file_meta = history_music_meta @@ -4225,7 +2193,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): else: # 加入列表 if self.__put_to_jobview(transfer_task): - self.__register_scrape_batch_task(transfer_task) + self._register_scrape_batch_task(transfer_task) transfer_tasks.append(transfer_task) else: logger.debug(f"{file_path.name} 已在整理列表中，跳过") @@ -4234,7 +2202,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): finally: file_items.clear() del file_items - self.__close_scrape_batch(transfer_batch_id) + self._close_scrape_batch(transfer_batch_id) # 实时整理 preview_items: List[dict] = [] @@ -4323,7 +2291,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): # 预览模式不走默认回调，这里需要手动收敛任务状态，避免残留 running self.jobview.fail_task(transfer_task) self.jobview.try_remove_job(transfer_task) - if preview and (not preview_items or preview_items[-1].get("source") != transfer_task.fileitem.path): + if preview and ( + not preview_items or preview_items[-1].get("source") != transfer_task.fileitem.path): preview_items.append( { "source": transfer_task.fileitem.path, @@ -4461,7 +2430,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): ]: args_error() return - state, errmsg = self.__re_transfer( + state, errmsg = self._re_transfer( logid=int(logid), mtype=MediaType(type_str), media_source=normalized_source, @@ -4481,338 +2450,6 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton): ) return - @staticmethod - def build_failed_transfer_buttons( - history_id: Optional[int], - ) -> Optional[List[List[dict]]]: - """ - 构建整理失败通知的操作按钮。 - """ - if not history_id: - return None - return [ - [ - {"text": "重试", "callback_data": f"transfer_retry_{history_id}"}, - { - "text": "智能助手接管", - "callback_data": f"transfer_ai_retry_{history_id}", - }, - ] - ] - - def redo_transfer_history(self, history_id: int) -> Tuple[bool, str]: - """ - 按历史记录直接重新整理，自动重新识别媒体信息。 - """ - return self.__re_transfer(logid=history_id) - - @staticmethod - def parse_failed_transfer_callback( - callback_data: str, - ) -> Optional[tuple[str, int]]: - """ - 解析整理失败通知按钮回调。 - """ - for prefix, action in ( - ("transfer_retry_", "retry"), - ("transfer_ai_retry_", "ai_retry"), - ): - if callback_data.startswith(prefix): - history_id = callback_data.replace(prefix, "", 1) - if history_id.isdigit(): - return action, int(history_id) - return None - - def handle_failed_transfer_callback( - self, - *, - callback_data: str, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - ) -> bool: - """ - 处理整理失败通知中的重试类按钮。 - """ - callback = self.parse_failed_transfer_callback(callback_data) - if not callback: - return False - - action, history_id = callback - if action == "retry": - self._retry_transfer_history( - history_id=history_id, - channel=channel, - source=source, - userid=userid, - username=username, - ) - else: - self._take_over_transfer_history_by_ai( - history_id=history_id, - channel=channel, - source=source, - userid=userid, - username=username, - ) - return True - - def _retry_transfer_history( - self, - history_id: int, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - ) -> None: - """ - 立即重新整理一条失败的整理记录。 - """ - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title=f"开始重新整理记录 #{history_id} ...", - save_history=False, - ) - ) - - state, errmsg = self.redo_transfer_history(history_id) - if state: - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title=f"整理记录 #{history_id} 已重新整理", - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - return - - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title="重新整理失败", - text=errmsg, - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - - def _take_over_transfer_history_by_ai( - self, - history_id: int, - channel: MessageChannel, - source: str, - userid: Union[str, int], - username: str, - ) -> None: - """ - 由智能助手接管一条失败的整理记录。 - """ - - if not settings.AI_AGENT_ENABLE: - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title="MoviePilot智能助手未启用，请在系统设置中启用", - save_history=False, - ) - ) - return - - history = TransferHistoryOper().get(history_id) - if not history: - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title="重新整理失败", - text=f"整理记录 #{history_id} 不存在", - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - return - - redo_prompt = build_manual_redo_prompt(history) - - self.post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title=f"已将整理记录 #{history_id} 交给智能助手处理", - text="处理完成后会在这里回复结果。", - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - - async def _run_ai_takeover(): - final_output = "" - - def _capture_output(text_output: str): - nonlocal final_output - final_output = text_output or "" - - try: - await agent_manager.run_background_prompt( - message=redo_prompt, - session_prefix=f"__agent_manual_redo_{history_id}", - output_callback=_capture_output, - reply_mode=ReplyMode.CAPTURE_ONLY, - allow_message_tools=False, - ) - await self.async_post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title="智能助手整理完成", - text=final_output.strip() - or f"整理记录 #{history_id} 已由智能助手处理完成。", - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - except Exception as e: - await self.async_post_message( - Notification( - channel=channel, - source=source, - userid=userid, - username=username, - title="智能助手整理失败", - text=str(e), - link=settings.MP_DOMAIN("#/history"), - save_history=False, - ) - ) - - asyncio.run_coroutine_threadsafe(_run_ai_takeover(), global_vars.loop) - - - def __re_transfer( - self, - logid: int, - mtype: MediaType = None, - media_source: Optional[MediaSource] = None, - media_id: Optional[str] = None, - ) -> Tuple[bool, str]: - """ - 根据历史记录，重新识别整理，只支持简单条件 - :param logid: 历史记录ID - :param mtype: 媒体类型 - :param media_source: 媒体数据源 - :param media_id: 数据源原生 ID，必须与 media_source 成对提供 - """ - # 查询历史记录 - history: TransferHistory = TransferHistoryOper().get(logid) - if not history: - logger.error(f"整理记录不存在，ID：{logid}") - return False, "整理记录不存在" - # 按源目录路径重新整理 - src_path = Path(history.src) - if not src_path.exists(): - return False, f"源目录不存在：{src_path}" - # 查询媒体信息 - explicit_identity = media_source is not None or media_id is not None - if explicit_identity and (not media_source or not media_id): - return False, "媒体重新识别需要同时提供 media_source 和 media_id" - if mtype and media_source and media_id: - mediainfo = MediaChain().recognize_media( - mtype=mtype, - media_source=media_source, - media_id=media_id, - music_type=( - getattr(history, "music_type", None) - if mtype == MediaType.MUSIC - else None - ), - episode_group=history.episode_group, - ) - if mediainfo and not isinstance(mediainfo, MusicInfo): - # 更新媒体图片 - self.obtain_images(mediainfo=mediainfo) - elif history.media_source and history.media_id: - try: - history_type = mtype or MediaType(history.type) - except ValueError: - history_type = mtype - mediainfo = MediaChain().recognize_media( - mtype=history_type, - media_source=history.media_source, - media_id=history.media_id, - music_type=( - getattr(history, "music_type", None) - if history_type == MediaType.MUSIC - else None - ), - episode_group=history.episode_group, - ) - mtype = history_type - if mediainfo and not isinstance(mediainfo, MusicInfo): - self.obtain_images(mediainfo=mediainfo) - elif mtype == MediaType.MUSIC or self._is_music_retry_source(history, src_path): - # 音乐重新整理走音乐识别链，避免默认影视识别误入 TMDB - mtype = MediaType.MUSIC - mediainfo = self._recognize_music_retry_media(history, src_path) - else: - recognize_context = MediaChain().recognize_by_path( - str(src_path), - episode_group=history.episode_group, - obtain_images=True, - ) - mediainfo = recognize_context.media_info if recognize_context else None - # 音乐专辑目录允许无预识别信息，由整理链按音频后缀逐文件解析识别 - if not mediainfo and not (mtype == MediaType.MUSIC and src_path.is_dir()): - return False, ( - f"未识别到媒体信息，类型：{mtype.value if mtype else None}，" - f"media_source：{media_source}，media_id：{media_id}" - ) - # 重新执行整理 - if mediainfo: - logger.info(f"{src_path.name} 识别为：{mediainfo.title_year}") - - # 删除旧的已整理文件 - if history.dest_fileitem: - # 解析目标文件对象 - dest_fileitem = FileItem(**history.dest_fileitem) - StorageChain().delete_file(dest_fileitem) - - # 强制整理 - if history.src_fileitem: - state, errmsg = self.do_transfer( - fileitem=FileItem(**history.src_fileitem), - mediainfo=mediainfo, - mtype=mtype, - download_hash=history.download_hash, - force=True, - background=False, - manual=True, - ) - if not state: - return False, errmsg - - return True, "" - def manual_transfer( self, fileitem: FileItem, diff --git a/app/domain/context.py b/app/domain/context.py index abec1d13f..d78d87b5d 100644 --- a/app/domain/context.py +++ b/app/domain/context.py @@ -1490,6 +1490,11 @@ class MediaInfo: meta = MetaInfo(self.title) season = meta.begin_season if meta.begin_season is not None else 1 episodes_count = info.get("total_episodes") or info.get("eps") + # bangumi 返回的集数可能为字符串，统一转整型避免拼接/范围构造异常 + try: + episodes_count = int(episodes_count) if episodes_count else 0 + except (TypeError, ValueError): + episodes_count = 0 if episodes_count: self.seasons[season] = list(range(1, episodes_count + 1)) self.number_of_episodes = episodes_count diff --git a/app/factory.py b/app/factory.py index 370e544e9..fb36aa9bf 100644 --- a/app/factory.py +++ b/app/factory.py @@ -8,6 +8,7 @@ from fastapi.responses import JSONResponse from starlette.exceptions import HTTPException from app.api.response import ResponseAPIRoute +from app.application.plugins import register_api_app from app.runtime.config import settings from app.runtime.localization import LocaleHelper from app.runtime.log import logger @@ -326,3 +327,7 @@ def create_app() -> FastAPI: # 创建 FastAPI 应用实例 app = create_app() + +# 向 application 层插件路由服务注入应用实例，插件 API 的动态注册/移除 +# 统一经服务完成，避免 api.endpoints 反向依赖本模块。 +register_api_app(app) diff --git a/app/modules/_base/__init__.py b/app/modules/_base/__init__.py new file mode 100644 index 000000000..203021c7b --- /dev/null +++ b/app/modules/_base/__init__.py @@ -0,0 +1,15 @@ +"""模块业务样板基类包。 + +沉淀各内置模块逐字复制的业务样板，模块发现规则 +（`ModuleHelper.load`）会跳过 `_` 前缀的包与类，因此本包不会被识别为可实例化模块。 +""" + +from app.modules._base.downloader import _DownloaderModuleBase +from app.modules._base.mediaserver import _MediaServerModuleBase +from app.modules._base.notification import _MessageChannelModuleBase + +__all__ = [ + "_DownloaderModuleBase", + "_MessageChannelModuleBase", + "_MediaServerModuleBase", +] diff --git a/app/modules/_base/downloader.py b/app/modules/_base/downloader.py new file mode 100644 index 000000000..ec60913ab --- /dev/null +++ b/app/modules/_base/downloader.py @@ -0,0 +1,109 @@ +"""下载器模块业务样板基类。 + +沉淀三个内置下载器模块（qbittorrent/transmission/rtorrent）逐字复制的样板： +连接测试、定时重连、种子信息读取与查询状态归一。差异化逻辑 +（任务添加、原始状态映射、任务列表构建）仍留在各模块。 +""" +from pathlib import Path +from typing import Optional, Tuple, Union + +from torrentool.torrent import Torrent + +from app.domain import torrent as torrent_rules +from app.modules import _DownloaderBase, _ModuleBase, TService +from app.runtime.cache import FileCache +from app.runtime.log import logger +from app.schemas.types import TorrentQueryStatus, TorrentStatus + + +class _DownloaderModuleBase(_ModuleBase, _DownloaderBase[TService]): + """ + 下载器模块业务样板基类。 + """ + + def test(self) -> Optional[Tuple[bool, str]]: + """ + 测试模块连接性 + """ + if not self.get_instances(): + return None + for name, server in self.get_instances().items(): + if server.is_inactive(): + server.reconnect() + if not server.transfer_info(): + return False, f"无法连接{self.get_name()}下载器：{name}" + return True, "" + + def scheduler_job(self) -> None: + """ + 定时任务，每10分钟调用一次 + """ + for name, server in self.get_instances().items(): + if server.is_inactive(): + logger.info(f"{self.get_name()}下载器 {name} 连接断开，尝试重连 ...") + server.reconnect() + + def _get_torrent_info(self, content: Union[Path, str, bytes]) \ + -> Tuple[Optional[Torrent], Optional[bytes]]: + """ + 读取种子内容，返回解析后的种子信息与原始内容，磁力链接不解析 + """ + torrent_info, torrent_content = None, None + try: + if isinstance(content, Path): + if content.exists(): + torrent_content = content.read_bytes() + else: + # 读取缓存的种子文件 + torrent_content = FileCache().get( + content.as_posix(), region="torrents" + ) + else: + torrent_content = content + + if torrent_content: + # 检查是否为磁力链接 + if torrent_rules.is_magnet_link(torrent_content): + return None, torrent_content + else: + torrent_info = Torrent.from_string(torrent_content) + + return torrent_info, torrent_content + except Exception as e: + logger.error(f"获取种子名称失败：{e}") + return None, None + + @staticmethod + def _normalize_query_status( + status: Optional[Union[TorrentStatus, TorrentQueryStatus, str]] + ) -> TorrentQueryStatus: + """ + 归一任务查询状态。 + """ + status_value = getattr(status, "value", status) + status_text = str(status_value or "").strip().lower() + if not status_text or status_text in {"all", "全部"}: + return TorrentQueryStatus.ALL + if status_text in { + TorrentStatus.TRANSFER.value, + TorrentQueryStatus.TRANSFER.value, + "transfer", + }: + return TorrentQueryStatus.TRANSFER + if status_text in { + TorrentStatus.DOWNLOADING.value, + TorrentQueryStatus.DOWNLOADING.value, + "downloading", + }: + return TorrentQueryStatus.DOWNLOADING + if status_text in { + TorrentQueryStatus.COMPLETED.value, + "complete", + "seeding", + "完成", + "已完成", + }: + return TorrentQueryStatus.COMPLETED + if status_text in {TorrentQueryStatus.PAUSED.value, "pause", "暂停", "已暂停"}: + return TorrentQueryStatus.PAUSED + return TorrentQueryStatus.ALL diff --git a/app/modules/_base/mediaserver.py b/app/modules/_base/mediaserver.py new file mode 100644 index 000000000..4b4b3aee9 --- /dev/null +++ b/app/modules/_base/mediaserver.py @@ -0,0 +1,192 @@ +"""媒体服务器模块业务样板基类。 + +沉淀各媒体服务器模块逐字复制的样板：用户辅助认证、媒体存在性检查、 +定时重连与连接测试。服务器差异（认证 API、存在性检查端点、连接探测方式） +通过类属性与钩子方法保留在各模块。 +""" +from typing import Optional, Tuple + +from app import schemas +from app.application.mediaserver import MusicMediaServerHelper +from app.domain.context import MediaInfo +from app.modules import _MediaServerBase, _ModuleBase, TService +from app.runtime.events import eventmanager +from app.runtime.log import logger +from app.schemas.types import ChainEventType, MediaType + + +class _MediaServerModuleBase(_ModuleBase, _MediaServerBase[TService]): + """ + 媒体服务器模块业务样板基类。 + """ + + # 媒体库标识（用于 ExistMediaInfo.server_type，如 "emby"），子类覆写 + _server_type_value: str = "" + + def user_authenticate( + self, + credentials: schemas.AuthCredentials, + service_name: Optional[str] = None, + ) -> Optional[schemas.AuthCredentials]: + """ + 使用媒体服务器用户辅助完成用户认证 + + :param credentials: 认证数据 + :param service_name: 指定要认证的媒体服务器名称，若为 None 则认证所有服务器 + :return: 认证数据 + """ + if not credentials or credentials.grant_type != "password": + return None + # 确定要认证的服务器列表 + if service_name: + # 如果指定了服务名，获取该服务实例 + servers = ( + [(service_name, server)] + if (server := self.get_instance(service_name)) + else [] + ) + else: + # 如果没有指定服务名，遍历所有服务 + servers = self.get_instances().items() + # 遍历要认证的服务器 + for name, server in servers: + # 触发认证拦截事件 + intercept_event = eventmanager.send_event( + etype=ChainEventType.AuthIntercept, + data=schemas.AuthInterceptCredentials( + username=credentials.username, + channel=self.get_name(), + service=name, + status="triggered", + ), + ) + if intercept_event and intercept_event.event_data: + intercept_data: schemas.AuthInterceptCredentials = intercept_event.event_data + if intercept_data.cancel: + continue + token = server.authenticate(credentials.username, credentials.password) + if token: + credentials.channel = self.get_name() + credentials.service = name + credentials.token = token + return credentials + return None + + def media_exists( + self, + mediainfo: MediaInfo, + itemid: Optional[str] = None, + server: Optional[str] = None, + ) -> Optional[schemas.ExistMediaInfo]: + """ + 判断媒体文件是否存在 + + :param mediainfo: 识别的媒体信息 + :param itemid: 媒体服务器ItemID + :param server: 媒体服务器名称 + :return: 如不存在返回None，存在时返回信息，包括每季已存在所有集{type: movie/tv, seasons: {season: [episodes]}} + """ + if server: + servers = [(server, self.get_instance(server))] + else: + servers = self.get_instances().items() + for name, s in servers: + if not s: + continue + if mediainfo.type == MediaType.MUSIC: + # 部分服务器未实现音乐查询，退化为空列表 + matches = getattr(s, "get_music", lambda **_: [])( + **MusicMediaServerHelper.search_params(mediainfo) + ) + match = MusicMediaServerHelper.find_match(mediainfo, matches) + if match: + return schemas.ExistMediaInfo( + type=MediaType.MUSIC, + server_type=self._server_type_value, + server=name, + itemid=match.item_id, + ) + continue + if mediainfo.type == MediaType.MOVIE: + if itemid: + movie = s.get_iteminfo(itemid) + if movie: + logger.info(f"媒体库 {name} 中找到了 {movie}") + return schemas.ExistMediaInfo( + type=MediaType.MOVIE, + server_type=self._server_type_value, + server=name, + itemid=movie.item_id + ) + movies = s.get_movies(title=mediainfo.title, + year=mediainfo.year, + media_source=mediainfo.media_source, + media_id=mediainfo.media_id) + if not movies: + logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") + continue + else: + logger.info(f"媒体库 {name} 中找到了 {movies}") + return schemas.ExistMediaInfo( + type=MediaType.MOVIE, + server_type=self._server_type_value, + server=name, + itemid=movies[0].item_id + ) + else: + itemid, tvs = s.get_tv_episodes(title=mediainfo.title, + year=mediainfo.year, + media_source=mediainfo.media_source, + media_id=mediainfo.media_id, + item_id=itemid) + if not tvs: + logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") + continue + else: + logger.info(f"{mediainfo.title_year} 在媒体库 {name} 中找到 了这些季集：{tvs}") + return schemas.ExistMediaInfo( + type=MediaType.TV, + seasons=tvs, + server_type=self._server_type_value, + server=name, + itemid=itemid + ) + return None + + def scheduler_job(self) -> None: + """ + 定时任务，每10分钟调用一次 + """ + # 定时重连 + for name, server in self.get_instances().items(): + if self._is_inactive(server): + logger.info(f"{self.get_name()}服务器 {name} 连接断开，尝试重连 ...") + server.reconnect() + + def _is_inactive(self, server) -> bool: + """ + 定时重连的失活判断钩子，子类可覆写（如增加配置完整性检查）。 + """ + return server.is_inactive() + + def test(self) -> Optional[Tuple[bool, str]]: + """ + 测试模块连接性 + """ + if not self.get_instances(): + return None + for name, server in self.get_instances().items(): + error = self._test_server(server, name) + if error: + return False, error + return True, "" + + def _test_server(self, server, name: str) -> Optional[str]: + """ + 连接测试钩子，返回失败信息，None 表示就绪，子类可覆写。 + """ + if server.is_inactive(): + server.reconnect() + if not server.get_user(): + return f"无法连接{self.get_name()}服务器：{name}" + return None diff --git a/app/modules/_base/notification.py b/app/modules/_base/notification.py new file mode 100644 index 000000000..83f49557a --- /dev/null +++ b/app/modules/_base/notification.py @@ -0,0 +1,149 @@ +"""消息渠道模块业务样板基类。 + +沉淀各消息渠道模块逐字复制的样板：管理员判断、连接测试、 +斜杠命令注册。渠道差异（客户端类型、菜单 API、前置条件）通过 +类属性与钩子方法保留在各模块。 +""" +import copy +from typing import Dict, List, Optional, Tuple, Union + +from app.application.messaging.agent import ( + matches_channel_admin, + resolve_config_principal_ids, +) +from app.foundation.collections import DictUtils +from app.modules import _MessageBase, _ModuleBase, TService +from app.runtime.events import eventmanager +from app.runtime.log import logger +from app.schemas import CommandRegisterEventData +from app.schemas.types import ChainEventType + + +class _MessageChannelModuleBase(_ModuleBase, _MessageBase[TService]): + """ + 消息渠道模块业务样板基类。 + """ + + # 管理员配置键，子类覆写（如 "TELEGRAM_ADMINS"） + _admin_config_key: str = "" + # 命令注册事件源标识，默认取模块名，子类可覆写 + _command_origin: Optional[str] = None + + @classmethod + def _get_admins(cls, config: Optional[dict]) -> List[str]: + """ + 解析渠道管理员配置，兼容逗号分隔和首尾空白。 + """ + return sorted(resolve_config_principal_ids(config, cls._admin_config_key)) + + def _should_reject_admin_command( + self, + config: Optional[dict], + *user_ids: Optional[Union[str, int]], + ) -> bool: + """ + 判断命令或命令型按钮回调是否应因非管理员身份被拒绝。 + """ + if not self._get_admins(config): + return False + # 模块实例未初始化时 self._channel 为空，退回静态子类型声明 + channel = self._channel or self.get_subtype() + return not matches_channel_admin( + channel, + config, + *user_ids, + ) + + def test(self) -> Optional[Tuple[bool, str]]: + """ + 测试模块连接性 + """ + if not self.get_instances(): + return None + for name, client in self.get_instances().items(): + state, message = self._test_connection(client) + if not state: + suffix = f"：{message}" if message else "" + return False, f"{self.get_name()} {name} 未就绪{suffix}" + return True, "" + + def _test_connection(self, client) -> Tuple[bool, str]: + """ + 连接测试钩子，返回 (是否就绪, 失败信息)，子类可覆写。 + """ + return bool(client.get_state()), "" + + def register_commands(self, commands: Dict[str, dict]) -> None: + """ + 注册命令，实现这个函数接收系统可用的命令菜单 + + :param commands: 命令字典 + """ + for client_config in self.get_configs().values(): + if not self._commands_enabled(client_config.config): + continue + + client = self.get_instance(client_config.name) + if not client: + continue + + # 触发事件，允许调整命令数据，这里需要进行深复制，避免实例共享 + scoped_commands = copy.deepcopy(commands) + event = eventmanager.send_event( + ChainEventType.CommandRegister, + CommandRegisterEventData( + commands=scoped_commands, + origin=self._command_origin or self.get_name(), + service=client_config.name, + ), + ) + + # 如果事件返回有效的 event_data，使用事件中调整后的命令 + if event and event.event_data: + event_data: CommandRegisterEventData = event.event_data + # 如果事件被取消，跳过命令注册，并清理菜单 + if event_data.cancel: + self._delete_commands(client) + logger.debug( + f"Command registration for {client_config.name} canceled by event: {event_data.source}" + ) + continue + scoped_commands = event_data.commands or {} + if not scoped_commands: + logger.debug("Filtered commands are empty, skipping registration.") + self._delete_commands(client) + + # scoped_commands 必须是 commands 的子集 + filtered_scoped_commands = DictUtils.filter_keys_to_subset( + scoped_commands, + commands, + ) + # 如果 filtered_scoped_commands 为空，则跳过注册 + if not filtered_scoped_commands: + logger.debug("Filtered commands are empty, skipping registration.") + self._delete_commands(client) + continue + # 对比调整后的命令与当前命令 + if filtered_scoped_commands != commands: + logger.debug( + f"Command set has changed, Updating new commands: {filtered_scoped_commands}" + ) + self._apply_commands(client, filtered_scoped_commands) + + def _commands_enabled(self, config: Optional[dict]) -> bool: + """ + 命令注册前置条件钩子，返回 False 时跳过该实例，子类可覆写。 + """ + return True + + def _delete_commands(self, client) -> None: + """ + 清理已注册命令的钩子，子类可覆写（如改用菜单 API）。 + """ + client.delete_commands() + + def _apply_commands(self, client, commands: Dict[str, dict]) -> None: + """ + 应用命令集合的钩子，子类可覆写（如改用菜单 API）。 + """ + client.register_commands(commands) diff --git a/app/modules/discord/__init__.py b/app/modules/discord/__init__.py index f2e6d7ca3..575ba879e 100644 --- a/app/modules/discord/__init__.py +++ b/app/modules/discord/__init__.py @@ -1,27 +1,23 @@ -import copy import json from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import quote, unquote from app.domain.context import MediaInfo, Context -from app.runtime.events import eventmanager from app.application.messaging.agent import ( matches_channel_admin, register_channel_admin_resolver, resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.schemas import ( - CommandRegisterEventData, CommingMessage, MessageChannel, MessageResponse, Notification, ) -from app.schemas.types import ChainEventType, ModuleType +from app.schemas.types import ModuleType from app.adapters.network.http import RequestUtils -from app.foundation.collections import DictUtils try: from app.modules.discord.discord import Discord @@ -36,7 +32,9 @@ register_channel_admin_resolver( ) -class DiscordModule(_ModuleBase, _MessageBase[Discord]): +class DiscordModule(_MessageChannelModuleBase[Discord]): + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "DISCORD_ADMINS" _IMAGE_SUFFIXES = ( ".png", ".jpg", @@ -107,51 +105,9 @@ class DiscordModule(_ModuleBase, _MessageBase[Discord]): except Exception as err: logger.error(f"停止Discord模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"Discord {name} Bot 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 Discord 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("DISCORD_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 Discord 命令或命令型按钮回调是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - candidates = [ - str(user_id).strip() - for user_id in user_ids - if user_id is not None and str(user_id).strip() - ] - return not any(candidate in admins for candidate in candidates) - @staticmethod def _send_admin_denied( client: Optional[Discord], @@ -556,54 +512,6 @@ class DiscordModule(_ModuleBase, _MessageBase[Discord]): return True return False - def register_commands(self, commands: Dict[str, dict]) -> None: - """ - 注册命令，实现这个函数接收系统可用的命令菜单。 - - :param commands: 命令字典 - """ - for client_config in self.get_configs().values(): - client = self.get_instance(client_config.name) - if not client: - continue - - scoped_commands = copy.deepcopy(commands) - event = eventmanager.send_event( - ChainEventType.CommandRegister, - CommandRegisterEventData( - commands=scoped_commands, - origin="Discord", - service=client_config.name, - ), - ) - - if event and event.event_data: - event_data: CommandRegisterEventData = event.event_data - if event_data.cancel: - client.delete_commands() - logger.debug( - f"Command registration for {client_config.name} canceled by event: {event_data.source}" - ) - continue - scoped_commands = event_data.commands or {} - if not scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - - filtered_scoped_commands = DictUtils.filter_keys_to_subset( - scoped_commands, - commands, - ) - if not filtered_scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - continue - if filtered_scoped_commands != commands: - logger.debug( - f"Command set has changed, Updating new commands: {filtered_scoped_commands}" - ) - client.register_commands(filtered_scoped_commands) - def mark_message_processing_started( self, channel: MessageChannel, diff --git a/app/modules/emby/__init__.py b/app/modules/emby/__init__.py index 02649d02e..28c6d9fbd 100644 --- a/app/modules/emby/__init__.py +++ b/app/modules/emby/__init__.py @@ -1,16 +1,16 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from app import schemas -from app.domain.context import MediaInfo -from app.runtime.events import eventmanager -from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _MediaServerBase, _ModuleBase +from app.modules._base import _MediaServerModuleBase from app.modules.emby.emby import Emby -from app.schemas.types import MediaType, ModuleType, ChainEventType, MediaServerType +from app.schemas.types import ModuleType, MediaServerType -class EmbyModule(_ModuleBase, _MediaServerBase[Emby]): +class EmbyModule(_MediaServerModuleBase[Emby]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "emby" def init_module(self) -> None: """ @@ -47,70 +47,9 @@ class EmbyModule(_ModuleBase, _MediaServerBase[Emby]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.get_user(): - return False, f"无法连接Emby服务器：{name}" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - # 定时重连 - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"Emby服务器 {name} 连接断开，尝试重连 ...") - server.reconnect() - - def user_authenticate(self, credentials: schemas.AuthCredentials, service_name: Optional[str] = None) \ - -> Optional[schemas.AuthCredentials]: - """ - 使用Emby用户辅助完成用户认证 - :param credentials: 认证数据 - :param service_name: 指定要认证的媒体服务器名称，若为 None 则认证所有服务 - :return: 认证数据 - """ - # Emby认证 - if not credentials or credentials.grant_type != "password": - return None - # 确定要认证的服务器列表 - if service_name: - # 如果指定了服务名，获取该服务实例 - servers = [(service_name, server)] if (server := self.get_instance(service_name)) else [] - else: - # 如果没有指定服务名，遍历所有服务 - servers = self.get_instances().items() - # 遍历要认证的服务器 - for name, server in servers: - # 触发认证拦截事件 - intercept_event = eventmanager.send_event( - etype=ChainEventType.AuthIntercept, - data=schemas.AuthInterceptCredentials(username=credentials.username, channel=self.get_name(), - service=name, status="triggered") - ) - if intercept_event and intercept_event.event_data: - intercept_data: schemas.AuthInterceptCredentials = intercept_event.event_data - if intercept_data.cancel: - continue - token = server.authenticate(credentials.username, credentials.password) - if token: - credentials.channel = self.get_name() - credentials.service = name - credentials.token = token - return credentials - return None - def webhook_parser(self, body: Any, form: Any, args: Any) -> Optional[schemas.WebhookEventInfo]: """ 解析Webhook报文体 @@ -136,79 +75,6 @@ class EmbyModule(_ModuleBase, _MediaServerBase[Emby]): return result return None - def media_exists(self, mediainfo: MediaInfo, itemid: Optional[str] = None, - server: Optional[str] = None) -> Optional[schemas.ExistMediaInfo]: - """ - 判断媒体文件是否存在 - :param mediainfo: 识别的媒体信息 - :param itemid: 媒体服务器ItemID - :param server: 媒体服务器名称 - :return: 如不存在返回None，存在时返回信息，包括每季已存在所有集{type: movie/tv, seasons: {season: [episodes]}} - """ - if server: - servers = [(server, self.get_instance(server))] - else: - servers = self.get_instances().items() - for name, s in servers: - if not s: - continue - if mediainfo.type == MediaType.MUSIC: - matches = s.get_music(**MusicMediaServerHelper.search_params(mediainfo)) - match = MusicMediaServerHelper.find_match(mediainfo, matches) - if match: - return schemas.ExistMediaInfo( - type=MediaType.MUSIC, - server_type="emby", - server=name, - itemid=match.item_id, - ) - continue - if mediainfo.type == MediaType.MOVIE: - if itemid: - movie = s.get_iteminfo(itemid) - if movie: - logger.info(f"媒体库 {name} 中找到了 {movie}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="emby", - server=name, - itemid=movie.item_id - ) - movies = s.get_movies(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id) - if not movies: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"媒体库 {name} 中找到了 {movies}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="emby", - server=name, - itemid=movies[0].item_id - ) - else: - itemid, tvs = s.get_tv_episodes(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - item_id=itemid) - if not tvs: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"{mediainfo.title_year} 在媒体库 {name} 中找到了这些季集：{tvs}") - return schemas.ExistMediaInfo( - type=MediaType.TV, - seasons=tvs, - server_type="emby", - server=name, - itemid=itemid - ) - return None - def media_statistic(self, server: Optional[str] = None) -> Optional[List[schemas.Statistic]]: """ 媒体数量统计 diff --git a/app/modules/feishu/__init__.py b/app/modules/feishu/__init__.py index f9b5c1def..21f54562f 100644 --- a/app/modules/feishu/__init__.py +++ b/app/modules/feishu/__init__.py @@ -3,7 +3,7 @@ from typing import Any, List, Optional, Tuple, Union from app.domain.context import Context, MediaInfo from app.application.messaging.agent import register_channel_admin_resolver, resolve_config_principal_ids from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.feishu.feishu import Feishu from app.schemas import CommingMessage, MessageChannel, MessageResponse, Notification from app.schemas.types import ModuleType @@ -17,7 +17,7 @@ register_channel_admin_resolver( ) -class FeishuModule(_ModuleBase, _MessageBase[Feishu]): +class FeishuModule(_MessageChannelModuleBase[Feishu]): def init_module(self) -> None: super().init_service(service_name=Feishu.__name__.lower(), service_type=Feishu) self._channel = MessageChannel.Feishu @@ -46,15 +46,6 @@ class FeishuModule(_ModuleBase, _MessageBase[Feishu]): except Exception as err: logger.error(f"停止飞书模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"飞书 {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: """通知模块通过系统通知配置控制实例化，这里不额外设置环境开关。""" return None diff --git a/app/modules/jellyfin/__init__.py b/app/modules/jellyfin/__init__.py index d9344668b..517a88edf 100644 --- a/app/modules/jellyfin/__init__.py +++ b/app/modules/jellyfin/__init__.py @@ -1,17 +1,16 @@ from typing import Any, Dict, Generator, List, Optional, Tuple, Union from app import schemas -from app.domain.context import MediaInfo -from app.runtime.events import eventmanager -from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _MediaServerBase, _ModuleBase +from app.modules._base import _MediaServerModuleBase from app.modules.jellyfin.jellyfin import Jellyfin -from app.schemas import AuthCredentials, AuthInterceptCredentials -from app.schemas.types import MediaType, ModuleType, ChainEventType, MediaServerType +from app.schemas.types import ModuleType, MediaServerType -class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]): +class JellyfinModule(_MediaServerModuleBase[Jellyfin]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "jellyfin" def init_module(self) -> None: """ @@ -48,70 +47,9 @@ class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]): def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - # 定时重连 - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"Jellyfin {name} 服务器连接断开，尝试重连 ...") - server.reconnect() - def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.get_user(): - return False, f"无法连接Jellyfin服务器：{name}" - return True, "" - - def user_authenticate(self, credentials: AuthCredentials, service_name: Optional[str] = None) \ - -> Optional[AuthCredentials]: - """ - 使用Jellyfin用户辅助完成用户认证 - :param credentials: 认证数据 - :param service_name: 指定要认证的媒体服务器名称，若为 None 则认证所有服务 - :return: 认证数据 - """ - # Jellyfin认证 - if not credentials or credentials.grant_type != "password": - return None - # 确定要认证的服务器列表 - if service_name: - # 如果指定了服务名，获取该服务实例 - servers = [(service_name, server)] if (server := self.get_instance(service_name)) else [] - else: - # 如果没有指定服务名，遍历所有服务 - servers = self.get_instances().items() - # 遍历要认证的服务器 - for name, server in servers: - # 触发认证拦截事件 - intercept_event = eventmanager.send_event( - etype=ChainEventType.AuthIntercept, - data=AuthInterceptCredentials(username=credentials.username, channel=self.get_name(), - service=name, status="triggered") - ) - if intercept_event and intercept_event.event_data: - intercept_data: AuthInterceptCredentials = intercept_event.event_data - if intercept_data.cancel: - continue - token = server.authenticate(credentials.username, credentials.password) - if token: - credentials.channel = self.get_name() - credentials.service = name - credentials.token = token - return credentials - return None - def webhook_parser(self, body: Any, form: Any, args: Any) -> Optional[schemas.WebhookEventInfo]: """ 解析Webhook报文体 @@ -137,79 +75,6 @@ class JellyfinModule(_ModuleBase, _MediaServerBase[Jellyfin]): return result return None - def media_exists(self, mediainfo: MediaInfo, itemid: Optional[str] = None, - server: Optional[str] = None) -> Optional[schemas.ExistMediaInfo]: - """ - 判断媒体文件是否存在 - :param mediainfo: 识别的媒体信息 - :param itemid: 媒体服务器ItemID - :param server: 媒体服务器名称 - :return: 如不存在返回None，存在时返回信息，包括每季已存在所有集{type: movie/tv, seasons: {season: [episodes]}} - """ - if server: - servers = [(server, self.get_instance(server))] - else: - servers = self.get_instances().items() - for name, s in servers: - if not s: - continue - if mediainfo.type == MediaType.MUSIC: - matches = s.get_music(**MusicMediaServerHelper.search_params(mediainfo)) - match = MusicMediaServerHelper.find_match(mediainfo, matches) - if match: - return schemas.ExistMediaInfo( - type=MediaType.MUSIC, - server_type="jellyfin", - server=name, - itemid=match.item_id, - ) - continue - if mediainfo.type == MediaType.MOVIE: - if itemid: - movie = s.get_iteminfo(itemid) - if movie: - logger.info(f"媒体库 {name} 中找到了 {movie}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="jellyfin", - server=name, - itemid=movie.item_id - ) - movies = s.get_movies(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id) - if not movies: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"媒体库 {name} 中找到了 {movies}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="jellyfin", - server=name, - itemid=movies[0].item_id - ) - else: - itemid, tvs = s.get_tv_episodes(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - item_id=itemid) - if not tvs: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"{mediainfo.title_year} 在媒体库 {name} 中找到了这些季集：{tvs}") - return schemas.ExistMediaInfo( - type=MediaType.TV, - seasons=tvs, - server_type="jellyfin", - server=name, - itemid=itemid - ) - return None - def media_statistic(self, server: Optional[str] = None) -> Optional[List[schemas.Statistic]]: """ 媒体数量统计 diff --git a/app/modules/plex/__init__.py b/app/modules/plex/__init__.py index c8dd4b0be..531879b69 100644 --- a/app/modules/plex/__init__.py +++ b/app/modules/plex/__init__.py @@ -5,13 +5,16 @@ from app.domain.context import MediaInfo from app.runtime.events import eventmanager from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _ModuleBase, _MediaServerBase +from app.modules._base import _MediaServerModuleBase from app.modules.plex.plex import Plex from app.schemas import AuthCredentials, AuthInterceptCredentials from app.schemas.types import MediaType, ModuleType, ChainEventType, MediaServerType -class PlexModule(_ModuleBase, _MediaServerBase[Plex]): +class PlexModule(_MediaServerModuleBase[Plex]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "plex" def init_module(self) -> None: """ @@ -54,32 +57,17 @@ class PlexModule(_ModuleBase, _MediaServerBase[Plex]): except Exception as err: logger.error(f"停止Plex模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.get_librarys(): - return False, f"无法连接Plex服务器：{name}" - return True, "" + def _test_server(self, server, name: str) -> Optional[str]: + """Plex 用媒体库列表探测连接状态。""" + if server.is_inactive(): + server.reconnect() + if not server.get_librarys(): + return f"无法连接Plex服务器：{name}" + return None def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - # 定时重连 - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"Plex {name} 服务器连接断开，尝试重连 ...") - server.reconnect() - def user_authenticate(self, credentials: AuthCredentials, service_name: Optional[str] = None) \ -> Optional[AuthCredentials]: """ diff --git a/app/modules/qbittorrent/__init__.py b/app/modules/qbittorrent/__init__.py index ee5425df4..178586faf 100644 --- a/app/modules/qbittorrent/__init__.py +++ b/app/modules/qbittorrent/__init__.py @@ -2,14 +2,12 @@ from pathlib import Path from typing import Set, Tuple, Optional, Union, List, Dict from qbittorrentapi import TorrentFilesList -from torrentool.torrent import Torrent from app import schemas -from app.runtime.cache import FileCache from app.runtime.config import settings from app.domain.metainfo import MetaInfo from app.runtime.log import logger -from app.modules import _ModuleBase, _DownloaderBase +from app.modules._base import _DownloaderModuleBase from app.modules.qbittorrent.qbittorrent import Qbittorrent from app.schemas import DownloaderTorrent from app.schemas.types import ( @@ -19,7 +17,6 @@ from app.schemas.types import ( TorrentQueryStatus, TorrentStatus, ) -from app.domain import torrent as torrent_rules from app.foundation import size as size_tools from app.foundation import temporal as time_tools from app.foundation import text as text_tools @@ -44,7 +41,7 @@ _TORRENT_FILES_RETRY_TIMES = 5 _TORRENT_FILES_RETRY_INTERVAL = 1 -class QbittorrentModule(_ModuleBase, _DownloaderBase[Qbittorrent]): +class QbittorrentModule(_DownloaderModuleBase[Qbittorrent]): """ qBittorrent 下载器模块，负责下载任务添加、文件选择和任务管理。 """ @@ -90,34 +87,12 @@ class QbittorrentModule(_ModuleBase, _DownloaderBase[Qbittorrent]): """ pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.transfer_info(): - return False, f"无法连接Qbittorrent下载器：{name}" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: """ 返回控制模块启用状态的配置项 """ pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"Qbittorrent下载器 {name} 连接断开，尝试重连 ...") - server.reconnect() - def download(self, content: Union[Path, str, bytes], download_dir: Path, cookie: str, episodes: Set[int] = None, category: Optional[str] = None, label: Optional[str] = None, downloader: Optional[str] = None) -> Optional[Tuple[Optional[str], Optional[str], Optional[str], str]]: @@ -132,39 +107,11 @@ class QbittorrentModule(_ModuleBase, _DownloaderBase[Qbittorrent]): :param downloader: 下载器 :return: 下载器名称、种子Hash、种子文件布局、错误原因 """ - - def __get_torrent_info() -> Tuple[Optional[Torrent], Optional[bytes]]: - """ - 获取种子名称 - """ - torrent_info, torrent_content = None, None - try: - if isinstance(content, Path): - if content.exists(): - torrent_content = content.read_bytes() - else: - # 读取缓存的种子文件 - torrent_content = FileCache().get(content.as_posix(), region="torrents") - else: - torrent_content = content - - if torrent_content: - # 检查是否为磁力链接 - if torrent_rules.is_magnet_link(torrent_content): - return None, torrent_content - else: - torrent_info = Torrent.from_string(torrent_content) - - return torrent_info, torrent_content - except Exception as e: - logger.error(f"获取种子名称失败：{e}") - return None, None - if not content: return None, None, None, "下载内容为空" # 读取种子的名称 - torrent_from_file, content = __get_torrent_info() + torrent_from_file, content = self._get_torrent_info(content) # 检查是否为磁力链接 is_magnet = isinstance(content, str) and content.startswith("magnet:") or isinstance(content, bytes) and content.startswith( @@ -302,7 +249,7 @@ class QbittorrentModule(_ModuleBase, _DownloaderBase[Qbittorrent]): else: servers: Dict[str, Qbittorrent] = self.get_instances() ret_torrents = [] - query_status = self.__normalize_query_status(status) + query_status = self._normalize_query_status(status) query_tags = None if include_all_tags else settings.TORRENT_TAG def __get_torrent_path(torrent_data: dict) -> Path: @@ -408,41 +355,6 @@ class QbittorrentModule(_ModuleBase, _DownloaderBase[Qbittorrent]): return None return ret_torrents # noqa - @staticmethod - def __normalize_query_status( - status: Optional[Union[TorrentStatus, TorrentQueryStatus, str]] - ) -> TorrentQueryStatus: - """ - 归一任务查询状态。 - """ - status_value = getattr(status, "value", status) - status_text = str(status_value or "").strip().lower() - if not status_text or status_text in {"all", "全部"}: - return TorrentQueryStatus.ALL - if status_text in { - TorrentStatus.TRANSFER.value, - TorrentQueryStatus.TRANSFER.value, - "transfer", - }: - return TorrentQueryStatus.TRANSFER - if status_text in { - TorrentStatus.DOWNLOADING.value, - TorrentQueryStatus.DOWNLOADING.value, - "downloading", - }: - return TorrentQueryStatus.DOWNLOADING - if status_text in { - TorrentQueryStatus.COMPLETED.value, - "complete", - "seeding", - "完成", - "已完成", - }: - return TorrentQueryStatus.COMPLETED - if status_text in {TorrentQueryStatus.PAUSED.value, "pause", "暂停", "已暂停"}: - return TorrentQueryStatus.PAUSED - return TorrentQueryStatus.ALL - @staticmethod def __normalize_torrent_state(state: Optional[Union[str, int]]) -> str: """ diff --git a/app/modules/qqbot/__init__.py b/app/modules/qqbot/__init__.py index 47583a1e4..2c16a1392 100644 --- a/app/modules/qqbot/__init__.py +++ b/app/modules/qqbot/__init__.py @@ -15,7 +15,7 @@ from app.application.messaging.agent import ( resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.qqbot.qqbot import QQBot from app.schemas import CommingMessage, MessageChannel, Notification from app.schemas.types import ModuleType @@ -30,9 +30,12 @@ register_channel_admin_resolver( ) -class QQBotModule(_ModuleBase, _MessageBase[QQBot]): +class QQBotModule(_MessageChannelModuleBase[QQBot]): """QQ Bot 通知模块""" + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "QQBOT_ADMINS" + _IMAGE_SUFFIXES = ( ".png", ".jpg", @@ -86,46 +89,9 @@ class QQBotModule(_ModuleBase, _MessageBase[QQBot]): except Exception as err: logger.error(f"停止QQ Bot模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - if not client.get_state(): - return False, f"QQ Bot {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 QQ 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("QQBOT_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 QQ 斜杠命令是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - return not matches_channel_admin( - MessageChannel.QQ, - config, - *user_ids, - ) - @staticmethod def _send_admin_denied( client: Optional[QQBot], userid: Optional[Union[str, int]] diff --git a/app/modules/rtorrent/__init__.py b/app/modules/rtorrent/__init__.py index 8dc71388e..9d1a655c4 100644 --- a/app/modules/rtorrent/__init__.py +++ b/app/modules/rtorrent/__init__.py @@ -1,14 +1,11 @@ from pathlib import Path from typing import Set, Tuple, Optional, Union, List, Dict -from torrentool.torrent import Torrent - from app import schemas -from app.runtime.cache import FileCache from app.runtime.config import settings from app.domain.metainfo import MetaInfo from app.runtime.log import logger -from app.modules import _ModuleBase, _DownloaderBase +from app.modules._base import _DownloaderModuleBase from app.modules.rtorrent.rtorrent import Rtorrent from app.schemas import DownloaderTorrent from app.schemas.types import ( @@ -18,13 +15,12 @@ from app.schemas.types import ( TorrentQueryStatus, TorrentStatus, ) -from app.domain import torrent as torrent_rules from app.foundation import size as size_tools from app.foundation import temporal as time_tools from app.foundation import text as text_tools -class RtorrentModule(_ModuleBase, _DownloaderBase[Rtorrent]): +class RtorrentModule(_DownloaderModuleBase[Rtorrent]): def init_module(self) -> None: """ 初始化模块 @@ -61,31 +57,9 @@ class RtorrentModule(_ModuleBase, _DownloaderBase[Rtorrent]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.transfer_info(): - return False, f"无法连接rTorrent下载器：{name}" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"rTorrent下载器 {name} 连接断开，尝试重连 ...") - server.reconnect() - def download( self, content: Union[Path, str, bytes], @@ -108,38 +82,11 @@ class RtorrentModule(_ModuleBase, _DownloaderBase[Rtorrent]): :return: 下载器名称、种子Hash、种子文件布局、错误原因 """ - def __get_torrent_info() -> Tuple[Optional[Torrent], Optional[bytes]]: - """ - 获取种子名称 - """ - torrent_info, torrent_content = None, None - try: - if isinstance(content, Path): - if content.exists(): - torrent_content = content.read_bytes() - else: - torrent_content = FileCache().get( - content.as_posix(), region="torrents" - ) - else: - torrent_content = content - - if torrent_content: - if torrent_rules.is_magnet_link(torrent_content): - return None, torrent_content - else: - torrent_info = Torrent.from_string(torrent_content) - - return torrent_info, torrent_content - except Exception as e: - logger.error(f"获取种子名称失败：{e}") - return None, None - if not content: return None, None, None, "下载内容为空" # 读取种子的名称 - torrent_from_file, content = __get_torrent_info() + torrent_from_file, content = self._get_torrent_info(content) # 检查是否为磁力链接 is_magnet = ( isinstance(content, str) @@ -311,7 +258,7 @@ class RtorrentModule(_ModuleBase, _DownloaderBase[Rtorrent]): else: servers: Dict[str, Rtorrent] = self.get_instances() ret_torrents = [] - query_status = self.__normalize_query_status(status) + query_status = self._normalize_query_status(status) query_tags = None if include_all_tags else settings.TORRENT_TAG def __get_torrent_path(torrent_data: dict) -> Path: @@ -424,41 +371,6 @@ class RtorrentModule(_ModuleBase, _DownloaderBase[Rtorrent]): return None return ret_torrents # noqa - @staticmethod - def __normalize_query_status( - status: Optional[Union[TorrentStatus, TorrentQueryStatus, str]] - ) -> TorrentQueryStatus: - """ - 归一任务查询状态。 - """ - status_value = getattr(status, "value", status) - status_text = str(status_value or "").strip().lower() - if not status_text or status_text in {"all", "全部"}: - return TorrentQueryStatus.ALL - if status_text in { - TorrentStatus.TRANSFER.value, - TorrentQueryStatus.TRANSFER.value, - "transfer", - }: - return TorrentQueryStatus.TRANSFER - if status_text in { - TorrentStatus.DOWNLOADING.value, - TorrentQueryStatus.DOWNLOADING.value, - "downloading", - }: - return TorrentQueryStatus.DOWNLOADING - if status_text in { - TorrentQueryStatus.COMPLETED.value, - "complete", - "seeding", - "完成", - "已完成", - }: - return TorrentQueryStatus.COMPLETED - if status_text in {TorrentQueryStatus.PAUSED.value, "pause", "暂停", "已暂停"}: - return TorrentQueryStatus.PAUSED - return TorrentQueryStatus.ALL - @staticmethod def __normalize_torrent_state( state: Optional[Union[int, str]], diff --git a/app/modules/slack/__init__.py b/app/modules/slack/__init__.py index 631ae5304..42a6b57c4 100644 --- a/app/modules/slack/__init__.py +++ b/app/modules/slack/__init__.py @@ -1,28 +1,24 @@ -import copy import json import re from typing import Any, Dict, List, Optional, Tuple, Union from urllib.parse import quote, unquote from app.domain.context import MediaInfo, Context -from app.runtime.events import eventmanager from app.application.messaging.agent import ( matches_channel_admin, register_channel_admin_resolver, resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.slack.slack import Slack from app.schemas import ( - CommandRegisterEventData, CommingMessage, MessageChannel, MessageResponse, Notification, ) -from app.schemas.types import ChainEventType, ModuleType -from app.foundation.collections import DictUtils +from app.schemas.types import ModuleType register_channel_admin_resolver( @@ -31,7 +27,9 @@ register_channel_admin_resolver( ) -class SlackModule(_ModuleBase, _MessageBase[Slack]): +class SlackModule(_MessageChannelModuleBase[Slack]): + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "SLACK_ADMINS" PROCESSING_REACTION = "eyes" _AUDIO_SUFFIXES = ( ".mp3", @@ -88,51 +86,9 @@ class SlackModule(_ModuleBase, _MessageBase[Slack]): except Exception as err: logger.error(f"停止Slack模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"Slack {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 Slack 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("SLACK_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 Slack 命令或命令型按钮回调是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - candidates = [ - str(user_id).strip() - for user_id in user_ids - if user_id is not None and str(user_id).strip() - ] - return not any(candidate in admins for candidate in candidates) - @staticmethod def _send_admin_denied(client: Optional[Slack], userid: Optional[Union[str, int]]) -> None: """ @@ -688,54 +644,6 @@ class SlackModule(_ModuleBase, _MessageBase[Slack]): return True return False - def register_commands(self, commands: Dict[str, dict]) -> None: - """ - 注册命令，实现这个函数接收系统可用的命令菜单。 - - :param commands: 命令字典 - """ - for client_config in self.get_configs().values(): - client = self.get_instance(client_config.name) - if not client: - continue - - scoped_commands = copy.deepcopy(commands) - event = eventmanager.send_event( - ChainEventType.CommandRegister, - CommandRegisterEventData( - commands=scoped_commands, - origin="Slack", - service=client_config.name, - ), - ) - - if event and event.event_data: - event_data: CommandRegisterEventData = event.event_data - if event_data.cancel: - client.delete_commands() - logger.debug( - f"Command registration for {client_config.name} canceled by event: {event_data.source}" - ) - continue - scoped_commands = event_data.commands or {} - if not scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - - filtered_scoped_commands = DictUtils.filter_keys_to_subset( - scoped_commands, - commands, - ) - if not filtered_scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - continue - if filtered_scoped_commands != commands: - logger.debug( - f"Command set has changed, Updating new commands: {filtered_scoped_commands}" - ) - client.register_commands(filtered_scoped_commands) - def mark_message_processing_started( self, channel: MessageChannel, diff --git a/app/modules/synologychat/__init__.py b/app/modules/synologychat/__init__.py index d47d8af02..364206528 100644 --- a/app/modules/synologychat/__init__.py +++ b/app/modules/synologychat/__init__.py @@ -9,7 +9,7 @@ from app.application.messaging.agent import ( resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.synologychat.synologychat import SynologyChat from app.schemas import MessageChannel, CommingMessage, Notification from app.schemas.types import ModuleType @@ -22,7 +22,9 @@ register_channel_admin_resolver( ) -class SynologyChatModule(_ModuleBase, _MessageBase[SynologyChat]): +class SynologyChatModule(_MessageChannelModuleBase[SynologyChat]): + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "SYNOLOGYCHAT_ADMINS" _IMAGE_SUFFIXES = ( ".png", ".jpg", @@ -84,51 +86,9 @@ class SynologyChatModule(_ModuleBase, _MessageBase[SynologyChat]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"Synology Chat {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 Synology Chat 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("SYNOLOGYCHAT_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 Synology Chat 斜杠命令是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - candidates = [ - str(user_id).strip() - for user_id in user_ids - if user_id is not None and str(user_id).strip() - ] - return not any(candidate in admins for candidate in candidates) - @staticmethod def _send_admin_denied( client: Optional[SynologyChat], userid: Optional[Union[str, int]] diff --git a/app/modules/telegram/__init__.py b/app/modules/telegram/__init__.py index fd2841210..c10e1ba85 100644 --- a/app/modules/telegram/__init__.py +++ b/app/modules/telegram/__init__.py @@ -1,28 +1,24 @@ -import copy import json import re from typing import Dict, Optional, Union, List, Tuple, Any from app.domain.context import MediaInfo, Context -from app.runtime.events import eventmanager from app.application.messaging.agent import ( matches_channel_admin, register_channel_admin_resolver, resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.telegram.telegram import Telegram from app.schemas import ( MessageChannel, CommingMessage, Notification, - CommandRegisterEventData, NotificationConf, MessageResponse, ) -from app.schemas.types import ModuleType, ChainEventType -from app.foundation.collections import DictUtils +from app.schemas.types import ModuleType register_channel_admin_resolver( @@ -33,11 +29,14 @@ register_channel_admin_resolver( ) -class TelegramModule(_ModuleBase, _MessageBase[Telegram]): +class TelegramModule(_MessageChannelModuleBase[Telegram]): """ Telegram 通知模块，负责模块生命周期、消息解析和通知发送。 """ + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "TELEGRAM_ADMINS" + def init_module(self) -> None: """ 初始化模块 @@ -83,53 +82,12 @@ class TelegramModule(_ModuleBase, _MessageBase[Telegram]): except Exception as err: logger.error(f"停止Telegram模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"Telegram {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: """ 获取模块初始化配置项。 """ pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 Telegram 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("TELEGRAM_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 Telegram 命令或命令型按钮回调是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - return not matches_channel_admin( - MessageChannel.Telegram, - config, - *user_ids, - ) - def message_parser( self, source: str, body: Any, form: Any, args: Any ) -> Optional[CommingMessage]: @@ -795,58 +753,6 @@ class TelegramModule(_ModuleBase, _MessageBase[Telegram]): ) return None - def register_commands(self, commands: Dict[str, dict]): - """ - 注册命令，实现这个函数接收系统可用的命令菜单 - :param commands: 命令字典 - """ - for client_config in self.get_configs().values(): - client = self.get_instance(client_config.name) - if not client: - continue - - # 触发事件，允许调整命令数据，这里需要进行深复制，避免实例共享 - scoped_commands = copy.deepcopy(commands) - event = eventmanager.send_event( - ChainEventType.CommandRegister, - CommandRegisterEventData( - commands=scoped_commands, - origin="Telegram", - service=client_config.name, - ), - ) - - # 如果事件返回有效的 event_data，使用事件中调整后的命令 - if event and event.event_data: - event_data: CommandRegisterEventData = event.event_data - # 如果事件被取消，跳过命令注册，并清理菜单 - if event_data.cancel: - client.delete_commands() - logger.debug( - f"Command registration for {client_config.name} canceled by event: {event_data.source}" - ) - continue - scoped_commands = event_data.commands or {} - if not scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - - # scoped_commands 必须是 commands 的子集 - filtered_scoped_commands = DictUtils.filter_keys_to_subset( - scoped_commands, commands - ) - # 如果 filtered_scoped_commands 为空，则跳过注册 - if not filtered_scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_commands() - continue - # 对比调整后的命令与当前命令 - if filtered_scoped_commands != commands: - logger.debug( - f"Command set has changed, Updating new commands: {filtered_scoped_commands}" - ) - client.register_commands(filtered_scoped_commands) - def download_telegram_file_to_base64(self, file_id: str, source: str) -> Optional[str]: """ 下载Telegram文件并转为base64 diff --git a/app/modules/transmission/__init__.py b/app/modules/transmission/__init__.py index 0b2a72b85..118dfb5b3 100644 --- a/app/modules/transmission/__init__.py +++ b/app/modules/transmission/__init__.py @@ -1,15 +1,13 @@ from pathlib import Path from typing import Set, Tuple, Optional, Union, List, Dict -from torrentool.torrent import Torrent from transmission_rpc import File from app import schemas -from app.runtime.cache import FileCache from app.runtime.config import settings from app.domain.metainfo import MetaInfo from app.runtime.log import logger -from app.modules import _ModuleBase, _DownloaderBase +from app.modules._base import _DownloaderModuleBase from app.modules.transmission.transmission import Transmission from app.schemas import DownloaderTorrent from app.schemas.types import ( @@ -19,7 +17,6 @@ from app.schemas.types import ( TorrentQueryStatus, TorrentStatus, ) -from app.domain import torrent as torrent_rules from app.foundation import size as size_tools from app.foundation import temporal as time_tools @@ -32,7 +29,7 @@ _TRANSMISSION_PAUSED_STATES = { } -class TransmissionModule(_ModuleBase, _DownloaderBase[Transmission]): +class TransmissionModule(_DownloaderModuleBase[Transmission]): def init_module(self) -> None: """ @@ -69,32 +66,9 @@ class TransmissionModule(_ModuleBase, _DownloaderBase[Transmission]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive(): - server.reconnect() - if not server.transfer_info(): - return False, f"无法连接Transmission下载器：{name}" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - # 定时重连 - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"Transmission下载器 {name} 连接断开，尝试重连 ...") - server.reconnect() - def download(self, content: Union[Path, str, bytes], download_dir: Path, cookie: str, episodes: Set[int] = None, category: Optional[str] = None, label: Optional[str] = None, downloader: Optional[str] = None) -> Optional[Tuple[Optional[str], Optional[str], Optional[str], str]]: @@ -110,38 +84,11 @@ class TransmissionModule(_ModuleBase, _DownloaderBase[Transmission]): :return: 下载器名称、种子Hash、种子文件布局、错误原因 """ - def __get_torrent_info() -> Tuple[Optional[Torrent], Optional[bytes]]: - """ - 获取种子名称 - """ - torrent_info, torrent_content = None, None - try: - if isinstance(content, Path): - if content.exists(): - torrent_content = content.read_bytes() - else: - # 读取缓存的种子文件 - torrent_content = FileCache().get(content.as_posix(), region="torrents") - else: - torrent_content = content - - if torrent_content: - # 检查是否为磁力链接 - if torrent_rules.is_magnet_link(torrent_content): - return None, torrent_content - else: - torrent_info = Torrent.from_string(torrent_content) - - return torrent_info, torrent_content - except Exception as e: - logger.error(f"获取种子名称失败：{e}") - return None, None - if not content: return None, None, None, "下载内容为空" # 读取种子的名称 - torrent_from_file, content = __get_torrent_info() + torrent_from_file, content = self._get_torrent_info(content) # 检查是否为磁力链接 is_magnet = isinstance(content, str) and content.startswith("magnet:") or isinstance(content, bytes) and content.startswith( @@ -261,7 +208,7 @@ class TransmissionModule(_ModuleBase, _DownloaderBase[Transmission]): else: servers: Dict[str, Transmission] = self.get_instances() ret_torrents = [] - query_status = self.__normalize_query_status(status) + query_status = self._normalize_query_status(status) query_tags = None if include_all_tags else settings.TORRENT_TAG def __get_torrent_attr(torrent_data, *attr_names): @@ -406,41 +353,6 @@ class TransmissionModule(_ModuleBase, _DownloaderBase[Transmission]): return None return ret_torrents # noqa - @staticmethod - def __normalize_query_status( - status: Optional[Union[TorrentStatus, TorrentQueryStatus, str]] - ) -> TorrentQueryStatus: - """ - 归一任务查询状态。 - """ - status_value = getattr(status, "value", status) - status_text = str(status_value or "").strip().lower() - if not status_text or status_text in {"all", "全部"}: - return TorrentQueryStatus.ALL - if status_text in { - TorrentStatus.TRANSFER.value, - TorrentQueryStatus.TRANSFER.value, - "transfer", - }: - return TorrentQueryStatus.TRANSFER - if status_text in { - TorrentStatus.DOWNLOADING.value, - TorrentQueryStatus.DOWNLOADING.value, - "downloading", - }: - return TorrentQueryStatus.DOWNLOADING - if status_text in { - TorrentQueryStatus.COMPLETED.value, - "complete", - "seeding", - "完成", - "已完成", - }: - return TorrentQueryStatus.COMPLETED - if status_text in {TorrentQueryStatus.PAUSED.value, "pause", "暂停", "已暂停"}: - return TorrentQueryStatus.PAUSED - return TorrentQueryStatus.ALL - @staticmethod def __normalize_torrent_state(status: Optional[str]) -> str: """ diff --git a/app/modules/trimemedia/__init__.py b/app/modules/trimemedia/__init__.py index 6f04d4614..817e4dadb 100644 --- a/app/modules/trimemedia/__init__.py +++ b/app/modules/trimemedia/__init__.py @@ -1,17 +1,16 @@ from typing import Any, Generator, List, Optional, Tuple, Union from app import schemas -from app.domain.context import MediaInfo -from app.runtime.events import eventmanager -from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _MediaServerBase, _ModuleBase +from app.modules._base import _MediaServerModuleBase from app.modules.trimemedia.trimemedia import TrimeMedia -from app.schemas import AuthCredentials, AuthInterceptCredentials -from app.schemas.types import ChainEventType, MediaServerType, MediaType, ModuleType +from app.schemas.types import MediaServerType, ModuleType -class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]): +class TrimeMediaModule(_MediaServerModuleBase[TrimeMedia]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "trimemedia" def init_module(self) -> None: """ @@ -52,15 +51,9 @@ class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]): def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - # 定时重连 - for name, server in self.get_instances().items(): - if server.is_configured() and server.is_inactive(): - logger.info(f"飞牛影视 {name} 连接断开，尝试重连 ...") - server.reconnect() + def _is_inactive(self, server) -> bool: + """未配置的实例不参与定时重连。""" + return server.is_configured() and server.is_inactive() def stop(self) -> None: """停止模块""" @@ -71,65 +64,12 @@ class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]): except Exception as err: logger.error(f"停止飞牛影视模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if not server.is_configured(): - return False, f"飞牛影视配置不完整：{name}" - if server.is_inactive() and not server.reconnect(): - return False, f"无法连接飞牛影视：{name}" - return True, "" - - def user_authenticate( - self, credentials: AuthCredentials, service_name: Optional[str] = None - ) -> Optional[AuthCredentials]: - """ - 使用飞牛影视用户辅助完成用户认证 - - :param credentials: 认证数据 - :param service_name: 指定要认证的媒体服务器名称，若为 None 则认证所有服务 - :return: 认证数据 - """ - # 飞牛影视认证 - if not credentials or credentials.grant_type != "password": - return None - # 确定要认证的服务器列表 - if service_name: - # 如果指定了服务名，获取该服务实例 - servers = ( - [(service_name, server)] - if (server := self.get_instance(service_name)) - else [] - ) - else: - # 如果没有指定服务名，遍历所有服务 - servers = self.get_instances().items() - # 遍历要认证的服务器 - for name, server in servers: - # 触发认证拦截事件 - intercept_event = eventmanager.send_event( - etype=ChainEventType.AuthIntercept, - data=AuthInterceptCredentials( - username=credentials.username, - channel=self.get_name(), - service=name, - status="triggered", - ), - ) - if intercept_event and intercept_event.event_data: - intercept_data: AuthInterceptCredentials = intercept_event.event_data - if intercept_data.cancel: - continue - token = server.authenticate(credentials.username, credentials.password) - if token: - credentials.channel = self.get_name() - credentials.service = name - credentials.token = token - return credentials + def _test_server(self, server, name: str) -> Optional[str]: + """飞牛影视用配置完整性与重连结果探测连接状态。""" + if not server.is_configured(): + return f"{self.get_name()}配置不完整：{name}" + if server.is_inactive() and not server.reconnect(): + return f"无法连接{self.get_name()}：{name}" return None def webhook_parser( @@ -160,92 +100,6 @@ class TrimeMediaModule(_ModuleBase, _MediaServerBase[TrimeMedia]): return result return None - def media_exists( - self, - mediainfo: MediaInfo, - itemid: Optional[str] = None, - server: Optional[str] = None, - ) -> Optional[schemas.ExistMediaInfo]: - """ - 判断媒体文件是否存在 - - :param mediainfo: 识别的媒体信息 - :param itemid: 媒体服务器ItemID - :param server: 媒体服务器名称 - :return: 如不存在返回None，存在时返回信息，包括每季已存在所有集{type: movie/tv, seasons: {season: [episodes]}} - """ - if server: - servers = [(server, self.get_instance(server))] - else: - servers = self.get_instances().items() - for name, s in servers: - if not s: - continue - if mediainfo.type == MediaType.MUSIC: - matches = getattr(s, "get_music", lambda **_: [])( - **MusicMediaServerHelper.search_params(mediainfo) - ) - match = MusicMediaServerHelper.find_match(mediainfo, matches) - if match: - return schemas.ExistMediaInfo( - type=MediaType.MUSIC, - server_type="trimemedia", - server=name, - itemid=match.item_id, - ) - continue - if mediainfo.type == MediaType.MOVIE: - if itemid: - movie = s.get_iteminfo(itemid) - if movie: - logger.info(f"媒体库 {name} 中找到了 {movie}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="trimemedia", - server=name, - itemid=movie.item_id, - ) - movies = s.get_movies( - title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - ) - if not movies: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"媒体库 {name} 中找到了 {movies}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="trimemedia", - server=name, - itemid=movies[0].item_id, - ) - else: - itemid, tvs = s.get_tv_episodes( - title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - item_id=itemid, - ) - if not tvs: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info( - f"{mediainfo.title_year} 在媒体库 {name} 中找到了这些季集：{tvs}" - ) - return schemas.ExistMediaInfo( - type=MediaType.TV, - seasons=tvs, - server_type="trimemedia", - server=name, - itemid=itemid, - ) - return None - def media_statistic( self, server: Optional[str] = None ) -> Optional[List[schemas.Statistic]]: diff --git a/app/modules/ugreen/__init__.py b/app/modules/ugreen/__init__.py index 5b1a75db5..5c05440fa 100644 --- a/app/modules/ugreen/__init__.py +++ b/app/modules/ugreen/__init__.py @@ -1,17 +1,16 @@ from typing import Any, Generator, List, Optional, Tuple, Union from app import schemas -from app.domain.context import MediaInfo -from app.runtime.events import eventmanager -from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _MediaServerBase, _ModuleBase +from app.modules._base import _MediaServerModuleBase from app.modules.ugreen.ugreen import Ugreen -from app.schemas import AuthCredentials, AuthInterceptCredentials -from app.schemas.types import ChainEventType, MediaServerType, MediaType, ModuleType +from app.schemas.types import MediaServerType, ModuleType -class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]): +class UgreenModule(_MediaServerModuleBase[Ugreen]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "ugreen" def init_module(self) -> None: """ @@ -52,14 +51,9 @@ class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]): def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - for name, server in self.get_instances().items(): - if server.is_configured() and server.is_inactive(): - logger.info(f"绿联影视 {name} 连接断开，尝试重连 ...") - server.reconnect() + def _is_inactive(self, server) -> bool: + """未配置的实例不参与定时重连。""" + return server.is_configured() and server.is_inactive() def stop(self) -> None: """停止模块""" @@ -70,57 +64,12 @@ class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]): except Exception as err: logger.error(f"停止绿联影视模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if not server.is_configured(): - return False, f"绿联影视配置不完整：{name}" - if server.is_inactive() and not server.reconnect(): - return False, f"无法连接绿联影视：{name}" - return True, "" - - def user_authenticate( - self, credentials: AuthCredentials, service_name: Optional[str] = None - ) -> Optional[AuthCredentials]: - """ - 使用绿联影视用户辅助完成用户认证 - """ - if not credentials or credentials.grant_type != "password": - return None - - if service_name: - servers = ( - [(service_name, server)] - if (server := self.get_instance(service_name)) - else [] - ) - else: - servers = self.get_instances().items() - - for name, server in servers: - intercept_event = eventmanager.send_event( - etype=ChainEventType.AuthIntercept, - data=AuthInterceptCredentials( - username=credentials.username, - channel=self.get_name(), - service=name, - status="triggered", - ), - ) - if intercept_event and intercept_event.event_data: - intercept_data: AuthInterceptCredentials = intercept_event.event_data - if intercept_data.cancel: - continue - token = server.authenticate(credentials.username, credentials.password) - if token: - credentials.channel = self.get_name() - credentials.service = name - credentials.token = token - return credentials + def _test_server(self, server, name: str) -> Optional[str]: + """绿联影视用配置完整性与重连结果探测连接状态。""" + if not server.is_configured(): + return f"{self.get_name()}配置不完整：{name}" + if server.is_inactive() and not server.reconnect(): + return f"无法连接{self.get_name()}：{name}" return None def webhook_parser( @@ -146,84 +95,6 @@ class UgreenModule(_ModuleBase, _MediaServerBase[Ugreen]): return result return None - def media_exists( - self, - mediainfo: MediaInfo, - itemid: Optional[str] = None, - server: Optional[str] = None, - ) -> Optional[schemas.ExistMediaInfo]: - """ - 判断媒体文件是否存在 - """ - if server: - servers = [(server, self.get_instance(server))] - else: - servers = self.get_instances().items() - - for name, s in servers: - if not s: - continue - if mediainfo.type == MediaType.MUSIC: - matches = getattr(s, "get_music", lambda **_: [])( - **MusicMediaServerHelper.search_params(mediainfo) - ) - match = MusicMediaServerHelper.find_match(mediainfo, matches) - if match: - return schemas.ExistMediaInfo( - type=MediaType.MUSIC, - server_type="ugreen", - server=name, - itemid=match.item_id, - ) - continue - if mediainfo.type == MediaType.MOVIE: - if itemid: - movie = s.get_iteminfo(itemid) - if movie: - logger.info(f"媒体库 {name} 中找到了 {movie}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="ugreen", - server=name, - itemid=movie.item_id, - ) - movies = s.get_movies( - title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - ) - if not movies: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - logger.info(f"媒体库 {name} 中找到了 {movies}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="ugreen", - server=name, - itemid=movies[0].item_id, - ) - - itemid, tvs = s.get_tv_episodes( - title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - item_id=itemid, - ) - if not tvs: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - logger.info(f"{mediainfo.title_year} 在媒体库 {name} 中找到了这些季集：{tvs}") - return schemas.ExistMediaInfo( - type=MediaType.TV, - seasons=tvs, - server_type="ugreen", - server=name, - itemid=itemid, - ) - return None - def media_statistic( self, server: Optional[str] = None ) -> Optional[List[schemas.Statistic]]: diff --git a/app/modules/vocechat/__init__.py b/app/modules/vocechat/__init__.py index 2f4b8564b..560231471 100644 --- a/app/modules/vocechat/__init__.py +++ b/app/modules/vocechat/__init__.py @@ -9,7 +9,7 @@ from app.application.messaging.agent import ( resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.modules.vocechat.vocechat import VoceChat from app.schemas import MessageChannel, CommingMessage, Notification from app.schemas.types import ModuleType @@ -21,7 +21,9 @@ register_channel_admin_resolver( ) -class VoceChatModule(_ModuleBase, _MessageBase[VoceChat]): +class VoceChatModule(_MessageChannelModuleBase[VoceChat]): + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "VOCECHAT_ADMINS" _IMAGE_SUFFIXES = ( ".png", ".jpg", @@ -83,51 +85,9 @@ class VoceChatModule(_ModuleBase, _MessageBase[VoceChat]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"VoceChat {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析 VoceChat 管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("VOCECHAT_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, - config: Optional[dict], - *user_ids: Optional[Union[str, int]], - ) -> bool: - """ - 判断 VoceChat 斜杠命令是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - candidates = [ - str(user_id).strip() - for user_id in user_ids - if user_id is not None and str(user_id).strip() - ] - return not any(candidate in admins for candidate in candidates) - @staticmethod def _send_admin_denied( client: Optional[VoceChat], userid: Optional[Union[str, int]] diff --git a/app/modules/wechat/__init__.py b/app/modules/wechat/__init__.py index 92aece764..c1c841838 100644 --- a/app/modules/wechat/__init__.py +++ b/app/modules/wechat/__init__.py @@ -1,4 +1,3 @@ -import copy import json import re import xml.dom.minidom @@ -6,21 +5,19 @@ from typing import Optional, Union, List, Tuple, Any, Dict from urllib.parse import quote from app.domain.context import Context, MediaInfo -from app.runtime.events import eventmanager from app.application.messaging.agent import ( matches_channel_admin, register_channel_admin_resolver, resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _ModuleBase, _MessageBase +from app.modules._base import _MessageChannelModuleBase from app.adapters.external.wechat_crypt import WXBizMsgCrypt from app.modules.wechat.wechat import WeChat from app.modules.wechat.wechatbot import WeChatBot -from app.schemas import MessageChannel, CommingMessage, Notification, CommandRegisterEventData -from app.schemas.types import ModuleType, ChainEventType +from app.schemas import MessageChannel, CommingMessage, Notification +from app.schemas.types import ModuleType from app.foundation.dom import DomUtils -from app.foundation.collections import DictUtils def _resolve_wechat_admin_ids(config: Optional[dict]) -> set[str]: @@ -34,7 +31,12 @@ def _resolve_wechat_admin_ids(config: Optional[dict]) -> set[str]: register_channel_admin_resolver(MessageChannel.Wechat, _resolve_wechat_admin_ids) -class WechatModule(_ModuleBase, _MessageBase[WeChat]): +class WechatModule(_MessageChannelModuleBase[WeChat]): + + # 管理员配置键，与渠道 resolver 保持一致 + _admin_config_key = "WECHAT_ADMINS" + # 命令注册事件源标识固定为 WeChat（get_name 为“企业微信”） + _command_origin = "WeChat" def init_module(self) -> None: """ @@ -82,51 +84,12 @@ class WechatModule(_ModuleBase, _MessageBase[WeChat]): def _is_bot_mode(config: dict) -> bool: return (config or {}).get("WECHAT_MODE", "app") == "bot" - @staticmethod - def _get_admins(config: Optional[dict]) -> List[str]: - """ - 解析企业微信管理员配置，兼容逗号分隔和首尾空白。 - """ - return [ - admin.strip() - for admin in str((config or {}).get("WECHAT_ADMINS") or "").split(",") - if admin.strip() - ] - - @classmethod - def _should_reject_admin_command( - cls, config: Optional[dict], user_id: Optional[str] - ) -> bool: - """ - 判断企业微信菜单或斜杠命令是否应因非管理员身份被拒绝。 - """ - admins = cls._get_admins(config) - if not admins: - return False - return not matches_channel_admin( - MessageChannel.Wechat, - config, - user_id, - ) - @classmethod def _create_client(cls, conf): if cls._is_bot_mode(conf.config): return WeChatBot(name=conf.name, **conf.config) return WeChat(name=conf.name, **conf.config) - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state = client.get_state() - if not state: - return False, f"企业微信 {name} 未就绪" - return True, "" - def init_setting(self) -> Tuple[str, Union[str, bool]]: pass @@ -457,54 +420,22 @@ class WechatModule(_ModuleBase, _MessageBase[WeChat]): client.send_torrents_msg(title=message.title, torrents=torrents, userid=message.userid, link=message.link) - def register_commands(self, commands: Dict[str, dict]): + def _commands_enabled(self, config: Optional[dict]) -> bool: """ - 注册命令，实现这个函数接收系统可用的命令菜单 - :param commands: 命令字典 + 菜单注册前置条件：智能机器人模式无传统菜单，缺少解密参数时无法调用菜单 API。 """ - for client_config in self.get_configs().values(): - if self._is_bot_mode(client_config.config): - logger.debug(f"{client_config.name} 为智能机器人模式，跳过传统菜单初始化") - continue - # 如果没有配置消息解密相关参数，则也没有必要进行菜单初始化 - if not client_config.config.get("WECHAT_ENCODING_AESKEY") or not client_config.config.get("WECHAT_TOKEN"): - logger.debug(f"{client_config.name} 缺少消息解密参数，跳过后续菜单初始化") - continue + if self._is_bot_mode(config): + logger.debug("智能机器人模式，跳过传统菜单初始化") + return False + if not config.get("WECHAT_ENCODING_AESKEY") or not config.get("WECHAT_TOKEN"): + logger.debug("缺少消息解密参数，跳过菜单初始化") + return False + return True - client = self.get_instance(client_config.name) - if not client: - continue + def _delete_commands(self, client) -> None: + """企业微信使用自定义菜单 API 清理命令。""" + client.delete_menus() - # 触发事件，允许调整命令数据，这里需要进行深复制，避免实例共享 - scoped_commands = copy.deepcopy(commands) - event = eventmanager.send_event( - ChainEventType.CommandRegister, - CommandRegisterEventData(commands=scoped_commands, origin="WeChat", service=client_config.name) - ) - - # 如果事件返回有效的 event_data，使用事件中调整后的命令 - if event and event.event_data: - event_data: CommandRegisterEventData = event.event_data - # 如果事件被取消，跳过命令注册，并清理菜单 - if event_data.cancel: - client.delete_menus() - logger.debug( - f"Command registration for {client_config.name} canceled by event: {event_data.source}" - ) - continue - scoped_commands = event_data.commands or {} - if not scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_menus() - - # scoped_commands 必须是 commands 的子集 - filtered_scoped_commands = DictUtils.filter_keys_to_subset(scoped_commands, commands) - # 如果 filtered_scoped_commands 为空，则跳过注册 - if not filtered_scoped_commands: - logger.debug("Filtered commands are empty, skipping registration.") - client.delete_menus() - continue - # 对比调整后的命令与当前命令 - if filtered_scoped_commands != commands: - logger.debug(f"Command set has changed, Updating new commands: {filtered_scoped_commands}") - client.create_menus(filtered_scoped_commands) + def _apply_commands(self, client, commands: Dict[str, dict]) -> None: + """企业微信使用自定义菜单 API 注册命令。""" + client.create_menus(commands) diff --git a/app/modules/wechatclawbot/__init__.py b/app/modules/wechatclawbot/__init__.py index d6a8f5734..adca12d74 100644 --- a/app/modules/wechatclawbot/__init__.py +++ b/app/modules/wechatclawbot/__init__.py @@ -9,7 +9,7 @@ from app.application.messaging.agent import ( resolve_config_principal_ids, ) from app.runtime.log import logger -from app.modules import _MessageBase, _ModuleBase +from app.modules._base import _MessageChannelModuleBase from app.modules.wechatclawbot.wechatclawbot import WechatClawBot from app.schemas import CommingMessage, Notification from app.schemas.types import MessageChannel, ModuleType, NotificationAction @@ -23,7 +23,7 @@ register_channel_admin_resolver( ) -class WechatClawBotModule(_ModuleBase, _MessageBase[WechatClawBot]): +class WechatClawBotModule(_MessageChannelModuleBase[WechatClawBot]): def __init__(self): """初始化模块级去重缓存，拦截 iLink 偶发的重复回放消息。""" super().__init__() @@ -69,15 +69,9 @@ class WechatClawBotModule(_ModuleBase, _MessageBase[WechatClawBot]): except Exception as err: logger.error(f"停止微信 ClawBot 模块实例失败：{err}") - def test(self) -> Optional[Tuple[bool, str]]: - """测试模块连接性。""" - if not self.get_instances(): - return None - for name, client in self.get_instances().items(): - state, message = client.test_connection() - if not state: - return False, f"微信 ClawBot {name} 未就绪：{message}" - return True, "" + def _test_connection(self, client) -> Tuple[bool, str]: + """微信 ClawBot 的连接探测返回 (状态, 信息)。""" + return client.test_connection() def init_setting(self) -> Tuple[str, Union[str, bool]]: """初始化模块设置。""" diff --git a/app/modules/zspace/__init__.py b/app/modules/zspace/__init__.py index 526e3f24d..1d8e606a3 100644 --- a/app/modules/zspace/__init__.py +++ b/app/modules/zspace/__init__.py @@ -1,17 +1,17 @@ from typing import Any, Generator, List, Optional, Tuple, Union from app import schemas -from app.domain.context import MediaInfo -from app.runtime.events import eventmanager -from app.application.mediaserver import MusicMediaServerHelper from app.runtime.log import logger -from app.modules import _MediaServerBase, _ModuleBase +from app.modules._base import _MediaServerModuleBase from app.modules.zspace.zspace import ZSpace from app.schemas import AuthCredentials, AuthInterceptCredentials -from app.schemas.types import ChainEventType, MediaServerType, MediaType, ModuleType +from app.schemas.types import ChainEventType, MediaServerType, ModuleType -class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]): +class ZSpaceModule(_MediaServerModuleBase[ZSpace]): + + # 媒体库标识（ExistMediaInfo.server_type） + _server_type_value = "zspace" def init_module(self) -> None: """ @@ -48,63 +48,17 @@ class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]): def stop(self): pass - def test(self) -> Optional[Tuple[bool, str]]: - """ - 测试模块连接性 - """ - if not self.get_instances(): - return None - for name, server in self.get_instances().items(): - if server.is_inactive() and not server.reconnect(): - return False, f"无法连接极影视服务器：{name}" - if not server.user: - return False, f"无法连接极影视服务器：{name}" - return True, "" + def _test_server(self, server, name: str) -> Optional[str]: + """极影视用重连结果与用户信息探测连接状态。""" + if server.is_inactive() and not server.reconnect(): + return f"无法连接{self.get_name()}服务器：{name}" + if not server.user: + return f"无法连接{self.get_name()}服务器：{name}" + return None def init_setting(self) -> Tuple[str, Union[str, bool]]: pass - def scheduler_job(self) -> None: - """ - 定时任务，每10分钟调用一次 - """ - for name, server in self.get_instances().items(): - if server.is_inactive(): - logger.info(f"极影视服务器 {name} 连接断开，尝试重连 ...") - server.reconnect() - - def user_authenticate(self, credentials: AuthCredentials, service_name: Optional[str] = None) \ - -> Optional[AuthCredentials]: - """ - 使用极影视用户辅助完成用户认证 - :param credentials: 认证数据 - :param service_name: 指定要认证的媒体服务器名称，若为 None 则认证所有服务 - :return: 认证数据 - """ - if not credentials or credentials.grant_type != "password": - return None - if service_name: - servers = [(service_name, server)] if (server := self.get_instance(service_name)) else [] - else: - servers = self.get_instances().items() - for name, server in servers: - intercept_event = eventmanager.send_event( - etype=ChainEventType.AuthIntercept, - data=AuthInterceptCredentials(username=credentials.username, channel=self.get_name(), - service=name, status="triggered") - ) - if intercept_event and intercept_event.event_data: - intercept_data: AuthInterceptCredentials = intercept_event.event_data - if intercept_data.cancel: - continue - token = server.authenticate(credentials.username, credentials.password) - if token: - credentials.channel = self.get_name() - credentials.service = name - credentials.token = token - return credentials - return None - def webhook_parser(self, body: Any, form: Any, args: Any) -> Optional[schemas.WebhookEventInfo]: """ 解析Webhook报文体 @@ -130,81 +84,6 @@ class ZSpaceModule(_ModuleBase, _MediaServerBase[ZSpace]): return result return None - def media_exists(self, mediainfo: MediaInfo, itemid: Optional[str] = None, - server: Optional[str] = None) -> Optional[schemas.ExistMediaInfo]: - """ - 判断媒体文件是否存在 - :param mediainfo: 识别的媒体信息 - :param itemid: 媒体服务器ItemID - :param server: 媒体服务器名称 - :return: 如不存在返回None，存在时返回信息，包括每季已存在所有集{type: movie/tv, seasons: {season: [episodes]}} - """ - if server: - servers = [(server, self.get_instance(server))] - else: - servers = self.get_instances().items() - for name, s in servers: - if not s: - continue - if mediainfo.type == MediaType.MUSIC: - matches = getattr(s, "get_music", lambda **_: [])( - **MusicMediaServerHelper.search_params(mediainfo) - ) - match = MusicMediaServerHelper.find_match(mediainfo, matches) - if match: - return schemas.ExistMediaInfo( - type=MediaType.MUSIC, - server_type="zspace", - server=name, - itemid=match.item_id, - ) - continue - if mediainfo.type == MediaType.MOVIE: - if itemid: - movie = s.get_iteminfo(itemid) - if movie: - logger.info(f"媒体库 {name} 中找到了 {movie}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="zspace", - server=name, - itemid=movie.item_id - ) - movies = s.get_movies(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id) - if not movies: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"媒体库 {name} 中找到了 {movies}") - return schemas.ExistMediaInfo( - type=MediaType.MOVIE, - server_type="zspace", - server=name, - itemid=movies[0].item_id - ) - else: - itemid, tvs = s.get_tv_episodes(title=mediainfo.title, - year=mediainfo.year, - media_source=mediainfo.media_source, - media_id=mediainfo.media_id, - item_id=itemid) - if not tvs: - logger.info(f"{mediainfo.title_year} 没有在媒体库 {name} 中") - continue - else: - logger.info(f"{mediainfo.title_year} 在媒体库 {name} 中找到了这些季集：{tvs}") - return schemas.ExistMediaInfo( - type=MediaType.TV, - seasons=tvs, - server_type="zspace", - server=name, - itemid=itemid - ) - return None - def media_statistic(self, server: Optional[str] = None) -> Optional[List[schemas.Statistic]]: """ 媒体数量统计 diff --git a/app/runtime/compat/imports.py b/app/runtime/compat/imports.py index 32f34305d..c23226510 100644 --- a/app/runtime/compat/imports.py +++ b/app/runtime/compat/imports.py @@ -188,11 +188,14 @@ class LegacySymbolOverlayLoader(importlib.abc.Loader): module.__getattr__ = resolve_export module.__dir__ = list_exports + # 兼容符号不并入 __all__：避免 `from import *` 在包初始化期 + # 急切解析旧符号、反向拉起应用层模块形成循环导入；显式导入与属性 + # 访问仍由上方 __getattr__ 惰性解析兜底 public_names = { name for name in module.__dict__ if not name.startswith("_") } declared_exports = set(previous_all or ()) if had_all else public_names - module.__all__ = sorted(declared_exports | set(exports)) + module.__all__ = sorted(declared_exports) module.__dict__[self._STATE_KEY] = { "__getattr__": previous_getattr, "__dir__": previous_dir, diff --git a/app/runtime/compat/manifest.py b/app/runtime/compat/manifest.py index b16d833b1..a40f2076a 100644 --- a/app/runtime/compat/manifest.py +++ b/app/runtime/compat/manifest.py @@ -680,6 +680,18 @@ PACKAGE_EXPORTS: Dict[str, Dict[str, SymbolAlias]] = { # 物理模块仍存在、仅部分公开符号迁走时，由导入器在标准 Loader 执行后叠加惰性符号路由。 # canonical 源码不反向依赖兼容层，目标符号也只在旧调用方真正取用时加载。 SYMBOL_ALIASES: Dict[str, Dict[str, SymbolAlias]] = { + "app.agent.orchestrator": { + "AgentChain": SymbolAlias( + target_module="app.chain.agent", + target_name="AgentChain", + replacement="app.chain.agent.AgentChain", + ), + "ReplyMode": SymbolAlias( + target_module="app.schemas.agent", + target_name="ReplyMode", + replacement="app.schemas.agent.ReplyMode", + ), + }, "app.chain.message": { "MediaInteractionChain": SymbolAlias( target_module="app.chain.interaction", diff --git a/app/scheduler.py b/app/scheduler.py index e009910e5..a0b2811ba 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -51,7 +51,8 @@ from app.runtime.scheduling import TimerUtils lock = threading.Lock() SCHEDULER_PROGRESS_PREFIX = "scheduler" -AGENT_TASK_JOB_PREFIX = "agent-task" +# Agent 自主定时任务前缀下沉到 application 门面，此处保留兼容导出。 +from app.application.scheduling import AGENT_TASK_JOB_PREFIX # noqa: E402 class SchedulerChain(ChainBase): diff --git a/app/schemas/agent.py b/app/schemas/agent.py index 9c229d3fa..447791bf6 100644 --- a/app/schemas/agent.py +++ b/app/schemas/agent.py @@ -1,6 +1,7 @@ """AI智能体相关数据模型""" from datetime import datetime +from enum import Enum from typing import Any, List, Literal, Optional, Union from langchain_core.messages import BaseMessage @@ -9,6 +10,13 @@ from pydantic import BaseModel, Field, ConfigDict, field_serializer from app.schemas.common import JsonData +class ReplyMode(str, Enum): + """Agent 最终回复处理模式（chain 与 agent 层共享的值域）。""" + + DISPATCH = "dispatch" + CAPTURE_ONLY = "capture_only" + + class ConversationMemory(BaseModel): """对话记忆模型""" diff --git a/app/startup/agent_initializer.py b/app/startup/agent_initializer.py index 13bbcedd1..f56f4c39d 100644 --- a/app/startup/agent_initializer.py +++ b/app/startup/agent_initializer.py @@ -1,7 +1,21 @@ +from app.agent.llm import AgentCapabilityManager, LLMHelper from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services from app.runtime.config import settings from app.runtime.log import logger +# 导入期即向 application 门面注册实现，保证任何先于 initialize 的 +# 链层调用都能通过门面取到 Agent 服务对象。 +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + class AgentInitializer: """ diff --git a/app/startup/command_initializer.py b/app/startup/command_initializer.py index 013660e1d..51102d3ab 100644 --- a/app/startup/command_initializer.py +++ b/app/startup/command_initializer.py @@ -1,5 +1,9 @@ +from app.application.commands import register_command_class from app.command import Command +# 导入期即向 application 门面注册命令类，保证工具调用时不依赖静态边。 +register_command_class(Command) + def init_command(): """ diff --git a/app/startup/scheduler_initializer.py b/app/startup/scheduler_initializer.py index 49f771457..28d958792 100644 --- a/app/startup/scheduler_initializer.py +++ b/app/startup/scheduler_initializer.py @@ -1,5 +1,9 @@ +from app.application.scheduling import register_scheduler_class from app.scheduler import Scheduler +# 导入期即向 application 门面注册调度器类，保证工具调用时不依赖静态边。 +register_scheduler_class(Scheduler) + def init_scheduler(): """ diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 9c263ab3a..9cf6b55f3 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -201,6 +201,18 @@ do not belong here. Chains interact with modules exclusively through internals (classes, exceptions, constants) are forbidden, so every module stays pluggable and a chain never names a concrete module implementation. +Underscore-prefixed files in `app/chain/` are feature-domain mixins for +`ChainBase` and concrete chains, not chains themselves: `_recognition.py` +(`RecognitionMixin`), `_messaging.py` (`MessageProcessingMixin` / +`NotificationMixin`), `_interaction.py` (`InteractionChainMixin`, the shared +slash-command delegation for `remote_list` / `parse_callback` / +`handle_callback_interaction` / `handle_text_interaction`), `_music.py` +(`MusicSubscribeMixin`, the music single/album subscribe domain mixed into +`SubscribeChain`) and `_mixins.py` (TransferChain feature mixins). A concrete chain that exposes slash-command +interaction inherits `InteractionChainMixin`, injects its handler class via +`_interaction_handler_type` and implements only `_interaction_handler`; it must +not re-export application-layer interaction managers. + ### Module layer `app/modules/` contains pluggable downloaders, media servers, metadata sources, @@ -212,6 +224,20 @@ exceptions and value domains used by both modules and upper layers live in method names. The directory remains unchanged because discovery and plugin code depend on this established runtime root. +`app/modules/_base/` hosts the shared template base classes for module families +(`downloader.py`, `mediaserver.py`, `notification.py`), each combining the +family mixin with `_ModuleBase` and typed by `TService` (usage: +`class QbittorrentModule(_DownloaderModuleBase[Qbittorrent])`). The base classes +carry only verbatim-duplicated boilerplate — connection test, scheduled +reconnect, torrent-info reading, query-status normalization for downloaders; +authentication, media-exists check, inactive-server handling for media servers; +admin resolution and command registration for message channels — while +subclasses keep the differentiated API calls and override small hooks such as +`_test_connection`, `_test_server` and `_is_inactive`. Discovery already skips +the package (module discovery only enumerates first-level submodules and skips +underscore-prefixed names), so no new exclusion rules are needed; do not grow +this package with per-module business logic. + Channels and storages that need login management or temporary-parameter initialization follow one generic contract instead of per-target APIs: modules implement `channel_manage(channel, action, **params)` or @@ -303,6 +329,9 @@ policy. `app/db` therefore has no dependency on `app/domain`. |---|---| | `entrypoint -> chain / application / Oper` | Allowed according to workflow complexity | | `chain -> module (only via run_module dispatch) / application / Oper / canonical capability` | Allowed; direct `chain -> module` imports forbidden | +| `chain -> agent implementation` | Forbidden; chains reach Agent runtime only through `app/application/agent.py`, whose implementations are registered by `app/startup/agent_initializer.py` at import time | +| `agent.tools -> api / scheduler / command` | Forbidden; tools use `app/application/plugins.py`, `scheduling.py` and `commands.py` facades | +| `api -> factory` | Forbidden; the FastAPI instance is injected into `app/application/plugins.py` by the composition root after creation | | `application -> domain / runtime / adapter / Oper` | Allowed | | `module -> canonical capability / Oper` | Allowed | | `module -> module / chain` | Forbidden for new code | @@ -317,6 +346,11 @@ policy. `app/db` therefore has no dependency on `app/domain`. | Path | Purpose | |---|---| +| `app/application/agent.py` | Agent orchestration facade (`get_agent_manager` / `get_prompt_manager` / capability queries / prompt builders); Agent implementations register through `app/startup/agent_initializer.py`, no static `application -> agent` edge | +| `app/application/plugins.py` | Plugin API dynamic route registration/removal; the FastAPI instance is injected by `app/factory.py` after creation | +| `app/application/scheduling.py` | Runtime scheduler facade for Agent tools and endpoints; `Scheduler` class registered by `app/startup/scheduler_initializer.py` | +| `app/application/commands.py` | Command registry facade for Agent tools and endpoints; `Command` class registered by `app/startup/command_initializer.py` | +| `app/chain/agent.py` | `AgentChain(ChainBase)`: the chain-layer entry for Agent sessions; Agent runtime stays in `app/agent/` | | `app/runtime/config.py` | `ConfigModel`, `Settings` and deployment configuration | | `app/runtime/events.py` | `EventManager`, `Event` and event resolver registration | | `app/runtime/extensions/module_manager.py` | Module discovery and lifecycle | diff --git a/tests/test_agent_image_capability.py b/tests/test_agent_image_capability.py index 4760e78fb..b749576b7 100644 --- a/tests/test_agent_image_capability.py +++ b/tests/test_agent_image_capability.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + from unittest.mock import AsyncMock, patch from app.agent import MoviePilotAgent @@ -69,7 +84,7 @@ def test_handle_ai_message_routes_text_only_model_images_to_files(monkeypatch): } ], ) as prepare_files, patch( - "app.chain.message.agent_manager.process_message", new_callable=AsyncMock + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), diff --git a/tests/test_agent_image_support.py b/tests/test_agent_image_support.py index b2802af5e..913b6db0d 100644 --- a/tests/test_agent_image_support.py +++ b/tests/test_agent_image_support.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import asyncio import base64 import json @@ -451,7 +466,7 @@ class AgentImageSupportTest(unittest.TestCase): } ], ) as prepare_files, patch( - "app.chain.message.agent_manager.process_message", new_callable=AsyncMock + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), @@ -484,7 +499,7 @@ class AgentImageSupportTest(unittest.TestCase): with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( chain, "_get_or_create_session_id", return_value="session-1" ), patch( - "app.chain.message.agent_manager.process_message", new_callable=AsyncMock + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), diff --git a/tests/test_agent_interaction.py b/tests/test_agent_interaction.py index 6f6e7ea84..40c123c43 100644 --- a/tests/test_agent_interaction.py +++ b/tests/test_agent_interaction.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import asyncio import unittest from datetime import datetime @@ -195,7 +210,7 @@ class TestAgentInteraction(unittest.TestCase): ) as message_add, patch.object( chain, "edit_message", return_value=True ) as edit_message, patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", @@ -268,7 +283,7 @@ class TestAgentInteraction(unittest.TestCase): try: for channel in (MessageChannel.Telegram, MessageChannel.Feishu): with patch( - "app.chain.message.agent_manager.matches_secret_confirmation", + "app.application.agent._agent_manager.matches_secret_confirmation", return_value=True, ), patch.object( chain, diff --git a/tests/test_agent_message_routing.py b/tests/test_agent_message_routing.py index 4eb753f89..b75d43c0c 100644 --- a/tests/test_agent_message_routing.py +++ b/tests/test_agent_message_routing.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import asyncio from unittest.mock import AsyncMock, Mock, patch @@ -67,7 +82,7 @@ def test_explicit_ai_message_is_not_recorded_to_message_history(): with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( chain, "_record_user_message" ) as record_user_message, patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", @@ -90,7 +105,7 @@ def test_message_chain_passes_stable_channel_admin_principal_to_agent(): chain = MessageChain() with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", @@ -113,7 +128,7 @@ def test_message_chain_does_not_trust_channel_display_username(): chain = MessageChain() with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", @@ -136,7 +151,7 @@ def test_message_chain_uses_same_admin_contract_for_slack(): chain = MessageChain() with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", @@ -259,7 +274,7 @@ def test_agent_choice_callback_is_not_recorded_to_message_history(): ) as record_user_message, patch.object( chain, "edit_message", return_value=True ), patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", diff --git a/tests/test_agent_scheduled_tasks.py b/tests/test_agent_scheduled_tasks.py index d5c7476ee..88fdd3b87 100644 --- a/tests/test_agent_scheduled_tasks.py +++ b/tests/test_agent_scheduled_tasks.py @@ -431,6 +431,10 @@ async def test_interrupted_date_task_enable_toggle_stays_manual_only( scheduler = _build_agent_task_scheduler() scheduler.init_agent_task_jobs() monkeypatch.setattr("app.scheduler.Scheduler", lambda: scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: scheduler, + ) tool = _build_tool(UpdateAgentTaskTool, task.user_id) paused = json.loads(await tool.run(task_id=task.id, enabled=False)) @@ -458,6 +462,10 @@ async def test_interrupted_date_task_new_trigger_rearms_schedule(monkeypatch) -> scheduler = _build_agent_task_scheduler() scheduler.init_agent_task_jobs() monkeypatch.setattr("app.scheduler.Scheduler", lambda: scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: scheduler, + ) updated = json.loads( await _build_tool(UpdateAgentTaskTool, task.user_id).run( task_id=task.id, @@ -487,6 +495,10 @@ async def test_interrupted_date_task_rejects_past_trigger_while_pausing( scheduler = _build_agent_task_scheduler() scheduler.init_agent_task_jobs() monkeypatch.setattr("app.scheduler.Scheduler", lambda: scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: scheduler, + ) tool = _build_tool(UpdateAgentTaskTool, task.user_id) with pytest.raises(ValueError, match="必须晚于当前时间"): @@ -521,6 +533,10 @@ async def test_expired_date_task_rejects_enable_without_reschedule( scheduler = _build_agent_task_scheduler() scheduler.init_agent_task_jobs() monkeypatch.setattr("app.scheduler.Scheduler", lambda: scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: scheduler, + ) with pytest.raises(ValueError, match="必须晚于当前时间"): await _build_tool(UpdateAgentTaskTool, task.user_id).run( @@ -734,6 +750,10 @@ async def test_scheduler_tools_exclude_agent_tasks(monkeypatch) -> None: ] ) monkeypatch.setattr("app.scheduler.Scheduler", lambda: scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: scheduler, + ) tool = _build_tool(QuerySchedulersTool, "admin-user") result = json.loads(await tool.run()) @@ -763,6 +783,10 @@ async def test_agent_task_tools_manage_persistent_schedule(monkeypatch) -> None: user_id = f"user-{uuid4().hex}" fake_scheduler = _FakeAgentTaskScheduler() monkeypatch.setattr("app.scheduler.Scheduler", lambda: fake_scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: fake_scheduler, + ) create_tool = _build_tool(CreateAgentTaskTool, user_id) created = json.loads(await create_tool.ainvoke({ @@ -843,6 +867,10 @@ async def test_run_agent_task_enforces_owner_and_enabled_state(monkeypatch) -> N ) fake_scheduler = _FakeAgentTaskScheduler() monkeypatch.setattr("app.scheduler.Scheduler", lambda: fake_scheduler) + monkeypatch.setattr( + "app.application.scheduling._scheduler_class", + lambda: fake_scheduler, + ) other_user_result = await _build_tool( RunAgentTaskTool, diff --git a/tests/test_agent_session_status.py b/tests/test_agent_session_status.py index f4883500c..1d189740f 100644 --- a/tests/test_agent_session_status.py +++ b/tests/test_agent_session_status.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import asyncio import unittest from datetime import datetime, timedelta @@ -94,7 +109,7 @@ class TestAgentSessionStatus(unittest.TestCase): with ( patch( - "app.chain.message.agent_manager.get_session_status", + "app.application.agent._agent_manager.get_session_status", return_value=status, ), patch.object(chain, "post_message") as post_message, diff --git a/tests/test_api_response.py b/tests/test_api_response.py index b12f39313..194c5ba65 100644 --- a/tests/test_api_response.py +++ b/tests/test_api_response.py @@ -684,7 +684,7 @@ def test_openapi_success_models_have_no_implicit_empty_nested_schemas(): def test_plugin_routes_only_register_v1(monkeypatch): """插件动态路由只应注册 v1 地址并由应用统一路由类处理。""" - from app.api.endpoints import plugin as plugin_endpoint + from app.application import plugins class FakeApp: """记录动态注册路径的应用桩。""" @@ -715,15 +715,15 @@ def test_plugin_routes_only_register_v1(monkeypatch): ] fake_app = FakeApp() - monkeypatch.setattr(plugin_endpoint, "app", fake_app) - monkeypatch.setattr(plugin_endpoint, "PluginManager", FakePluginManager) + monkeypatch.setattr(plugins, "_api_app", fake_app) + monkeypatch.setattr(plugins, "PluginManager", FakePluginManager) - plugin_endpoint._update_plugin_api_routes("DemoPlugin", action="add") + plugins._update_plugin_api_routes("DemoPlugin", action="add") assert [route.path for route in fake_app.routes] == [ "/api/v1/plugin/DemoPlugin/health" ] - plugin_endpoint._update_plugin_api_routes("DemoPlugin", action="remove") + plugins._update_plugin_api_routes("DemoPlugin", action="remove") assert fake_app.routes == [] diff --git a/tests/test_architecture_dependencies.py b/tests/test_architecture_dependencies.py index ce59f29c8..aab554eda 100644 --- a/tests/test_architecture_dependencies.py +++ b/tests/test_architecture_dependencies.py @@ -1,6 +1,9 @@ import ast +from functools import lru_cache from pathlib import Path +import pytest + PROJECT_ROOT = Path(__file__).parents[1] APP_ROOT = PROJECT_ROOT / "app" @@ -97,11 +100,17 @@ FORBIDDEN_IMPORT_PREFIXES = { def _discover_modules() -> dict[str, Path]: - """建立实际 Python 模块名到源码路径的映射。""" + """建立实际 Python 模块名到源码路径的映射。 + + `app/plugins/` 由插件仓自治（包含独立第三方实现与未完成文件）， + 不参与宿主架构图分析。 + """ modules: dict[str, Path] = {} for path in APP_ROOT.rglob("*.py"): relative = path.relative_to(PROJECT_ROOT).with_suffix("") parts = list(relative.parts) + if parts[0] == "app" and parts[1] == "plugins": + continue if parts[-1] == "__init__": parts.pop() modules[".".join(parts)] = path @@ -431,7 +440,10 @@ def test_resource_adapter_does_not_restart_process(): def test_modules_do_not_import_other_modules_or_chain(): - """模块之间以及模块对链层的直接依赖被禁止，跨模块编排归链层。""" + """模块之间以及模块对链层的直接依赖被禁止，跨模块编排归链层。 + + `app.modules._base` 是模块共享样板基类包（模块发现会跳过），不视为业务模块。 + """ modules = _discover_modules() known_modules = set(modules) violations: dict[str, set[str]] = {} @@ -446,7 +458,7 @@ def test_modules_do_not_import_other_modules_or_chain(): if dependency.startswith("app.chain") or ( dependency.startswith("app.modules.") - and dependency.split(".")[2] != own_package + and dependency.split(".")[2] not in (own_package, "_base") ) } if forbidden: @@ -513,3 +525,103 @@ def test_chain_does_not_import_module_internals(): if forbidden: violations[module_name] = forbidden assert violations == {} + + +@lru_cache(maxsize=1) +def _build_module_graph() -> dict[str, set[str]]: + """构建非插件模块的完整静态依赖图，供既有包治理断言复用。 + + 纯静态 AST 分析无副作用，结果可安全缓存；多断言共享一次解析。 + """ + modules = _discover_modules() + known_modules = set(modules) + return { + name: _resolve_imports(name, path, known_modules) + for name, path in modules.items() + } + + +def test_chain_does_not_import_agent_implementation(): + """编排层不得反向依赖 Agent 实现，跨域编排经 application 门面。""" + violations: dict[str, set[str]] = {} + for module_name, dependencies in _build_module_graph().items(): + if not module_name.startswith("app.chain"): + continue + forbidden = { + dependency + for dependency in dependencies + if dependency.startswith("app.agent") + } + if forbidden: + violations[module_name] = forbidden + assert violations == {} + + +def test_agent_tools_do_not_import_entrypoint_internals(): + """Agent 工具不得穿透导入 HTTP 端点、调度器与命令注册表内部实现。 + + 工具对进程级状态的读写必须收敛到 application 门面， + 否则 agent 层与入口层互相穿透会形成不可测试的循环。 + """ + violations: dict[str, set[str]] = {} + for module_name, dependencies in _build_module_graph().items(): + if not module_name.startswith("app.agent.tools"): + continue + forbidden = { + dependency + for dependency in dependencies + if dependency.startswith(("app.api", "app.scheduler", "app.command")) + } + if forbidden: + violations[module_name] = forbidden + assert violations == {} + + +def test_api_does_not_import_factory(): + """装配器（factory）只允许 app.main 使用，HTTP 端点不得回引。""" + violations: dict[str, set[str]] = {} + for module_name, dependencies in _build_module_graph().items(): + if not module_name.startswith("app.api"): + continue + forbidden = { + dependency + for dependency in dependencies + if dependency.startswith("app.factory") + } + if forbidden: + violations[module_name] = forbidden + assert violations == {} + + +PROCESS_LEVEL_ROOTS = ( + "app.api", + "app.chain", + "app.agent", + "app.scheduler", + "app.command", + "app.monitor", + "app.startup", + "app.factory", +) + + +def test_process_level_packages_are_not_mutually_cyclic(): + """进程级根包之间不得形成跨包强连通分量。 + + 允许的环只存在于：单一包内部（modules 模块内、db 内、schemas 包内、 + agent 子域内、doctor 内）。跨根包的环意味着入口层、编排层与 Agent 层 + 互相穿透，破坏可插拔性与可测试性。 + """ + graph = _build_module_graph() + components = _strongly_connected_components(graph) + violations: list[list[str]] = [] + for component in components: + roots = { + name.split(".")[1] + for name in component + if name.startswith("app.") and name.count(".") >= 1 + } + involved = {root for root in roots if f"app.{root}" in PROCESS_LEVEL_ROOTS} + if len(involved) > 1: + violations.append(sorted(component)) + assert violations == [] diff --git a/tests/test_capability_registry.py b/tests/test_capability_registry.py index 82adc2586..79033d887 100644 --- a/tests/test_capability_registry.py +++ b/tests/test_capability_registry.py @@ -187,6 +187,8 @@ def test_current_host_module_manifests_follow_the_strict_nested_schema() -> None path for path in modules_root.iterdir() if path.is_dir() and (path / "__init__.py").is_file() + # 下划线前缀目录是内部基础包（如 _base），不是 host module，不参与清单校验 + and not path.name.startswith("_") } entrypoint_modules = {spec.entrypoint.split(":", maxsplit=1)[0] for spec in specs} diff --git a/tests/test_chain_layering.py b/tests/test_chain_layering.py index 9ec365a57..a3b3ef027 100644 --- a/tests/test_chain_layering.py +++ b/tests/test_chain_layering.py @@ -53,9 +53,13 @@ def test_chain_base_does_not_import_concrete_chains() -> None: """基础链不得反向导入任何具体处理链。""" imports = _imported_modules(CHAIN_ROOT / "__init__.py") + # 下划线前缀的内部模块（_messaging/_recognition 等）是 ChainBase 的 + # 功能域 mixin，不是具体处理链，允许导入 assert not { - module for module in imports + module + for module in imports if module.startswith("app.chain.") + and not module.removeprefix("app.chain.").startswith("_") } @@ -68,6 +72,7 @@ def test_legacy_music_chain_is_removed() -> None: ) for root in LEGACY_MUSIC_SCAN_ROOTS for path in root.rglob("*.py") + if "plugins" not in path.parts # 插件目录由插件仓自治，跳过 if "app.chain.music" in _imported_modules(path) } assert not violations diff --git a/tests/test_discord_command_registration.py b/tests/test_discord_command_registration.py index 3d0d810ce..0f7f59985 100644 --- a/tests/test_discord_command_registration.py +++ b/tests/test_discord_command_registration.py @@ -30,7 +30,7 @@ def test_discord_module_register_commands_filters_event_subset(): return_value={"discord-main": SimpleNamespace(name="discord-main", config={})}, ), patch.object(module, "get_instance", return_value=client), - patch("app.modules.discord.eventmanager.send_event", return_value=event), + patch("app.modules._base.notification.eventmanager.send_event", return_value=event), ): module.register_commands(original_commands) @@ -60,7 +60,7 @@ def test_discord_module_register_commands_deletes_when_event_canceled(): return_value={"discord-main": SimpleNamespace(name="discord-main", config={})}, ), patch.object(module, "get_instance", return_value=client), - patch("app.modules.discord.eventmanager.send_event", return_value=event), + patch("app.modules._base.notification.eventmanager.send_event", return_value=event), ): module.register_commands({"/sites": {"description": "管理站点"}}) diff --git a/tests/test_downloader_path_mapping.py b/tests/test_downloader_path_mapping.py index e275f5333..26dd586e4 100644 --- a/tests/test_downloader_path_mapping.py +++ b/tests/test_downloader_path_mapping.py @@ -109,6 +109,7 @@ def _load_transmission_module(): size_tools_module = types.ModuleType("app.foundation.size") temporal_tools_module = types.ModuleType("app.foundation.temporal") cache_module = types.ModuleType("app.runtime.cache") + base_module = types.ModuleType("app.modules._base") modules_module = types.ModuleType("app.modules") modules_module.__path__ = [] transmission_package_module = types.ModuleType("app.modules.transmission") @@ -131,6 +132,38 @@ def _load_transmission_module(): def __class_getitem__(cls, _item): return cls + class _DownloaderModuleBase(_ModuleBase, _DownloaderBase): + """隔离测试用的下载器模块基类桩，与 app.modules._base 行为对齐。""" + + def _get_torrent_info(self, content): + """与真实基类一致的种子信息读取，磁力链接不解析。""" + torrent_content = content + if isinstance(content, Path): + torrent_content = content.read_bytes() if content.exists() else None + torrent_info = None + if torrent_content and not torrent_rules_module.is_magnet_link(torrent_content): + torrent_info = torrentool_torrent_module.Torrent.from_string( + torrent_content + ) + return torrent_info, torrent_content + + @staticmethod + def _normalize_query_status(status): + """与真实基类一致的查询状态归一，返回隔离测试枚举。""" + status_value = getattr(status, "value", status) + status_text = str(status_value or "").strip().lower() + if not status_text or status_text in {"all", "全部"}: + return TorrentQueryStatus.ALL + if status_text in {"transfer", "transferring"}: + return TorrentQueryStatus.TRANSFER + if status_text in {"downloading"}: + return TorrentQueryStatus.DOWNLOADING + if status_text in {"complete", "completed", "seeding", "完成", "已完成"}: + return TorrentQueryStatus.COMPLETED + if status_text in {"pause", "paused", "暂停", "已暂停"}: + return TorrentQueryStatus.PAUSED + return TorrentQueryStatus.ALL + class _TransferTorrent: def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -210,6 +243,8 @@ def _load_transmission_module(): log_module.logger = _Logger() modules_module._ModuleBase = _ModuleBase modules_module._DownloaderBase = _DownloaderBase + modules_module._base = base_module + base_module._DownloaderModuleBase = _DownloaderModuleBase torrent_rules_module.is_magnet_link = _is_magnet_link size_tools_module.format_compact_size = _format_size temporal_tools_module.format_duration = _format_duration @@ -247,6 +282,7 @@ def _load_transmission_module(): "app.domain.metainfo": metainfo_module, "app.runtime.log": log_module, "app.modules": modules_module, + "app.modules._base": base_module, "app.modules.transmission": transmission_package_module, "app.modules.transmission.transmission": transmission_client_module, "app.schemas": schemas_module, diff --git a/tests/test_duplicate_code.py b/tests/test_duplicate_code.py new file mode 100644 index 000000000..b78d2f31c --- /dev/null +++ b/tests/test_duplicate_code.py @@ -0,0 +1,143 @@ +"""函数级重复代码门禁。 + +对非插件模块做归一化 AST 指纹比对：两个及以上不同模块中出现同构函数 +（变量名、字面量已归一，仅保留结构与属性名），且指纹规模超过阈值时告警。 + +存量复制粘贴以白名单标注，随各 Phase 清理后同步收紧；新增重复不得越过阈值。 +""" +import ast +from collections import defaultdict +from pathlib import Path + +PROJECT_ROOT = Path(__file__).parents[1] +APP_ROOT = PROJECT_ROOT / "app" + +# 指纹长度阈值：低于该值视为偶然相似，不告警。 +MIN_FINGERPRINT_SIZE = 1000 +# 参与告警的最小函数体节点数：过滤 setter/getter 等小函数。 +MIN_FUNCTION_SIZE = 40 +# 存量白名单：(模块名, 函数名) 集合，各 Phase 清理后同步移除。 +KNOWN_DUPLICATES = { + # 服务实现类（非模块类）的条目信息格式化样板（待后续 Phase 清理）。 + ("app.modules.jellyfin.jellyfin", "__format_item_info"), + ("app.modules.zspace.zspace", "__format_item_info"), +} + + +def _normalize(node: ast.AST) -> str: + """把函数 AST 归一化为指纹字符串。 + + 只保留控制流骨架与属性访问名，统一变量名与字面量， + 使改名、改常量的复制粘贴仍能被识别为同构。 + """ + parts: list[str] = [] + for child in ast.walk(node): + if isinstance(child, ast.FunctionDef): + parts.append(f"F:{child.name}") + elif isinstance(child, ast.Name): + parts.append(f"V:{child.id}") + elif isinstance(child, ast.Attribute): + parts.append(f"A:{child.attr}") + elif isinstance(child, ast.Constant): + parts.append("C:lit") + elif isinstance(child, ast.Call): + parts.append("call") + elif isinstance(child, ast.BinOp): + parts.append(f"op:{type(child.op).__name__}") + elif isinstance(child, ast.Compare): + parts.append("cmp") + elif isinstance(child, ast.UnaryOp): + parts.append("uop") + elif isinstance(child, ast.BoolOp): + parts.append("boolop") + elif isinstance(child, ast.If): + parts.append("if") + elif isinstance(child, ast.For): + parts.append("for") + elif isinstance(child, ast.While): + parts.append("while") + elif isinstance(child, ast.Try): + parts.append("try") + elif isinstance(child, ast.Return): + parts.append("return") + elif isinstance(child, ast.Assign): + parts.append("assign") + elif isinstance(child, ast.AnnAssign): + parts.append("annassign") + elif isinstance(child, ast.AugAssign): + parts.append("augassign") + elif isinstance(child, ast.Dict): + parts.append("dict") + elif isinstance(child, ast.List): + parts.append("list") + elif isinstance(child, ast.Subscript): + parts.append("sub") + elif isinstance(child, ast.Lambda): + parts.append("lambda") + elif isinstance(child, ast.Expr): + parts.append("expr") + elif isinstance(child, ast.With): + parts.append("with") + elif isinstance(child, ast.Yield): + parts.append("yield") + elif isinstance(child, ast.Import): + parts.append("import") + elif isinstance(child, ast.ImportFrom): + parts.append("importfrom") + elif isinstance(child, ast.Pass): + parts.append("pass") + elif isinstance(child, ast.arguments): + parts.append("args") + return "|".join(parts) + + +def _collect_duplicates() -> dict[int, list[tuple[str, str, int, int]]]: + """扫描非插件模块，按归一化指纹分组收集跨模块同构函数。""" + fingerprints: dict[int, list[tuple[str, str, int, int]]] = defaultdict(list) + for path in APP_ROOT.rglob("*.py"): + if path.relative_to(APP_ROOT).parts[0] == "plugins": + continue + try: + tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path)) + except SyntaxError: + continue + relative = path.relative_to(PROJECT_ROOT).with_suffix("") + parts = list(relative.parts) + if parts[-1] == "__init__": + parts.pop() + module_name = ".".join(parts) + for node in ast.walk(tree): + if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + continue + if sum(1 for _ in ast.walk(node)) < MIN_FUNCTION_SIZE: + continue + fingerprint = _normalize(node) + if len(fingerprint) < MIN_FINGERPRINT_SIZE: + continue + fingerprints[hash(fingerprint)].append( + (module_name, node.name, node.lineno, len(fingerprint)) + ) + return fingerprints + + +def test_no_new_large_duplicate_functions(): + """跨模块同构大函数（指纹 >= 1000）不得出现，存量白名单除外。""" + violations: list[list[tuple[str, str, int, int]]] = [] + for items in _collect_duplicates().values(): + modules = {module_name for module_name, _, _, _ in items} + if len(modules) < 2: + continue + leftovers = [ + item for item in items if (item[0], item[1]) not in KNOWN_DUPLICATES + ] + if leftovers: + violations.append(leftovers) + assert violations == [], ( + "检测到跨模块同构大函数（疑似复制粘贴），请提取公共基类/工具或" + "先加入 KNOWN_DUPLICATES 白名单并随对应 Phase 清理：\n" + + "\n".join( + f" {module_name}.{func_name} (line {line})" + for group in violations + for module_name, func_name, line, _ in group + ) + ) diff --git a/tests/test_episode_format_helper.py b/tests/test_episode_format_helper.py index a82413537..3ad3d6764 100644 --- a/tests/test_episode_format_helper.py +++ b/tests/test_episode_format_helper.py @@ -708,21 +708,21 @@ def test_transfer_chain_recommend_episode_format_passes_helper_data(monkeypatch) monkeypatch.setattr( chain, - "_TransferChain__resolve_episode_format_directory", + "_resolve_episode_format_directory", lambda item: directory, ) monkeypatch.setattr( chain, - "_TransferChain__get_episode_format_rules", + "_get_episode_format_rules", lambda: [], ) monkeypatch.setattr( chain, - "_TransferChain__get_episode_format_sample_files", + "_get_episode_format_sample_files", lambda item: [sample], ) monkeypatch.setattr( - "app.chain.transfer.EpisodeFormatRuleHelper.recommend", + "app.chain._mixins.EpisodeFormatRuleHelper.recommend", lambda self, rules, sample_files: (True, "", helper_data), ) @@ -773,11 +773,11 @@ def test_transfer_chain_recommend_episode_format_uses_selected_fileitems(monkeyp monkeypatch.setattr( chain, - "_TransferChain__get_episode_format_rules", + "_get_episode_format_rules", lambda: [], ) monkeypatch.setattr( - "app.chain.transfer.EpisodeFormatRuleHelper.recommend", + "app.chain._mixins.EpisodeFormatRuleHelper.recommend", lambda self, rules, sample_files: (True, "", { **helper_data, "received_samples": [item.name for item in sample_files], @@ -854,7 +854,7 @@ def test_transfer_chain_episode_format_samples_include_extra_files(monkeypatch): monkeypatch.setattr(chain, "_subtitle_exts", [".ass", ".ssa"], raising=False) monkeypatch.setattr(chain, "_audio_exts", [".mka", ".aac"], raising=False) - sample_files = TransferChain._TransferChain__get_episode_format_sample_files( + sample_files = TransferChain._get_episode_format_sample_files( chain, directory, ) diff --git a/tests/test_episode_group_recognition.py b/tests/test_episode_group_recognition.py index 0544bf724..c3aed1181 100644 --- a/tests/test_episode_group_recognition.py +++ b/tests/test_episode_group_recognition.py @@ -34,9 +34,9 @@ def test_recognize_media_uses_meta_episode_group(): ) with patch.object(chain, "run_module", return_value=mediainfo) as run_module, patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=True, - ), patch("app.chain.MoviePilotServerHelper.query_recognize_share") as query_mock: + ), patch("app.chain._recognition.MoviePilotServerHelper.query_recognize_share") as query_mock: result = chain.recognize_media(meta=meta, cache=False) assert result is mediainfo diff --git a/tests/test_manual_transfer_history.py b/tests/test_manual_transfer_history.py index 487997acc..2190497a6 100644 --- a/tests/test_manual_transfer_history.py +++ b/tests/test_manual_transfer_history.py @@ -36,12 +36,12 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -62,6 +62,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del "app.chain.transfer.TransferHistoryOper", lambda: history_oper, ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: history_oper) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -71,10 +72,17 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -85,6 +93,13 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del or True, ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + exists=lambda current_fileitem: True, + delete_media_file=lambda current_fileitem: deleted.append( + ("target", current_fileitem.path) + ) + or True, + )) monkeypatch.setattr( "app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1), diff --git a/tests/test_media_recognize_share.py b/tests/test_media_recognize_share.py index b0d07534e..60ab49e97 100644 --- a/tests/test_media_recognize_share.py +++ b/tests/test_media_recognize_share.py @@ -48,10 +48,10 @@ def test_report_shared_result_after_local_recognize_success(): mediainfo = _tmdb_media("测试电影", 100, MediaType.MOVIE, year="2024") with patch.object(chain, "run_module", return_value=mediainfo) as run_module, patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=True, ) as report_mock, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share" + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share" ) as query_mock: result = chain.recognize_media(meta=meta, cache=False) @@ -72,7 +72,7 @@ def test_query_shared_result_when_local_recognize_failed(): "run_module", side_effect=[None, shared_media], ) as run_module, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value={ "type": "tv", "media_source": "themoviedb", @@ -80,7 +80,7 @@ def test_query_shared_result_when_local_recognize_failed(): "season": 1, }, ) as query_mock, patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.TV, "media_source": MediaSource.TMDB, @@ -88,7 +88,7 @@ def test_query_shared_result_when_local_recognize_failed(): "season": 1, }, ), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ), patch.object( chain, @@ -119,7 +119,7 @@ def test_async_query_shared_result_when_local_recognize_failed(): "async_run_module", async_run_module, ), patch( - "app.chain.MoviePilotServerHelper.async_query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_query_recognize_share", AsyncMock(return_value={ "type": "tv", "media_source": "themoviedb", @@ -127,7 +127,7 @@ def test_async_query_shared_result_when_local_recognize_failed(): "season": 2, }), ) as query_mock, patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.TV, "media_source": MediaSource.TMDB, @@ -135,7 +135,7 @@ def test_async_query_shared_result_when_local_recognize_failed(): "season": 2, }, ), patch( - "app.chain.MoviePilotServerHelper.async_report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_report_recognize_share", AsyncMock(return_value=False), ), patch.object( chain, @@ -173,14 +173,14 @@ def test_backfill_local_cache_after_shared_recognize_success(): "run_module", side_effect=[None, shared_media, None], ) as run_module_mock, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value={ "type": "movie", "media_source": "themoviedb", "media_id": "700", }, ), patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.MOVIE, "media_source": MediaSource.TMDB, @@ -188,7 +188,7 @@ def test_backfill_local_cache_after_shared_recognize_success(): "season": None, }, ), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ): result = chain.recognize_media(meta=meta, cache=False) @@ -288,7 +288,7 @@ def test_report_shared_result_with_distinct_keyword_meta(): mediainfo = _tmdb_media("测试剧集", 402, MediaType.TV, year="2024") with patch.object(chain, "run_module", return_value=mediainfo), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=True, ) as report_mock: result = chain.recognize_media(meta=meta, share_meta=share_meta, cache=False) @@ -315,7 +315,7 @@ def test_query_shared_result_with_distinct_keyword_meta(): "run_module", side_effect=[None, shared_media], ), patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value={ "type": "tv", "media_source": "themoviedb", @@ -323,7 +323,7 @@ def test_query_shared_result_with_distinct_keyword_meta(): "season": 1, }, ) as query_mock, patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.TV, "media_source": MediaSource.TMDB, @@ -331,7 +331,7 @@ def test_query_shared_result_with_distinct_keyword_meta(): "season": 1, }, ), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ), patch.object( chain, @@ -359,10 +359,10 @@ def test_skip_report_when_local_recognize_hits_cache(): mediainfo.recognize_cache_hit = True with patch.object(chain, "run_module", return_value=mediainfo) as run_module, patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=True, ) as report_mock, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share" + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share" ) as query_mock: result = chain.recognize_media(meta=meta) @@ -385,10 +385,10 @@ def test_async_skip_report_when_local_recognize_hits_cache(): "async_run_module", AsyncMock(return_value=mediainfo), ) as async_run_module, patch( - "app.chain.MoviePilotServerHelper.async_report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_report_recognize_share", AsyncMock(return_value=True), ) as report_mock, patch( - "app.chain.MoviePilotServerHelper.async_query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_query_recognize_share", AsyncMock(), ) as query_mock: result = await chain.async_recognize_media(meta=meta) @@ -579,10 +579,10 @@ def test_chain_recognize_media_reports_music_share_result(): music = _music_info() with patch.object(chain, "recognize_music_from_source", return_value=music), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=True, ) as report_mock, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share" + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share" ) as query_mock: result = chain.recognize_media(meta=meta, cache=False) @@ -602,7 +602,7 @@ def test_chain_recognize_media_queries_music_share_when_local_failed(): "recognize_music_from_source", side_effect=[None, music], ) as recognize_source, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value={ "type": "music", "media_source": "musicbrainz", @@ -610,7 +610,7 @@ def test_chain_recognize_media_queries_music_share_when_local_failed(): "music_type": "recording", }, ), patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.MUSIC, "media_source": MediaSource.MusicBrainz, @@ -619,7 +619,7 @@ def test_chain_recognize_media_queries_music_share_when_local_failed(): "season": None, }, ), patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ), patch.object( chain, @@ -650,7 +650,7 @@ def test_chain_recognize_media_queries_music_share_after_local_fallback(): "recognize_music_from_source", side_effect=[fallback, music], ) as recognize_source, patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value={ "type": "music", "media_source": "musicbrainz", @@ -658,7 +658,7 @@ def test_chain_recognize_media_queries_music_share_after_local_fallback(): "music_type": "recording", }, ) as query_share, patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.MUSIC, "media_source": MediaSource.MusicBrainz, @@ -670,7 +670,7 @@ def test_chain_recognize_media_queries_music_share_after_local_fallback(): chain, "_update_local_recognize_cache", ), patch( - "app.chain.settings.MEDIA_RECOGNIZE_SHARE", + "app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True, ): result = chain.recognize_media(meta=meta, cache=False) @@ -697,7 +697,7 @@ def test_chain_async_recognize_media_queries_music_share_after_local_fallback(): "async_recognize_music_from_source", new=AsyncMock(side_effect=[fallback, music]), ) as recognize_source, patch( - "app.chain.MoviePilotServerHelper.async_query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_query_recognize_share", new=AsyncMock(return_value={ "type": "music", "media_source": "musicbrainz", @@ -705,7 +705,7 @@ def test_chain_async_recognize_media_queries_music_share_after_local_fallback(): "music_type": "recording", }), ) as query_share, patch( - "app.chain.MoviePilotServerHelper.to_recognize_params", + "app.chain._recognition.MoviePilotServerHelper.to_recognize_params", return_value={ "mtype": MediaType.MUSIC, "media_source": MediaSource.MusicBrainz, @@ -718,7 +718,7 @@ def test_chain_async_recognize_media_queries_music_share_after_local_fallback(): "_async_update_local_recognize_cache", new=AsyncMock(), ), patch( - "app.chain.settings.MEDIA_RECOGNIZE_SHARE", + "app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True, ): result = await chain.async_recognize_media(meta=meta, cache=False) @@ -749,12 +749,12 @@ def test_chain_recognize_media_skips_music_report_for_fallback_result(): fallback = MusicInfo(title="未知曲目", artists=["未知艺术家"]) with patch.object(chain, "recognize_music_from_source", return_value=fallback), patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value=None, ) as query_mock, patch( - "app.chain.MoviePilotServerHelper.report_recognize_share" + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share" ) as report_mock, patch( - "app.chain.settings.MEDIA_RECOGNIZE_SHARE", True + "app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True ): result = chain.recognize_media(meta=meta, cache=False) diff --git a/tests/test_media_recognize_share_statistics.py b/tests/test_media_recognize_share_statistics.py index 60f879c9e..23a159a65 100644 --- a/tests/test_media_recognize_share_statistics.py +++ b/tests/test_media_recognize_share_statistics.py @@ -29,8 +29,9 @@ def _shared_params(tmdb_id: int) -> dict: def _mock_counter(monkeypatch) -> Mock: """替换系统配置持久化入口并返回递增调用桩。""" increment = Mock() + # 计数逻辑在识别 mixin 中，按 _recognition 模块命名空间解析 SystemConfigOper monkeypatch.setattr( - "app.chain.SystemConfigOper", + "app.chain._recognition.SystemConfigOper", lambda: SimpleNamespace(increment=increment), ) return increment @@ -56,7 +57,7 @@ def test_sync_shared_recognize_success_increments_persisted_count(monkeypatch): type=MediaType.MOVIE, ) increment = _mock_counter(monkeypatch) - monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True) + monkeypatch.setattr("app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True) monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, media])) monkeypatch.setattr(chain, "_update_local_recognize_cache", Mock()) monkeypatch.setattr( @@ -85,7 +86,7 @@ def test_sync_shared_result_without_local_match_does_not_increment(monkeypatch): chain = _bare_chain() meta = _build_meta("共享识别失败电影") increment = _mock_counter(monkeypatch) - monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True) + monkeypatch.setattr("app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True) monkeypatch.setattr(chain, "run_module", Mock(side_effect=[None, None])) monkeypatch.setattr( MoviePilotServerHelper, @@ -121,7 +122,7 @@ def test_async_shared_recognize_success_increments_persisted_count(monkeypatch): type=MediaType.MOVIE, ) increment = _mock_counter(monkeypatch) - monkeypatch.setattr("app.chain.settings.MEDIA_RECOGNIZE_SHARE", True) + monkeypatch.setattr("app.chain._recognition.settings.MEDIA_RECOGNIZE_SHARE", True) monkeypatch.setattr( chain, "async_run_module", diff --git a/tests/test_media_source_routing.py b/tests/test_media_source_routing.py index 81593f46b..02268bc32 100644 --- a/tests/test_media_source_routing.py +++ b/tests/test_media_source_routing.py @@ -71,7 +71,7 @@ def test_explicit_source_recognition_reaches_modules_with_unified_identity() -> chain.run_module = Mock(return_value=media) with patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ): result = chain.recognize_media( @@ -105,7 +105,7 @@ def test_default_recognition_passes_empty_generic_identity() -> None: meta.type = MediaType.MOVIE with patch( - "app.chain.MoviePilotServerHelper.report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.report_recognize_share", return_value=False, ): result = chain.recognize_media(meta=meta) diff --git a/tests/test_message_notifications.py b/tests/test_message_notifications.py index 90c8b42b5..708e2a99a 100644 --- a/tests/test_message_notifications.py +++ b/tests/test_message_notifications.py @@ -170,7 +170,7 @@ def test_user_helper_message_does_not_enter_sse_queue() -> None: assert helper.get() is None -def test_notification_post_message_is_persisted_without_sse_queue() -> None: +def test_notification_post_message_is_persisted_without_sse_queue(monkeypatch) -> None: """ 业务通知通过消息链发送时只登记数据库，不进入前端 SSE 队列。 """ @@ -179,8 +179,10 @@ def test_notification_post_message_is_persisted_without_sse_queue() -> None: _reset_message_helper(helper) chain = ChainBase() - chain.messagequeue.send_message = Mock() - chain.eventmanager.send_event = Mock() + # messagequeue 是全局单例，用 monkeypatch 避免用例间污染 + send_message = Mock() + monkeypatch.setattr(chain.messagequeue, "send_message", send_message) + monkeypatch.setattr(chain.eventmanager, "send_event", Mock()) chain.post_message( Notification( @@ -195,10 +197,10 @@ def test_notification_post_message_is_persisted_without_sse_queue() -> None: assert messages[0].title == "下载完成" assert messages[0].mtype == NotificationType.Download.value assert helper.get() is None - chain.messagequeue.send_message.assert_called_once() + send_message.assert_called_once() -def test_agent_notification_post_message_is_persisted_without_sse_queue() -> None: +def test_agent_notification_post_message_is_persisted_without_sse_queue(monkeypatch) -> None: """ 智能体消息通过消息链发送时登记数据库，但不进入前端 SSE 队列。 """ @@ -207,8 +209,10 @@ def test_agent_notification_post_message_is_persisted_without_sse_queue() -> Non _reset_message_helper(helper) chain = ChainBase() - chain.messagequeue.send_message = Mock() - chain.eventmanager.send_event = Mock() + # messagequeue 是全局单例，用 monkeypatch 避免用例间污染 + send_message = Mock() + monkeypatch.setattr(chain.messagequeue, "send_message", send_message) + monkeypatch.setattr(chain.eventmanager, "send_event", Mock()) chain.post_message( Notification( @@ -223,18 +227,21 @@ def test_agent_notification_post_message_is_persisted_without_sse_queue() -> Non assert messages[0].title == "MoviePilot助手" assert messages[0].mtype == NotificationType.Agent.value assert helper.get() is None - chain.messagequeue.send_message.assert_called_once() + send_message.assert_called_once() -def test_transient_notification_post_message_skips_history_but_dispatches() -> None: +def test_transient_notification_post_message_skips_history_but_dispatches(monkeypatch) -> None: """ 标记为不保存历史的过程消息应跳过数据库登记，但仍正常派发。 """ _clear_messages() chain = ChainBase() - chain.messagequeue.send_message = Mock() - chain.eventmanager.send_event = Mock() + # messagequeue 是全局单例，用 monkeypatch 避免用例间污染 + send_message = Mock() + monkeypatch.setattr(chain.messagequeue, "send_message", send_message) + send_event = Mock() + monkeypatch.setattr(chain.eventmanager, "send_event", send_event) chain.post_message( Notification( @@ -245,12 +252,12 @@ def test_transient_notification_post_message_skips_history_but_dispatches() -> N ) assert MessageOper().list_by_page(page=1, count=10) == [] - assert "save_history" not in chain.eventmanager.send_event.call_args.kwargs["data"] - chain.eventmanager.send_event.assert_called_once() - chain.messagequeue.send_message.assert_called_once() + assert "save_history" not in send_event.call_args.kwargs["data"] + send_event.assert_called_once() + send_message.assert_called_once() -def test_transient_media_and_torrent_lists_skip_history_but_dispatch() -> None: +def test_transient_media_and_torrent_lists_skip_history_but_dispatch(monkeypatch) -> None: """ 传统交互候选列表标记为不保存历史时，只发送到渠道，不写入消息表。 """ @@ -267,7 +274,9 @@ def test_transient_media_and_torrent_lists_skip_history_but_dispatch() -> None: ), ) - chain.messagequeue.send_message = Mock() + # messagequeue 是全局单例，用 monkeypatch 避免用例间污染 + send_message = Mock() + monkeypatch.setattr(chain.messagequeue, "send_message", send_message) chain.post_medias_message( Notification(title="请选择媒体", save_history=False), @@ -279,4 +288,4 @@ def test_transient_media_and_torrent_lists_skip_history_but_dispatch() -> None: ) assert MessageOper().list_by_page(page=1, count=10) == [] - assert chain.messagequeue.send_message.call_count == 2 + assert send_message.call_count == 2 diff --git a/tests/test_music_plugin_recognize.py b/tests/test_music_plugin_recognize.py index 020c5c3c9..2b994c2c3 100644 --- a/tests/test_music_plugin_recognize.py +++ b/tests/test_music_plugin_recognize.py @@ -369,7 +369,7 @@ def test_chain_recognize_media_music_plugin_supplement(): with patch.object(chain, "recognize_music_from_source", return_value=fallback), \ patch.object(chain.eventmanager, "check", return_value=True), \ patch.object(chain.eventmanager, "send_event", return_value=event), \ - patch("app.chain.MoviePilotServerHelper.report_recognize_share") as report_mock: + patch("app.chain._recognition.MoviePilotServerHelper.report_recognize_share") as report_mock: result = chain.recognize_media(meta=meta, cache=False) assert result is not fallback diff --git a/tests/test_music_subscribe.py b/tests/test_music_subscribe.py index 51c830196..76b46daf3 100644 --- a/tests/test_music_subscribe.py +++ b/tests/test_music_subscribe.py @@ -132,9 +132,9 @@ def test_music_subscribe_reuses_search_download_and_finish_flow(): chain.filter_torrents = Mock(side_effect=lambda **kwargs: kwargs["torrent_list"]) with patch.object(SubscribeChain, "_recognize_music_subscribe", return_value=target), \ - patch("app.chain.subscribe.SearchChain", return_value=search_chain), \ - patch("app.chain.subscribe.DownloadChain", return_value=download_chain), \ - patch("app.chain.subscribe.SubscribeOper") as subscribe_oper: + patch("app.chain._music.SearchChain", return_value=search_chain), \ + patch("app.chain._music.DownloadChain", return_value=download_chain), \ + patch("app.chain._music.SubscribeOper") as subscribe_oper: subscribe_oper.return_value.get.return_value = subscribe chain._search_music_subscribe(subscribe) @@ -239,8 +239,8 @@ def test_music_best_version_persists_downloaded_rule_priority(): chain = SubscribeChain() chain.finish_subscribe_or_not = Mock() - with patch("app.chain.subscribe.DownloadChain", return_value=download_chain), \ - patch("app.chain.subscribe.SubscribeOper", return_value=subscribe_oper): + with patch("app.chain._music.DownloadChain", return_value=download_chain), \ + patch("app.chain._music.SubscribeOper", return_value=subscribe_oper): chain._download_music_subscribe(subscribe, _music_info(), [downloaded]) subscribe_oper.update.assert_called_once_with( @@ -441,8 +441,9 @@ def test_music_rss_match_reuses_cached_context_without_second_site_search(): torrent_helper.filter_torrent.return_value = True with patch.object(SubscribeChain, "_recognize_music_subscribe", return_value=target), \ patch("app.chain.subscribe.SubscribeOper", return_value=subscribe_oper), \ - patch("app.chain.subscribe.TorrentHelper", return_value=torrent_helper), \ - patch("app.chain.subscribe.DownloadChain", return_value=download_chain), \ + patch("app.chain._music.SubscribeOper", return_value=subscribe_oper), \ + patch("app.chain._music.TorrentHelper", return_value=torrent_helper), \ + patch("app.chain._music.DownloadChain", return_value=download_chain), \ patch("app.chain.subscribe.SearchChain") as search_chain, \ patch("app.chain.subscribe.MediaChain") as media_chain: chain.match({"music.example": [source_context]}) @@ -519,7 +520,7 @@ def test_legacy_music_without_identity_uses_recording_recognition_boundary(): media_chain = Mock() media_chain.recognize_media.return_value = recording - with patch("app.chain.subscribe.MediaChain", return_value=media_chain): + with patch("app.chain._music.MediaChain", return_value=media_chain): restored = SubscribeChain._recognize_music_subscribe(subscribe) assert restored is recording @@ -637,7 +638,7 @@ def test_recording_target_sync_clears_stale_album_track_count(): subscribe = _subscribe(total_tracks=11) subscribe_oper = Mock() - with patch("app.chain.subscribe.SubscribeOper", return_value=subscribe_oper): + with patch("app.chain._music.SubscribeOper", return_value=subscribe_oper): SubscribeChain._sync_music_subscribe_target(subscribe, _music_info()) subscribe_oper.update.assert_called_once_with(subscribe.id, {"total_tracks": None}) diff --git a/tests/test_music_transfer.py b/tests/test_music_transfer.py index d35aa3d93..4e1d7935b 100644 --- a/tests/test_music_transfer.py +++ b/tests/test_music_transfer.py @@ -68,6 +68,7 @@ def test_music_retry_restores_history_entity_namespace(tmp_path, monkeypatch): title="叶惠美", ) monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain) + monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: media_chain) result = TransferChain()._recognize_music_retry_media( history, @@ -188,8 +189,8 @@ def test_music_scrape_batch_event_preserves_each_track_context(): ) tasks.append(task) target_paths.append(target_path) - chain._TransferChain__register_scrape_batch_task(task) - chain._TransferChain__record_scrape_target( + chain._register_scrape_batch_task(task) + chain._record_scrape_target( task, TransferInfo( success=True, @@ -199,9 +200,9 @@ def test_music_scrape_batch_event_preserves_each_track_context(): ), ) - chain._TransferChain__close_scrape_batch(batch_id) + chain._close_scrape_batch(batch_id) for task in tasks: - chain._TransferChain__finish_scrape_batch_task(task) + chain._finish_scrape_batch_task(task) scrape_calls = [ call @@ -537,6 +538,7 @@ def test_success_file_aggregation_is_isolated_between_music_jobs_in_same_directo "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace()) monkeypatch.setattr( "app.chain.transfer.add_transfer_success", lambda **kwargs: SimpleNamespace(id=1), @@ -782,6 +784,7 @@ def test_downloader_process_forwards_music_history_type(tmp_path, monkeypatch): ), ) monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain) + monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: media_chain) monkeypatch.setattr(chain, "do_transfer", Mock(return_value=(True, ""))) monkeypatch.setattr(chain, "run_module", run_module) diff --git a/tests/test_music_workflows.py b/tests/test_music_workflows.py index c6ba86de1..232416d9c 100644 --- a/tests/test_music_workflows.py +++ b/tests/test_music_workflows.py @@ -435,7 +435,7 @@ def test_media_chain_default_recognition_only_queries_musicbrainz(monkeypatch): recognize_source = Mock(return_value=expected) monkeypatch.setattr(chain, "recognize_music_from_source", recognize_source) - with patch("app.chain.MoviePilotServerHelper.report_recognize_share"): + with patch("app.chain._recognition.MoviePilotServerHelper.report_recognize_share"): result = chain.recognize_media(meta=meta) assert result is expected @@ -484,7 +484,7 @@ def test_default_recognition_does_not_fallback_after_musicbrainz_miss(monkeypatc monkeypatch.setattr(chain, "recognize_music_from_source", recognize_source) with patch( - "app.chain.MoviePilotServerHelper.query_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.query_recognize_share", return_value=None, ): assert chain.recognize_media(meta=meta) is None @@ -506,7 +506,7 @@ def test_async_default_recognition_only_queries_musicbrainz(monkeypatch): monkeypatch.setattr(chain, "async_recognize_music_from_source", recognize_source) with patch( - "app.chain.MoviePilotServerHelper.async_report_recognize_share", + "app.chain._recognition.MoviePilotServerHelper.async_report_recognize_share", new=AsyncMock(), ): result = asyncio.run(chain.async_recognize_media(meta=meta)) diff --git a/tests/test_qbittorrent_compat.py b/tests/test_qbittorrent_compat.py index ebbedcc2b..cd3fdcb82 100644 --- a/tests/test_qbittorrent_compat.py +++ b/tests/test_qbittorrent_compat.py @@ -92,6 +92,40 @@ def _load_qbittorrent_modules(): def __class_getitem__(cls, _item): return cls + # 隔离环境下的下载器业务样板基类，镜像 app.modules._base.downloader 的行为 + class _DownloaderModuleBase(_ModuleBase, _DownloaderBase): + def test(self): + return True, "" + + def scheduler_job(self): + pass + + def _get_torrent_info(self, content): + torrent_info, torrent_content = None, None + if isinstance(content, Path): + torrent_content = content.read_bytes() if content.exists() else None + else: + torrent_content = content + if torrent_content: + if torrent_rules_module.is_magnet_link(torrent_content): + return None, torrent_content + torrent_info = torrentool_torrent_module.Torrent.from_string(torrent_content) + return torrent_info, torrent_content + + @staticmethod + def _normalize_query_status(status): + status_value = getattr(status, "value", status) + status_text = str(status_value or "").strip().lower() + if status_text in {"transfer", TorrentStatus.TRANSFER.value}: + return TorrentQueryStatus.TRANSFER + if status_text in {"downloading", TorrentStatus.DOWNLOADING.value}: + return TorrentQueryStatus.DOWNLOADING + if status_text in {"completed", "seeding", "complete", "完成", "已完成"}: + return TorrentQueryStatus.COMPLETED + if status_text in {"paused", "pause", "暂停", "已暂停"}: + return TorrentQueryStatus.PAUSED + return TorrentQueryStatus.ALL + class _Torrent: @staticmethod def from_string(content): @@ -143,6 +177,9 @@ def _load_qbittorrent_modules(): temporal_tools_module.format_duration = _format_duration modules_module._ModuleBase = _ModuleBase modules_module._DownloaderBase = _DownloaderBase + base_module = types.ModuleType("app.modules._base") + base_module._DownloaderModuleBase = _DownloaderModuleBase + modules_module._base = base_module torrentool_torrent_module.Torrent = _Torrent qbittorrentapi_module.TorrentDictionary = dict qbittorrentapi_module.TorrentFilesList = list @@ -186,6 +223,7 @@ def _load_qbittorrent_modules(): "app.domain.metainfo": metainfo_module, "app.runtime.log": log_module, "app.modules": modules_module, + "app.modules._base": base_module, "app.modules.qbittorrent": qbittorrent_package_module, "app.schemas": schemas_module, "app.schemas.types": schema_types_module, diff --git a/tests/test_recognize_source_selection.py b/tests/test_recognize_source_selection.py index 3c0db6375..b35e1e076 100644 --- a/tests/test_recognize_source_selection.py +++ b/tests/test_recognize_source_selection.py @@ -39,7 +39,7 @@ def test_recognize_media_with_source_only_uses_name_search(): with patch.object(chain, "_run_native_media_recognize", side_effect=fake_native), \ patch.object(chain, "_supplement_media_recognize", side_effect=lambda **kw: kw["mediainfo"]), \ - patch("app.chain.MoviePilotServerHelper"): + patch("app.chain._recognition.MoviePilotServerHelper"): result = chain.recognize_media(meta=meta, media_source=MediaSource.TMDB, cache=False) assert result is not None @@ -70,7 +70,7 @@ def test_async_recognize_media_with_source_only_uses_name_search(): with patch.object(chain, "_async_run_native_media_recognize", side_effect=fake_native), \ patch.object(chain, "_async_supplement_media_recognize", side_effect=fake_supplement), \ - patch("app.chain.MoviePilotServerHelper", helper): + patch("app.chain._recognition.MoviePilotServerHelper", helper): result = asyncio.run( chain.async_recognize_media(meta=meta, media_source=MediaSource.TMDB, cache=False) ) @@ -93,7 +93,7 @@ def test_recognize_media_accepts_string_source_only(): with patch.object(chain, "_run_native_media_recognize", side_effect=fake_native), \ patch.object(chain, "_supplement_media_recognize", side_effect=lambda **kw: kw["mediainfo"]), \ - patch("app.chain.MoviePilotServerHelper"): + patch("app.chain._recognition.MoviePilotServerHelper"): result = chain.recognize_media(meta=meta, media_source="themoviedb", cache=False) assert result is not None @@ -112,7 +112,7 @@ def test_recognize_media_meta_identity_same_source_uses_id(): with patch.object(chain, "_run_native_media_recognize", side_effect=fake_native), \ patch.object(chain, "_supplement_media_recognize", side_effect=lambda **kw: kw["mediainfo"]), \ - patch("app.chain.MoviePilotServerHelper"): + patch("app.chain._recognition.MoviePilotServerHelper"): result = chain.recognize_media(meta=meta, media_source=MediaSource.TMDB, cache=False) assert result is not None diff --git a/tests/test_slack_command_registration.py b/tests/test_slack_command_registration.py index 428aa2c4f..09c9226a1 100644 --- a/tests/test_slack_command_registration.py +++ b/tests/test_slack_command_registration.py @@ -29,7 +29,7 @@ def test_slack_module_register_commands_filters_event_subset(): return_value={"slack-main": SimpleNamespace(name="slack-main", config={})}, ), patch.object(module, "get_instance", return_value=client), - patch("app.modules.slack.eventmanager.send_event", return_value=event), + patch("app.modules._base.notification.eventmanager.send_event", return_value=event), ): module.register_commands(original_commands) diff --git a/tests/test_slash_command_interactions.py b/tests/test_slash_command_interactions.py index 0602c0295..9b4b5d2f8 100644 --- a/tests/test_slash_command_interactions.py +++ b/tests/test_slash_command_interactions.py @@ -12,9 +12,11 @@ ensure_optional_stub("pyquery", PyQuery=object) from app.chain.message import MessageChain from app.application.messaging.interaction import InteractionContext -from app.chain.site import SiteChain, site_interaction_manager +from app.chain.site import SiteChain +from app.application.messaging.site import site_interaction_manager from app.application.messaging.skill import skill_interaction_manager -from app.chain.subscribe import SubscribeChain, subscribe_interaction_manager +from app.chain.subscribe import SubscribeChain +from app.application.messaging.subscribe import subscribe_interaction_manager from app.schemas.types import MessageChannel diff --git a/tests/test_subscribe_chain.py b/tests/test_subscribe_chain.py index bbe8749cb..3d7ea6b64 100644 --- a/tests/test_subscribe_chain.py +++ b/tests/test_subscribe_chain.py @@ -19,6 +19,11 @@ def _load_subscribe_chain_class(): module = sys.modules[module_name] return module, module.SubscribeChain + # 交互处理器模块须在打桩上下文之外预先真实加载：其模块级会话管理器单例 + # 由 SlashInteractionManager 构造，若在桩内导入会绑定桩类并残留 sys.modules， + # 污染依赖真实会话管理器的后续测试 + import app.application.messaging.subscribe # noqa: F401 + stub_deps = {} def ensure_module(name: str, module: types.ModuleType): @@ -43,6 +48,12 @@ def _load_subscribe_chain_class(): chain_module.ChainBase = _ChainBase + # 链内功能域 mixin：交互四件套委托与音乐订阅域，隔离加载以空 mixin 注入 + interaction_mixin_module = ensure_module("app.chain._interaction", types.ModuleType("app.chain._interaction")) + interaction_mixin_module.InteractionChainMixin = type("InteractionChainMixin", (), {}) + music_mixin_module = ensure_module("app.chain._music", types.ModuleType("app.chain._music")) + music_mixin_module.MusicSubscribeMixin = type("MusicSubscribeMixin", (), {}) + class _MediaChain: """提供订阅链隔离测试所需的统一媒体识别接口。""" @@ -77,6 +88,8 @@ def _load_subscribe_chain_class(): def remove(self, *args, **kwargs): return None + # 真实导入 app.application.messaging.subscribe 需要 MessageGateway 类型符号 + interaction_module.MessageGateway = type("MessageGateway", (), {}) interaction_module.SlashInteractionManager = _SlashInteractionManager interaction_module.build_navigation_buttons = lambda *args, **kwargs: [] interaction_module.format_markdown_table = lambda *args, **kwargs: "" diff --git a/tests/test_system_notification_dispatch.py b/tests/test_system_notification_dispatch.py index 0f2894aef..812fa2ac7 100644 --- a/tests/test_system_notification_dispatch.py +++ b/tests/test_system_notification_dispatch.py @@ -36,7 +36,7 @@ class TestSystemNotificationDispatch(unittest.TestCase): text="任务完成", ) - with patch("app.chain.MessageTemplateHelper.render", return_value=message), patch.object( + with patch("app.chain._messaging.MessageTemplateHelper.render", return_value=message), patch.object( chain.messagehelper, "put" ), patch.object(chain.messageoper, "add"), patch.object( chain.eventmanager, "send_event" diff --git a/tests/test_telegram_typing_lifecycle.py b/tests/test_telegram_typing_lifecycle.py index dc01e7de7..dda99f46e 100644 --- a/tests/test_telegram_typing_lifecycle.py +++ b/tests/test_telegram_typing_lifecycle.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import asyncio import threading import time @@ -263,7 +278,7 @@ class TestTelegramTypingLifecycle(unittest.TestCase): ) as start_status, patch( "app.chain.message.settings.AI_AGENT_ENABLE", True ), patch( - "app.chain.message.agent_manager.process_message", + "app.application.agent._agent_manager.process_message", new_callable=AsyncMock, ) as process_message, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", diff --git a/tests/test_transfer_custom_words.py b/tests/test_transfer_custom_words.py index b2a4c0873..30563b137 100644 --- a/tests/test_transfer_custom_words.py +++ b/tests/test_transfer_custom_words.py @@ -7,7 +7,7 @@ """ from types import SimpleNamespace -import app.chain.transfer as transfer_module +import app.chain._mixins as mixins_module from app.chain.transfer import TransferChain @@ -26,7 +26,7 @@ def test_transfer_prefers_snapshot_over_live_lookup(monkeypatch): called["lookup"] = True return SimpleNamespace(custom_words="不应使用\n实时反查") - monkeypatch.setattr(transfer_module, "SubscribeChain", _GuardSubscribeChain) + monkeypatch.setattr(mixins_module, "SubscribeChain", _GuardSubscribeChain) history = _fake_history( custom_words="S04 => S01\n第 <> 集 >> EP+66", @@ -46,7 +46,7 @@ def test_transfer_falls_back_to_live_lookup_without_snapshot(monkeypatch): assert source == "Subscribe|{...}" return SimpleNamespace(custom_words="A => B") - monkeypatch.setattr(transfer_module, "SubscribeChain", _FakeSubscribeChain) + monkeypatch.setattr(mixins_module, "SubscribeChain", _FakeSubscribeChain) history = _fake_history(custom_words=None, note={"source": "Subscribe|{...}"}) result = TransferChain._get_subscribe_custom_words(history) @@ -61,7 +61,7 @@ def test_transfer_returns_none_when_unavailable(monkeypatch): def get_subscribe_by_source(self, source): return None - monkeypatch.setattr(transfer_module, "SubscribeChain", _NoneSubscribeChain) + monkeypatch.setattr(mixins_module, "SubscribeChain", _NoneSubscribeChain) # 无下载记录 assert TransferChain._get_subscribe_custom_words(None) is None diff --git a/tests/test_transfer_failed_retry_buttons.py b/tests/test_transfer_failed_retry_buttons.py index 495be8b57..b096b8891 100644 --- a/tests/test_transfer_failed_retry_buttons.py +++ b/tests/test_transfer_failed_retry_buttons.py @@ -1,3 +1,18 @@ +# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 +from app.agent.llm import AgentCapabilityManager, LLMHelper +from app.agent.orchestrator import agent_manager +from app.agent.prompt import prompt_manager +from app.agent.prompt.transfer_redo import build_manual_redo_prompt +from app.application.agent import register_agent_services + +register_agent_services( + agent_manager=agent_manager, + prompt_manager=prompt_manager, + capability_manager=AgentCapabilityManager, + llm_helper=LLMHelper, + manual_redo_prompt_builder=build_manual_redo_prompt, +) + import unittest import asyncio import sys @@ -134,10 +149,14 @@ class TestTransferFailedRetryButtons(unittest.TestCase): with patch( "app.chain.transfer.TransferHistoryOper" ) as history_oper_cls, patch( + # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 + "app.chain._mixins.TransferHistoryOper" + ) as mixins_history_oper_cls, patch( "app.chain.transfer.asyncio.run_coroutine_threadsafe", side_effect=_close_pending_coro, ) as run_task: history_oper_cls.return_value.get.return_value = history + mixins_history_oper_cls.return_value.get.return_value = history with patch.object(chain, "post_message") as post_message: chain.handle_failed_transfer_callback( callback_data="transfer_ai_retry_34", @@ -204,13 +223,17 @@ class TestTransferFailedRetryButtons(unittest.TestCase): with patch( "app.chain.transfer.TransferHistoryOper" ) as history_oper_cls, patch( - "app.chain.transfer.agent_manager.run_background_prompt", + # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 + "app.chain._mixins.TransferHistoryOper" + ) as mixins_history_oper_cls, patch( + "app.application.agent._agent_manager.run_background_prompt", side_effect=fake_run_background_prompt, ), patch( "app.chain.transfer.asyncio.run_coroutine_threadsafe", side_effect=_run_pending_coro, ): history_oper_cls.return_value.get.return_value = history + mixins_history_oper_cls.return_value.get.return_value = history with patch.object(chain, "post_message"), patch.object( chain, "async_post_message", side_effect=fake_async_post_message ): diff --git a/tests/test_transfer_job_manager.py b/tests/test_transfer_job_manager.py index 52abe07fd..7388aa136 100644 --- a/tests/test_transfer_job_manager.py +++ b/tests/test_transfer_job_manager.py @@ -454,8 +454,8 @@ class TransferJobManagerTest(unittest.TestCase): (source_fileitem, False) ] chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None def fake_handle_transfer(task, callback=None): planned_episodes.append(task.meta.begin_episode) @@ -1051,8 +1051,8 @@ class TransferJobManagerTest(unittest.TestCase): (main_fileitem, False) ] chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None def fake_handle_transfer(task, callback=None): planned.append(task.fileitem.path) @@ -1161,8 +1161,8 @@ class TransferJobManagerTest(unittest.TestCase): ) chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None def fake_handle_transfer(task, callback=None): planned.append((task.fileitem.path, task.meta.begin_episode)) @@ -1236,8 +1236,8 @@ class TransferJobManagerTest(unittest.TestCase): (main_fileitem, False) ] chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None def fake_handle_transfer(task, callback=None): planned.append((task.fileitem.path, task.meta.begin_episode)) @@ -1331,8 +1331,8 @@ class TransferJobManagerTest(unittest.TestCase): (other_title_fileitem, False), ] chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None def fake_handle_transfer(task, callback=None): planned.append((task.fileitem.path, task.meta.begin_episode)) @@ -1420,9 +1420,9 @@ class TransferJobManagerTest(unittest.TestCase): task.background = False task.manual = True self.assertTrue(chain._TransferChain__put_to_jobview(task)) - chain._TransferChain__register_scrape_batch_task(task) + chain._register_scrape_batch_task(task) - chain._TransferChain__close_scrape_batch(batch_id) + chain._close_scrape_batch(batch_id) transferinfos = [ TransferInfo( @@ -1474,7 +1474,7 @@ class TransferJobManagerTest(unittest.TestCase): storage_chain_cls.return_value.is_bluray_folder.return_value = False for task, transferinfo in zip(tasks, transferinfos): chain._TransferChain__default_callback(task, transferinfo) - chain._TransferChain__finish_scrape_batch_task(task) + chain._finish_scrape_batch_task(task) metadata_calls = [ call diff --git a/tests/test_transfer_mounted_disk_cleanup.py b/tests/test_transfer_mounted_disk_cleanup.py index c0e61474e..1c4732c31 100644 --- a/tests/test_transfer_mounted_disk_cleanup.py +++ b/tests/test_transfer_mounted_disk_cleanup.py @@ -31,10 +31,10 @@ def test_enabled_cleanup_skips_filesystem_detection(): 开关开启时应保持旧行为，且不产生额外文件系统检测。 """ with patch( - "app.chain.transfer.SystemUtils.is_network_filesystem" + "app.chain._mixins.SystemUtils.is_network_filesystem" ) as is_network_filesystem: should_delete = ( - TransferChain._TransferChain__should_delete_empty_source_directories( + TransferChain._should_delete_empty_source_directories( _make_task(), True, {}, @@ -50,11 +50,11 @@ def test_disabled_cleanup_keeps_mounted_local_source_directories(): 开关关闭时应保留网络或 FUSE 挂载的本地源目录。 """ with patch( - "app.chain.transfer.SystemUtils.is_network_filesystem", + "app.chain._mixins.SystemUtils.is_network_filesystem", return_value=True, ) as is_network_filesystem: should_delete = ( - TransferChain._TransferChain__should_delete_empty_source_directories( + TransferChain._should_delete_empty_source_directories( _make_task(), False, {}, @@ -72,11 +72,11 @@ def test_disabled_cleanup_still_deletes_ordinary_local_source_directories(): 开关关闭时普通本地文件系统仍应删除空目录。 """ with patch( - "app.chain.transfer.SystemUtils.is_network_filesystem", + "app.chain._mixins.SystemUtils.is_network_filesystem", return_value=False, ): should_delete = ( - TransferChain._TransferChain__should_delete_empty_source_directories( + TransferChain._should_delete_empty_source_directories( _make_task(download_path="/downloads"), False, {}, @@ -91,10 +91,10 @@ def test_disabled_cleanup_does_not_change_remote_storage_cleanup(): 开关关闭时非本地存储仍应执行原有空目录清理。 """ with patch( - "app.chain.transfer.SystemUtils.is_network_filesystem" + "app.chain._mixins.SystemUtils.is_network_filesystem" ) as is_network_filesystem: should_delete = ( - TransferChain._TransferChain__should_delete_empty_source_directories( + TransferChain._should_delete_empty_source_directories( _make_task(storage="alist", download_path="/downloads"), False, {}, @@ -111,12 +111,12 @@ def test_mounted_filesystem_detection_is_cached_by_source_directory(): """ mounted_filesystem_cache = {} with patch( - "app.chain.transfer.SystemUtils.is_network_filesystem", + "app.chain._mixins.SystemUtils.is_network_filesystem", return_value=True, ) as is_network_filesystem: for _ in range(2): should_delete = ( - TransferChain._TransferChain__should_delete_empty_source_directories( + TransferChain._should_delete_empty_source_directories( _make_task(), False, mounted_filesystem_cache, diff --git a/tests/test_transfer_movie_collection.py b/tests/test_transfer_movie_collection.py index 174858e9b..3d00ed21b 100644 --- a/tests/test_transfer_movie_collection.py +++ b/tests/test_transfer_movie_collection.py @@ -28,8 +28,8 @@ def _make_chain() -> TransferChain: (fileitem, False) ] chain._TransferChain__put_to_jobview = lambda task: True - chain._TransferChain__register_scrape_batch_task = lambda task: None - chain._TransferChain__close_scrape_batch = lambda batch_id: None + chain._register_scrape_batch_task = lambda task: None + chain._close_scrape_batch = lambda batch_id: None return chain @@ -107,6 +107,7 @@ def test_conflicting_download_history_recognizes_movie_by_file_meta(monkeypatch) "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_type_tmdbid=lambda **kwargs: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_type_tmdbid=lambda **kwargs: None)) monkeypatch.setattr( "app.chain.transfer.MediaChain", lambda: SimpleNamespace( @@ -117,6 +118,13 @@ def test_conflicting_download_history_recognizes_movie_by_file_meta(monkeypatch) supplement_tmdb_info=lambda media, _meta: media, ), ) + monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: SimpleNamespace( + recognize_media=lambda **kwargs: pytest.fail("不应按合集历史 ID 识别"), + recognize_by_meta=lambda meta, obtain_images: ( + recognized_meta.append(meta) or fallback_media + ), + supplement_tmdb_info=lambda media, _meta: media, + )) task = TransferTask( fileitem=FileItem( storage="local", @@ -182,12 +190,16 @@ def test_movie_collection_conflict_only_drops_automatic_media( "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr("app.chain.transfer.DownloadHistoryOper", lambda: history_oper) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: history_oper) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.StorageChain", lambda: SimpleNamespace()) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace()) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda *args, **kwargs: file_meta) # 用真 MediaInfo 而非 SimpleNamespace：它会被装进 TransferTask.mediainfo， diff --git a/tests/test_transfer_overwrite_declined.py b/tests/test_transfer_overwrite_declined.py index 6d5d07890..650653e59 100644 --- a/tests/test_transfer_overwrite_declined.py +++ b/tests/test_transfer_overwrite_declined.py @@ -53,7 +53,7 @@ def test_overwrite_declined_false_when_flag_not_set(): transferinfo = TransferInfo(success=False, overwrite_skipped=False) transferhis = make_history_oper(raise_on_query=True) - result = TransferChain._TransferChain__is_overwrite_declined( + result = TransferChain._is_overwrite_declined( task, transferinfo, transferhis ) @@ -67,7 +67,7 @@ def test_overwrite_declined_true_when_success_history_exists(): transferinfo = TransferInfo(success=False, overwrite_skipped=True) transferhis = make_history_oper(history=success_history) - result = TransferChain._TransferChain__is_overwrite_declined( + result = TransferChain._is_overwrite_declined( task, transferinfo, transferhis ) @@ -80,7 +80,7 @@ def test_overwrite_declined_false_when_no_history(): transferinfo = TransferInfo(success=False, overwrite_skipped=True) transferhis = make_history_oper(history=None) - result = TransferChain._TransferChain__is_overwrite_declined( + result = TransferChain._is_overwrite_declined( task, transferinfo, transferhis ) @@ -94,7 +94,7 @@ def test_overwrite_declined_false_when_only_failed_history(): transferinfo = TransferInfo(success=False, overwrite_skipped=True) transferhis = make_history_oper(history=failed_history, success_history=None) - result = TransferChain._TransferChain__is_overwrite_declined( + result = TransferChain._is_overwrite_declined( task, transferinfo, transferhis ) @@ -107,7 +107,7 @@ def test_overwrite_declined_false_when_query_raises(): transferinfo = TransferInfo(success=False, overwrite_skipped=True) transferhis = make_history_oper(raise_on_query=True) - result = TransferChain._TransferChain__is_overwrite_declined( + result = TransferChain._is_overwrite_declined( task, transferinfo, transferhis ) diff --git a/tests/test_transfer_stale_tasks.py b/tests/test_transfer_stale_tasks.py index 1b23b488d..90210bb0d 100644 --- a/tests/test_transfer_stale_tasks.py +++ b/tests/test_transfer_stale_tasks.py @@ -1,10 +1,10 @@ """整理任务失活收敛行为测试。""" -from app.chain import transfer from app.chain.transfer import JobManager from app.domain.meta.metabase import MetaBase from app.schemas import FileItem from app.application.transfer import TransferTask +from app.application import transfer as app_transfer from app.schemas.types import MediaType @@ -74,7 +74,7 @@ def _make_task(name: str = "Test.Show.S01E01.mkv") -> TransferTask: def test_external_running_task_expires_without_heartbeat(monkeypatch): """外部接管的运行中任务超过心跳期限后应被标记失败并清理。""" clock = [100.0] - monkeypatch.setattr(transfer, "monotonic", lambda: clock[0]) + monkeypatch.setattr(app_transfer, "monotonic", lambda: clock[0]) manager = JobManager() task = _make_task() assert manager.add_task(task) @@ -90,7 +90,7 @@ def test_external_running_task_expires_without_heartbeat(monkeypatch): def test_main_thread_execution_is_not_expired(monkeypatch): """主程序整理线程仍在执行的任务不应被失活检测伪清理。""" clock = [100.0] - monkeypatch.setattr(transfer, "monotonic", lambda: clock[0]) + monkeypatch.setattr(app_transfer, "monotonic", lambda: clock[0]) manager = JobManager() task = _make_task() assert manager.add_task(task) @@ -110,7 +110,7 @@ def test_main_thread_execution_is_not_expired(monkeypatch): def test_waiting_task_and_refreshed_heartbeat_do_not_expire(monkeypatch): """等待中任务不受失活期限影响，重复运行状态更新可刷新外部心跳。""" clock = [100.0] - monkeypatch.setattr(transfer, "monotonic", lambda: clock[0]) + monkeypatch.setattr(app_transfer, "monotonic", lambda: clock[0]) manager = JobManager() waiting_task = _make_task("Test.Show.S01E01.waiting.mkv") running_task = _make_task("Test.Show.S01E02.running.mkv") diff --git a/tests/test_transfer_sync_extra_files.py b/tests/test_transfer_sync_extra_files.py index c4549ec8f..3b02a50e6 100644 --- a/tests/test_transfer_sync_extra_files.py +++ b/tests/test_transfer_sync_extra_files.py @@ -112,12 +112,12 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -142,6 +142,7 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -151,10 +152,17 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", fake_meta_info_path) state, errmsg = TransferChain.do_transfer( @@ -201,12 +209,12 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -231,6 +239,7 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -240,10 +249,17 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -254,6 +270,13 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): ], ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + get_parent_item=lambda fileitem: parent_fileitem, + list_files=lambda fileitem, recursion=False: [ + main_fileitem, + subtitle_fileitem, + ], + )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", fake_meta_info_path) state, errmsg = TransferChain.do_transfer( @@ -299,12 +322,12 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -331,6 +354,7 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -340,10 +364,17 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -351,6 +382,10 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch list_files=fake_list_files, ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + get_parent_item=lambda fileitem: parent_fileitem, + list_files=fake_list_files, + )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(2)) state, errmsg = TransferChain.do_transfer( @@ -396,12 +431,12 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -417,6 +452,7 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -426,10 +462,17 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -475,12 +518,12 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -496,6 +539,7 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -505,10 +549,17 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -554,12 +605,12 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -584,6 +635,7 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -593,10 +645,17 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -607,6 +666,13 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat ], ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + get_parent_item=lambda fileitem: parent_fileitem, + list_files=lambda fileitem, recursion=False: [ + main_fileitem, + subtitle_fileitem, + ], + )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", fake_meta_info_path) state, errmsg = TransferChain.do_transfer( @@ -644,12 +710,12 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp monkeypatch.setattr(chain, "_TransferChain__put_to_jobview", lambda task: True) monkeypatch.setattr( chain, - "_TransferChain__register_scrape_batch_task", + "_register_scrape_batch_task", lambda task: None, ) monkeypatch.setattr( chain, - "_TransferChain__close_scrape_batch", + "_close_scrape_batch", lambda batch_id: None, ) @@ -665,6 +731,7 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -674,16 +741,26 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp get_by_path=lambda path: None, ), ) + monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + get_by_hash=lambda download_hash: None, + get_file_by_fullpath=lambda fullpath: None, + get_files_by_savepath=lambda savepath: [], + get_by_path=lambda path: None, + )) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, + )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -722,12 +799,16 @@ def test_cleanup_dest_fileitem_is_kept_when_episode_format_matches_nothing(monke "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, ), ) + monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, + )) state, errmsg = TransferChain.do_transfer( chain, @@ -760,6 +841,7 @@ def test_episode_format_matched_but_filtered_by_size_returns_failure(monkeypatch "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) state, errmsg = TransferChain.do_transfer( chain, @@ -800,6 +882,7 @@ def test_candidate_collection_checks_continue_callback(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) + monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) state, errmsg = TransferChain.do_transfer( chain, diff --git a/tests/test_transfer_tmdb_category.py b/tests/test_transfer_tmdb_category.py index 9a3c6be17..bae4ed2ed 100644 --- a/tests/test_transfer_tmdb_category.py +++ b/tests/test_transfer_tmdb_category.py @@ -79,12 +79,16 @@ def test_transfer_stops_when_automatic_category_has_no_tmdb_result(monkeypatch) "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(), ) + monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace()) monkeypatch.setattr( "app.chain.transfer.MediaChain", lambda: SimpleNamespace( supplement_tmdb_info=lambda media, _meta: media, ), ) + monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: SimpleNamespace( + supplement_tmdb_info=lambda media, _meta: media, + )) task = TransferTask( fileitem=FileItem( storage="local", From b8b59ae20a5abf875adab2851066a513d2d7eac9 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:44:45 +0800 Subject: [PATCH 3/8] refactor(runtime): activate managed resources on demand (#6334) --- AGENTS.md | 14 +- app/adapters/network/browser.py | 62 ++- app/adapters/system/display.py | 29 -- app/adapters/system/display/__init__.py | 53 +++ app/adapters/system/display/capability.toml | 12 + app/adapters/system/display/resource.py | 45 ++ app/runtime/compat/resource_imports.py | 196 ++++++++ .../extensions/managed_resource_adapter.py | 194 ++++++++ app/runtime/extensions/plugin_manager.py | 23 + app/runtime/managed_resources.py | 182 +++++++ app/sdk/browser.py | 34 ++ app/startup/managed_resources_initializer.py | 47 ++ app/startup/modules_initializer.py | 19 +- app/startup/plugins_initializer.py | 15 + docs/rules/05-architecture.md | 21 +- scripts/perf/README.md | 31 ++ scripts/perf/instrument/sitecustomize.py | 344 +++++++++++++- scripts/perf/moviepilot_docker_ab.py | 449 +++++++++++++++--- scripts/perf/test_scenarios.py | 430 +++++++++++++++++ tests/test_agent_lifecycle.py | 2 +- tests/test_browser_helper.py | 65 ++- tests/test_cache_system.py | 2 +- tests/test_display_resource.py | 99 ++++ tests/test_legacy_plugin_resource_imports.py | 382 +++++++++++++++ tests/test_lifecycle_shutdown.py | 33 +- tests/test_managed_resources.py | 339 +++++++++++++ tests/test_plugin_sdk.py | 62 +++ 27 files changed, 3068 insertions(+), 116 deletions(-) delete mode 100644 app/adapters/system/display.py create mode 100644 app/adapters/system/display/__init__.py create mode 100644 app/adapters/system/display/capability.toml create mode 100644 app/adapters/system/display/resource.py create mode 100644 app/runtime/compat/resource_imports.py create mode 100644 app/runtime/extensions/managed_resource_adapter.py create mode 100644 app/runtime/managed_resources.py create mode 100644 app/sdk/browser.py create mode 100644 app/startup/managed_resources_initializer.py create mode 100644 scripts/perf/test_scenarios.py create mode 100644 tests/test_display_resource.py create mode 100644 tests/test_legacy_plugin_resource_imports.py create mode 100644 tests/test_managed_resources.py diff --git a/AGENTS.md b/AGENTS.md index 23fd86dc7..4e9b9ea53 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -60,19 +60,19 @@ The legacy roots have no physical directories in the source tree. Current images |---|---|---|---| | `app/foundation/` | 无状态、无配置和无 I/O 的底层机制：反射/动态导入、加密、DOM、身份、集合、单例、文本、URL 和版本比较 | `settings`、DB/SystemConfig、网络请求、运行日志、MoviePilot 业务规则、旧导入路径 | `reflection.py`, `crypto.py`, `collections.py`, `text.py`, `url.py` | | `app/domain/` | Pure MoviePilot business semantics and models for media, recognition, sites, and torrents | Persistence, global settings reads, network/filesystem clients, Rust imports, service discovery, process lifecycle | `context.py`, `media.py`, `metainfo.py`, `scraper.py`, `meta/` | -| `app/runtime/` | 进程级运行机制和策略：配置、事件、完整日志、缓存契约/内存行为、并发、调度、限流、本地化、GC 和重启状态 | 具体外部产品、业务流程、Redis/文件缓存实现 | `config.py`, `events.py`, `log.py`, `cache.py`, `thread.py`, `state.py` | -| `app/runtime/extensions/` | 模块、插件和配置化服务实现的发现、注册与生命周期 | 通用反射机制、插件公开 API、无关业务流程 | `module_manager.py`, `plugin_manager.py`, `service_registry.py` | +| `app/runtime/` | 进程级运行机制和策略：配置、事件、完整日志、缓存契约/内存行为、托管资源门面、并发、调度、限流、本地化、GC 和重启状态 | 具体外部产品、业务流程、Redis/文件缓存实现 | `config.py`, `events.py`, `log.py`, `cache.py`, `managed_resources.py`, `thread.py`, `state.py` | +| `app/runtime/extensions/` | 模块、插件、配置化服务和托管资源实现的发现、注册与生命周期适配 | 通用反射机制、插件公开 API、无关业务流程 | `module_manager.py`, `plugin_manager.py`, `managed_resource_adapter.py`, `service_registry.py` | | `app/adapters/network/` | HTTP、浏览器、DNS、Cloudflare 和 IP 等通用网络技术适配 | RSS/站点业务编排、身份认证策略、命名外部产品流程 | `http.py`, `browser.py`, `doh.py`, `ip.py` | | `app/adapters/cache/` | Redis 与文件缓存等具体持久化实现 | 缓存协议、装饰器和进程内缓存策略 | `backends.py`, `redis.py` | -| `app/adapters/system/` | 操作系统、文件、进程、标准流、包/资源安装、显示和 Rust 加速适配 | 业务规则、进程重启决策 | `host.py`, `stdio.py`, `package.py`, `resource.py`, `rust.py`, `fsproxy.py` | +| `app/adapters/system/` | 操作系统、文件、进程、标准流、包/资源安装、显示和 Rust 加速适配 | 业务规则、进程重启决策 | `host.py`, `display/`, `stdio.py`, `package.py`, `resource.py`, `rust.py`, `fsproxy.py` | | `app/adapters/external/` | CookieCloud、插件市场、OCR、IP 归属和 MoviePilot Server 等命名外部生态 | 通用 HTTP/DNS/文件机制或可复用领域语义 | `market.py`, `server.py`, `cookiecloud.py`, `ocr.py`, `location.py`, `wechat_crypt.py` | | `app/application/` | 读取配置/持久化状态的聚焦应用服务和服务族规则 | 多领域 Chain 编排、底层通用机制、通用传输协议 | `recognition.py`, `filter.py`, `filter_rules.py`, `notification.py`, `mediaserver.py`, `rss.py`, `site/sites.*` | | `app/application/messaging/` | 消息渲染/路由、交互和 Agent 到消息桥接：`interaction.py` 通用交互契约和视图工具；`router.py` 统一交互优先级和回调分发；`site.py`/`subscribe.py`/`skill.py` 对应命令的会话、输入解析和视图；`media.py` 媒体交互状态（业务工作流仍由 `MediaInteractionChain` 执行）；`plugin.py` 插件输入接管和插件按钮回调；`agent.py` Agent 选择状态、回调协议和 WebAgent 消息桥接；`message.py` 通知渲染、模板和队列。不作为推荐给插件直接使用的公开 SDK | 认证策略、通用 HTTP、服务发现、仅端点使用的 Web Push 行为 | `message.py`, `interaction.py`, `router.py`, `agent.py` | | `app/application/security/` | 认证、授权、Cookie、Passkey、OTP/二次认证、路径/URL 安全、SSRF 和签名策略 | 通用 URL 解析、进程运行策略、普通业务校验 | `access.py`, `auth.py`, `cookie.py`, `passkey.py`, `otp.py`, `twofactor.py`, `url.py` | | `app/chain/` | Reusable use-case orchestration across modules, services, Oper classes, events, and caches; chains reach modules only through `run_module` dispatch on method-name contracts | Transport schemas, backend-specific protocol details, generic primitives, direct imports of module internals (classes, exceptions, constants) | `media.py`, `download.py`, `subscribe.py`, `transfer.py` | -| `app/startup/` | Composition root: inject providers/adapters, order initialization and shutdown, decide restart/lifecycle policy | Reusable business rules or adapter implementation details | `lifecycle.py`, `domain_initializer.py`, `cache_initializer.py`, `modules_initializer.py` | -| `app/sdk/` | Deliberately curated stable imports for new plugins | Canonical implementation logic or host-internal dependencies | `cache.py`, `logging.py`, `media.py`, `network.py`, `services.py` | -| `app/runtime/compat/` | 仅依赖标准库的精确旧导入路由和 DEBUG 诊断 | 业务实现、通配猜测、目标模块的提前导入 | `manifest.py`, `imports.py`, `diagnostics.py` | +| `app/startup/` | Composition root: inject providers/adapters, order initialization and shutdown, decide restart/lifecycle policy | Reusable business rules or adapter implementation details | `lifecycle.py`, `domain_initializer.py`, `cache_initializer.py`, `managed_resources_initializer.py`, `modules_initializer.py` | +| `app/sdk/` | Deliberately curated stable imports for new plugins | Canonical implementation logic or host-internal dependencies | `browser.py`, `cache.py`, `logging.py`, `media.py`, `network.py`, `services.py` | +| `app/runtime/compat/` | 仅依赖标准库的精确旧导入路由、资源前置扫描和 DEBUG 诊断 | 业务实现、通配猜测、目标模块的提前导入 | `manifest.py`, `imports.py`, `resource_imports.py`, `diagnostics.py` | 容易误分的三个边界必须按实际职责判断：`application/rss.py` 同时承担 Feed/种子语义、站点规则和浏览器回退，不是单纯 HTTP 传输；`application/site/sites.*` 及 `user.sites.v3.bin` 共同构成站点目录、认证和索引应用能力，只有下载安装机制留在 `adapters/system/resource.py`；`foundation/crypto.py` 只提供无状态 RSA/摘要/AES 算法，认证、签名、令牌和二次验证策略仍属于 `application/security/`。 @@ -156,4 +156,4 @@ For the full documentation map and cross-references, refer to: **[Documentation Hub Index](./docs/rules/README.md)** -*Last Updated: 2026-08-15* +*Last Updated: 2026-08-16* diff --git a/app/adapters/network/browser.py b/app/adapters/network/browser.py index 8a3202f09..06149c556 100644 --- a/app/adapters/network/browser.py +++ b/app/adapters/network/browser.py @@ -9,6 +9,10 @@ from urllib.parse import urlparse from app.runtime.config import settings from app.runtime.log import logger +from app.runtime.managed_resources import ( + acquire_managed_resource, + acquire_managed_resource_async, +) from app.adapters.network.http import RequestUtils, cookie_parse @@ -117,6 +121,47 @@ class BrowserPage(Protocol): ... +def launch_browser_context(headless: bool = True, **kwargs: Any) -> BrowserContext: + """ + 启动同步浏览器上下文；有界面模式先显式获取宿主显示资源。 + + :param headless: 是否使用无头模式 + :param kwargs: 浏览器实现接受的其余启动参数 + :return: 浏览器上下文 + """ + if not headless: + acquire_managed_resource( + "host.display", + reason="headed_browser_launch", + retry=True, + ) + from cloakbrowser import launch_context + + return launch_context(headless=headless, **kwargs) + + +async def launch_browser_context_async( + headless: bool = True, + **kwargs: Any, +) -> Any: + """ + 启动异步浏览器上下文；有界面模式等待宿主显示资源就绪。 + + :param headless: 是否使用无头模式 + :param kwargs: 浏览器实现接受的其余启动参数 + :return: 浏览器上下文 + """ + if not headless: + await acquire_managed_resource_async( + "host.display", + reason="headed_browser_launch", + retry=True, + ) + from cloakbrowser import launch_context_async + + return await launch_context_async(headless=headless, **kwargs) + + @dataclass class _BrowserSessionState: """保存一个可复用浏览器上下文及其页面游标。""" @@ -662,10 +707,7 @@ class BrowserSessionHelper: viewport: Optional[dict[str, int]] = None, ) -> BrowserContext: """按宿主反检测配置创建 CloakBrowser 上下文。""" - from cloakbrowser import launch_context - context_kwargs = { - "headless": headless, "humanize": settings.CLOAKBROWSER_HUMANIZE, "human_preset": settings.CLOAKBROWSER_HUMAN_PRESET, } @@ -673,7 +715,7 @@ class BrowserSessionHelper: context_kwargs["user_agent"] = user_agent if viewport: context_kwargs["viewport"] = viewport - return launch_context(**context_kwargs) + return launch_browser_context(headless=headless, **context_kwargs) def _get_or_create_session( self, @@ -883,13 +925,11 @@ class PlaywrightHelper: """ 启动 CloakBrowser 上下文。 """ - from cloakbrowser import launch_context - - return launch_context(headless=headless, - proxy=proxies, - user_agent=user_agent, - humanize=settings.CLOAKBROWSER_HUMANIZE, - human_preset=settings.CLOAKBROWSER_HUMAN_PRESET) + return launch_browser_context(headless=headless, + proxy=proxies, + user_agent=user_agent, + humanize=settings.CLOAKBROWSER_HUMANIZE, + human_preset=settings.CLOAKBROWSER_HUMAN_PRESET) @staticmethod def __fs_cookie_str(cookies: list) -> str: diff --git a/app/adapters/system/display.py b/app/adapters/system/display.py deleted file mode 100644 index 614e58dca..000000000 --- a/app/adapters/system/display.py +++ /dev/null @@ -1,29 +0,0 @@ -from pyvirtualdisplay import Display - -from app.runtime.log import logger -from app.foundation.singleton import Singleton -from app.adapters.system.host import SystemUtils - -import os - - -class DisplayHelper(metaclass=Singleton): - """在容器环境中管理浏览器所需的虚拟显示。""" - - def __init__(self): - """仅在 Docker 内启动虚拟显示服务。""" - self._display = None - if not SystemUtils.is_docker(): - return - try: - self._display = Display(visible=False, size=(1024, 768), extra_args=[os.environ['DISPLAY']]) - self._display.start() - except Exception as err: - logger.error(f"DisplayHelper init error: {str(err)}") - - def stop(self): - """停止已经启动的虚拟显示服务。""" - if self._display: - logger.info("正在停止虚拟显示...") - self._display.stop() - logger.info("虚拟显示已停止") diff --git a/app/adapters/system/display/__init__.py b/app/adapters/system/display/__init__.py new file mode 100644 index 000000000..f64aa0797 --- /dev/null +++ b/app/adapters/system/display/__init__.py @@ -0,0 +1,53 @@ +"""虚拟显示适配器及旧 DisplayHelper 兼容入口。""" + +from __future__ import annotations + +from importlib import import_module +from typing import Any + +from app.foundation.singleton import Singleton +from app.runtime.log import logger +from app.runtime.managed_resources import ( + acquire_managed_resource, + stop_managed_resource, +) + + +DISPLAY_CAPABILITY_ID = "host.display" + + +class DisplayHelper(metaclass=Singleton): + """保留旧构造 API，并把资源所有权委托给 host.display 能力。""" + + def __init__(self) -> None: + """显式构造旧门面时激活虚拟显示，失败保持旧 API 的日志语义。""" + try: + acquire_managed_resource( + DISPLAY_CAPABILITY_ID, + reason="legacy_display_helper", + retry=True, + ) + except Exception as error: + logger.error("DisplayHelper init error: %s", error) + + def stop(self) -> None: + """停止已激活的虚拟显示；未配置 Runtime 时保持幂等。""" + stop_managed_resource( + DISPLAY_CAPABILITY_ID, + reason="legacy_display_helper_stop", + ) + + +__all__ = ["DISPLAY_CAPABILITY_ID", "DisplayHelper", "VirtualDisplayResource"] + + +def __getattr__(name: str) -> Any: + """按需公开资源实现，普通兼容导入不加载显示后端。""" + if name != "VirtualDisplayResource": + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr( + import_module("app.adapters.system.display.resource"), + "VirtualDisplayResource", + ) + globals()[name] = value + return value diff --git a/app/adapters/system/display/capability.toml b/app/adapters/system/display/capability.toml new file mode 100644 index 000000000..ee17d37c4 --- /dev/null +++ b/app/adapters/system/display/capability.toml @@ -0,0 +1,12 @@ +schema_version = 1 +id = "host.display" +kind = "managed_resource.sync" +entrypoint = "app.adapters.system.display.resource:VirtualDisplayResource" +depends_on = [] + +[metadata] +name = "Virtual Display" + +[activation] +policy = "on_first_use" +watch = [] diff --git a/app/adapters/system/display/resource.py b/app/adapters/system/display/resource.py new file mode 100644 index 000000000..c91e730fb --- /dev/null +++ b/app/adapters/system/display/resource.py @@ -0,0 +1,45 @@ +"""虚拟显示进程的托管资源实现。""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +from app.adapters.system.host import SystemUtils +from app.runtime.log import logger + + +class VirtualDisplayResource: + """按需拥有一个容器内虚拟显示进程。""" + + def __init__(self) -> None: + self._display: Optional[Any] = None + + @property + def display(self) -> Optional[Any]: + """返回当前拥有的显示对象；未启动或已停止时为 None。""" + return self._display + + def start(self) -> None: + """仅在容器环境启动虚拟显示，重复启动保持幂等。""" + if self._display is not None or not SystemUtils.is_docker(): + return + from pyvirtualdisplay import Display + + display = Display( + visible=False, + size=(1024, 768), + extra_args=[os.environ["DISPLAY"]], + ) + self._display = display + display.start() + + def stop(self) -> None: + """停止当前资源拥有的显示进程，失败时保留句柄供 Runtime 重试。""" + display = self._display + if display is None: + return + logger.info("正在停止虚拟显示...") + display.stop() + self._display = None + logger.info("虚拟显示已停止") diff --git a/app/runtime/compat/resource_imports.py b/app/runtime/compat/resource_imports.py new file mode 100644 index 000000000..9abf6f36a --- /dev/null +++ b/app/runtime/compat/resource_imports.py @@ -0,0 +1,196 @@ +"""从旧插件源码导入中识别必须提前就绪的宿主资源。""" + +from __future__ import annotations + +import ast +import threading +import tokenize +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, FrozenSet, Iterable, Set, Tuple + + +@dataclass(frozen=True, slots=True) +class ResourceImportRule: + """描述第三方模块导入与宿主资源能力之间的静态映射。""" + + capability_id: str # 导入前必须准备的宿主能力标识 + module_prefixes: tuple[str, ...] # 按完整包边界匹配的第三方模块前缀 + headed_entrypoints: tuple[str, ...] # 已确认允许 headed 模式的公开入口 + + +# 旧插件可能绕过宿主浏览器门面直接调用 CloakBrowser。其六个 launch +# 入口均允许 headed 模式，因此导入该包或任意子模块时保守准备虚拟显示。 +RESOURCE_IMPORT_RULES: tuple[ResourceImportRule, ...] = ( + ResourceImportRule( + capability_id="host.display", + module_prefixes=("cloakbrowser",), + headed_entrypoints=( + "launch", + "launch_async", + "launch_context", + "launch_context_async", + "launch_persistent_context", + "launch_persistent_context_async", + ), + ), +) + + +_scan_cache_lock = threading.RLock() +_scan_cache: Dict[Path, Tuple[int, int, int, int, int, FrozenSet[str]]] = {} + + +class PluginResourceImportScanError(RuntimeError): + """表示单个插件源码无法生成可靠的精确资源集合。""" + + +def _all_resource_capabilities() -> FrozenSet[str]: + """扫描不完整时返回全部已登记资源，避免漏失导入前置条件。""" + return frozenset(rule.capability_id for rule in RESOURCE_IMPORT_RULES) + + +def _matches_module(module_name: str, module_prefixes: Iterable[str]) -> bool: + """按完整包边界匹配模块，避免相似名称产生误报。""" + return any( + module_name == prefix or module_name.startswith(f"{prefix}.") + for prefix in module_prefixes + ) + + +def _dynamic_import_aliases(tree: ast.AST) -> tuple[Set[str], Set[str]]: + """收集 importlib 模块及 import_module 函数的本地别名。""" + module_aliases = {"importlib"} + function_aliases: Set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for imported in node.names: + if imported.name == "importlib": + module_aliases.add(imported.asname or imported.name) + elif isinstance(node, ast.ImportFrom) and node.module == "importlib": + for imported in node.names: + if imported.name == "import_module": + function_aliases.add(imported.asname or imported.name) + return module_aliases, function_aliases + + +def _constant_dynamic_import( + node: ast.Call, + *, + importlib_aliases: Set[str], + import_module_aliases: Set[str], +) -> str | None: + """提取受支持动态导入调用中的常量模块名。""" + if not node.args: + return None + is_import_call = isinstance(node.func, ast.Name) and ( + node.func.id == "__import__" or node.func.id in import_module_aliases + ) + if ( + isinstance(node.func, ast.Attribute) + and isinstance(node.func.value, ast.Name) + and node.func.value.id in importlib_aliases + and node.func.attr == "import_module" + ): + is_import_call = True + if not is_import_call: + return None + argument = node.args[0] + if isinstance(argument, ast.Constant) and isinstance(argument.value, str): + return argument.value + return None + + +def _imported_modules(tree: ast.AST) -> FrozenSet[str]: + """提取静态导入以及可确定目标的动态导入模块名。""" + modules: Set[str] = set() + importlib_aliases, import_module_aliases = _dynamic_import_aliases(tree) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(imported.name for imported in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + modules.add(node.module) + elif isinstance(node, ast.Call): + module_name = _constant_dynamic_import( + node, + importlib_aliases=importlib_aliases, + import_module_aliases=import_module_aliases, + ) + if module_name: + modules.add(module_name) + return frozenset(modules) + + +def _scan_source(plugin_id: str, path: Path) -> FrozenSet[str]: + """读取并解析单个源码文件；不完整结果不能进入插件导入阶段。""" + try: + before_stat = path.stat() + # 热加载工具可能保留 mtime，等长替换也不会改变 size；ctime 与 inode/device + # 一并参与身份判断，避免把已替换源码误认为旧缓存。 + cache_key = ( + before_stat.st_mtime_ns, + before_stat.st_ctime_ns, + before_stat.st_size, + before_stat.st_dev, + before_stat.st_ino, + ) + with _scan_cache_lock: + cached = _scan_cache.get(path) + if cached and cached[:5] == cache_key: + return cached[5] + with tokenize.open(path) as source_file: + source = source_file.read() + tree = ast.parse(source, filename=str(path)) + after_stat = path.stat() + except (OSError, SyntaxError, UnicodeError) as error: + raise PluginResourceImportScanError( + f"无法扫描插件 {plugin_id} 源码 {path.name}：{error}" + ) from error + after_key = ( + after_stat.st_mtime_ns, + after_stat.st_ctime_ns, + after_stat.st_size, + after_stat.st_dev, + after_stat.st_ino, + ) + if cache_key != after_key: + raise PluginResourceImportScanError( + f"扫描插件 {plugin_id} 时源码 {path.name} 发生变化" + ) + + capabilities: Set[str] = set() + for module_name in _imported_modules(tree): + for rule in RESOURCE_IMPORT_RULES: + if _matches_module(module_name, rule.module_prefixes): + capabilities.add(rule.capability_id) + result = frozenset(capabilities) + with _scan_cache_lock: + _scan_cache[path] = (*cache_key, result) + return result + + +def scan_plugin_resource_imports( + plugin_id: str, + plugin_dir: Path, +) -> tuple[str, ...]: + """递归扫描插件源码并返回导入前必须准备的 capability ID。""" + if not plugin_dir.is_dir(): + raise PluginResourceImportScanError( + f"插件 {plugin_id} 源码目录不存在：{plugin_dir}" + ) + + capabilities: Set[str] = set() + try: + source_files = sorted(plugin_dir.rglob("*.py")) + except OSError: + return tuple(sorted(_all_resource_capabilities())) + for path in source_files: + if "__pycache__" in path.parts: + continue + try: + capabilities.update(_scan_source(plugin_id, path)) + except PluginResourceImportScanError: + # Python 最终只会导入真实依赖链；无法解析的残留或平台专用文件不应 + # 阻断整个插件，但必须按最保守资源集合准备后再交给 loader 判断。 + capabilities.update(_all_resource_capabilities()) + return tuple(sorted(capabilities)) diff --git a/app/runtime/extensions/managed_resource_adapter.py b/app/runtime/extensions/managed_resource_adapter.py new file mode 100644 index 000000000..86c9e5d47 --- /dev/null +++ b/app/runtime/extensions/managed_resource_adapter.py @@ -0,0 +1,194 @@ +"""Managed Resource 的声明发现与 Capability Runtime 适配器。""" + +from __future__ import annotations + +import asyncio +import importlib +import inspect +from pathlib import Path +from typing import Any, Iterable + +from app.runtime.capabilities.errors import CapabilityAdapterContractError +from app.runtime.capabilities.model import ( + ActivationPolicy, + AdapterExecutionMode, + CapabilitySpec, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.managed_resources import ( + MANAGED_RESOURCE_ASYNC_KIND, + MANAGED_RESOURCE_SYNC_KIND, +) + + +_DEFAULT_RESOURCE_ROOT = Path(__file__).resolve().parents[2] / "adapters" +_RESOURCE_KINDS = {MANAGED_RESOURCE_SYNC_KIND, MANAGED_RESOURCE_ASYNC_KIND} + + +def _load_entrypoint(spec: CapabilitySpec) -> Any: + """解析声明中的 canonical 实现对象，不创建资源实例。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = importlib.import_module(module_name) + try: + return getattr(module, symbol_name) + except AttributeError as error: + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 未公开 Managed Resource 实现" + ) from error + + +def _create_candidate(spec: CapabilitySpec, implementation: Any) -> Any: + """通过零参数工厂创建资源候选，实例在 start 成功前不可见。""" + if not callable(implementation): + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 不是可调用的 Managed Resource 工厂" + ) + candidate = implementation() + if candidate is None or inspect.isawaitable(candidate): + close = getattr(candidate, "close", None) + if callable(close): + close() + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 必须同步返回资源候选" + ) + return candidate + + +def _resource_method(spec: CapabilitySpec, candidate: Any, name: str) -> Any: + """读取必需生命周期方法并生成稳定合同错误。""" + callback = getattr(candidate, name, None) + if not callable(callback): + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 的资源候选缺少 {name}()" + ) + return callback + + +class SyncManagedResourceAdapter: + """把同步 start/stop 资源接入 Capability Runtime。""" + + execution_mode = AdapterExecutionMode.SYNC + + @staticmethod + def materialize(spec: CapabilitySpec) -> Any: + """解析资源工厂。""" + return _load_entrypoint(spec) + + @staticmethod + def create( + spec: CapabilitySpec, + implementation: Any, + _generation: int, + _previous: Any = None, + ) -> Any: + """创建尚未发布的同步资源候选。""" + return _create_candidate(spec, implementation) + + @staticmethod + def start(spec: CapabilitySpec, candidate: Any, _generation: int) -> None: + """启动同步候选；同步 kind 不接受 awaitable 返回值。""" + result = _resource_method(spec, candidate, "start")() + if inspect.isawaitable(result): + close = getattr(result, "close", None) + if callable(close): + close() + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.start() 返回 awaitable，与同步 kind 不匹配" + ) + + @staticmethod + def stop(spec: CapabilitySpec, instance: Any, _generation: int) -> None: + """停止同步资源，异常交由 Runtime 保留资源所有权并支持重试。""" + result = _resource_method(spec, instance, "stop")() + if inspect.isawaitable(result): + close = getattr(result, "close", None) + if callable(close): + close() + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.stop() 返回 awaitable，与同步 kind 不匹配" + ) + + @staticmethod + def cleanup( + spec: CapabilitySpec, + candidate: Any, + generation: int, + _error: BaseException, + ) -> None: + """启动失败时按同一 stop 合同清理尚未发布的候选。""" + SyncManagedResourceAdapter.stop(spec, candidate, generation) + + +class AsyncManagedResourceAdapter: + """把异步 start/stop 资源接入 Capability Runtime。""" + + execution_mode = AdapterExecutionMode.ASYNC + + @staticmethod + async def materialize(spec: CapabilitySpec) -> Any: + """在线程中解析资源工厂，避免第三方导入阻塞事件循环。""" + return await asyncio.to_thread(_load_entrypoint, spec) + + @staticmethod + async def create( + spec: CapabilitySpec, + implementation: Any, + _generation: int, + _previous: Any = None, + ) -> Any: + """创建尚未发布的异步资源候选。""" + return _create_candidate(spec, implementation) + + @staticmethod + async def start(spec: CapabilitySpec, candidate: Any, _generation: int) -> None: + """等待异步候选完成启动。""" + result = _resource_method(spec, candidate, "start")() + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.start() 必须返回 awaitable" + ) + await result + + @staticmethod + async def stop(spec: CapabilitySpec, instance: Any, _generation: int) -> None: + """等待异步资源完成停止。""" + result = _resource_method(spec, instance, "stop")() + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.stop() 必须返回 awaitable" + ) + await result + + @staticmethod + async def cleanup( + spec: CapabilitySpec, + candidate: Any, + generation: int, + _error: BaseException, + ) -> None: + """启动失败时等待同一 stop 合同清理候选。""" + await AsyncManagedResourceAdapter.stop(spec, candidate, generation) + + +def _validate_registry(registry: CapabilityRegistry) -> None: + """固定类别级声明合同，资源只能由显式首用触发。""" + for spec in registry.list_specs(): + if set(spec.metadata) != {"name"}: + raise ValueError(f"{spec.source}: Managed Resource metadata 只能包含 name") + if spec.activation is not ActivationPolicy.ON_FIRST_USE: + raise ValueError(f"{spec.source}: Managed Resource 必须使用 on_first_use") + if spec.selector is not None or spec.watch: + raise ValueError(f"{spec.source}: Managed Resource 不接受配置 selector 或 watch") + + +def build_managed_resource_registry( + roots: Iterable[Path | str] | None = None, +) -> CapabilityRegistry: + """从 data-only manifest 构建不导入资源实现的注册表。""" + registry = CapabilityRegistry.discover( + tuple(roots) if roots is not None else (_DEFAULT_RESOURCE_ROOT,), + kinds=_RESOURCE_KINDS, + selector_schemas={}, + ) + _validate_registry(registry) + return registry diff --git a/app/runtime/extensions/plugin_manager.py b/app/runtime/extensions/plugin_manager.py index 4c73b1395..7ae04c49a 100644 --- a/app/runtime/extensions/plugin_manager.py +++ b/app/runtime/extensions/plugin_manager.py @@ -37,6 +37,7 @@ from app.schemas.types import EventType, SystemConfigKey LegacyDiagnosticsConfigurator = Callable[..., None] LegacyImportScanner = Callable[..., None] +LegacyPluginImportPreparer = Callable[..., None] PluginInstallReporter = Callable[..., None] SiteAuthLevelProvider = Callable[[], int] @@ -45,6 +46,10 @@ def _ignore_legacy_diagnostics(**_kwargs) -> None: """在启动组合根尚未注入兼容服务时保持插件加载可用。""" +def _ignore_plugin_resource_imports(**_kwargs) -> None: + """未进入应用启动组合时不主动创建进程级宿主资源。""" + + def _unavailable_site_auth_level() -> int: """站点能力尚未装配时返回未认证等级。""" return 0 @@ -54,6 +59,9 @@ _legacy_diagnostics_configurator: LegacyDiagnosticsConfigurator = ( _ignore_legacy_diagnostics ) _legacy_import_scanner: LegacyImportScanner = _ignore_legacy_diagnostics +_legacy_plugin_import_preparer: LegacyPluginImportPreparer = ( + _ignore_plugin_resource_imports +) _plugin_install_reporter: PluginInstallReporter = _ignore_legacy_diagnostics _site_auth_level_provider: SiteAuthLevelProvider = _unavailable_site_auth_level @@ -69,6 +77,14 @@ def configure_plugin_legacy_import_services( _legacy_import_scanner = import_scanner +def configure_plugin_resource_import_preparer( + preparer: LegacyPluginImportPreparer, +) -> None: + """注入旧插件导入前的宿主资源准备器。""" + global _legacy_plugin_import_preparer + _legacy_plugin_import_preparer = preparer + + def configure_plugin_install_reporter(reporter: PluginInstallReporter) -> None: """由启动组合根注入插件安装上报器，避免扩展层依赖远程服务。""" global _plugin_install_reporter @@ -318,6 +334,13 @@ class PluginManager(ConfigReloadMixin, metaclass=Singleton): module_name = f"app.plugins.{plugin_dir.name}" logger.debug(f"正在导入插件模块：{module_name}") + # 旧插件可能直接导入带宿主资源前置条件的第三方包。资源必须在 + # Python 执行插件模块顶层代码前就绪，否则导入副作用无法安全回滚。 + _legacy_plugin_import_preparer( + plugin_id=plugin_dir.name, + plugin_dir=plugin_dir, + ) + _legacy_import_scanner( plugin_id=plugin_dir.name, plugin_dir=plugin_dir, diff --git a/app/runtime/managed_resources.py b/app/runtime/managed_resources.py new file mode 100644 index 000000000..d6a21122b --- /dev/null +++ b/app/runtime/managed_resources.py @@ -0,0 +1,182 @@ +"""进程级托管资源的轻量调用门面。""" + +from __future__ import annotations + +import asyncio +import threading +from typing import Any, Optional, Protocol + + +MANAGED_RESOURCE_SYNC_KIND = "managed_resource.sync" +MANAGED_RESOURCE_ASYNC_KIND = "managed_resource.async" + + +class ManagedResourceRuntime(Protocol): + """Managed Resource 门面依赖的最小 Capability Runtime 合同。""" + + @property + def is_shutdown(self) -> bool: + """返回 Runtime 是否已进入不可逆关闭态。""" + + def get_spec(self, capability_id: str) -> Any: + """返回资源声明。""" + + def get_running(self, capability_id: str) -> Any: + """只查询已发布实例。""" + + def snapshot(self, capability_id: str) -> Any: + """返回资源状态快照。""" + + def observations(self, capability_id: Optional[str] = None) -> tuple[Any, ...]: + """返回资源转换观测。""" + + def activate(self, capability_id: str, *, reason: str, retry: bool = False) -> Any: + """通过同步 adapter 激活资源。""" + + async def activate_async( + self, + capability_id: str, + *, + reason: str, + retry: bool = False, + ) -> Any: + """通过异步 adapter 激活资源。""" + + def stop(self, capability_id: str, *, reason: str) -> None: + """通过同步 adapter 停止资源。""" + + async def stop_async(self, capability_id: str, *, reason: str) -> None: + """通过异步 adapter 停止资源。""" + + async def shutdown_async(self, *, reason: str) -> None: + """关闭混合同步和异步 adapter 的 Runtime。""" + + +_runtime_lock = threading.RLock() +_managed_resource_runtime: Optional[ManagedResourceRuntime] = None + + +def configure_managed_resource_runtime(runtime: ManagedResourceRuntime) -> None: + """由启动组合层注入唯一的 Managed Resource Runtime。""" + if runtime is None: + raise ValueError("Managed Resource Runtime 不能为空") + global _managed_resource_runtime + with _runtime_lock: + _managed_resource_runtime = runtime + + +def _runtime(*, required: bool) -> Optional[ManagedResourceRuntime]: + """读取当前 Runtime；资源使用路径要求启动组合已经完成装配。""" + with _runtime_lock: + runtime = _managed_resource_runtime + if runtime is None and required: + raise RuntimeError("Managed Resource Runtime 尚未初始化") + return runtime + + +def _resource_kind(runtime: ManagedResourceRuntime, capability_id: str) -> str: + """返回声明的执行模式；未知资源继续沿用 Runtime 的领域错误。""" + spec = runtime.get_spec(capability_id) + if spec is None: + runtime.get_running(capability_id) + raise RuntimeError(f"未知 Managed Resource：{capability_id}") + return str(spec.kind) + + +def acquire_managed_resource( + capability_id: str, + *, + reason: str, + retry: bool = True, +) -> Any: + """同步激活一个声明为同步模式的托管资源。""" + runtime = _runtime(required=True) + kind = _resource_kind(runtime, capability_id) + if kind != MANAGED_RESOURCE_SYNC_KIND: + raise RuntimeError(f"异步 Managed Resource 不能通过同步入口激活：{capability_id}") + return runtime.activate(capability_id, reason=reason, retry=retry) + + +async def acquire_managed_resource_async( + capability_id: str, + *, + reason: str, + retry: bool = True, +) -> Any: + """异步激活资源；同步资源移交工作线程，避免阻塞事件循环。""" + runtime = _runtime(required=True) + kind = _resource_kind(runtime, capability_id) + if kind == MANAGED_RESOURCE_ASYNC_KIND: + return await runtime.activate_async( + capability_id, + reason=reason, + retry=retry, + ) + if kind == MANAGED_RESOURCE_SYNC_KIND: + return await asyncio.to_thread( + runtime.activate, + capability_id, + reason=reason, + retry=retry, + ) + raise RuntimeError(f"未知 Managed Resource kind：{kind}") + + +def get_running_managed_resource(capability_id: str) -> Any: + """只查询已发布资源；Runtime 未配置时返回 None，不触发初始化。""" + runtime = _runtime(required=False) + if runtime is None: + return None + return runtime.get_running(capability_id) + + +def managed_resource_snapshot(capability_id: str) -> Any: + """返回资源状态快照；Runtime 未配置时返回 None。""" + runtime = _runtime(required=False) + if runtime is None: + return None + return runtime.snapshot(capability_id) + + +def managed_resource_observations( + capability_id: Optional[str] = None, +) -> tuple[Any, ...]: + """返回资源转换观测；Runtime 未配置时返回空快照。""" + runtime = _runtime(required=False) + if runtime is None: + return () + return runtime.observations(capability_id) + + +def stop_managed_resource(capability_id: str, *, reason: str) -> None: + """同步停止资源；Runtime 未配置时保持幂等且不反向初始化。""" + runtime = _runtime(required=False) + if runtime is None: + return + kind = _resource_kind(runtime, capability_id) + if kind != MANAGED_RESOURCE_SYNC_KIND: + raise RuntimeError(f"异步 Managed Resource 不能通过同步入口停止：{capability_id}") + runtime.stop(capability_id, reason=reason) + + +async def stop_managed_resource_async(capability_id: str, *, reason: str) -> None: + """异步停止资源；同步资源移交工作线程。""" + runtime = _runtime(required=False) + if runtime is None: + return + kind = _resource_kind(runtime, capability_id) + if kind == MANAGED_RESOURCE_ASYNC_KIND: + await runtime.stop_async(capability_id, reason=reason) + return + if kind == MANAGED_RESOURCE_SYNC_KIND: + await asyncio.to_thread(runtime.stop, capability_id, reason=reason) + return + raise RuntimeError(f"未知 Managed Resource kind：{kind}") + + +async def shutdown_managed_resource_runtime(*, reason: str) -> None: + """关闭已配置 Runtime；未配置时直接返回，绝不因关闭而创建资源。""" + runtime = _runtime(required=False) + if runtime is None: + return + await runtime.shutdown_async(reason=reason) diff --git a/app/sdk/browser.py b/app/sdk/browser.py new file mode 100644 index 000000000..c98e00413 --- /dev/null +++ b/app/sdk/browser.py @@ -0,0 +1,34 @@ +"""插件可依赖的轻量浏览器启动接口。""" + +from __future__ import annotations + +from typing import Any + + +def launch_browser_context(headless: bool = True, **kwargs: Any) -> Any: + """ + 启动同步浏览器上下文，并由宿主协调所需进程资源。 + + :param headless: 是否使用无头模式 + :param kwargs: 浏览器实现接受的其余启动参数 + :return: 浏览器上下文 + """ + from app.adapters.network.browser import launch_browser_context as launch + + return launch(headless=headless, **kwargs) + + +async def launch_browser_context_async(headless: bool = True, **kwargs: Any) -> Any: + """ + 启动异步浏览器上下文，并由宿主协调所需进程资源。 + + :param headless: 是否使用无头模式 + :param kwargs: 浏览器实现接受的其余启动参数 + :return: 浏览器上下文 + """ + from app.adapters.network.browser import launch_browser_context_async as launch + + return await launch(headless=headless, **kwargs) + + +__all__ = ["launch_browser_context", "launch_browser_context_async"] diff --git a/app/startup/managed_resources_initializer.py b/app/startup/managed_resources_initializer.py new file mode 100644 index 000000000..4a394768a --- /dev/null +++ b/app/startup/managed_resources_initializer.py @@ -0,0 +1,47 @@ +"""Managed Resource 的启动组合与进程关闭入口。""" + +from __future__ import annotations + +import threading +from typing import Optional + +from app.runtime.capabilities.runtime import CapabilityRuntime +from app.runtime.extensions.managed_resource_adapter import ( + AsyncManagedResourceAdapter, + SyncManagedResourceAdapter, + build_managed_resource_registry, +) +from app.runtime.managed_resources import ( + MANAGED_RESOURCE_ASYNC_KIND, + MANAGED_RESOURCE_SYNC_KIND, + configure_managed_resource_runtime, +) + + +_runtime_lock = threading.RLock() +_managed_resource_runtime: Optional[CapabilityRuntime] = None + + +def init_managed_resources() -> CapabilityRuntime: + """构建并注入资源 Runtime；只发现声明，不物化或启动任何资源。""" + global _managed_resource_runtime + with _runtime_lock: + if _managed_resource_runtime is None: + _managed_resource_runtime = CapabilityRuntime( + build_managed_resource_registry(), + adapters={ + MANAGED_RESOURCE_SYNC_KIND: SyncManagedResourceAdapter(), + MANAGED_RESOURCE_ASYNC_KIND: AsyncManagedResourceAdapter(), + }, + ) + configure_managed_resource_runtime(_managed_resource_runtime) + return _managed_resource_runtime + + +async def stop_managed_resources() -> None: + """关闭已经初始化的资源 Runtime；未初始化时不执行发现或激活。""" + with _runtime_lock: + runtime = _managed_resource_runtime + if runtime is None: + return + await runtime.shutdown_async(reason="application_shutdown") diff --git a/app/startup/modules_initializer.py b/app/startup/modules_initializer.py index a54b29ef9..f59a41b7b 100644 --- a/app/startup/modules_initializer.py +++ b/app/startup/modules_initializer.py @@ -22,7 +22,6 @@ from app.runtime.extensions.module_manager import ModuleManager from app.runtime.events import EventManager from app.runtime.state import SystemHelper from app.runtime.thread import ThreadHelper -from app.adapters.system.display import DisplayHelper from app.adapters.network.doh import DohHelper from app.adapters.system.resource import ( ResourceHelper, @@ -36,6 +35,10 @@ from app.command import CommandChain from app.schemas import Notification, NotificationType from app.schemas.types import SystemConfigKey from app.startup.agent_initializer import init_agent, stop_agent +from app.startup.managed_resources_initializer import ( + init_managed_resources, + stop_managed_resources, +) from app.application.security.access import set_superuser_token_payload_provider from app.application.security.auth import build_superuser_token_payload from app.application.image import configure_wallpaper_providers @@ -170,6 +173,13 @@ def update_resources() -> None: logger.error(f"资源更新完成但自动重启失败：{message}") +def close_browser_sessions() -> None: + """在托管资源关闭前释放所有浏览器上下文及其工作线程。""" + from app.adapters.network.browser import BrowserSessionHelper + + BrowserSessionHelper.close_all_sessions() + + async def stop_modules(): """ 服务关闭 @@ -186,7 +196,8 @@ async def stop_modules(): await run_step("AI智能体", stop_agent) await run_step("模块", lambda: ModuleManager().shutdown()) await run_step("事件消费", lambda: EventManager().stop()) - await run_step("虚拟显示", lambda: DisplayHelper().stop()) + await run_step("浏览器会话", close_browser_sessions) + await run_step("托管资源", stop_managed_resources) await run_step("DoH服务", lambda: DohHelper().shutdown()) await run_step("线程池", lambda: ThreadHelper().shutdown()) await run_step("消息服务", stop_message) @@ -201,12 +212,12 @@ async def init_modules(): """ 启动模块 """ + # 托管资源只在这里装配声明与 adapter，具体资源仍由首个消费者显式激活。 + init_managed_resources() # 应用服务不反向依赖 Chain，由启动组合层注入壁纸来源。 configure_wallpaper_services() # 认证访问层不反向依赖数据库实现，由启动组合层注入载荷提供器。 set_superuser_token_payload_provider(build_superuser_token_payload) - # 虚拟显示 - DisplayHelper() # DoH DohHelper() # 站点管理 diff --git a/app/startup/plugins_initializer.py b/app/startup/plugins_initializer.py index b311e0e0c..2fac784b0 100644 --- a/app/startup/plugins_initializer.py +++ b/app/startup/plugins_initializer.py @@ -1,25 +1,40 @@ +from pathlib import Path + from app.runtime.compat.diagnostics import ( configure_legacy_import_diagnostics, scan_plugin_legacy_imports, ) +from app.runtime.compat.resource_imports import scan_plugin_resource_imports from app.runtime.config import global_vars from app.runtime.extensions.plugin_manager import ( PluginManager, configure_plugin_install_reporter, configure_plugin_legacy_import_services, + configure_plugin_resource_import_preparer, configure_site_auth_level_provider, ) +from app.runtime.managed_resources import acquire_managed_resource from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.adapters.external.server import MoviePilotServerHelper from app.runtime.log import logger +def _prepare_legacy_plugin_import(*, plugin_id: str, plugin_dir: Path) -> None: + """在执行旧插件顶层代码前准备其静态导入所需的宿主资源。""" + for capability_id in scan_plugin_resource_imports(plugin_id, plugin_dir): + acquire_managed_resource( + capability_id, + reason="legacy_plugin_import", + ) + + def _configure_plugin_services() -> None: """把兼容诊断、远程上报和站点认证等级装配到插件管理器。""" configure_plugin_legacy_import_services( diagnostics_configurator=configure_legacy_import_diagnostics, import_scanner=scan_plugin_legacy_imports, ) + configure_plugin_resource_import_preparer(_prepare_legacy_plugin_import) configure_plugin_install_reporter(MoviePilotServerHelper.install_plugin_reg) configure_site_auth_level_provider(lambda: SitesHelper().auth_level) diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 9cf6b55f3..80fc531d3 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -78,9 +78,10 @@ create additional top-level directory categories. | `app/runtime/events.py` | Event contracts, dispatch and resolver registration | | `app/runtime/log.py` | Complete console/plugin/file logging runtime and shutdown | | `app/runtime/cache.py` | Cache protocols, memory implementations, decorators and proxies | +| `app/runtime/managed_resources.py` | Provider-neutral acquisition, observation and shutdown facade for process-owned optional resources | | `app/runtime/state.py` | Process restart and update state | -| `app/runtime/extensions/` | Module, plugin and configured-service discovery/registration/lifecycle | -| `app/runtime/compat/` | Standard-library-only exact legacy import routing and DEBUG diagnostics | +| `app/runtime/extensions/` | Module, plugin, configured-service and managed-resource discovery/registration/lifecycle adapters | +| `app/runtime/compat/` | Standard-library-only exact legacy import routing, resource preflight scanning and DEBUG diagnostics | `app/startup/` remains the established composition root and is not nested under runtime. It injects providers and callbacks, orders initialization/shutdown and @@ -109,6 +110,16 @@ site extension owns the configured catalog/authentication/index capability and lives in `app/application/site/`; only its download and file installation mechanism remains in `app/adapters/system/resource.py`. +可选的进程级技术资源使用 Managed Resource 合同：实现及其 data-only +`capability.toml` 与适配器同目录，`runtime/extensions` 只解释通用的同步/异步 +`start`、`stop` 生命周期，`startup` 负责构建 Capability Runtime。声明必须使用 +`on_first_use`，普通启动只发现声明；消费者通过 `app/runtime/managed_resources.py` +显式获取资源。关闭路径先释放消费者，再关闭已初始化 Runtime，未使用的资源不得因关闭而物化。 +Runtime 关闭后不可逆；完整应用生命周期的再次启动必须由新进程承载，不能在同一解释器中重建局部资源域。 +插件需要浏览器时使用 `app.sdk.browser`，由宿主浏览器适配器协调资源，不直接依赖资源实现。 +旧插件若直接导入有资源前置条件的第三方包，compat 在插件 import 前递归扫描源码并保守准备资源； +无法精确解析的文件按全部已登记资源降级，最终可导入性仍由 Python loader 判断。 + `app/foundation/crypto.py` stays in foundation because it contains only generic RSA, digest and CryptoJS-compatible AES primitives and has no settings, policy, I/O or logging. Authentication, token, passkey, signing and two-factor policy @@ -355,8 +366,12 @@ policy. `app/db` therefore has no dependency on `app/domain`. | `app/runtime/events.py` | `EventManager`, `Event` and event resolver registration | | `app/runtime/extensions/module_manager.py` | Module discovery and lifecycle | | `app/runtime/extensions/plugin_manager.py` | Plugin discovery and lifecycle | +| `app/runtime/extensions/managed_resource_adapter.py` | Data-only managed-resource registry and sync/async lifecycle adapters | +| `app/runtime/managed_resources.py` | Lightweight acquisition, state observation and shutdown facade | | `app/foundation/reflection.py` | Generic reflection and Python module discovery | | `app/adapters/network/http.py` | Shared synchronous and asynchronous HTTP clients | +| `app/adapters/network/browser.py` | Browser launch facade and browser session implementation | +| `app/adapters/system/display/` | On-first-use virtual display resource and legacy `DisplayHelper` facade | | `app/application/rss.py` | Configured RSS retrieval and parsing | | `app/application/site/sites.*` | Generated site catalog, authentication and index capability plus its colocated data bundle | | `app/runtime/cache.py` | Cache contracts, memory backend, decorators and proxies | @@ -369,7 +384,7 @@ policy. `app/db` therefore has no dependency on `app/domain`. | `app/application/security/url.py` | URL/path validation, SSRF protection and signed image policy | | `app/application/mediaserver.py` | Configured media-server discovery and identity matching | | `app/runtime/compat/manifest.py` | Exact legacy-to-canonical import manifest | -| `app/sdk/` | Stable plugin imports | +| `app/sdk/` | Stable plugin imports, including provider-neutral browser launch functions | Run `tests/test_architecture_dependencies.py` after every ownership or import change. It rejects physical legacy or retired canonical sources, forbidden diff --git a/scripts/perf/README.md b/scripts/perf/README.md index 42597b6ac..a2b074069 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -50,6 +50,37 @@ ${PYTHON} scripts/perf/moviepilot_docker_ab.py \ 开发 harness 时可以用小数分钟做短冒烟，例如 `--points 0,0.02`。正式数据必须保持 `1,5,10,30`。 +未指定 `--scenario` 时仍使用 `idle-default`，样本目录和 Docker 资源名称与既有命令保持一致。 + +## 浏览器激活场景 + +浏览器场景使用 campaign browser seed 的独立克隆卷，不直接挂载或写入固定来源卷，也不会在样本 +阶段下载浏览器。容器保持 internal network；探针只发送信号，`app.sdk.browser` 的导入、浏览器上下文 +创建和本地 `data:` 页面校验都发生在主 MoviePilot Python 进程中。 + +非默认浏览器场景用于候选实现的 After 激活门禁；Before 不具备新 SDK，且旧实现启动时已经常驻 +Xvfb，因此不能用同一个 `0 → 0` / `0 → 1` 不变量衡量。三轮 Before/After 空载收益仍由默认 +`run` 的 `idle-default` 场景完成，浏览器场景用三个隔离的 After sample 记录冷激活成本。 + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-002-headless \ + sample --variant after --index 1 --scenario browser-headless --points 1,5,10,30 + +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-002-headed \ + sample --variant after --index 1 --scenario browser-headed --points 1,5,10,30 +``` + +- `browser-headless`：一次真实 headless context 激活，要求 Xvfb `0 → 0`； +- `browser-headed`：主进程内两个线程通过屏障并发调用 + `launch_browser_context(headless=False)`；要求两个真实 SDK 冷启动调用成功、额外上下文关闭后只保留一个、 + Capability observation 只有一个 `headed_browser_launch` generation/start，且 Xvfb `0 → 1`； +- 激活完成后再开始 `1/5/10/30m` 计时，JSON 保留激活前后 Engine 网络、working set、进程 + PSS/USS/RSS/线程、Xvfb 数量/PSS、`sys.modules` 和进程内 marker； +- 非默认场景结果保存在 `samples//-/`，可与同 campaign 的 idle 样本并存， + Markdown 中位数会按场景分组，不会混算。 + ## 完整三组 A/B ```bash diff --git a/scripts/perf/instrument/sitecustomize.py b/scripts/perf/instrument/sitecustomize.py index 086e242dc..2fabc4f58 100644 --- a/scripts/perf/instrument/sitecustomize.py +++ b/scripts/perf/instrument/sitecustomize.py @@ -1,12 +1,41 @@ -"""MoviePilot Docker A/B 测量时使用的最小 ``sys.modules`` 快照探针。""" +"""MoviePilot Docker 测量进程内使用的最小诊断探针。""" + +from __future__ import annotations import os import signal import sys +# 场景激活依赖只在收到信号后加载，避免改变 idle-default 的 import 基线。 +# pylint: disable=import-outside-toplevel _OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR") +_SCENARIO = os.environ.get("MP_PERF_SCENARIO", "idle-default") +_ACTIVATION_TIMEOUT = float(os.environ.get("MP_PERF_ACTIVATION_TIMEOUT", "120")) _snapshot_index = 0 +_activation_started = False +_browser_resources: list[object] = [] + + +def _utc_now() -> str: + """返回稳定、可机器解析的 UTC 时间。""" + from datetime import datetime, timezone + + return datetime.now(timezone.utc).isoformat() + + +def _atomic_write_json(path, payload: dict[str, object]) -> None: + """原子发布结果，避免采集端读取到半写 marker。""" + import json + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_suffix(f"{path.suffix}.tmp") + with temporary_path.open("w", encoding="utf-8") as output: + json.dump(payload, output, ensure_ascii=False, indent=2, sort_keys=True) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary_path, path) def _dump_modules(_signum, _frame) -> None: @@ -31,5 +60,318 @@ def _dump_modules(_signum, _frame) -> None: os.replace(temporary_path, final_path) +def _launch_one_browser( + *, + index: int, + headless: bool, + launcher, + start_gate, +) -> tuple[dict[str, object], list[object]]: + """启动一个本地 data URL 浏览器上下文并返回可序列化结果。""" + import time + + if start_gate is not None: + start_gate.wait(timeout=min(_ACTIVATION_TIMEOUT, 10)) + started_at = time.perf_counter() + retained: list[object] = [] + result: dict[str, object] = { + "index": index, + "headless": headless, + "started_at_monotonic": started_at, + } + try: + context = launcher(headless=headless) + retained.append(context) + page = context.new_page() + retained.append(page) + page.goto("data:text/html,MoviePilot Browser Probe") + title = page.title() + result.update( + { + "success": title == "MoviePilot Browser Probe", + "page_title": title, + "context_type": type(context).__name__, + } + ) + if not result["success"]: + result["error"] = "本地 data URL 标题校验失败" + except Exception as error: # pragma: no cover - 真实浏览器错误由 marker 保存 + result.update( + { + "success": False, + "error_type": type(error).__name__, + "error": str(error), + } + ) + result["elapsed_seconds"] = time.perf_counter() - started_at + return result, retained + + +def _enum_value(value): + """把 runtime 枚举降为 JSON 标量。""" + return getattr(value, "value", value) + + +def _read_display_runtime() -> dict[str, object]: + """读取 host.display 的只读状态和观测，不触发资源激活。""" + try: + from app.runtime.managed_resources import ( + managed_resource_observations, + managed_resource_snapshot, + ) + + snapshot = managed_resource_snapshot("host.display") + observations = managed_resource_observations("host.display") + return { + "available": True, + "snapshot": { + "capability_id": snapshot.capability_id, + "materialization": _enum_value(snapshot.materialization), + "lifecycle": _enum_value(snapshot.lifecycle), + "generation": snapshot.generation, + "visible": snapshot.visible, + "error": snapshot.error, + }, + "observations": [ + { + "capability_id": item.capability_id, + "generation": item.generation, + "operation": item.operation, + "outcome": item.outcome, + "reason": item.reason, + "materialization": _enum_value(item.materialization), + "lifecycle": _enum_value(item.lifecycle), + "duration_ms": item.duration_ms, + "error": item.error, + } + for item in observations + ], + } + except Exception as error: # pragma: no cover - 核心未就绪或真实 runtime 错误 + return { + "available": False, + "error_type": type(error).__name__, + "error": str(error), + } + + +def _close_browser_resources(resources: list[object]) -> list[dict[str, str]]: + """逆序关闭一次探针创建的页面与上下文，并返回可序列化错误。""" + errors: list[dict[str, str]] = [] + for resource in reversed(resources): + close = getattr(resource, "close", None) + if not callable(close): + continue + try: + close() + except Exception as error: # pragma: no cover - 真实浏览器错误由 marker 保存 + errors.append( + { + "resource_type": type(resource).__name__, + "error_type": type(error).__name__, + "error": str(error), + } + ) + return errors + + +def _activate_browser_scenario( + scenario: str, + launcher=None, +) -> dict[str, object]: + """通过公开 SDK 执行真实浏览器激活，headed 使用并发冷启动探针。""" + import threading + import time + + if scenario not in {"browser-headless", "browser-headed"}: + raise ValueError(f"场景不支持浏览器激活：{scenario}") + if launcher is None: + from app.sdk.browser import launch_browser_context + + launcher = launch_browser_context + + display_before = _read_display_runtime() + headless = scenario == "browser-headless" + concurrency = 1 if headless else 2 + launch_results: list[dict[str, object] | None] = [None] * concurrency + cleanup_error_slots: list[list[dict[str, str]]] = [ + [] for _index in range(concurrency) + ] + retained_slots = [False] * concurrency + start_gate = None if headless else threading.Barrier(concurrency) + completion_gate = None if headless else threading.Barrier(concurrency) + + def launch(index: int) -> None: + result, resources = _launch_one_browser( + index=index, + headless=headless, + launcher=launcher, + start_gate=start_gate, + ) + launch_results[index] = result + if headless: + if result.get("success"): + _browser_resources.extend(resources) + result["retained"] = True + retained_slots[index] = True + else: + cleanup_error_slots[index] = _close_browser_resources(resources) + return + + try: + completion_gate.wait(timeout=min(_ACTIVATION_TIMEOUT, 30)) + except threading.BrokenBarrierError: + cleanup_error_slots[index] = _close_browser_resources(resources) + result.update( + { + "success": False, + "error_type": "BrokenBarrierError", + "error": "并发浏览器启动未能完成同线程清理协调", + } + ) + return + + successful_indices = [ + candidate_index + for candidate_index, item in enumerate(launch_results) + if item is not None and bool(item.get("success")) + ] + retained_index = min(successful_indices) if successful_indices else None + if index == retained_index: + # 保留对象不再跨线程使用；容器退出会回收浏览器及其 worker 进程。 + _browser_resources.extend(resources) + result["retained"] = True + retained_slots[index] = True + else: + # Playwright sync/greenlet 对象必须在创建它的线程内关闭。 + cleanup_error_slots[index] = _close_browser_resources(resources) + + if headless: + launch(0) + else: + threads = [ + threading.Thread( + target=launch, + args=(index,), + name=f"mp-perf-browser-launch-{index}", + daemon=True, + ) + for index in range(concurrency) + ] + for thread in threads: + thread.start() + deadline = time.monotonic() + _ACTIVATION_TIMEOUT + for thread in threads: + thread.join(timeout=max(deadline - time.monotonic(), 0)) + + serialized_launches = [ + item + if item is not None + else { + "index": index, + "success": False, + "error_type": "TimeoutError", + "error": "浏览器启动未在进程内超时前完成", + } + for index, item in enumerate(launch_results) + ] + launch_starts = [ + float(item["started_at_monotonic"]) + for item in serialized_launches + if "started_at_monotonic" in item + ] + successful_indices = [ + index + for index, item in enumerate(serialized_launches) + if bool(item.get("success")) + ] + cleanup_errors = [ + error for slot_errors in cleanup_error_slots for error in slot_errors + ] + retained_count = sum(retained_slots) + + expected_successes = concurrency + browser_success = ( + len(successful_indices) == expected_successes and not cleanup_errors + ) + return { + "requested": True, + "headless": headless, + "concurrency": concurrency, + "successes": len(successful_indices), + "retained_contexts": retained_count, + "launches": serialized_launches, + "cleanup_errors": cleanup_errors, + "success": browser_success, + "managed_resource": { + "before": display_before, + "after": _read_display_runtime(), + }, + "single_flight_probe": { + "requested": not headless, + "concurrent_callers": concurrency if not headless else 0, + "successful_callers": len(successful_indices) if not headless else 0, + "barrier_used": not headless, + "launch_start_spread_ms": ( + (max(launch_starts) - min(launch_starts)) * 1000 + if launch_starts + else None + ), + "all_callers_succeeded": len(successful_indices) == expected_successes, + "calls": serialized_launches if not headless else [], + }, + } + + +def _run_activation() -> None: + """在目标解释器的工作线程中运行激活并发布完成 marker。""" + if not _OUTPUT_DIR: + return + import time + from pathlib import Path + + started_at = time.perf_counter() + result: dict[str, object] = { + "schema_version": 1, + "scenario": _SCENARIO, + "pid": os.getpid(), + "started_at": _utc_now(), + } + try: + result["browser"] = _activate_browser_scenario(_SCENARIO) + result["success"] = bool(result["browser"]["success"]) + except Exception as error: # pragma: no cover - 真实集成错误由 marker 保存 + result.update( + { + "success": False, + "error_type": type(error).__name__, + "error": str(error), + } + ) + result["elapsed_seconds"] = time.perf_counter() - started_at + result["completed_at"] = _utc_now() + _atomic_write_json( + Path(_OUTPUT_DIR) / f"activation-{os.getpid()}.json", + result, + ) + + +def _request_activation(_signum, _frame) -> None: + """SIGUSR2 只调度一次工作线程，真实 import 与启动仍在目标进程内完成。""" + global _activation_started + if not _OUTPUT_DIR or _activation_started: + return + import threading + + _activation_started = True + threading.Thread( + target=_run_activation, + name="mp-perf-scenario-activation", + daemon=True, + ).start() + + if _OUTPUT_DIR and hasattr(signal, "SIGUSR1"): signal.signal(signal.SIGUSR1, _dump_modules) +if _OUTPUT_DIR and hasattr(signal, "SIGUSR2"): + signal.signal(signal.SIGUSR2, _request_activation) diff --git a/scripts/perf/moviepilot_docker_ab.py b/scripts/perf/moviepilot_docker_ab.py index 797085cda..5a2a97829 100644 --- a/scripts/perf/moviepilot_docker_ab.py +++ b/scripts/perf/moviepilot_docker_ab.py @@ -31,6 +31,8 @@ DEFAULT_SUBSTRATE = ( "sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720" ) DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed" +DEFAULT_SCENARIO = "idle-default" +SCENARIOS = (DEFAULT_SCENARIO, "browser-headless", "browser-headed") CAMPAIGN_LABEL = "org.moviepilot.perf.campaign" ROLE_LABEL = "org.moviepilot.perf.role" SOURCE_LABEL = "org.moviepilot.perf.source-commit" @@ -587,6 +589,10 @@ def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, s { "PYTHONPATH": "/opt/moviepilot-perf/instrument", "MP_PERF_OUTPUT_DIR": "/opt/moviepilot-perf/out/modules", + "MP_PERF_SCENARIO": getattr(args, "scenario", DEFAULT_SCENARIO), + "MP_PERF_ACTIVATION_TIMEOUT": str( + getattr(args, "activation_timeout", 180) + ), } ) return environment @@ -1103,7 +1109,9 @@ def sample_volume_names( index: int, ) -> tuple[str, str]: """返回单个样本的隔离配置和浏览器卷名称。""" - prefix = f"{resource_prefix(args)}-{variant}-{index}" + scenario = getattr(args, "scenario", DEFAULT_SCENARIO) + scenario_segment = "" if scenario == DEFAULT_SCENARIO else f"-{scenario}" + prefix = f"{resource_prefix(args)}{scenario_segment}-{variant}-{index}" return f"{prefix}-config", f"{prefix}-browser" @@ -1113,11 +1121,190 @@ def sample_result_directory( index: int, ) -> Path: """返回单个样本的原始结果目录。""" - return campaign_directory(args) / "samples" / f"{variant}-{index}" + scenario = getattr(args, "scenario", DEFAULT_SCENARIO) + sample_root = campaign_directory(args) / "samples" + if scenario == DEFAULT_SCENARIO: + return sample_root / f"{variant}-{index}" + return sample_root / scenario / f"{variant}-{index}" + + +def capture_activation_snapshot( + container, + output_dir: Path, + phase: str, +) -> dict[str, Any]: + """采集浏览器激活边界的 Engine、进程和进程内 import 状态。""" + engine = capture_engine_stats(container) + processes = capture_processes(container) + modules = capture_modules(container, output_dir, processes["main_python"]) + return { + "phase": phase, + "captured_at": utc_now(), + "engine": engine, + "processes": processes, + "modules": modules, + } + + +def evaluate_browser_activation( + scenario: str, + pre: dict[str, Any], + post: dict[str, Any], + marker: dict[str, Any], + expected_pid: Optional[int] = None, +) -> dict[str, Any]: + """按场景不变量判断浏览器与 display 的真实激活是否有效。""" + pre_xvfb = pre["processes"]["xvfb"] + post_xvfb = post["processes"]["xvfb"] + browser = marker.get("browser") or {} + managed_resource = browser.get("managed_resource") or {} + managed_before = managed_resource.get("before") or {} + managed_after = managed_resource.get("after") or {} + before_observations = managed_before.get("observations") or [] + after_observations = managed_after.get("observations") or [] + observation_prefix_matches = ( + after_observations[: len(before_observations)] == before_observations + ) + new_observations = ( + after_observations[len(before_observations) :] + if observation_prefix_matches + else after_observations + ) + display_starts = [ + item + for item in new_observations + if item.get("operation") == "activate" and item.get("outcome") == "started" + ] + display_successes = [ + item + for item in new_observations + if item.get("operation") == "activate" and item.get("outcome") == "succeeded" + ] + display_start_reasons = [item.get("reason") for item in display_starts] + before_generation = (managed_before.get("snapshot") or {}).get("generation") + after_generation = (managed_after.get("snapshot") or {}).get("generation") + errors: list[str] = [] + if marker.get("scenario") != scenario: + errors.append("进程内 marker 的场景与采集请求不一致") + if expected_pid is not None and marker.get("pid") != expected_pid: + errors.append("进程内 marker 不是目标 MoviePilot Python 进程写出") + if not marker.get("success") or not browser.get("success"): + errors.append("主 MoviePilot Python 进程未完成浏览器激活") + if browser.get("retained_contexts") != 1: + errors.append("激活后必须保留一个浏览器上下文供 post activation 采样") + if not managed_before.get("available") or not managed_after.get("available"): + errors.append("主进程未提供 host.display managed resource 观测") + process_single_flight = browser.get("single_flight_probe") or {} + + single_flight = { + "requested": scenario == "browser-headed", + "concurrent_callers": int(process_single_flight.get("concurrent_callers") or 0), + "successful_callers": int(process_single_flight.get("successful_callers") or 0), + "xvfb_process_delta": int(post_xvfb["count"]) - int(pre_xvfb["count"]), + "generation_before": before_generation, + "generation_after": after_generation, + "activation_start_count": len(display_starts), + "activation_success_count": len(display_successes), + "activation_start_reasons": display_start_reasons, + "observation_prefix_matches": observation_prefix_matches, + "passed": None, + } + if scenario == "browser-headless": + if pre_xvfb["count"] != 0 or post_xvfb["count"] != 0: + errors.append("headless 激活前后都不得存在 Xvfb") + if display_starts or before_generation != after_generation: + errors.append("headless 激活不得申请 host.display") + elif scenario == "browser-headed": + if pre_xvfb["count"] != 0: + errors.append("headed 冷激活前必须没有 Xvfb") + if post_xvfb["count"] != 1: + errors.append("headed 并发激活后必须恰好存在一个 Xvfb") + single_flight["passed"] = ( + single_flight["concurrent_callers"] == 2 + and single_flight["successful_callers"] == 2 + and single_flight["xvfb_process_delta"] == 1 + and single_flight["activation_start_count"] == 1 + and single_flight["activation_success_count"] == 1 + and single_flight["activation_start_reasons"] == ["headed_browser_launch"] + and before_generation is not None + and after_generation == before_generation + 1 + ) + if not single_flight["passed"]: + errors.append("headed 并发请求未证明 display single-flight") + else: + errors.append(f"未知浏览器场景：{scenario}") + + return { + "passed": not errors, + "errors": errors, + "expected": ("Xvfb 0→0" if scenario == "browser-headless" else "Xvfb 0→1"), + "observed": { + "pre_xvfb_count": pre_xvfb["count"], + "pre_xvfb_pss_kib": pre_xvfb["pss_kib"], + "post_xvfb_count": post_xvfb["count"], + "post_xvfb_pss_kib": post_xvfb["pss_kib"], + }, + "single_flight": single_flight, + } + + +def activate_browser_scenario( + container, + output_dir: Path, + scenario: str, + timeout: float, +) -> dict[str, Any]: + """通过 SIGUSR2 让目标 MoviePilot 解释器执行场景激活并回收 marker。""" + pre = capture_activation_snapshot(container, output_dir, "pre-activation") + main_python = pre["processes"]["main_python"] + if not main_python: + raise HarnessError("未找到主 Python 进程，无法触发浏览器场景") + marker_path = output_dir / "modules" / f"activation-{main_python['pid']}.json" + marker_path.unlink(missing_ok=True) + + requested_at = time.monotonic() + result = container.exec_run(["kill", "-USR2", str(main_python["pid"])]) + if result.exit_code != 0: + raise HarnessError("向主 Python 进程发送场景激活信号失败") + deadline = requested_at + timeout + while time.monotonic() < deadline: + if marker_path.exists(): + break + if not container_running(container): + raise HarnessError("等待场景激活 marker 时容器提前退出") + time.sleep(0.05) + if not marker_path.exists(): + raise HarnessError(f"浏览器场景激活在 {timeout:.0f}s 内未完成") + + marker_received_at = time.monotonic() + marker = json.loads(marker_path.read_text(encoding="utf-8")) + post = capture_activation_snapshot(container, output_dir, "post-activation") + validation = evaluate_browser_activation( + scenario, + pre, + post, + marker, + expected_pid=main_python["pid"], + ) + return { + "scenario": scenario, + "trigger": "SIGUSR2-to-main-python", + "main_python_pid": main_python["pid"], + "orchestrator_elapsed_seconds": marker_received_at - requested_at, + "post_capture_elapsed_seconds": time.monotonic() - marker_received_at, + "worker_elapsed_seconds": marker.get("elapsed_seconds"), + "pre": pre, + "post": post, + "marker": marker, + "validation": validation, + } def command_sample(args: argparse.Namespace) -> dict[str, Any]: """执行一个隔离样本并在约定时间点采集完整指标。""" + scenario = getattr(args, "scenario", DEFAULT_SCENARIO) + if scenario != DEFAULT_SCENARIO and args.variant != "after": + raise HarnessError("浏览器激活场景只用于验证包含 app.sdk.browser 的 After 候选") client = require_docker_client() build = load_build_manifest(args) config_seed, browser_seed = require_seed_volumes(client, args) @@ -1144,13 +1331,21 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: clone_volume(client, image, browser_seed, browser_volume.name) browser_before = volume_fingerprint(client, image, browser_volume.name) network = ensure_internal_network(client, args) - container_name = f"{resource_prefix(args)}-{args.variant}-{args.index}" + scenario_segment = "" if scenario == DEFAULT_SCENARIO else f"-{scenario}" + container_name = ( + f"{resource_prefix(args)}{scenario_segment}-{args.variant}-{args.index}" + ) + role = ( + f"sample-{args.variant}-{args.index}" + if scenario == DEFAULT_SCENARIO + else f"sample-{scenario}-{args.variant}-{args.index}" + ) container = create_app_container( client, args, image=image, name=container_name, - role=f"sample-{args.variant}-{args.index}", + role=role, config_volume=config_volume.name, browser_volume=browser_volume.name, network_name=network.name, @@ -1161,6 +1356,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: "campaign": args.campaign, "variant": args.variant, "sample_index": args.index, + "scenario": scenario, "source_commit": build[f"{args.variant}_commit"], "image": image, "started_at": utc_now(), @@ -1171,6 +1367,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: "network": "internal", "database": "sqlite-seed-clone", "browser": "prewarmed-seed-clone", + "scenario": scenario, }, "browser_before": browser_before, "measurements": [], @@ -1191,9 +1388,27 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: result["http_ready_seconds"] = ready_seconds result["settled_seconds"] = settled_at - started_at result["settled_wait_seconds_after_ready"] = settled_at - ready_at + measurement_origin_at = settled_at + + if scenario != DEFAULT_SCENARIO: + activation = activate_browser_scenario( + container, + output_dir, + scenario, + args.activation_timeout, + ) + result["activation"] = activation + atomic_write_json(output_dir / "result.partial.json", result) + if not activation["validation"]["passed"]: + details = "; ".join(activation["validation"]["errors"]) + raise HarnessError(f"{scenario} 场景激活不满足验收条件：{details}") + measurement_origin_at = time.monotonic() + result["measurement_origin"] = "post-activation" + else: + result["measurement_origin"] = "settled" for point in args.points: - deadline = settled_at + point * 60 + deadline = measurement_origin_at + point * 60 remaining = deadline - time.monotonic() if remaining > 0: time.sleep(remaining) @@ -1201,7 +1416,12 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: raise HarnessError(f"容器在 {point:g}m 采样前退出") print(f"[{args.variant}-{args.index}] sampling {point:g}m") result["measurements"].append( - capture_measurement(container, output_dir, point, settled_at) + capture_measurement( + container, + output_dir, + point, + measurement_origin_at, + ) ) atomic_write_json(output_dir / "result.partial.json", result) assert_no_app_env(container) @@ -1236,7 +1456,7 @@ def load_sample_results(args: argparse.Namespace) -> list[dict[str, Any]]: results = [] if not sample_root.exists(): return results - for path in sorted(sample_root.glob("*/result.json")): + for path in sorted(sample_root.rglob("result.json")): results.append(json.loads(path.read_text(encoding="utf-8"))) return results @@ -1282,6 +1502,10 @@ def build_markdown_report( samples: list[dict[str, Any]], ) -> str: """生成不含本机路径和凭据的 Markdown 汇总。""" + scenarios = sorted( + {sample.get("scenario", DEFAULT_SCENARIO) for sample in samples} + ) or [DEFAULT_SCENARIO] + show_scenario = any(scenario != DEFAULT_SCENARIO for scenario in scenarios) points = sorted( { float(measurement["target_minute"]) @@ -1314,7 +1538,8 @@ def build_markdown_report( ) headers = ( - ["版本", "样本", "HTTP ready(s)"] + (["场景"] if show_scenario else []) + + ["版本", "样本", "HTTP ready(s)"] + [f"{point:g}m WS(MiB)" for point in points] + [ "末次 Python PSS(MiB)", @@ -1333,11 +1558,12 @@ def build_markdown_report( for sample in sorted( samples, key=lambda item: ( + item.get("scenario", DEFAULT_SCENARIO), variant_order.get(item["variant"], 99), item["sample_index"], ), ): - row = [ + row = ([sample.get("scenario", DEFAULT_SCENARIO)] if show_scenario else []) + [ sample["variant"], str(sample["sample_index"]), f"{sample.get('http_ready_seconds', 0):.2f}" @@ -1388,53 +1614,149 @@ def build_markdown_report( ) lines.append("| " + " | ".join(row) + " |") + activated_samples = [sample for sample in samples if sample.get("activation")] + if activated_samples: + activation_headers = [ + "场景", + "版本", + "样本", + "激活(s)", + "Pre WS(MiB)", + "Post WS(MiB)", + "Pre Python PSS(MiB)", + "Post Python PSS(MiB)", + "Pre Xvfb", + "Post Xvfb", + "Post Xvfb PSS(MiB)", + "Activation RX Δ(KiB)", + "Activation TX Δ(KiB)", + "Browser", + "Single-flight generation/start", + "验收", + ] + lines.extend( + [ + "", + "## 场景激活", + "", + "| " + " | ".join(activation_headers) + " |", + "| " + " | ".join(["---"] * len(activation_headers)) + " |", + ] + ) + for sample in sorted( + activated_samples, + key=lambda item: ( + item.get("scenario", DEFAULT_SCENARIO), + variant_order.get(item["variant"], 99), + item["sample_index"], + ), + ): + activation = sample["activation"] + pre = activation["pre"] + post = activation["post"] + marker = activation["marker"] + validation = activation["validation"] + pre_python = pre["processes"].get("main_python") or {} + post_python = post["processes"].get("main_python") or {} + single_flight = validation["single_flight"] + activation_row = [ + sample.get("scenario", DEFAULT_SCENARIO), + sample["variant"], + str(sample["sample_index"]), + f"{float(activation.get('worker_elapsed_seconds') or 0):.2f}", + format_mib(pre["engine"]["working_set_bytes"]), + format_mib(post["engine"]["working_set_bytes"]), + format_kib_as_mib(pre_python.get("pss_kib")), + format_kib_as_mib(post_python.get("pss_kib")), + str(pre["processes"]["xvfb"]["count"]), + str(post["processes"]["xvfb"]["count"]), + format_kib_as_mib(post["processes"]["xvfb"]["pss_kib"]), + format_bytes_as_kib( + post["engine"]["network_rx_bytes"] + - pre["engine"]["network_rx_bytes"] + ), + format_bytes_as_kib( + post["engine"]["network_tx_bytes"] + - pre["engine"]["network_tx_bytes"] + ), + "成功" if marker.get("success") else "失败", + ( + f"{single_flight.get('generation_after')}/" + f"{single_flight.get('activation_start_count')}" + if single_flight.get("passed") is True + else "不适用" + if single_flight.get("passed") is None + else "失败" + ), + "通过" if validation["passed"] else "失败", + ] + lines.append("| " + " | ".join(activation_row) + " |") + lines.extend(["", "## 中位数对照", ""]) - if points: - lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |") - lines.append("| --- | ---: | ---: | ---: | ---: |") - for point in points: - before_values = [ - measurement_at(sample, point)["engine"]["working_set_bytes"] - for sample in samples - if sample["variant"] == "before" and measurement_at(sample, point) - ] - after_values = [ - measurement_at(sample, point)["engine"]["working_set_bytes"] - for sample in samples - if sample["variant"] == "after" and measurement_at(sample, point) - ] - before_median = median(before_values) - after_median = median(after_values) - if before_median is None or after_median is None: - lines.append(f"| {point:g}m | — | — | — | — |") - continue - delta = after_median - before_median - percent = delta / before_median * 100 if before_median else 0 - lines.append( - f"| {point:g}m | {format_mib(before_median)} | " - f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | {percent:.1f}% |" + for scenario in scenarios: + scenario_samples = [ + sample + for sample in samples + if sample.get("scenario", DEFAULT_SCENARIO) == scenario + ] + if show_scenario: + lines.extend([f"### `{scenario}`", ""]) + if points: + lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |") + lines.append("| --- | ---: | ---: | ---: | ---: |") + for point in points: + before_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in scenario_samples + if sample["variant"] == "before" and measurement_at(sample, point) + ] + after_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in scenario_samples + if sample["variant"] == "after" and measurement_at(sample, point) + ] + before_median = median(before_values) + after_median = median(after_values) + if before_median is None or after_median is None: + lines.append(f"| {point:g}m | — | — | — | — |") + continue + delta = after_median - before_median + percent = delta / before_median * 100 if before_median else 0 + lines.append( + f"| {point:g}m | {format_mib(before_median)} | " + f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | " + f"{percent:.1f}% |" + ) + lines.append("") + + lines.extend(["## 启动时间", ""]) + for scenario in scenarios: + scenario_samples = [ + sample + for sample in samples + if sample.get("scenario", DEFAULT_SCENARIO) == scenario + ] + ready_before = median( + sample["http_ready_seconds"] + for sample in scenario_samples + if sample["variant"] == "before" and "http_ready_seconds" in sample + ) + ready_after = median( + sample["http_ready_seconds"] + for sample in scenario_samples + if sample["variant"] == "after" and "http_ready_seconds" in sample + ) + scenario_prefix = f"`{scenario}`：" if show_scenario else "" + if ready_before is not None and ready_after is not None: + startup_change = ( + (ready_after - ready_before) / ready_before * 100 if ready_before else 0 ) - ready_before = median( - sample["http_ready_seconds"] - for sample in samples - if sample["variant"] == "before" and "http_ready_seconds" in sample - ) - ready_after = median( - sample["http_ready_seconds"] - for sample in samples - if sample["variant"] == "after" and "http_ready_seconds" in sample - ) - lines.extend(["", "## 启动时间", ""]) - if ready_before is not None and ready_after is not None: - startup_change = ( - (ready_after - ready_before) / ready_before * 100 if ready_before else 0 - ) - lines.append( - f"Before 中位数 {ready_before:.2f}s，After 中位数 {ready_after:.2f}s，" - f"变化 {startup_change:.1f}%。" - ) - else: - lines.append("样本尚不完整。") + lines.append( + f"{scenario_prefix}Before 中位数 {ready_before:.2f}s，" + f"After 中位数 {ready_after:.2f}s，变化 {startup_change:.1f}%。" + ) + else: + lines.append(f"{scenario_prefix}样本尚不完整。") lines.extend( [ "", @@ -1605,6 +1927,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--memory", default="2g") parser.add_argument("--ready-timeout", type=int, default=300) parser.add_argument("--settle-timeout", type=int, default=300) + parser.add_argument( + "--activation-timeout", + type=int, + default=180, + help="进程内场景激活完成 marker 的等待秒数", + ) parser.add_argument("--stop-timeout", type=int, default=120) subparsers = parser.add_subparsers(dest="command", required=True) @@ -1624,6 +1952,12 @@ def build_parser() -> argparse.ArgumentParser: sample.add_argument( "--points", type=parse_points, default=parse_points("1,5,10,30") ) + sample.add_argument( + "--scenario", + choices=SCENARIOS, + default=DEFAULT_SCENARIO, + help="样本场景；默认保持 PERF-001 idle-default 行为", + ) sample.add_argument("--replace", action="store_true") run = subparsers.add_parser("run", help="完整执行 build、seed 和三组平衡 A/B") @@ -1652,7 +1986,12 @@ def main(argv: Optional[list[str]] = None) -> int: args.browser_source_volume = DEFAULT_BROWSER_SOURCE_VOLUME if args.cpus <= 0: parser.error("--cpus 必须大于 0") - if args.ready_timeout <= 0 or args.settle_timeout <= 0 or args.stop_timeout <= 0: + if ( + args.ready_timeout <= 0 + or args.settle_timeout <= 0 + or args.activation_timeout <= 0 + or args.stop_timeout <= 0 + ): parser.error("timeout 必须大于 0") try: if args.command == "build": diff --git a/scripts/perf/test_scenarios.py b/scripts/perf/test_scenarios.py new file mode 100644 index 000000000..013952e2f --- /dev/null +++ b/scripts/perf/test_scenarios.py @@ -0,0 +1,430 @@ +"""PERF Docker harness 场景协议的无 Docker fake 测试。""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import os +import sys +import threading +import time +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + + +PERF_DIR = Path(__file__).resolve().parent + + +def load_module(name: str, path: Path): + """从脚本路径加载模块，避免要求 scripts 变成运行时 Python package。""" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def snapshot(xvfb_count: int, xvfb_pss_kib: int = 0) -> dict: + """构造只包含验收字段的进程快照。""" + return { + "processes": { + "xvfb": {"count": xvfb_count, "pss_kib": xvfb_pss_kib}, + } + } + + +def managed_resource(before_generation: int, after_generation: int) -> dict: + """构造 host.display single-flight 观测。""" + observations = [] + if after_generation > before_generation: + observations = [ + { + "operation": "activate", + "outcome": "started", + "generation": after_generation, + "reason": "headed_browser_launch", + }, + { + "operation": "activate", + "outcome": "succeeded", + "generation": after_generation, + }, + ] + return { + "before": { + "available": True, + "snapshot": {"generation": before_generation}, + "observations": [], + }, + "after": { + "available": True, + "snapshot": {"generation": after_generation}, + "observations": observations, + }, + } + + +class FakePage: + """验证本地 data URL 的同步页面替身。""" + + def __init__(self) -> None: + self.url = "" + + def goto(self, url: str) -> None: + self.url = url + + def title(self) -> str: + assert self.url.startswith("data:text/html,") + return "MoviePilot Browser Probe" + + def close(self) -> None: + """模拟 Playwright page 对称关闭。""" + + +class FakeContext: + """浏览器上下文替身。""" + + def new_page(self) -> FakePage: + return FakePage() + + def close(self) -> None: + """模拟 CloakBrowser context 对称关闭。""" + + +def test_default_cli_and_paths_keep_idle_contract(tmp_path: Path) -> None: + """未指定 scenario 时保持既有 idle 命令和资源路径。""" + harness = load_module("moviepilot_perf_cli", PERF_DIR / "moviepilot_docker_ab.py") + args = harness.build_parser().parse_args( + [ + "--campaign", + "fake", + "--output-dir", + str(tmp_path), + "sample", + "--variant", + "after", + "--index", + "1", + ] + ) + + assert args.scenario == "idle-default" + assert harness.sample_volume_names(args, "after", 1) == ( + "mpperf-fake-after-1-config", + "mpperf-fake-after-1-browser", + ) + assert harness.sample_result_directory(args, "after", 1) == ( + tmp_path / "fake" / "samples" / "after-1" + ) + + +def test_browser_scenario_uses_isolated_resource_and_result_names( + tmp_path: Path, +) -> None: + """不同激活场景不会覆盖 idle 样本或彼此复用可写卷。""" + harness = load_module("moviepilot_perf_paths", PERF_DIR / "moviepilot_docker_ab.py") + args = argparse.Namespace( + campaign="fake", + output_dir=tmp_path, + scenario="browser-headed", + ) + + assert harness.sample_volume_names(args, "after", 2) == ( + "mpperf-fake-browser-headed-after-2-config", + "mpperf-fake-browser-headed-after-2-browser", + ) + assert harness.sample_result_directory(args, "after", 2) == ( + tmp_path / "fake" / "samples" / "browser-headed" / "after-2" + ) + + +def test_browser_scenario_rejects_before_without_touching_docker() -> None: + """旧基线不具备 SDK/display 冷启动不变量，非默认场景只接受 After。""" + harness = load_module( + "moviepilot_perf_after_only", PERF_DIR / "moviepilot_docker_ab.py" + ) + args = argparse.Namespace(scenario="browser-headless", variant="before") + + with pytest.raises(harness.HarnessError, match="After"): + harness.command_sample(args) + + +def test_activation_validation_enforces_headless_and_headed_invariants() -> None: + """headless 保持无 Xvfb，headed 两调用只能产生一个 Xvfb。""" + harness = load_module( + "moviepilot_perf_validation", + PERF_DIR / "moviepilot_docker_ab.py", + ) + headless_marker = { + "scenario": "browser-headless", + "pid": 42, + "success": True, + "browser": { + "success": True, + "concurrency": 1, + "successes": 1, + "retained_contexts": 1, + "managed_resource": managed_resource(0, 0), + "single_flight_probe": { + "concurrent_callers": 0, + "successful_callers": 0, + }, + }, + } + headed_marker = { + "scenario": "browser-headed", + "pid": 42, + "success": True, + "browser": { + "success": True, + "concurrency": 2, + "successes": 2, + "retained_contexts": 1, + "managed_resource": managed_resource(0, 1), + "single_flight_probe": { + "concurrent_callers": 2, + "successful_callers": 2, + }, + }, + } + + headless = harness.evaluate_browser_activation( + "browser-headless", + snapshot(0), + snapshot(0), + headless_marker, + expected_pid=42, + ) + headed = harness.evaluate_browser_activation( + "browser-headed", + snapshot(0), + snapshot(1, 72 * 1024), + headed_marker, + expected_pid=42, + ) + invalid = harness.evaluate_browser_activation( + "browser-headed", + snapshot(0), + snapshot(2, 144 * 1024), + headed_marker, + expected_pid=42, + ) + + assert headless["passed"] is True + assert headed["passed"] is True + assert headed["single_flight"]["passed"] is True + assert headed["single_flight"]["generation_after"] == 1 + assert headed["single_flight"]["activation_start_count"] == 1 + assert invalid["passed"] is False + assert invalid["single_flight"]["passed"] is False + + +def test_sitecustomize_acquires_headed_display_concurrently_in_same_process() -> None: + """headed probe 并发走公开 SDK 冷启动，并只保留一个上下文。""" + probe = load_module( + "moviepilot_perf_sitecustomize", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + browser_calls: list[tuple[int, bool]] = [] + closed_contexts: list[tuple[int, int]] = [] + lock = threading.Lock() + + class TrackedContext(FakeContext): + """记录并发探针关闭的额外浏览器上下文。""" + + def __init__(self, index: int) -> None: + self.index = index + + def close(self) -> None: + closed_contexts.append((self.index, threading.get_ident())) + + def launcher(*, headless: bool) -> FakeContext: + with lock: + browser_calls.append((threading.get_ident(), headless)) + index = len(browser_calls) - 1 + return TrackedContext(index) + + result = probe._activate_browser_scenario( + "browser-headed", + launcher=launcher, + ) + + assert result["success"] is True + assert result["successes"] == 2 + assert result["retained_contexts"] == 1 + assert len(browser_calls) == 2 + assert len({thread_id for thread_id, _headless in browser_calls}) == 2 + assert all(headless is False for _thread_id, headless in browser_calls) + assert len(closed_contexts) == 1 + closed_index, closed_thread_id = closed_contexts[0] + assert closed_thread_id == browser_calls[closed_index][0] + assert result["single_flight_probe"]["barrier_used"] is True + + +def test_sitecustomize_headless_uses_one_headless_context() -> None: + """headless probe 只启动一个无显示上下文。""" + probe = load_module( + "moviepilot_perf_sitecustomize_headless", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + calls: list[bool] = [] + + def launcher(*, headless: bool) -> FakeContext: + calls.append(headless) + return FakeContext() + + result = probe._activate_browser_scenario("browser-headless", launcher=launcher) + + assert result["success"] is True + assert calls == [True] + assert result["single_flight_probe"]["requested"] is False + + +def test_sitecustomize_serializes_managed_resource_facade(monkeypatch) -> None: + """进程探针按公开只读 facade 记录 generation 与 activate observation。""" + observation = SimpleNamespace( + capability_id="host.display", + generation=1, + operation="activate", + outcome="started", + reason="fake", + materialization="materialized", + lifecycle="starting", + duration_ms=0.5, + error=None, + ) + runtime_snapshot = SimpleNamespace( + capability_id="host.display", + materialization="materialized", + lifecycle="running", + generation=1, + visible=True, + error=None, + ) + + facade = ModuleType("app.runtime.managed_resources") + + def managed_resource_snapshot(capability_id: str): + assert capability_id == "host.display" + return runtime_snapshot + + def managed_resource_observations(capability_id=None): + assert capability_id == "host.display" + return (observation,) + + facade.managed_resource_snapshot = managed_resource_snapshot + facade.managed_resource_observations = managed_resource_observations + monkeypatch.setitem(sys.modules, "app.runtime.managed_resources", facade) + probe = load_module( + "moviepilot_perf_sitecustomize_observation", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + + result = probe._read_display_runtime() + + assert result["available"] is True + assert result["snapshot"]["generation"] == 1 + assert result["observations"][0]["operation"] == "activate" + assert result["observations"][0]["outcome"] == "started" + + +def test_sitecustomize_signal_worker_publishes_atomic_marker(tmp_path: Path) -> None: + """信号回调只调度目标进程工作线程，并发布带 PID 的完成 marker。""" + probe = load_module( + "moviepilot_perf_sitecustomize_marker", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + probe._OUTPUT_DIR = str(tmp_path) + probe._SCENARIO = "browser-headless" + probe._activation_started = False + probe._activate_browser_scenario = lambda scenario: { + "success": scenario == "browser-headless" + } + + probe._request_activation(None, None) + marker_path = tmp_path / f"activation-{os.getpid()}.json" + deadline = time.monotonic() + 2 + while time.monotonic() < deadline and not marker_path.exists(): + time.sleep(0.01) + + payload = json.loads(marker_path.read_text(encoding="utf-8")) + assert payload["pid"] == os.getpid() + assert payload["scenario"] == "browser-headless" + assert payload["success"] is True + assert not list(tmp_path.glob("*.tmp")) + + +def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> None: + """非默认场景报告包含激活证据，并按场景隔离中位数。""" + harness = load_module( + "moviepilot_perf_report", PERF_DIR / "moviepilot_docker_ab.py" + ) + process_data = { + "main_python": {"pss_kib": 400 * 1024, "uss_kib": 390 * 1024, "threads": 8}, + "xvfb": {"count": 0, "pss_kib": 0}, + } + post_process_data = { + "main_python": {"pss_kib": 410 * 1024, "uss_kib": 400 * 1024, "threads": 10}, + "xvfb": {"count": 1, "pss_kib": 72 * 1024}, + } + activation = { + "worker_elapsed_seconds": 1.25, + "pre": { + "engine": { + "working_set_bytes": 500 * 1024 * 1024, + "network_rx_bytes": 1024, + "network_tx_bytes": 512, + }, + "processes": process_data, + }, + "post": { + "engine": { + "working_set_bytes": 600 * 1024 * 1024, + "network_rx_bytes": 3072, + "network_tx_bytes": 1536, + }, + "processes": post_process_data, + }, + "marker": {"success": True}, + "validation": { + "passed": True, + "single_flight": {"passed": True}, + }, + } + sample = { + "scenario": "browser-headed", + "variant": "after", + "sample_index": 1, + "http_ready_seconds": 7.0, + "activation": activation, + "measurements": [ + { + "target_minute": 1.0, + "engine": { + "working_set_bytes": 610 * 1024 * 1024, + "network_rx_bytes": 1024, + "network_tx_bytes": 512, + }, + "processes": post_process_data, + "modules": {"count": 3000}, + } + ], + } + build = { + "campaign": "fake", + "platform": "linux/arm64", + "before_commit": "before", + "after_commit": "after", + "substrate": {"reference": "frozen"}, + } + + report = harness.build_markdown_report(build, None, [sample]) + + assert "## 场景激活" in report + assert "browser-headed" in report + assert "Single-flight" in report + assert "### `browser-headed`" in report + assert "1.25" in report diff --git a/tests/test_agent_lifecycle.py b/tests/test_agent_lifecycle.py index 90f56b064..a14dce458 100644 --- a/tests/test_agent_lifecycle.py +++ b/tests/test_agent_lifecycle.py @@ -108,13 +108,13 @@ async def test_agent_initialization_failure_does_not_stop_module_startup( monkeypatch.setattr(modules_initializer, "init_agent", agent_initializer.init_agent) for name in ( - "DisplayHelper", "DohHelper", "SitesHelper", "ResourceHelper", "ModuleManager", ): monkeypatch.setattr(modules_initializer, name, MagicMock()) + monkeypatch.setattr(modules_initializer, "init_managed_resources", MagicMock()) monkeypatch.setattr(modules_initializer, "user_auth", MagicMock()) monkeypatch.setattr(modules_initializer.EventManager, "start", MagicMock()) for name in ( diff --git a/tests/test_browser_helper.py b/tests/test_browser_helper.py index a012637e7..7e66aa6b9 100644 --- a/tests/test_browser_helper.py +++ b/tests/test_browser_helper.py @@ -1,15 +1,23 @@ from __future__ import annotations import json +import asyncio +import sys import threading from concurrent.futures import ThreadPoolExecutor +from types import ModuleType from typing import Optional -from unittest.mock import patch +from unittest.mock import AsyncMock, MagicMock, call, patch import pytest from app.agent.tools.impl.browse_webpage import BrowserAction, BrowseWebpageTool -from app.adapters.network.browser import BrowserSessionHelper, PlaywrightHelper +from app.adapters.network.browser import ( + BrowserSessionHelper, + PlaywrightHelper, + launch_browser_context, + launch_browser_context_async, +) class _FakeResponse: @@ -224,6 +232,59 @@ def test_legacy_browser_type_constructor_is_accepted(): assert source == "ok" +def test_sync_browser_facade_activates_display_only_for_headed_mode(monkeypatch): + """同步启动仅在明确有界面模式获取 host.display，参数原样交给浏览器。""" + provider = ModuleType("cloakbrowser") + launch_context = MagicMock(return_value=object()) + provider.launch_context = launch_context + monkeypatch.setitem(sys.modules, "cloakbrowser", provider) + activate = MagicMock() + monkeypatch.setattr( + "app.adapters.network.browser.acquire_managed_resource", + activate, + ) + + headless_context = launch_browser_context(headless=True, locale="zh-CN") + headed_context = launch_browser_context(headless=False, locale="zh-CN") + + assert headless_context is launch_context.return_value + assert headed_context is launch_context.return_value + activate.assert_called_once_with( + "host.display", + reason="headed_browser_launch", + retry=True, + ) + assert launch_context.call_args_list == [ + call(headless=True, locale="zh-CN"), + call(headless=False, locale="zh-CN"), + ] + + +def test_async_browser_facade_waits_for_display_before_provider(monkeypatch): + """异步有界面启动必须等待显示资源完成激活后再创建浏览器上下文。""" + events: list[str] = [] + provider = ModuleType("cloakbrowser") + + async def provider_launch(**_kwargs): + events.append("provider") + return object() + + provider.launch_context_async = provider_launch + monkeypatch.setitem(sys.modules, "cloakbrowser", provider) + + async def activate(*_args, **_kwargs): + events.append("display") + + monkeypatch.setattr( + "app.adapters.network.browser.acquire_managed_resource_async", + AsyncMock(side_effect=activate), + ) + + asyncio.run(launch_browser_context_async(headless=False, timezone="Asia/Shanghai")) + + assert events == ["display", "provider"] + + def test_browser_session_helper_blocks_private_network_by_default(): """默认应阻止 Agent 浏览器访问本机或私网地址。""" with pytest.raises(ValueError, match="默认不允许访问本机或私网地址"): diff --git a/tests/test_cache_system.py b/tests/test_cache_system.py index 5bb256d45..02fc6f74c 100644 --- a/tests/test_cache_system.py +++ b/tests/test_cache_system.py @@ -142,7 +142,7 @@ def test_init_modules_does_not_clear_package_tool_cache(monkeypatch): raise AssertionError("init_modules must not clear package tool cache directly") monkeypatch.setattr(modules_initializer, "clear_package_tool_cache", fail_if_called) - monkeypatch.setattr(modules_initializer, "DisplayHelper", lambda: None) + monkeypatch.setattr(modules_initializer, "init_managed_resources", lambda: None) monkeypatch.setattr(modules_initializer, "DohHelper", lambda: None) monkeypatch.setattr(modules_initializer, "SitesHelper", lambda: None) monkeypatch.setattr( diff --git a/tests/test_display_resource.py b/tests/test_display_resource.py new file mode 100644 index 000000000..23455543a --- /dev/null +++ b/tests/test_display_resource.py @@ -0,0 +1,99 @@ +"""虚拟显示托管资源与旧 API 的兼容测试。""" + +from __future__ import annotations + +import sys +from types import ModuleType + +from app.adapters.system.display import DisplayHelper +from app.adapters.system.display.resource import VirtualDisplayResource +from app.foundation.singleton import Singleton + + +def test_virtual_display_skips_host_process_outside_docker(monkeypatch) -> None: + """非容器环境启动资源时不得创建虚拟显示进程。""" + monkeypatch.setattr( + "app.adapters.system.display.resource.SystemUtils.is_docker", + lambda: False, + ) + resource = VirtualDisplayResource() + + resource.start() + resource.stop() + + assert resource.display is None + + +def test_virtual_display_starts_and_stops_owned_process(monkeypatch) -> None: + """容器环境只停止当前资源实际拥有的显示进程。""" + events: list[object] = [] + + class FakeDisplay: + """记录 pyvirtualdisplay 的构造与生命周期。""" + + def __init__(self, **kwargs) -> None: + events.append(("create", kwargs)) + + def start(self) -> None: + events.append("start") + + def stop(self) -> None: + events.append("stop") + + pyvirtualdisplay = ModuleType("pyvirtualdisplay") + pyvirtualdisplay.Display = FakeDisplay + monkeypatch.setitem(sys.modules, "pyvirtualdisplay", pyvirtualdisplay) + monkeypatch.setattr( + "app.adapters.system.display.resource.SystemUtils.is_docker", + lambda: True, + ) + monkeypatch.setenv("DISPLAY", ":99") + resource = VirtualDisplayResource() + + resource.start() + resource.stop() + resource.stop() + + assert events == [ + ( + "create", + { + "visible": False, + "size": (1024, 768), + "extra_args": [":99"], + }, + ), + "start", + "stop", + ] + assert resource.display is None + + +def test_display_helper_keeps_legacy_constructor_and_stop_contract(monkeypatch) -> None: + """旧构造入口显式激活 host.display，stop 只停止已配置 Runtime。""" + events: list[tuple[str, str]] = [] + singleton_key = (DisplayHelper, (), frozenset()) + previous = Singleton._instances.pop(singleton_key, None) + monkeypatch.setattr( + "app.adapters.system.display.acquire_managed_resource", + lambda capability_id, *, reason, retry: events.append( + ("activate", capability_id) + ), + ) + monkeypatch.setattr( + "app.adapters.system.display.stop_managed_resource", + lambda capability_id, *, reason: events.append(("stop", capability_id)), + ) + try: + helper = DisplayHelper() + assert DisplayHelper() is helper + helper.stop() + finally: + Singleton._instances.pop(singleton_key, None) + if previous is not None: + Singleton._instances[singleton_key] = previous + + assert events == [ + ("activate", "host.display"), + ("stop", "host.display"), + ] diff --git a/tests/test_legacy_plugin_resource_imports.py b/tests/test_legacy_plugin_resource_imports.py new file mode 100644 index 000000000..5c874f7ab --- /dev/null +++ b/tests/test_legacy_plugin_resource_imports.py @@ -0,0 +1,382 @@ +"""旧插件资源导入扫描与加载前准备合同测试。""" + +from __future__ import annotations + +import importlib +import os +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +from app.runtime.compat import resource_imports +from app.runtime.compat.resource_imports import ( + PluginResourceImportScanError, + RESOURCE_IMPORT_RULES, + scan_plugin_resource_imports, +) +from app.runtime.extensions import plugin_manager as plugin_manager_module +from app.runtime.extensions.plugin_manager import PluginManager +from app.startup import plugins_initializer + + +_HEADED_CLOAKBROWSER_ENTRYPOINTS = ( + "launch", + "launch_async", + "launch_context", + "launch_context_async", + "launch_persistent_context", + "launch_persistent_context_async", +) + + +def _write_plugin(root: Path, plugin_id: str, source: str) -> Path: + """写入一个仅用于 AST 扫描的最小插件源码目录。""" + plugin_dir = root / plugin_id.lower() + plugin_dir.mkdir(parents=True) + (plugin_dir / "__init__.py").write_text(source, encoding="utf-8") + return plugin_dir + + +@pytest.mark.parametrize( + "source", + ( + "import cloakbrowser\n", + "from cloakbrowser import *\n", + "from cloakbrowser.browser import launch_context\n", + "__import__('cloakbrowser')\n", + "import importlib\nimportlib.import_module('cloakbrowser.browser')\n", + "import importlib as loader\nloader.import_module('cloakbrowser')\n", + "from importlib import import_module as load\nload('cloakbrowser')\n", + ), +) +def test_cloakbrowser_import_shapes_require_display( + tmp_path: Path, + source: str, +) -> None: + """静态、星号、子模块及常量动态导入均准备虚拟显示。""" + plugin_dir = _write_plugin(tmp_path, "SamplePlugin", source) + + assert scan_plugin_resource_imports("SamplePlugin", plugin_dir) == ("host.display",) + + +@pytest.mark.parametrize( + "entrypoint", + _HEADED_CLOAKBROWSER_ENTRYPOINTS, +) +def test_all_headed_capable_cloakbrowser_entrypoints_require_display( + tmp_path: Path, + entrypoint: str, +) -> None: + """CloakBrowser 六类允许 headed 模式的入口共用同一资源规则。""" + plugin_dir = _write_plugin( + tmp_path, + "HeadedPlugin", + f"from cloakbrowser import {entrypoint}\n", + ) + + assert scan_plugin_resource_imports("HeadedPlugin", plugin_dir) == ("host.display",) + assert RESOURCE_IMPORT_RULES[0].headed_entrypoints == ( + _HEADED_CLOAKBROWSER_ENTRYPOINTS + ) + + +@pytest.mark.parametrize( + ("plugin_id", "source"), + ( + ("DynamicWechat", "from cloakbrowser import launch_context_async\n"), + ("ContractCheck", "from cloakbrowser import launch_context\n"), + ("InvitesSignin", "from cloakbrowser import launch_context\n"), + ( + "WeatherWidget", + "__import__('cloakbrowser')\nfrom cloakbrowser import launch_context\n", + ), + ( + "P115StrmHelper", + "from cloakbrowser import launch_context as _cloak_launch_context\n", + ), + ), +) +def test_current_direct_cloakbrowser_plugin_shapes_require_display( + tmp_path: Path, + plugin_id: str, + source: str, +) -> None: + """当前五种直接 CloakBrowser 插件导入形态均命中 host.display。""" + plugin_dir = _write_plugin(tmp_path, plugin_id, source) + + assert scan_plugin_resource_imports(plugin_id, plugin_dir) == ("host.display",) + + +def test_sdk_browser_import_does_not_require_legacy_resource(tmp_path: Path) -> None: + """宿主 SDK 浏览器门面自行按 headless 参数协调资源，不应被保守扫描。""" + plugin_dir = _write_plugin( + tmp_path, + "SdkPlugin", + "from app.sdk.browser import launch_browser_context_async\n", + ) + + assert scan_plugin_resource_imports("SdkPlugin", plugin_dir) == () + + +def test_scanner_reuses_successful_file_result( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """未变化源码在热加载扫描时复用按文件状态缓存的结果。""" + plugin_dir = _write_plugin(tmp_path, "CachedPlugin", "import cloakbrowser\n") + parse_calls = 0 + original_parse = resource_imports.ast.parse + + def count_parse(*args, **kwargs): + nonlocal parse_calls + parse_calls += 1 + return original_parse(*args, **kwargs) + + monkeypatch.setattr(resource_imports.ast, "parse", count_parse) + + assert scan_plugin_resource_imports("CachedPlugin", plugin_dir) == ("host.display",) + assert scan_plugin_resource_imports("CachedPlugin", plugin_dir) == ("host.display",) + assert parse_calls == 1 + + +def test_scanner_invalidates_cache_when_source_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """文件大小或修改时间变化后重新解析，不沿用旧能力集合。""" + plugin_dir = _write_plugin( + tmp_path, + "ChangedPlugin", + "from app.sdk.browser import launch_browser_context\n", + ) + source_path = plugin_dir / "__init__.py" + parse_calls = 0 + original_parse = resource_imports.ast.parse + + def count_parse(*args, **kwargs): + nonlocal parse_calls + parse_calls += 1 + return original_parse(*args, **kwargs) + + monkeypatch.setattr(resource_imports.ast, "parse", count_parse) + + assert scan_plugin_resource_imports("ChangedPlugin", plugin_dir) == () + source_path.write_text( + "from cloakbrowser.browser import launch_persistent_context_async\n", + encoding="utf-8", + ) + assert scan_plugin_resource_imports("ChangedPlugin", plugin_dir) == ( + "host.display", + ) + assert parse_calls == 2 + + +def test_scanner_invalidates_equal_size_source_with_preserved_mtime( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """等长热更新即使保留 mtime，也不能复用替换前的导入结论。""" + plugin_dir = _write_plugin(tmp_path, "ReplacedPlugin", "import cloakbrowser\n") + source_path = plugin_dir / "__init__.py" + original_stat = source_path.stat() + parse_calls = 0 + original_parse = resource_imports.ast.parse + + def count_parse(*args, **kwargs): + nonlocal parse_calls + parse_calls += 1 + return original_parse(*args, **kwargs) + + monkeypatch.setattr(resource_imports.ast, "parse", count_parse) + + assert scan_plugin_resource_imports("ReplacedPlugin", plugin_dir) == ( + "host.display", + ) + source_path.write_text("import cloakbrowsex\n", encoding="utf-8") + os.utime( + source_path, + ns=(original_stat.st_atime_ns, original_stat.st_mtime_ns), + ) + + assert scan_plugin_resource_imports("ReplacedPlugin", plugin_dir) == () + assert parse_calls == 2 + + +def test_scanner_conservatively_prepares_resources_for_invalid_source( + tmp_path: Path, +) -> None: + """未被导入的残留语法文件不得阻断插件，但必须准备全部资源。""" + plugin_dir = _write_plugin( + tmp_path, + "BrokenPlugin", + "from app.sdk.browser import launch_browser_context\n", + ) + (plugin_dir / "unused.py").write_text( + "from cloakbrowser import (\n", + encoding="utf-8", + ) + + assert scan_plugin_resource_imports("BrokenPlugin", plugin_dir) == ("host.display",) + + +def test_scanner_conservatively_prepares_resources_for_read_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """源码读取失败时按全部资源准备，不能降级为空资源集合。""" + plugin_dir = _write_plugin( + tmp_path, + "UnreadablePlugin", + "from app.sdk.browser import launch_browser_context\n", + ) + + original_open = resource_imports.tokenize.open + + def guarded_open(path: Path): + if Path(path).parent == plugin_dir: + raise OSError("fixture read failure") + return original_open(path) + + monkeypatch.setattr(resource_imports.tokenize, "open", guarded_open) + + assert scan_plugin_resource_imports("UnreadablePlugin", plugin_dir) == ( + "host.display", + ) + + +def test_scanner_conservatively_prepares_resources_for_walk_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """目录遍历失败时准备全部资源，后续导入仍由 Python loader 判断。""" + plugin_dir = _write_plugin(tmp_path, "WalkErrorPlugin", "plugin_name = 'ok'\n") + + def fail_walk(_path: Path, _pattern: str): + raise OSError("fixture walk failure") + + monkeypatch.setattr(Path, "rglob", fail_walk) + + assert scan_plugin_resource_imports("WalkErrorPlugin", plugin_dir) == ( + "host.display", + ) + + +def test_scanner_honors_python_source_encoding_cookie(tmp_path: Path) -> None: + """合法的非 UTF-8 Python 源码按 PEP 263 声明解析。""" + plugin_dir = tmp_path / "encodedplugin" + plugin_dir.mkdir() + (plugin_dir / "__init__.py").write_bytes( + "# -*- coding: latin-1 -*-\n# café\nimport cloakbrowser\n".encode("latin-1") + ) + + assert scan_plugin_resource_imports("EncodedPlugin", plugin_dir) == ( + "host.display", + ) + + +def _fake_plugin_module(module_name: str) -> ModuleType: + """构造满足 PluginManager 类发现合同的内存模块。""" + module = ModuleType(module_name) + plugin_type = type( + module_name.rsplit(".", maxsplit=1)[-1].title(), + (), + { + "init_plugin": lambda self, _config: None, + "plugin_name": "Fixture", + }, + ) + setattr(module, plugin_type.__name__, plugin_type) + return module + + +def test_plugin_preparer_runs_before_import_in_non_debug_and_isolates_failures( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """扫描或资源失败只阻止对应插件，后续插件仍按准备后导入的顺序加载。""" + plugins_root = tmp_path / "app" / "plugins" + for plugin_id in ("scanfailed", "resourcefailed", "healthy"): + _write_plugin(plugins_root, plugin_id, "plugin_name = 'Fixture'\n") + + events: list[tuple[str, str]] = [] + + def prepare(*, plugin_id: str, plugin_dir: Path) -> None: + assert plugin_dir.name == plugin_id + events.append(("prepare", plugin_id)) + if plugin_id == "scanfailed": + raise PluginResourceImportScanError("fixture scan failure") + if plugin_id == "resourcefailed": + raise RuntimeError("fixture resource activation failure") + + def import_plugin(module_name: str) -> ModuleType: + plugin_id = module_name.rsplit(".", maxsplit=1)[-1] + assert events[-1] == ("prepare", plugin_id) + events.append(("import", plugin_id)) + return _fake_plugin_module(module_name) + + monkeypatch.setattr( + plugin_manager_module, + "settings", + SimpleNamespace(ROOT_PATH=tmp_path, DEBUG=False), + ) + monkeypatch.setattr( + plugin_manager_module, + "_legacy_plugin_import_preparer", + prepare, + ) + monkeypatch.setattr( + plugin_manager_module, + "_legacy_import_scanner", + lambda **_kwargs: None, + ) + monkeypatch.setattr(importlib, "import_module", import_plugin) + + plugins = PluginManager._load_selective_plugins( + None, + ["ScanFailed", "ResourceFailed", "Healthy"], + lambda plugin_type: hasattr(plugin_type, "init_plugin"), + ) + + assert [plugin.__name__ for plugin in plugins] == ["Healthy"] + assert ("prepare", "scanfailed") in events + assert ("prepare", "resourcefailed") in events + assert ("prepare", "healthy") in events + assert ("import", "scanfailed") not in events + assert ("import", "resourcefailed") not in events + assert ("import", "healthy") in events + + +def test_startup_preparer_activates_scanner_results_generically( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """组合根逐项激活扫描结果，并使用稳定的旧插件导入原因。""" + events: list[tuple[str, str, str]] = [] + plugin_dir = _write_plugin(tmp_path, "LegacyPlugin", "import cloakbrowser\n") + monkeypatch.setattr( + plugins_initializer, + "scan_plugin_resource_imports", + lambda plugin_id, path: ( + events.append(("scan", plugin_id, path.name)) + or ("host.display", "fixture.resource") + ), + ) + monkeypatch.setattr( + plugins_initializer, + "acquire_managed_resource", + lambda capability_id, *, reason: events.append( + ("acquire", capability_id, reason) + ), + ) + + plugins_initializer._prepare_legacy_plugin_import( + plugin_id="LegacyPlugin", + plugin_dir=plugin_dir, + ) + + assert events == [ + ("scan", "LegacyPlugin", "legacyplugin"), + ("acquire", "host.display", "legacy_plugin_import"), + ("acquire", "fixture.resource", "legacy_plugin_import"), + ] diff --git a/tests/test_lifecycle_shutdown.py b/tests/test_lifecycle_shutdown.py index 6feb4d568..9e92576df 100644 --- a/tests/test_lifecycle_shutdown.py +++ b/tests/test_lifecycle_shutdown.py @@ -366,7 +366,6 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: for name, method_name in ( ("ModuleManager", "shutdown"), ("EventManager", "stop"), - ("DisplayHelper", "stop"), ("DohHelper", "shutdown"), ("ThreadHelper", "shutdown"), ("RedisHelper", "close"), @@ -381,11 +380,24 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: key = name.removesuffix("Helper").removesuffix("Manager").lower() dependencies[key] = getattr(instance, method_name) - for name in ("stop_message", "stop_frontend", "clear_temp"): + for name in ( + "close_browser_sessions", + "stop_message", + "stop_frontend", + "clear_temp", + ): dependency = MagicMock() monkeypatch.setattr(modules_initializer, name, dependency) dependencies[name] = dependency + stop_managed_resources = AsyncMock() + monkeypatch.setattr( + modules_initializer, + "stop_managed_resources", + stop_managed_resources, + ) + dependencies["stop_managed_resources"] = stop_managed_resources + async_redis = MagicMock() async_redis.close = AsyncMock() monkeypatch.setattr( @@ -400,6 +412,23 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: return dependencies +def test_browser_sessions_close_before_managed_resources(monkeypatch) -> None: + """显示等宿主资源必须晚于浏览器会话释放，避免存活上下文失去依赖。""" + calls: list[str] = [] + monkeypatch.setattr(modules_initializer, "stop_agent", AsyncMock()) + dependencies = _patch_module_shutdown_dependencies(monkeypatch) + dependencies["close_browser_sessions"].side_effect = lambda: calls.append("browser") + + async def stop_resources() -> None: + calls.append("resources") + + dependencies["stop_managed_resources"].side_effect = stop_resources + + asyncio.run(modules_initializer.stop_modules()) + + assert calls == ["browser", "resources"] + + def test_shared_http_close_waits_for_real_lru_eviction(monkeypatch): """最终 HTTP 关闭必须等待真实 LRU 淘汰任务并消费其异常""" diff --git a/tests/test_managed_resources.py b/tests/test_managed_resources.py new file mode 100644 index 000000000..0ba671c09 --- /dev/null +++ b/tests/test_managed_resources.py @@ -0,0 +1,339 @@ +"""Managed Resource 与 Capability Runtime 的集成合同测试。""" + +from __future__ import annotations + +import asyncio +import subprocess +import sys +import threading +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import ModuleType +from unittest.mock import MagicMock + +import pytest + +from app.runtime.capabilities.errors import ( + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.runtime import CapabilityRuntime +from app.runtime.extensions.managed_resource_adapter import ( + AsyncManagedResourceAdapter, + SyncManagedResourceAdapter, + build_managed_resource_registry, +) +from app.runtime import managed_resources as managed_resource_facade +from app.runtime.managed_resources import ( + MANAGED_RESOURCE_ASYNC_KIND, + MANAGED_RESOURCE_SYNC_KIND, + acquire_managed_resource, + acquire_managed_resource_async, + configure_managed_resource_runtime, + managed_resource_observations, + managed_resource_snapshot, + shutdown_managed_resource_runtime, +) + + +PROJECT_ROOT = Path(__file__).parents[1] + + +@pytest.fixture(autouse=True) +def isolate_managed_resource_facade(monkeypatch: pytest.MonkeyPatch) -> None: + """每个用例使用独立 Runtime，避免不可逆关闭态泄漏到后续测试。""" + monkeypatch.setattr( + managed_resource_facade, + "_managed_resource_runtime", + None, + ) + + +def _write_manifest( + root: Path, *, capability_id: str, kind: str, entrypoint: str +) -> None: + """写入一个最小 on-first-use 托管资源声明。""" + resource_dir = root / capability_id.replace(".", "_") + resource_dir.mkdir(parents=True) + (resource_dir / "capability.toml").write_text( + "\n".join( + ( + "schema_version = 1", + f'id = "{capability_id}"', + f'kind = "{kind}"', + f'entrypoint = "{entrypoint}"', + "depends_on = []", + "", + "[metadata]", + f'name = "{capability_id}"', + "", + "[activation]", + 'policy = "on_first_use"', + "watch = []", + "", + ) + ), + encoding="utf-8", + ) + + +def _runtime(root: Path) -> CapabilityRuntime: + """构造同时支持同步与异步资源的测试 Runtime。""" + registry = build_managed_resource_registry((root,)) + return CapabilityRuntime( + registry, + adapters={ + MANAGED_RESOURCE_SYNC_KIND: SyncManagedResourceAdapter(), + MANAGED_RESOURCE_ASYNC_KIND: AsyncManagedResourceAdapter(), + }, + ) + + +def test_sync_managed_resource_is_single_flight( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """并发首用只能发布一个同步资源实例。""" + module_name = "fixture_sync_managed_resource" + module = ModuleType(module_name) + + class SyncResource: + """记录同步资源的创建、启动和停止次数。""" + + instances: list["SyncResource"] = [] + + def __init__(self) -> None: + self.started = 0 + self.stopped = 0 + type(self).instances.append(self) + + def start(self) -> None: + self.started += 1 + + def stop(self) -> None: + self.stopped += 1 + + module.SyncResource = SyncResource + monkeypatch.setitem(sys.modules, module_name, module) + _write_manifest( + tmp_path, + capability_id="fixture.sync", + kind=MANAGED_RESOURCE_SYNC_KIND, + entrypoint=f"{module_name}:SyncResource", + ) + runtime = _runtime(tmp_path) + configure_managed_resource_runtime(runtime) + + barrier = threading.Barrier(8) + + def activate() -> SyncResource: + barrier.wait(timeout=2) + return acquire_managed_resource("fixture.sync", reason="test") + + with ThreadPoolExecutor(max_workers=8) as executor: + resources = list(executor.map(lambda _index: activate(), range(8))) + + assert len({id(resource) for resource in resources}) == 1 + assert len(SyncResource.instances) == 1 + assert SyncResource.instances[0].started == 1 + assert managed_resource_snapshot("fixture.sync").generation == 1 + assert [ + observation.outcome + for observation in managed_resource_observations("fixture.sync") + if observation.operation == "activate" + ] == ["started", "succeeded"] + + asyncio.run(shutdown_managed_resource_runtime(reason="test_shutdown")) + + assert SyncResource.instances[0].stopped == 1 + with pytest.raises(CapabilityRuntimeClosedError): + acquire_managed_resource("fixture.sync", reason="after_shutdown") + + +def test_async_managed_resource_uses_async_adapter( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """异步资源通过异步 Runtime 入口启动并关闭。""" + module_name = "fixture_async_managed_resource" + module = ModuleType(module_name) + + class AsyncResource: + """记录异步资源生命周期调用。""" + + instances: list["AsyncResource"] = [] + + def __init__(self) -> None: + self.events: list[str] = [] + type(self).instances.append(self) + + async def start(self) -> None: + self.events.append("start") + + async def stop(self) -> None: + self.events.append("stop") + + module.AsyncResource = AsyncResource + monkeypatch.setitem(sys.modules, module_name, module) + _write_manifest( + tmp_path, + capability_id="fixture.async", + kind=MANAGED_RESOURCE_ASYNC_KIND, + entrypoint=f"{module_name}:AsyncResource", + ) + runtime = _runtime(tmp_path) + configure_managed_resource_runtime(runtime) + + async def exercise() -> AsyncResource: + resource = await acquire_managed_resource_async( + "fixture.async", + reason="test", + ) + await shutdown_managed_resource_runtime(reason="test_shutdown") + return resource + + resource = asyncio.run(exercise()) + + assert resource.events == ["start", "stop"] + assert AsyncResource.instances == [resource] + + +def test_failed_start_is_cleaned_before_explicit_retry( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """启动失败的候选必须先清理，显式 retry 才能发布下一代资源。""" + module_name = "fixture_retry_managed_resource" + module = ModuleType(module_name) + + class RetryResource: + """首个候选启动失败，后续候选正常启动。""" + + instances: list["RetryResource"] = [] + + def __init__(self) -> None: + self.events: list[str] = [] + self.fail_start = not type(self).instances + type(self).instances.append(self) + + def start(self) -> None: + self.events.append("start") + if self.fail_start: + raise RuntimeError("start failed") + + def stop(self) -> None: + self.events.append("stop") + + module.RetryResource = RetryResource + monkeypatch.setitem(sys.modules, module_name, module) + _write_manifest( + tmp_path, + capability_id="fixture.retry", + kind=MANAGED_RESOURCE_SYNC_KIND, + entrypoint=f"{module_name}:RetryResource", + ) + configure_managed_resource_runtime(_runtime(tmp_path)) + + with pytest.raises(CapabilityOperationError, match="start failed"): + acquire_managed_resource( + "fixture.retry", + reason="first_use", + retry=False, + ) + + resource = acquire_managed_resource( + "fixture.retry", + reason="retry", + retry=True, + ) + + assert RetryResource.instances[0].events == ["start", "stop"] + assert resource is RetryResource.instances[1] + assert resource.events == ["start"] + + asyncio.run(shutdown_managed_resource_runtime(reason="test_shutdown")) + + assert resource.events == ["start", "stop"] + + +def test_shutdown_does_not_materialize_unused_resource( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """关闭未激活 Runtime 时不得构造资源或调用 start。""" + module_name = "fixture_unused_managed_resource" + module = ModuleType(module_name) + + class UnusedResource: + """任何实例化都表示关闭路径发生反向激活。""" + + def __init__(self) -> None: + raise AssertionError("unused resource must not be materialized") + + def start(self) -> None: + raise AssertionError("unused resource must not start") + + def stop(self) -> None: + raise AssertionError("unused resource must not stop") + + module.UnusedResource = UnusedResource + monkeypatch.setitem(sys.modules, module_name, module) + _write_manifest( + tmp_path, + capability_id="fixture.unused", + kind=MANAGED_RESOURCE_SYNC_KIND, + entrypoint=f"{module_name}:UnusedResource", + ) + configure_managed_resource_runtime(_runtime(tmp_path)) + + asyncio.run(shutdown_managed_resource_runtime(reason="test_shutdown")) + + +def test_startup_initializer_discovers_manifest_without_importing_resource() -> None: + """启动装配只能读取声明，不得提前导入或构造虚拟显示实现。""" + script = """ +import asyncio +import sys +from app.startup.managed_resources_initializer import ( + init_managed_resources, + stop_managed_resources, +) + +runtime = init_managed_resources() +assert runtime.get_running("host.display") is None +assert "app.adapters.system.display.resource" not in sys.modules +assert "pyvirtualdisplay" not in sys.modules +asyncio.run(stop_managed_resources()) +assert "app.adapters.system.display.resource" not in sys.modules +assert "pyvirtualdisplay" not in sys.modules +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_startup_shutdown_without_init_does_not_build_registry(monkeypatch) -> None: + """未执行启动装配时，关闭入口不得通过发现声明反向初始化 Runtime。""" + from app.startup import managed_resources_initializer + + build_registry = MagicMock(side_effect=AssertionError("must not discover")) + monkeypatch.setattr( + managed_resources_initializer, + "_managed_resource_runtime", + None, + ) + monkeypatch.setattr( + managed_resources_initializer, + "build_managed_resource_registry", + build_registry, + ) + + asyncio.run(managed_resources_initializer.stop_managed_resources()) + + build_registry.assert_not_called() diff --git a/tests/test_plugin_sdk.py b/tests/test_plugin_sdk.py index b20437765..9a24f4e71 100644 --- a/tests/test_plugin_sdk.py +++ b/tests/test_plugin_sdk.py @@ -1,4 +1,8 @@ import importlib +import subprocess +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock from app.sdk.cache import Cache, cached from app.sdk.config import settings @@ -12,6 +16,9 @@ from app.sdk.utilities import StringUtils as UtilityStringUtils from app.sdk.utilities import decrypt, encrypt +PROJECT_ROOT = Path(__file__).parents[1] + + def test_sdk_exports_canonical_plugin_interfaces(): """SDK 应复用 canonical 对象，不复制实现或制造第二套单例。""" from app.domain.context import MediaInfo as CanonicalMediaInfo @@ -68,3 +75,58 @@ def test_legacy_common_crypto_aliases_round_trip(): passphrase = b"0123456789abcdef" assert legacy_decrypt(legacy_encrypt(message, passphrase), passphrase) == message + + +def test_browser_sdk_import_is_provider_free(): + """仅导入浏览器 SDK 不得加载浏览器或虚拟显示实现。""" + script = """ +import sys +import app.sdk.browser + +for name in ( + "cloakbrowser", + "pyvirtualdisplay", + "app.adapters.network.browser", + "app.adapters.system.display.resource", +): + assert name not in sys.modules, name +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=PROJECT_ROOT, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_browser_sdk_delegates_sync_and_async_launch(monkeypatch): + """SDK 只转发浏览器参数，不复制宿主生命周期实现。""" + from app.sdk import browser as browser_sdk + from app.adapters.network import browser as browser_adapter + + sync_context = object() + async_context = object() + sync_launch = MagicMock(return_value=sync_context) + async_launch = AsyncMock(return_value=async_context) + monkeypatch.setattr(browser_adapter, "launch_browser_context", sync_launch) + monkeypatch.setattr(browser_adapter, "launch_browser_context_async", async_launch) + + assert browser_sdk.launch_browser_context(headless=False, locale="zh-CN") is sync_context + + async def run_async(): + return await browser_sdk.launch_browser_context_async( + headless=True, + timezone="Asia/Shanghai", + ) + + import asyncio + + assert asyncio.run(run_async()) is async_context + sync_launch.assert_called_once_with(headless=False, locale="zh-CN") + async_launch.assert_awaited_once_with( + headless=True, + timezone="Asia/Shanghai", + ) From 6424f76e70f0cfe73b5b1a1405af6daea6e7638e Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 16 Aug 2026 16:56:53 +0800 Subject: [PATCH 4/8] =?UTF-8?q?fix(notification):=20=E9=A3=9E=E4=B9=A6/QQ/?= =?UTF-8?q?=E7=88=AA=E7=88=AA=E6=9C=BA=E5=99=A8=E4=BA=BA=E8=B7=B3=E8=BF=87?= =?UTF-8?q?=E5=91=BD=E4=BB=A4=E6=B3=A8=E5=86=8C=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=20AttributeError?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 通知基类默认命令注册钩子调用 client.register_commands，但飞书 lark_oapi、 QQBot、WechatClawBot 客户端均无命令注册/删除 API（飞书机器人无斜杠命令概念）， 启动时 FeishuModule.register_commands 抛 AttributeError。 三个模块覆写 _commands_enabled 返回 False 跳过注册，与企业微信走菜单 API 的钩子模式一致；Telegram/Slack/Discord 客户端具备该 API，不受影响。 --- app/modules/feishu/__init__.py | 7 +++++++ app/modules/qqbot/__init__.py | 7 +++++++ app/modules/wechatclawbot/__init__.py | 7 +++++++ 3 files changed, 21 insertions(+) diff --git a/app/modules/feishu/__init__.py b/app/modules/feishu/__init__.py index 21f54562f..54d0d85d8 100644 --- a/app/modules/feishu/__init__.py +++ b/app/modules/feishu/__init__.py @@ -38,6 +38,13 @@ class FeishuModule(_MessageChannelModuleBase[Feishu]): def get_priority() -> int: return 2 + def _commands_enabled(self, config: Optional[dict]) -> bool: + """ + 飞书机器人无斜杠命令概念，lark_oapi Client 也不提供命令注册/删除 API， + 跳过命令注册，避免基类默认钩子调用不存在的 client.register_commands。 + """ + return False + def stop(self) -> None: """停止模块""" for client in self.get_instances().values(): diff --git a/app/modules/qqbot/__init__.py b/app/modules/qqbot/__init__.py index 2c16a1392..3d402b877 100644 --- a/app/modules/qqbot/__init__.py +++ b/app/modules/qqbot/__init__.py @@ -81,6 +81,13 @@ class QQBotModule(_MessageChannelModuleBase[QQBot]): def get_priority() -> int: return 10 + def _commands_enabled(self, config: Optional[dict]) -> bool: + """ + QQ 机器人客户端未提供命令注册/删除 API，跳过命令注册， + 避免基类默认钩子调用不存在的 client.register_commands。 + """ + return False + def stop(self) -> None: """停止模块""" for client in self.get_instances().values(): diff --git a/app/modules/wechatclawbot/__init__.py b/app/modules/wechatclawbot/__init__.py index adca12d74..f349b0406 100644 --- a/app/modules/wechatclawbot/__init__.py +++ b/app/modules/wechatclawbot/__init__.py @@ -61,6 +61,13 @@ class WechatClawBotModule(_MessageChannelModuleBase[WechatClawBot]): """获取模块优先级。""" return 2 + def _commands_enabled(self, config: Optional[dict]) -> bool: + """ + 微信爪爪机器人客户端未提供命令注册/删除 API，跳过命令注册， + 避免基类默认钩子调用不存在的 client.register_commands。 + """ + return False + def stop(self) -> None: """停止模块""" for client in self.get_instances().values(): From 5810f6bc2bf5e2eb9f770d621cc816f865235256 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 16 Aug 2026 17:13:51 +0800 Subject: [PATCH 5/8] =?UTF-8?q?refactor(application):=20filter=5Frules.py?= =?UTF-8?q?=20=E6=9B=B4=E5=90=8D=20rules.py=20=E5=B9=B6=E5=90=88=E5=B9=B6?= =?UTF-8?q?=20RuleHelper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 规则域收敛为单一事实来源 app/application/rules.py： - filter_rules.py 更名为 rules.py（内置规则集 + RuleParser） - filter.py 仅含 66 行 RuleHelper（用户规则组配置访问），并入 rules.py - 同步更新 5 处引用方导入、nettest stub 目标与架构文档 - 兼容映射 app.helper.rule 指向新路径，legacy 导入仍可用 --- app/agent/tools/impl/_filter_rule_utils.py | 6 +- app/api/endpoints/system.py | 2 +- app/application/filter.py | 66 ----------------- app/application/{filter_rules.py => rules.py} | 72 ++++++++++++++++++- app/modules/filter/__init__.py | 6 +- app/runtime/compat/manifest.py | 2 +- app/sdk/services.py | 2 +- docs/rules/05-architecture.md | 2 +- tests/test_system_nettest.py | 2 +- tests/test_torrent_filter.py | 2 +- 10 files changed, 82 insertions(+), 80 deletions(-) delete mode 100644 app/application/filter.py rename app/application/{filter_rules.py => rules.py} (80%) diff --git a/app/agent/tools/impl/_filter_rule_utils.py b/app/agent/tools/impl/_filter_rule_utils.py index 6d8fd1dac..99aa6e38b 100644 --- a/app/agent/tools/impl/_filter_rule_utils.py +++ b/app/agent/tools/impl/_filter_rule_utils.py @@ -7,9 +7,9 @@ from typing import Any, Dict, Iterable, Optional from app.runtime.events import eventmanager from app.db.oper.subscribe import SubscribeOper from app.db.oper.systemconfig import SystemConfigOper -from app.application.filter import RuleHelper -from app.application.filter_rules import RuleParser -from app.application.filter_rules import BUILTIN_RULE_SET +from app.application.rules import RuleHelper +from app.application.rules import RuleParser +from app.application.rules import BUILTIN_RULE_SET from app.schemas import CustomRule, FilterRuleGroup from app.schemas.event import ConfigChangeEventData from app.schemas.types import EventType, SystemConfigKey diff --git a/app/api/endpoints/system.py b/app/api/endpoints/system.py index af033238e..e24aeb53e 100644 --- a/app/api/endpoints/system.py +++ b/app/api/endpoints/system.py @@ -41,7 +41,7 @@ from app.adapters.external.market import ( ) from app.application.messaging.message import MessageHelper from app.runtime.progress import ProgressHelper -from app.application.filter import RuleHelper +from app.application.rules import RuleHelper from app.adapters.external.server import MoviePilotServerHelper from app.runtime.state import SystemHelper from app.runtime.log import logger diff --git a/app/application/filter.py b/app/application/filter.py deleted file mode 100644 index 47ac7698c..000000000 --- a/app/application/filter.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import List, Optional - -from app.db.oper.systemconfig import SystemConfigOper -from app.domain.context import MediaInfo -from app.schemas import CustomRule, FilterRuleGroup -from app.schemas.types import SystemConfigKey - - -class RuleHelper: - """读取用户过滤规则配置，并按媒体上下文选择适用规则组。""" - - @staticmethod - def get_rule_groups() -> List[FilterRuleGroup]: - """返回用户配置的全部过滤规则组。""" - rule_groups: List[dict] = SystemConfigOper().get( - SystemConfigKey.UserFilterRuleGroups - ) - if not rule_groups: - return [] - return [FilterRuleGroup(**group) for group in rule_groups] - - def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]: - """按名称返回过滤规则组。""" - return next( - (group for group in self.get_rule_groups() if group.name == group_name), - None, - ) - - def get_rule_group_by_media( - self, - media: Optional[MediaInfo] = None, - group_names: Optional[list] = None, - ) -> List[FilterRuleGroup]: - """按媒体类型、分类和候选名称筛选适用规则组。""" - rule_groups = self.get_rule_groups() - if group_names: - rule_groups = [ - group for group in rule_groups if group.name in group_names - ] - return [ - group - for group in rule_groups - if not group.media_type - or ( - media - and ( - (not group.category and group.media_type == media.type.value) - or group.category == media.category - ) - ) - ] - - @staticmethod - def get_custom_rules() -> List[CustomRule]: - """返回用户配置的全部自定义过滤规则。""" - rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules) - if not rules: - return [] - return [CustomRule(**rule) for rule in rules] - - def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]: - """按 ID 返回一条自定义过滤规则。""" - return next( - (rule for rule in self.get_custom_rules() if rule.id == rule_id), - None, - ) diff --git a/app/application/filter_rules.py b/app/application/rules.py similarity index 80% rename from app/application/filter_rules.py rename to app/application/rules.py index 86905e227..e51f4c6a8 100644 --- a/app/application/filter_rules.py +++ b/app/application/rules.py @@ -1,11 +1,79 @@ -"""过滤规则解析器与内置规则定义，过滤模块与 Agent 工具共享同一事实来源。""" +""" +规则域：用户规则组配置访问、内置规则定义与规则解析器， +过滤模块与 Agent 工具共享同一事实来源。 +""" import threading -from typing import Dict +from typing import Dict, List, Optional from pyparsing import Forward, Literal, Word, alphas, infix_notation, opAssoc, alphanums, Combine, nums, ParseResults from app.adapters.system import rust as rust_accel +from app.db.oper.systemconfig import SystemConfigOper +from app.domain.context import MediaInfo +from app.schemas import CustomRule, FilterRuleGroup +from app.schemas.types import SystemConfigKey + + +class RuleHelper: + """读取用户过滤规则配置，并按媒体上下文选择适用规则组。""" + + @staticmethod + def get_rule_groups() -> List[FilterRuleGroup]: + """返回用户配置的全部过滤规则组。""" + rule_groups: List[dict] = SystemConfigOper().get( + SystemConfigKey.UserFilterRuleGroups + ) + if not rule_groups: + return [] + return [FilterRuleGroup(**group) for group in rule_groups] + + def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]: + """按名称返回过滤规则组。""" + return next( + (group for group in self.get_rule_groups() if group.name == group_name), + None, + ) + + def get_rule_group_by_media( + self, + media: Optional[MediaInfo] = None, + group_names: Optional[list] = None, + ) -> List[FilterRuleGroup]: + """按媒体类型、分类和候选名称筛选适用规则组。""" + rule_groups = self.get_rule_groups() + if group_names: + rule_groups = [ + group for group in rule_groups if group.name in group_names + ] + return [ + group + for group in rule_groups + if not group.media_type + or ( + media + and ( + (not group.category and group.media_type == media.type.value) + or group.category == media.category + ) + ) + ] + + @staticmethod + def get_custom_rules() -> List[CustomRule]: + """返回用户配置的全部自定义过滤规则。""" + rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules) + if not rules: + return [] + return [CustomRule(**rule) for rule in rules] + + def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]: + """按 ID 返回一条自定义过滤规则。""" + return next( + (rule for rule in self.get_custom_rules() if rule.id == rule_id), + None, + ) + # 内置规则只在这里维护一份，便于过滤模块和 Agent 工具共享同一套事实来源。 BUILTIN_RULE_SET: Dict[str, dict] = { diff --git a/app/modules/filter/__init__.py b/app/modules/filter/__init__.py index 5ce370aa2..2d5a590aa 100644 --- a/app/modules/filter/__init__.py +++ b/app/modules/filter/__init__.py @@ -5,11 +5,11 @@ from typing import List, Tuple, Union, Dict, Optional from app.domain.context import TorrentInfo, MediaInfo from app.domain.metainfo import MetaInfo, clear_rust_parse_options_cache, _rust_parse_options -from app.application.filter import RuleHelper +from app.application.rules import RuleHelper from app.runtime.log import logger from app.modules import _ModuleBase -from app.application.filter_rules import RuleParser -from app.application.filter_rules import BUILTIN_RULE_SET +from app.application.rules import RuleParser +from app.application.rules import BUILTIN_RULE_SET from app.schemas.types import ModuleType, OtherModulesType, SystemConfigKey from app.adapters.system import rust as rust_accel from app.foundation import size as size_tools diff --git a/app/runtime/compat/manifest.py b/app/runtime/compat/manifest.py index a40f2076a..9964774d0 100644 --- a/app/runtime/compat/manifest.py +++ b/app/runtime/compat/manifest.py @@ -543,7 +543,7 @@ MODULE_ALIASES: Dict[str, ModuleAlias] = { introduced="v3.0.0", owner="application", ), "app.helper.rule": ModuleAlias( - target="app.application.filter", replacement="app.sdk.services", + target="app.application.rules", replacement="app.sdk.services", introduced="v3.0.0", owner="application", ), "app.helper.scraper": ModuleAlias( diff --git a/app/sdk/services.py b/app/sdk/services.py index a52c4de10..f91e89264 100644 --- a/app/sdk/services.py +++ b/app/sdk/services.py @@ -3,7 +3,7 @@ from app.runtime.extensions.service_registry import ServiceBaseHelper, ServiceConfigHelper from app.runtime.state import SystemHelper from app.application.downloader import DownloaderHelper -from app.application.filter import RuleHelper +from app.application.rules import RuleHelper from app.application.mediaserver import ( MediaServerIdentityHelper, MediaServerHelper, diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 80fc531d3..6e3712d71 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -379,7 +379,7 @@ policy. `app/db` therefore has no dependency on `app/domain`. | `app/adapters/system/resource.py` | Runtime resource detection/download/installation | | `app/adapters/system/fsproxy.py` | Timeout-guarded local filesystem operations in a killable subprocess (with colocated `fsworker.py`) | | `app/adapters/external/wechat_crypt.py` | WeChat enterprise-message XML encryption/decryption protocol | -| `app/application/filter_rules.py` | Built-in torrent filter rule set and rule parser | +| `app/application/rules.py` | Rule domain: user rule-group config access (`RuleHelper`), built-in torrent filter rule set and rule parser | | `app/adapters/external/market.py` | Plugin repository discovery and installation | | `app/application/security/url.py` | URL/path validation, SSRF protection and signed image policy | | `app/application/mediaserver.py` | Configured media-server discovery and identity matching | diff --git a/tests/test_system_nettest.py b/tests/test_system_nettest.py index 9a0ff15af..029bc2b6d 100644 --- a/tests/test_system_nettest.py +++ b/tests/test_system_nettest.py @@ -51,7 +51,7 @@ _STUB_MODULES = dict([ _stub("app.application.mediaserver", MediaServerHelper=_Dummy), _stub("app.application.messaging.message", MessageHelper=_Dummy), _stub("app.runtime.progress", ProgressHelper=_Dummy), - _stub("app.application.filter", RuleHelper=_Dummy), + _stub("app.application.rules", RuleHelper=_Dummy), _stub("app.adapters.external.server", MoviePilotServerHelper=_Dummy), _stub("app.runtime.state", SystemHelper=_Dummy), _stub("app.application.image", ImageHelper=_Dummy), diff --git a/tests/test_torrent_filter.py b/tests/test_torrent_filter.py index 135080edc..d1fb9e7dc 100644 --- a/tests/test_torrent_filter.py +++ b/tests/test_torrent_filter.py @@ -5,7 +5,7 @@ from unittest.mock import patch from app.domain.context import MediaInfo, TorrentInfo from app.application.torrent import TorrentHelper from app.modules.filter import FilterModule -from app.application.filter_rules import BUILTIN_RULE_SET +from app.application.rules import BUILTIN_RULE_SET from app.adapters.system import rust as rust_accel From 3e417c1ac51694bf7b3e4e45113063d81d6bfb92 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Sun, 16 Aug 2026 17:25:49 +0800 Subject: [PATCH 6/8] =?UTF-8?q?refactor(chain):=20=5Fmixins.py=20=E6=9B=B4?= =?UTF-8?q?=E5=90=8D=20=5Ftransfer.py=EF=BC=8C=E7=BB=9F=E4=B8=80=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=9F=9F=20mixin=20=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TransferChain 的 7 个功能域 mixin 集合文件按域更名 _transfer.py， 与 _recognition/_messaging/_interaction/_music 的单域命名约定对齐； 同步更新 transfer.py 导入、9 个测试文件 patch 目标与架构文档。 另含 chain 基类导入排序优化与未使用导入清理（copy/datetime 已随 dispatch 方法迁入 _messaging.py）。 --- app/chain/__init__.py | 16 +++--- app/chain/{_mixins.py => _transfer.py} | 0 app/chain/transfer.py | 2 +- docs/rules/05-architecture.md | 2 +- tests/test_episode_format_helper.py | 4 +- tests/test_manual_transfer_history.py | 8 +-- tests/test_music_transfer.py | 6 +-- tests/test_transfer_custom_words.py | 2 +- tests/test_transfer_failed_retry_buttons.py | 4 +- tests/test_transfer_mounted_disk_cleanup.py | 10 ++-- tests/test_transfer_movie_collection.py | 12 ++--- tests/test_transfer_sync_extra_files.py | 58 ++++++++++----------- tests/test_transfer_tmdb_category.py | 4 +- 13 files changed, 63 insertions(+), 65 deletions(-) rename app/chain/{_mixins.py => _transfer.py} (100%) diff --git a/app/chain/__init__.py b/app/chain/__init__.py index b6f9b4749..6c6498213 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -1,25 +1,26 @@ from __future__ import annotations -import copy import inspect import pickle import traceback from abc import ABCMeta from collections.abc import Callable -from datetime import datetime from pathlib import Path from typing import Optional, Any, Tuple, List, Set, Union, Dict from fastapi.concurrency import run_in_threadpool -from app.runtime.cache import FileCache, AsyncFileCache +from app.application.messaging.message import MessageHelper, MessageQueueManager +from app.chain._messaging import MessageProcessingMixin, NotificationMixin +from app.chain._recognition import RecognitionMixin +from app.db.oper.message import MessageOper from app.domain.context import Context, MediaInfo, SubtitleInfo, TorrentInfo -from app.runtime.events import EventManager from app.domain.meta.metabase import MetaBase +from app.foundation.reflection import ObjectUtils +from app.runtime.cache import FileCache, AsyncFileCache +from app.runtime.events import EventManager from app.runtime.extensions.module_manager import ModuleManager from app.runtime.extensions.plugin_manager import PluginManager -from app.db.oper.message import MessageOper -from app.application.messaging.message import MessageHelper, MessageQueueManager from app.runtime.log import logger from app.schemas import ( RateLimitExceededException, @@ -41,9 +42,6 @@ from app.schemas.types import ( MediaImageType, EventType, ) -from app.foundation.reflection import ObjectUtils -from app.chain._messaging import MessageProcessingMixin, NotificationMixin -from app.chain._recognition import RecognitionMixin class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, diff --git a/app/chain/_mixins.py b/app/chain/_transfer.py similarity index 100% rename from app/chain/_mixins.py rename to app/chain/_transfer.py diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 2c3c86b76..eefbc8193 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -57,7 +57,7 @@ from app.schemas.types import ( from app.runtime.reload import ConfigReloadMixin from app.application.transfer import (FailedRetryScheduler, JobManager, TransferQueue, TransferTask, job_lock) -from app.chain._mixins import (EpisodeFormatMixin, FailedRetryMixin, +from app.chain._transfer import (EpisodeFormatMixin, FailedRetryMixin, FileFilterMixin, FileKeyMixin, HistoryMatchMixin, ManualHistoryMixin, ScrapeBatchMixin) diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 6e3712d71..637f9ad98 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -219,7 +219,7 @@ Underscore-prefixed files in `app/chain/` are feature-domain mixins for slash-command delegation for `remote_list` / `parse_callback` / `handle_callback_interaction` / `handle_text_interaction`), `_music.py` (`MusicSubscribeMixin`, the music single/album subscribe domain mixed into -`SubscribeChain`) and `_mixins.py` (TransferChain feature mixins). A concrete chain that exposes slash-command +`SubscribeChain`) and `_transfer.py` (TransferChain feature mixins). A concrete chain that exposes slash-command interaction inherits `InteractionChainMixin`, injects its handler class via `_interaction_handler_type` and implements only `_interaction_handler`; it must not re-export application-layer interaction managers. diff --git a/tests/test_episode_format_helper.py b/tests/test_episode_format_helper.py index 3ad3d6764..4bb22be05 100644 --- a/tests/test_episode_format_helper.py +++ b/tests/test_episode_format_helper.py @@ -722,7 +722,7 @@ def test_transfer_chain_recommend_episode_format_passes_helper_data(monkeypatch) lambda item: [sample], ) monkeypatch.setattr( - "app.chain._mixins.EpisodeFormatRuleHelper.recommend", + "app.chain._transfer.EpisodeFormatRuleHelper.recommend", lambda self, rules, sample_files: (True, "", helper_data), ) @@ -777,7 +777,7 @@ def test_transfer_chain_recommend_episode_format_uses_selected_fileitems(monkeyp lambda: [], ) monkeypatch.setattr( - "app.chain._mixins.EpisodeFormatRuleHelper.recommend", + "app.chain._transfer.EpisodeFormatRuleHelper.recommend", lambda self, rules, sample_files: (True, "", { **helper_data, "received_samples": [item.name for item in sample_files], diff --git a/tests/test_manual_transfer_history.py b/tests/test_manual_transfer_history.py index 2190497a6..12665826e 100644 --- a/tests/test_manual_transfer_history.py +++ b/tests/test_manual_transfer_history.py @@ -62,7 +62,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del "app.chain.transfer.TransferHistoryOper", lambda: history_oper, ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: history_oper) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: history_oper) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -72,7 +72,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -82,7 +82,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -93,7 +93,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del or True, ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( exists=lambda current_fileitem: True, delete_media_file=lambda current_fileitem: deleted.append( ("target", current_fileitem.path) diff --git a/tests/test_music_transfer.py b/tests/test_music_transfer.py index 4e1d7935b..d0825c375 100644 --- a/tests/test_music_transfer.py +++ b/tests/test_music_transfer.py @@ -68,7 +68,7 @@ def test_music_retry_restores_history_entity_namespace(tmp_path, monkeypatch): title="叶惠美", ) monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain) - monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: media_chain) + monkeypatch.setattr("app.chain._transfer.MediaChain", lambda: media_chain) result = TransferChain()._recognize_music_retry_media( history, @@ -538,7 +538,7 @@ def test_success_file_aggregation_is_isolated_between_music_jobs_in_same_directo "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace()) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace()) monkeypatch.setattr( "app.chain.transfer.add_transfer_success", lambda **kwargs: SimpleNamespace(id=1), @@ -784,7 +784,7 @@ def test_downloader_process_forwards_music_history_type(tmp_path, monkeypatch): ), ) monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain) - monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: media_chain) + monkeypatch.setattr("app.chain._transfer.MediaChain", lambda: media_chain) monkeypatch.setattr(chain, "do_transfer", Mock(return_value=(True, ""))) monkeypatch.setattr(chain, "run_module", run_module) diff --git a/tests/test_transfer_custom_words.py b/tests/test_transfer_custom_words.py index 30563b137..531ae89a3 100644 --- a/tests/test_transfer_custom_words.py +++ b/tests/test_transfer_custom_words.py @@ -7,7 +7,7 @@ """ from types import SimpleNamespace -import app.chain._mixins as mixins_module +import app.chain._transfer as mixins_module from app.chain.transfer import TransferChain diff --git a/tests/test_transfer_failed_retry_buttons.py b/tests/test_transfer_failed_retry_buttons.py index b096b8891..223d9e4e5 100644 --- a/tests/test_transfer_failed_retry_buttons.py +++ b/tests/test_transfer_failed_retry_buttons.py @@ -150,7 +150,7 @@ class TestTransferFailedRetryButtons(unittest.TestCase): "app.chain.transfer.TransferHistoryOper" ) as history_oper_cls, patch( # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 - "app.chain._mixins.TransferHistoryOper" + "app.chain._transfer.TransferHistoryOper" ) as mixins_history_oper_cls, patch( "app.chain.transfer.asyncio.run_coroutine_threadsafe", side_effect=_close_pending_coro, @@ -224,7 +224,7 @@ class TestTransferFailedRetryButtons(unittest.TestCase): "app.chain.transfer.TransferHistoryOper" ) as history_oper_cls, patch( # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 - "app.chain._mixins.TransferHistoryOper" + "app.chain._transfer.TransferHistoryOper" ) as mixins_history_oper_cls, patch( "app.application.agent._agent_manager.run_background_prompt", side_effect=fake_run_background_prompt, diff --git a/tests/test_transfer_mounted_disk_cleanup.py b/tests/test_transfer_mounted_disk_cleanup.py index 1c4732c31..ad11c0629 100644 --- a/tests/test_transfer_mounted_disk_cleanup.py +++ b/tests/test_transfer_mounted_disk_cleanup.py @@ -31,7 +31,7 @@ def test_enabled_cleanup_skips_filesystem_detection(): 开关开启时应保持旧行为，且不产生额外文件系统检测。 """ with patch( - "app.chain._mixins.SystemUtils.is_network_filesystem" + "app.chain._transfer.SystemUtils.is_network_filesystem" ) as is_network_filesystem: should_delete = ( TransferChain._should_delete_empty_source_directories( @@ -50,7 +50,7 @@ def test_disabled_cleanup_keeps_mounted_local_source_directories(): 开关关闭时应保留网络或 FUSE 挂载的本地源目录。 """ with patch( - "app.chain._mixins.SystemUtils.is_network_filesystem", + "app.chain._transfer.SystemUtils.is_network_filesystem", return_value=True, ) as is_network_filesystem: should_delete = ( @@ -72,7 +72,7 @@ def test_disabled_cleanup_still_deletes_ordinary_local_source_directories(): 开关关闭时普通本地文件系统仍应删除空目录。 """ with patch( - "app.chain._mixins.SystemUtils.is_network_filesystem", + "app.chain._transfer.SystemUtils.is_network_filesystem", return_value=False, ): should_delete = ( @@ -91,7 +91,7 @@ def test_disabled_cleanup_does_not_change_remote_storage_cleanup(): 开关关闭时非本地存储仍应执行原有空目录清理。 """ with patch( - "app.chain._mixins.SystemUtils.is_network_filesystem" + "app.chain._transfer.SystemUtils.is_network_filesystem" ) as is_network_filesystem: should_delete = ( TransferChain._should_delete_empty_source_directories( @@ -111,7 +111,7 @@ def test_mounted_filesystem_detection_is_cached_by_source_directory(): """ mounted_filesystem_cache = {} with patch( - "app.chain._mixins.SystemUtils.is_network_filesystem", + "app.chain._transfer.SystemUtils.is_network_filesystem", return_value=True, ) as is_network_filesystem: for _ in range(2): diff --git a/tests/test_transfer_movie_collection.py b/tests/test_transfer_movie_collection.py index 3d00ed21b..b420cbfdc 100644 --- a/tests/test_transfer_movie_collection.py +++ b/tests/test_transfer_movie_collection.py @@ -107,7 +107,7 @@ def test_conflicting_download_history_recognizes_movie_by_file_meta(monkeypatch) "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_type_tmdbid=lambda **kwargs: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_type_tmdbid=lambda **kwargs: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_type_tmdbid=lambda **kwargs: None)) monkeypatch.setattr( "app.chain.transfer.MediaChain", lambda: SimpleNamespace( @@ -118,7 +118,7 @@ def test_conflicting_download_history_recognizes_movie_by_file_meta(monkeypatch) supplement_tmdb_info=lambda media, _meta: media, ), ) - monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.MediaChain", lambda: SimpleNamespace( recognize_media=lambda **kwargs: pytest.fail("不应按合集历史 ID 识别"), recognize_by_meta=lambda meta, obtain_images: ( recognized_meta.append(meta) or fallback_media @@ -190,16 +190,16 @@ def test_movie_collection_conflict_only_drops_automatic_media( "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr("app.chain.transfer.DownloadHistoryOper", lambda: history_oper) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: history_oper) + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: history_oper) monkeypatch.setattr( "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.StorageChain", lambda: SimpleNamespace()) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace()) + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace()) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda *args, **kwargs: file_meta) # 用真 MediaInfo 而非 SimpleNamespace：它会被装进 TransferTask.mediainfo， diff --git a/tests/test_transfer_sync_extra_files.py b/tests/test_transfer_sync_extra_files.py index 3b02a50e6..7e33d8269 100644 --- a/tests/test_transfer_sync_extra_files.py +++ b/tests/test_transfer_sync_extra_files.py @@ -142,7 +142,7 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -152,7 +152,7 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -162,7 +162,7 @@ def test_sync_extra_subtitle_inherits_matching_video_episode(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", fake_meta_info_path) state, errmsg = TransferChain.do_transfer( @@ -239,7 +239,7 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -249,7 +249,7 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -259,7 +259,7 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -270,7 +270,7 @@ def test_single_subtitle_transfer_reuses_same_name_video_episode(monkeypatch): ], ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( get_parent_item=lambda fileitem: parent_fileitem, list_files=lambda fileitem, recursion=False: [ main_fileitem, @@ -354,7 +354,7 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -364,7 +364,7 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -374,7 +374,7 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -382,7 +382,7 @@ def test_single_video_transfer_lists_parent_once_for_same_name_extra(monkeypatch list_files=fake_list_files, ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( get_parent_item=lambda fileitem: parent_fileitem, list_files=fake_list_files, )) @@ -452,7 +452,7 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -462,7 +462,7 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -472,7 +472,7 @@ def test_episode_format_filters_extra_files_before_sync_planning(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -539,7 +539,7 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -549,7 +549,7 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -559,7 +559,7 @@ def test_episode_format_keeps_matching_extra_files_following_main(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -635,7 +635,7 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -645,7 +645,7 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -655,7 +655,7 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( @@ -666,7 +666,7 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat ], ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( get_parent_item=lambda fileitem: parent_fileitem, list_files=lambda fileitem, recursion=False: [ main_fileitem, @@ -731,7 +731,7 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace(get_by_src=lambda src, storage=None: None)) monkeypatch.setattr( "app.chain.transfer.DownloadHistoryOper", lambda: SimpleNamespace( @@ -741,7 +741,7 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp get_by_path=lambda path: None, ), ) - monkeypatch.setattr("app.chain._mixins.DownloadHistoryOper", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.DownloadHistoryOper", lambda: SimpleNamespace( get_by_hash=lambda download_hash: None, get_file_by_fullpath=lambda fullpath: None, get_files_by_savepath=lambda savepath: [], @@ -751,14 +751,14 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) @@ -799,14 +799,14 @@ def test_cleanup_dest_fileitem_is_kept_when_episode_format_matches_nothing(monke "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) monkeypatch.setattr( "app.chain.transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, ), ) - monkeypatch.setattr("app.chain._mixins.StorageChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, )) @@ -841,7 +841,7 @@ def test_episode_format_matched_but_filtered_by_size_returns_failure(monkeypatch "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) state, errmsg = TransferChain.do_transfer( chain, @@ -882,7 +882,7 @@ def test_candidate_collection_checks_continue_callback(monkeypatch): "app.chain.transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None), ) - monkeypatch.setattr("app.chain._mixins.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) + monkeypatch.setattr("app.chain._transfer.SystemConfigOper", lambda: SimpleNamespace(get=lambda key: None)) state, errmsg = TransferChain.do_transfer( chain, diff --git a/tests/test_transfer_tmdb_category.py b/tests/test_transfer_tmdb_category.py index bae4ed2ed..6a7818951 100644 --- a/tests/test_transfer_tmdb_category.py +++ b/tests/test_transfer_tmdb_category.py @@ -79,14 +79,14 @@ def test_transfer_stops_when_automatic_category_has_no_tmdb_result(monkeypatch) "app.chain.transfer.TransferHistoryOper", lambda: SimpleNamespace(), ) - monkeypatch.setattr("app.chain._mixins.TransferHistoryOper", lambda: SimpleNamespace()) + monkeypatch.setattr("app.chain._transfer.TransferHistoryOper", lambda: SimpleNamespace()) monkeypatch.setattr( "app.chain.transfer.MediaChain", lambda: SimpleNamespace( supplement_tmdb_info=lambda media, _meta: media, ), ) - monkeypatch.setattr("app.chain._mixins.MediaChain", lambda: SimpleNamespace( + monkeypatch.setattr("app.chain._transfer.MediaChain", lambda: SimpleNamespace( supplement_tmdb_info=lambda media, _meta: media, )) task = TransferTask( From e5f0c530690895965f486d09c921e090ffe1a9d8 Mon Sep 17 00:00:00 2001 From: LinFei83 Date: Sun, 16 Aug 2026 17:28:38 +0800 Subject: [PATCH 7/8] =?UTF-8?q?MCP=20get=5Fsearch=5Fresults=20=E5=A2=9E?= =?UTF-8?q?=E5=8A=A0=20include=5Flabels=20=E6=8C=89=E9=9C=80=E8=BF=94?= =?UTF-8?q?=E5=9B=9E=E7=A7=8D=E5=AD=90=E6=A0=87=E7=AD=BE=20(#6335)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 让 Agent 在筛选命中标签时能按需查看 labels，默认不返回以免拉长上下文。 Co-authored-by: Cursor (cherry picked from commit 6a02e7de21c110d758e3fc44e79150ac1d4d7949) --- app/agent/tools/impl/_torrent_search_utils.py | 4 ++++ app/agent/tools/impl/get_search_results.py | 9 ++++++++- docs/mcp-api.md | 2 +- skills/moviepilot-cli/SKILL.md | 4 ++-- tests/test_agent_get_search_results_tool.py | 11 +++++++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/app/agent/tools/impl/_torrent_search_utils.py b/app/agent/tools/impl/_torrent_search_utils.py index f12cff5b0..6445e7008 100644 --- a/app/agent/tools/impl/_torrent_search_utils.py +++ b/app/agent/tools/impl/_torrent_search_utils.py @@ -133,6 +133,7 @@ def simplify_search_result( context: Context, index: int, include_description: bool = False, + include_labels: bool = False, ) -> dict: """ 精简单条搜索结果 @@ -140,6 +141,7 @@ def simplify_search_result( :param context: 搜索结果上下文 :param index: 搜索结果在原始缓存中的序号 :param include_description: 是否返回种子简介 + :param include_labels: 是否返回种子标签 :return: 精简后的搜索结果 """ simplified = {} @@ -162,6 +164,8 @@ def simplify_search_result( } if include_description: simplified["torrent_info"]["description"] = torrent_info.description + if include_labels: + simplified["torrent_info"]["labels"] = torrent_info.labels or [] if media_info: if getattr(media_info, "type", None) == MediaType.MUSIC: diff --git a/app/agent/tools/impl/get_search_results.py b/app/agent/tools/impl/get_search_results.py index b4603d112..128a66de2 100644 --- a/app/agent/tools/impl/get_search_results.py +++ b/app/agent/tools/impl/get_search_results.py @@ -42,6 +42,10 @@ class GetSearchResultsInput(BaseModel): False, description="Whether to include torrent descriptions in returned results", ) + include_labels: Optional[bool] = Field( + False, + description="Whether to include torrent labels in returned results", + ) show_filter_options: Optional[bool] = Field( False, description="Whether to return only optional filter options for re-checking available conditions", @@ -79,6 +83,7 @@ class GetSearchResultsTool(MoviePilotTool): title_pattern: Optional[str] = None, content_pattern: Optional[str] = None, include_description: bool = False, + include_labels: bool = False, show_filter_options: bool = False, page: Optional[int] = 1, **kwargs, @@ -96,6 +101,7 @@ class GetSearchResultsTool(MoviePilotTool): :param title_pattern: 仅匹配种子标题的正则表达式 :param content_pattern: 匹配种子标题、简介和标签的正则表达式 :param include_description: 是否在结果中返回种子简介 + :param include_labels: 是否在结果中返回种子标签 :param show_filter_options: 是否只返回可用筛选项 :param page: 分页页码 :param kwargs: 工具框架附加参数 @@ -103,7 +109,7 @@ class GetSearchResultsTool(MoviePilotTool): """ page = max(1, page or 1) logger.info( - f"执行工具: {self.name}, 参数: site={site}, season={season}, free_state={free_state}, video_code={video_code}, edition={edition}, resolution={resolution}, release_group={release_group}, title_pattern={title_pattern}, content_pattern={content_pattern}, include_description={include_description}, show_filter_options={show_filter_options}, page={page}" + f"执行工具: {self.name}, 参数: site={site}, season={season}, free_state={free_state}, video_code={video_code}, edition={edition}, resolution={resolution}, release_group={release_group}, title_pattern={title_pattern}, content_pattern={content_pattern}, include_description={include_description}, include_labels={include_labels}, show_filter_options={show_filter_options}, page={page}" ) try: @@ -193,6 +199,7 @@ class GetSearchResultsTool(MoviePilotTool): item, index, include_description=include_description, + include_labels=include_labels, ) for item, index in zip(page_items, page_indices) ] diff --git a/docs/mcp-api.md b/docs/mcp-api.md index a5cc8a76a..9c8db1891 100644 --- a/docs/mcp-api.md +++ b/docs/mcp-api.md @@ -300,7 +300,7 @@ TMDB 缓存查询响应的 `data` 包含 `count`、`recognized`、`unrecognized` Agent 音乐流程与影视共用同一采集管线，但实体边界不同：单曲通过 `music_type=recording` 按一个文件处理；专辑通过 `music_type=album` 类似电视剧整季包，按一个目录/资源处理并校验总曲目数；艺术家不是采集目标。`add_subscribe` / `update_subscribe` 支持音乐音质筛选字段和 `best_version` 音质洗版；`query_subscribes` 会返回筛选条件及当前音质快照。`scrape_metadata(media_type="music")` 会按策略写音频标签、封面和歌词，并返回歌词新增、已存在、未匹配和失败数量。 -`get_search_results` 可使用 `title_pattern` 对种子标题执行正则筛选，也可使用 `content_pattern` 联合匹配种子标题、简介和标签。`title_pattern` 保持仅匹配标题的兼容语义；需要在结果中查看种子简介时，传入 `include_description=true`。两种正则参数与站点、分辨率等结构化筛选条件同时传入时按 AND 关系组合。 +`get_search_results` 可使用 `title_pattern` 对种子标题执行正则筛选，也可使用 `content_pattern` 联合匹配种子标题、简介和标签。`title_pattern` 保持仅匹配标题的兼容语义；需要在结果中查看种子简介时，传入 `include_description=true`；需要查看种子标签时，传入 `include_labels=true`。两种正则参数与站点、分辨率等结构化筛选条件同时传入时按 AND 关系组合。 #### Agent 自主定时任务工具 diff --git a/skills/moviepilot-cli/SKILL.md b/skills/moviepilot-cli/SKILL.md index 8803f312f..f64897fa9 100644 --- a/skills/moviepilot-cli/SKILL.md +++ b/skills/moviepilot-cli/SKILL.md @@ -105,8 +105,8 @@ Filter values must come from the `filter_options` returned by `search_torrents` Fetch results with selected filters: `moviepilot tool run get_search_results resolution='1080p,2160p' free_state='免费,50%'` -To filter subtitle, audio, DIY, translation, or release notes that may appear outside the title, use `content_pattern`. It matches the torrent title, description, and labels while `title_pattern` continues to match the title only. Set `include_description=true` when the description is needed to explain why a result matched: -`moviepilot tool run get_search_results content_pattern='特效字幕|国语|DIY' include_description=true` +To filter subtitle, audio, DIY, translation, or release notes that may appear outside the title, use `content_pattern`. It matches the torrent title, description, and labels while `title_pattern` continues to match the title only. Set `include_description=true` when the description is needed to explain why a result matched, and `include_labels=true` when the labels should be returned: +`moviepilot tool run get_search_results content_pattern='特效字幕|国语|DIY' include_description=true include_labels=true` If empty, tell the user which filter to relax and ask before retrying. diff --git a/tests/test_agent_get_search_results_tool.py b/tests/test_agent_get_search_results_tool.py index 250216d79..f0670ff4c 100644 --- a/tests/test_agent_get_search_results_tool.py +++ b/tests/test_agent_get_search_results_tool.py @@ -54,6 +54,17 @@ def test_simplify_search_result_only_includes_description_when_requested(): assert detailed_result["torrent_info"]["description"] == "简繁特效字幕" +def test_simplify_search_result_only_includes_labels_when_requested(): + """精简结果应按参数控制标签输出，避免默认增加上下文长度。""" + context = _build_context("Movie.2026.1080p", labels=["官译", "特效"]) + + default_result = simplify_search_result(context, 1) + detailed_result = simplify_search_result(context, 1, include_labels=True) + + assert "labels" not in default_result["torrent_info"] + assert detailed_result["torrent_info"]["labels"] == ["官译", "特效"] + + def test_content_pattern_matches_title_description_and_labels(): """内容正则应联合匹配标题、简介和标签，并可返回命中的简介。""" items = [ From 5b367011c5b938ab4dbd7563f652f06a1c94716a Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sun, 16 Aug 2026 19:25:24 +0800 Subject: [PATCH 8/8] =?UTF-8?q?refactor(agent):=20=E6=8C=89=E9=9C=80?= =?UTF-8?q?=E5=8A=A0=E8=BD=BD=20Agent=20=E8=BF=90=E8=A1=8C=E6=97=B6=20(#63?= =?UTF-8?q?36)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/agent/capabilities/__init__.py | 8 + app/agent/capabilities/adapter.py | 218 ++++++ .../capabilities/manager/capability.toml | 12 + .../moviepilot_type/capability.toml | 12 + .../capabilities/service/capability.toml | 16 + .../capabilities/tool_factory/capability.toml | 12 + app/agent/contracts.py | 35 + app/agent/llm/__init__.py | 68 +- app/agent/orchestrator.py | 265 ++++--- app/agent/runtime_loader.py | 137 ++++ app/agent/tools/base.py | 64 +- app/agent/tools/manager.py | 200 ++++-- app/api/endpoints/agent.py | 109 ++- app/api/endpoints/anthropic.py | 80 ++- app/api/endpoints/history.py | 16 +- app/api/endpoints/llm.py | 14 +- app/api/endpoints/openai.py | 279 +++++++- app/application/agent.py | 121 ++-- app/application/transfer.py | 9 +- app/chain/_transfer.py | 7 +- app/chain/message.py | 84 ++- app/chain/search.py | 8 +- app/scheduler.py | 16 +- app/schemas/agent.py | 9 +- app/schemas/types.py | 7 + app/startup/agent_initializer.py | 181 ++++- docs/rules/05-architecture.md | 5 +- scripts/perf/README.md | 56 ++ scripts/perf/instrument/sitecustomize.py | 260 ++++++- scripts/perf/moviepilot_docker_ab.py | 381 +++++++++- scripts/perf/test_scenarios.py | 444 +++++++++++- tests/test_agent_api_lazy_imports.py | 460 ++++++++++++ tests/test_agent_background_output.py | 35 +- tests/test_agent_cancellation.py | 4 + tests/test_agent_doctor_tool.py | 17 +- tests/test_agent_graph_cache.py | 2 +- tests/test_agent_image_capability.py | 21 +- tests/test_agent_image_support.py | 36 +- tests/test_agent_interaction.py | 28 +- tests/test_agent_lazy_initializer.py | 245 +++++++ tests/test_agent_lazy_runtime_boundary.py | 322 +++++++++ tests/test_agent_lifecycle.py | 177 +++++ tests/test_agent_message_routing.py | 57 +- tests/test_agent_protocol_lifecycle.py | 216 ++++++ tests/test_agent_recognize_captcha_tool.py | 17 +- tests/test_agent_resource_flow_permissions.py | 24 +- tests/test_agent_runtime_loader.py | 674 ++++++++++++++++++ tests/test_agent_scheduled_tasks.py | 23 +- tests/test_agent_session_status.py | 19 +- tests/test_agent_system_settings_tools.py | 12 +- tests/test_agent_tool_streaming.py | 4 +- tests/test_architecture_dependencies.py | 10 + tests/test_search_ai_recommend.py | 8 + tests/test_telegram_typing_lifecycle.py | 22 +- tests/test_transfer_failed_retry_buttons.py | 44 +- tests/test_web_agent_stream.py | 77 +- 56 files changed, 5059 insertions(+), 628 deletions(-) create mode 100644 app/agent/capabilities/__init__.py create mode 100644 app/agent/capabilities/adapter.py create mode 100644 app/agent/capabilities/manager/capability.toml create mode 100644 app/agent/capabilities/moviepilot_type/capability.toml create mode 100644 app/agent/capabilities/service/capability.toml create mode 100644 app/agent/capabilities/tool_factory/capability.toml create mode 100644 app/agent/contracts.py create mode 100644 app/agent/runtime_loader.py create mode 100644 tests/test_agent_api_lazy_imports.py create mode 100644 tests/test_agent_lazy_initializer.py create mode 100644 tests/test_agent_lazy_runtime_boundary.py create mode 100644 tests/test_agent_protocol_lifecycle.py create mode 100644 tests/test_agent_runtime_loader.py diff --git a/app/agent/capabilities/__init__.py b/app/agent/capabilities/__init__.py new file mode 100644 index 000000000..fe4aa18e2 --- /dev/null +++ b/app/agent/capabilities/__init__.py @@ -0,0 +1,8 @@ +"""Agent Capability 声明与通用入口适配器。""" + +AGENT_ENTRYPOINT_KIND = "agent_entrypoint" +AGENT_SERVICE_KIND = "agent_service" +AGENT_MANAGER_CAPABILITY_ID = "agent.manager" +AGENT_SERVICE_CAPABILITY_ID = "agent.service" +MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID = "agent.moviepilot_type" +TOOL_FACTORY_CAPABILITY_ID = "agent.tool_factory" diff --git a/app/agent/capabilities/adapter.py b/app/agent/capabilities/adapter.py new file mode 100644 index 000000000..d6b160394 --- /dev/null +++ b/app/agent/capabilities/adapter.py @@ -0,0 +1,218 @@ +"""Agent canonical entrypoint 的 Capability Runtime 适配器。""" + +from __future__ import annotations + +import asyncio +import importlib +import inspect +from pathlib import Path +from typing import Any, Iterable, Mapping + +from app.agent.capabilities import AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND +from app.runtime.capabilities.errors import CapabilityAdapterContractError +from app.runtime.capabilities.model import ( + ActivationPolicy, + AdapterExecutionMode, + CapabilitySpec, + SelectorSchema, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.config import settings + + +_DEFAULT_CAPABILITY_ROOT = Path(__file__).resolve().parent +_SETTING_SELECTOR = "setting_truthy" + + +def _validate_setting_selector(config: Mapping[str, Any]) -> None: + """限制 selector 只能读取已声明的应用设置。""" + key = config["key"] + if not isinstance(key, str) or not key or not hasattr(settings, key): + raise ValueError(f"未知应用设置：{key!r}") + + +AGENT_SELECTOR_SCHEMAS = { + _SETTING_SELECTOR: SelectorSchema( + required_fields=frozenset({"key"}), + validator=_validate_setting_selector, + ) +} + + +def _load_entrypoint(spec: CapabilitySpec) -> Any: + """按 manifest 解析 canonical 符号，不创建额外业务对象。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = importlib.import_module(module_name) + try: + return getattr(module, symbol_name) + except AttributeError as error: + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 未公开 Agent entrypoint" + ) from error + + +def _lifecycle_method(spec: CapabilitySpec, candidate: Any, name: str) -> Any: + """读取 Agent Service 必需的异步生命周期方法。""" + callback = getattr(candidate, name, None) + if not callable(callback): + raise CapabilityAdapterContractError( + f"{spec.entrypoint} 的 Agent Service 缺少 {name}()" + ) + return callback + + +class AgentEntrypointAdapter: + """把 canonical Python 符号作为无资源副作用的同步能力发布。""" + + execution_mode = AdapterExecutionMode.SYNC + + @staticmethod + def materialize(spec: CapabilitySpec) -> Any: + """按 manifest entrypoint 导入 canonical 符号。""" + return _load_entrypoint(spec) + + @staticmethod + def create( + _spec: CapabilitySpec, + implementation: Any, + _generation: int, + _previous: Any = None, + ) -> Any: + """发布 canonical 符号本身，不创建第二份业务对象。""" + return implementation + + @staticmethod + def start( + _spec: CapabilitySpec, + _candidate: Any, + _generation: int, + ) -> None: + """entrypoint 不拥有业务资源，初始化由独立 service 能力负责。""" + + @staticmethod + def stop( + _spec: CapabilitySpec, + _instance: Any, + _generation: int, + ) -> None: + """撤销入口可见性；业务资源由独立 service 能力关闭。""" + + @staticmethod + def cleanup( + _spec: CapabilitySpec, + _candidate: Any, + _generation: int, + _error: BaseException, + ) -> None: + """entrypoint 启动无副作用，因此失败候选无需额外释放。""" + + +class AgentServiceAdapter: + """把具备 initialize/close 的 canonical 对象接入异步资源生命周期。""" + + execution_mode = AdapterExecutionMode.ASYNC + + @staticmethod + async def materialize(spec: CapabilitySpec) -> Any: + """在线程中导入 canonical service，避免阻塞应用事件循环。""" + return await asyncio.to_thread(_load_entrypoint, spec) + + @staticmethod + async def create( + _spec: CapabilitySpec, + implementation: Any, + _generation: int, + _previous: Any = None, + ) -> Any: + """复用 canonical service，不复制其内部队列和后台任务所有权。""" + return implementation + + @staticmethod + async def start( + spec: CapabilitySpec, + candidate: Any, + _generation: int, + ) -> None: + """等待 service 在当前应用事件循环完成初始化。""" + result = _lifecycle_method(spec, candidate, "initialize")() + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.initialize() 必须返回 awaitable" + ) + await result + + @staticmethod + async def stop( + spec: CapabilitySpec, + instance: Any, + _generation: int, + ) -> None: + """等待 service 停止后台任务并释放其资源。""" + result = _lifecycle_method(spec, instance, "close")() + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"{spec.entrypoint}.close() 必须返回 awaitable" + ) + await result + + @staticmethod + async def cleanup( + spec: CapabilitySpec, + candidate: Any, + generation: int, + _error: BaseException, + ) -> None: + """初始化失败或关闭竞态时按相同 close 合同释放部分资源。""" + await AgentServiceAdapter.stop(spec, candidate, generation) + + +def _validate_registry(registry: CapabilityRegistry) -> None: + """固定 entrypoint 物化轴与 service 资源轴的声明合同。""" + for spec in registry.list_specs(): + if set(spec.metadata) != {"name"}: + raise ValueError(f"{spec.source}: Agent Capability metadata 只能包含 name") + if spec.kind == AGENT_ENTRYPOINT_KIND: + if spec.activation is not ActivationPolicy.ON_FIRST_USE: + raise ValueError( + f"{spec.source}: Agent entrypoint 必须使用 on_first_use" + ) + if spec.selector is not None or spec.watch: + raise ValueError( + f"{spec.source}: Agent entrypoint 不接受 selector 或 watch" + ) + continue + if spec.activation is not ActivationPolicy.WHEN_CONFIGURED: + raise ValueError(f"{spec.source}: Agent Service 必须使用 when_configured") + selector = spec.selector + if selector is None or selector.kind != _SETTING_SELECTOR: + raise ValueError(f"{spec.source}: Agent Service 必须声明 setting_truthy") + selector_key = str(selector.config["key"]) + if spec.watch != (selector_key,): + raise ValueError( + f"{spec.source}: Agent Service watch 必须只包含 selector key" + ) + + +def build_agent_capability_registry( + roots: Iterable[Path | str] | None = None, +) -> CapabilityRegistry: + """发现 data-only Agent manifests，不导入编排器、Provider 或工具实现。""" + registry = CapabilityRegistry.discover( + tuple(roots) if roots is not None else (_DEFAULT_CAPABILITY_ROOT,), + kinds={AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND}, + selector_schemas=AGENT_SELECTOR_SCHEMAS, + ) + _validate_registry(registry) + return registry + + +def should_run_agent_service(spec: CapabilitySpec) -> bool: + """依据 manifest selector 判断 service 是否应拥有运行实例。""" + selector = spec.selector + if ( + spec.kind != AGENT_SERVICE_KIND + or selector is None + or selector.kind != _SETTING_SELECTOR + ): + raise ValueError(f"{spec.source}: 不是可协调的 Agent Service 声明") + return bool(getattr(settings, selector.config["key"])) diff --git a/app/agent/capabilities/manager/capability.toml b/app/agent/capabilities/manager/capability.toml new file mode 100644 index 000000000..fe26d9e38 --- /dev/null +++ b/app/agent/capabilities/manager/capability.toml @@ -0,0 +1,12 @@ +schema_version = 1 +id = "agent.manager" +kind = "agent_entrypoint" +entrypoint = "app.agent.orchestrator:agent_manager" +depends_on = [] + +[metadata] +name = "Agent Manager" + +[activation] +policy = "on_first_use" +watch = [] diff --git a/app/agent/capabilities/moviepilot_type/capability.toml b/app/agent/capabilities/moviepilot_type/capability.toml new file mode 100644 index 000000000..663dbc87a --- /dev/null +++ b/app/agent/capabilities/moviepilot_type/capability.toml @@ -0,0 +1,12 @@ +schema_version = 1 +id = "agent.moviepilot_type" +kind = "agent_entrypoint" +entrypoint = "app.agent.orchestrator:MoviePilotAgent" +depends_on = [] + +[metadata] +name = "MoviePilot Agent Type" + +[activation] +policy = "on_first_use" +watch = [] diff --git a/app/agent/capabilities/service/capability.toml b/app/agent/capabilities/service/capability.toml new file mode 100644 index 000000000..99b16c3c9 --- /dev/null +++ b/app/agent/capabilities/service/capability.toml @@ -0,0 +1,16 @@ +schema_version = 1 +id = "agent.service" +kind = "agent_service" +entrypoint = "app.agent.orchestrator:agent_manager" +depends_on = [] + +[metadata] +name = "Agent Service" + +[activation] +policy = "when_configured" +watch = ["AI_AGENT_ENABLE"] + +[activation.selector] +kind = "setting_truthy" +key = "AI_AGENT_ENABLE" diff --git a/app/agent/capabilities/tool_factory/capability.toml b/app/agent/capabilities/tool_factory/capability.toml new file mode 100644 index 000000000..7982e7429 --- /dev/null +++ b/app/agent/capabilities/tool_factory/capability.toml @@ -0,0 +1,12 @@ +schema_version = 1 +id = "agent.tool_factory" +kind = "agent_entrypoint" +entrypoint = "app.agent.tools.factory:MoviePilotToolFactory" +depends_on = [] + +[metadata] +name = "Agent Tool Factory" + +[activation] +policy = "on_first_use" +watch = [] diff --git a/app/agent/contracts.py b/app/agent/contracts.py new file mode 100644 index 000000000..242465691 --- /dev/null +++ b/app/agent/contracts.py @@ -0,0 +1,35 @@ +"""Agent 轻量公共合同，不触发模型、工具或编排运行时加载。""" + +import uuid +from datetime import datetime +from typing import Any, Optional + +from app.schemas.types import ReplyMode + + +def build_display_message( + role: str, + content: str = "", + attachments: Optional[list[dict]] = None, + status: str = "done", +) -> dict[str, Any]: + """构造前后端共享的 Agent 会话展示消息。""" + normalized_content = content or "" + return { + "id": f"{role}-{uuid.uuid4().hex}", + "role": role, + "content": normalized_content, + "createdAt": int(datetime.now().timestamp() * 1000), + "status": status, + "tools": [], + "segments": ( + [{"type": "text", "content": normalized_content}] + if normalized_content + else [] + ), + "attachments": attachments or [], + "choices": [], + } + + +__all__ = ["ReplyMode", "build_display_message"] diff --git a/app/agent/llm/__init__.py b/app/agent/llm/__init__.py index 488aba3e1..32c2be776 100644 --- a/app/agent/llm/__init__.py +++ b/app/agent/llm/__init__.py @@ -1,20 +1,56 @@ -"""Agent 内部使用的 LLM 适配层。""" +"""Agent 内部使用的 LLM 适配层，公开对象按需解析。""" -from app.agent.llm.helper import LLMHelper, LLMTestError, LLMTestTimeout -from app.agent.llm.capability import ( - AgentCapabilityManager, - AgentCapabilityProvider, - AudioCapabilityProvider, - MiMoAudioProvider, - OpenAIChatAudioProvider, - OpenAIAudioProvider, -) -from app.agent.llm.provider import ( - LLMProviderAuthError, - LLMProviderError, - LLMProviderManager, - render_auth_result_html, -) +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from app.agent.llm.capability import ( + AgentCapabilityManager, + AgentCapabilityProvider, + AudioCapabilityProvider, + MiMoAudioProvider, + OpenAIAudioProvider, + OpenAIChatAudioProvider, + ) + from app.agent.llm.helper import LLMHelper, LLMTestError, LLMTestTimeout + from app.agent.llm.provider import ( + LLMProviderAuthError, + LLMProviderError, + LLMProviderManager, + render_auth_result_html, + ) + + +_EXPORT_MODULES = { + "LLMHelper": "app.agent.llm.helper", + "LLMTestError": "app.agent.llm.helper", + "LLMTestTimeout": "app.agent.llm.helper", + "AgentCapabilityManager": "app.agent.llm.capability", + "AgentCapabilityProvider": "app.agent.llm.capability", + "AudioCapabilityProvider": "app.agent.llm.capability", + "MiMoAudioProvider": "app.agent.llm.capability", + "OpenAIChatAudioProvider": "app.agent.llm.capability", + "OpenAIAudioProvider": "app.agent.llm.capability", + "LLMProviderAuthError": "app.agent.llm.provider", + "LLMProviderError": "app.agent.llm.provider", + "LLMProviderManager": "app.agent.llm.provider", + "render_auth_result_html": "app.agent.llm.provider", +} + + +def __getattr__(name: str) -> Any: + """首次访问公开对象时只加载其所属适配模块。""" + module_name = _EXPORT_MODULES.get(name) + if module_name is None: + raise AttributeError(f"module 'app.agent.llm' has no attribute {name!r}") + value = getattr(import_module(module_name), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """让延迟公开对象继续支持交互式发现。""" + return sorted(set(globals()) | set(_EXPORT_MODULES)) __all__ = [ "LLMHelper", diff --git a/app/agent/orchestrator.py b/app/agent/orchestrator.py index 8bf21d07a..f66aa2fcb 100644 --- a/app/agent/orchestrator.py +++ b/app/agent/orchestrator.py @@ -4,6 +4,7 @@ import json import re import traceback import uuid +import warnings from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Callable, Dict, List, Optional @@ -16,12 +17,10 @@ from langchain_core.messages import ( # noqa: F401 SystemMessage, ) -import warnings -warnings.filterwarnings("ignore", message=".*allowed_objects.*") - from langgraph.checkpoint.memory import InMemorySaver from app.agent.callback import StreamingHandler +from app.agent.contracts import ReplyMode, build_display_message from app.agent.llm import LLMHelper from app.agent.llm.server_tools import ServerToolRegistry from app.agent.memory import memory_manager @@ -60,7 +59,6 @@ from app.agent.policy import ( ) from app.agent.runtime import agent_runtime_manager from app.agent.mcp import agent_mcp_manager -from app.agent.tools.factory import MoviePilotToolFactory from app.agent.tools.catalog import ToolCatalogSnapshot from app.agent.tools.impl.mcp import ( create_external_mcp_tools, @@ -76,11 +74,12 @@ from app.db.oper.agenttask import AgentTaskOper from app.db.oper.user import UserOper from app.runtime.log import logger from app.schemas import AgentLLMProviderEventData, AgentTokensUsageEventData, Notification, NotificationType -from app.schemas.agent import ReplyMode from app.schemas.message import ChannelCapabilityManager, ChannelCapability from app.schemas.types import ChainEventType, EventType, MessageChannel from app.foundation.identity import SYSTEM_INTERNAL_USER_ID +warnings.filterwarnings("ignore", message=".*allowed_objects.*") + def _finish_processing_status(status: Optional[dict], user_id: Optional[str] = None) -> None: """结束入站消息的渠道处理状态。""" @@ -393,11 +392,6 @@ class MoviePilotAgent: # 流式token管理 self.stream_handler = StreamingHandler() - @staticmethod - def _current_timestamp_ms() -> int: - """返回当前毫秒时间戳。""" - return int(datetime.now().timestamp() * 1000) - @classmethod def build_display_message( cls, @@ -409,22 +403,12 @@ class MoviePilotAgent: """ 构造可展示的 Agent 会话消息。 """ - normalized_content = content or "" - return { - "id": f"{role}-{uuid.uuid4().hex}", - "role": role, - "content": normalized_content, - "createdAt": cls._current_timestamp_ms(), - "status": status, - "tools": [], - "segments": ( - [{"type": "text", "content": normalized_content}] - if normalized_content - else [] - ), - "attachments": attachments or [], - "choices": [], - } + return build_display_message( + role=role, + content=content, + attachments=attachments, + status=status, + ) def _should_save_display_history(self) -> bool: """ @@ -1560,7 +1544,9 @@ class MoviePilotAgent: """ 初始化主 Agent 本地工具实例。 """ - return MoviePilotToolFactory.create_tools( + from app.agent.runtime_loader import get_tool_factory + + return get_tool_factory().create_tools( session_id=self.session_id, user_id=self.user_id, channel=self.channel, @@ -1575,14 +1561,17 @@ class MoviePilotAgent: self, ) -> tuple[ToolCatalogSnapshot, ToolCatalogSnapshot]: """在同一插件 revision 窗口内建立主图和子图工具目录。""" + from app.agent.runtime_loader import get_tool_factory + + tool_factory = get_tool_factory() plugin_manager = PluginManager() - for _attempt in range(MoviePilotToolFactory.CATALOG_BUILD_MAX_ATTEMPTS): + for _attempt in range(tool_factory.CATALOG_BUILD_MAX_ATTEMPTS): before_revision = plugin_manager.get_plugin_agent_tools_revision() tools = self._initialize_tools() subagent_tools = self._initialize_subagent_tools() after_revision = plugin_manager.get_plugin_agent_tools_revision() if before_revision == after_revision: - factory_revision = MoviePilotToolFactory.catalog_factory_revision() + factory_revision = tool_factory.catalog_factory_revision() return ( ToolCatalogSnapshot.from_tools( tools, @@ -1670,12 +1659,19 @@ class MoviePilotAgent: (tool_catalog.signature, subagent_catalog.signature) if tool_catalog is not None and subagent_catalog is not None else ( - MoviePilotToolFactory.catalog_factory_revision(), + self._tool_factory_revision(), PluginManager().get_plugin_agent_tools_revision(), ) ), ) + @staticmethod + def _tool_factory_revision() -> str: + """在目录签名确实需要时解析工具工厂版本。""" + from app.agent.runtime_loader import get_tool_factory + + return get_tool_factory().catalog_factory_revision() + def _get_cached_agent( self, signature: tuple[Any, ...], streaming: bool ) -> Optional[Any]: @@ -1722,7 +1718,9 @@ class MoviePilotAgent: """ 初始化子代理专用静默工具列表。 """ - return MoviePilotToolFactory.create_tools( + from app.agent.runtime_loader import get_tool_factory + + return get_tool_factory().create_tools( session_id=self.session_id, user_id=self.user_id, channel=self.channel, @@ -1907,8 +1905,10 @@ class MoviePilotAgent: logger.debug(f"复用会话内 Agent 图: session_id={self.session_id}") return cached_agent max_tools = settings.LLM_MAX_TOOLS + from app.agent.runtime_loader import get_tool_factory + always_include_tools = ( - MoviePilotToolFactory.get_tool_selector_always_include_names(tools) + get_tool_factory().get_tool_selector_always_include_names(tools) ) if subagent_task_tools: always_include_tools.extend( @@ -2438,9 +2438,16 @@ class _MessageTask: protected_output_callback: Optional[Callable[[str], Optional[bool]]] = None notification_callback: Optional[Callable[[Any], None]] = None agent_factory: Optional[Callable[..., MoviePilotAgent]] = None + agent_setup: Optional[Callable[[MoviePilotAgent], None]] = None completion_future: Optional[asyncio.Future] = None +class AgentManagerUnavailableError(RuntimeError): + """AgentManager 未运行或已开始关闭，不能再接收新任务。""" + + code = "agent_manager_unavailable" + + class AgentManager: """ AI智能体管理器 @@ -2458,6 +2465,9 @@ class AgentManager: self._idle_cleanup_task: Optional[asyncio.Task] = None self._idle_session_ttl = timedelta(hours=24) self._idle_cleanup_interval = 60 * 60 + # 接收门禁与队列写入共用一把锁，确保关闭开始后不会再创建 worker。 + self._lifecycle_lock = asyncio.Lock() + self._accepting_tasks = False def get_session_status(self, session_id: str) -> dict[str, Any]: """获取会话当前模型与 token 使用状态。""" @@ -2504,40 +2514,51 @@ class AgentManager: """ 初始化管理器 """ - memory_manager.initialize() - if self._idle_cleanup_task and not self._idle_cleanup_task.done(): - return - self._idle_cleanup_task = asyncio.create_task(self._cleanup_idle_sessions()) + async with self._lifecycle_lock: + if self._accepting_tasks: + return + memory_manager.initialize() + if not self._idle_cleanup_task or self._idle_cleanup_task.done(): + self._idle_cleanup_task = asyncio.create_task( + self._cleanup_idle_sessions() + ) + self._accepting_tasks = True async def close(self): """ 关闭管理器 """ - if self._idle_cleanup_task: - self._idle_cleanup_task.cancel() - try: - await self._idle_cleanup_task - except asyncio.CancelledError: - pass - self._idle_cleanup_task = None - await memory_manager.close() - # 取消所有会话worker - for task in list(self._session_workers.values()): - task.cancel() - # 等待所有worker结束 - for session_id, task in list(self._session_workers.items()): - try: - await task - except asyncio.CancelledError: - pass - self._session_workers.clear() - for queue in list(self._session_queues.values()): - self._discard_queued_messages(queue) - self._session_queues.clear() - self._session_last_used.clear() - for agent in list(self.active_agents.values()): - await agent.cleanup() - self.active_agents.clear() + async with self._lifecycle_lock: + # 门禁必须先关闭；锁内完成清理可阻止等待中的请求在收口期间重新入队。 + self._accepting_tasks = False + if self._idle_cleanup_task: + self._idle_cleanup_task.cancel() + try: + await self._idle_cleanup_task + except asyncio.CancelledError: + pass + self._idle_cleanup_task = None + # 取消所有会话worker + for task in list(self._session_workers.values()): + task.cancel() + # 等待所有worker结束 + for session_id, task in list(self._session_workers.items()): + try: + await task + except asyncio.CancelledError: + pass + self._session_workers.clear() + for queue in list(self._session_queues.values()): + self._discard_queued_messages( + queue, + error=AgentManagerUnavailableError("AgentManager 已关闭"), + ) + self._session_queues.clear() + self._session_last_used.clear() + for agent in list(self.active_agents.values()): + await agent.cleanup() + self.active_agents.clear() + await memory_manager.close() def _record_session_activity(self, session_id: str, user_id: str) -> None: """ @@ -2607,6 +2628,7 @@ class AgentManager: protected_output_callback: Optional[Callable[[str], Optional[bool]]] = None, notification_callback: Optional[Callable[[Any], None]] = None, agent_factory: Optional[Callable[..., MoviePilotAgent]] = None, + agent_setup: Optional[Callable[[MoviePilotAgent], None]] = None, wait_for_completion: bool = False, ) -> str: """ @@ -2635,38 +2657,40 @@ class AgentManager: protected_output_callback=protected_output_callback, notification_callback=notification_callback, agent_factory=agent_factory, + agent_setup=agent_setup, completion_future=completion_future, ) - self._record_session_activity(session_id, user_id) + async with self._lifecycle_lock: + if not self._accepting_tasks: + raise AgentManagerUnavailableError("AgentManager 未运行或已关闭") + self._record_session_activity(session_id, user_id) - # 获取或创建会话队列 - if session_id not in self._session_queues: - self._session_queues[session_id] = asyncio.Queue() + # 获取或创建会话队列 + if session_id not in self._session_queues: + self._session_queues[session_id] = asyncio.Queue() - queue = self._session_queues[session_id] - queue_size = queue.qsize() + queue = self._session_queues[session_id] + queue_size = queue.qsize() - # 如果队列中已有等待的消息，通知用户消息已排队 - if queue_size > 0 or ( - session_id in self._session_workers - and not self._session_workers[session_id].done() - ): - logger.info( - f"会话 {session_id} 有任务正在处理，消息已排队等待 " - f"(队列中待处理: {queue_size} 条)" - ) + # 如果队列中已有等待的消息，通知用户消息已排队 + if queue_size > 0 or ( + session_id in self._session_workers + and not self._session_workers[session_id].done() + ): + logger.info( + f"会话 {session_id} 有任务正在处理，消息已排队等待 " + f"(队列中待处理: {queue_size} 条)" + ) - # 放入队列 - await queue.put(task) - - # 确保该会话有一个worker在运行 - if ( - session_id not in self._session_workers - or self._session_workers[session_id].done() - ): - self._session_workers[session_id] = asyncio.create_task( - self._session_worker(session_id) - ) + # 放入队列并创建 worker 与关闭门禁保持原子关系。 + await queue.put(task) + if ( + session_id not in self._session_workers + or self._session_workers[session_id].done() + ): + self._session_workers[session_id] = asyncio.create_task( + self._session_worker(session_id) + ) if completion_future: return await completion_future @@ -2698,7 +2722,12 @@ class AgentManager: task.completion_future.set_result(result) except asyncio.CancelledError: if task.completion_future and not task.completion_future.done(): - task.completion_future.cancel() + if self._accepting_tasks: + task.completion_future.cancel() + else: + task.completion_future.set_exception( + AgentManagerUnavailableError("AgentManager 已关闭") + ) raise except Exception as e: logger.error(f"处理会话 {session_id} 的消息失败: {e}") @@ -2723,7 +2752,10 @@ class AgentManager: self._session_queues.pop(session_id, None) @staticmethod - def _discard_queued_messages(queue: asyncio.Queue) -> None: + def _discard_queued_messages( + queue: asyncio.Queue, + error: Optional[Exception] = None, + ) -> None: """丢弃会话队列时同步结束等待任务完成的调用方。""" while not queue.empty(): try: @@ -2731,7 +2763,10 @@ class AgentManager: except asyncio.QueueEmpty: break if task.completion_future and not task.completion_future.done(): - task.completion_future.cancel() + if error is None: + task.completion_future.cancel() + else: + task.completion_future.set_exception(error) queue.task_done() @staticmethod @@ -2810,6 +2845,9 @@ class AgentManager: if task.notification_callback is not None and hasattr(agent, "set_notification_callback"): agent.set_notification_callback(task.notification_callback) + if task.agent_setup is not None: + task.agent_setup(agent) + process_kwargs = { "images": task.images, "files": task.files, @@ -2824,6 +2862,11 @@ class AgentManager: 与 clear_session 不同，此方法不会销毁Agent实例或清除记忆， 用户可以在停止后继续对话。 """ + async with self._lifecycle_lock: + return await self._stop_current_task_locked(session_id) + + async def _stop_current_task_locked(self, session_id: str): + """在 lifecycle 互斥域内停止会话 worker。""" stopped = False worker = self._session_workers.get(session_id) @@ -2831,7 +2874,7 @@ class AgentManager: if queue and self._session_queues.get(session_id) is queue: self._session_queues.pop(session_id, None) - # 先摘下旧队列；清理期间的新消息进入新队列，但等待旧 worker 完全退出后再执行。 + # 先摘下旧队列再等待 worker 退出；lifecycle 锁保证清理期间不会并发建立新队列。 if worker: worker.cancel() if queue: @@ -2869,6 +2912,11 @@ class AgentManager: """ 清空会话 """ + async with self._lifecycle_lock: + await self._clear_session_locked(session_id=session_id, user_id=user_id) + + async def _clear_session_locked(self, session_id: str, user_id: str) -> None: + """在 lifecycle 互斥域内释放会话、Agent 与记忆。""" self._session_last_used.pop(session_id, None) # 取消该会话的worker if session_id in self._session_workers: @@ -2879,8 +2927,10 @@ class AgentManager: pass self._session_workers.pop(session_id, None) # noqa - # 清理队列 - self._session_queues.pop(session_id, None) + # 清理队列时同步结束未执行请求，避免 wait_for_completion 调用方永久等待。 + queue = self._session_queues.pop(session_id, None) + if queue: + self._discard_queued_messages(queue) # 清理agent if session_id in self.active_agents: @@ -2890,8 +2940,8 @@ class AgentManager: memory_manager.clear_memory(session_id, user_id) logger.info(f"会话 {session_id} 的记忆已清空") - @staticmethod async def run_background_prompt( + self, message: str, session_prefix: str = "__agent_background", output_callback: Optional[Callable[[str], None]] = None, @@ -2909,22 +2959,21 @@ class AgentManager: elif allow_message_tools is None: allow_message_tools = True - agent = MoviePilotAgent( - session_id=session_id, - user_id=user_id, - channel=None, - source=None, - username=settings.SUPERUSER, - replay_mode=reply_mode, - output_callback=output_callback, - allow_message_tools=allow_message_tools, - ) - try: - await agent.process(message) + await self.process_message( + session_id=session_id, + user_id=user_id, + message=message, + channel=None, + source=None, + username=settings.SUPERUSER, + reply_mode=reply_mode, + output_callback=output_callback, + allow_message_tools=allow_message_tools, + wait_for_completion=True, + ) finally: - await agent.cleanup() - memory_manager.clear_memory(session_id, user_id) + await self.clear_session(session_id=session_id, user_id=user_id) async def execute_scheduled_task( self, diff --git a/app/agent/runtime_loader.py b/app/agent/runtime_loader.py new file mode 100644 index 000000000..1006970d8 --- /dev/null +++ b/app/agent/runtime_loader.py @@ -0,0 +1,137 @@ +"""Agent 重量级 canonical 对象的轻量首用入口。""" + +from __future__ import annotations + +import threading +from typing import Any + +from app.agent.capabilities import ( + AGENT_ENTRYPOINT_KIND, + AGENT_MANAGER_CAPABILITY_ID, + AGENT_SERVICE_CAPABILITY_ID, + AGENT_SERVICE_KIND, + MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID, + TOOL_FACTORY_CAPABILITY_ID, +) +from app.agent.capabilities.adapter import ( + AgentEntrypointAdapter, + AgentServiceAdapter, + build_agent_capability_registry, + should_run_agent_service, +) +from app.runtime.capabilities.model import CapabilityMaterializationState +from app.runtime.capabilities.runtime import CapabilityRuntime + + +_runtime_lock = threading.RLock() +_agent_runtime: CapabilityRuntime | None = None + + +def _build_agent_runtime() -> CapabilityRuntime: + """装配 Agent Runtime；构建阶段只解析 manifests。""" + return CapabilityRuntime( + build_agent_capability_registry(), + adapters={ + AGENT_ENTRYPOINT_KIND: AgentEntrypointAdapter(), + AGENT_SERVICE_KIND: AgentServiceAdapter(), + }, + ) + + +def _ensure_runtime() -> CapabilityRuntime: + """返回进程唯一 Runtime，同进程关闭后不重新创建。""" + global _agent_runtime + with _runtime_lock: + if _agent_runtime is None: + _agent_runtime = _build_agent_runtime() + return _agent_runtime + + +def _materialize_entrypoint(capability_id: str) -> Any: + """通过通用 Runtime 完成并发 single-flight 物化，不声明资源运行态。""" + return _ensure_runtime().materialize( + capability_id, + reason="agent_entrypoint_first_use", + ) + + +def get_agent_manager() -> Any: + """返回 canonical Agent Manager；关闭门禁生效后稳定拒绝首用。""" + return _materialize_entrypoint(AGENT_MANAGER_CAPABILITY_ID) + + +async def reconcile_agent_service( + *, + reason: str, + changed_keys: set[str] | None = None, + retry: bool = False, +) -> Any | None: + """按 manifest watch/selector 协调唯一 Agent Service 生命周期。""" + runtime = _ensure_runtime() + spec = runtime.get_spec(AGENT_SERVICE_CAPABILITY_ID) + if spec is None: + raise RuntimeError("缺少 agent.service capability") + if changed_keys is not None and not changed_keys.intersection(spec.watch): + return runtime.get_running(AGENT_SERVICE_CAPABILITY_ID) + if not should_run_agent_service(spec): + # stop_async 会等待并发首启后再撤销实例；未物化能力则保持零导入。 + await runtime.stop_async( + AGENT_SERVICE_CAPABILITY_ID, + reason=reason, + ) + return None + return await runtime.activate_async( + AGENT_SERVICE_CAPABILITY_ID, + reason=reason, + retry=retry, + ) + + +async def activate_agent_service(*, retry: bool = False) -> Any | None: + """执行启动期协调；selector 未启用时保持 service 未物化。""" + return await reconcile_agent_service( + reason="agent_service_startup_reconcile", + retry=retry, + ) + + +def get_running_agent_manager() -> Any | None: + """只读返回 RUNNING Agent Service；未构建 Runtime 时不触发声明发现。""" + with _runtime_lock: + runtime = _agent_runtime + if runtime is None: + return None + return runtime.get_running(AGENT_SERVICE_CAPABILITY_ID) + + +def get_moviepilot_agent_type() -> type: + """返回 canonical MoviePilotAgent 类型。""" + agent_type = _materialize_entrypoint(MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID) + if not isinstance(agent_type, type): + raise TypeError("MoviePilot Agent entrypoint 必须是类型") + return agent_type + + +def get_tool_factory() -> type: + """返回 canonical 工具工厂类型。""" + factory_type = _materialize_entrypoint(TOOL_FACTORY_CAPABILITY_ID) + if not isinstance(factory_type, type): + raise TypeError("Agent Tool Factory entrypoint 必须是类型") + return factory_type + + +def is_tool_factory_materialized() -> bool: + """只读判断工具工厂是否已解析；未建 Runtime 时不触发发现或导入。""" + with _runtime_lock: + runtime = _agent_runtime + if runtime is None: + return False + return ( + runtime.snapshot(TOOL_FACTORY_CAPABILITY_ID).materialization + is CapabilityMaterializationState.RESOLVED + ) + + +async def begin_agent_shutdown() -> None: + """不可逆关闭首用闸门，并等待全部同步及异步能力释放。""" + await _ensure_runtime().shutdown_async(reason="application_shutdown") diff --git a/app/agent/tools/base.py b/app/agent/tools/base.py index 8769b7f51..7b3230f19 100644 --- a/app/agent/tools/base.py +++ b/app/agent/tools/base.py @@ -5,12 +5,11 @@ from abc import ABCMeta, abstractmethod from concurrent.futures import ThreadPoolExecutor from functools import partial from pathlib import Path -from typing import Any, Callable, ClassVar, Optional +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Protocol from langchain_core.tools import BaseTool from pydantic import PrivateAttr -from app.agent.callback import StreamingHandler from app.agent.policy.sanitizer import ( summarize_error, summarize_input, @@ -25,6 +24,54 @@ from app.runtime.log import logger from app.schemas import Notification from app.schemas.types import MessageChannel, NotificationType +if TYPE_CHECKING: + from app.agent.callback import StreamingHandler as _StreamingHandlerProtocol +else: + class _StreamingHandlerProtocol(Protocol): + """工具执行仅依赖的流式缓冲合同。""" + + @property + def is_streaming(self) -> bool: + """是否正在收集流式输出。""" + ... + + @property + def is_auto_flushing(self) -> bool: + """是否由渠道编辑能力自动刷新缓冲。""" + ... + + @property + def last_buffer_char(self) -> str: + """返回缓冲区最后一个字符。""" + ... + + def emit(self, token: str) -> str: + """追加流式文本并返回实际追加内容。""" + ... + + async def take(self) -> str: + """取出并清空当前缓冲内容。""" + ... + + def record_tool_call( + self, + tool_name: str, + tool_message: Optional[str] = None, + tool_kwargs: Optional[dict[str, Any]] = None, + ) -> None: + """记录一次待汇总的工具调用。""" + ... + + + +def __getattr__(name: str) -> Any: + """显式访问历史 StreamingHandler 符号时返回 canonical 实现。""" + if name == "StreamingHandler": + from app.agent.callback import StreamingHandler + + return StreamingHandler + raise AttributeError(f"module 'app.agent.tools.base' has no attribute {name!r}") + class ToolChain(ChainBase): pass @@ -206,7 +253,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta): _channel: Optional[str] = PrivateAttr(default=None) _source: Optional[str] = PrivateAttr(default=None) _username: Optional[str] = PrivateAttr(default=None) - _stream_handler: Optional[StreamingHandler] = PrivateAttr(default=None) + _stream_handler: Optional[_StreamingHandlerProtocol] = PrivateAttr(default=None) _require_admin: bool = PrivateAttr(default=False) _agent_context: dict = PrivateAttr(default_factory=dict) @@ -387,7 +434,9 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta): self._source = source self._username = username - def set_stream_handler(self, stream_handler: StreamingHandler): + def set_stream_handler( + self, stream_handler: Optional[_StreamingHandlerProtocol] + ) -> None: """ 设置回调处理器 """ @@ -642,3 +691,10 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta): save_history=False, ) ) + + +# 普通导入保持 callback 冷态；显式导入或历史星号导入仍解析真实类。 +__all__ = sorted( + {name for name in globals() if not name.startswith("_")} + | {"StreamingHandler"} +) diff --git a/app/agent/tools/manager.py b/app/agent/tools/manager.py index 5e3420884..8be289458 100644 --- a/app/agent/tools/manager.py +++ b/app/agent/tools/manager.py @@ -1,24 +1,16 @@ +from __future__ import annotations + import json import threading import uuid -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from app.agent.policy import ( - DEFAULT_TOOL_POLICY_ORCHESTRATOR, - AgentToolPolicyOrchestrator, - AuthSource, - PrincipalType, - ToolOrigin, - ToolPolicyContext, - call_policy_hook, - summarize_error, -) -from app.agent.tools.base import ToolExecutionTimeoutError, format_tool_result_for_agent -from app.agent.tools.factory import MoviePilotToolFactory -from app.agent.tools.catalog import ToolCatalogSnapshot -from app.runtime.extensions.plugin_manager import PluginManager from app.runtime.log import logger +if TYPE_CHECKING: + from app.agent.policy import AgentToolPolicyOrchestrator, ToolPolicyContext + from app.agent.tools.catalog import ToolCatalogSnapshot + class ToolDefinition: """ @@ -53,31 +45,33 @@ class MoviePilotToolsManager: self.user_id = user_id self.session_id = session_id self.is_admin = is_admin - self.policy_orchestrator = ( - policy_orchestrator or DEFAULT_TOOL_POLICY_ORCHESTRATOR - ) - self._policy_context = ToolPolicyContext( - session_id=session_id, - user_id=user_id, - origin=ToolOrigin.OPERATOR_DIRECT, - principal_type=PrincipalType.SYSTEM_ADMIN_INTEGRATION, - auth_source=AuthSource.API_TOKEN, - channel=None, - source="api", - agent_context={"is_admin": is_admin}, - ) + self.policy_orchestrator = policy_orchestrator + self._policy_context: Optional[ToolPolicyContext] = None self.tools: List[Any] = [] self.catalog: Optional[ToolCatalogSnapshot] = None self._tools_lock = threading.Lock() self._plugin_agent_tools_revision = -1 - self._load_tools() + self._catalog_materialized = False + self._catalog_managed_by_factory = False - def _load_tools(self) -> None: + @staticmethod + def _summarize_error(error: Exception) -> str: + """仅在错误路径加载策略脱敏器，保持默认导入轻量。""" + from app.agent.policy import summarize_error + + return summarize_error(error) + + def _load_tools_locked(self) -> None: """ - 加载所有MoviePilot工具 + 在 manager 锁内加载所有 MoviePilot 工具。 + + 工厂负责插件 revision 前后稳定窗口；manager 只发布完整快照，避免 + 并发调用观察到一半刷新后的工具列表。 """ + from app.agent.runtime_loader import get_tool_factory + try: - catalog = MoviePilotToolFactory.create_catalog( + catalog = get_tool_factory().create_catalog( session_id=self.session_id, user_id=self.user_id, channel=None, @@ -89,17 +83,43 @@ class MoviePilotToolsManager: self.catalog = catalog self.tools = catalog.tools self._plugin_agent_tools_revision = catalog.plugin_revision + self._catalog_materialized = True + self._catalog_managed_by_factory = True logger.info(f"成功加载 {len(self.tools)} 个工具") except Exception as e: - logger.error(f"加载工具失败: {summarize_error(e)}") + logger.error(f"加载工具失败: {self._summarize_error(e)}") self.tools = [] self.catalog = None self._plugin_agent_tools_revision = -1 + self._catalog_materialized = False + self._catalog_managed_by_factory = False + + def _load_tools(self) -> None: + """兼容显式刷新入口，并保证外部调用仍原子发布完整目录。""" + with self._tools_lock: + self._load_tools_locked() def _ensure_tools_current(self) -> None: """ - 在插件工具注册表变化后惰性刷新工具实例。 + 首次使用时加载目录，并在插件注册表变化后惰性刷新工具实例。 """ + # 调用方可能显式注入工具实例；这些实例仍由调用方拥有，manager 不应 + # 在第一次查询时用全量目录覆盖它们。 + if not self._catalog_materialized and self.tools: + self._catalog_materialized = True + return + + if self._catalog_materialized and not self._catalog_managed_by_factory: + return + + if not self._catalog_materialized: + with self._tools_lock: + if not self._catalog_materialized: + self._load_tools_locked() + return + + from app.runtime.extensions.plugin_manager import PluginManager + plugin_manager = PluginManager() if ( self._plugin_agent_tools_revision @@ -112,7 +132,41 @@ class MoviePilotToolsManager: == plugin_manager.get_plugin_agent_tools_revision() ): return - self._load_tools() + self._load_tools_locked() + + def _ensure_policy_runtime( + self, + ) -> tuple[AgentToolPolicyOrchestrator, ToolPolicyContext]: + """返回 direct 入口的策略对象，仅在真实工具调用前完成构造。""" + policy_orchestrator = self.policy_orchestrator + policy_context = self._policy_context + if policy_orchestrator is not None and policy_context is not None: + return policy_orchestrator, policy_context + + from app.agent.policy import ( + DEFAULT_TOOL_POLICY_ORCHESTRATOR, + AuthSource, + PrincipalType, + ToolOrigin, + ToolPolicyContext, + ) + + if policy_orchestrator is None: + policy_orchestrator = DEFAULT_TOOL_POLICY_ORCHESTRATOR + if policy_context is None: + policy_context = ToolPolicyContext( + session_id=self.session_id, + user_id=self.user_id, + origin=ToolOrigin.OPERATOR_DIRECT, + principal_type=PrincipalType.SYSTEM_ADMIN_INTEGRATION, + auth_source=AuthSource.API_TOKEN, + channel=None, + source="api", + agent_context={"is_admin": self.is_admin}, + ) + self.policy_orchestrator = policy_orchestrator + self._policy_context = policy_context + return policy_orchestrator, policy_context def list_tools(self) -> List[ToolDefinition]: """ @@ -122,8 +176,10 @@ class MoviePilotToolsManager: 工具定义列表 """ self._ensure_tools_current() + with self._tools_lock: + tools = list(self.tools) tools_list = [] - for tool in self.tools: + for tool in tools: if getattr(tool, "_require_admin", False) and not self.is_admin: continue # 获取工具的输入参数模型 @@ -156,26 +212,31 @@ class MoviePilotToolsManager: 工具实例，如果未找到返回None """ self._ensure_tools_current() - return next( - (tool for tool in self.tools if tool.name == tool_name), - None, - ) + with self._tools_lock: + return next( + (tool for tool in self.tools if tool.name == tool_name), + None, + ) def get_strict_tool(self, tool_name: str) -> Optional[Any]: """按当前目录唯一身份解析严格调用，重名时稳定失败。""" self._ensure_tools_current() - if self.catalog is None or [ - id(tool) for tool in self.catalog.tools - ] != [id(tool) for tool in self.tools]: - self.catalog = ToolCatalogSnapshot.from_tools( - self.tools, - plugin_revision=self._plugin_agent_tools_revision, - factory_revision=MoviePilotToolFactory.catalog_factory_revision(), - ) - if self.catalog is None: - return None - entry = self.catalog.resolve_unique(tool_name) - return entry.tool if entry else None + with self._tools_lock: + if self.catalog is None or [ + id(tool) for tool in self.catalog.tools + ] != [id(tool) for tool in self.tools]: + from app.agent.runtime_loader import get_tool_factory + from app.agent.tools.catalog import ToolCatalogSnapshot + + self.catalog = ToolCatalogSnapshot.from_tools( + self.tools, + plugin_revision=self._plugin_agent_tools_revision, + factory_revision=get_tool_factory().catalog_factory_revision(), + ) + if self.catalog is None: + return None + entry = self.catalog.resolve_unique(tool_name) + return entry.tool if entry else None @staticmethod def _resolve_field_schema(field_info: Dict[str, Any]) -> Dict[str, Any]: @@ -265,7 +326,7 @@ class MoviePilotToolsManager: schema = args_schema.model_json_schema() properties = schema.get("properties", {}) except Exception as e: - logger.warning(f"获取工具schema失败: {summarize_error(e)}") + logger.warning(f"获取工具schema失败: {MoviePilotToolsManager._summarize_error(e)}") return arguments # 规范化参数 @@ -320,7 +381,14 @@ class MoviePilotToolsManager: ) return error_msg + from app.agent.policy import call_policy_hook + from app.agent.tools.base import ( + ToolExecutionTimeoutError, + format_tool_result_for_agent, + ) + observation = None + policy_orchestrator = None try: permission_error = self._check_tool_permission(tool_instance) if permission_error: @@ -328,11 +396,12 @@ class MoviePilotToolsManager: # 规范化参数类型 normalized_arguments = self._normalize_arguments(tool_instance, arguments) - self._policy_context.agent_context["is_admin"] = self.is_admin + policy_orchestrator, policy_context = self._ensure_policy_runtime() + policy_context.agent_context["is_admin"] = self.is_admin observation = call_policy_hook( "start", - self.policy_orchestrator.start, - context=self._policy_context, + policy_orchestrator.start, + context=policy_context, tool=tool_instance, arguments=normalized_arguments, ) @@ -346,28 +415,29 @@ class MoviePilotToolsManager: max_chars=getattr(tool_instance, "result_max_chars", None), ) except ToolExecutionTimeoutError as e: - if observation: - call_policy_hook("fail", self.policy_orchestrator.fail, observation, e) - logger.warning(summarize_error(e)) + if observation is not None and policy_orchestrator is not None: + call_policy_hook("fail", policy_orchestrator.fail, observation, e) + error_summary = self._summarize_error(e) + logger.warning(error_summary) return format_tool_result_for_agent( - summarize_error(e), + error_summary, tool_name=tool_name, max_chars=getattr(tool_instance, "result_max_chars", None), ) except Exception as e: - if observation: - call_policy_hook("fail", self.policy_orchestrator.fail, observation, e) - error_summary = summarize_error(e) + if observation is not None and policy_orchestrator is not None: + call_policy_hook("fail", policy_orchestrator.fail, observation, e) + error_summary = self._summarize_error(e) logger.error(f"调用工具 {tool_name} 时发生错误: {error_summary}") error_msg = json.dumps( {"error": f"调用工具 '{tool_name}' 时发生错误: {error_summary}"}, ensure_ascii=False, ) return error_msg - if observation: + if observation is not None and policy_orchestrator is not None: call_policy_hook( "finish", - self.policy_orchestrator.finish, + policy_orchestrator.finish, observation, str_result, ) diff --git a/app/api/endpoints/agent.py b/app/api/endpoints/agent.py index fb53ecd2b..d77504d93 100644 --- a/app/api/endpoints/agent.py +++ b/app/api/endpoints/agent.py @@ -20,10 +20,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from app import schemas from app.api.response import ResponseAPIRouter -from app.agent.callback import StreamingHandler -from app.agent.orchestrator import MoviePilotAgent, ReplyMode, agent_manager +from app.agent.contracts import ReplyMode, build_display_message from app.agent.llm.capability import AgentCapabilityManager from app.agent.mcp import agent_mcp_manager +from app.agent.runtime_loader import ( + get_moviepilot_agent_type, + get_running_agent_manager, +) from app.chain.message import MessageChain from app.command import Command from app.runtime.config import global_vars, settings @@ -254,7 +257,7 @@ async def test_agent_mcp_server( ) -class _WebAgentStreamingHandler(StreamingHandler): +class _WebAgentStreamingHandlerMixin: """ Web 前端专用流式处理器，将工具提示和文本统一回调给 SSE。 """ @@ -342,7 +345,28 @@ class _WebAgentStreamingHandler(StreamingHandler): return True -class _WebAgentMoviePilotAgent(MoviePilotAgent): +def _get_web_agent_streaming_handler_type() -> type: + """首次构造 Web Agent 时才解析完整流式处理器实现。""" + global _WEB_AGENT_STREAMING_HANDLER_TYPE + if _WEB_AGENT_STREAMING_HANDLER_TYPE is not None: + return _WEB_AGENT_STREAMING_HANDLER_TYPE + with _WEB_AGENT_STREAMING_HANDLER_TYPE_LOCK: + if _WEB_AGENT_STREAMING_HANDLER_TYPE is None: + from app.agent.callback import StreamingHandler + + _WEB_AGENT_STREAMING_HANDLER_TYPE = type( + "_RuntimeWebAgentStreamingHandler", + (_WebAgentStreamingHandlerMixin, StreamingHandler), + {"__module__": __name__}, + ) + return _WEB_AGENT_STREAMING_HANDLER_TYPE + + +_WEB_AGENT_STREAMING_HANDLER_TYPE_LOCK = Lock() +_WEB_AGENT_STREAMING_HANDLER_TYPE: Optional[type] = None + + +class _WebAgentMoviePilotAgentMixin: """ Web 前端专用 Agent，强制使用流式推理。 """ @@ -355,7 +379,9 @@ class _WebAgentMoviePilotAgent(MoviePilotAgent): ) -> None: super().__init__(*args, **kwargs) self._notification_callback = notification_callback - self.stream_handler = _WebAgentStreamingHandler(self._emit_output) + self.stream_handler = _get_web_agent_streaming_handler_type()( + self._emit_output + ) def _should_stream(self) -> bool: """Web 对话实时输出，复用会话执行后台任务时改用非流式广播。""" @@ -381,7 +407,9 @@ class _WebAgentMoviePilotAgent(MoviePilotAgent): :param output_callback: 当前请求的输出回调 """ self.output_callback = output_callback - if output_callback and isinstance(self.stream_handler, _WebAgentStreamingHandler): + if output_callback and isinstance( + self.stream_handler, _WebAgentStreamingHandlerMixin + ): self.stream_handler.set_emit_callback(self._emit_output) async def _is_system_admin_context(self) -> bool: @@ -420,6 +448,30 @@ class _WebAgentMoviePilotAgent(MoviePilotAgent): logger.debug(f"Web智能体输出回调失败: {e}") +def _build_web_agent_type(agent_base_type: type) -> type: + """为 Web 通道组合唯一的运行时 Agent 类型。""" + return type( + "_RuntimeWebAgentMoviePilotAgent", + (_WebAgentMoviePilotAgentMixin, agent_base_type), + {"__module__": __name__}, + ) + + +_WEB_AGENT_TYPE_LOCK = Lock() +_WEB_AGENT_TYPE: Optional[type] = None + + +def _get_web_agent_type() -> type: + """在真实 Web Agent 调用边界 single-flight 解析运行时类型。""" + global _WEB_AGENT_TYPE + if _WEB_AGENT_TYPE is not None: + return _WEB_AGENT_TYPE + with _WEB_AGENT_TYPE_LOCK: + if _WEB_AGENT_TYPE is None: + _WEB_AGENT_TYPE = _build_web_agent_type(get_moviepilot_agent_type()) + return _WEB_AGENT_TYPE + + def _build_web_agent_session_id(user: User, session_id: Optional[str]) -> str: """ 构建前端 Agent 会话 ID。 @@ -1131,7 +1183,7 @@ def _build_web_agent_display_message_from_events( :param events: 已转换的 WebAgent SSE 事件列表 :return: 可持久化的助手展示消息 """ - message = MoviePilotAgent.build_display_message( + message = build_display_message( role="assistant", status="streaming", ) @@ -1725,7 +1777,8 @@ async def get_agent_chat_session( if server_session_id != session_id: chat = await _get_accessible_agent_chat(oper, server_session_id, current_user) if not chat: - if agent_manager.is_session_busy(server_session_id): + manager = get_running_agent_manager() + if manager and manager.is_session_busy(server_session_id): return schemas.Response( success=True, data={ @@ -1737,7 +1790,10 @@ async def get_agent_chat_session( ) return schemas.Response(success=False, message="会话不存在或无权访问") data = AgentChatOper.to_detail(chat) - data["is_processing"] = agent_manager.is_session_busy(chat.session_id) + manager = get_running_agent_manager() + data["is_processing"] = bool( + manager and manager.is_session_busy(chat.session_id) + ) return schemas.Response(success=True, data=data) @@ -1836,7 +1892,8 @@ async def stop_web_agent_session_task( if chat and not _can_access_agent_chat(chat, current_user): return schemas.Response(success=False, message="会话不存在或无权访问") - stopped = await agent_manager.stop_current_task(server_session_id) + manager = get_running_agent_manager() + stopped = await manager.stop_current_task(server_session_id) if manager else False return schemas.Response( success=True, data={"stopped": stopped}, @@ -1881,7 +1938,8 @@ async def web_agent_stream( ) is_secret_confirmation_control = ( is_secret_confirmation_candidate - and agent_manager.matches_secret_confirmation( + and (manager := get_running_agent_manager()) is not None + and manager.matches_secret_confirmation( session_id, str(current_user.id), channel=MessageChannel.WebAgent.value, @@ -1943,7 +2001,7 @@ async def web_agent_stream( display_messages = [] if payload.echo_user: display_messages.append( - MoviePilotAgent.build_display_message( + build_display_message( role="user", content=display_prompt or prompt, attachments=user_attachments, @@ -2038,6 +2096,19 @@ async def web_agent_stream( media_type="text/event-stream", ) + manager = get_running_agent_manager() + if manager is None: + return StreamingResponse( + iter([ + _build_web_agent_sse( + "error", + {"message": "智能助手服务尚未就绪，请稍后重试。"}, + locale=locale, + ) + ]), + media_type="text/event-stream", + ) + transcript = _transcribe_web_agent_audio_refs(payload.audio_refs or []) prompt = _merge_web_agent_prompt_with_transcript(prompt, transcript) display_prompt = _merge_web_agent_prompt_with_transcript(display_prompt, transcript) @@ -2077,7 +2148,7 @@ async def web_agent_stream( ) display_messages = [] if payload.echo_user and not is_secret_confirmation_control: - user_display_message = MoviePilotAgent.build_display_message( + user_display_message = build_display_message( role="user", content=display_prompt or prompt, attachments=user_attachments, @@ -2085,7 +2156,7 @@ async def web_agent_stream( if payload.choice_selection: user_display_message["choice_selection"] = payload.choice_selection display_messages.append(user_display_message) - assistant_display_message = MoviePilotAgent.build_display_message( + assistant_display_message = build_display_message( role="assistant", status="streaming", ) @@ -2132,7 +2203,10 @@ async def web_agent_stream( async def run_agent() -> None: """后台执行 Agent，并将结果写入事件队列。""" try: - await agent_manager.process_message( + runtime_manager = get_running_agent_manager() + if runtime_manager is None: + raise RuntimeError("智能助手服务尚未就绪，请稍后重试。") + await runtime_manager.process_message( session_id=session_id, user_id=str(current_user.id), message=prompt, @@ -2151,9 +2225,12 @@ async def web_agent_stream( else None ), notification_callback=notification_callback, - agent_factory=_WebAgentMoviePilotAgent, + agent_factory=_get_web_agent_type(), wait_for_completion=True, ) + except asyncio.CancelledError: + # 显式停止会话沿用正常终止语义；服务关闭会由 manager 的稳定异常分支处理。 + pass except Exception as err: logger.error(f"Web智能助手执行失败: {str(err)}") error_event = { diff --git a/app/api/endpoints/anthropic.py b/app/api/endpoints/anthropic.py index 2a9d619a0..f18cbaf97 100644 --- a/app/api/endpoints/anthropic.py +++ b/app/api/endpoints/anthropic.py @@ -9,16 +9,17 @@ from fastapi.responses import JSONResponse, StreamingResponse from app import schemas from app.api.endpoints.openai import ( MODEL_ID, - _CollectingMoviePilotAgent, + _is_manager_unavailable, + _run_managed_agent, ) from app.api.openai_utils import ( build_anthropic_messages, build_prompt, build_session_id, ) +from app.agent.runtime_loader import get_running_agent_manager from app.runtime.config import settings from app.application.security.access import anthropic_api_key_header -from app.schemas.types import MessageChannel ANTHROPIC_ERROR_RESPONSES = { 400: {"model": schemas.AnthropicErrorResponse, "description": "请求格式错误"}, @@ -60,19 +61,31 @@ def _check_auth(api_key: Optional[str]) -> Optional[JSONResponse]: async def _stream_anthropic_response( - agent: _CollectingMoviePilotAgent, + manager, + session_id: str, + user_id: str, prompt: str, images: List[str], ) -> AsyncIterator[str]: event_queue: asyncio.Queue = asyncio.Queue() - if hasattr(agent.stream_handler, "bind_queue"): - agent.stream_handler.bind_queue(event_queue) message_id = f"msg_{uuid.uuid4().hex}" async def _run_agent(): try: - await agent.process(prompt, images=images, files=None) + await _run_managed_agent( + manager=manager, + session_id=session_id, + user_id=user_id, + username="anthropic-client", + source="anthropic", + prompt=prompt, + images=images, + stream_mode=True, + event_queue=event_queue, + ) + except asyncio.CancelledError: + await event_queue.put({"error": "MoviePilot AI agent is unavailable."}) except Exception as exc: await event_queue.put({"error": str(exc)}) finally: @@ -87,7 +100,12 @@ async def _stream_anthropic_response( if item is None: break if isinstance(item, dict) and item.get("error"): - raise RuntimeError(str(item["error"])) + yield ( + "event: error\n" + f"data: {json.dumps({'type': 'error', 'error': {'type': 'api_error', 'message': str(item['error'])}}, ensure_ascii=False)}\n\n" + ) + yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'}, ensure_ascii=False)}\n\n" + return text = str(item or "") if not text: continue @@ -96,6 +114,7 @@ async def _stream_anthropic_response( yield f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': 0}}, ensure_ascii=False)}\n\n" yield f"event: message_stop\ndata: {json.dumps({'type': 'message_stop'}, ensure_ascii=False)}\n\n" finally: + await manager.clear_session(session_id=session_id, user_id=user_id) if not task.done(): task.cancel() try: @@ -132,6 +151,13 @@ async def messages( 503, error_type="api_error", ) + manager = get_running_agent_manager() + if manager is None: + return _anthropic_error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="api_error", + ) normalized_messages = build_anthropic_messages(payload.system, payload.messages) try: @@ -141,19 +167,15 @@ async def messages( session_seed = anthropic_version or "anthropic" session_id = build_session_id(f"{session_seed}:{uuid.uuid4().hex}", SESSION_PREFIX) - # 兼容接口的 API_TOKEN 客户端按管理员级 MoviePilot Agent 集成处理。 - agent = _CollectingMoviePilotAgent( - session_id=session_id, - user_id=session_id, - channel=MessageChannel.Web.value, - source="anthropic", - username="anthropic-client", - stream_mode=payload.stream, - ) - if payload.stream: return StreamingResponse( - _stream_anthropic_response(agent=agent, prompt=prompt, images=images), + _stream_anthropic_response( + manager=manager, + session_id=session_id, + user_id=session_id, + prompt=prompt, + images=images, + ), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -162,14 +184,32 @@ async def messages( }, ) + collected_messages = [] try: - result = await agent.process(prompt, images=images, files=None) + result, collected_messages = await _run_managed_agent( + manager=manager, + session_id=session_id, + user_id=session_id, + username="anthropic-client", + source="anthropic", + prompt=prompt, + images=images, + stream_mode=False, + ) except Exception as exc: + if _is_manager_unavailable(exc): + return _anthropic_error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="api_error", + ) return _anthropic_error_response(str(exc), 500, error_type="api_error") + finally: + await manager.clear_session(session_id=session_id, user_id=session_id) content = "\n\n".join( message.strip() - for message in agent.collected_messages + for message in collected_messages if message and message.strip() ).strip() if not content and result: diff --git a/app/api/endpoints/history.py b/app/api/endpoints/history.py index b6f1f5560..d14cdb13d 100644 --- a/app/api/endpoints/history.py +++ b/app/api/endpoints/history.py @@ -9,7 +9,8 @@ from sqlalchemy.orm import Session from app import schemas from app.api.response import ResponseAPIRouter -from app.agent.orchestrator import ReplyMode, agent_manager +from app.agent.contracts import ReplyMode +from app.agent.runtime_loader import get_running_agent_manager from app.agent.prompt.transfer_redo import ( build_batch_manual_redo_prompt, build_manual_redo_prompt, @@ -31,6 +32,7 @@ from app.runtime.progress import ProgressHelper from app.application.history import clear_transfer_failures from app.schemas.types import EventType from app.foundation.text import cut as jieba_cut +from app.runtime.log import logger router = ResponseAPIRouter() @@ -58,7 +60,11 @@ def _start_ai_redo_task(history_id: int, prompt: str, progress_key: str): async def runner(): try: - await agent_manager.run_background_prompt( + manager = get_running_agent_manager() + if manager is None: + logger.warning("智能助手服务未运行，跳过单条整理历史 AI 重做") + raise RuntimeError("智能助手服务未运行") + await manager.run_background_prompt( message=prompt, session_prefix=f"__agent_manual_redo_{history_id}", output_callback=update_output, @@ -103,7 +109,11 @@ def _start_batch_ai_redo_task( async def runner(): try: - await agent_manager.run_background_prompt( + manager = get_running_agent_manager() + if manager is None: + logger.warning("智能助手服务未运行，跳过批量整理历史 AI 重做") + raise RuntimeError("智能助手服务未运行") + await manager.run_background_prompt( message=prompt, session_prefix="__agent_manual_redo_batch", output_callback=update_output, diff --git a/app/api/endpoints/llm.py b/app/api/endpoints/llm.py index 33eeb0057..711332e2d 100644 --- a/app/api/endpoints/llm.py +++ b/app/api/endpoints/llm.py @@ -5,13 +5,19 @@ from fastapi.responses import HTMLResponse from app import schemas from app.api.response import ResponseAPIRouter -from app.agent.llm import LLMProviderManager, render_auth_result_html from app.db.models import User from app.api.deps import get_current_active_superuser_async router = ResponseAPIRouter() +def _get_llm_provider_manager_type() -> type: + """在真实管理请求边界解析 provider 运行时。""" + from app.agent.llm.provider import LLMProviderManager + + return LLMProviderManager + + @router.post( "/manage", summary="LLM提供商统一管理", @@ -37,7 +43,7 @@ async def manage_provider( "callback_url", str(request.url_for("llm_provider_auth_callback", provider_id=payload.target)), ) - result = await LLMProviderManager().provider_manage( + result = await _get_llm_provider_manager_type()().provider_manage( payload.target, payload.action, **params ) return schemas.Response( @@ -70,11 +76,13 @@ async def llm_provider_auth_callback( """ 处理需要浏览器回跳的 OAuth provider。 """ - success, message = await LLMProviderManager().handle_chatgpt_callback( + success, message = await _get_llm_provider_manager_type()().handle_chatgpt_callback( provider_id, code, state, error, error_description, ) + from app.agent.llm.provider import render_auth_result_html + return HTMLResponse(content=render_auth_result_html(success, message)) diff --git a/app/api/endpoints/openai.py b/app/api/endpoints/openai.py index 18458ead6..22f13eb6e 100644 --- a/app/api/endpoints/openai.py +++ b/app/api/endpoints/openai.py @@ -2,6 +2,7 @@ import asyncio import json import time import uuid +from threading import Lock from typing import AsyncIterator, List, Optional, Tuple from fastapi import APIRouter, Request, Security @@ -15,8 +16,11 @@ from app.api.openai_utils import ( build_responses_input, build_session_id, ) -from app.agent.callback import StreamingHandler -from app.agent.orchestrator import MoviePilotAgent +from app.agent.runtime_loader import ( + get_moviepilot_agent_type, + get_running_agent_manager, +) +from app.agent.contracts import ReplyMode from app.runtime.config import settings from app.application.security.access import openai_bearer_scheme from app.schemas.types import MessageChannel @@ -35,7 +39,7 @@ MODEL_ID = "moviepilot-agent" SESSION_PREFIX = "openai:" -class _CollectingMoviePilotAgent(MoviePilotAgent): +class _CollectingMoviePilotAgentMixin: """ 捕获 Agent 最终输出，避免再通过消息渠道二次发送。 """ @@ -45,11 +49,38 @@ class _CollectingMoviePilotAgent(MoviePilotAgent): self.collected_messages: List[str] = [] self.stream_mode = stream_mode if stream_mode: - self.stream_handler = _OpenAIStreamingHandler() + self.stream_handler = _get_openai_streaming_handler_type()() def _should_stream(self) -> bool: return self.stream_mode + def configure_protocol_request( + self, + *, + stream_mode: bool, + event_queue: Optional[asyncio.Queue], + ) -> None: + """切换请求级输出目标，并保持已编译工具引用的 handler identity。""" + self.collected_messages = [] + self.stream_mode = stream_mode + if isinstance(self.stream_handler, _OpenAIStreamingHandlerMixin): + self.stream_handler.bind_queue(event_queue if stream_mode else None) + return + if not stream_mode: + return + self.stream_handler = _get_openai_streaming_handler_type()() + self.stream_handler.bind_queue(event_queue) + # 已编译工具持有旧 handler；identity 变化时必须重建图和工具目录。 + self._compiled_agent_bundle = None + + def release_protocol_request( + self, + event_queue: Optional[asyncio.Queue], + ) -> None: + """释放已结束请求的输出队列，不影响同会话已重绑的新请求。""" + if isinstance(self.stream_handler, _OpenAIStreamingHandlerMixin): + self.stream_handler.unbind_queue(event_queue) + async def send_agent_message(self, message: str, title: str = ""): text = (message or "").strip() if title and text: @@ -62,7 +93,7 @@ class _CollectingMoviePilotAgent(MoviePilotAgent): self.stream_handler.emit(text) -class _OpenAIStreamingHandler(StreamingHandler): +class _OpenAIStreamingHandlerMixin: """ 将 Agent 流式输出转发到 OpenAI SSE 队列，不向站内消息系统落消息。 """ @@ -71,9 +102,15 @@ class _OpenAIStreamingHandler(StreamingHandler): super().__init__() self._event_queue: Optional[asyncio.Queue] = None - def bind_queue(self, queue: asyncio.Queue): + def bind_queue(self, queue: Optional[asyncio.Queue]): + """绑定当前协议请求的输出队列。""" self._event_queue = queue + def unbind_queue(self, queue: Optional[asyncio.Queue]) -> None: + """仅当仍指向该请求时解除绑定，避免清掉已排队的新请求。""" + if self._event_queue is queue: + self._event_queue = None + def emit(self, token: str): emitted = super().emit(token) if emitted and self._event_queue is not None: @@ -121,18 +158,67 @@ class _OpenAIStreamingHandler(StreamingHandler): return True, final_text +def _get_openai_streaming_handler_type() -> type: + """首次兼容协议调用时才解析完整流式处理器。""" + global _OPENAI_STREAMING_HANDLER_TYPE + if _OPENAI_STREAMING_HANDLER_TYPE is not None: + return _OPENAI_STREAMING_HANDLER_TYPE + with _OPENAI_STREAMING_HANDLER_TYPE_LOCK: + if _OPENAI_STREAMING_HANDLER_TYPE is None: + from app.agent.callback import StreamingHandler + + _OPENAI_STREAMING_HANDLER_TYPE = type( + "_RuntimeOpenAIStreamingHandler", + (_OpenAIStreamingHandlerMixin, StreamingHandler), + {"__module__": __name__}, + ) + return _OPENAI_STREAMING_HANDLER_TYPE + + +_OPENAI_STREAMING_HANDLER_TYPE_LOCK = Lock() +_OPENAI_STREAMING_HANDLER_TYPE: Optional[type] = None + + +def _build_collecting_agent_type(agent_base_type: type) -> type: + """为 OpenAI 与 Anthropic 兼容协议组合唯一的运行时类型。""" + return type( + "_RuntimeCollectingMoviePilotAgent", + (_CollectingMoviePilotAgentMixin, agent_base_type), + {"__module__": __name__}, + ) + + +_COLLECTING_AGENT_TYPE_LOCK = Lock() +_COLLECTING_AGENT_TYPE: Optional[type] = None + + +def _get_collecting_agent_type() -> type: + """在首个真实兼容协议请求边界 single-flight 解析 Agent 类型。""" + global _COLLECTING_AGENT_TYPE + if _COLLECTING_AGENT_TYPE is not None: + return _COLLECTING_AGENT_TYPE + with _COLLECTING_AGENT_TYPE_LOCK: + if _COLLECTING_AGENT_TYPE is None: + _COLLECTING_AGENT_TYPE = _build_collecting_agent_type( + get_moviepilot_agent_type() + ) + return _COLLECTING_AGENT_TYPE + + def _sse_payload(data: dict) -> str: return f"data: {json.dumps(data, ensure_ascii=False)}\n\n" async def _stream_response( - agent: _CollectingMoviePilotAgent, + manager, + session_id: str, + user_id: str, + username: str, prompt: str, images: List[str], + cleanup_session: bool, ) -> AsyncIterator[str]: event_queue: asyncio.Queue = asyncio.Queue() - if isinstance(agent.stream_handler, _OpenAIStreamingHandler): - agent.stream_handler.bind_queue(event_queue) created = int(time.time()) completion_id = f"chatcmpl-{uuid.uuid4().hex}" @@ -140,7 +226,19 @@ async def _stream_response( async def _run_agent(): try: - await agent.process(prompt, images=images, files=None) + await _run_managed_agent( + manager=manager, + session_id=session_id, + user_id=user_id, + username=username, + source="openai", + prompt=prompt, + images=images, + stream_mode=True, + event_queue=event_queue, + ) + except asyncio.CancelledError: + await event_queue.put({"error": "MoviePilot AI agent is unavailable."}) except Exception as exc: await event_queue.put({"error": str(exc)}) finally: @@ -170,7 +268,17 @@ async def _stream_response( if item is None: break if isinstance(item, dict) and item.get("error"): - raise RuntimeError(str(item["error"])) + yield _sse_payload( + { + "error": { + "message": str(item["error"]), + "type": "server_error", + "code": "agent_execution_failed", + } + } + ) + yield "data: [DONE]\n\n" + return text = str(item or "") if not text: continue @@ -208,6 +316,10 @@ async def _stream_response( ) yield "data: [DONE]\n\n" finally: + if cleanup_session: + await manager.clear_session(session_id=session_id, user_id=user_id) + elif not task.done(): + await manager.stop_current_task(session_id) if not task.done(): task.cancel() try: @@ -218,6 +330,57 @@ async def _stream_response( await task +def _is_manager_unavailable(error: BaseException) -> bool: + """识别 manager acceptance gate 的稳定错误，不导入完整编排模块。""" + return getattr(error, "code", None) == "agent_manager_unavailable" + + +async def _run_managed_agent( + *, + manager, + session_id: str, + user_id: str, + username: str, + source: str, + prompt: str, + images: List[str], + stream_mode: bool, + event_queue: Optional[asyncio.Queue] = None, +) -> tuple[str, List[str]]: + """通过 AgentManager 执行协议请求，并在 worker 内配置请求级输出。""" + agent_holder = {} + + def configure_agent(agent) -> None: + agent.configure_protocol_request( + stream_mode=stream_mode, + event_queue=event_queue, + ) + agent_holder["agent"] = agent + + try: + result = await manager.process_message( + session_id=session_id, + user_id=user_id, + message=prompt, + images=images, + files=None, + channel=MessageChannel.Web.value, + source=source, + username=username, + reply_mode=ReplyMode.CAPTURE_ONLY, + allow_message_tools=True, + agent_factory=_get_collecting_agent_type(), + agent_setup=configure_agent, + wait_for_completion=True, + ) + agent = agent_holder.get("agent") + return result, list(agent.collected_messages if agent else []) + finally: + agent = agent_holder.get("agent") + if agent is not None: + agent.release_protocol_request(event_queue) + + def _error_response( message: str, status_code: int, @@ -310,6 +473,14 @@ async def chat_completions( error_type="server_error", code="ai_agent_disabled", ) + manager = get_running_agent_manager() + if manager is None: + return _error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="server_error", + code="ai_agent_unavailable", + ) if not payload.messages: return _error_response( @@ -337,19 +508,17 @@ async def chat_completions( session_id = build_session_id(session_key, SESSION_PREFIX) username = str(payload.user or "openai-client") - # 兼容接口的 API_TOKEN 客户端按管理员级 MoviePilot Agent 集成处理。 - agent = _CollectingMoviePilotAgent( - session_id=session_id, - user_id=session_key, - channel=MessageChannel.Web.value, - source="openai", - username=username, - stream_mode=payload.stream, - ) - if payload.stream: return StreamingResponse( - _stream_response(agent=agent, prompt=prompt, images=images), + _stream_response( + manager=manager, + session_id=session_id, + user_id=session_key, + username=username, + prompt=prompt, + images=images, + cleanup_session=not use_server_session, + ), media_type="text/event-stream", headers={ "Cache-Control": "no-cache", @@ -358,19 +527,39 @@ async def chat_completions( }, ) + collected_messages = [] try: - result = await agent.process(prompt, images=images, files=None) + result, collected_messages = await _run_managed_agent( + manager=manager, + session_id=session_id, + user_id=session_key, + username=username, + source="openai", + prompt=prompt, + images=images, + stream_mode=False, + ) except Exception as exc: + if _is_manager_unavailable(exc): + return _error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="server_error", + code="ai_agent_unavailable", + ) return _error_response( str(exc), 500, error_type="server_error", code="agent_execution_failed", ) + finally: + if not use_server_session: + await manager.clear_session(session_id=session_id, user_id=session_key) content = "\n\n".join( message.strip() - for message in agent.collected_messages + for message in collected_messages if message and message.strip() ).strip() if not content and result: @@ -403,6 +592,14 @@ async def responses( error_type="server_error", code="ai_agent_disabled", ) + manager = get_running_agent_manager() + if manager is None: + return _error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="server_error", + code="ai_agent_unavailable", + ) if payload.stream: return _error_response( @@ -430,29 +627,39 @@ async def responses( session_key = str(payload.user or uuid.uuid4()) session_id = build_session_id(session_key, SESSION_PREFIX) - # 兼容接口的 API_TOKEN 客户端按管理员级 MoviePilot Agent 集成处理。 - agent = _CollectingMoviePilotAgent( - session_id=session_id, - user_id=session_key, - channel=MessageChannel.Web.value, - source="openai.responses", - username=str(payload.user or "openai-client"), - stream_mode=False, - ) - + collected_messages = [] try: - result = await agent.process(prompt, images=images, files=None) + result, collected_messages = await _run_managed_agent( + manager=manager, + session_id=session_id, + user_id=session_key, + username=str(payload.user or "openai-client"), + source="openai.responses", + prompt=prompt, + images=images, + stream_mode=False, + ) except Exception as exc: + if _is_manager_unavailable(exc): + return _error_response( + "MoviePilot AI agent is unavailable.", + 503, + error_type="server_error", + code="ai_agent_unavailable", + ) return _error_response( str(exc), 500, error_type="server_error", code="agent_execution_failed", ) + finally: + if not payload.user: + await manager.clear_session(session_id=session_id, user_id=session_key) content = "\n\n".join( message.strip() - for message in agent.collected_messages + for message in collected_messages if message and message.strip() ).strip() if not content and result: diff --git a/app/application/agent.py b/app/application/agent.py index cab4fd58f..efec398d1 100644 --- a/app/application/agent.py +++ b/app/application/agent.py @@ -1,26 +1,46 @@ """Agent 编排服务门面。 chain 层需要触发 Agent 后台任务、渲染提示词、查询模型能力时统一经本模块调用。 -具体实现由 app.agent 在启动时注册，形成依赖倒置： +具体实现由 startup 组合根注册，形成依赖倒置： - chain -> application.agent <- agent（startup 在导入期注册） + chain -> application.agent <- startup -> agent -静态依赖图上 application 不依赖 agent，agent 作为入口层向 application -注册实现，从而拆除 chain <-> agent 的互指环。 - -注意：本模块禁止静态导入 app.agent 下的任何模块（含函数内导入）， -否则会形成 agent -> chain -> application -> agent 的新环。 -未注册时的兜底注册由 startup/agent_initializer 在导入期完成。 +门面保存 provider 而非重量级实现对象，注册本身不会物化 Agent、LLM 或工具树。 +本模块禁止静态或函数内导入 app.agent，否则会重新形成跨层循环依赖。 """ from typing import Any, Callable, Optional -# 注册表：启动期由 startup/agent_initializer 填充。 -_agent_manager: Any = None -_prompt_manager: Any = None -_agent_capability_manager: Any = None -_llm_helper: Any = None -_manual_redo_prompt_builder: Optional[Callable[[Any], str]] = None +Provider = Callable[[], Any] + +# provider 注册表由 startup/agent_initializer 在组合根装配。 +_agent_manager_provider: Optional[Provider] = None +_running_agent_manager_provider: Optional[Provider] = None +_prompt_manager_provider: Optional[Provider] = None +_agent_capability_manager_provider: Optional[Provider] = None +_llm_helper_provider: Optional[Provider] = None +_manual_redo_prompt_builder_provider: Optional[Provider] = None + + +def register_agent_service_providers( + *, + agent_manager_provider: Provider, + running_agent_manager_provider: Provider, + prompt_manager_provider: Provider, + capability_manager_provider: Provider, + llm_helper_provider: Provider, + manual_redo_prompt_builder_provider: Provider, +) -> None: + """注册 Agent 服务 provider，保持组合根装配阶段零重量实现导入。""" + global _agent_manager_provider, _running_agent_manager_provider + global _prompt_manager_provider, _agent_capability_manager_provider + global _llm_helper_provider, _manual_redo_prompt_builder_provider + _agent_manager_provider = agent_manager_provider + _running_agent_manager_provider = running_agent_manager_provider + _prompt_manager_provider = prompt_manager_provider + _agent_capability_manager_provider = capability_manager_provider + _llm_helper_provider = llm_helper_provider + _manual_redo_prompt_builder_provider = manual_redo_prompt_builder_provider def register_agent_services( @@ -30,38 +50,40 @@ def register_agent_services( llm_helper: Any, manual_redo_prompt_builder: Optional[Callable[[Any], str]] = None, ) -> None: - """注册 Agent 服务实现（由 startup 组合根在导入期调用）。""" - global _agent_manager, _prompt_manager, _agent_capability_manager, _llm_helper - global _manual_redo_prompt_builder - _agent_manager = agent_manager - _prompt_manager = prompt_manager - _agent_capability_manager = capability_manager - _llm_helper = llm_helper - _manual_redo_prompt_builder = manual_redo_prompt_builder + """兼容直接对象注入；生产组合根应注册惰性 provider。""" + register_agent_service_providers( + agent_manager_provider=lambda: agent_manager, + running_agent_manager_provider=lambda: agent_manager, + prompt_manager_provider=lambda: prompt_manager, + capability_manager_provider=lambda: capability_manager, + llm_helper_provider=lambda: llm_helper, + manual_redo_prompt_builder_provider=lambda: manual_redo_prompt_builder, + ) -def _ensure_registered() -> None: - """校验 Agent 服务已注册。 - - 正常启动路径由 startup/agent_initializer 在导入期注册；未注册时 - 直接抛出带指引的错误，避免在此处静态导入 app.agent 破坏依赖方向。 - """ - if _agent_manager is None: +def _resolve(provider: Optional[Provider], service_name: str) -> Any: + """解析已注册服务；缺少组合根装配时给出稳定错误。""" + if provider is None: raise RuntimeError( - "Agent 服务未注册：请先导入 app.startup.agent_initializer 完成组合根装配" + f"Agent 服务 {service_name} 未注册：" + "请先导入 app.startup.agent_initializer 完成组合根装配" ) + return provider() def get_agent_manager() -> Any: - """返回 AgentManager 单例。""" - _ensure_registered() - return _agent_manager + """返回 canonical AgentManager；调用可能触发实现物化。""" + return _resolve(_agent_manager_provider, "agent_manager") + + +def get_running_agent_manager() -> Any | None: + """返回已进入 RUNNING 的 AgentManager，不触发实现物化。""" + return _resolve(_running_agent_manager_provider, "running_agent_manager") def get_prompt_manager() -> Any: - """返回提示词管理器。""" - _ensure_registered() - return _prompt_manager + """按需返回提示词管理器。""" + return _resolve(_prompt_manager_provider, "prompt_manager") def supports_image_input( @@ -71,8 +93,8 @@ def supports_image_input( base_url_preset: Optional[str] = None, ) -> bool: """判断当前模型是否启用了图片输入能力。""" - _ensure_registered() - return _llm_helper.supports_image_input( + llm_helper = _resolve(_llm_helper_provider, "llm_helper") + return llm_helper.supports_image_input( provider=provider, model=model, base_url=base_url, @@ -82,19 +104,28 @@ def supports_image_input( def is_audio_input_available() -> bool: """判断语音输入能力是否可用。""" - _ensure_registered() - return _agent_capability_manager.is_audio_input_available() + capability_manager = _resolve( + _agent_capability_manager_provider, + "agent_capability_manager", + ) + return capability_manager.is_audio_input_available() def transcribe_audio(content: bytes, filename: str = "input.ogg") -> Optional[str]: """把音频内容转写为文本。""" - _ensure_registered() - return _agent_capability_manager.transcribe_audio(content, filename=filename) + capability_manager = _resolve( + _agent_capability_manager_provider, + "agent_capability_manager", + ) + return capability_manager.transcribe_audio(content, filename=filename) def build_manual_redo_prompt(history: Any) -> str: """构造整理记录 AI 重新整理提示词（builder 由 agent 层注册）。""" - _ensure_registered() - if _manual_redo_prompt_builder is None: + builder = _resolve( + _manual_redo_prompt_builder_provider, + "manual_redo_prompt_builder", + ) + if builder is None: raise RuntimeError("整理记录重新整理提示词构建器未注册") - return _manual_redo_prompt_builder(history) + return builder(history) diff --git a/app/application/transfer.py b/app/application/transfer.py index 6bdbdf50f..0554169b7 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -24,7 +24,7 @@ from pydantic import BaseModel, ConfigDict from app import schemas from app.adapters.system.host import SystemUtils -from app.application.agent import get_agent_manager, get_prompt_manager +from app.application.agent import get_prompt_manager, get_running_agent_manager from app.domain.context import MediaInfo, MusicInfo from app.domain.media import normalize_music_type from app.domain.meta.metabase import MetaBase @@ -959,7 +959,11 @@ class FailedRetryScheduler: ) try: - await get_agent_manager().run_background_prompt( + manager = get_running_agent_manager() + if manager is None: + logger.warning("智能助手服务未运行，跳过整理失败自动重试") + return + await manager.run_background_prompt( message=self._build_retry_transfer_prompt(history_ids), session_prefix="__agent_retry_transfer_batch", reply_mode=ReplyMode.DISPATCH, @@ -972,4 +976,3 @@ class FailedRetryScheduler: f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}" ) - diff --git a/app/chain/_transfer.py b/app/chain/_transfer.py index d780b4124..063490b7c 100644 --- a/app/chain/_transfer.py +++ b/app/chain/_transfer.py @@ -14,7 +14,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from app import schemas from app.adapters.system.host import SystemUtils -from app.application.agent import build_manual_redo_prompt, get_agent_manager +from app.application.agent import build_manual_redo_prompt, get_running_agent_manager from app.application.formatting import EpisodeFormatRuleHelper from app.application.history import clear_transfer_failures, resolve_history from app.application.transfer import TransferTask, job_lock @@ -1418,7 +1418,10 @@ class FailedRetryMixin: final_output = text_output or "" try: - await get_agent_manager().run_background_prompt( + manager = get_running_agent_manager() + if manager is None: + raise RuntimeError("智能助手服务未运行") + await manager.run_background_prompt( message=redo_prompt, session_prefix=f"__agent_manual_redo_{history_id}", output_callback=_capture_output, diff --git a/app/chain/message.py b/app/chain/message.py index 15ee57d73..6a9d237c0 100644 --- a/app/chain/message.py +++ b/app/chain/message.py @@ -12,7 +12,7 @@ from typing import Any, Optional, Dict, Union, List, Tuple from urllib.parse import unquote, urlparse from app.application.agent import ( - get_agent_manager, + get_running_agent_manager, is_audio_input_available, supports_image_input, transcribe_audio, @@ -70,9 +70,14 @@ class MessageChain(ChainBase): """ if not session_id: return + manager = get_running_agent_manager() + if manager is None: + return clear_task = None try: - clear_task = get_agent_manager().clear_session(session_id=session_id, user_id=str(userid)) + clear_task = manager.clear_session( + session_id=session_id, user_id=str(userid) + ) asyncio.run_coroutine_threadsafe( clear_task, global_vars.loop, @@ -350,7 +355,8 @@ class MessageChain(ChainBase): if not session_info: return False session_id, _ = session_info - if not get_agent_manager().matches_secret_confirmation( + manager = get_running_agent_manager() + if manager is None or not manager.matches_secret_confirmation( session_id, str(userid), channel=channel.value, @@ -968,19 +974,21 @@ class MessageChain(ChainBase): # 如果有会话ID，同时清除智能体的会话记忆 if session_id: + manager = get_running_agent_manager() clear_task = None - try: - clear_task = get_agent_manager().clear_session( - session_id=session_id, user_id=str(userid) - ) - asyncio.run_coroutine_threadsafe( - clear_task, - global_vars.loop, - ) - except Exception as e: - if clear_task: - clear_task.close() - logger.warning(f"清除智能体会话记忆失败: {e}") + if manager is not None: + try: + clear_task = manager.clear_session( + session_id=session_id, user_id=str(userid) + ) + asyncio.run_coroutine_threadsafe( + clear_task, + global_vars.loop, + ) + except Exception as e: + if clear_task: + clear_task.close() + logger.warning(f"清除智能体会话记忆失败: {e}") self.post_message( Notification( @@ -1017,12 +1025,16 @@ class MessageChain(ChainBase): session_info = self._user_sessions.get(userid) if session_info: session_id, _ = session_info + manager = get_running_agent_manager() try: - future = asyncio.run_coroutine_threadsafe( - get_agent_manager().stop_current_task(session_id=session_id), - global_vars.loop, - ) - stopped = future.result(timeout=10) + if manager is None: + stopped = False + else: + future = asyncio.run_coroutine_threadsafe( + manager.stop_current_task(session_id=session_id), + global_vars.loop, + ) + stopped = future.result(timeout=10) except Exception as e: logger.warning(f"停止Agent推理失败: {e}") stopped = False @@ -1184,7 +1196,19 @@ class MessageChain(ChainBase): return session_id, _ = session_info - status = get_agent_manager().get_session_status(session_id=session_id) + manager = get_running_agent_manager() + if manager is None: + self.post_message( + Notification( + channel=channel, + source=source, + title="您当前没有活跃的智能体会话", + userid=userid, + save_history=False, + ) + ) + return + status = manager.get_session_status(session_id=session_id) self.post_message( Notification( channel=channel, @@ -1229,6 +1253,20 @@ class MessageChain(ChainBase): ) return False + manager = get_running_agent_manager() + if manager is None: + self.post_message( + Notification( + channel=channel, + source=source, + userid=userid, + username=username, + title="MoviePilot智能助手服务尚未就绪，请稍后重试", + save_history=False, + ) + ) + return False + images = CommingMessage.MessageImage.normalize_list(images) # 提取用户消息 @@ -1337,7 +1375,7 @@ class MessageChain(ChainBase): process_kwargs["has_audio_input"] = True # 在事件循环中处理 asyncio.run_coroutine_threadsafe( - get_agent_manager().process_message(**process_kwargs), + manager.process_message(**process_kwargs), global_vars.loop, ) return True @@ -1854,4 +1892,4 @@ class MessageChain(ChainBase): return base64.b64decode(payload) except Exception as e: logger.error(e) - return None \ No newline at end of file + return None diff --git a/app/chain/search.py b/app/chain/search.py index 9e87768bd..df4cbb434 100644 --- a/app/chain/search.py +++ b/app/chain/search.py @@ -509,7 +509,7 @@ class SearchChain(ChainBase): """ 通过统一后台提示词机制执行资源推荐。 """ - from app.application.agent import get_agent_manager, get_prompt_manager + from app.application.agent import get_prompt_manager, get_running_agent_manager from app.schemas.agent import ReplyMode prompt = get_prompt_manager().render_system_task_message( @@ -521,7 +521,11 @@ class SearchChain(ChainBase): def on_output(text: str): full_output[0] = text - await get_agent_manager().run_background_prompt( + manager = get_running_agent_manager() + if manager is None: + logger.warning("智能助手服务未运行，跳过搜索结果 AI 推荐") + raise RuntimeError("智能助手服务未运行") + await manager.run_background_prompt( message=prompt, session_prefix="__agent_search_recommend", output_callback=on_output, diff --git a/app/scheduler.py b/app/scheduler.py index a0b2811ba..40b1232ee 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1221,10 +1221,14 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass): :param trigger_source: 触发入口，scheduled-自动调度，manual-显式立即执行 :return: 执行是否成功及结果摘要 """ - from app.agent.orchestrator import agent_manager + from app.agent.runtime_loader import get_running_agent_manager try: - return await agent_manager.execute_scheduled_task( + manager = get_running_agent_manager() + if manager is None: + logger.warning("智能助手服务未运行，跳过 Agent 定时任务") + return False, "智能助手服务未运行" + return await manager.execute_scheduled_task( task_id, trigger_source=trigger_source, ) @@ -1537,9 +1541,13 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass): """ 智能体心跳唤醒：检查并执行待处理的定时任务 """ - from app.agent.orchestrator import agent_manager + from app.agent.runtime_loader import get_running_agent_manager - await agent_manager.heartbeat_check_jobs() + manager = get_running_agent_manager() + if manager is None: + logger.debug("智能助手服务未运行，跳过心跳任务") + return + await manager.heartbeat_check_jobs() def user_auth(self): """ diff --git a/app/schemas/agent.py b/app/schemas/agent.py index 447791bf6..65f5cdab8 100644 --- a/app/schemas/agent.py +++ b/app/schemas/agent.py @@ -1,20 +1,13 @@ """AI智能体相关数据模型""" from datetime import datetime -from enum import Enum from typing import Any, List, Literal, Optional, Union from langchain_core.messages import BaseMessage from pydantic import BaseModel, Field, ConfigDict, field_serializer from app.schemas.common import JsonData - - -class ReplyMode(str, Enum): - """Agent 最终回复处理模式（chain 与 agent 层共享的值域）。""" - - DISPATCH = "dispatch" - CAPTURE_ONLY = "capture_only" +from app.schemas.types import ReplyMode class ConversationMemory(BaseModel): diff --git a/app/schemas/types.py b/app/schemas/types.py index d46efbdf4..57f4c0e23 100644 --- a/app/schemas/types.py +++ b/app/schemas/types.py @@ -22,6 +22,13 @@ MUSIC_SUBSCRIBABLE_TYPES = frozenset({ MUSIC_ENTITY_ALBUM, }) + +class ReplyMode(str, Enum): + """Agent 最终回复的投递策略，供编排层与调用层共享。""" + + DISPATCH = "dispatch" + CAPTURE_ONLY = "capture_only" + # ListenBrainz 音乐探索能力的参数取值域契约，供入口层校验、链层与模块实现共用 # ListenBrainz 全站统计支持的周期，取值与官方统计页面完全一致 LISTENBRAINZ_CHART_RANGES = ( diff --git a/app/startup/agent_initializer.py b/app/startup/agent_initializer.py index f56f4c39d..fc9f51889 100644 --- a/app/startup/agent_initializer.py +++ b/app/startup/agent_initializer.py @@ -1,20 +1,81 @@ -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services -from app.runtime.config import settings -from app.runtime.log import logger +from typing import Any -# 导入期即向 application 门面注册实现，保证任何先于 initialize 的 -# 链层调用都能通过门面取到 Agent 服务对象。 -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, +from app.agent.runtime_loader import ( + activate_agent_service, + begin_agent_shutdown, + get_agent_manager as get_runtime_agent_manager, + get_running_agent_manager as get_runtime_running_agent_manager, + is_tool_factory_materialized, + reconcile_agent_service, ) +from app.application.agent import register_agent_service_providers +from app.runtime.config import settings +from app.runtime.events import Event, eventmanager +from app.runtime.log import logger +from app.schemas.types import EventType + + +# 嵌入式启动器可显式注入 manager；常规进程使用 Capability Runtime。 +agent_manager: Any = None + + +def _event_changed_keys(event: Event | None) -> set[str]: + """兼容对象和 dict 两种配置事件载荷。""" + if event is None: + return set() + event_data = event.event_data + if isinstance(event_data, dict): + keys = event_data.get("key", set()) + else: + keys = getattr(event_data, "key", set()) + if isinstance(keys, str): + return {keys} + return {str(key) for key in (keys or set())} + + +def _get_agent_manager() -> Any: + """兼容显式注入对象，否则按需解析 canonical manager。""" + return agent_manager if agent_manager is not None else get_runtime_agent_manager() + + +def _get_running_agent_manager() -> Any | None: + """只返回已运行实例，状态探测不得触发 Agent 物化。""" + if agent_initializer._compat_injected: + return agent_initializer._manager + return get_runtime_running_agent_manager() + + +def _get_prompt_manager() -> Any: + """首个提示词调用才导入模板管理器。""" + from app.agent.prompt import prompt_manager + + return prompt_manager + + +def _get_capability_manager() -> Any: + """首个多模态调用才导入 Agent 能力管理器。""" + from app.agent.llm import AgentCapabilityManager + + return AgentCapabilityManager + + +def _get_llm_helper() -> Any: + """首个模型能力查询才导入 LLM helper。""" + from app.agent.llm import LLMHelper + + return LLMHelper + + +def _get_manual_redo_prompt_builder() -> Any: + """首个整理接管请求才导入对应提示词构建器。""" + from app.agent.prompt.transfer_redo import build_manual_redo_prompt + + return build_manual_redo_prompt + + +async def _handle_agent_config_changed(event: Event) -> None: + """把配置事件交给当前全局 initializer，避免监听器持有过期实例。""" + await agent_initializer.handle_config_changed(event) class AgentInitializer: @@ -24,17 +85,33 @@ class AgentInitializer: def __init__(self): self._initialized = False + self._manager: Any = None + self._compat_injected = False + self._shutdown_complete = False + eventmanager.add_event_listener( + EventType.ConfigChanged, + _handle_agent_config_changed, + ) async def initialize(self) -> bool: """ 初始化AI智能体管理器 """ try: - if not settings.AI_AGENT_ENABLE: - logger.info("AI智能体功能未启用") - return True - - await agent_manager.initialize() + self._shutdown_complete = False + if agent_manager is not None: + if not settings.AI_AGENT_ENABLE: + logger.info("AI智能体功能未启用") + return True + self._manager = agent_manager + self._compat_injected = True + await agent_manager.initialize() + else: + self._manager = await activate_agent_service() + self._compat_injected = False + if self._manager is None: + logger.info("AI智能体功能未启用") + return True self._initialized = True logger.info("AI智能体管理器初始化成功") return True @@ -43,16 +120,38 @@ class AgentInitializer: logger.error(f"AI智能体管理器初始化失败: {e}") return False - async def cleanup(self) -> None: - """ - 清理AI智能体管理器 - """ + async def handle_config_changed(self, event: Event) -> None: + """仅在 manifest watch 命中时协调 service，关闭态保持 fail closed。""" + changed_keys = _event_changed_keys(event) + if not changed_keys or self._compat_injected or self._shutdown_complete: + return try: - if not self._initialized: - return - await agent_manager.close() + self._manager = await reconcile_agent_service( + reason="agent_service_config_changed", + changed_keys=changed_keys, + retry=True, + ) + self._initialized = self._manager is not None + except Exception as error: + self._manager = None self._initialized = False - logger.info("AI智能体管理器已关闭") + logger.debug(f"配置变更协调AI智能体失败: {error}") + + async def cleanup(self) -> None: + """清理 initializer 引用；显式注入对象同时在此关闭。""" + try: + manager = self._manager + compat_injected = self._compat_injected + if manager is None: + return + try: + if compat_injected: + await manager.close() + logger.info("AI智能体管理器已关闭") + finally: + self._initialized = False + self._manager = None + self._compat_injected = False except Exception as e: logger.debug(f"关闭AI智能体管理器时发生错误: {e}") @@ -61,16 +160,22 @@ class AgentInitializer: # 全局AI智能体初始化器实例 agent_initializer = AgentInitializer() +# application 门面仅保存 provider；下列注册不会导入 Agent 实现。 +register_agent_service_providers( + agent_manager_provider=_get_agent_manager, + running_agent_manager_provider=_get_running_agent_manager, + prompt_manager_provider=_get_prompt_manager, + capability_manager_provider=_get_capability_manager, + llm_helper_provider=_get_llm_helper, + manual_redo_prompt_builder_provider=_get_manual_redo_prompt_builder, +) + async def init_agent() -> bool: """ 在应用事件循环中初始化AI智能体。 """ try: - if not settings.AI_AGENT_ENABLE: - logger.info("AI智能体功能未启用") - return True - return await agent_initializer.initialize() except Exception as e: @@ -83,6 +188,16 @@ async def stop_agent(): 停止AI智能体（异步版本，用于在应用关闭时调用） """ try: - await agent_initializer.cleanup() + if not agent_initializer._shutdown_complete: + if agent_initializer._compat_injected: + await agent_initializer.cleanup() + else: + await begin_agent_shutdown() + await agent_initializer.cleanup() + agent_initializer._shutdown_complete = True + if is_tool_factory_materialized(): + from app.agent.tools.base import shutdown_blocking_executors + + shutdown_blocking_executors(cancel_futures=True) except Exception as e: logger.error(f"停止AI智能体时发生错误: {e}") diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 637f9ad98..37cdcd663 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -340,7 +340,7 @@ policy. `app/db` therefore has no dependency on `app/domain`. |---|---| | `entrypoint -> chain / application / Oper` | Allowed according to workflow complexity | | `chain -> module (only via run_module dispatch) / application / Oper / canonical capability` | Allowed; direct `chain -> module` imports forbidden | -| `chain -> agent implementation` | Forbidden; chains reach Agent runtime only through `app/application/agent.py`, whose implementations are registered by `app/startup/agent_initializer.py` at import time | +| `chain -> agent implementation` | Forbidden; chains reach Agent runtime only through `app/application/agent.py`; `app/startup/agent_initializer.py` registers lightweight providers at import time, and implementations are materialized only when the capability is enabled or first used | | `agent.tools -> api / scheduler / command` | Forbidden; tools use `app/application/plugins.py`, `scheduling.py` and `commands.py` facades | | `api -> factory` | Forbidden; the FastAPI instance is injected into `app/application/plugins.py` by the composition root after creation | | `application -> domain / runtime / adapter / Oper` | Allowed | @@ -357,7 +357,8 @@ policy. `app/db` therefore has no dependency on `app/domain`. | Path | Purpose | |---|---| -| `app/application/agent.py` | Agent orchestration facade (`get_agent_manager` / `get_prompt_manager` / capability queries / prompt builders); Agent implementations register through `app/startup/agent_initializer.py`, no static `application -> agent` edge | +| `app/application/agent.py` | Agent orchestration facade (`get_agent_manager` / `get_prompt_manager` / capability queries / prompt builders); lightweight providers register through `app/startup/agent_initializer.py`, with no static `application -> agent` edge | +| `app/agent/runtime_loader.py` | Agent-specific capability discovery and canonical entrypoint/service materialization; reuses the generic Capability Runtime while keeping Agent ownership under `app/agent/` | | `app/application/plugins.py` | Plugin API dynamic route registration/removal; the FastAPI instance is injected by `app/factory.py` after creation | | `app/application/scheduling.py` | Runtime scheduler facade for Agent tools and endpoints; `Scheduler` class registered by `app/startup/scheduler_initializer.py` | | `app/application/commands.py` | Command registry facade for Agent tools and endpoints; `Command` class registered by `app/startup/command_initializer.py` | diff --git a/scripts/perf/README.md b/scripts/perf/README.md index a2b074069..11d29fd26 100644 --- a/scripts/perf/README.md +++ b/scripts/perf/README.md @@ -81,6 +81,62 @@ Xvfb，因此不能用同一个 `0 → 0` / `0 → 1` 不变量衡量。三轮 B - 非默认场景结果保存在 `samples//-/`，可与同 campaign 的 idle 样本并存， Markdown 中位数会按场景分组，不会混算。 +## Agent 惰性物化场景 + +PERF-003 在既有 `AI_AGENT_ENABLE=false` 固定配置下增加两个 After-only 场景。探针只向主 MoviePilot +Python 进程发送信号；OpenAPI 生成和工具目录构造均发生在该解释器内，不通过 `docker exec` 启动 +第二个 Python，也不调用真实 Agent、LLM provider 或外部 MCP。 + +先以 `f2e548e1` 冻结 Before，候选提交完成后把 `AFTER_COMMIT` 替换为其精确 commit： + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-003 \ + build --before-ref f2e548e1 --after-ref AFTER_COMMIT + +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-003 \ + seed --browser-source-volume mp-perf-v3-browser-seed --replace +``` + +正式 idle-default 三组 A/B 仍使用原 `run` 合同；下面两个动作场景在同一 build/seed 后单独采 After， +不会覆盖 idle 结果： + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-003 \ + run --before-ref f2e548e1 --after-ref AFTER_COMMIT \ + --browser-source-volume mp-perf-v3-browser-seed \ + --points 1,5,10,30 --replace --keep-resources + +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-003 \ + sample --variant after --index 1 --scenario agent-disabled-router --points 1,5,10,30 + +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-003 \ + sample --variant after --index 2 --scenario agent-tool-catalog --points 1,5,10,30 +``` + +- `agent-disabled-router`：直接从主进程 FastAPI app 生成完整 OpenAPI，确认 Agent、LLM、MCP、OpenAI、 + Anthropic 路由在禁用态仍存在，同时 callback、LLM helper、工具域、orchestrator、LangGraph 和 provider SDK + 前后保持 0，工具工厂不物化； +- `agent-tool-catalog`：通过主进程已有的 `moviepilot_tool_manager.list_tools()` 首次构建现有工具目录和 JSON Schema， + 要求动作前工具域未物化，动作后仅工具 base/catalog/factory/impl 物化；目录还必须无身份碰撞、Schema + digest 完整，重复读取复用同一 snapshot/revision。结果记录工具数、Schema 摘要、plugin revision 与 + factory revision； +- 固定哨兵覆盖 `app.agent.orchestrator`、`app.agent.callback`、`app.agent.llm.helper`、工具 + `base/catalog/factory/impl`、`langgraph`、`langchain`、`langchain_core`、`openai`、`anthropic`、 + `google.genai`、`boto3`、`botocore`。其中 `langchain/langchain_core` 可能由完整 Schema 聚合形成既有 + 基线，只记录数量与变化，不作为禁用态归零门禁； +- JSON 保留动作前后 Engine、PSS/USS、线程、完整 `sys.modules`、materialization observation、revision、 + 网络累计值和浏览器卷指纹；动作前后容器网络收发必须为 0，Markdown 另汇总 Agent 场景与各定时点的 + 模块哨兵峰值； +- 启用态 Agent 生命周期不会在该无凭据场景中伪造。现有 `get_running_agent_manager()` 是严格只读、 + non-materializing 的运行态 getter，`begin_agent_shutdown()` 也只是关闭轴；二者都不是安全启用入口。 + 启用态必须由正式 startup/service lifecycle 驱动，只有宿主形成明确不创建 provider/client、不会外联的 + 公共初始化合同后，才适合加入同一测量门禁。 + ## 完整三组 A/B ```bash diff --git a/scripts/perf/instrument/sitecustomize.py b/scripts/perf/instrument/sitecustomize.py index 2fabc4f58..3fa40a18e 100644 --- a/scripts/perf/instrument/sitecustomize.py +++ b/scripts/perf/instrument/sitecustomize.py @@ -12,6 +12,20 @@ import sys _OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR") _SCENARIO = os.environ.get("MP_PERF_SCENARIO", "idle-default") _ACTIVATION_TIMEOUT = float(os.environ.get("MP_PERF_ACTIVATION_TIMEOUT", "120")) +_AGENT_SCENARIOS = {"agent-disabled-router", "agent-tool-catalog"} +_AGENT_HEAVY_MODULE_PREFIXES = tuple( + prefix + for prefix in os.environ.get( + "MP_PERF_AGENT_MODULE_PREFIXES", + ( + "app.agent.orchestrator,app.agent.callback,app.agent.llm.helper," + "app.agent.tools.base,app.agent.tools.catalog," + "app.agent.tools.factory,app.agent.tools.impl,langgraph,langchain," + "langchain_core,openai,anthropic,google.genai,boto3,botocore" + ), + ).split(",") + if prefix +) _snapshot_index = 0 _activation_started = False _browser_resources: list[object] = [] @@ -112,6 +126,244 @@ def _enum_value(value): return getattr(value, "value", value) +def _stable_digest(value: object) -> str: + """计算不依赖对象地址的 JSON 摘要。""" + import hashlib + import json + + content = json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(content.encode("utf-8")).hexdigest() + + +def _agent_module_observation() -> dict[str, object]: + """记录 Agent 重模块在目标解释器中的精确加载状态。""" + prefix_counts = { + prefix: sum( + 1 + for module_name in sys.modules + if module_name == prefix or module_name.startswith(f"{prefix}.") + ) + for prefix in _AGENT_HEAVY_MODULE_PREFIXES + } + matching_modules = sorted( + module_name + for module_name in sys.modules + if any( + module_name == prefix or module_name.startswith(f"{prefix}.") + for prefix in _AGENT_HEAVY_MODULE_PREFIXES + ) + ) + return { + "total_modules": len(sys.modules), + "prefix_counts": prefix_counts, + "matching_modules": matching_modules, + "matching_sha256": _stable_digest(matching_modules), + } + + +def _read_agent_runtime() -> dict[str, object]: + """读取轻量 Agent loader 的公开只读状态，不触发 capability 首用。""" + try: + from app.agent.runtime_loader import is_tool_factory_materialized + + return { + "available": True, + "tool_factory_materialized": is_tool_factory_materialized(), + } + except Exception as error: # pragma: no cover - 候选未就绪或真实 runtime 错误 + return { + "available": False, + "error_type": type(error).__name__, + "error": str(error), + } + + +def _probe_router_openapi(app_instance=None, settings_object=None) -> dict[str, object]: + """在主进程中生成 OpenAPI，并验证禁用态 Agent 路由仍完整存在。""" + if app_instance is None: + from app.factory import app as app_instance + if settings_object is None: + from app.runtime.config import settings as settings_object + + required_paths = ( + "/api/v1/message/agent/stream", + "/api/v1/message/agent/sessions", + "/api/v1/openai/v1/chat/completions", + "/api/v1/openai/v1/responses", + "/api/v1/anthropic/v1/messages", + "/api/v1/llm/manage", + "/api/v1/mcp", + "/api/v1/mcp/tools", + ) + schema = app_instance.openapi() + route_paths = sorted( + { + str(route.path) + for route in app_instance.routes + if getattr(route, "path", None) + } + ) + openapi_paths = sorted((schema.get("paths") or {}).keys()) + missing_routes = [path for path in required_paths if path not in route_paths] + missing_openapi_paths = [ + path for path in required_paths if path not in openapi_paths + ] + agent_enabled = bool(settings_object.AI_AGENT_ENABLE) + return { + "success": not agent_enabled + and not missing_routes + and not missing_openapi_paths, + "ai_agent_enable": agent_enabled, + "required_paths": list(required_paths), + "missing_routes": missing_routes, + "missing_openapi_paths": missing_openapi_paths, + "route_count": len(route_paths), + "openapi_path_count": len(openapi_paths), + "openapi_sha256": _stable_digest(schema), + "openapi_title": (schema.get("info") or {}).get("title"), + "openapi_version": (schema.get("info") or {}).get("version"), + } + + +def _probe_tool_catalog(manager=None) -> dict[str, object]: + """通过稳定工具管理入口首次生成目录与 JSON Schema。""" + if manager is None: + from app.agent.tools.manager import moviepilot_tool_manager + + manager = moviepilot_tool_manager + definitions = manager.list_tools() + catalog = manager.catalog + serialized_definitions = [ + { + "name": definition.name, + "input_schema": definition.input_schema, + } + for definition in definitions + ] + schema_count = sum( + isinstance(definition.input_schema, dict) for definition in definitions + ) + entries = catalog.entries if catalog is not None else () + collisions = catalog.collisions if catalog is not None else {} + source_counts: dict[str, int] = {} + serialized_entries = [] + for entry in entries: + source_counts[entry.source] = source_counts.get(entry.source, 0) + 1 + serialized_entries.append( + { + "name": entry.name, + "source": entry.source, + "schema_digest": entry.schema_digest, + } + ) + first_catalog_sha256 = _stable_digest(serialized_entries) + first_schemas_sha256 = _stable_digest(serialized_definitions) + repeated_definitions = manager.list_tools() + repeated_catalog = manager.catalog + repeated_serialized_definitions = [ + { + "name": definition.name, + "input_schema": definition.input_schema, + } + for definition in repeated_definitions + ] + repeated_entries = repeated_catalog.entries if repeated_catalog is not None else () + repeated_serialized_entries = [ + { + "name": entry.name, + "source": entry.source, + "schema_digest": entry.schema_digest, + } + for entry in repeated_entries + ] + repeated_catalog_sha256 = _stable_digest(repeated_serialized_entries) + repeated_schemas_sha256 = _stable_digest(repeated_serialized_definitions) + schema_digests_complete = all( + isinstance(entry.schema_digest, str) and len(entry.schema_digest) == 64 + for entry in entries + ) + repeat_revision_unchanged = bool( + catalog is not None + and repeated_catalog is not None + and repeated_catalog.plugin_revision == catalog.plugin_revision + and repeated_catalog.factory_revision == catalog.factory_revision + ) + repeat_stable = bool( + repeated_catalog is catalog + and len(repeated_definitions) == len(definitions) + and repeated_catalog_sha256 == first_catalog_sha256 + and repeated_schemas_sha256 == first_schemas_sha256 + and repeat_revision_unchanged + ) + return { + "success": bool(definitions) + and catalog is not None + and len(entries) == len(definitions) + and schema_count == len(definitions) + and not collisions + and schema_digests_complete + and repeat_stable, + "tool_count": len(definitions), + "schema_count": schema_count, + "catalog_entry_count": len(entries), + "collision_names": sorted(collisions), + "plugin_revision": catalog.plugin_revision if catalog is not None else None, + "factory_revision": catalog.factory_revision if catalog is not None else None, + "schemas_sha256": first_schemas_sha256, + "catalog_sha256": first_catalog_sha256, + "source_counts": source_counts, + "schema_digests_complete": schema_digests_complete, + "repeat_tool_count": len(repeated_definitions), + "repeat_catalog_same_object": repeated_catalog is catalog, + "repeat_catalog_sha256": repeated_catalog_sha256, + "repeat_schemas_sha256": repeated_schemas_sha256, + "repeat_revision_unchanged": repeat_revision_unchanged, + "repeat_stable": repeat_stable, + } + + +def _activate_agent_scenario( + scenario: str, + *, + app_instance=None, + settings_object=None, + tool_manager=None, + runtime_reader=None, +) -> dict[str, object]: + """执行 Agent 禁用态路由或首次工具目录的进程内场景。""" + if scenario not in _AGENT_SCENARIOS: + raise ValueError(f"场景不支持 Agent 激活：{scenario}") + runtime_reader = runtime_reader or _read_agent_runtime + modules_before = _agent_module_observation() + runtime_before = runtime_reader() + + if scenario == "agent-disabled-router": + action = _probe_router_openapi( + app_instance=app_instance, + settings_object=settings_object, + ) + else: + action = _probe_tool_catalog(manager=tool_manager) + + modules_after = _agent_module_observation() + runtime_after = runtime_reader() + return { + "requested": True, + "action": scenario.removeprefix("agent-"), + "success": bool(action.get("success")), + "modules": {"before": modules_before, "after": modules_after}, + "observations": {"before": runtime_before, "after": runtime_after}, + "router_openapi": action if scenario == "agent-disabled-router" else None, + "tool_catalog": action if scenario == "agent-tool-catalog" else None, + } + + def _read_display_runtime() -> dict[str, object]: """读取 host.display 的只读状态和观测，不触发资源激活。""" try: @@ -338,8 +590,12 @@ def _run_activation() -> None: "started_at": _utc_now(), } try: - result["browser"] = _activate_browser_scenario(_SCENARIO) - result["success"] = bool(result["browser"]["success"]) + if _SCENARIO in _AGENT_SCENARIOS: + result["agent"] = _activate_agent_scenario(_SCENARIO) + result["success"] = bool(result["agent"]["success"]) + else: + result["browser"] = _activate_browser_scenario(_SCENARIO) + result["success"] = bool(result["browser"]["success"]) except Exception as error: # pragma: no cover - 真实集成错误由 marker 保存 result.update( { diff --git a/scripts/perf/moviepilot_docker_ab.py b/scripts/perf/moviepilot_docker_ab.py index 5a2a97829..ee667a708 100644 --- a/scripts/perf/moviepilot_docker_ab.py +++ b/scripts/perf/moviepilot_docker_ab.py @@ -32,7 +32,9 @@ DEFAULT_SUBSTRATE = ( ) DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed" DEFAULT_SCENARIO = "idle-default" -SCENARIOS = (DEFAULT_SCENARIO, "browser-headless", "browser-headed") +BROWSER_SCENARIOS = ("browser-headless", "browser-headed") +AGENT_SCENARIOS = ("agent-disabled-router", "agent-tool-catalog") +SCENARIOS = (DEFAULT_SCENARIO, *BROWSER_SCENARIOS, *AGENT_SCENARIOS) CAMPAIGN_LABEL = "org.moviepilot.perf.campaign" ROLE_LABEL = "org.moviepilot.perf.role" SOURCE_LABEL = "org.moviepilot.perf.source-commit" @@ -43,6 +45,35 @@ CRITICAL_SUBSTRATE_PATHS = ( "scripts/uv-pip-compat.sh", ) SEED_COMPATIBILITY_PATHS = ("database/versions",) +AGENT_HEAVY_MODULE_PREFIXES = ( + "app.agent.orchestrator", + "app.agent.callback", + "app.agent.llm.helper", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "langgraph", + "langchain", + "langchain_core", + "openai", + "anthropic", + "google.genai", + "boto3", + "botocore", +) +AGENT_SCHEMA_BASELINE_PREFIXES = ("langchain", "langchain_core") +AGENT_NONMATERIALIZATION_PREFIXES = tuple( + prefix + for prefix in AGENT_HEAVY_MODULE_PREFIXES + if prefix not in AGENT_SCHEMA_BASELINE_PREFIXES +) +AGENT_TOOL_CATALOG_PREFIXES = ( + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", +) MODULE_PREFIXES = ( "lark_oapi", "slack_bolt", @@ -50,12 +81,10 @@ MODULE_PREFIXES = ( "discord", "plexapi", "telebot", - "langgraph", - "langchain", "app.agent", - "app.agent.orchestrator", "app.agent.tools", "app.modules", + *AGENT_HEAVY_MODULE_PREFIXES, ) BALANCED_RUN_ORDER = ( ("before", 1), @@ -593,6 +622,7 @@ def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, s "MP_PERF_ACTIVATION_TIMEOUT": str( getattr(args, "activation_timeout", 180) ), + "MP_PERF_AGENT_MODULE_PREFIXES": ",".join(AGENT_HEAVY_MODULE_PREFIXES), } ) return environment @@ -1133,7 +1163,7 @@ def capture_activation_snapshot( output_dir: Path, phase: str, ) -> dict[str, Any]: - """采集浏览器激活边界的 Engine、进程和进程内 import 状态。""" + """采集场景动作边界的 Engine、进程和进程内 import 状态。""" engine = capture_engine_stats(container) processes = capture_processes(container) modules = capture_modules(container, output_dir, processes["main_python"]) @@ -1248,17 +1278,195 @@ def evaluate_browser_activation( } -def activate_browser_scenario( +def _agent_prefix_counts(snapshot: dict[str, Any]) -> dict[str, int]: + """从模块快照提取 PERF-003 Agent 重模块哨兵。""" + counts = snapshot["modules"].get("prefix_counts") or {} + return { + prefix: int(counts.get(prefix) or 0) for prefix in AGENT_HEAVY_MODULE_PREFIXES + } + + +def evaluate_agent_activation( + scenario: str, + pre: dict[str, Any], + post: dict[str, Any], + marker: dict[str, Any], + expected_pid: Optional[int] = None, +) -> dict[str, Any]: + """验证禁用态路由与首次工具目录的惰性物化不变量。""" + agent = marker.get("agent") or {} + observations = agent.get("observations") or {} + runtime_before = observations.get("before") or {} + runtime_after = observations.get("after") or {} + prefix_before = _agent_prefix_counts(pre) + prefix_after = _agent_prefix_counts(post) + forbidden_before = { + prefix: prefix_before[prefix] + for prefix in AGENT_NONMATERIALIZATION_PREFIXES + if prefix_before[prefix] + } + pre_xvfb = pre["processes"]["xvfb"] + post_xvfb = post["processes"]["xvfb"] + network_delta = { + "rx_bytes": int(post["engine"]["network_rx_bytes"]) + - int(pre["engine"]["network_rx_bytes"]), + "tx_bytes": int(post["engine"]["network_tx_bytes"]) + - int(pre["engine"]["network_tx_bytes"]), + } + errors: list[str] = [] + + if marker.get("scenario") != scenario: + errors.append("进程内 marker 的场景与采集请求不一致") + if expected_pid is not None and marker.get("pid") != expected_pid: + errors.append("进程内 marker 不是目标 MoviePilot Python 进程写出") + if not marker.get("success") or not agent.get("success"): + errors.append("主 MoviePilot Python 进程未完成 Agent 场景动作") + if forbidden_before: + errors.append("Agent 场景动作前已经加载必须延迟物化的模块") + if pre_xvfb["count"] != 0 or post_xvfb["count"] != 0: + errors.append("Agent 场景不得物化 Xvfb") + if not runtime_before.get("available") or not runtime_after.get("available"): + errors.append("主进程未提供轻量 Agent runtime 只读观测") + if runtime_before.get("tool_factory_materialized") is not False: + errors.append("Agent 场景动作前工具工厂必须未物化") + if any(network_delta.values()): + errors.append("Agent 场景动作产生了容器网络收发") + + revision = {"plugin": None, "factory": None} + action_summary: dict[str, Any] + if scenario == "agent-disabled-router": + router = agent.get("router_openapi") or {} + if router.get("ai_agent_enable") is not False: + errors.append("router/OpenAPI 场景必须运行在 AI_AGENT_ENABLE=false") + if router.get("missing_routes") or router.get("missing_openapi_paths"): + errors.append("禁用态缺少 Agent 相关 router 或 OpenAPI path") + forbidden_after = { + prefix: prefix_after[prefix] + for prefix in AGENT_NONMATERIALIZATION_PREFIXES + if prefix_after[prefix] + } + if forbidden_after: + errors.append("生成完整 OpenAPI 后加载了必须延迟物化的模块") + if runtime_after.get("tool_factory_materialized") is not False: + errors.append("生成完整 OpenAPI 不得物化工具工厂") + action_summary = { + "route_count": router.get("route_count"), + "openapi_path_count": router.get("openapi_path_count"), + "openapi_sha256": router.get("openapi_sha256"), + } + elif scenario == "agent-tool-catalog": + catalog = agent.get("tool_catalog") or {} + if prefix_after["app.agent.tools.factory"] < 1: + errors.append("首次工具目录动作后未加载工具工厂") + if prefix_after["app.agent.tools.impl"] < 1: + errors.append("首次工具目录动作后未加载工具实现") + allowed_prefixes = { + *AGENT_TOOL_CATALOG_PREFIXES, + *AGENT_SCHEMA_BASELINE_PREFIXES, + } + unexpected_prefixes = { + prefix: count + for prefix, count in prefix_after.items() + if prefix not in allowed_prefixes and count + } + if unexpected_prefixes: + errors.append("首次工具目录动作加载了非目录所需的 Agent/provider 重模块") + if runtime_after.get("tool_factory_materialized") is not True: + errors.append("首次工具目录动作后工具工厂未标记为已物化") + if ( + not catalog.get("success") + or not catalog.get("tool_count") + or catalog.get("schema_count") != catalog.get("tool_count") + or catalog.get("catalog_entry_count") != catalog.get("tool_count") + or catalog.get("collision_names") + or not catalog.get("schema_digests_complete") + or not catalog.get("repeat_stable") + or catalog.get("repeat_tool_count") != catalog.get("tool_count") + ): + errors.append("工具目录、JSON Schema 或重复读取稳定性不满足合同") + if catalog.get("plugin_revision") is None or not catalog.get( + "factory_revision" + ): + errors.append("工具目录缺少 plugin/factory revision") + revision = { + "plugin": catalog.get("plugin_revision"), + "factory": catalog.get("factory_revision"), + } + action_summary = { + "tool_count": catalog.get("tool_count"), + "schema_count": catalog.get("schema_count"), + "schemas_sha256": catalog.get("schemas_sha256"), + "collision_names": catalog.get("collision_names") or [], + "repeat_stable": catalog.get("repeat_stable"), + } + else: + errors.append(f"未知 Agent 场景：{scenario}") + action_summary = {} + + return { + "passed": not errors, + "errors": errors, + "expected": ( + "router/OpenAPI 完整且必须延迟物化的模块保持 0" + if scenario == "agent-disabled-router" + else "首次工具目录后仅物化工具域及 Schema 基线" + ), + "observed": { + "pre_xvfb_count": pre_xvfb["count"], + "post_xvfb_count": post_xvfb["count"], + "prefix_before": prefix_before, + "prefix_after": prefix_after, + "network_delta": network_delta, + "tool_factory_materialized_before": runtime_before.get( + "tool_factory_materialized" + ), + "tool_factory_materialized_after": runtime_after.get( + "tool_factory_materialized" + ), + }, + "action": action_summary, + "revision": revision, + } + + +def evaluate_scenario_activation( + scenario: str, + pre: dict[str, Any], + post: dict[str, Any], + marker: dict[str, Any], + expected_pid: Optional[int] = None, +) -> dict[str, Any]: + """按场景族分派外部采样验收。""" + if scenario in BROWSER_SCENARIOS: + return evaluate_browser_activation( + scenario, + pre, + post, + marker, + expected_pid=expected_pid, + ) + if scenario in AGENT_SCENARIOS: + return evaluate_agent_activation( + scenario, + pre, + post, + marker, + expected_pid=expected_pid, + ) + raise HarnessError(f"未知激活场景：{scenario}") + + +def activate_sample_scenario( container, output_dir: Path, scenario: str, timeout: float, ) -> dict[str, Any]: - """通过 SIGUSR2 让目标 MoviePilot 解释器执行场景激活并回收 marker。""" + """通过 SIGUSR2 让目标 MoviePilot 解释器执行动作并回收 marker。""" pre = capture_activation_snapshot(container, output_dir, "pre-activation") main_python = pre["processes"]["main_python"] if not main_python: - raise HarnessError("未找到主 Python 进程，无法触发浏览器场景") + raise HarnessError("未找到主 Python 进程，无法触发测量场景") marker_path = output_dir / "modules" / f"activation-{main_python['pid']}.json" marker_path.unlink(missing_ok=True) @@ -1274,12 +1482,12 @@ def activate_browser_scenario( raise HarnessError("等待场景激活 marker 时容器提前退出") time.sleep(0.05) if not marker_path.exists(): - raise HarnessError(f"浏览器场景激活在 {timeout:.0f}s 内未完成") + raise HarnessError(f"测量场景动作在 {timeout:.0f}s 内未完成") marker_received_at = time.monotonic() marker = json.loads(marker_path.read_text(encoding="utf-8")) post = capture_activation_snapshot(container, output_dir, "post-activation") - validation = evaluate_browser_activation( + validation = evaluate_scenario_activation( scenario, pre, post, @@ -1304,7 +1512,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: """执行一个隔离样本并在约定时间点采集完整指标。""" scenario = getattr(args, "scenario", DEFAULT_SCENARIO) if scenario != DEFAULT_SCENARIO and args.variant != "after": - raise HarnessError("浏览器激活场景只用于验证包含 app.sdk.browser 的 After 候选") + raise HarnessError("非默认场景只用于验证包含候选公共 API 的 After 版本") client = require_docker_client() build = load_build_manifest(args) config_seed, browser_seed = require_seed_volumes(client, args) @@ -1391,7 +1599,7 @@ def command_sample(args: argparse.Namespace) -> dict[str, Any]: measurement_origin_at = settled_at if scenario != DEFAULT_SCENARIO: - activation = activate_browser_scenario( + activation = activate_sample_scenario( container, output_dir, scenario, @@ -1614,8 +1822,13 @@ def build_markdown_report( ) lines.append("| " + " | ".join(row) + " |") - activated_samples = [sample for sample in samples if sample.get("activation")] - if activated_samples: + browser_activated_samples = [ + sample + for sample in samples + if sample.get("activation") + and sample.get("scenario", DEFAULT_SCENARIO) in BROWSER_SCENARIOS + ] + if browser_activated_samples: activation_headers = [ "场景", "版本", @@ -1644,7 +1857,7 @@ def build_markdown_report( ] ) for sample in sorted( - activated_samples, + browser_activated_samples, key=lambda item: ( item.get("scenario", DEFAULT_SCENARIO), variant_order.get(item["variant"], 99), @@ -1692,6 +1905,144 @@ def build_markdown_report( ] lines.append("| " + " | ".join(activation_row) + " |") + agent_activated_samples = [ + sample + for sample in samples + if sample.get("activation") + and sample.get("scenario", DEFAULT_SCENARIO) in AGENT_SCENARIOS + ] + if agent_activated_samples: + agent_headers = [ + "场景", + "样本", + "动作(s)", + "Pre/Post WS(MiB)", + "Pre/Post Python PSS(MiB)", + "Pre/Post sys.modules", + "Factory observation", + "模块哨兵 Pre", + "模块哨兵 Post", + "Router/OpenAPI 或 Tools/Schemas", + "Plugin/Factory revision", + "Action RX/TX Δ(KiB)", + "验收", + ] + lines.extend( + [ + "", + "## Agent 场景动作", + "", + "| " + " | ".join(agent_headers) + " |", + "| " + " | ".join(["---"] * len(agent_headers)) + " |", + ] + ) + + def format_prefix_counts(counts: dict[str, int]) -> str: + """仅展开已加载前缀，全部未加载时输出明确零状态。""" + loaded = [f"{prefix}={count}" for prefix, count in counts.items() if count] + return ", ".join(loaded) if loaded else "全部 0" + + for sample in sorted( + agent_activated_samples, + key=lambda item: ( + item.get("scenario", DEFAULT_SCENARIO), + item["sample_index"], + ), + ): + activation = sample["activation"] + pre = activation["pre"] + post = activation["post"] + validation = activation["validation"] + observed = validation["observed"] + action = validation["action"] + revision = validation["revision"] + pre_python = pre["processes"].get("main_python") or {} + post_python = post["processes"].get("main_python") or {} + if sample.get("scenario") == "agent-disabled-router": + action_result = ( + f"{action.get('route_count')}/{action.get('openapi_path_count')}" + ) + else: + action_result = ( + f"{action.get('tool_count')}/{action.get('schema_count')}; " + f"repeat={'Y' if action.get('repeat_stable') else 'N'}; " + f"collision={len(action.get('collision_names') or [])}" + ) + factory_revision = str(revision.get("factory") or "") + revision_result = ( + f"{revision.get('plugin')}/{factory_revision[:12]}" + if factory_revision + else "不适用" + ) + agent_row = [ + sample.get("scenario", DEFAULT_SCENARIO), + str(sample["sample_index"]), + f"{float(activation.get('worker_elapsed_seconds') or 0):.2f}", + f"{format_mib(pre['engine']['working_set_bytes'])}/" + f"{format_mib(post['engine']['working_set_bytes'])}", + f"{format_kib_as_mib(pre_python.get('pss_kib'))}/" + f"{format_kib_as_mib(post_python.get('pss_kib'))}", + f"{pre['modules'].get('count')}/{post['modules'].get('count')}", + ( + f"{observed.get('tool_factory_materialized_before')}→" + f"{observed.get('tool_factory_materialized_after')}" + ), + format_prefix_counts(observed.get("prefix_before") or {}), + format_prefix_counts(observed.get("prefix_after") or {}), + action_result, + revision_result, + ( + f"{format_bytes_as_kib(post['engine']['network_rx_bytes'] - pre['engine']['network_rx_bytes'])}/" + f"{format_bytes_as_kib(post['engine']['network_tx_bytes'] - pre['engine']['network_tx_bytes'])}" + ), + "通过" if validation["passed"] else "失败", + ] + lines.append("| " + " | ".join(agent_row) + " |") + + sentinel_samples = [sample for sample in samples if sample.get("measurements")] + if sentinel_samples: + lines.extend( + [ + "", + "## Agent 模块哨兵", + "", + "每行记录该样本所有定时采样点的最大模块数；精确时间点数据保留在 JSON。", + "`langchain` 与 `langchain_core` 只记录 Schema 基线，不参与归零门禁。", + "", + "| 场景 | 版本 | 样本 | 重模块峰值 |", + "| --- | --- | --- | --- |", + ] + ) + for sample in sorted( + sentinel_samples, + key=lambda item: ( + item.get("scenario", DEFAULT_SCENARIO), + variant_order.get(item["variant"], 99), + item["sample_index"], + ), + ): + peaks = { + prefix: max( + int( + measurement.get("modules", {}) + .get("prefix_counts", {}) + .get(prefix, 0) + ) + for measurement in sample["measurements"] + ) + for prefix in AGENT_HEAVY_MODULE_PREFIXES + } + peak_text = ( + ", ".join( + f"{prefix}={count}" for prefix, count in peaks.items() if count + ) + or "全部 0" + ) + lines.append( + f"| {sample.get('scenario', DEFAULT_SCENARIO)} | " + f"{sample['variant']} | {sample['sample_index']} | {peak_text} |" + ) + lines.extend(["", "## 中位数对照", ""]) for scenario in scenarios: scenario_samples = [ diff --git a/scripts/perf/test_scenarios.py b/scripts/perf/test_scenarios.py index 013952e2f..f111bcc40 100644 --- a/scripts/perf/test_scenarios.py +++ b/scripts/perf/test_scenarios.py @@ -36,6 +36,29 @@ def snapshot(xvfb_count: int, xvfb_pss_kib: int = 0) -> dict: } +def agent_snapshot(prefix_counts: dict[str, int], xvfb_count: int = 0) -> dict: + """构造包含 Agent 模块哨兵的场景边界快照。""" + return { + "engine": { + "working_set_bytes": 500 * 1024 * 1024, + "network_rx_bytes": 1024, + "network_tx_bytes": 512, + }, + "processes": { + "main_python": { + "pss_kib": 400 * 1024, + "uss_kib": 390 * 1024, + "threads": 8, + }, + "xvfb": {"count": xvfb_count, "pss_kib": 0}, + }, + "modules": { + "count": 3000, + "prefix_counts": prefix_counts, + }, + } + + def managed_resource(before_generation: int, after_generation: int) -> dict: """构造 host.display single-flight 观测。""" observations = [] @@ -141,12 +164,54 @@ def test_browser_scenario_uses_isolated_resource_and_result_names( ) -def test_browser_scenario_rejects_before_without_touching_docker() -> None: - """旧基线不具备 SDK/display 冷启动不变量，非默认场景只接受 After。""" +def test_agent_scenarios_are_explicit_and_keep_idle_prefix_contract() -> None: + """PERF-003 暴露完整哨兵，并把 Schema 基线排除在归零门禁外。""" + harness = load_module( + "moviepilot_perf_agent_cli", + PERF_DIR / "moviepilot_docker_ab.py", + ) + expected_prefixes = { + "app.agent.orchestrator", + "app.agent.callback", + "app.agent.llm.helper", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "langgraph", + "langchain", + "langchain_core", + "openai", + "anthropic", + "google.genai", + "boto3", + "botocore", + } + + assert set(harness.AGENT_SCENARIOS) == { + "agent-disabled-router", + "agent-tool-catalog", + } + assert set(harness.AGENT_HEAVY_MODULE_PREFIXES) == expected_prefixes + assert expected_prefixes.issubset(harness.MODULE_PREFIXES) + assert set(harness.AGENT_SCHEMA_BASELINE_PREFIXES) == { + "langchain", + "langchain_core", + } + assert not set(harness.AGENT_SCHEMA_BASELINE_PREFIXES).intersection( + harness.AGENT_NONMATERIALIZATION_PREFIXES + ) + + +@pytest.mark.parametrize("scenario", ["browser-headless", "agent-disabled-router"]) +def test_non_default_scenario_rejects_before_without_touching_docker( + scenario: str, +) -> None: + """旧基线不具备候选公共合同，所有非默认场景只接受 After。""" harness = load_module( "moviepilot_perf_after_only", PERF_DIR / "moviepilot_docker_ab.py" ) - args = argparse.Namespace(scenario="browser-headless", variant="before") + args = argparse.Namespace(scenario=scenario, variant="before") with pytest.raises(harness.HarnessError, match="After"): harness.command_sample(args) @@ -222,6 +287,132 @@ def test_activation_validation_enforces_headless_and_headed_invariants() -> None assert invalid["single_flight"]["passed"] is False +def test_agent_activation_validation_enforces_lazy_boundaries() -> None: + """禁用态延迟重 Agent 域，首次目录只允许工具域物化。""" + harness = load_module( + "moviepilot_perf_agent_validation", + PERF_DIR / "moviepilot_docker_ab.py", + ) + zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES} + catalog_loaded = dict(zero) + catalog_loaded["app.agent.tools.base"] = 1 + catalog_loaded["app.agent.tools.catalog"] = 1 + catalog_loaded["app.agent.tools.factory"] = 1 + catalog_loaded["app.agent.tools.impl"] = 82 + schema_baseline = dict(zero) + schema_baseline["langchain_core"] = 5 + router_marker = { + "scenario": "agent-disabled-router", + "pid": 42, + "success": True, + "agent": { + "success": True, + "observations": { + "before": { + "available": True, + "tool_factory_materialized": False, + }, + "after": { + "available": True, + "tool_factory_materialized": False, + }, + }, + "router_openapi": { + "success": True, + "ai_agent_enable": False, + "missing_routes": [], + "missing_openapi_paths": [], + "route_count": 200, + "openapi_path_count": 180, + "openapi_sha256": "schema", + }, + }, + } + catalog_marker = { + "scenario": "agent-tool-catalog", + "pid": 42, + "success": True, + "agent": { + "success": True, + "observations": { + "before": { + "available": True, + "tool_factory_materialized": False, + }, + "after": { + "available": True, + "tool_factory_materialized": True, + }, + }, + "tool_catalog": { + "success": True, + "tool_count": 82, + "schema_count": 82, + "catalog_entry_count": 82, + "collision_names": [], + "plugin_revision": 0, + "factory_revision": "factory-revision", + "schemas_sha256": "schemas", + "schema_digests_complete": True, + "repeat_tool_count": 82, + "repeat_stable": True, + }, + }, + } + + router = harness.evaluate_agent_activation( + "agent-disabled-router", + agent_snapshot(schema_baseline), + agent_snapshot(schema_baseline), + router_marker, + expected_pid=42, + ) + catalog = harness.evaluate_agent_activation( + "agent-tool-catalog", + agent_snapshot(zero), + agent_snapshot(catalog_loaded), + catalog_marker, + expected_pid=42, + ) + invalid_loaded = dict(catalog_loaded) + invalid_loaded["app.agent.orchestrator"] = 1 + invalid = harness.evaluate_agent_activation( + "agent-tool-catalog", + agent_snapshot(zero), + agent_snapshot(invalid_loaded), + catalog_marker, + expected_pid=42, + ) + callback_loaded = dict(catalog_loaded) + callback_loaded["app.agent.callback"] = 1 + invalid_callback = harness.evaluate_agent_activation( + "agent-tool-catalog", + agent_snapshot(zero), + agent_snapshot(callback_loaded), + catalog_marker, + expected_pid=42, + ) + network_post = agent_snapshot(catalog_loaded) + network_post["engine"]["network_tx_bytes"] += 1 + invalid_network = harness.evaluate_agent_activation( + "agent-tool-catalog", + agent_snapshot(zero), + network_post, + catalog_marker, + expected_pid=42, + ) + + assert router["passed"] is True + assert router["action"]["openapi_path_count"] == 180 + assert catalog["passed"] is True + assert catalog["revision"]["factory"] == "factory-revision" + assert invalid["passed"] is False + assert any("非目录" in error for error in invalid["errors"]) + assert invalid_callback["passed"] is False + assert invalid_network["passed"] is False + assert any("网络" in error for error in invalid_network["errors"]) + + def test_sitecustomize_acquires_headed_display_concurrently_in_same_process() -> None: """headed probe 并发走公开 SDK 冷启动，并只保留一个上下文。""" probe = load_module( @@ -283,6 +474,161 @@ def test_sitecustomize_headless_uses_one_headless_context() -> None: assert result["single_flight_probe"]["requested"] is False +def test_sitecustomize_router_probe_generates_complete_openapi_without_http() -> None: + """禁用态探针直接读取主进程 app，不发起 HTTP 或外部请求。""" + probe = load_module( + "moviepilot_perf_sitecustomize_router", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + required_paths = [ + "/api/v1/message/agent/stream", + "/api/v1/message/agent/sessions", + "/api/v1/openai/v1/chat/completions", + "/api/v1/openai/v1/responses", + "/api/v1/anthropic/v1/messages", + "/api/v1/llm/manage", + "/api/v1/mcp", + "/api/v1/mcp/tools", + ] + + class FakeApp: + """只实现 Router/OpenAPI 探针使用的 FastAPI 合同。""" + + routes = [SimpleNamespace(path=path) for path in required_paths] + + @staticmethod + def openapi() -> dict: + return { + "info": {"title": "MoviePilot", "version": "v3"}, + "paths": {path: {"get": {}} for path in required_paths}, + } + + result = probe._probe_router_openapi( + app_instance=FakeApp(), + settings_object=SimpleNamespace(AI_AGENT_ENABLE=False), + ) + + assert result["success"] is True + assert result["route_count"] == len(required_paths) + assert result["openapi_path_count"] == len(required_paths) + assert result["missing_routes"] == [] + assert result["missing_openapi_paths"] == [] + + +def test_sitecustomize_tool_catalog_probe_records_schema_and_revisions() -> None: + """首次目录探针保留工具数、Schema 摘要和双 revision。""" + probe = load_module( + "moviepilot_perf_sitecustomize_catalog", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + definitions = [ + SimpleNamespace( + name="query_media", + input_schema={"type": "object", "properties": {}}, + ), + SimpleNamespace( + name="add_subscribe", + input_schema={"type": "object", "properties": {"title": {}}}, + ), + ] + catalog = SimpleNamespace( + entries=( + SimpleNamespace( + name="query_media", + source="builtin", + schema_digest="a" * 64, + ), + SimpleNamespace( + name="add_subscribe", + source="builtin", + schema_digest="b" * 64, + ), + ), + collisions={}, + plugin_revision=7, + factory_revision="factory-revision", + ) + + class FakeManager: + """按真实管理器合同在 list_tools 后发布 catalog。""" + + def __init__(self) -> None: + self.catalog = None + + def list_tools(self): + self.catalog = catalog + return definitions + + result = probe._probe_tool_catalog(manager=FakeManager()) + + assert result["success"] is True + assert result["tool_count"] == 2 + assert result["schema_count"] == 2 + assert result["plugin_revision"] == 7 + assert result["factory_revision"] == "factory-revision" + assert len(result["schemas_sha256"]) == 64 + assert len(result["catalog_sha256"]) == 64 + assert result["source_counts"] == {"builtin": 2} + assert result["schema_digests_complete"] is True + assert result["repeat_catalog_same_object"] is True + assert result["repeat_revision_unchanged"] is True + assert result["repeat_stable"] is True + + +def test_sitecustomize_agent_scenario_records_before_and_after_observations( + monkeypatch, +) -> None: + """Agent 场景在同一目标解释器内记录模块与 materialization 边界。""" + probe = load_module( + "moviepilot_perf_sitecustomize_agent", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + module_observations = iter( + [ + {"total_modules": 100, "prefix_counts": {}, "matching_modules": []}, + { + "total_modules": 190, + "prefix_counts": { + "app.agent.tools.factory": 1, + "app.agent.tools.impl": 82, + }, + "matching_modules": ["app.agent.tools.factory"], + }, + ] + ) + runtime_observations = iter( + [ + {"available": True, "tool_factory_materialized": False}, + {"available": True, "tool_factory_materialized": True}, + ] + ) + monkeypatch.setattr( + probe, + "_agent_module_observation", + lambda: next(module_observations), + ) + monkeypatch.setattr( + probe, + "_probe_tool_catalog", + lambda manager=None: { + "success": True, + "tool_count": 82, + "schema_count": 82, + }, + ) + + result = probe._activate_agent_scenario( + "agent-tool-catalog", + runtime_reader=lambda: next(runtime_observations), + ) + + assert result["success"] is True + assert result["observations"]["before"]["tool_factory_materialized"] is False + assert result["observations"]["after"]["tool_factory_materialized"] is True + assert result["modules"]["before"]["total_modules"] == 100 + assert result["modules"]["after"]["total_modules"] == 190 + + def test_sitecustomize_serializes_managed_resource_facade(monkeypatch) -> None: """进程探针按公开只读 facade 记录 generation 与 activate observation。""" observation = SimpleNamespace( @@ -357,6 +703,28 @@ def test_sitecustomize_signal_worker_publishes_atomic_marker(tmp_path: Path) -> assert not list(tmp_path.glob("*.tmp")) +def test_sitecustomize_signal_worker_dispatches_agent_scenario(tmp_path: Path) -> None: + """SIGUSR2 worker 对 Agent 场景也在当前 PID 发布完整 marker。""" + probe = load_module( + "moviepilot_perf_sitecustomize_agent_marker", + PERF_DIR / "instrument" / "sitecustomize.py", + ) + probe._OUTPUT_DIR = str(tmp_path) + probe._SCENARIO = "agent-disabled-router" + probe._activate_agent_scenario = lambda scenario: { + "success": scenario == "agent-disabled-router" + } + + probe._run_activation() + + marker_path = tmp_path / f"activation-{os.getpid()}.json" + payload = json.loads(marker_path.read_text(encoding="utf-8")) + assert payload["pid"] == os.getpid() + assert payload["scenario"] == "agent-disabled-router" + assert payload["agent"]["success"] is True + assert "browser" not in payload + + def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> None: """非默认场景报告包含激活证据，并按场景隔离中位数。""" harness = load_module( @@ -428,3 +796,73 @@ def test_markdown_reports_activation_and_keeps_scenario_medians_separate() -> No assert "Single-flight" in report assert "### `browser-headed`" in report assert "1.25" in report + + +def test_markdown_reports_agent_observation_revision_and_sentinel() -> None: + """Agent 场景报告展示物化边界、revision 与定时哨兵峰值。""" + harness = load_module( + "moviepilot_perf_agent_report", + PERF_DIR / "moviepilot_docker_ab.py", + ) + zero = {prefix: 0 for prefix in harness.AGENT_HEAVY_MODULE_PREFIXES} + loaded = dict(zero) + loaded["app.agent.tools.factory"] = 1 + loaded["app.agent.tools.impl"] = 82 + pre = agent_snapshot(zero) + post = agent_snapshot(loaded) + activation = { + "worker_elapsed_seconds": 2.5, + "pre": pre, + "post": post, + "marker": {"success": True}, + "validation": { + "passed": True, + "observed": { + "prefix_before": zero, + "prefix_after": loaded, + "tool_factory_materialized_before": False, + "tool_factory_materialized_after": True, + }, + "action": { + "tool_count": 82, + "schema_count": 82, + "repeat_stable": True, + "collision_names": [], + }, + "revision": {"plugin": 7, "factory": "1234567890abcdef"}, + }, + } + sample = { + "scenario": "agent-tool-catalog", + "variant": "after", + "sample_index": 1, + "http_ready_seconds": 7.0, + "activation": activation, + "measurements": [ + { + "target_minute": 1.0, + "engine": post["engine"], + "processes": post["processes"], + "modules": { + "count": 3082, + "prefix_counts": loaded, + }, + } + ], + } + build = { + "campaign": "fake", + "platform": "linux/arm64", + "before_commit": "before", + "after_commit": "after", + "substrate": {"reference": "frozen"}, + } + + report = harness.build_markdown_report(build, None, [sample]) + + assert "## Agent 场景动作" in report + assert "False→True" in report + assert "82/82; repeat=Y; collision=0" in report + assert "7/1234567890ab" in report + assert "## Agent 模块哨兵" in report + assert "app.agent.tools.impl=82" in report diff --git a/tests/test_agent_api_lazy_imports.py b/tests/test_agent_api_lazy_imports.py new file mode 100644 index 000000000..c3f5eb23e --- /dev/null +++ b/tests/test_agent_api_lazy_imports.py @@ -0,0 +1,460 @@ +"""Agent API 路由与禁用响应的延迟加载合同。""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +def _run_isolated(script: str, config_dir: Path) -> dict: + """在隔离解释器中执行路由探针，并返回末行 JSON 结果。""" + env = os.environ.copy() + env.update( + { + "AI_AGENT_ENABLE": "false", + "API_TOKEN": "test-agent-api-token-1234", + "CONFIG_DIR": str(config_dir), + "PYTHONDONTWRITEBYTECODE": "1", + } + ) + completed = subprocess.run( + [sys.executable, "-c", script], + check=True, + capture_output=True, + text=True, + env=env, + ) + lines = [line for line in completed.stdout.splitlines() if line.strip()] + return json.loads(lines[-1]) + + +def test_full_api_openapi_keeps_agent_runtime_cold(tmp_path: Path) -> None: + """完整路由与 OpenAPI 注册不得物化 Agent、工具或模型运行时。""" + result = _run_isolated( + r''' +import json +import socket +import sys +import types + +network_attempts = [] + +def block_network(*args, **kwargs): + network_attempts.append(repr(args[:2])) + raise AssertionError("router import attempted network access") + +socket.create_connection = block_network +socket.getaddrinfo = block_network +socket.socket.connect = block_network + +sites = types.ModuleType("app.application.site.sites") +sites.SitesHelper = type("SitesHelper", (), {}) +sites.__file__ = "" +sys.modules["app.application.site.sites"] = sites + +from fastapi import FastAPI +from app.startup.routers_initializer import init_routers + +app = FastAPI() +init_routers(app) +paths = set(app.openapi()["paths"]) +required_paths = { + "/api/v1/message/agent/stream", + "/api/v1/message/agent/sessions", + "/api/v1/openai/v1/chat/completions", + "/api/v1/openai/v1/responses", + "/api/v1/anthropic/v1/messages", + "/api/v1/llm/manage", + "/api/v1/mcp", + "/api/v1/mcp/tools", +} +forbidden = ( + "app.agent.callback", + "app.agent.llm.helper", + "app.agent.orchestrator", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "langgraph", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) +) +print(json.dumps({ + "loaded": loaded, + "missing_paths": sorted(required_paths - paths), + "network_attempts": network_attempts, +})) +''', + tmp_path / "router-import", + ) + + assert result == { + "loaded": [], + "missing_paths": [], + "network_attempts": [], + } + + +def test_disabled_protocol_requests_preserve_503_without_runtime_load( + tmp_path: Path, +) -> None: + """禁用态兼容协议保持 503，并且不会因构造响应加载 Agent。""" + result = _run_isolated( + r''' +import asyncio +import json +import socket +import sys +import types +from types import SimpleNamespace + +network_attempts = [] + +def block_network(*args, **kwargs): + network_attempts.append(repr(args[:2])) + raise AssertionError("disabled request attempted network access") + +socket.create_connection = block_network +socket.getaddrinfo = block_network +socket.socket.connect = block_network + +sites = types.ModuleType("app.application.site.sites") +sites.SitesHelper = type("SitesHelper", (), {}) +sites.__file__ = "" +sys.modules["app.application.site.sites"] = sites + +from fastapi.security import HTTPAuthorizationCredentials +from app import schemas +from app.api.endpoints.anthropic import messages as anthropic_messages +from app.api.endpoints.openai import chat_completions, responses +from app.runtime.config import settings + +credentials = HTTPAuthorizationCredentials( + scheme="Bearer", + credentials=settings.API_TOKEN, +) +request = SimpleNamespace(headers={}) + +async def run_requests(): + chat_response = await chat_completions( + payload=schemas.OpenAIChatCompletionsRequest( + messages=[schemas.OpenAIChatMessage(role="user", content="hello")] + ), + request=request, + credentials=credentials, + ) + responses_response = await responses( + payload=schemas.OpenAIResponsesRequest(input="hello"), + credentials=credentials, + ) + anthropic_response = await anthropic_messages( + payload=schemas.AnthropicMessagesRequest( + messages=[schemas.AnthropicMessage(role="user", content="hello")] + ), + x_api_key=settings.API_TOKEN, + ) + return chat_response, responses_response, anthropic_response + +protocol_responses = asyncio.run(run_requests()) +forbidden = ( + "app.agent.callback", + "app.agent.llm.helper", + "app.agent.orchestrator", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "langgraph", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) +) +print(json.dumps({ + "loaded": loaded, + "network_attempts": network_attempts, + "status_codes": [response.status_code for response in protocol_responses], + "bodies": [json.loads(response.body) for response in protocol_responses], +}, ensure_ascii=False)) +''', + tmp_path / "disabled-requests", + ) + + assert result["loaded"] == [] + assert result["network_attempts"] == [] + assert result["status_codes"] == [503, 503, 503] + assert result["bodies"][0]["error"]["code"] == "ai_agent_disabled" + assert result["bodies"][1]["error"]["code"] == "ai_agent_disabled" + assert result["bodies"][2]["error"]["type"] == "api_error" + + +def test_runtime_agent_type_factories_are_single_flight(tmp_path: Path) -> None: + """并发首次解析必须返回同一 class，避免会话复用误判构造器已变化。""" + result = _run_isolated( + r''' +import json +import sys +import threading +import time +import types + +sites = types.ModuleType("app.application.site.sites") +sites.SitesHelper = type("SitesHelper", (), {}) +sites.__file__ = "" +sys.modules["app.application.site.sites"] = sites + +from app.api.endpoints import agent, openai + +def exercise(module, factory_name, getter_name): + calls = [] + call_lock = threading.Lock() + start = threading.Barrier(8) + + class RuntimeAgent: + pass + + def get_runtime_type(): + with call_lock: + calls.append(1) + time.sleep(0.02) + return RuntimeAgent + + setattr(module, getter_name, get_runtime_type) + factory = getattr(module, factory_name) + results = [] + + def resolve(): + start.wait() + results.append(factory()) + + threads = [threading.Thread(target=resolve) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + return len(calls), all(result is results[0] for result in results) + +web_calls, web_identity = exercise( + agent, + "_get_web_agent_type", + "get_moviepilot_agent_type", +) +collecting_calls, collecting_identity = exercise( + openai, + "_get_collecting_agent_type", + "get_moviepilot_agent_type", +) +print(json.dumps({ + "web_calls": web_calls, + "web_identity": web_identity, + "collecting_calls": collecting_calls, + "collecting_identity": collecting_identity, +})) +''', + tmp_path / "agent-type-single-flight", + ) + + assert result == { + "web_calls": 1, + "web_identity": True, + "collecting_calls": 1, + "collecting_identity": True, + } + + +def test_persistent_protocol_agent_rebinds_stream_queue_without_stale_output( + tmp_path: Path, +) -> None: + """稳定协议会话复用 Agent 时必须保留 handler identity 并切换请求队列。""" + result = _run_isolated( + r''' +import asyncio +import json +import sys +import types + +sites = types.ModuleType("app.application.site.sites") +sites.SitesHelper = type("SitesHelper", (), {}) +sites.__file__ = "" +sys.modules["app.application.site.sites"] = sites + +from app.api.endpoints import openai + +class RuntimeAgent: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.stream_handler = object() + self._compiled_agent_bundle = object() + +class TestStreamingHandler(openai._OpenAIStreamingHandlerMixin): + pass + +openai._get_openai_streaming_handler_type = lambda: TestStreamingHandler +agent_type = openai._build_collecting_agent_type(RuntimeAgent) +agent = agent_type(session_id="stable", user_id="api") +first_queue = asyncio.Queue() +second_queue = asyncio.Queue() + +agent.configure_protocol_request(stream_mode=True, event_queue=first_queue) +handler = agent.stream_handler +handler._event_queue.put_nowait("first") +compiled_bundle = object() +agent._compiled_agent_bundle = compiled_bundle + +agent.configure_protocol_request(stream_mode=True, event_queue=second_queue) +agent.release_protocol_request(first_queue) +handler._event_queue.put_nowait("second") +agent.release_protocol_request(second_queue) + +print(json.dumps({ + "same_handler": agent.stream_handler is handler, + "same_bundle": agent._compiled_agent_bundle is compiled_bundle, + "first": first_queue.get_nowait(), + "first_empty": first_queue.empty(), + "second": second_queue.get_nowait(), + "second_empty": second_queue.empty(), + "released": handler._event_queue is None, +})) +''', + tmp_path / "protocol-stream-rebind", + ) + + assert result == { + "same_handler": True, + "same_bundle": True, + "first": "first", + "first_empty": True, + "second": "second", + "second_empty": True, + "released": True, + } + + +def test_protocol_routes_follow_agent_service_lifecycle(tmp_path: Path) -> None: + """服务未运行时返回 503，运行态仍执行原有兼容协议响应流程。""" + result = _run_isolated( + r''' +import asyncio +import json +import socket +import sys +import types +from types import SimpleNamespace + +network_attempts = [] + +def block_network(*args, **kwargs): + network_attempts.append(repr(args[:2])) + raise AssertionError("protocol lifecycle test attempted network access") + +socket.create_connection = block_network +socket.getaddrinfo = block_network +socket.socket.connect = block_network + +sites = types.ModuleType("app.application.site.sites") +sites.SitesHelper = type("SitesHelper", (), {}) +sites.__file__ = "" +sys.modules["app.application.site.sites"] = sites + +from fastapi.security import HTTPAuthorizationCredentials +from app import schemas +from app.api.endpoints import anthropic, openai +from app.runtime.config import settings + +settings.AI_AGENT_ENABLE = True +credentials = HTTPAuthorizationCredentials( + scheme="Bearer", + credentials=settings.API_TOKEN, +) +request = SimpleNamespace(headers={}) +chat_payload = schemas.OpenAIChatCompletionsRequest( + messages=[schemas.OpenAIChatMessage(role="user", content="hello")] +) +anthropic_payload = schemas.AnthropicMessagesRequest( + messages=[schemas.AnthropicMessage(role="user", content="hello")] +) + +async def run_unavailable(): + return ( + await openai.chat_completions(chat_payload, request, credentials), + await anthropic.messages( + anthropic_payload, + x_api_key=settings.API_TOKEN, + ), + ) + +unavailable = asyncio.run(run_unavailable()) + +class RuntimeAgent: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.stream_handler = object() + self._compiled_agent_bundle = None + + async def process(self, _prompt, **_kwargs): + return "runtime reply" + +class RunningManager: + async def process_message(self, **kwargs): + agent = kwargs["agent_factory"]( + session_id=kwargs["session_id"], + user_id=kwargs["user_id"], + channel=kwargs["channel"], + source=kwargs["source"], + username=kwargs["username"], + ) + kwargs["agent_setup"](agent) + return await agent.process( + kwargs["message"], + images=kwargs["images"], + files=kwargs["files"], + ) + + async def clear_session(self, **_kwargs): + return None + +running_manager = RunningManager() +openai.get_running_agent_manager = lambda: running_manager +anthropic.get_running_agent_manager = lambda: running_manager +openai.get_moviepilot_agent_type = lambda: RuntimeAgent + +async def run_available(): + return ( + await openai.chat_completions(chat_payload, request, credentials), + await anthropic.messages( + anthropic_payload, + x_api_key=settings.API_TOKEN, + ), + ) + +available = asyncio.run(run_available()) +openai_body = json.loads(available[0].body) +print(json.dumps({ + "unavailable_status": [response.status_code for response in unavailable], + "unavailable_codes": [ + json.loads(unavailable[0].body)["error"]["code"], + json.loads(unavailable[1].body)["error"]["type"], + ], + "available_openai": openai_body["choices"][0]["message"]["content"], + "available_anthropic": available[1].content[0].text, + "network_attempts": network_attempts, +}, ensure_ascii=False)) +''', + tmp_path / "protocol-service-lifecycle", + ) + + assert result == { + "unavailable_status": [503, 503], + "unavailable_codes": ["ai_agent_unavailable", "api_error"], + "available_openai": "runtime reply", + "available_anthropic": "runtime reply", + "network_attempts": [], + } diff --git a/tests/test_agent_background_output.py b/tests/test_agent_background_output.py index 736a55225..bed906a57 100644 --- a/tests/test_agent_background_output.py +++ b/tests/test_agent_background_output.py @@ -407,7 +407,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): patch("app.agent.orchestrator.prompt_manager.get_agent_prompt", return_value="PROMPT"), patch("app.agent.orchestrator.create_subagent_middlewares", return_value=([], [])), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -462,7 +462,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): patch("app.agent.orchestrator.prompt_manager.get_agent_prompt", return_value="PROMPT"), patch("app.agent.orchestrator.create_subagent_middlewares", return_value=([], [])), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -514,7 +514,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): patch("app.agent.orchestrator.prompt_manager.get_agent_prompt", return_value="PROMPT"), patch("app.agent.orchestrator.create_subagent_middlewares", return_value=([], [])), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -603,7 +603,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): patch("app.agent.orchestrator.prompt_manager.get_agent_prompt", return_value="PROMPT"), patch("app.agent.orchestrator.create_subagent_middlewares", return_value=([], [])), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -667,7 +667,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): ), ), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -718,7 +718,7 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): patch("app.agent.orchestrator.prompt_manager.get_agent_prompt", return_value="PROMPT"), patch("app.agent.orchestrator.create_subagent_middlewares", return_value=([], [])), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( @@ -766,23 +766,24 @@ class AgentBackgroundOutputTest(unittest.IsolatedAsyncioTestCase): async def test_run_background_prompt_forces_disable_message_tools_when_capture_only(self): captured = {} + manager = AgentManager() - async def fake_process(self, message, images=None, files=None): - captured["message"] = message - captured["reply_mode"] = self.reply_mode - captured["allow_message_tools"] = self.allow_message_tools - captured["user_id"] = self.user_id + async def fake_process(task): + captured["message"] = task.message + captured["reply_mode"] = task.reply_mode + captured["allow_message_tools"] = task.allow_message_tools + captured["user_id"] = task.user_id - with ( - patch.object(MoviePilotAgent, "process", new=fake_process), - patch.object(MoviePilotAgent, "cleanup", new=AsyncMock()), - patch.object(memory_manager, "clear_memory"), - ): - await AgentManager.run_background_prompt( + manager._process_message_internal = fake_process + await manager.initialize() + try: + await manager.run_background_prompt( message="background task", reply_mode=ReplyMode.CAPTURE_ONLY, allow_message_tools=True, ) + finally: + await manager.close() self.assertEqual("background task", captured["message"]) self.assertEqual(ReplyMode.CAPTURE_ONLY, captured["reply_mode"]) diff --git a/tests/test_agent_cancellation.py b/tests/test_agent_cancellation.py index 4ac905288..f8aa80828 100644 --- a/tests/test_agent_cancellation.py +++ b/tests/test_agent_cancellation.py @@ -40,6 +40,7 @@ def test_stop_current_task_cancels_waiters_and_allows_next_message(): async def _run_scenario(): manager = AgentManager() + await manager.initialize() started = asyncio.Event() async def _block_current_task(_task): @@ -96,6 +97,7 @@ def test_stop_current_task_cancels_waiters_and_allows_next_message(): second_waiter, return_exceptions=True, ) + await manager.close() asyncio.run(_run_scenario()) @@ -105,6 +107,7 @@ def test_stop_queues_new_message_until_cancellation_cleanup_finishes(): async def _run_scenario(): manager = AgentManager() + await manager.initialize() current_started = asyncio.Event() cancellation_cleanup_started = asyncio.Event() release_cleanup = asyncio.Event() @@ -149,5 +152,6 @@ def test_stop_queues_new_message_until_cancellation_cleanup_finishes(): assert await asyncio.wait_for(next_waiter, timeout=1) == "next-completed" with pytest.raises(asyncio.CancelledError): await current_waiter + await manager.close() asyncio.run(_run_scenario()) diff --git a/tests/test_agent_doctor_tool.py b/tests/test_agent_doctor_tool.py index b747729b9..830b0c000 100644 --- a/tests/test_agent_doctor_tool.py +++ b/tests/test_agent_doctor_tool.py @@ -3,6 +3,7 @@ import json from datetime import datetime from unittest.mock import patch +from app.agent.tools.catalog import ToolCatalogSnapshot from app.agent.tools.factory import MoviePilotToolFactory from app.agent.tools.impl.query_doctor_report import QueryDoctorReportTool from app.agent.tools.manager import MoviePilotToolsManager @@ -97,14 +98,20 @@ def test_query_doctor_report_compact_mode_omits_details(): def test_mcp_tool_manager_exposes_doctor_report_tool(): """MCP 工具管理器应暴露 doctor 诊断报告工具。""" tool = QueryDoctorReportTool(session_id="doctor-session", user_id="10001") + catalog = ToolCatalogSnapshot.from_tools( + [tool], plugin_revision=0, factory_revision="test" + ) - with patch( - "app.agent.tools.manager.MoviePilotToolFactory.create_tools", - return_value=[tool], - ): + with patch.object( + MoviePilotToolFactory, + "create_catalog", + return_value=catalog, + ) as create_catalog: manager = MoviePilotToolsManager(is_admin=True) + create_catalog.assert_not_called() + tool_definitions = manager.list_tools() + create_catalog.assert_called_once() - tool_definitions = manager.list_tools() assert [item.name for item in tool_definitions] == ["query_doctor_report"] schema = tool_definitions[0].input_schema assert "deep" in schema["properties"] diff --git a/tests/test_agent_graph_cache.py b/tests/test_agent_graph_cache.py index ab957e0a3..3561db338 100644 --- a/tests/test_agent_graph_cache.py +++ b/tests/test_agent_graph_cache.py @@ -434,7 +434,7 @@ async def test_graph_keeps_mcp_first_winner_and_catalogs_all_collisions( side_effect=_capture_subagents, ), patch( - "app.agent.orchestrator.MoviePilotToolFactory.get_tool_selector_always_include_names", + "app.agent.tools.factory.MoviePilotToolFactory.get_tool_selector_always_include_names", return_value=[], ), patch( diff --git a/tests/test_agent_image_capability.py b/tests/test_agent_image_capability.py index b749576b7..a36d0b0ca 100644 --- a/tests/test_agent_image_capability.py +++ b/tests/test_agent_image_capability.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - from unittest.mock import AsyncMock, patch from app.agent import MoviePilotAgent @@ -84,11 +69,13 @@ def test_handle_ai_message_routes_text_only_model_images_to_files(monkeypatch): } ], ) as prepare_files, patch( - "app.application.agent._agent_manager.process_message", new_callable=AsyncMock - ) as process_message, patch( + "app.chain.message.get_running_agent_manager" + ) as get_running_manager, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), ): + process_message = AsyncMock() + get_running_manager.return_value.process_message = process_message chain._handle_ai_message( text="/ai 帮我看看这张图", channel=MessageChannel.Telegram, diff --git a/tests/test_agent_image_support.py b/tests/test_agent_image_support.py index 913b6db0d..2d10ca787 100644 --- a/tests/test_agent_image_support.py +++ b/tests/test_agent_image_support.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import asyncio import base64 import json @@ -301,15 +286,14 @@ class AgentImageSupportTest(unittest.TestCase): "feishu://file/om_audio/file_audio/voice.opus", ] - with patch.object( - AgentCapabilityManager, "is_audio_input_available", return_value=True + with patch( + "app.chain.message.is_audio_input_available", return_value=True ), patch.object( chain, "run_module", side_effect=[b"slack", b"discord", b"qq", b"vocechat", b"synology", b"feishu"], - ) as run_module, patch.object( - AgentCapabilityManager, - "transcribe_audio", + ) as run_module, patch( + "app.chain.message.transcribe_audio", side_effect=[ "slack text", "discord text", @@ -466,11 +450,13 @@ class AgentImageSupportTest(unittest.TestCase): } ], ) as prepare_files, patch( - "app.application.agent._agent_manager.process_message", new_callable=AsyncMock - ) as process_message, patch( + "app.chain.message.get_running_agent_manager" + ) as get_running_manager, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), ) as run_coroutine_threadsafe: + process_message = AsyncMock() + get_running_manager.return_value.process_message = process_message chain._handle_ai_message( text="/ai 帮我看看这张图", channel=MessageChannel.Telegram, @@ -499,11 +485,13 @@ class AgentImageSupportTest(unittest.TestCase): with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( chain, "_get_or_create_session_id", return_value="session-1" ), patch( - "app.application.agent._agent_manager.process_message", new_callable=AsyncMock - ) as process_message, patch( + "app.chain.message.get_running_agent_manager" + ) as get_running_manager, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: coro.close(), ): + process_message = AsyncMock() + get_running_manager.return_value.process_message = process_message chain._handle_ai_message( text="帮我推荐一部电影", channel=MessageChannel.Telegram, diff --git a/tests/test_agent_interaction.py b/tests/test_agent_interaction.py index 40c123c43..9e21e7ef0 100644 --- a/tests/test_agent_interaction.py +++ b/tests/test_agent_interaction.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import asyncio import unittest from datetime import datetime @@ -210,12 +195,13 @@ class TestAgentInteraction(unittest.TestCase): ) as message_add, patch.object( chain, "edit_message", return_value=True ) as edit_message, patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager" + ) as get_running_manager, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): + process_message = AsyncMock() + get_running_manager.return_value.process_message = process_message handled = chain._handle_callback( callback_data=f"agent_interaction:choice:{request.request_id}:1", context=InteractionContext( @@ -282,9 +268,11 @@ class TestAgentInteraction(unittest.TestCase): try: for channel in (MessageChannel.Telegram, MessageChannel.Feishu): + manager = Mock() + manager.matches_secret_confirmation.return_value = True with patch( - "app.application.agent._agent_manager.matches_secret_confirmation", - return_value=True, + "app.chain.message.get_running_agent_manager", + return_value=manager, ), patch.object( chain, "_handle_ai_message", diff --git a/tests/test_agent_lazy_initializer.py b/tests/test_agent_lazy_initializer.py new file mode 100644 index 000000000..f9d8bdd65 --- /dev/null +++ b/tests/test_agent_lazy_initializer.py @@ -0,0 +1,245 @@ +from __future__ import annotations + +import sys +import types +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.runtime.capabilities.errors import CapabilityRuntimeClosedError +from app.startup import agent_initializer + + +@pytest.mark.anyio +async def test_disabled_initializer_does_not_materialize_manager(monkeypatch) -> None: + """功能关闭时启动阶段不得解析完整 Agent 模块。""" + activate = AsyncMock(return_value=None) + monkeypatch.setattr( + agent_initializer, + "activate_agent_service", + activate, + ) + initializer = agent_initializer.AgentInitializer() + + assert await initializer.initialize() is True + assert initializer._initialized is False + activate.assert_awaited_once_with() + + +@pytest.mark.anyio +async def test_cleanup_without_initialized_manager_does_not_query(monkeypatch) -> None: + """清理空状态只能关闭已持有资源，不能为清理而触发首次导入。""" + activate = AsyncMock(side_effect=AssertionError("service activated")) + monkeypatch.setattr(agent_initializer, "activate_agent_service", activate) + + await agent_initializer.AgentInitializer().cleanup() + + activate.assert_not_awaited() + + +@pytest.mark.anyio +async def test_failed_initialize_keeps_manager_for_shutdown_cleanup( + monkeypatch, +) -> None: + """初始化中途失败时仍须保留实际 manager，供应用关闭释放部分资源。""" + manager = AsyncMock() + manager.initialize.side_effect = RuntimeError("partial initialization") + monkeypatch.setattr(agent_initializer.settings, "AI_AGENT_ENABLE", True) + monkeypatch.setattr(agent_initializer, "agent_manager", manager) + initializer = agent_initializer.AgentInitializer() + + assert await initializer.initialize() is False + await initializer.cleanup() + + manager.close.assert_awaited_once_with() + assert initializer._manager is None + + +@pytest.mark.anyio +async def test_compat_stop_closes_injected_manager_without_building_runtime( + monkeypatch, +) -> None: + """显式注入对象由兼容路径关闭，不为其构建空 Capability Runtime。""" + events = [] + manager = AsyncMock() + manager.initialize.side_effect = lambda: events.append("initialize") + manager.close.side_effect = lambda: events.append("close") + monkeypatch.setattr(agent_initializer.settings, "AI_AGENT_ENABLE", True) + monkeypatch.setattr(agent_initializer, "agent_manager", manager) + shutdown = AsyncMock(side_effect=lambda: events.append("shutdown_gate")) + monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown) + monkeypatch.setattr( + agent_initializer, + "agent_initializer", + agent_initializer.AgentInitializer(), + ) + + assert await agent_initializer.init_agent() is True + await agent_initializer.stop_agent() + + assert events == ["initialize", "close"] + manager.initialize.assert_awaited_once_with() + manager.close.assert_awaited_once_with() + shutdown.assert_not_awaited() + + +@pytest.mark.anyio +async def test_production_initializer_delegates_lifecycle_to_runtime( + monkeypatch, +) -> None: + """生产路径只协调 service，不得再次手工 initialize 或 close canonical manager。""" + manager = AsyncMock() + activate = AsyncMock(return_value=manager) + monkeypatch.setattr(agent_initializer, "agent_manager", None) + monkeypatch.setattr(agent_initializer, "activate_agent_service", activate) + initializer = agent_initializer.AgentInitializer() + + assert await initializer.initialize() is True + await initializer.cleanup() + + activate.assert_awaited_once_with() + manager.initialize.assert_not_awaited() + manager.close.assert_not_awaited() + assert initializer._manager is None + + +@pytest.mark.anyio +async def test_production_stop_seals_runtime_without_manually_closing_manager( + monkeypatch, +) -> None: + """生产关闭由 Runtime 关闸并释放 service，initializer 只清理自身引用。""" + manager = AsyncMock() + initializer = agent_initializer.AgentInitializer() + initializer._manager = manager + initializer._initialized = True + initializer._compat_injected = False + shutdown = AsyncMock() + monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", shutdown) + monkeypatch.setattr(agent_initializer, "agent_initializer", initializer) + monkeypatch.setattr( + agent_initializer, + "is_tool_factory_materialized", + lambda: False, + ) + + await agent_initializer.stop_agent() + + shutdown.assert_awaited_once_with() + manager.close.assert_not_awaited() + assert initializer._manager is None + + +@pytest.mark.anyio +async def test_config_listener_delegates_watch_filter_to_runtime(monkeypatch) -> None: + """配置监听器只转交 changed keys，不维护第二份启用开关。""" + manager = AsyncMock() + reconcile = AsyncMock(return_value=manager) + monkeypatch.setattr(agent_initializer, "reconcile_agent_service", reconcile) + initializer = agent_initializer.AgentInitializer() + event = agent_initializer.Event( + agent_initializer.EventType.ConfigChanged, + {"key": {"AI_AGENT_ENABLE"}}, + ) + + await initializer.handle_config_changed(event) + + reconcile.assert_awaited_once_with( + reason="agent_service_config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + assert initializer._manager is manager + assert initializer._initialized is True + + +def test_config_listener_registration_is_idempotent_and_instance_free() -> None: + """重复构造 initializer 不得累积监听器或持有过期实例。""" + subscribers = getattr( + agent_initializer.eventmanager, + "_EventManager__broadcast_subscribers", + ) + AgentInitializer = agent_initializer.AgentInitializer + AgentInitializer() + AgentInitializer() + + listeners = tuple( + subscribers.get(agent_initializer.EventType.ConfigChanged, {}).values() + ) + matching = [ + listener + for listener in listeners + if listener is agent_initializer._handle_agent_config_changed + ] + assert len(matching) == 1 + assert agent_initializer._handle_agent_config_changed.__closure__ is None + + +@pytest.mark.anyio +async def test_config_event_after_shutdown_is_fail_closed(monkeypatch) -> None: + """关闭后的配置事件不得把 service 重新标为初始化成功。""" + reconcile = AsyncMock(side_effect=CapabilityRuntimeClosedError("closed")) + monkeypatch.setattr(agent_initializer, "reconcile_agent_service", reconcile) + initializer = agent_initializer.AgentInitializer() + initializer._shutdown_complete = True + event = agent_initializer.Event( + agent_initializer.EventType.ConfigChanged, + {"key": "AI_AGENT_ENABLE"}, + ) + + await initializer.handle_config_changed(event) + + reconcile.assert_not_awaited() + assert initializer._manager is None + assert initializer._initialized is False + + +@pytest.mark.anyio +async def test_stop_skips_tool_executor_cleanup_when_factory_is_unresolved( + monkeypatch, +) -> None: + """工具能力从未解析时，关闭路径不得为线程池清理导入工具基础模块。""" + fake_base = types.ModuleType("app.agent.tools.base") + cleanup = MagicMock() + fake_base.shutdown_blocking_executors = cleanup + monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base) + monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock()) + monkeypatch.setattr( + agent_initializer, + "is_tool_factory_materialized", + lambda: False, + ) + monkeypatch.setattr( + agent_initializer, + "agent_initializer", + agent_initializer.AgentInitializer(), + ) + + await agent_initializer.stop_agent() + + cleanup.assert_not_called() + + +@pytest.mark.anyio +async def test_stop_closes_tool_executor_after_factory_materialization( + monkeypatch, +) -> None: + """工具能力已解析时，应取消仍排队的阻塞工具任务。""" + fake_base = types.ModuleType("app.agent.tools.base") + cleanup = MagicMock() + fake_base.shutdown_blocking_executors = cleanup + monkeypatch.setitem(sys.modules, "app.agent.tools.base", fake_base) + monkeypatch.setattr(agent_initializer, "begin_agent_shutdown", AsyncMock()) + monkeypatch.setattr( + agent_initializer, + "is_tool_factory_materialized", + lambda: True, + ) + monkeypatch.setattr( + agent_initializer, + "agent_initializer", + agent_initializer.AgentInitializer(), + ) + + await agent_initializer.stop_agent() + + cleanup.assert_called_once_with(cancel_futures=True) diff --git a/tests/test_agent_lazy_runtime_boundary.py b/tests/test_agent_lazy_runtime_boundary.py new file mode 100644 index 000000000..40827576e --- /dev/null +++ b/tests/test_agent_lazy_runtime_boundary.py @@ -0,0 +1,322 @@ +"""Agent 工具与 LLM 入口的延迟加载合同测试。""" + +from __future__ import annotations + +import json +import subprocess +import sys +import threading +import time +from types import SimpleNamespace +from unittest.mock import patch + + +def _run_isolated(script: str) -> dict: + """在全新解释器中执行导入探针，避免当前 pytest 模块缓存干扰。""" + completed = subprocess.run( + [sys.executable, "-c", script], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + +def test_mcp_router_import_keeps_agent_tool_runtime_cold() -> None: + """默认 API 路由加载不得提前物化工具目录或 Agent 编排。""" + result = _run_isolated( + """ +import json +import sys + +import app.api.endpoints.mcp + +forbidden = ( + "app.agent.callback", + "app.agent.orchestrator", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "anthropic", + "boto3", + "google.genai", + "langchain", + "langgraph", + "openai", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) +) +print(json.dumps({"loaded": loaded})) +""" + ) + + assert result == {"loaded": []} + + +def test_agent_initializer_import_only_registers_lazy_providers() -> None: + """组合根导入只注册 provider，不得提前加载 Agent 重量实现。""" + result = _run_isolated( + """ +import json +import sys + +import app.startup.agent_initializer + +forbidden = ( + "app.agent.orchestrator", + "app.agent.llm.capability", + "app.agent.llm.helper", + "app.agent.llm.provider", + "app.agent.prompt", + "app.agent.tools.base", + "app.agent.tools.factory", + "app.agent.tools.impl", + "anthropic", + "langchain", + "langgraph", + "openai", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) +) +print(json.dumps({"loaded": loaded})) +""" + ) + + assert result == {"loaded": []} + + +def test_manager_constructor_and_llm_facade_are_lightweight() -> None: + """构造全局 manager 与导入 LLM facade 都不加载真实目录或 provider。""" + result = _run_isolated( + """ +import json +import sys + +from app.agent.tools.manager import MoviePilotToolsManager +import app.agent.llm + +manager = MoviePilotToolsManager(session_id="lazy", user_id="api") +forbidden = ( + "app.agent.llm.capability", + "app.agent.llm.helper", + "app.agent.llm.provider", + "app.agent.tools.base", + "app.agent.tools.catalog", + "app.agent.tools.factory", + "app.agent.tools.impl", + "anthropic", + "boto3", + "google.genai", + "langchain", + "langchain_core", + "openai", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in forbidden) +) +print(json.dumps({ + "loaded": loaded, + "tools": manager.tools, + "catalog": manager.catalog, +})) +""" + ) + + assert result == {"loaded": [], "tools": [], "catalog": None} + + +def test_tool_catalog_materialization_does_not_load_streaming_callback() -> None: + """工具目录和 schema 首用不应加载仅在真实编排中需要的回调实现。""" + result = _run_isolated( + """ +import json +import sys +from typing import get_args, get_type_hints + +from app.testing.bootstrap import ensure_sites_stub + +ensure_sites_stub() +from app.agent.runtime_loader import get_tool_factory + +factory = get_tool_factory() +catalog = factory.create_catalog(session_id="lazy", user_id="api") +from app.agent.tools.base import MoviePilotTool + +hints = get_type_hints(MoviePilotTool.set_stream_handler) +handler_args = get_args(hints["stream_handler"]) +print(json.dumps({ + "callback_loaded": "app.agent.callback" in sys.modules, + "catalog_entries": len(catalog.entries), + "handler_types": [item.__name__ for item in handler_args], +})) +""" + ) + + assert result["callback_loaded"] is False + assert result["catalog_entries"] > 0 + assert result["handler_types"] == ["_StreamingHandlerProtocol", "NoneType"] + + +def test_legacy_streaming_handler_import_keeps_canonical_identity() -> None: + """历史显式与星号导入必须按需返回真实 callback 类。""" + result = _run_isolated( + """ +import json +import sys + +import app.agent.tools.base as base +cold_before_explicit = "app.agent.callback" not in sys.modules +from app.agent.tools.base import StreamingHandler +from app.agent.callback import StreamingHandler as CanonicalStreamingHandler + +namespace = {} +exec("from app.agent.tools.base import *", namespace) +print(json.dumps({ + "cold_before_explicit": cold_before_explicit, + "explicit_identity": StreamingHandler is CanonicalStreamingHandler, + "star_identity": namespace["StreamingHandler"] is CanonicalStreamingHandler, +})) +""" + ) + + assert result == { + "cold_before_explicit": True, + "explicit_identity": True, + "star_identity": True, + } + + +def test_manager_first_catalog_use_is_single_flight(monkeypatch) -> None: + """并发首次查询只能在 manager 锁内建立一次会话工具快照。""" + from app.agent import runtime_loader + from app.agent.tools.manager import MoviePilotToolsManager + + calls: list[tuple[str, str]] = [] + fake_tool = SimpleNamespace( + name="demo", + description="demo tool", + args_schema=None, + _require_admin=False, + ) + + class _Factory: + """记录目录构造次数的轻量工厂替身。""" + + @classmethod + def create_catalog(cls, **kwargs): + calls.append((kwargs["session_id"], kwargs["user_id"])) + time.sleep(0.05) + return SimpleNamespace(tools=[fake_tool], plugin_revision=0) + + monkeypatch.setattr(runtime_loader, "get_tool_factory", lambda: _Factory) + manager = MoviePilotToolsManager(session_id="session", user_id="user") + results: list[list[str]] = [] + + def _list_tools() -> None: + results.append([tool.name for tool in manager.list_tools()]) + + threads = [threading.Thread(target=_list_tools) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert calls == [("session", "user")] + assert results == [["demo"], ["demo"]] + assert manager.tools == [fake_tool] + assert manager.catalog is not None + + +def test_legacy_explicit_tool_refresh_keeps_atomic_catalog_contract( + monkeypatch, +) -> None: + """插件显式刷新旧入口应继续发布同一次构造的完整目录快照。""" + from app.agent import runtime_loader + from app.agent.tools.manager import MoviePilotToolsManager + + calls: list[int] = [] + fake_tool = SimpleNamespace(name="plugin_tool") + fake_catalog = SimpleNamespace( + tools=[fake_tool], + plugin_revision=9, + ) + + class _Factory: + """提供固定 revision 快照的轻量工厂替身。""" + + @classmethod + def create_catalog(cls, **_kwargs): + calls.append(1) + return fake_catalog + + monkeypatch.setattr(runtime_loader, "get_tool_factory", lambda: _Factory) + manager = MoviePilotToolsManager(session_id="session", user_id="user") + + manager._load_tools() + + assert calls == [1] + assert manager.catalog is fake_catalog + assert manager.tools == [fake_tool] + assert manager._plugin_agent_tools_revision == 9 + + +def test_reply_mode_identity_and_display_message_contract() -> None: + """旧编排路径必须复用同一枚举，展示消息委托保持原有结构。""" + from app.agent.contracts import ReplyMode, build_display_message + from app.agent.orchestrator import MoviePilotAgent + from app.agent.orchestrator import ReplyMode as LegacyReplyMode + + assert LegacyReplyMode is ReplyMode + + contract_message = build_display_message( + role="assistant", + content="done", + attachments=[{"name": "report.txt"}], + status="streaming", + ) + legacy_message = MoviePilotAgent.build_display_message( + role="assistant", + content="done", + attachments=[{"name": "report.txt"}], + status="streaming", + ) + + for message in (contract_message, legacy_message): + assert message["id"].startswith("assistant-") + assert isinstance(message["createdAt"], int) + message.pop("id") + message.pop("createdAt") + assert legacy_message == contract_message + + +def test_llm_facade_resolves_only_requested_public_module() -> None: + """访问 capability 导出时不应顺带加载 helper 或 provider registry。""" + result = _run_isolated( + """ +import json +import sys + +import app.agent.llm as llm +capability = llm.AgentCapabilityManager +print(json.dumps({ + "module": capability.__module__, + "helper_loaded": "app.agent.llm.helper" in sys.modules, + "provider_loaded": "app.agent.llm.provider" in sys.modules, +})) +""" + ) + + assert result == { + "module": "app.agent.llm.capability", + "helper_loaded": False, + "provider_loaded": False, + } diff --git a/tests/test_agent_lifecycle.py b/tests/test_agent_lifecycle.py index a14dce458..39d61077e 100644 --- a/tests/test_agent_lifecycle.py +++ b/tests/test_agent_lifecycle.py @@ -5,6 +5,7 @@ import pytest import app.agent.orchestrator as agent_module from app.agent import AgentManager +from app.agent.orchestrator import AgentManagerUnavailableError from app.agent.memory import MemoryManager from app.startup import agent_initializer, modules_initializer @@ -154,3 +155,179 @@ async def test_disabled_agent_does_not_create_background_tasks(monkeypatch) -> N assert await agent_initializer.init_agent() is True manager.initialize.assert_not_awaited() + + +@pytest.mark.anyio +async def test_agent_manager_acceptance_gate_rejects_stale_references( + monkeypatch, +) -> None: + """未启动和关闭后的 manager 引用不得创建队列、worker 或 Agent。""" + manager = AgentManager() + memory_manager = MemoryManager() + monkeypatch.setattr(agent_module, "memory_manager", memory_manager) + + with pytest.raises(AgentManagerUnavailableError): + await manager.process_message("before-init", "1", "hello") + + await manager.initialize() + manager._process_message_internal = AsyncMock(return_value="accepted") + assert await manager.process_message( + "running", + "1", + "hello", + wait_for_completion=True, + ) == "accepted" + await manager.close() + + with pytest.raises(AgentManagerUnavailableError): + await manager.process_message("after-close", "1", "hello") + assert manager._session_queues == {} + assert manager._session_workers == {} + assert manager.active_agents == {} + + +@pytest.mark.anyio +async def test_agent_manager_close_serializes_racing_enqueue_and_clear( + monkeypatch, +) -> None: + """关闭、临时会话清理和迟到请求必须串行收口且只清理一次。""" + manager = AgentManager() + memory_manager = MemoryManager() + monkeypatch.setattr(agent_module, "memory_manager", memory_manager) + started = asyncio.Event() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + created = [] + cleanup_calls = [] + + class BlockingAgent: + """用于放大 close 与请求级 clear 竞态窗口。""" + + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + created.append(self) + + async def process(self, _message, **_kwargs): + started.set() + await asyncio.Event().wait() + + async def cleanup(self): + cleanup_calls.append(self) + cleanup_started.set() + await release_cleanup.wait() + + await manager.initialize() + waiter = asyncio.create_task( + manager.process_message( + "closing", + "1", + "hello", + agent_factory=BlockingAgent, + wait_for_completion=True, + ) + ) + await asyncio.wait_for(started.wait(), timeout=1) + close_task = asyncio.create_task(manager.close()) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + late_enqueue = asyncio.create_task( + manager.process_message("late", "1", "hello") + ) + request_clear = asyncio.create_task(manager.clear_session("closing", "1")) + await asyncio.sleep(0) + assert not late_enqueue.done() + assert not request_clear.done() + + release_cleanup.set() + await asyncio.wait_for(close_task, timeout=1) + with pytest.raises(AgentManagerUnavailableError): + await late_enqueue + await request_clear + with pytest.raises(AgentManagerUnavailableError): + await waiter + + assert len(created) == 1 + assert cleanup_calls == created + assert manager._session_queues == {} + assert manager._session_workers == {} + assert manager.active_agents == {} + + +@pytest.mark.anyio +async def test_clear_session_settles_current_and_queued_waiters(monkeypatch) -> None: + """清空会话必须同时结束正在执行和尚未执行的等待请求。""" + manager = AgentManager() + memory_manager = MemoryManager() + monkeypatch.setattr(agent_module, "memory_manager", memory_manager) + started = asyncio.Event() + + async def block_current(_task): + started.set() + await asyncio.Event().wait() + + manager._process_message_internal = block_current + await manager.initialize() + current_waiter = asyncio.create_task( + manager.process_message( + "session-with-queue", + "1", + "current", + wait_for_completion=True, + ) + ) + await asyncio.wait_for(started.wait(), timeout=1) + queued_waiter = asyncio.create_task( + manager.process_message( + "session-with-queue", + "1", + "queued", + wait_for_completion=True, + ) + ) + await asyncio.sleep(0) + + await asyncio.wait_for( + manager.clear_session("session-with-queue", "1"), + timeout=1, + ) + + with pytest.raises(asyncio.CancelledError): + await current_waiter + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(queued_waiter, timeout=1) + assert "session-with-queue" not in manager._session_queues + assert "session-with-queue" not in manager._session_workers + await manager.close() + + +@pytest.mark.anyio +async def test_background_prompt_is_owned_and_cancelled_by_manager_close( + monkeypatch, +) -> None: + """后台 prompt 必须进入 manager worker，关闭时同步结束且不残留临时会话。""" + manager = AgentManager() + memory_manager = MemoryManager() + monkeypatch.setattr(agent_module, "memory_manager", memory_manager) + started = asyncio.Event() + + async def block_background(task): + assert task.session_id.startswith("__managed_background_") + started.set() + await asyncio.Event().wait() + + manager._process_message_internal = block_background + await manager.initialize() + execution = asyncio.create_task( + manager.run_background_prompt( + "background", + session_prefix="__managed_background", + ) + ) + await asyncio.wait_for(started.wait(), timeout=1) + assert len(manager._session_workers) == 1 + + await manager.close() + with pytest.raises(AgentManagerUnavailableError): + await execution + assert manager._session_queues == {} + assert manager._session_workers == {} + assert manager.active_agents == {} diff --git a/tests/test_agent_message_routing.py b/tests/test_agent_message_routing.py index b75d43c0c..cbd9d177e 100644 --- a/tests/test_agent_message_routing.py +++ b/tests/test_agent_message_routing.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import asyncio from unittest.mock import AsyncMock, Mock, patch @@ -78,13 +63,13 @@ def test_explicit_ai_message_bypasses_pending_media_interaction(): def test_explicit_ai_message_is_not_recorded_to_message_history(): """显式 /ai 消息不登记到数据库或实时消息队列。""" chain = MessageChain() + manager = Mock(process_message=AsyncMock()) with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( chain, "_record_user_message" ) as record_user_message, patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", return_value=manager + ), patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): @@ -97,17 +82,17 @@ def test_explicit_ai_message_is_not_recorded_to_message_history(): ) record_user_message.assert_not_called() - process_message.assert_called_once() + manager.process_message.assert_called_once() def test_message_chain_passes_stable_channel_admin_principal_to_agent(): """消息链应将渠道适配器生成的管理员事实传给 Agent。""" chain = MessageChain() + manager = Mock(process_message=AsyncMock()) with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", return_value=manager + ), patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): @@ -120,17 +105,17 @@ def test_message_chain_passes_stable_channel_admin_principal_to_agent(): text="/ai 检查系统状态", ) - assert process_message.call_args.kwargs["is_channel_admin"] is True + assert manager.process_message.call_args.kwargs["is_channel_admin"] is True def test_message_chain_does_not_trust_channel_display_username(): """消息链应保留适配器给出的明确非管理员结论。""" chain = MessageChain() + manager = Mock(process_message=AsyncMock()) with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", return_value=manager + ), patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): @@ -143,17 +128,17 @@ def test_message_chain_does_not_trust_channel_display_username(): text="/ai 检查系统状态", ) - assert process_message.call_args.kwargs["is_channel_admin"] is False + assert manager.process_message.call_args.kwargs["is_channel_admin"] is False def test_message_chain_uses_same_admin_contract_for_slack(): """管理员事实透传应复用于其他消息渠道，而不是 Telegram 特判。""" chain = MessageChain() + manager = Mock(process_message=AsyncMock()) with patch.object(settings, "AI_AGENT_ENABLE", True), patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", return_value=manager + ), patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): @@ -166,7 +151,7 @@ def test_message_chain_uses_same_admin_contract_for_slack(): text="/ai 检查系统状态", ) - assert process_message.call_args.kwargs["is_channel_admin"] is True + assert manager.process_message.call_args.kwargs["is_channel_admin"] is True def test_ask_user_choice_message_is_not_recorded_to_message_history(): @@ -267,6 +252,7 @@ def test_agent_choice_callback_is_not_recorded_to_message_history(): AgentInteractionOption(label="电视剧", value="我选择电视剧"), ], ) + manager = Mock(process_message=AsyncMock()) try: with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( @@ -274,9 +260,8 @@ def test_agent_choice_callback_is_not_recorded_to_message_history(): ) as record_user_message, patch.object( chain, "edit_message", return_value=True ), patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", return_value=manager + ), patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ): @@ -296,5 +281,5 @@ def test_agent_choice_callback_is_not_recorded_to_message_history(): agent_interaction_manager.clear() record_user_message.assert_not_called() - process_message.assert_called_once() - assert process_message.call_args.kwargs["is_channel_admin"] is False + manager.process_message.assert_called_once() + assert manager.process_message.call_args.kwargs["is_channel_admin"] is False diff --git a/tests/test_agent_protocol_lifecycle.py b/tests/test_agent_protocol_lifecycle.py new file mode 100644 index 000000000..e2475b368 --- /dev/null +++ b/tests/test_agent_protocol_lifecycle.py @@ -0,0 +1,216 @@ +"""兼容协议请求的 AgentManager ownership 与关闭竞态合同。""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import patch + +from fastapi.security import HTTPAuthorizationCredentials + +from app import schemas +from app.api.endpoints import anthropic, openai +from app.runtime.config import settings + +_API_TOKEN = "test-agent-protocol-token" + + +class _ManagerClosedError(RuntimeError): + """模拟 enqueue 时 manager 已关闭的 acceptance gate 错误。""" + + code = "agent_manager_unavailable" + + +class _ClosingManager: + """拒绝新任务并记录请求级清理的 manager 替身。""" + + def __init__(self) -> None: + self.process_calls = [] + self.clear_calls = [] + + async def process_message(self, **kwargs): + self.process_calls.append(kwargs) + raise _ManagerClosedError("AgentManager 已关闭") + + async def clear_session(self, **kwargs): + self.clear_calls.append(kwargs) + + async def stop_current_task(self, _session_id): + return False + + +async def _collect(response) -> str: + """收集 StreamingResponse 的全部文本块。""" + chunks = [] + async for chunk in response.body_iterator: + chunks.append(chunk.decode("utf-8") if isinstance(chunk, bytes) else chunk) + return "".join(chunks) + + +def test_streaming_protocols_reject_config_disable_before_manager_lookup() -> None: + """配置关闭后流式请求保持 503，且不得接触运行态 manager。""" + credentials = HTTPAuthorizationCredentials( + scheme="Bearer", + credentials=_API_TOKEN, + ) + openai_payload = schemas.OpenAIChatCompletionsRequest( + messages=[schemas.OpenAIChatMessage(role="user", content="hello")], + stream=True, + ) + anthropic_payload = schemas.AnthropicMessagesRequest( + messages=[schemas.AnthropicMessage(role="user", content="hello")], + stream=True, + ) + + async def scenario(): + return ( + await openai.chat_completions( + openai_payload, + SimpleNamespace(headers={}), + credentials, + ), + await anthropic.messages( + anthropic_payload, + x_api_key=_API_TOKEN, + ), + ) + + with patch.object(settings, "AI_AGENT_ENABLE", False), patch.object( + settings, + "API_TOKEN", + _API_TOKEN, + ), patch.object( + openai, + "get_running_agent_manager", + ) as openai_manager, patch.object( + anthropic, + "get_running_agent_manager", + ) as anthropic_manager: + responses = asyncio.run(scenario()) + + assert [response.status_code for response in responses] == [503, 503] + openai_manager.assert_not_called() + anthropic_manager.assert_not_called() + + +def test_openai_stream_rejects_shutdown_race_and_cleans_request_session() -> None: + """随机 OpenAI 流在 enqueue 竞态失败时返回协议错误并清理临时会话。""" + manager = _ClosingManager() + credentials = HTTPAuthorizationCredentials( + scheme="Bearer", + credentials=_API_TOKEN, + ) + payload = schemas.OpenAIChatCompletionsRequest( + messages=[schemas.OpenAIChatMessage(role="user", content="hello")], + stream=True, + ) + + async def scenario() -> str: + response = await openai.chat_completions( + payload, + SimpleNamespace(headers={}), + credentials, + ) + return await _collect(response) + + with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( + settings, + "API_TOKEN", + _API_TOKEN, + ), patch.object( + openai, + "get_running_agent_manager", + return_value=manager, + ): + body = asyncio.run(scenario()) + + assert '"type": "server_error"' in body + assert "data: [DONE]" in body + assert len(manager.process_calls) == 1 + assert manager.process_calls[0]["wait_for_completion"] is True + assert callable(manager.process_calls[0]["agent_setup"]) + assert len(manager.clear_calls) == 1 + + +def test_anthropic_stream_rejects_shutdown_race_and_cleans_request_session() -> None: + """Anthropic 流在 enqueue 竞态失败时返回 error 终态并清理临时会话。""" + manager = _ClosingManager() + payload = schemas.AnthropicMessagesRequest( + messages=[schemas.AnthropicMessage(role="user", content="hello")], + stream=True, + ) + + async def scenario() -> str: + response = await anthropic.messages( + payload, + x_api_key=_API_TOKEN, + ) + return await _collect(response) + + with patch.object(settings, "AI_AGENT_ENABLE", True), patch.object( + settings, + "API_TOKEN", + _API_TOKEN, + ), patch.object( + anthropic, + "get_running_agent_manager", + return_value=manager, + ): + body = asyncio.run(scenario()) + + assert "event: error" in body + assert "event: message_stop" in body + assert len(manager.process_calls) == 1 + assert manager.process_calls[0]["wait_for_completion"] is True + assert callable(manager.process_calls[0]["agent_setup"]) + assert len(manager.clear_calls) == 1 + + +def test_managed_protocol_request_releases_its_stream_queue() -> None: + """协议请求完成后不应由持久会话 Agent 继续强引用请求队列。""" + event_queue = asyncio.Queue() + created_agents = [] + + class ProtocolAgent: + """记录请求绑定与释放的最小协议 Agent。""" + + def __init__(self, **_kwargs): + self.collected_messages = ["done"] + self.bound_queue = None + created_agents.append(self) + + def configure_protocol_request(self, *, stream_mode, event_queue): + assert stream_mode is True + self.bound_queue = event_queue + + def release_protocol_request(self, queue): + if self.bound_queue is queue: + self.bound_queue = None + + class RunningManager: + """在 worker 边界执行 agent_setup 的 manager 替身。""" + + async def process_message(self, **kwargs): + agent = kwargs["agent_factory"]() + kwargs["agent_setup"](agent) + return "done" + + async def scenario(): + with patch.object( + openai, + "_get_collecting_agent_type", + return_value=ProtocolAgent, + ): + return await openai._run_managed_agent( + manager=RunningManager(), + session_id="persistent", + user_id="1", + username="api", + source="openai", + prompt="hello", + images=[], + stream_mode=True, + event_queue=event_queue, + ) + + assert asyncio.run(scenario()) == ("done", ["done"]) + assert len(created_agents) == 1 + assert created_agents[0].bound_queue is None diff --git a/tests/test_agent_recognize_captcha_tool.py b/tests/test_agent_recognize_captcha_tool.py index 345156f08..3a386c5ce 100644 --- a/tests/test_agent_recognize_captcha_tool.py +++ b/tests/test_agent_recognize_captcha_tool.py @@ -3,6 +3,7 @@ import base64 import json from unittest.mock import patch +from app.agent.tools.catalog import ToolCatalogSnapshot from app.agent.tools.factory import MoviePilotToolFactory from app.agent.tools.impl.recognize_captcha import RecognizeCaptchaTool from app.agent.tools.manager import MoviePilotToolsManager @@ -45,14 +46,20 @@ def test_factory_registers_recognize_captcha_tool(): def test_mcp_tool_manager_exposes_recognize_captcha_schema(): """MCP 工具管理器应暴露验证码识别工具参数。""" tool = RecognizeCaptchaTool(session_id="captcha-session", user_id="10001") + catalog = ToolCatalogSnapshot.from_tools( + [tool], plugin_revision=0, factory_revision="test" + ) - with patch( - "app.agent.tools.manager.MoviePilotToolFactory.create_tools", - return_value=[tool], - ): + with patch.object( + MoviePilotToolFactory, + "create_catalog", + return_value=catalog, + ) as create_catalog: manager = MoviePilotToolsManager(is_admin=True) + create_catalog.assert_not_called() + tool_definitions = manager.list_tools() + create_catalog.assert_called_once() - tool_definitions = manager.list_tools() schema = tool_definitions[0].input_schema assert [item.name for item in tool_definitions] == ["recognize_captcha"] diff --git a/tests/test_agent_resource_flow_permissions.py b/tests/test_agent_resource_flow_permissions.py index 6f594677a..8df8b4040 100644 --- a/tests/test_agent_resource_flow_permissions.py +++ b/tests/test_agent_resource_flow_permissions.py @@ -5,6 +5,8 @@ import json from types import SimpleNamespace from unittest.mock import AsyncMock, patch +from app.agent.tools.catalog import ToolCatalogSnapshot +from app.agent.tools.factory import MoviePilotToolFactory from app.agent.tools.impl.edit_file import EditFileTool from app.agent.tools.impl.list_directory import ListDirectoryTool from app.agent.tools.impl.query_downloaders import QueryDownloadersTool @@ -28,10 +30,16 @@ def test_non_admin_manager_exposes_resource_flow_helper_tools(): """普通用户应能看到搜索、订阅、下载流程所需的辅助工具。""" site_tool = QuerySitesTool(session_id="session-1", user_id="10001") downloader_tool = QueryDownloadersTool(session_id="session-1", user_id="10001") + catalog = ToolCatalogSnapshot.from_tools( + [site_tool, downloader_tool], + plugin_revision=0, + factory_revision="test", + ) - with patch( - "app.agent.tools.manager.MoviePilotToolFactory.create_tools", - return_value=[site_tool, downloader_tool], + with patch.object( + MoviePilotToolFactory, + "create_catalog", + return_value=catalog, ): manager = MoviePilotToolsManager(is_admin=False) @@ -48,10 +56,14 @@ def test_non_admin_manager_exposes_restricted_file_tools(): EditFileTool(session_id="session-1", user_id="10001"), ListDirectoryTool(session_id="session-1", user_id="10001"), ] + catalog = ToolCatalogSnapshot.from_tools( + tools, plugin_revision=0, factory_revision="test" + ) - with patch( - "app.agent.tools.manager.MoviePilotToolFactory.create_tools", - return_value=tools, + with patch.object( + MoviePilotToolFactory, + "create_catalog", + return_value=catalog, ): manager = MoviePilotToolsManager(is_admin=False) diff --git a/tests/test_agent_runtime_loader.py b/tests/test_agent_runtime_loader.py new file mode 100644 index 000000000..2cb7fa57a --- /dev/null +++ b/tests/test_agent_runtime_loader.py @@ -0,0 +1,674 @@ +from __future__ import annotations + +import asyncio +import importlib +import sys +import threading +import types +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from app.runtime.capabilities.errors import ( + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.model import ( + CapabilityLifecycleState, + CapabilityMaterializationState, +) + + +@pytest.fixture +def runtime_loader(monkeypatch): + """为每个用例提供未构建、未关闭的 Agent Capability Runtime。""" + from app.agent import runtime_loader as module + + monkeypatch.setattr(module, "_agent_runtime", None) + for implementation_module in ( + "app.agent.orchestrator", + "app.agent.tools.factory", + ): + monkeypatch.delitem(sys.modules, implementation_module, raising=False) + return module + + +class _FakeManager: + """记录异步 service 生命周期，并支持测试控制初始化时序。""" + + def __init__(self) -> None: + self.initialize_calls = 0 + self.close_calls = 0 + self.fail_initialize = False + self.initialize_entered: asyncio.Event | None = None + self.initialize_release: asyncio.Event | None = None + + async def initialize(self) -> None: + self.initialize_calls += 1 + if self.initialize_entered is not None: + self.initialize_entered.set() + if self.initialize_release is not None: + await self.initialize_release.wait() + if self.fail_initialize: + raise RuntimeError("service initialization failed") + + async def close(self) -> None: + self.close_calls += 1 + + +def _fake_agent_modules(manager: object | None = None) -> dict[str, types.ModuleType]: + orchestrator = types.ModuleType("app.agent.orchestrator") + orchestrator.agent_manager = manager if manager is not None else object() + orchestrator.MoviePilotAgent = type("MoviePilotAgent", (), {}) + tools = types.ModuleType("app.agent.tools.factory") + tools.MoviePilotToolFactory = type("MoviePilotToolFactory", (), {}) + return { + orchestrator.__name__: orchestrator, + tools.__name__: tools, + } + + +def test_registry_discovery_does_not_import_agent_implementation( + runtime_loader, + monkeypatch, +) -> None: + """声明发现只能读取 TOML，不得导入 Agent、LLM 或工具实现。""" + imported = [] + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: imported.append(name), + ) + + runtime = runtime_loader._ensure_runtime() + + assert {spec.id for spec in runtime.list_specs()} == { + "agent.manager", + "agent.moviepilot_type", + "agent.service", + "agent.tool_factory", + } + assert imported == [] + + +def test_concurrent_manager_first_use_is_single_flight( + runtime_loader, + monkeypatch, +) -> None: + """并发首用必须只导入一次并向全部调用者发布同一 canonical 对象。""" + modules = _fake_agent_modules() + import_calls = [] + import_entered = threading.Event() + import_release = threading.Event() + + def import_module(name: str): + import_calls.append(name) + import_entered.set() + assert import_release.wait(timeout=5) + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + import_module, + ) + with ThreadPoolExecutor(max_workers=8) as executor: + futures = [executor.submit(runtime_loader.get_agent_manager) for _ in range(8)] + assert import_entered.wait(timeout=5) + import_release.set() + results = [future.result(timeout=5) for future in futures] + + assert all( + result is modules["app.agent.orchestrator"].agent_manager for result in results + ) + assert ( + runtime_loader.get_moviepilot_agent_type() + is modules["app.agent.orchestrator"].MoviePilotAgent + ) + manager_snapshot = runtime_loader._agent_runtime.snapshot("agent.manager") + assert manager_snapshot.materialization is CapabilityMaterializationState.RESOLVED + assert manager_snapshot.lifecycle is CapabilityLifecycleState.DISCOVERED + assert manager_snapshot.visible is False + assert import_calls == ["app.agent.orchestrator"] + + +def test_tool_factory_has_independent_first_use_entrypoint( + runtime_loader, + monkeypatch, +) -> None: + """工具工厂可独立首用，不需要先解析完整 Agent 编排模块。""" + modules = _fake_agent_modules() + import_calls = [] + + def import_module(name: str): + import_calls.append(name) + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + import_module, + ) + + assert runtime_loader.is_tool_factory_materialized() is False + assert ( + runtime_loader.get_tool_factory() + is modules["app.agent.tools.factory"].MoviePilotToolFactory + ) + assert runtime_loader.is_tool_factory_materialized() is True + assert import_calls == ["app.agent.tools.factory"] + + +def test_materialization_query_does_not_construct_runtime( + runtime_loader, + monkeypatch, +) -> None: + """只读物化查询在 Runtime 未构建时必须直接返回 False。""" + build_calls = [] + monkeypatch.setattr( + runtime_loader, + "_build_agent_runtime", + lambda: build_calls.append(True), + ) + + assert runtime_loader.is_tool_factory_materialized() is False + assert runtime_loader.get_running_agent_manager() is None + assert build_calls == [] + + +@pytest.mark.anyio +async def test_service_first_and_entrypoint_first_share_canonical_identity( + runtime_loader, + monkeypatch, +) -> None: + """资源轴和兼容物化轴无论谁先解析，都必须共享 canonical manager。""" + manager = _FakeManager() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + + def import_module(name: str): + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + import_module, + ) + + service_first = await runtime_loader.activate_agent_service() + compat_after = runtime_loader.get_agent_manager() + + assert service_first is manager + assert compat_after is manager + assert runtime_loader.get_running_agent_manager() is manager + assert manager.initialize_calls == 1 + snapshot = runtime_loader._agent_runtime.snapshot("agent.service") + assert snapshot.lifecycle is CapabilityLifecycleState.RUNNING + assert snapshot.visible is True + + await runtime_loader.begin_agent_shutdown() + + assert manager.close_calls == 1 + assert runtime_loader.get_running_agent_manager() is None + with pytest.raises(CapabilityRuntimeClosedError): + runtime_loader.get_agent_manager() + with pytest.raises(CapabilityRuntimeClosedError): + runtime_loader.get_moviepilot_agent_type() + with pytest.raises(CapabilityRuntimeClosedError): + runtime_loader.get_tool_factory() + with pytest.raises(CapabilityRuntimeClosedError): + await runtime_loader.activate_agent_service() + + +@pytest.mark.anyio +async def test_entrypoint_first_then_service_initializes_same_manager_once( + runtime_loader, + monkeypatch, +) -> None: + """兼容 getter 先物化时不初始化，随后 service 只初始化同一对象一次。""" + manager = _FakeManager() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + + def import_module(name: str): + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + import_module, + ) + + assert runtime_loader.get_agent_manager() is manager + assert manager.initialize_calls == 0 + assert await runtime_loader.activate_agent_service() is manager + assert await runtime_loader.activate_agent_service() is manager + assert manager.initialize_calls == 1 + + +@pytest.mark.anyio +async def test_concurrent_service_first_use_initializes_once( + runtime_loader, + monkeypatch, +) -> None: + """并发 service 首启只能 initialize 一次并发布同一实例。""" + manager = _FakeManager() + manager.initialize_entered = asyncio.Event() + manager.initialize_release = asyncio.Event() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + + tasks = [ + asyncio.create_task(runtime_loader.activate_agent_service()) for _ in range(8) + ] + await manager.initialize_entered.wait() + assert runtime_loader.get_agent_manager() is manager + assert runtime_loader.get_running_agent_manager() is None + manager.initialize_release.set() + results = await asyncio.gather(*tasks) + + assert all(result is manager for result in results) + assert manager.initialize_calls == 1 + assert runtime_loader.get_running_agent_manager() is manager + + +@pytest.mark.anyio +async def test_concurrent_service_and_entrypoint_first_use_share_identity( + runtime_loader, + monkeypatch, +) -> None: + """两个 spec 并发首解析时也只能引用同一个 canonical manager。""" + manager = _FakeManager() + modules = _fake_agent_modules(manager) + import_barrier = threading.Barrier(2) + import_calls = [] + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + + def import_module(name: str): + import_calls.append(name) + import_barrier.wait(timeout=5) + monkeypatch.setitem(sys.modules, name, modules[name]) + return modules[name] + + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + import_module, + ) + + service_task = asyncio.create_task(runtime_loader.activate_agent_service()) + entrypoint_task = asyncio.create_task( + asyncio.to_thread(runtime_loader.get_agent_manager) + ) + service, entrypoint = await asyncio.gather(service_task, entrypoint_task) + + assert service is manager + assert entrypoint is manager + assert runtime_loader.get_running_agent_manager() is manager + assert manager.initialize_calls == 1 + assert import_calls == ["app.agent.orchestrator"] * 2 + + await runtime_loader.begin_agent_shutdown() + + +@pytest.mark.anyio +async def test_service_failure_is_not_published_and_requires_retry( + runtime_loader, + monkeypatch, +) -> None: + """初始化失败必须清理候选、保持不可见，并要求显式 retry。""" + manager = _FakeManager() + manager.fail_initialize = True + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + + with pytest.raises(CapabilityOperationError, match="initialization failed"): + await runtime_loader.activate_agent_service() + + assert runtime_loader.get_running_agent_manager() is None + failed = runtime_loader._agent_runtime.snapshot("agent.service") + assert failed.lifecycle is CapabilityLifecycleState.FAILED + assert failed.visible is False + assert manager.close_calls == 1 + with pytest.raises(CapabilityOperationError, match="retry=True"): + await runtime_loader.activate_agent_service() + + manager.fail_initialize = False + assert await runtime_loader.activate_agent_service(retry=True) is manager + assert manager.initialize_calls == 2 + + +@pytest.mark.anyio +async def test_shutdown_racing_service_first_use_fails_closed( + runtime_loader, + monkeypatch, +) -> None: + """关闭与 service 首启竞争时不得发布对象，并须清理已初始化候选。""" + manager = _FakeManager() + manager.initialize_entered = asyncio.Event() + manager.initialize_release = asyncio.Event() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + + activation = asyncio.create_task(runtime_loader.activate_agent_service()) + await manager.initialize_entered.wait() + shutdown = asyncio.create_task(runtime_loader.begin_agent_shutdown()) + await asyncio.sleep(0) + manager.initialize_release.set() + + with pytest.raises(CapabilityRuntimeClosedError): + await activation + await shutdown + + assert runtime_loader.get_running_agent_manager() is None + assert manager.initialize_calls == 1 + assert manager.close_calls == 1 + snapshot = runtime_loader._agent_runtime.snapshot("agent.service") + assert snapshot.lifecycle is CapabilityLifecycleState.STOPPED + assert snapshot.visible is False + + +@pytest.mark.anyio +async def test_disabled_service_reconcile_stays_unmaterialized( + runtime_loader, + monkeypatch, +) -> None: + """selector 为 false 时协调结果为空且不得导入 orchestrator。""" + imported = [] + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: imported.append(name), + ) + + assert await runtime_loader.activate_agent_service() is None + assert runtime_loader.get_running_agent_manager() is None + snapshot = runtime_loader._agent_runtime.snapshot("agent.service") + assert snapshot.materialization is CapabilityMaterializationState.UNRESOLVED + assert snapshot.lifecycle is CapabilityLifecycleState.STOPPED + assert imported == [] + + +@pytest.mark.anyio +async def test_empty_shutdown_does_not_import_agent_or_tools( + runtime_loader, + monkeypatch, +) -> None: + """空载关闭只解析 data-only manifests，不导入编排器或工具实现。""" + imported = [] + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: imported.append(name), + ) + + await runtime_loader.begin_agent_shutdown() + + assert imported == [] + assert runtime_loader.get_running_agent_manager() is None + assert runtime_loader.is_tool_factory_materialized() is False + + +@pytest.mark.anyio +async def test_disable_reconcile_waits_for_concurrent_service_start( + runtime_loader, + monkeypatch, +) -> None: + """关闭配置与首启竞争时必须等待启动并最终撤销实例。""" + manager = _FakeManager() + manager.initialize_entered = asyncio.Event() + manager.initialize_release = asyncio.Event() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + + activation = asyncio.create_task(runtime_loader.activate_agent_service()) + await manager.initialize_entered.wait() + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + disable = asyncio.create_task( + runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + ) + await asyncio.sleep(0) + manager.initialize_release.set() + + assert await activation is manager + assert await disable is None + assert runtime_loader.get_running_agent_manager() is None + assert manager.initialize_calls == 1 + assert manager.close_calls == 1 + + +@pytest.mark.anyio +async def test_config_reconcile_hot_switches_service_generations( + runtime_loader, + monkeypatch, +) -> None: + """watch 命中的配置切换应停止并重启同一 canonical service。""" + manager = _FakeManager() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + + assert await runtime_loader.activate_agent_service() is None + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"UNRELATED"}, + retry=True, + ) + is None + ) + assert manager.initialize_calls == 0 + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is manager + ) + assert manager.initialize_calls == 1 + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is None + ) + assert manager.close_calls == 1 + assert runtime_loader.get_running_agent_manager() is None + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is manager + ) + assert manager.initialize_calls == 2 + assert runtime_loader._agent_runtime.snapshot("agent.service").generation == 3 + + +@pytest.mark.anyio +async def test_config_reconcile_after_shutdown_cannot_restart_service( + runtime_loader, + monkeypatch, +) -> None: + """Runtime 关闭后即使 selector 再次为 true，配置协调也必须拒绝重启。""" + manager = _FakeManager() + modules = _fake_agent_modules(manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + monkeypatch.setattr( + "app.agent.capabilities.adapter.importlib.import_module", + lambda name: ( + monkeypatch.setitem(sys.modules, name, modules[name]) or modules[name] + ), + ) + + assert await runtime_loader.activate_agent_service() is manager + await runtime_loader.begin_agent_shutdown() + + with pytest.raises(CapabilityRuntimeClosedError): + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + assert manager.initialize_calls == 1 + assert manager.close_calls == 1 + assert runtime_loader.get_running_agent_manager() is None + + +@pytest.mark.anyio +async def test_real_agent_manager_can_restart_across_config_generations( + runtime_loader, + monkeypatch, +) -> None: + """真实 AgentManager 在配置热切换后必须重新建立运行代际。""" + orchestrator = importlib.import_module("app.agent.orchestrator") + + manager = orchestrator.AgentManager() + memory_events = [] + + async def close_memory() -> None: + memory_events.append("close") + + monkeypatch.setattr( + orchestrator.memory_manager, + "initialize", + lambda: memory_events.append("initialize"), + ) + monkeypatch.setattr(orchestrator.memory_manager, "close", close_memory) + monkeypatch.setattr(orchestrator, "agent_manager", manager) + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + + assert await runtime_loader.activate_agent_service() is None + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is manager + ) + assert manager._accepting_tasks is True + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + False, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is None + ) + assert manager._accepting_tasks is False + + monkeypatch.setattr( + "app.agent.capabilities.adapter.settings.AI_AGENT_ENABLE", + True, + ) + assert ( + await runtime_loader.reconcile_agent_service( + reason="config_changed", + changed_keys={"AI_AGENT_ENABLE"}, + retry=True, + ) + is manager + ) + assert manager._accepting_tasks is True + assert memory_events == ["initialize", "close", "initialize"] + + await runtime_loader.begin_agent_shutdown() + + assert manager._accepting_tasks is False + assert memory_events == ["initialize", "close", "initialize", "close"] diff --git a/tests/test_agent_scheduled_tasks.py b/tests/test_agent_scheduled_tasks.py index 88fdd3b87..eaaac8c6c 100644 --- a/tests/test_agent_scheduled_tasks.py +++ b/tests/test_agent_scheduled_tasks.py @@ -385,7 +385,15 @@ async def test_interrupted_date_task_manual_run_disables_and_removes_job( scheduler.init_agent_task_jobs() process_message = AsyncMock(return_value="执行完成") - monkeypatch.setattr("app.agent.orchestrator.agent_manager.process_message", process_message) + manager = SimpleNamespace( + execute_scheduled_task=AgentManager.execute_scheduled_task, + process_message=process_message, + ) + manager.execute_scheduled_task = AgentManager.execute_scheduled_task.__get__(manager) + monkeypatch.setattr( + "app.agent.runtime_loader.get_running_agent_manager", + lambda: manager, + ) assert await scheduler.execute_agent_task( task.id, @@ -410,7 +418,10 @@ async def test_scheduler_propagates_scheduled_trigger_source(monkeypatch) -> Non task = _add_agent_task("cron", "0 * * * *", "scheduled-source") scheduler = _build_agent_task_scheduler() execute = AsyncMock(return_value=(True, "执行完成")) - monkeypatch.setattr("app.agent.orchestrator.agent_manager.execute_scheduled_task", execute) + monkeypatch.setattr( + "app.agent.runtime_loader.get_running_agent_manager", + lambda: SimpleNamespace(execute_scheduled_task=execute), + ) assert await scheduler.execute_agent_task(task.id) == (True, "执行完成") execute.assert_awaited_once_with(task.id, trigger_source="scheduled") @@ -1048,6 +1059,7 @@ async def test_agent_manager_close_finishes_active_and_queued_scheduled_tasks() for index in range(2) ] manager = AgentManager() + await manager.initialize() started = asyncio.Event() async def block_current_task(_task): @@ -1069,11 +1081,14 @@ async def test_agent_manager_close_finishes_active_and_queued_scheduled_tasks() await manager.close() results = await asyncio.gather(*executions, return_exceptions=True) - assert all(isinstance(result, asyncio.CancelledError) for result in results) + assert all( + result == (False, "Agent 定时任务执行失败：AgentManager 已关闭") + for result in results + ) for task in tasks: completed = AgentTaskOper().get(task.id) assert completed.last_status == "failed" - assert completed.last_result == "Agent 定时任务已取消" + assert completed.last_result == "Agent 定时任务执行失败：AgentManager 已关闭" assert completed.run_count == 1 diff --git a/tests/test_agent_session_status.py b/tests/test_agent_session_status.py index 1d189740f..50c4d1b6b 100644 --- a/tests/test_agent_session_status.py +++ b/tests/test_agent_session_status.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import asyncio import unittest from datetime import datetime, timedelta @@ -109,8 +94,8 @@ class TestAgentSessionStatus(unittest.TestCase): with ( patch( - "app.application.agent._agent_manager.get_session_status", - return_value=status, + "app.chain.message.get_running_agent_manager", + return_value=SimpleNamespace(get_session_status=lambda **_: status), ), patch.object(chain, "post_message") as post_message, ): diff --git a/tests/test_agent_system_settings_tools.py b/tests/test_agent_system_settings_tools.py index aabe73ad9..a22159a4d 100644 --- a/tests/test_agent_system_settings_tools.py +++ b/tests/test_agent_system_settings_tools.py @@ -5,6 +5,8 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest +from app.agent.tools.catalog import ToolCatalogSnapshot +from app.agent.tools.factory import MoviePilotToolFactory from app.agent.tools.impl._system_setting_utils import list_setting_specs from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool from app.agent.tools.impl.update_system_settings import UpdateSystemSettingsTool @@ -330,10 +332,14 @@ class TestAgentSystemSettingsTools(unittest.TestCase): def test_tool_manager_blocks_admin_tools_for_non_admin_context(self): tool = QuerySystemSettingsTool(session_id="session-1", user_id="10001") + catalog = ToolCatalogSnapshot.from_tools( + [tool], plugin_revision=0, factory_revision="test" + ) - with patch( - "app.agent.tools.manager.MoviePilotToolFactory.create_tools", - return_value=[tool], + with patch.object( + MoviePilotToolFactory, + "create_catalog", + return_value=catalog, ): manager = MoviePilotToolsManager(is_admin=False) result = asyncio.run( diff --git a/tests/test_agent_tool_streaming.py b/tests/test_agent_tool_streaming.py index fb0e9987d..558af59ef 100644 --- a/tests/test_agent_tool_streaming.py +++ b/tests/test_agent_tool_streaming.py @@ -12,7 +12,7 @@ from app.agent.callback import StreamingHandler from app.agent.middleware.subagents import is_subagent_stream_metadata from app.agent.tools.base import MoviePilotTool from app.agent.tools.impl.send_voice_message import SendVoiceMessageTool -from app.api.endpoints.openai import _OpenAIStreamingHandler +from app.api.endpoints.openai import _get_openai_streaming_handler_type from app.runtime.config import settings from app.schemas.message import MessageResponse from app.schemas.types import MessageChannel, NotificationType @@ -321,7 +321,7 @@ class TestAgentToolStreaming: def test_openai_streaming_handler_flushes_pending_summary_to_queue(self): """校验 OpenAI 流式处理器将待发送摘要推入队列。""" async def _run(): - handler = _OpenAIStreamingHandler() + handler = _get_openai_streaming_handler_type()() queue: asyncio.Queue = asyncio.Queue() handler.bind_queue(queue) await handler.start_streaming() diff --git a/tests/test_architecture_dependencies.py b/tests/test_architecture_dependencies.py index aab554eda..42bfb72bd 100644 --- a/tests/test_architecture_dependencies.py +++ b/tests/test_architecture_dependencies.py @@ -557,6 +557,16 @@ def test_chain_does_not_import_agent_implementation(): assert violations == {} +def test_agent_application_facade_does_not_import_agent_implementation(): + """Agent application 门面只能接收组合根注入，不能反向解析具体实现。""" + dependencies = _build_module_graph()["app.application.agent"] + assert { + dependency + for dependency in dependencies + if dependency.startswith("app.agent") + } == set() + + def test_agent_tools_do_not_import_entrypoint_internals(): """Agent 工具不得穿透导入 HTTP 端点、调度器与命令注册表内部实现。 diff --git a/tests/test_search_ai_recommend.py b/tests/test_search_ai_recommend.py index 9b3ca080b..be2739fe3 100644 --- a/tests/test_search_ai_recommend.py +++ b/tests/test_search_ai_recommend.py @@ -112,6 +112,14 @@ class SearchChainAIRecommendTest(unittest.IsolatedAsyncioTestCase): "render_system_task_message", return_value="PROMPT", ), + patch( + "app.application.agent.get_prompt_manager", + return_value=prompt_manager, + ), + patch( + "app.application.agent.get_running_agent_manager", + return_value=agent_manager, + ), patch.object( agent_manager, "run_background_prompt", diff --git a/tests/test_telegram_typing_lifecycle.py b/tests/test_telegram_typing_lifecycle.py index dda99f46e..abfec8039 100644 --- a/tests/test_telegram_typing_lifecycle.py +++ b/tests/test_telegram_typing_lifecycle.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import asyncio import threading import time @@ -278,14 +263,15 @@ class TestTelegramTypingLifecycle(unittest.TestCase): ) as start_status, patch( "app.chain.message.settings.AI_AGENT_ENABLE", True ), patch( - "app.application.agent._agent_manager.process_message", - new_callable=AsyncMock, - ) as process_message, patch( + "app.chain.message.get_running_agent_manager", + ) as get_running_manager, patch( "app.chain.message.asyncio.run_coroutine_threadsafe", side_effect=lambda coro, _loop: (coro.close(), Mock())[1], ), patch.object( chain, "_mark_message_processing_finished" ) as finish_status: + process_message = AsyncMock() + get_running_manager.return_value.process_message = process_message chain.handle_message( channel=MessageChannel.Telegram, source="telegram-test", diff --git a/tests/test_transfer_failed_retry_buttons.py b/tests/test_transfer_failed_retry_buttons.py index 223d9e4e5..a08021841 100644 --- a/tests/test_transfer_failed_retry_buttons.py +++ b/tests/test_transfer_failed_retry_buttons.py @@ -1,18 +1,3 @@ -# 把真实 Agent 服务注册进 application 门面（幂等），供测试 patch 门面背后的单例方法。 -from app.agent.llm import AgentCapabilityManager, LLMHelper -from app.agent.orchestrator import agent_manager -from app.agent.prompt import prompt_manager -from app.agent.prompt.transfer_redo import build_manual_redo_prompt -from app.application.agent import register_agent_services - -register_agent_services( - agent_manager=agent_manager, - prompt_manager=prompt_manager, - capability_manager=AgentCapabilityManager, - llm_helper=LLMHelper, - manual_redo_prompt_builder=build_manual_redo_prompt, -) - import unittest import asyncio import sys @@ -147,16 +132,15 @@ class TestTransferFailedRetryButtons(unittest.TestCase): with patch.object(settings, "AI_AGENT_ENABLE", True): with patch( - "app.chain.transfer.TransferHistoryOper" - ) as history_oper_cls, patch( - # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 "app.chain._transfer.TransferHistoryOper" - ) as mixins_history_oper_cls, patch( - "app.chain.transfer.asyncio.run_coroutine_threadsafe", + ) as history_oper_cls, patch( + "app.chain._transfer.build_manual_redo_prompt", + return_value="retry transfer prompt", + ), patch( + "app.chain._transfer.asyncio.run_coroutine_threadsafe", side_effect=_close_pending_coro, ) as run_task: history_oper_cls.return_value.get.return_value = history - mixins_history_oper_cls.return_value.get.return_value = history with patch.object(chain, "post_message") as post_message: chain.handle_failed_transfer_callback( callback_data="transfer_ai_retry_34", @@ -219,21 +203,23 @@ class TestTransferFailedRetryButtons(unittest.TestCase): async def fake_async_post_message(*args, **kwargs): return None + from app.agent.prompt.transfer_redo import build_manual_redo_prompt + + manager = SimpleNamespace(run_background_prompt=fake_run_background_prompt) with patch.object(settings, "AI_AGENT_ENABLE", True): with patch( - "app.chain.transfer.TransferHistoryOper" - ) as history_oper_cls, patch( - # mixin 中按自身模块命名空间解析 TransferHistoryOper，需同步镜像 "app.chain._transfer.TransferHistoryOper" - ) as mixins_history_oper_cls, patch( - "app.application.agent._agent_manager.run_background_prompt", - side_effect=fake_run_background_prompt, + ) as history_oper_cls, patch( + "app.chain._transfer.build_manual_redo_prompt", + side_effect=build_manual_redo_prompt, ), patch( - "app.chain.transfer.asyncio.run_coroutine_threadsafe", + "app.chain._transfer.get_running_agent_manager", + return_value=manager, + ), patch( + "app.chain._transfer.asyncio.run_coroutine_threadsafe", side_effect=_run_pending_coro, ): history_oper_cls.return_value.get.return_value = history - mixins_history_oper_cls.return_value.get.return_value = history with patch.object(chain, "post_message"), patch.object( chain, "async_post_message", side_effect=fake_async_post_message ): diff --git a/tests/test_web_agent_stream.py b/tests/test_web_agent_stream.py index e908cb561..d5948f499 100644 --- a/tests/test_web_agent_stream.py +++ b/tests/test_web_agent_stream.py @@ -6,10 +6,11 @@ from threading import Event as ThreadEvent from types import SimpleNamespace from unittest.mock import AsyncMock, patch +import pytest + from app import schemas from app.agent import ReplyMode, agent_manager from app.api.endpoints.agent import ( - _WebAgentMoviePilotAgent, _WebAgentEventPublisher, _WEB_AGENT_FILE_REGISTRY, _WEB_AGENT_NOTICE_QUEUES, @@ -23,6 +24,7 @@ from app.api.endpoints.agent import ( _collect_web_agent_traditional_events, _dispatch_web_agent_notice_event, _extract_web_agent_notification_from_event_data, + _get_web_agent_type, _has_web_agent_traditional_interaction, _prepare_web_agent_audio_attachment_path, _transcribe_web_agent_audio_refs, @@ -41,6 +43,26 @@ from app.schemas.message import ChannelCapability, ChannelCapabilityManager from app.schemas.types import EventType, MessageChannel, NotificationType +@pytest.fixture(autouse=True) +def _running_agent_service(): + """本文件验证运行态 Web Agent 行为，显式提供已启动的 canonical manager。""" + was_accepting = agent_manager._accepting_tasks + agent_manager._accepting_tasks = True + MessageChain._user_sessions.clear() + try: + with patch( + "app.api.endpoints.agent.get_running_agent_manager", + return_value=agent_manager, + ), patch( + "app.chain.message.get_running_agent_manager", + return_value=agent_manager, + ): + yield + finally: + MessageChain._user_sessions.clear() + agent_manager._accepting_tasks = was_accepting + + def test_split_web_agent_output_extracts_verbose_tool_message(): """应将啰嗦模式工具提示拆成独立工具事件，并保留渠道展示文案。""" events = _split_web_agent_output("准备查询。\n\n⚙️ => 查询站点\n\n已完成") @@ -358,7 +380,7 @@ def test_has_web_agent_traditional_interaction_detects_pending_skills(): def test_web_agent_admin_context_uses_current_user_id(): """Web Agent 工具权限应按当前登录用户 ID 判断管理员身份。""" - agent = _WebAgentMoviePilotAgent( + agent = _get_web_agent_type()( session_id="web-agent:session", user_id="7", channel=MessageChannel.WebAgent.value, @@ -378,7 +400,7 @@ def test_web_agent_admin_context_uses_current_user_id(): def test_web_agent_reused_for_background_task_disables_streaming(): """Web Agent 被后台任务复用且渠道已清空时应改用非流式广播。""" - agent = _WebAgentMoviePilotAgent( + agent = _get_web_agent_type()( session_id="web-agent:scheduled-session", user_id="7", channel=None, @@ -394,7 +416,7 @@ def test_web_agent_reused_for_background_task_disables_streaming(): def test_web_agent_output_callback_receives_only_new_text(): """WebAgent 外部回调应接收增量，同时内部仍保留完整输出。""" outputs = [] - agent = _WebAgentMoviePilotAgent( + agent = _get_web_agent_type()( session_id="web-agent:incremental-output", user_id="7", channel=MessageChannel.WebAgent.value, @@ -414,7 +436,7 @@ def test_web_agent_output_callback_receives_only_new_text(): def test_web_agent_tool_summary_is_emitted_before_following_text(): """Web 工具状态应在调用发生时输出，不能拖到正文结束后。""" outputs = [] - agent = _WebAgentMoviePilotAgent( + agent = _get_web_agent_type()( session_id="web-agent:tool-order", user_id="7", channel=MessageChannel.WebAgent.value, @@ -716,8 +738,8 @@ def test_web_agent_stream_binds_session_to_agent_manager(): try: with patch("app.api.endpoints.agent.settings.AI_AGENT_ENABLE", True), patch( - "app.api.endpoints.agent._WebAgentMoviePilotAgent", - FakeWebAgent, + "app.api.endpoints.agent._get_web_agent_type", + return_value=FakeWebAgent, ): body = asyncio.run(scenario()) @@ -812,8 +834,8 @@ def test_web_agent_stream_emits_secret_result_only_as_protected_event(): try: with patch("app.api.endpoints.agent.settings.AI_AGENT_ENABLE", True), patch( - "app.api.endpoints.agent._WebAgentMoviePilotAgent", - FakeProtectedAgent, + "app.api.endpoints.agent._get_web_agent_type", + return_value=FakeProtectedAgent, ), patch( "app.api.endpoints.agent._save_web_agent_display_snapshot", ) as save_snapshot: @@ -1128,8 +1150,8 @@ def test_web_agent_stop_finishes_stream_without_error(): MessageChain, "bind_user_session", ), patch( - "app.api.endpoints.agent._WebAgentMoviePilotAgent", - BlockingWebAgent, + "app.api.endpoints.agent._get_web_agent_type", + return_value=BlockingWebAgent, ), patch( "app.api.endpoints.agent._save_web_agent_display_snapshot", ): @@ -1143,6 +1165,39 @@ def test_web_agent_stop_finishes_stream_without_error(): assert '"type": "error"' not in body +def test_web_agent_stream_rechecks_running_service_before_enqueue(): + """响应建立后服务若已关闭，生成器必须稳定返回错误且不向旧 manager 入队。""" + payload = schemas.AgentWebChatRequest( + text="检查状态", + session_id="shutdown-race", + ) + request = SimpleNamespace(is_disconnected=AsyncMock(return_value=False)) + user = SimpleNamespace(id=1, name="admin", is_superuser=True) + stale_manager = SimpleNamespace(process_message=AsyncMock()) + + async def scenario(): + response = await web_agent_stream(payload, request, user) + return "".join(await _collect_streaming_response(response)) + + with patch("app.api.endpoints.agent.settings.AI_AGENT_ENABLE", True), patch( + "app.api.endpoints.agent._is_web_agent_traditional_message", + return_value=False, + ), patch( + "app.api.endpoints.agent._has_web_agent_traditional_interaction", + return_value=False, + ), patch( + "app.api.endpoints.agent.get_running_agent_manager", + side_effect=[stale_manager, None], + ), patch( + "app.api.endpoints.agent._save_web_agent_display_snapshot", + ): + body = asyncio.run(scenario()) + + assert '"type": "error"' in body + assert '"type": "done"' in body + stale_manager.process_message.assert_not_awaited() + + def test_web_agent_traditional_stream_keeps_alive_and_saves_after_done(): """传统消息等待期间应保活，且展示快照不能阻塞终态。""" payload = schemas.AgentWebChatRequest(text="/状态", session_id="traditional-heartbeat")