diff --git a/app/api/endpoints/message.py b/app/api/endpoints/message.py index d9c758a58..5b109f326 100644 --- a/app/api/endpoints/message.py +++ b/app/api/endpoints/message.py @@ -1,9 +1,10 @@ +from __future__ import annotations + import json import time -from typing import Union, Any, List, Optional +from typing import Protocol, Union, Any, List, Optional from fastapi import BackgroundTasks, Depends, Request -from pywebpush import WebPushException, webpush from sqlalchemy.ext.asyncio import AsyncSession from starlette.responses import PlainTextResponse @@ -27,7 +28,13 @@ router = ResponseAPIRouter() _WNS_DEFAULT_TTL = 86400 -def is_webpush_subscription_gone(error: WebPushException) -> bool: +class WebPushError(Protocol): + """Web Push 订阅状态判断所需的最小异常协议。""" + + response: Any # 推送服务响应，状态码字段由具体 SDK 提供 + + +def is_webpush_subscription_gone(error: WebPushError) -> bool: """判断 Web Push 订阅是否已在浏览器或推送服务侧失效。""" response: Any = getattr(error, "response", None) status_code = getattr(response, "status_code", None) or getattr( @@ -359,6 +366,8 @@ def send_notification( """ 发送webpush通知 """ + from pywebpush import WebPushException, webpush + for sub in global_vars.get_subscriptions(): try: webpush( diff --git a/app/api/endpoints/system.py b/app/api/endpoints/system.py index 56645b66d..af033238e 100644 --- a/app/api/endpoints/system.py +++ b/app/api/endpoints/system.py @@ -1405,8 +1405,9 @@ def modulelist(_: schemas.TokenPayload = Depends(verify_token)): 查询已加载的模块ID列表 """ modules = [] - for module_id, module in ModuleManager().get_modules().items(): - name = module.get_name() + for spec in ModuleManager().list_specs(): + module_id = spec.id + name = str(spec.metadata["name"]) modules.append( { "id": module_id, diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 5cb69f08b..9d5e95234 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -1,9 +1,11 @@ +from __future__ import annotations + import copy import inspect import pickle import traceback from abc import ABCMeta -from collections.abc import Callable +from collections.abc import Callable, Sequence from datetime import datetime from pathlib import Path from typing import Optional, Any, Tuple, List, Set, Union, Dict diff --git a/app/foundation/singleton.py b/app/foundation/singleton.py index db5a088e8..a4d711fa2 100644 --- a/app/foundation/singleton.py +++ b/app/foundation/singleton.py @@ -9,6 +9,7 @@ class Singleton(abc.ABCMeta, type): """ _instances: dict = {} + _lock = threading.RLock() def get_existing_instance(cls, *args, **kwargs): """按相同参数返回已创建实例，不触发初始化""" @@ -18,9 +19,10 @@ class Singleton(abc.ABCMeta, type): def __call__(cls, *args, **kwargs): """按类和构造参数创建或复用实例。""" key = (cls, args, frozenset(kwargs.items())) - if key not in cls._instances: - cls._instances[key] = super().__call__(*args, **kwargs) - return cls._instances[key] + with cls._lock: + if key not in cls._instances: + cls._instances[key] = super().__call__(*args, **kwargs) + return cls._instances[key] class AbstractSingleton(abc.ABC, metaclass=Singleton): @@ -36,6 +38,7 @@ class SingletonClass(abc.ABCMeta, type): """ _instances: dict = {} + _lock = threading.RLock() def get_existing_instance(cls): """返回已创建实例，不触发初始化""" @@ -43,9 +46,10 @@ class SingletonClass(abc.ABCMeta, type): def __call__(cls, *args, **kwargs): """按类创建或复用唯一实例。""" - if cls not in cls._instances: - cls._instances[cls] = super().__call__(*args, **kwargs) - return cls._instances[cls] + with cls._lock: + if cls not in cls._instances: + cls._instances[cls] = super().__call__(*args, **kwargs) + return cls._instances[cls] class AbstractSingletonClass(abc.ABC, metaclass=SingletonClass): diff --git a/app/modules/__init__.py b/app/modules/__init__.py index ee8edbee7..50eca16f5 100644 --- a/app/modules/__init__.py +++ b/app/modules/__init__.py @@ -3,7 +3,7 @@ from abc import abstractmethod, ABCMeta from typing import Generic, Tuple, Union, TypeVar, Type, Dict, Optional, Callable from pathlib import Path -from app.runtime.extensions.service_registry import ServiceConfigHelper +from app.runtime.extensions.service_config import ServiceConfigHelper from app.runtime.log import logger from app.schemas import Notification, NotificationConf, MediaServerConf, DownloaderConf from app.schemas.types import ModuleType, DownloaderType, MediaServerType, MessageChannel, StorageSchema, \ @@ -17,6 +17,9 @@ class _ModuleBase(ConfigReloadMixin, metaclass=ABCMeta): 输入参数与输出参数一致的，或没有输出的，可以被多个模块重复实现 """ + # Host Module 的配置事件由统一 Adapter 协调，避免同一 generation 被双重重载。 + CONFIG_RELOAD_MANAGED_EXTERNALLY = True + def __init__(self) -> None: """初始化模块生命周期锁""" super().__init__() diff --git a/app/modules/acoustid/capability.toml b/app/modules/acoustid/capability.toml new file mode 100644 index 000000000..0243c6d1c --- /dev/null +++ b/app/modules/acoustid/capability.toml @@ -0,0 +1,19 @@ +schema_version = 1 +id = "AcoustIdModule" +kind = "host_module" +entrypoint = "app.modules.acoustid:AcoustIdModule" +depends_on = [] + +[metadata] +name = "AcoustID" +type = "other" +subtype = "AcoustId" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["ACOUSTID_API_KEY"] + +[activation.selector] +kind = "setting_truthy" +key = "ACOUSTID_API_KEY" diff --git a/app/modules/anilist/capability.toml b/app/modules/anilist/capability.toml new file mode 100644 index 000000000..0d76d82f4 --- /dev/null +++ b/app/modules/anilist/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "AniListModule" +kind = "host_module" +entrypoint = "app.modules.anilist:AniListModule" +depends_on = [] + +[metadata] +name = "AniList" +type = "mediarecognize" +subtype = "AniList" +priority = 4 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST"] diff --git a/app/modules/bangumi/capability.toml b/app/modules/bangumi/capability.toml new file mode 100644 index 000000000..ad0668eab --- /dev/null +++ b/app/modules/bangumi/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "BangumiModule" +kind = "host_module" +entrypoint = "app.modules.bangumi:BangumiModule" +depends_on = [] + +[metadata] +name = "Bangumi" +type = "mediarecognize" +subtype = "Bangumi" +priority = 3 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST"] diff --git a/app/modules/discord/capability.toml b/app/modules/discord/capability.toml new file mode 100644 index 000000000..33d7008cb --- /dev/null +++ b/app/modules/discord/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "DiscordModule" +kind = "host_module" +entrypoint = "app.modules.discord:DiscordModule" +depends_on = [] + +[metadata] +name = "Discord" +type = "notification" +subtype = "Discord" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "discord" +enabled_field = "enabled" diff --git a/app/modules/douban/capability.toml b/app/modules/douban/capability.toml new file mode 100644 index 000000000..5acea8cee --- /dev/null +++ b/app/modules/douban/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "DoubanModule" +kind = "host_module" +entrypoint = "app.modules.douban:DoubanModule" +depends_on = [] + +[metadata] +name = "豆瓣" +type = "mediarecognize" +subtype = "Douban" +priority = 2 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/emby/capability.toml b/app/modules/emby/capability.toml new file mode 100644 index 000000000..e61047eaf --- /dev/null +++ b/app/modules/emby/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "EmbyModule" +kind = "host_module" +entrypoint = "app.modules.emby:EmbyModule" +depends_on = [] + +[metadata] +name = "Emby" +type = "mediaserver" +subtype = "Emby" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "emby" +enabled_field = "enabled" diff --git a/app/modules/fanart/capability.toml b/app/modules/fanart/capability.toml new file mode 100644 index 000000000..94439c182 --- /dev/null +++ b/app/modules/fanart/capability.toml @@ -0,0 +1,19 @@ +schema_version = 1 +id = "FanartModule" +kind = "host_module" +entrypoint = "app.modules.fanart:FanartModule" +depends_on = [] + +[metadata] +name = "Fanart" +type = "other" +subtype = "Fanart" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["FANART_API_KEY"] + +[activation.selector] +kind = "setting_truthy" +key = "FANART_API_KEY" diff --git a/app/modules/feishu/capability.toml b/app/modules/feishu/capability.toml new file mode 100644 index 000000000..67203bf20 --- /dev/null +++ b/app/modules/feishu/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "FeishuModule" +kind = "host_module" +entrypoint = "app.modules.feishu:FeishuModule" +depends_on = [] + +[metadata] +name = "飞书" +type = "notification" +subtype = "Feishu" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "feishu" +enabled_field = "enabled" diff --git a/app/modules/filemanager/capability.toml b/app/modules/filemanager/capability.toml new file mode 100644 index 000000000..ef6a278de --- /dev/null +++ b/app/modules/filemanager/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "FileManagerModule" +kind = "host_module" +entrypoint = "app.modules.filemanager:FileManagerModule" +depends_on = [] + +[metadata] +name = "文件整理" +type = "other" +subtype = "FileManager" +priority = 4 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/filter/capability.toml b/app/modules/filter/capability.toml new file mode 100644 index 000000000..3eadfcebd --- /dev/null +++ b/app/modules/filter/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "FilterModule" +kind = "host_module" +entrypoint = "app.modules.filter:FilterModule" +depends_on = [] + +[metadata] +name = "过滤器" +type = "other" +subtype = "Filter" +priority = 4 + +[activation] +policy = "bootstrap" +watch = ["CustomFilterRules", "CustomIdentifiers", "CustomReleaseGroups", "Customization"] diff --git a/app/modules/indexer/capability.toml b/app/modules/indexer/capability.toml new file mode 100644 index 000000000..f0e5b4f39 --- /dev/null +++ b/app/modules/indexer/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "IndexerModule" +kind = "host_module" +entrypoint = "app.modules.indexer:IndexerModule" +depends_on = [] + +[metadata] +name = "站点索引" +type = "indexer" +subtype = "Indexer" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/jellyfin/capability.toml b/app/modules/jellyfin/capability.toml new file mode 100644 index 000000000..2d420fb76 --- /dev/null +++ b/app/modules/jellyfin/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "JellyfinModule" +kind = "host_module" +entrypoint = "app.modules.jellyfin:JellyfinModule" +depends_on = [] + +[metadata] +name = "Jellyfin" +type = "mediaserver" +subtype = "Jellyfin" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "jellyfin" +enabled_field = "enabled" diff --git a/app/modules/listenbrainz/capability.toml b/app/modules/listenbrainz/capability.toml new file mode 100644 index 000000000..92466961b --- /dev/null +++ b/app/modules/listenbrainz/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "ListenBrainzModule" +kind = "host_module" +entrypoint = "app.modules.listenbrainz:ListenBrainzModule" +depends_on = [] + +[metadata] +name = "ListenBrainz" +type = "other" +subtype = "ListenBrainz" +priority = 5 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/lrclib/capability.toml b/app/modules/lrclib/capability.toml new file mode 100644 index 000000000..23ad94652 --- /dev/null +++ b/app/modules/lrclib/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "LrclibModule" +kind = "host_module" +entrypoint = "app.modules.lrclib:LrclibModule" +depends_on = [] + +[metadata] +name = "LRCLIB" +type = "other" +subtype = "Lrclib" +priority = 5 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/musicbrainz/capability.toml b/app/modules/musicbrainz/capability.toml new file mode 100644 index 000000000..55c7a1c0d --- /dev/null +++ b/app/modules/musicbrainz/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "MusicBrainzModule" +kind = "host_module" +entrypoint = "app.modules.musicbrainz:MusicBrainzModule" +depends_on = [] + +[metadata] +name = "MusicBrainz" +type = "mediarecognize" +subtype = "MusicBrainz" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/navidrome/capability.toml b/app/modules/navidrome/capability.toml new file mode 100644 index 000000000..9a41ebe03 --- /dev/null +++ b/app/modules/navidrome/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "NavidromeModule" +kind = "host_module" +entrypoint = "app.modules.navidrome:NavidromeModule" +depends_on = [] + +[metadata] +name = "Navidrome" +type = "mediaserver" +subtype = "Navidrome" +priority = 7 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "navidrome" +enabled_field = "enabled" diff --git a/app/modules/plex/capability.toml b/app/modules/plex/capability.toml new file mode 100644 index 000000000..0cf679309 --- /dev/null +++ b/app/modules/plex/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "PlexModule" +kind = "host_module" +entrypoint = "app.modules.plex:PlexModule" +depends_on = [] + +[metadata] +name = "Plex" +type = "mediaserver" +subtype = "Plex" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "plex" +enabled_field = "enabled" diff --git a/app/modules/postgresql/capability.toml b/app/modules/postgresql/capability.toml new file mode 100644 index 000000000..ef05508b4 --- /dev/null +++ b/app/modules/postgresql/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "PostgreSQLModule" +kind = "host_module" +entrypoint = "app.modules.postgresql:PostgreSQLModule" +depends_on = [] + +[metadata] +name = "PostgreSQL" +type = "other" +subtype = "PostgreSQL" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/qbittorrent/capability.toml b/app/modules/qbittorrent/capability.toml new file mode 100644 index 000000000..28d247aa7 --- /dev/null +++ b/app/modules/qbittorrent/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "QbittorrentModule" +kind = "host_module" +entrypoint = "app.modules.qbittorrent:QbittorrentModule" +depends_on = [] + +[metadata] +name = "Qbittorrent" +type = "downloader" +subtype = "Qbittorrent" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "qbittorrent" +enabled_field = "enabled" diff --git a/app/modules/qqbot/capability.toml b/app/modules/qqbot/capability.toml new file mode 100644 index 000000000..557b74a5b --- /dev/null +++ b/app/modules/qqbot/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "QQBotModule" +kind = "host_module" +entrypoint = "app.modules.qqbot:QQBotModule" +depends_on = [] + +[metadata] +name = "QQ" +type = "notification" +subtype = "QQ" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "qqbot" +enabled_field = "enabled" diff --git a/app/modules/redis/capability.toml b/app/modules/redis/capability.toml new file mode 100644 index 000000000..94d9b497a --- /dev/null +++ b/app/modules/redis/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "RedisModule" +kind = "host_module" +entrypoint = "app.modules.redis:RedisModule" +depends_on = [] + +[metadata] +name = "Redis缓存" +type = "other" +subtype = "Redis" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/rtorrent/capability.toml b/app/modules/rtorrent/capability.toml new file mode 100644 index 000000000..6206dd06d --- /dev/null +++ b/app/modules/rtorrent/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "RtorrentModule" +kind = "host_module" +entrypoint = "app.modules.rtorrent:RtorrentModule" +depends_on = [] + +[metadata] +name = "Rtorrent" +type = "downloader" +subtype = "Rtorrent" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "rtorrent" +enabled_field = "enabled" diff --git a/app/modules/slack/capability.toml b/app/modules/slack/capability.toml new file mode 100644 index 000000000..042e3a020 --- /dev/null +++ b/app/modules/slack/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "SlackModule" +kind = "host_module" +entrypoint = "app.modules.slack:SlackModule" +depends_on = [] + +[metadata] +name = "Slack" +type = "notification" +subtype = "Slack" +priority = 3 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "slack" +enabled_field = "enabled" diff --git a/app/modules/subtitle/capability.toml b/app/modules/subtitle/capability.toml new file mode 100644 index 000000000..92d4d71f6 --- /dev/null +++ b/app/modules/subtitle/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "SubtitleModule" +kind = "host_module" +entrypoint = "app.modules.subtitle:SubtitleModule" +depends_on = [] + +[metadata] +name = "站点字幕" +type = "other" +subtype = "Subtitle" +priority = 0 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/synologychat/capability.toml b/app/modules/synologychat/capability.toml new file mode 100644 index 000000000..89bafdd2f --- /dev/null +++ b/app/modules/synologychat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "SynologyChatModule" +kind = "host_module" +entrypoint = "app.modules.synologychat:SynologyChatModule" +depends_on = [] + +[metadata] +name = "Synology Chat" +type = "notification" +subtype = "SynologyChat" +priority = 5 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "synologychat" +enabled_field = "enabled" diff --git a/app/modules/telegram/capability.toml b/app/modules/telegram/capability.toml new file mode 100644 index 000000000..aae376504 --- /dev/null +++ b/app/modules/telegram/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TelegramModule" +kind = "host_module" +entrypoint = "app.modules.telegram:TelegramModule" +depends_on = [] + +[metadata] +name = "Telegram" +type = "notification" +subtype = "Telegram" +priority = 0 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "telegram" +enabled_field = "enabled" diff --git a/app/modules/theaudiodb/capability.toml b/app/modules/theaudiodb/capability.toml new file mode 100644 index 000000000..5aa2fcc9a --- /dev/null +++ b/app/modules/theaudiodb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheAudioDbModule" +kind = "host_module" +entrypoint = "app.modules.theaudiodb:TheAudioDbModule" +depends_on = [] + +[metadata] +name = "TheAudioDB" +type = "mediarecognize" +subtype = "TheAudioDB" +priority = 1 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/themoviedb/capability.toml b/app/modules/themoviedb/capability.toml new file mode 100644 index 000000000..80cbe9815 --- /dev/null +++ b/app/modules/themoviedb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheMovieDbModule" +kind = "host_module" +entrypoint = "app.modules.themoviedb:TheMovieDbModule" +depends_on = [] + +[metadata] +name = "TheMovieDb" +type = "mediarecognize" +subtype = "TMDB" +priority = 1 + +[activation] +policy = "bootstrap" +watch = ["PROXY_HOST", "TMDB_API_DOMAIN", "TMDB_API_KEY", "TMDB_LOCALE"] diff --git a/app/modules/thetvdb/capability.toml b/app/modules/thetvdb/capability.toml new file mode 100644 index 000000000..bba680f9f --- /dev/null +++ b/app/modules/thetvdb/capability.toml @@ -0,0 +1,15 @@ +schema_version = 1 +id = "TheTvDbModule" +kind = "host_module" +entrypoint = "app.modules.thetvdb:TheTvDbModule" +depends_on = [] + +[metadata] +name = "TheTvDb" +type = "mediarecognize" +subtype = "TVDB" +priority = 4 + +[activation] +policy = "bootstrap" +watch = [] diff --git a/app/modules/transmission/capability.toml b/app/modules/transmission/capability.toml new file mode 100644 index 000000000..f63977037 --- /dev/null +++ b/app/modules/transmission/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TransmissionModule" +kind = "host_module" +entrypoint = "app.modules.transmission:TransmissionModule" +depends_on = [] + +[metadata] +name = "Transmission" +type = "downloader" +subtype = "Transmission" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Downloaders"] + +[activation.selector] +kind = "system_config_item" +key = "Downloaders" +match_field = "type" +match_value = "transmission" +enabled_field = "enabled" diff --git a/app/modules/trimemedia/capability.toml b/app/modules/trimemedia/capability.toml new file mode 100644 index 000000000..3ca94566e --- /dev/null +++ b/app/modules/trimemedia/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "TrimeMediaModule" +kind = "host_module" +entrypoint = "app.modules.trimemedia:TrimeMediaModule" +depends_on = [] + +[metadata] +name = "飞牛影视" +type = "mediaserver" +subtype = "TrimeMedia" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "trimemedia" +enabled_field = "enabled" diff --git a/app/modules/ugreen/capability.toml b/app/modules/ugreen/capability.toml new file mode 100644 index 000000000..46288aa82 --- /dev/null +++ b/app/modules/ugreen/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "UgreenModule" +kind = "host_module" +entrypoint = "app.modules.ugreen:UgreenModule" +depends_on = [] + +[metadata] +name = "绿联影视" +type = "mediaserver" +subtype = "Ugreen" +priority = 5 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "ugreen" +enabled_field = "enabled" diff --git a/app/modules/vocechat/capability.toml b/app/modules/vocechat/capability.toml new file mode 100644 index 000000000..830f3e6a9 --- /dev/null +++ b/app/modules/vocechat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "VoceChatModule" +kind = "host_module" +entrypoint = "app.modules.vocechat:VoceChatModule" +depends_on = [] + +[metadata] +name = "VoceChat" +type = "notification" +subtype = "VoceChat" +priority = 4 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "vocechat" +enabled_field = "enabled" diff --git a/app/modules/webpush/capability.toml b/app/modules/webpush/capability.toml new file mode 100644 index 000000000..ed9484185 --- /dev/null +++ b/app/modules/webpush/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WebPushModule" +kind = "host_module" +entrypoint = "app.modules.webpush:WebPushModule" +depends_on = [] + +[metadata] +name = "WebPush" +type = "notification" +subtype = "WebPush" +priority = 6 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "webpush" +enabled_field = "enabled" diff --git a/app/modules/wechat/capability.toml b/app/modules/wechat/capability.toml new file mode 100644 index 000000000..511625b01 --- /dev/null +++ b/app/modules/wechat/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WechatModule" +kind = "host_module" +entrypoint = "app.modules.wechat:WechatModule" +depends_on = [] + +[metadata] +name = "企业微信" +type = "notification" +subtype = "Wechat" +priority = 1 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "wechat" +enabled_field = "enabled" diff --git a/app/modules/wechatclawbot/capability.toml b/app/modules/wechatclawbot/capability.toml new file mode 100644 index 000000000..c64cf1975 --- /dev/null +++ b/app/modules/wechatclawbot/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "WechatClawBotModule" +kind = "host_module" +entrypoint = "app.modules.wechatclawbot:WechatClawBotModule" +depends_on = [] + +[metadata] +name = "微信 ClawBot" +type = "notification" +subtype = "WechatClawBot" +priority = 2 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "wechatclawbot" +enabled_field = "enabled" diff --git a/app/modules/zspace/capability.toml b/app/modules/zspace/capability.toml new file mode 100644 index 000000000..66e2fc95b --- /dev/null +++ b/app/modules/zspace/capability.toml @@ -0,0 +1,22 @@ +schema_version = 1 +id = "ZSpaceModule" +kind = "host_module" +entrypoint = "app.modules.zspace:ZSpaceModule" +depends_on = [] + +[metadata] +name = "极影视" +type = "mediaserver" +subtype = "ZSpace" +priority = 6 + +[activation] +policy = "when_configured" +watch = ["MediaServers"] + +[activation.selector] +kind = "system_config_item" +key = "MediaServers" +match_field = "type" +match_value = "zspace" +enabled_field = "enabled" diff --git a/app/runtime/capabilities/__init__.py b/app/runtime/capabilities/__init__.py new file mode 100644 index 000000000..ee512115f --- /dev/null +++ b/app/runtime/capabilities/__init__.py @@ -0,0 +1,47 @@ +"""MoviePilot 内部能力运行时的惰性公共导出。""" + +from __future__ import annotations + +from importlib import import_module +from typing import Any + + +_EXPORT_MODULES = { + "ActivationPolicy": "app.runtime.capabilities.model", + "AdapterExecutionMode": "app.runtime.capabilities.model", + "AsyncCapabilityAdapter": "app.runtime.capabilities.model", + "CapabilityAdapterContractError": "app.runtime.capabilities.errors", + "CapabilityAdapterModeError": "app.runtime.capabilities.errors", + "CapabilityError": "app.runtime.capabilities.errors", + "CapabilityLifecycleState": "app.runtime.capabilities.model", + "CapabilityManifestError": "app.runtime.capabilities.errors", + "CapabilityMaterializationState": "app.runtime.capabilities.model", + "CapabilityObservation": "app.runtime.capabilities.model", + "CapabilityOperationError": "app.runtime.capabilities.errors", + "CapabilityRegistry": "app.runtime.capabilities.registry", + "CapabilityRuntime": "app.runtime.capabilities.runtime", + "CapabilityRuntimeClosedError": "app.runtime.capabilities.errors", + "CapabilitySnapshot": "app.runtime.capabilities.model", + "CapabilitySpec": "app.runtime.capabilities.model", + "SelectorSchema": "app.runtime.capabilities.model", + "SelectorSpec": "app.runtime.capabilities.model", + "SyncCapabilityAdapter": "app.runtime.capabilities.model", + "UnknownCapabilityError": "app.runtime.capabilities.errors", +} + +__all__ = sorted(_EXPORT_MODULES) + + +def __getattr__(name: str) -> Any: + """仅在显式访问公共符号时导入对应叶模块，并缓存 canonical 对象。""" + module_name = _EXPORT_MODULES.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(module_name), name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """向交互式工具公开稳定导出名，而不触发叶模块导入。""" + return sorted(set(globals()) | set(__all__)) diff --git a/app/runtime/capabilities/errors.py b/app/runtime/capabilities/errors.py new file mode 100644 index 000000000..977409996 --- /dev/null +++ b/app/runtime/capabilities/errors.py @@ -0,0 +1,32 @@ +class CapabilityError(RuntimeError): + """Capability Runtime 错误基类。""" + + +class CapabilityManifestError(CapabilityError, ValueError): + """能力声明不完整、冲突或违反当前 schema。""" + + +class UnknownCapabilityError(CapabilityError, KeyError): + """请求了 Registry 中不存在的能力。""" + + +class CapabilityAdapterModeError(CapabilityError, TypeError): + """调用入口与适配器并发模型不匹配。""" + + +class CapabilityAdapterContractError(CapabilityError, TypeError): + """适配器回调返回值不符合已声明的并发模型。""" + + +class CapabilityOperationError(CapabilityError): + """能力物化或资源生命周期转换失败。""" + + def __init__(self, capability_id: str, operation: str, error: BaseException): + self.capability_id = capability_id + self.operation = operation + self.error = error + super().__init__(f"能力 {capability_id} 执行 {operation} 失败：{error}") + + +class CapabilityRuntimeClosedError(CapabilityError): + """Runtime 已进入关闭态，禁止启动或重新物化能力。""" diff --git a/app/runtime/capabilities/model.py b/app/runtime/capabilities/model.py new file mode 100644 index 000000000..2e9346b59 --- /dev/null +++ b/app/runtime/capabilities/model.py @@ -0,0 +1,182 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from types import MappingProxyType +from typing import Any, Awaitable, Callable, Mapping, Optional, Protocol + + +class ActivationPolicy(str, Enum): + """能力的启动触发策略。""" + + BOOTSTRAP = "bootstrap" + WHEN_CONFIGURED = "when_configured" + ON_FIRST_USE = "on_first_use" + + +class AdapterExecutionMode(str, Enum): + """领域适配器执行回调所使用的并发模型。""" + + SYNC = "sync" + ASYNC = "async" + + +class CapabilityMaterializationState(str, Enum): + """Python 实现对象的解析状态。""" + + UNRESOLVED = "unresolved" + RESOLVING = "resolving" + RESOLVED = "resolved" + FAILED = "failed" + + +class CapabilityLifecycleState(str, Enum): + """能力所拥有外部资源的生命周期状态。""" + + DISCOVERED = "discovered" + STARTING = "starting" + RUNNING = "running" + RELOADING = "reloading" + STOPPING = "stopping" + STOPPED = "stopped" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class SelectorSchema: + """声明一个 selector 可接受的精确参数集合。""" + + required_fields: frozenset[str] = frozenset() + optional_fields: frozenset[str] = frozenset() + validator: Optional[Callable[[Mapping[str, Any]], None]] = None + + +@dataclass(frozen=True, slots=True) +class SelectorSpec: + """由领域适配器解释的配置选择器。""" + + kind: str + config: Mapping[str, Any] + + +@dataclass(frozen=True, slots=True) +class CapabilitySpec: + """从 data-only manifest 构建的不可变能力声明。""" + + schema_version: int + id: str + kind: str + entrypoint: str + activation: ActivationPolicy + metadata: Mapping[str, Any] + selector: Optional[SelectorSpec] + watch: tuple[str, ...] + depends_on: tuple[str, ...] + source: Path + + +@dataclass(frozen=True, slots=True) +class CapabilitySnapshot: + """能力状态的只读快照。""" + + capability_id: str + materialization: CapabilityMaterializationState + lifecycle: CapabilityLifecycleState + generation: int + visible: bool + error: Optional[str] + + +@dataclass(frozen=True, slots=True) +class CapabilityObservation: + """单次状态转换的可复现观测记录。""" + + capability_id: str + generation: int + operation: str + outcome: str + reason: str + materialization: CapabilityMaterializationState + lifecycle: CapabilityLifecycleState + duration_ms: float + error: Optional[str] + + +class SyncCapabilityAdapter(Protocol): + """同步领域适配器合同。""" + + execution_mode: AdapterExecutionMode + + def materialize(self, spec: CapabilitySpec) -> Any: + """解析 manifest entrypoint 对应的 canonical 实现对象。""" + + def create( + self, + spec: CapabilitySpec, + implementation: Any, + generation: int, + previous: Any = None, + ) -> Any: + """创建尚未对外发布的候选资源。""" + + def start(self, spec: CapabilitySpec, candidate: Any, generation: int) -> None: + """启动候选资源；返回前候选不会对普通查询可见。""" + + def stop(self, spec: CapabilitySpec, instance: Any, generation: int) -> None: + """停止已经撤销运行态可见性的资源。""" + + def cleanup( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> None: + """清理由失败启动留下的候选资源。""" + + +class AsyncCapabilityAdapter(Protocol): + """异步领域适配器合同。""" + + execution_mode: AdapterExecutionMode + + def materialize(self, spec: CapabilitySpec) -> Awaitable[Any]: + """异步解析 manifest entrypoint 对应的 canonical 实现对象。""" + + def create( + self, + spec: CapabilitySpec, + implementation: Any, + generation: int, + previous: Any = None, + ) -> Awaitable[Any]: + """异步创建尚未对外发布的候选资源。""" + + def start( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + ) -> Awaitable[None]: + """异步启动候选资源。""" + + def stop( + self, + spec: CapabilitySpec, + instance: Any, + generation: int, + ) -> Awaitable[None]: + """异步停止已经撤销运行态可见性的资源。""" + + def cleanup( + self, + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> Awaitable[None]: + """异步清理由失败启动留下的候选资源。""" + + +EMPTY_MAPPING: Mapping[str, Any] = MappingProxyType({}) diff --git a/app/runtime/capabilities/registry.py b/app/runtime/capabilities/registry.py new file mode 100644 index 000000000..7a1a8ceca --- /dev/null +++ b/app/runtime/capabilities/registry.py @@ -0,0 +1,287 @@ +from __future__ import annotations + +import re +import tomllib +from pathlib import Path +from types import MappingProxyType +from typing import Any, Collection, Iterable, Mapping + +from app.runtime.capabilities.errors import ( + CapabilityManifestError, + UnknownCapabilityError, +) +from app.runtime.capabilities.model import ( + ActivationPolicy, + CapabilitySpec, + SelectorSchema, + SelectorSpec, +) + + +_SCHEMA_VERSION = 1 +_MANIFEST_NAME = "capability.toml" +_TOP_LEVEL_FIELDS = frozenset({ + "schema_version", + "id", + "kind", + "entrypoint", + "metadata", + "activation", + "depends_on", +}) +_REQUIRED_FIELDS = _TOP_LEVEL_FIELDS +_ACTIVATION_FIELDS = frozenset({"policy", "watch", "selector"}) +_ACTIVATION_REQUIRED_FIELDS = frozenset({"policy", "watch"}) +_IDENTIFIER_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9_.-]{0,127}$") +_KIND_PATTERN = re.compile(r"^[a-z][a-z0-9_.-]{0,63}$") +_ENTRYPOINT_PATTERN = re.compile( + r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*" + r":[A-Za-z_][A-Za-z0-9_]*$" +) + + +def _freeze(value: Any, *, field: str) -> Any: + """把 TOML 容器递归转换为不可变结构，并拒绝非配置标量。""" + if value is None or isinstance(value, (str, int, float, bool)): + return value + if isinstance(value, list): + return tuple(_freeze(item, field=field) for item in value) + if isinstance(value, dict): + if not all(isinstance(key, str) and key for key in value): + raise CapabilityManifestError(f"{field} 包含非法键") + return MappingProxyType({ + key: _freeze(item, field=f"{field}.{key}") + for key, item in value.items() + }) + raise CapabilityManifestError(f"{field} 包含不支持的 TOML 值类型 {type(value).__name__}") + + +def _string_list(value: Any, *, field: str, path: Path) -> tuple[str, ...]: + """校验无重复的非空字符串列表。""" + if not isinstance(value, list): + raise CapabilityManifestError(f"{path}: {field} 必须是字符串数组") + if any(not isinstance(item, str) or not item.strip() for item in value): + raise CapabilityManifestError(f"{path}: {field} 只能包含非空字符串") + normalized = tuple(item.strip() for item in value) + if len(set(normalized)) != len(normalized): + raise CapabilityManifestError(f"{path}: {field} 不能包含重复值") + return normalized + + +class CapabilityRegistry: + """只读取 data-only manifest 的不可变能力注册表。""" + + def __init__( + self, + specs: Mapping[str, CapabilitySpec], + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> None: + self._specs = MappingProxyType(dict(specs)) + self._kinds = frozenset(kinds) + self._selector_schemas = MappingProxyType(dict(selector_schemas)) + + @classmethod + def discover( + cls, + roots: Iterable[Path | str], + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> "CapabilityRegistry": + """扫描全部声明根；任何根或 manifest 非法都会阻止 Registry 构建。""" + normalized_kinds = frozenset(kinds) + if not normalized_kinds: + raise CapabilityManifestError("至少需要注册一个 capability kind") + for kind in normalized_kinds: + if not isinstance(kind, str) or not _KIND_PATTERN.fullmatch(kind): + raise CapabilityManifestError(f"非法 capability kind：{kind!r}") + + normalized_selectors = dict(selector_schemas) + for selector_type, schema in normalized_selectors.items(): + if not isinstance(selector_type, str) or not _KIND_PATTERN.fullmatch(selector_type): + raise CapabilityManifestError(f"非法 selector type：{selector_type!r}") + if not isinstance(schema, SelectorSchema): + raise CapabilityManifestError(f"selector {selector_type} 未提供 SelectorSchema") + overlap = schema.required_fields & schema.optional_fields + if overlap: + raise CapabilityManifestError( + f"selector {selector_type} 字段同时声明为 required/optional：{sorted(overlap)}" + ) + + specs: dict[str, CapabilitySpec] = {} + normalized_roots = tuple(Path(root) for root in roots) + if not normalized_roots: + raise CapabilityManifestError("至少需要一个 capability 声明根") + for root in normalized_roots: + if not root.is_dir(): + raise CapabilityManifestError(f"声明根不存在或不是目录：{root}") + manifests = sorted(root.rglob(_MANIFEST_NAME)) + if not manifests: + raise CapabilityManifestError(f"声明根没有 {_MANIFEST_NAME}：{root}") + for manifest_path in manifests: + spec = cls._load_manifest( + manifest_path, + kinds=normalized_kinds, + selector_schemas=normalized_selectors, + ) + previous = specs.get(spec.id) + if previous: + raise CapabilityManifestError( + f"capability id 重复：{spec.id}，来源 {previous.source} 与 {spec.source}" + ) + specs[spec.id] = spec + return cls( + specs, + kinds=normalized_kinds, + selector_schemas=normalized_selectors, + ) + + @classmethod + def _load_manifest( + cls, + path: Path, + *, + kinds: Collection[str], + selector_schemas: Mapping[str, SelectorSchema], + ) -> CapabilitySpec: + try: + with path.open("rb") as file: + data = tomllib.load(file) + except (OSError, tomllib.TOMLDecodeError) as error: + raise CapabilityManifestError(f"无法读取 {path}：{error}") from error + + unknown_fields = set(data) - _TOP_LEVEL_FIELDS + if unknown_fields: + raise CapabilityManifestError(f"{path}: 未知字段 {sorted(unknown_fields)}") + missing_fields = _REQUIRED_FIELDS - set(data) + if missing_fields: + raise CapabilityManifestError(f"{path}: 缺少字段 {sorted(missing_fields)}") + + schema_version = data["schema_version"] + if type(schema_version) is not int or schema_version != _SCHEMA_VERSION: + raise CapabilityManifestError( + f"{path}: 不支持 schema_version={schema_version!r}" + ) + + capability_id = data["id"] + if not isinstance(capability_id, str) or not _IDENTIFIER_PATTERN.fullmatch(capability_id): + raise CapabilityManifestError(f"{path}: 非法 capability id={capability_id!r}") + + kind = data["kind"] + if not isinstance(kind, str) or kind not in kinds: + raise CapabilityManifestError(f"{path}: 未注册 capability kind={kind!r}") + + entrypoint = data["entrypoint"] + if not isinstance(entrypoint, str) or not _ENTRYPOINT_PATTERN.fullmatch(entrypoint): + raise CapabilityManifestError(f"{path}: 非法 entrypoint={entrypoint!r}") + + metadata = data["metadata"] + if not isinstance(metadata, dict): + raise CapabilityManifestError(f"{path}: metadata 必须是 table") + name = metadata.get("name") + if not isinstance(name, str) or not name.strip(): + raise CapabilityManifestError(f"{path}: metadata.name 必须是非空字符串") + immutable_metadata = _freeze(metadata, field="metadata") + + activation_data = data["activation"] + if not isinstance(activation_data, dict): + raise CapabilityManifestError(f"{path}: activation 必须是 table") + unknown_activation_fields = set(activation_data) - _ACTIVATION_FIELDS + missing_activation_fields = _ACTIVATION_REQUIRED_FIELDS - set(activation_data) + if unknown_activation_fields or missing_activation_fields: + raise CapabilityManifestError( + f"{path}: activation 字段非法，missing={sorted(missing_activation_fields)} " + f"unknown={sorted(unknown_activation_fields)}" + ) + try: + activation = ActivationPolicy(activation_data["policy"]) + except (TypeError, ValueError) as error: + raise CapabilityManifestError( + f"{path}: 非法 activation.policy={activation_data['policy']!r}" + ) from error + + selector = cls._parse_selector( + path, + activation=activation, + data=activation_data.get("selector"), + selector_schemas=selector_schemas, + ) + watch = _string_list(activation_data["watch"], field="activation.watch", path=path) + depends_on = _string_list(data["depends_on"], field="depends_on", path=path) + if depends_on: + raise CapabilityManifestError( + f"{path}: 当前 schema 不支持非空 depends_on={list(depends_on)!r}" + ) + + return CapabilitySpec( + schema_version=schema_version, + id=capability_id, + kind=kind, + entrypoint=entrypoint, + activation=activation, + metadata=immutable_metadata, + selector=selector, + watch=watch, + depends_on=depends_on, + source=path, + ) + + @staticmethod + def _parse_selector( + path: Path, + *, + activation: ActivationPolicy, + data: Any, + selector_schemas: Mapping[str, SelectorSchema], + ) -> SelectorSpec | None: + if activation is not ActivationPolicy.WHEN_CONFIGURED: + if data is not None: + raise CapabilityManifestError( + f"{path}: activation={activation.value} 时不允许 selector" + ) + return None + if not isinstance(data, dict): + raise CapabilityManifestError(f"{path}: when_configured 必须提供 selector table") + selector_kind = data.get("kind") + if not isinstance(selector_kind, str) or selector_kind not in selector_schemas: + raise CapabilityManifestError(f"{path}: 未注册 selector kind={selector_kind!r}") + config = {key: value for key, value in data.items() if key != "kind"} + schema = selector_schemas[selector_kind] + missing = schema.required_fields - set(config) + unknown = set(config) - schema.required_fields - schema.optional_fields + if missing or unknown: + raise CapabilityManifestError( + f"{path}: selector {selector_kind} 字段非法，" + f"missing={sorted(missing)} unknown={sorted(unknown)}" + ) + immutable_config = _freeze(config, field="selector") + if schema.validator: + try: + schema.validator(immutable_config) + except Exception as error: + raise CapabilityManifestError( + f"{path}: selector {selector_kind} 校验失败：{error}" + ) from error + return SelectorSpec(kind=selector_kind, config=immutable_config) + + @property + def kinds(self) -> frozenset[str]: + """返回该 Registry 接受的 capability kind。""" + return self._kinds + + def get_spec(self, capability_id: str) -> CapabilitySpec | None: + """查询声明；不存在时返回 None。""" + return self._specs.get(capability_id) + + def require_spec(self, capability_id: str) -> CapabilitySpec: + """查询必需声明；不存在时给出稳定的领域错误。""" + spec = self.get_spec(capability_id) + if spec is None: + raise UnknownCapabilityError(f"未知 capability：{capability_id}") + return spec + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """按 ID 返回稳定排序的声明快照。""" + return tuple(self._specs[key] for key in sorted(self._specs)) diff --git a/app/runtime/capabilities/runtime.py b/app/runtime/capabilities/runtime.py new file mode 100644 index 000000000..59712ac0b --- /dev/null +++ b/app/runtime/capabilities/runtime.py @@ -0,0 +1,1369 @@ +from __future__ import annotations + +import asyncio +import inspect +import sys +import threading +import time +from collections import deque +from concurrent.futures import Future +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, Callable, Mapping, Optional + +from app.runtime.capabilities.errors import ( + CapabilityAdapterContractError, + CapabilityAdapterModeError, + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.model import ( + AdapterExecutionMode, + CapabilityLifecycleState, + CapabilityMaterializationState, + CapabilityObservation, + CapabilitySnapshot, + CapabilitySpec, +) +from app.runtime.capabilities.registry import CapabilityRegistry + + +@dataclass(slots=True) +class _CapabilityState: + """Runtime 私有状态；所有可变字段均由 lock 保护。""" + + spec: CapabilitySpec + lock: threading.RLock = field(default_factory=threading.RLock) + materialization: CapabilityMaterializationState = CapabilityMaterializationState.UNRESOLVED + lifecycle: CapabilityLifecycleState = CapabilityLifecycleState.DISCOVERED + generation: int = 0 + implementation: Any = None + instance: Any = None + # 撤销可见性后仍未确认释放的资源由 Runtime 持有，禁止并行创建第二实例。 + pending_stop: Any = None + pending_cleanup: Any = None + pending_cleanup_error: Optional[BaseException] = None + last_error: Optional[BaseException] = None + inflight: Optional[Future[Any]] = None + inflight_operation: Optional[str] = None + + +class CapabilityRuntime: + """协调能力实现物化和资源生命周期的通用运行时。""" + + def __init__( + self, + registry: CapabilityRegistry, + *, + adapters: Mapping[str, Any], + observer: Optional[Callable[[CapabilityObservation], None]] = None, + observation_limit: int = 1024, + ) -> None: + if observation_limit <= 0: + raise ValueError("observation_limit 必须大于 0") + missing_adapters = {spec.kind for spec in registry.list_specs()} - set(adapters) + if missing_adapters: + raise CapabilityAdapterContractError( + f"缺少 capability adapter：{sorted(missing_adapters)}" + ) + self._registry = registry + self._adapters = MappingProxyType(dict(adapters)) + self._states = { + spec.id: _CapabilityState(spec=spec) + for spec in registry.list_specs() + } + self._observer = observer + self._observations: deque[CapabilityObservation] = deque(maxlen=observation_limit) + self._observation_lock = threading.Lock() + self._shutdown_lock = threading.Lock() + self._shutdown = False + + @property + def is_shutdown(self) -> bool: + """返回 Runtime 是否已进入不可逆关闭态。""" + return self._shutdown + + def _state(self, capability_id: str) -> _CapabilityState: + self._registry.require_spec(capability_id) + return self._states[capability_id] + + def get_spec(self, capability_id: str) -> CapabilitySpec | None: + """返回保留在 Registry 中的声明，不受运行失败影响。""" + return self._registry.get_spec(capability_id) + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """返回全部声明；FAILED 能力不会从列表中消失。""" + return self._registry.list_specs() + + def _adapter(self, state: _CapabilityState, expected: AdapterExecutionMode) -> Any: + adapter = self._adapters[state.spec.kind] + actual = getattr(adapter, "execution_mode", None) + if actual is not expected: + raise CapabilityAdapterModeError( + f"能力 {state.spec.id} 的 adapter mode={actual!r}，" + f"不能通过 {expected.value} 入口调用" + ) + return adapter + + def _ensure_open(self) -> None: + if self._shutdown: + raise CapabilityRuntimeClosedError("Capability Runtime 已关闭") + + @staticmethod + def _sync_callback(adapter: Any, name: str, *args) -> Any: + callback = getattr(adapter, name, None) + if not callable(callback): + raise CapabilityAdapterContractError(f"同步 adapter 缺少 {name}()") + result = callback(*args) + if inspect.isawaitable(result): + close = getattr(result, "close", None) + if callable(close): + close() + raise CapabilityAdapterContractError( + f"同步 adapter 的 {name}() 不能返回 awaitable" + ) + return result + + @staticmethod + async def _async_callback(adapter: Any, name: str, *args) -> Any: + callback = getattr(adapter, name, None) + if not callable(callback): + raise CapabilityAdapterContractError(f"异步 adapter 缺少 {name}()") + result = callback(*args) + if not inspect.isawaitable(result): + raise CapabilityAdapterContractError( + f"异步 adapter 的 {name}() 必须返回 awaitable" + ) + return await result + + @staticmethod + def _wait_sync(future: Future[Any]) -> Any: + return future.result() + + @staticmethod + async def _wait_async(future: Future[Any]) -> Any: + return await asyncio.shield(asyncio.wrap_future(future)) + + def _emit( + self, + state: _CapabilityState, + *, + generation: int, + operation: str, + outcome: str, + reason: str, + started_at: float, + error: Optional[BaseException] = None, + ) -> None: + with state.lock: + observation = CapabilityObservation( + capability_id=state.spec.id, + generation=generation, + operation=operation, + outcome=outcome, + reason=reason, + materialization=state.materialization, + lifecycle=state.lifecycle, + duration_ms=max(0.0, (time.monotonic() - started_at) * 1000), + error=str(error) if error else None, + ) + with self._observation_lock: + self._observations.append(observation) + if self._observer: + try: + self._observer(observation) + except Exception: + # 观测消费者不能改变能力生命周期的成功或失败语义。 + pass + + @staticmethod + def _finish_transition( + state: _CapabilityState, + future: Future[Any], + *, + result: Any = None, + error: Optional[BaseException] = None, + ) -> None: + with state.lock: + if state.inflight is future: + state.inflight = None + state.inflight_operation = None + if error is None: + future.set_result(result) + else: + future.set_exception(error) + + def _calibrate_consumer_materialization(self, state: _CapabilityState) -> None: + """从 sys.modules 校准显式 consumer import，不触发任何新导入。""" + module_name, symbol_name = state.spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict) or symbol_name not in namespace: + return + canonical = namespace[symbol_name] + with state.lock: + if state.inflight is not None: + return + if state.materialization is CapabilityMaterializationState.RESOLVED: + if state.implementation is not canonical: + state.last_error = CapabilityAdapterContractError( + f"能力 {state.spec.id} 的 canonical implementation identity 发生变化" + ) + return + state.implementation = canonical + state.materialization = CapabilityMaterializationState.RESOLVED + if state.lifecycle is CapabilityLifecycleState.DISCOVERED: + state.last_error = None + generation = state.generation + self._emit( + state, + generation=generation, + operation="consumer_materialize", + outcome="succeeded", + reason="sys_modules", + started_at=time.monotonic(), + ) + + def snapshot(self, capability_id: str) -> CapabilitySnapshot: + """返回状态快照，并校准外部显式导入产生的 canonical 对象。""" + state = self._state(capability_id) + self._calibrate_consumer_materialization(state) + with state.lock: + return CapabilitySnapshot( + capability_id=state.spec.id, + materialization=state.materialization, + lifecycle=state.lifecycle, + generation=state.generation, + visible=state.instance is not None, + error=str(state.last_error) if state.last_error else None, + ) + + def observations(self, capability_id: Optional[str] = None) -> tuple[CapabilityObservation, ...]: + """返回按发生顺序记录的不可变观测快照。""" + with self._observation_lock: + items = tuple(self._observations) + if capability_id is None: + return items + return tuple(item for item in items if item.capability_id == capability_id) + + def get_running(self, capability_id: str) -> Any: + """只查询已发布实例，不触发物化或启动。""" + state = self._state(capability_id) + with state.lock: + return state.instance + + def materialize(self, capability_id: str, *, reason: str, retry: bool = False) -> Any: + """通过同步 adapter 显式物化实现，不启动领域资源。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.materialization is CapabilityMaterializationState.RESOLVED: + return state.implementation + if state.materialization is CapabilityMaterializationState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + "materialize", + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "materialize" + state.materialization = CapabilityMaterializationState.RESOLVING + waiter = None + waiter_operation = None + owner = (generation, future) + if waiter is not None: + result = self._wait_sync(waiter) + if waiter_operation == "materialize": + return result + continue + break + + generation, future = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="materialize", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + implementation = self._sync_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.last_error = None + self._finish_transition(state, future, result=implementation) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return implementation + except BaseException as error: + with state.lock: + state.materialization = CapabilityMaterializationState.FAILED + state.last_error = error + operation_error = self._wrap_error(state.spec.id, "materialize", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + raise operation_error from error + + async def materialize_async( + self, + capability_id: str, + *, + reason: str, + retry: bool = False, + ) -> Any: + """通过异步 adapter 显式物化实现，不阻塞事件循环。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.materialization is CapabilityMaterializationState.RESOLVED: + return state.implementation + if state.materialization is CapabilityMaterializationState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + "materialize", + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "materialize" + state.materialization = CapabilityMaterializationState.RESOLVING + waiter = None + waiter_operation = None + owner = (generation, future) + if waiter is not None: + result = await self._wait_async(waiter) + if waiter_operation == "materialize": + return result + continue + break + + generation, future = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="materialize", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + implementation = await self._async_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.last_error = None + self._finish_transition(state, future, result=implementation) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return implementation + except BaseException as error: + with state.lock: + state.materialization = CapabilityMaterializationState.FAILED + state.last_error = error + operation_error = self._wrap_error(state.spec.id, "materialize", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="materialize", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + raise operation_error from error + + @staticmethod + def _canonical_implementation(spec: CapabilitySpec, implementation: Any) -> Any: + if implementation is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {spec.id} 的 implementation" + ) + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict) or symbol_name not in namespace: + return implementation + canonical = namespace[symbol_name] + if implementation is not canonical: + raise CapabilityAdapterContractError( + f"adapter 返回的 {spec.id} 实现不是 sys.modules 中的 canonical 对象" + ) + return canonical + + @staticmethod + def _wrap_error(capability_id: str, operation: str, error: BaseException) -> BaseException: + if isinstance(error, (CapabilityOperationError, CapabilityRuntimeClosedError)): + return error + return CapabilityOperationError(capability_id, operation, error) + + def activate(self, capability_id: str, *, reason: str, retry: bool = False) -> Any: + """同步启动能力，并在 start 成功后原子发布候选实例。""" + return self._activate_sync(capability_id, reason=reason, retry=retry, previous=None) + + async def activate_async( + self, + capability_id: str, + *, + reason: str, + retry: bool = False, + ) -> Any: + """异步启动能力，并以 Future 协调并发调用者。""" + return await self._activate_async(capability_id, reason=reason, retry=retry, previous=None) + + def _activate_sync( + self, + capability_id: str, + *, + reason: str, + retry: bool, + previous: Any, + operation: str = "activate", + ) -> Any: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if operation == "activate" and state.instance is not None: + return state.instance + if state.pending_stop is not None and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.lifecycle is CapabilityLifecycleState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = operation + pending_stop = state.pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPING + if pending_stop is not None + else CapabilityLifecycleState.STARTING + ) + if state.implementation is None: + state.materialization = CapabilityMaterializationState.RESOLVING + pending_cleanup = state.pending_cleanup + pending_error = state.pending_cleanup_error + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + implementation = state.implementation + waiter = None + waiter_operation = None + owner = ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) + if waiter is not None: + result = self._wait_sync(waiter) + if waiter_operation == operation: + return result + continue + break + + ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation=operation, + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + try: + if pending_stop is not None: + self._sync_callback( + adapter, + "stop", + state.spec, + pending_stop, + generation, + ) + pending_stop = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STARTING + if pending_cleanup is not None: + try: + self._sync_callback( + adapter, + "cleanup", + state.spec, + pending_cleanup, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + except BaseException: + with state.lock: + state.pending_cleanup = pending_cleanup + state.pending_cleanup_error = pending_error + raise + if implementation is None: + implementation = self._sync_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + candidate = self._sync_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + self._sync_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation=operation, + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = self._cleanup_failed_sync( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.implementation = implementation + state.materialization = ( + CapabilityMaterializationState.RESOLVED + if implementation is not None + else CapabilityMaterializationState.FAILED + ) + state.instance = None + if pending_stop is not None: + state.pending_stop = pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, operation, final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation=operation, + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + async def _activate_async( + self, + capability_id: str, + *, + reason: str, + retry: bool, + previous: Any, + operation: str = "activate", + ) -> Any: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + self._calibrate_consumer_materialization(state) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if operation == "activate" and state.instance is not None: + return state.instance + if state.pending_stop is not None and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.lifecycle is CapabilityLifecycleState.FAILED and not retry: + raise CapabilityOperationError( + state.spec.id, + operation, + RuntimeError("能力处于 FAILED，必须显式 retry=True"), + ) + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = operation + pending_stop = state.pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPING + if pending_stop is not None + else CapabilityLifecycleState.STARTING + ) + if state.implementation is None: + state.materialization = CapabilityMaterializationState.RESOLVING + pending_cleanup = state.pending_cleanup + pending_error = state.pending_cleanup_error + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + implementation = state.implementation + waiter = None + waiter_operation = None + owner = ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) + if waiter is not None: + result = await self._wait_async(waiter) + if waiter_operation == operation: + return result + continue + break + + ( + generation, + future, + implementation, + pending_stop, + pending_cleanup, + pending_error, + ) = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation=operation, + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + try: + if pending_stop is not None: + await self._async_callback( + adapter, + "stop", + state.spec, + pending_stop, + generation, + ) + pending_stop = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STARTING + if pending_cleanup is not None: + try: + await self._async_callback( + adapter, + "cleanup", + state.spec, + pending_cleanup, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + except BaseException: + with state.lock: + state.pending_cleanup = pending_cleanup + state.pending_cleanup_error = pending_error + raise + if implementation is None: + implementation = await self._async_callback(adapter, "materialize", state.spec) + implementation = self._canonical_implementation(state.spec, implementation) + candidate = await self._async_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + await self._async_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.implementation = implementation + state.materialization = CapabilityMaterializationState.RESOLVED + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation=operation, + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = await self._cleanup_failed_async( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.implementation = implementation + state.materialization = ( + CapabilityMaterializationState.RESOLVED + if implementation is not None + else CapabilityMaterializationState.FAILED + ) + state.instance = None + if pending_stop is not None: + state.pending_stop = pending_stop + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, operation, final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation=operation, + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + def _cleanup_failed_sync( + self, + state: _CapabilityState, + adapter: Any, + candidate: Any, + generation: int, + error: BaseException, + ) -> Optional[BaseException]: + if candidate is None: + return None + try: + self._sync_callback( + adapter, + "cleanup", + state.spec, + candidate, + generation, + error, + ) + return None + except BaseException as cleanup_error: + with state.lock: + state.pending_cleanup = candidate + state.pending_cleanup_error = cleanup_error + return RuntimeError(f"{error}; cleanup failed: {cleanup_error}") + + async def _cleanup_failed_async( + self, + state: _CapabilityState, + adapter: Any, + candidate: Any, + generation: int, + error: BaseException, + ) -> Optional[BaseException]: + if candidate is None: + return None + try: + await self._async_callback( + adapter, + "cleanup", + state.spec, + candidate, + generation, + error, + ) + return None + except BaseException as cleanup_error: + with state.lock: + state.pending_cleanup = candidate + state.pending_cleanup_error = cleanup_error + return RuntimeError(f"{error}; cleanup failed: {cleanup_error}") + + def reload(self, capability_id: str, *, reason: str) -> Any: + """撤销当前实例后，通过同步 adapter 完成新 generation 的资源切换。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + else: + waiter = None + waiter_operation = None + if state.pending_stop is not None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.instance is None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("只有 RUNNING 能力可以 reload"), + ) + previous = state.instance + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "reload" + state.instance = None + state.lifecycle = CapabilityLifecycleState.RELOADING + implementation = state.implementation + break + result = self._wait_sync(waiter) + if waiter_operation == "reload": + return result + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="reload", + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + previous_released = False + try: + self._sync_callback(adapter, "stop", state.spec, previous, generation) + previous_released = True + candidate = self._sync_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + self._sync_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation="reload", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = self._cleanup_failed_sync( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.instance = None + if not previous_released: + state.pending_stop = previous + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, "reload", final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="reload", + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + async def reload_async(self, capability_id: str, *, reason: str) -> Any: + """撤销当前实例后，通过异步 adapter 完成新 generation 的资源切换。""" + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + while True: + with self._shutdown_lock: + self._ensure_open() + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + else: + waiter = None + waiter_operation = None + if state.pending_stop is not None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("能力仍持有未释放资源，必须先重试 stop"), + ) + if state.instance is None: + raise CapabilityOperationError( + state.spec.id, + "reload", + RuntimeError("只有 RUNNING 能力可以 reload"), + ) + previous = state.instance + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "reload" + state.instance = None + state.lifecycle = CapabilityLifecycleState.RELOADING + implementation = state.implementation + break + result = await self._wait_async(waiter) + if waiter_operation == "reload": + return result + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="reload", + outcome="started", + reason=reason, + started_at=started_at, + ) + candidate = None + previous_released = False + try: + await self._async_callback(adapter, "stop", state.spec, previous, generation) + previous_released = True + candidate = await self._async_callback( + adapter, + "create", + state.spec, + implementation, + generation, + previous, + ) + if candidate is None: + raise CapabilityAdapterContractError( + f"adapter 没有返回 {state.spec.id} 的 candidate" + ) + await self._async_callback(adapter, "start", state.spec, candidate, generation) + self._ensure_open() + with state.lock: + state.instance = candidate + state.lifecycle = CapabilityLifecycleState.RUNNING + state.last_error = None + self._finish_transition(state, future, result=candidate) + self._emit( + state, + generation=generation, + operation="reload", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + return candidate + except BaseException as error: + cleanup_error = await self._cleanup_failed_async( + state, + adapter, + candidate, + generation, + error, + ) + final_error = cleanup_error or error + with state.lock: + state.instance = None + if not previous_released: + state.pending_stop = previous + state.lifecycle = ( + CapabilityLifecycleState.STOPPED + if isinstance(error, CapabilityRuntimeClosedError) + else CapabilityLifecycleState.FAILED + ) + state.last_error = final_error + operation_error = self._wrap_error(state.spec.id, "reload", final_error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="reload", + outcome="failed", + reason=reason, + started_at=started_at, + error=final_error, + ) + raise operation_error from error + + def stop(self, capability_id: str, *, reason: str) -> None: + """同步撤销并停止能力资源；物化实现保留供后续显式重启。""" + self._stop_sync(capability_id, reason=reason, shutdown=False) + + def _stop_sync(self, capability_id: str, *, reason: str, shutdown: bool) -> None: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.SYNC) + while True: + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + stop_owner = ( + state.instance + if state.instance is not None + else state.pending_stop + ) + pending = state.pending_cleanup + pending_error = state.pending_cleanup_error + if stop_owner is None and pending is None: + if state.lifecycle is not CapabilityLifecycleState.FAILED: + state.lifecycle = CapabilityLifecycleState.STOPPED + return + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "stop" + state.instance = None + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + state.lifecycle = CapabilityLifecycleState.STOPPING + waiter = None + waiter_operation = None + owner = (generation, future, stop_owner, pending, pending_error) + if waiter is not None: + try: + self._wait_sync(waiter) + except BaseException: + if not shutdown: + raise + if waiter_operation == "stop": + return + continue + break + + generation, future, stop_owner, pending, pending_error = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="stop", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + if stop_owner is not None: + self._sync_callback(adapter, "stop", state.spec, stop_owner, generation) + stop_owner = None + if pending is not None: + self._sync_callback( + adapter, + "cleanup", + state.spec, + pending, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + pending = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STOPPED + state.last_error = None + self._finish_transition(state, future, result=None) + self._emit( + state, + generation=generation, + operation="stop", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + except BaseException as error: + with state.lock: + state.lifecycle = CapabilityLifecycleState.FAILED + state.last_error = error + if stop_owner is not None: + state.pending_stop = stop_owner + if pending is not None: + state.pending_cleanup = pending + state.pending_cleanup_error = error + operation_error = self._wrap_error(state.spec.id, "stop", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="stop", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + if not shutdown: + raise operation_error from error + + async def stop_async(self, capability_id: str, *, reason: str) -> None: + """异步撤销并停止能力资源。""" + await self._stop_async(capability_id, reason=reason, shutdown=False) + + async def _stop_async(self, capability_id: str, *, reason: str, shutdown: bool) -> None: + state = self._state(capability_id) + adapter = self._adapter(state, AdapterExecutionMode.ASYNC) + while True: + with state.lock: + if state.inflight is not None: + waiter = state.inflight + waiter_operation = state.inflight_operation + owner = None + else: + stop_owner = ( + state.instance + if state.instance is not None + else state.pending_stop + ) + pending = state.pending_cleanup + pending_error = state.pending_cleanup_error + if stop_owner is None and pending is None: + if state.lifecycle is not CapabilityLifecycleState.FAILED: + state.lifecycle = CapabilityLifecycleState.STOPPED + return + state.generation += 1 + generation = state.generation + future: Future[Any] = Future() + state.inflight = future + state.inflight_operation = "stop" + state.instance = None + state.pending_stop = None + state.pending_cleanup = None + state.pending_cleanup_error = None + state.lifecycle = CapabilityLifecycleState.STOPPING + waiter = None + waiter_operation = None + owner = (generation, future, stop_owner, pending, pending_error) + if waiter is not None: + try: + await self._wait_async(waiter) + except BaseException: + if not shutdown: + raise + if waiter_operation == "stop": + return + continue + break + + generation, future, stop_owner, pending, pending_error = owner + started_at = time.monotonic() + self._emit( + state, + generation=generation, + operation="stop", + outcome="started", + reason=reason, + started_at=started_at, + ) + try: + if stop_owner is not None: + await self._async_callback( + adapter, + "stop", + state.spec, + stop_owner, + generation, + ) + stop_owner = None + if pending is not None: + await self._async_callback( + adapter, + "cleanup", + state.spec, + pending, + generation, + pending_error or RuntimeError("pending cleanup"), + ) + pending = None + with state.lock: + state.lifecycle = CapabilityLifecycleState.STOPPED + state.last_error = None + self._finish_transition(state, future, result=None) + self._emit( + state, + generation=generation, + operation="stop", + outcome="succeeded", + reason=reason, + started_at=started_at, + ) + except BaseException as error: + with state.lock: + state.lifecycle = CapabilityLifecycleState.FAILED + state.last_error = error + if stop_owner is not None: + state.pending_stop = stop_owner + if pending is not None: + state.pending_cleanup = pending + state.pending_cleanup_error = error + operation_error = self._wrap_error(state.spec.id, "stop", error) + self._finish_transition(state, future, error=operation_error) + self._emit( + state, + generation=generation, + operation="stop", + outcome="failed", + reason=reason, + started_at=started_at, + error=error, + ) + if not shutdown: + raise operation_error from error + + def shutdown(self, *, reason: str) -> None: + """不可逆关闭仅含同步 adapter 的 Runtime，并阻止并发首启重新发布。""" + async_kinds = { + kind + for kind, adapter in self._adapters.items() + if getattr(adapter, "execution_mode", None) is AdapterExecutionMode.ASYNC + } + if async_kinds: + raise CapabilityAdapterModeError( + f"同步 shutdown 不能处理异步 adapter：{sorted(async_kinds)}" + ) + with self._shutdown_lock: + self._shutdown = True + for spec in self._registry.list_specs(): + self._stop_sync(spec.id, reason=reason, shutdown=True) + + async def shutdown_async(self, *, reason: str) -> None: + """不可逆关闭混合同步/异步 adapter 的 Runtime。""" + with self._shutdown_lock: + self._shutdown = True + for spec in self._registry.list_specs(): + adapter = self._adapters[spec.kind] + if getattr(adapter, "execution_mode", None) is AdapterExecutionMode.ASYNC: + await self._stop_async(spec.id, reason=reason, shutdown=True) + else: + await asyncio.to_thread( + self._stop_sync, + spec.id, + reason=reason, + shutdown=True, + ) diff --git a/app/runtime/events.py b/app/runtime/events.py index aea2174d6..983ca1ec4 100644 --- a/app/runtime/events.py +++ b/app/runtime/events.py @@ -415,21 +415,28 @@ class EventManager(metaclass=Singleton): 同步方式调度链式事件，按优先级顺序逐个调用事件处理器，并记录每个处理器的处理时间 :param event: 要调度的事件对象 """ - handlers = self.__chain_subscribers.get(event.event_type, {}) + # 运行期可以动态注册或移除处理器；当前事件始终使用调度开始时的快照。 + with self.__lock: + handlers = tuple( + self.__chain_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for chain event: {event}") return False # 过滤出启用的处理器 - enabled_handlers = {handler_id: (priority, handler) for handler_id, (priority, handler) in handlers.items() - if self.__is_handler_enabled(handler)} + enabled_handlers = tuple( + (handler_id, priority, handler) + for handler_id, (priority, handler) in handlers + if self.__is_handler_enabled(handler) + ) if not enabled_handlers: logger.debug(f"No enabled handlers found for chain event: {event}. Skipping execution.") return False self.__log_event_lifecycle(event, "Started") - for handler_id, (priority, handler) in enabled_handlers.items(): + for handler_id, priority, handler in enabled_handlers: start_time = time.time() self.__safe_invoke_handler(handler, event) logger.debug( @@ -444,21 +451,28 @@ class EventManager(metaclass=Singleton): 异步方式调度链式事件，按优先级顺序逐个调用事件处理器，并记录每个处理器的处理时间 :param event: 要调度的事件对象 """ - handlers = self.__chain_subscribers.get(event.event_type, {}) + # 快照在锁内建立、在锁外执行，处理器可以安全地修改后续订阅。 + with self.__lock: + handlers = tuple( + self.__chain_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for chain event: {event}") return False # 过滤出启用的处理器 - enabled_handlers = {handler_id: (priority, handler) for handler_id, (priority, handler) in handlers.items() - if self.__is_handler_enabled(handler)} + enabled_handlers = tuple( + (handler_id, priority, handler) + for handler_id, (priority, handler) in handlers + if self.__is_handler_enabled(handler) + ) if not enabled_handlers: logger.debug(f"No enabled handlers found for chain event: {event}. Skipping execution.") return False self.__log_event_lifecycle(event, "Started") - for handler_id, (priority, handler) in enabled_handlers.items(): + for handler_id, priority, handler in enabled_handlers: start_time = time.time() await self.__safe_invoke_handler_async(handler, event) logger.debug( @@ -473,7 +487,11 @@ class EventManager(metaclass=Singleton): 异步方式调度广播事件，通过线程池逐个调用事件处理器 :param event: 要调度的事件对象 """ - handlers = self.__broadcast_subscribers.get(event.event_type, {}) + # 快照隔离当前调度与运行期订阅变更；变更从下一个事件开始生效。 + with self.__lock: + handlers = tuple( + self.__broadcast_subscribers.get(event.event_type, {}).items() + ) if not handlers: logger.debug(f"No handlers found for broadcast event: {event}") return @@ -481,7 +499,7 @@ class EventManager(metaclass=Singleton): if event.event_type == EventType.MessageAction and isinstance(event.event_data, dict): target_plugin_id = event.event_data.get("__mp_target_plugin_id") # 为每个处理器提供独立的事件实例，防止某个处理器对 event_data 的修改影响其他处理器 - for handler_id, handler in handlers.items(): + for handler_id, handler in handlers: if target_plugin_id and not self.__should_dispatch_to_target_plugin( handler, handler_id, str(target_plugin_id) ): diff --git a/app/runtime/extensions/host_module_adapter.py b/app/runtime/extensions/host_module_adapter.py new file mode 100644 index 000000000..633dd4bdf --- /dev/null +++ b/app/runtime/extensions/host_module_adapter.py @@ -0,0 +1,260 @@ +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping + +from app.runtime.capabilities.model import ( + ActivationPolicy, + AdapterExecutionMode, + CapabilitySpec, + SelectorSchema, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.config import settings +from app.runtime.extensions.service_config import ServiceConfigHelper +from app.schemas.types import ( + DownloaderType, + MediaRecognizeType, + MediaServerType, + MessageChannel, + ModuleType, + OtherModulesType, + StorageSchema, + SystemConfigKey, +) + + +HOST_MODULE_KIND = "host_module" +_SETTING_SELECTOR = "setting_truthy" +_SERVICE_SELECTOR = "system_config_item" +_MODULE_ROOT = Path(__file__).resolve().parents[2] / "modules" +_SERVICE_CONFIG_GETTERS = MappingProxyType({ + SystemConfigKey.Downloaders.value: ServiceConfigHelper.get_downloader_configs, + SystemConfigKey.MediaServers.value: ServiceConfigHelper.get_mediaserver_configs, + SystemConfigKey.Notifications.value: ServiceConfigHelper.get_notification_configs, +}) +_SUBTYPE_NAMES = frozenset( + item.name + for enum_type in ( + DownloaderType, + MediaServerType, + MessageChannel, + StorageSchema, + OtherModulesType, + MediaRecognizeType, + ) + for item in enum_type +) + + +@dataclass(frozen=True, slots=True) +class HostModuleConfigSnapshot: + """一次 reconcile 使用的不可变配置视图，避免每个能力重复查询配置。""" + + settings: Mapping[str, Any] + services: Mapping[str, tuple[Any, ...]] + + +def _validate_setting_selector(config: Mapping[str, Any]) -> None: + """限制 setting selector 只能读取已声明的应用设置。""" + key = config["key"] + if not isinstance(key, str) or not key or not hasattr(settings, key): + raise ValueError(f"未知应用设置：{key!r}") + + +def _validate_service_selector(config: Mapping[str, Any]) -> None: + """限制服务 selector 使用经过 Schema 校验的三个宿主服务配置。""" + key = config["key"] + if key not in _SERVICE_CONFIG_GETTERS: + raise ValueError(f"不支持的服务配置：{key!r}") + if config["match_field"] != "type": + raise ValueError("服务 selector 的 match_field 必须是 type") + if config["enabled_field"] != "enabled": + raise ValueError("服务 selector 的 enabled_field 必须是 enabled") + match_value = config["match_value"] + if not isinstance(match_value, str) or not match_value: + raise ValueError("服务 selector 的 match_value 必须是非空字符串") + + +HOST_MODULE_SELECTOR_SCHEMAS = MappingProxyType({ + _SETTING_SELECTOR: SelectorSchema( + required_fields=frozenset({"key"}), + validator=_validate_setting_selector, + ), + _SERVICE_SELECTOR: SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + validator=_validate_service_selector, + ), +}) + + +def _validate_manifest_inventory(registry: CapabilityRegistry) -> None: + """校验一级模块包与 manifest 一一对应，并固定宿主声明合同。""" + module_packages = { + child.name + for child in _MODULE_ROOT.iterdir() + if child.is_dir() + and not child.name.startswith("_") + and (child / "__init__.py").is_file() + } + specs = registry.list_specs() + manifest_packages = {spec.source.parent.name for spec in specs} + if module_packages != manifest_packages: + missing = sorted(module_packages - manifest_packages) + unknown = sorted(manifest_packages - module_packages) + raise ValueError( + f"Host Module manifest inventory 不一致：missing={missing} unknown={unknown}" + ) + + allowed_metadata = {"name", "type", "subtype", "priority"} + module_type_values = {item.value for item in ModuleType} + for spec in specs: + if spec.source.parent.parent != _MODULE_ROOT: + raise ValueError(f"Host Module manifest 必须位于一级模块包：{spec.source}") + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + expected_module = f"app.modules.{spec.source.parent.name}" + if module_name != expected_module or symbol_name != spec.id: + raise ValueError( + f"{spec.source}: entrypoint 必须指向同包且类名等于 capability id" + ) + if set(spec.metadata) != allowed_metadata: + raise ValueError( + f"{spec.source}: metadata 字段必须是 {sorted(allowed_metadata)}" + ) + if spec.metadata["type"] not in module_type_values: + raise ValueError(f"{spec.source}: 非法 metadata.type={spec.metadata['type']!r}") + if spec.metadata["subtype"] not in _SUBTYPE_NAMES: + raise ValueError( + f"{spec.source}: 非法 metadata.subtype={spec.metadata['subtype']!r}" + ) + priority = spec.metadata["priority"] + if isinstance(priority, bool) or not isinstance(priority, int): + raise ValueError(f"{spec.source}: metadata.priority 必须是整数") + if spec.activation is ActivationPolicy.WHEN_CONFIGURED: + selector_key = str(spec.selector.config["key"]) + if selector_key not in spec.watch: + raise ValueError( + f"{spec.source}: activation.watch 必须包含 selector 配置键" + ) + + +def build_host_module_registry() -> CapabilityRegistry: + """从现有物理模块包构建 import-free Host Module Registry。""" + registry = CapabilityRegistry.discover( + (_MODULE_ROOT,), + kinds={HOST_MODULE_KIND}, + selector_schemas=HOST_MODULE_SELECTOR_SCHEMAS, + ) + _validate_manifest_inventory(registry) + return registry + + +def capture_host_module_config( + specs: tuple[CapabilitySpec, ...], +) -> HostModuleConfigSnapshot: + """对本轮涉及的设置和服务配置各读取一次并冻结容器。""" + setting_keys: set[str] = set() + service_keys: set[str] = set() + for spec in specs: + selector = spec.selector + if selector is None: + continue + key = str(selector.config["key"]) + if selector.kind == _SETTING_SELECTOR: + setting_keys.add(key) + elif selector.kind == _SERVICE_SELECTOR: + service_keys.add(key) + + setting_values = { + key: getattr(settings, key) + for key in sorted(setting_keys) + } + service_values = { + key: tuple(_SERVICE_CONFIG_GETTERS[key]()) + for key in sorted(service_keys) + } + return HostModuleConfigSnapshot( + settings=MappingProxyType(setting_values), + services=MappingProxyType(service_values), + ) + + +def should_run_host_module( + spec: CapabilitySpec, + snapshot: HostModuleConfigSnapshot, +) -> bool: + """依据有限 selector 语法判断能力是否应拥有运行资源。""" + if spec.activation is ActivationPolicy.BOOTSTRAP: + return True + if spec.activation is ActivationPolicy.ON_FIRST_USE: + return False + selector = spec.selector + if selector is None: + return False + if selector.kind == _SETTING_SELECTOR: + return bool(snapshot.settings[selector.config["key"]]) + if selector.kind == _SERVICE_SELECTOR: + config = selector.config + return any( + getattr(item, config["match_field"]) == config["match_value"] + and bool(getattr(item, config["enabled_field"])) + for item in snapshot.services[config["key"]] + ) + raise ValueError(f"未支持的 Host Module selector：{selector.kind}") + + +class HostModuleAdapter: + """把现有模块类接入 Capability Runtime，保持类路径与对象 identity 不变。""" + + execution_mode = AdapterExecutionMode.SYNC + + @staticmethod + def materialize(spec: CapabilitySpec) -> type: + """按 manifest entrypoint 导入并返回原始模块类。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + implementation = getattr(importlib.import_module(module_name), symbol_name) + if not isinstance(implementation, type): + raise TypeError(f"{spec.entrypoint} 不是模块类") + return implementation + + @staticmethod + def create( + spec: CapabilitySpec, + implementation: type, + generation: int, + previous: Any = None, + ) -> Any: + """首次创建实例；配置重载继续使用原实例以保留既有模块语义。""" + del spec, generation + return previous if previous is not None else implementation() + + @staticmethod + def start(spec: CapabilitySpec, candidate: Any, generation: int) -> None: + """初始化候选实例拥有的连接、线程或客户端资源。""" + del spec, generation + candidate.init_module() + + @staticmethod + def stop(spec: CapabilitySpec, instance: Any, generation: int) -> None: + """停止实例拥有的资源；Runtime 会先撤销其运行态可见性。""" + del spec, generation + instance.stop() + + @staticmethod + def cleanup( + spec: CapabilitySpec, + candidate: Any, + generation: int, + error: BaseException, + ) -> None: + """启动失败后尽力回收候选实例已创建的部分资源。""" + del spec, generation, error + candidate.stop() diff --git a/app/runtime/extensions/module_manager.py b/app/runtime/extensions/module_manager.py index dd189cecc..88efff72d 100644 --- a/app/runtime/extensions/module_manager.py +++ b/app/runtime/extensions/module_manager.py @@ -1,22 +1,42 @@ -import traceback -from typing import Generator, Optional, Tuple, Any, Union, List +from __future__ import annotations + +import sys +import threading +from typing import Any, Generator, List, Optional, Tuple, Union -from app.runtime.config import settings -from app.runtime.events import EventHandlerBinding, eventmanager -from app.foundation.reflection import ModuleHelper -from app.runtime.log import logger -from app.schemas.types import EventType, ModuleType, DownloaderType, MediaServerType, MessageChannel, StorageSchema, \ - OtherModulesType, MediaRecognizeType from app.foundation.reflection import ObjectUtils from app.foundation.singleton import Singleton +from app.runtime.capabilities.model import ( + CapabilityLifecycleState, + CapabilityObservation, + CapabilitySpec, +) +from app.runtime.capabilities.runtime import CapabilityRuntime +from app.runtime.config import settings +from app.runtime.events import Event, EventHandlerBinding, eventmanager +from app.runtime.extensions.host_module_adapter import ( + HOST_MODULE_KIND, + HostModuleAdapter, + build_host_module_registry, + capture_host_module_config, + should_run_host_module, +) +from app.runtime.log import logger +from app.schemas.types import ( + DownloaderType, + EventType, + MediaRecognizeType, + MediaServerType, + MessageChannel, + ModuleType, + OtherModulesType, + StorageSchema, +) class ModuleManager(metaclass=Singleton): - """ - 模块管理器 - """ + """以 Capability Runtime 管理宿主模块，并保留旧插件同步查询合同。""" - # 子模块类型集合 SubType = Union[ DownloaderType, MediaServerType, @@ -26,176 +46,286 @@ class ModuleManager(metaclass=Singleton): MediaRecognizeType, ] - def __init__(self): - """初始化模块注册表并装载当前启用的运行模块。""" - # 模块列表 - self._modules: dict = {} - # 运行态模块列表 - self._running_modules: dict = {} - # 事件总线通过该解析器绑定已启用的模块实例。 + def __init__(self) -> None: + """发现 data-only manifest，并按当前配置激活所需宿主模块。""" + self._lock = threading.RLock() + self._lifecycle_lock = threading.RLock() + self._modules: dict[str, type] = {} + self._running_modules: dict[str, Any] = {} + registry = build_host_module_registry() + self._runtime = CapabilityRuntime( + registry, + adapters={HOST_MODULE_KIND: HostModuleAdapter()}, + observer=self._observe_transition, + ) + # pkgutil 的既有发现顺序按一级包名稳定排列，兼容视图继续保持该顺序。 + self._specs = tuple( + sorted(self._runtime.list_specs(), key=lambda item: item.source.parent.name) + ) eventmanager.register_handler_instance_resolver( "modules", self.resolve_event_handler_instance, ) + eventmanager.add_event_listener( + EventType.ConfigChanged, + self.handle_config_changed, + ) self.load_modules() + @staticmethod + def _observe_transition(observation: CapabilityObservation) -> None: + """把 Runtime 的稳定转换结果接入现有日志面。""" + if observation.outcome == "failed": + logger.error( + "Host Module %s %s 失败：%s", + observation.capability_id, + observation.operation, + observation.error, + ) + elif observation.outcome == "succeeded": + logger.debug( + "Host Module %s %s 完成，generation=%s，耗时=%.2fms", + observation.capability_id, + observation.operation, + observation.generation, + observation.duration_ms, + ) + + @staticmethod + def _event_changed_keys(event: Optional[Event]) -> set[str]: + """兼容对象和 dict 两种配置事件载荷。""" + if not event: + return set() + event_data = event.event_data + if isinstance(event_data, dict): + keys = event_data.get("key", set()) + else: + keys = getattr(event_data, "key", set()) + if isinstance(keys, str): + return {keys} + return {str(key) for key in (keys or set())} + + def _remember_materialized(self, module_id: str, implementation: type) -> type: + """更新旧 `_modules` 视图，但不改变能力资源生命周期。""" + with self._lock: + self._modules[module_id] = implementation + return implementation + + def _consumer_materialized_class(self, spec: CapabilitySpec) -> Optional[type]: + """识别插件显式旧导入产生的真实类，不触发新的 Python import。""" + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + module = sys.modules.get(module_name) + namespace = getattr(module, "__dict__", None) if module is not None else None + if not isinstance(namespace, dict): + return None + implementation = namespace.get(symbol_name) + return implementation if isinstance(implementation, type) else None + + def _refresh_running_projection(self) -> None: + """从 Runtime 已发布实例重建插件可见的运行模块字典。""" + running = { + spec.id: instance + for spec in self._specs + if (instance := self._runtime.get_running(spec.id)) is not None + } + with self._lock: + self._running_modules = running + def resolve_event_handler_instance( - self, - owner_class: type, + self, + owner_class: type, ) -> Optional[EventHandlerBinding]: - """为模块声明的事件方法解析当前运行实例。""" - module_id = owner_class.__name__ - if module_id not in self._modules: - return None - module = self._running_modules.get(module_id) - owner_name = module_id - if module and callable(getattr(module, "get_name", None)): - owner_name = module.get_name() - return EventHandlerBinding( - instance=module, - owner_name=owner_name, + """按 canonical class identity 绑定当前 generation，停止态阻断 fallback 构造。""" + for spec in self._specs: + with self._lock: + implementation = self._modules.get(spec.id) + if implementation is None: + implementation = self._consumer_materialized_class(spec) + if implementation is not None: + self._remember_materialized(spec.id, implementation) + # 同步 Runtime 的物化观测，但不创建或启动实例。 + self._runtime.snapshot(spec.id) + if implementation is not owner_class: + continue + return EventHandlerBinding( + instance=self._runtime.get_running(spec.id), + owner_name=str(spec.metadata["name"]), + ) + return None + + def _reconcile( + self, + *, + reason: str, + changed_keys: Optional[set[str]] = None, + reload_running: bool = False, + ) -> None: + """以一次配置快照串行协调需要启动、重载或停止的能力。""" + with self._lifecycle_lock: + selected = tuple( + spec + for spec in self._specs + if changed_keys is None or changed_keys.intersection(spec.watch) + ) + snapshot = capture_host_module_config(selected) + for spec in selected: + desired = should_run_host_module(spec, snapshot) + running = self._runtime.get_running(spec.id) + try: + if desired and running is None: + instance = self._runtime.activate( + spec.id, + reason=reason, + retry=True, + ) + self._remember_materialized(spec.id, type(instance)) + elif desired and running is not None and reload_running: + instance = self._runtime.reload(spec.id, reason=reason) + self._remember_materialized(spec.id, type(instance)) + elif not desired and running is not None: + self._runtime.stop(spec.id, reason=reason) + except Exception: + # 单能力失败由 Runtime 完整记录；其它无依赖能力继续 reconcile。 + continue + self._refresh_running_projection() + + def load_modules(self) -> None: + """按当前配置启动未运行模块；已运行模块保持当前 generation。""" + self._reconcile(reason="module_manager_load") + + def handle_config_changed(self, event: Event) -> None: + """配置变更时仅协调 watch 命中的能力，并保证单一生命周期 writer。""" + changed_keys = self._event_changed_keys(event) + if not changed_keys: + return + self._reconcile( + reason="config_changed", + changed_keys=changed_keys, + reload_running=True, ) - def load_modules(self): - """ - 加载所有模块 - """ - # 扫描模块目录 - modules = ModuleHelper.load( - "app.modules", - filter_func=lambda _, obj: hasattr(obj, 'init_module') and hasattr(obj, 'init_setting') - ) - self._running_modules = {} - self._modules = {} - for module in modules: - module_id = module.__name__ - self._modules[module_id] = module - try: - # 生成实例 - _module = module() - # 初始化模块 - if self.check_setting(_module.init_setting()): - # 通过模板开关控制加载 - _module.init_module() - self._running_modules[module_id] = _module - logger.debug(f"Moudle Loaded：{module_id}") - except Exception as err: - logger.error(f"Load Moudle Error：{module_id}，{str(err)} - {traceback.format_exc()}", exc_info=True) - - def stop(self): - """ - 停止所有模块 - """ + def stop(self) -> None: + """停止全部运行模块但保留 Runtime，使旧插件可随后再次 load。""" logger.info("正在停止所有模块...") - for module_id, module in self._running_modules.items(): - try: - module.stop() - logger.debug(f"Moudle Stoped：{module_id}") - except Exception as err: - logger.error(f"Stop Moudle Error：{module_id}，{str(err)} - {traceback.format_exc()}", exc_info=True) + with self._lifecycle_lock: + for spec in reversed(self._specs): + snapshot = self._runtime.snapshot(spec.id) + if ( + self._runtime.get_running(spec.id) is None + and snapshot.lifecycle is not CapabilityLifecycleState.FAILED + ): + continue + try: + self._runtime.stop(spec.id, reason="module_manager_stop") + except Exception: + continue + self._refresh_running_projection() logger.info("所有模块停止完成") - def reload(self): - """ - 重新加载所有模块 - """ - self.stop() - self.load_modules() - eventmanager.send_event(etype=EventType.ModuleReload, data={}) + def shutdown(self) -> None: + """进程关闭时不可逆停止 Runtime，阻止并发能力重新发布。""" + logger.info("正在关闭模块运行时...") + with self._lifecycle_lock: + self._runtime.shutdown(reason="application_shutdown") + self._refresh_running_projection() + logger.info("模块运行时关闭完成") + + def reload(self) -> None: + """保留旧插件可观察的 stop、load、ModuleReload 同步顺序。""" + with self._lifecycle_lock: + self.stop() + self.load_modules() + eventmanager.send_event(etype=EventType.ModuleReload, data={}) def test(self, modleid: str) -> Tuple[bool, str]: - """ - 测试模块 - """ - if modleid not in self._running_modules: + """测试已运行模块；未启用模块保持旧合同返回 `(False, "")`。""" + module = self.get_running_module(modleid) + if module is None: return False, "" - module = self._running_modules[modleid] - if hasattr(module, "test") \ - and ObjectUtils.check_method(getattr(module, "test")): + if hasattr(module, "test") and ObjectUtils.check_method(module.test): result = module.test() - if not result: - return False, "" - return result + return result if result else (False, "") return True, "模块不支持测试" @staticmethod def check_setting(setting: Optional[tuple]) -> bool: - """ - 检查开关是否己打开，开关使用,分隔多个值，符合其中即代表开启 - """ + """保留旧模块开关的 truthy 与 membership 判定语义。""" if not setting: return True switch, value = setting option = getattr(settings, switch) if not option: return False - if option and value is True: + if value is True: return True - if value in option: - return True - return False + return value in option def get_running_module(self, module_id: str) -> Any: - """ - 根据模块id获取模块运行实例 - """ - if not module_id: + """根据模块 ID 返回已发布的运行实例，不触发物化。""" + if not module_id or self._runtime.get_spec(module_id) is None: return None - if not self._running_modules: - return None - return self._running_modules.get(module_id) + return self._runtime.get_running(module_id) + + def _running_snapshot(self) -> tuple[Any, ...]: + """直接读取 Runtime 发布视图，转换期间不暴露旧或候选实例。""" + return tuple( + instance + for spec in self._specs + if (instance := self._runtime.get_running(spec.id)) is not None + ) def get_running_modules(self, method: str) -> Generator: - """ - 获取实现了同一方法的模块列表 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, method) \ - and ObjectUtils.check_method(getattr(module, method)): + """返回实现了指定方法的运行模块快照。""" + for module in self._running_snapshot(): + candidate = getattr(module, method, None) + if callable(candidate) and ObjectUtils.check_method(candidate): yield module def get_running_type_modules(self, module_type: ModuleType) -> Generator: - """ - 获取指定类型的模块列表 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, 'get_type') \ - and module.get_type() == module_type: + """返回指定类型的运行模块快照。""" + for module in self._running_snapshot(): + if module.get_type() == module_type: yield module def get_running_subtype_module(self, module_subtype: SubType) -> Generator: - """ - 获取指定子类型的模块 - """ - if not self._running_modules: - return - for _, module in self._running_modules.items(): - if hasattr(module, 'get_subtype') \ - and module.get_subtype() == module_subtype: + """返回指定子类型的运行模块快照。""" + for module in self._running_snapshot(): + if module.get_subtype() == module_subtype: yield module def get_module(self, module_id: str) -> Any: - """ - 根据模块id获取模块 - """ - if not module_id: + """显式物化并返回 canonical 模块类；失败保持旧合同返回 None。""" + if not module_id or self._runtime.get_spec(module_id) is None: return None - if not self._modules: + with self._lock: + implementation = self._modules.get(module_id) + if implementation is not None: + return implementation + try: + implementation = self._runtime.materialize( + module_id, + reason="compat_get_module", + retry=True, + ) + except Exception: return None - return self._modules.get(module_id) + return self._remember_materialized(module_id, implementation) - def get_modules(self) -> dict: - """ - 获取模块列表 - """ - return self._modules + def get_modules(self) -> dict[str, type]: + """兼容性显式物化全部真实类；单个失败不阻断其它模块。""" + for spec in self._specs: + self.get_module(spec.id) + with self._lock: + return dict(self._modules) def get_module_ids(self) -> List[str]: - """ - 获取模块id列表 - """ - return list(self._modules.keys()) + """从 manifest 返回全部模块 ID，不物化实现。""" + return [spec.id for spec in self._specs] + + def list_specs(self) -> tuple[CapabilitySpec, ...]: + """返回全部轻量模块声明，包含物化或启动失败的能力。""" + return self._specs + + def get_specs(self) -> tuple[CapabilitySpec, ...]: + """兼容内部调用命名，返回与 `list_specs` 相同的声明快照。""" + return self.list_specs() diff --git a/app/runtime/extensions/service_config.py b/app/runtime/extensions/service_config.py new file mode 100644 index 000000000..39a083fda --- /dev/null +++ b/app/runtime/extensions/service_config.py @@ -0,0 +1,76 @@ +from typing import List, Optional, Type + +from pydantic import ValidationError + +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.log import logger +from app.schemas import ( + DownloaderConf, + MediaServerConf, + NotificationConf, + NotificationSwitchConf, +) +from app.schemas.types import NotificationType, SystemConfigKey + + +class ServiceConfigHelper: + """读取并校验通知、下载器和媒体服务器的宿主配置。""" + + @staticmethod + def get_configs(config_key: SystemConfigKey, conf_type: Type) -> List: + """按指定 Schema 过滤单条非法配置，避免影响同组其它服务。""" + config_data = SystemConfigOper().get(config_key) + if not config_data: + return [] + configs = [] + for conf in config_data: + if not isinstance(conf, dict): + logger.warning(f"{config_key.value} 配置格式不正确，已跳过：{conf}") + continue + try: + configs.append(conf_type(**conf)) + except ValidationError as err: + logger.error( + f"{config_key.value} 配置 {conf.get('name')} 校验失败，已跳过：{err}" + ) + return configs + + @staticmethod + def get_downloader_configs() -> List[DownloaderConf]: + """返回已通过结构校验的下载器配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.Downloaders, + DownloaderConf, + ) + + @staticmethod + def get_mediaserver_configs() -> List[MediaServerConf]: + """返回已通过结构校验的媒体服务器配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.MediaServers, + MediaServerConf, + ) + + @staticmethod + def get_notification_configs() -> List[NotificationConf]: + """返回已通过结构校验的通知配置。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.Notifications, + NotificationConf, + ) + + @staticmethod + def get_notification_switches() -> List[NotificationSwitchConf]: + """返回已通过结构校验的通知场景开关。""" + return ServiceConfigHelper.get_configs( + SystemConfigKey.NotificationSwitchs, + NotificationSwitchConf, + ) + + @staticmethod + def get_notification_switch(mtype: NotificationType) -> Optional[str]: + """返回指定通知场景的目标范围。""" + for switch in ServiceConfigHelper.get_notification_switches(): + if switch.type == mtype.value: + return switch.action + return None diff --git a/app/runtime/extensions/service_registry.py b/app/runtime/extensions/service_registry.py index 38f559270..610953501 100644 --- a/app/runtime/extensions/service_registry.py +++ b/app/runtime/extensions/service_registry.py @@ -1,84 +1,18 @@ from typing import Dict, List, Optional, Type, TypeVar, Generic, Iterator -from pydantic import ValidationError - -from app.runtime.extensions.module_manager import ModuleManager from app.db.oper.systemconfig import SystemConfigOper -from app.runtime.log import logger -from app.schemas import DownloaderConf, MediaServerConf, NotificationConf, NotificationSwitchConf, ServiceInfo -from app.schemas.types import NotificationType, SystemConfigKey, ModuleType +from app.runtime.extensions.module_manager import ModuleManager +from app.runtime.extensions.service_config import ServiceConfigHelper +from app.schemas import ServiceInfo +from app.schemas.types import SystemConfigKey, ModuleType TConf = TypeVar("TConf") - -class ServiceConfigHelper: - """ - 配置帮助类，获取不同类型的服务配置 - """ - - @staticmethod - def get_configs(config_key: SystemConfigKey, conf_type: Type) -> List: - """ - 通用获取配置的方法，根据 config_key 获取相应的配置并返回指定类型的配置列表 - - :param config_key: 系统配置的 key - :param conf_type: 用于实例化配置对象的类类型 - :return: 配置对象列表 - """ - config_data = SystemConfigOper().get(config_key) - if not config_data: - return [] - configs = [] - for conf in config_data: - if not isinstance(conf, dict): - logger.warn(f"{config_key.value} 配置格式不正确，已跳过：{conf}") - continue - try: - # 直接使用 conf_type 来实例化配置对象 - configs.append(conf_type(**conf)) - except ValidationError as e: - # 单条配置存在非法值时跳过，避免影响其它服务的初始化 - logger.error(f"{config_key.value} 配置 {conf.get('name')} 校验失败，已跳过：{e}") - return configs - - @staticmethod - def get_downloader_configs() -> List[DownloaderConf]: - """ - 获取下载器的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.Downloaders, DownloaderConf) - - @staticmethod - def get_mediaserver_configs() -> List[MediaServerConf]: - """ - 获取媒体服务器的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.MediaServers, MediaServerConf) - - @staticmethod - def get_notification_configs() -> List[NotificationConf]: - """ - 获取消息通知渠道的配置 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.Notifications, NotificationConf) - - @staticmethod - def get_notification_switches() -> List[NotificationSwitchConf]: - """ - 获取消息通知场景的开关 - """ - return ServiceConfigHelper.get_configs(SystemConfigKey.NotificationSwitchs, NotificationSwitchConf) - - @staticmethod - def get_notification_switch(mtype: NotificationType) -> Optional[str]: - """ - 获取指定类型的消息通知场景的开关 - """ - switchs = ServiceConfigHelper.get_notification_switches() - for switch in switchs: - if switch.type == mtype.value: - return switch.action - return None +__all__ = [ + "ServiceBaseHelper", + "ServiceConfigHelper", + "SystemConfigOper", +] class ServiceBaseHelper(Generic[TConf]): diff --git a/app/runtime/reload.py b/app/runtime/reload.py index 8d4bc6948..dac3e52a1 100644 --- a/app/runtime/reload.py +++ b/app/runtime/reload.py @@ -14,10 +14,16 @@ class ConfigReloadMixin: 可选地重写 get_reload_name 方法提供模块名称（用于日志显示） """ + # 统一生命周期管理器可以继承此 Mixin 的重载方法，但由外部唯一负责事件绑定。 + CONFIG_RELOAD_MANAGED_EXTERNALLY: bool = False + def __init_subclass__(cls, **kwargs): """为声明了 CONFIG_WATCH 的子类生成配置变更处理器。""" super().__init_subclass__(**kwargs) + if getattr(cls, "CONFIG_RELOAD_MANAGED_EXTERNALLY", False): + return + config_watch = getattr(cls, "CONFIG_WATCH", None) if not config_watch: return diff --git a/app/startup/modules_initializer.py b/app/startup/modules_initializer.py index 0fdedcf14..a54b29ef9 100644 --- a/app/startup/modules_initializer.py +++ b/app/startup/modules_initializer.py @@ -184,7 +184,7 @@ async def stop_modules(): logger.error(f"关闭{name}失败：{err}") await run_step("AI智能体", stop_agent) - await run_step("模块", lambda: ModuleManager().stop()) + await run_step("模块", lambda: ModuleManager().shutdown()) await run_step("事件消费", lambda: EventManager().stop()) await run_step("虚拟显示", lambda: DisplayHelper().stop()) await run_step("DoH服务", lambda: DohHelper().shutdown()) diff --git a/scripts/perf/README.md b/scripts/perf/README.md new file mode 100644 index 000000000..42597b6ac --- /dev/null +++ b/scripts/perf/README.md @@ -0,0 +1,106 @@ +# MoviePilot Docker A/B Harness + +该工具用同一个冻结 Docker substrate 对两个 Git commit 做源码级 A/B。派生镜像会先清空 +`/app`，复制目标 commit 的完整 `git archive`，再从 substrate 注入镜像构建阶段生成的插件目录、 +`sites.*.so` 和 `user.sites.v3.bin`。如果依赖或 Docker substrate 输入发生变化，工具会拒绝继续， +避免把外部构建输入漂移误算成性能收益。 + +工具不会读取工作区或容器内的 `app.env`，不挂载真实配置、媒体目录或 Docker socket。固定配置只使用 +内置实验室占位凭据，样本不发布主机端口，并运行在无外网的 internal network。 + +## 一次性预热浏览器 + +默认从固定命名 volume `mp-perf-v3-browser-seed` 复制 CloakBrowser 缓存。该 volume 只需联网预热 +一次： + +```bash +docker volume create mp-perf-v3-browser-seed +docker run --rm \ + --mount source=mp-perf-v3-browser-seed,target=/moviepilot/.cloakbrowser \ + --entrypoint python3 \ + jxxghp/moviepilot-v3@sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720 \ + -m cloakbrowser install +``` + +也可以在 `seed` 或 `run` 时显式传入 `--allow-browser-download`，但该选项会让 seed 阶段联网； +正式 Before/After 样本仍然只克隆预热结果并使用 internal network。 + +## 分阶段执行 + +所有全局参数必须放在子命令前。结果默认写到系统临时目录下的 +`moviepilot-perf-results//`。 + +```bash +PYTHON=../.venv/bin/python +CAMPAIGN=v3-perf-001 + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + build --before-ref upstream/v3 --after-ref HEAD + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + seed + +${PYTHON} scripts/perf/moviepilot_docker_ab.py \ + --campaign "${CAMPAIGN}" \ + sample --variant before --index 1 --points 1,5,10,30 +``` + +开发 harness 时可以用小数分钟做短冒烟，例如 `--points 0,0.02`。正式数据必须保持 +`1,5,10,30`。 + +## 完整三组 A/B + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-001 \ + run \ + --before-ref upstream/v3 \ + --after-ref HEAD \ + --points 1,5,10,30 +``` + +执行顺序固定为： + +```text +Before-1 → After-1 → After-2 → Before-2 → Before-3 → After-3 +``` + +每个样本都从 SQLite 和浏览器 seed 克隆新的命名 volume。样本结束后立即移除容器和样本卷； +完整 `run` 结束后还会移除 campaign seed 与 internal network。派生镜像和本地结果保留，便于复核。 + +## 输出 + +```text +// +├── build.json +├── seed.json +├── results.json +├── report.md +└── samples/ + ├── before-1/ + │ ├── result.json + │ ├── container.log + │ └── modules/ + └── ... +``` + +- `results.json`：Engine stats、进程 PSS/USS/RSS/线程、网络累计值和模块前缀计数； +- `report.md`：三次原值与中位数 Before/After 汇总； +- `modules/`：由目标 MoviePilot Python 进程自身写出的完整 `sys.modules` 名称清单； +- `container.log`：已移除实验室凭据值和本地实例 UUID。 + +容器 working set 统一按 Docker Engine API 的 +`memory_stats.usage - memory_stats.stats.inactive_file` 计算。进程 PSS 仅用于归因，不能替代容器指标。 + +## 清理 + +清理严格限定到 campaign 标签；不会执行 Docker prune： + +```bash +../.venv/bin/python scripts/perf/moviepilot_docker_ab.py \ + --campaign v3-perf-001 cleanup --images +``` + +本地 JSON、Markdown 和日志不会被 `cleanup` 删除。 diff --git a/scripts/perf/instrument/collect_proc.sh b/scripts/perf/instrument/collect_proc.sh new file mode 100644 index 000000000..8cabc6333 --- /dev/null +++ b/scripts/perf/instrument/collect_proc.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# 读取采样开始时已经存在的容器进程，避免把采样器自身计入结果。 + +set -uo pipefail + +self_pid="$$" +process_dirs=(/proc/[0-9]*) + +printf 'pid\tppid\tthreads\trss_kib\tpss_kib\tuss_kib\tcomm\texe\tcmdline\n' + +for process_dir in "${process_dirs[@]}"; do + pid="${process_dir##*/}" + if [[ "${pid}" == "${self_pid}" ]] || [[ ! -r "${process_dir}/status" ]]; then + continue + fi + + status_values="$(awk ' + /^PPid:/ { ppid=$2 } + /^Threads:/ { threads=$2 } + END { printf "%d %d", ppid + 0, threads + 0 } + ' "${process_dir}/status" 2>/dev/null || true)" + read -r ppid threads <<< "${status_values:-0 0}" + + rss_kib=0 + pss_kib=0 + uss_kib=0 + if [[ -r "${process_dir}/smaps_rollup" ]]; then + memory_values="$(awk ' + /^Rss:/ { rss=$2 } + /^Pss:/ { pss=$2 } + /^Private_Clean:/ { uss += $2 } + /^Private_Dirty:/ { uss += $2 } + /^Private_Hugetlb:/ { uss += $2 } + END { printf "%d %d %d", rss + 0, pss + 0, uss + 0 } + ' "${process_dir}/smaps_rollup" 2>/dev/null || true)" + read -r rss_kib pss_kib uss_kib <<< "${memory_values:-0 0 0}" + fi + + comm="" + if [[ -r "${process_dir}/comm" ]]; then + IFS= read -r comm < "${process_dir}/comm" || true + fi + executable="$(readlink "${process_dir}/exe" 2>/dev/null || true)" + command_line="$(tr '\000\t' ' ' < "${process_dir}/cmdline" 2>/dev/null || true)" + comm="${comm//$'\t'/ }" + executable="${executable//$'\t'/ }" + command_line="${command_line//$'\t'/ }" + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "${pid}" "${ppid:-0}" "${threads:-0}" \ + "${rss_kib:-0}" "${pss_kib:-0}" "${uss_kib:-0}" \ + "${comm}" "${executable}" "${command_line}" +done diff --git a/scripts/perf/instrument/sitecustomize.py b/scripts/perf/instrument/sitecustomize.py new file mode 100644 index 000000000..086e242dc --- /dev/null +++ b/scripts/perf/instrument/sitecustomize.py @@ -0,0 +1,35 @@ +"""MoviePilot Docker A/B 测量时使用的最小 ``sys.modules`` 快照探针。""" + +import os +import signal +import sys + + +_OUTPUT_DIR = os.environ.get("MP_PERF_OUTPUT_DIR") +_snapshot_index = 0 + + +def _dump_modules(_signum, _frame) -> None: + """收到 SIGUSR1 时原子写出当前解释器已经导入的模块名称。""" + global _snapshot_index + if not _OUTPUT_DIR: + return + + _snapshot_index += 1 + os.makedirs(_OUTPUT_DIR, exist_ok=True) + final_path = os.path.join( + _OUTPUT_DIR, + f"modules-{os.getpid()}-{_snapshot_index}.txt", + ) + temporary_path = f"{final_path}.tmp" + with open(temporary_path, "w", encoding="utf-8") as output: + for module_name in sorted(sys.modules): + output.write(module_name) + output.write("\n") + output.flush() + os.fsync(output.fileno()) + os.replace(temporary_path, final_path) + + +if _OUTPUT_DIR and hasattr(signal, "SIGUSR1"): + signal.signal(signal.SIGUSR1, _dump_modules) diff --git a/scripts/perf/moviepilot_docker_ab.py b/scripts/perf/moviepilot_docker_ab.py new file mode 100644 index 000000000..797085cda --- /dev/null +++ b/scripts/perf/moviepilot_docker_ab.py @@ -0,0 +1,1677 @@ +"""对两个 MoviePilot Git commit 执行可复现的 Docker 内存 A/B 测量。""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import shutil +import statistics +import subprocess +import sys +import tarfile +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Optional + +try: + import docker +except ImportError: # pragma: no cover - 仅用于让 --help 在缺依赖环境仍可使用 + docker = None + + +SCRIPT_DIR = Path(__file__).resolve().parent +PROJECT_ROOT = SCRIPT_DIR.parents[1] +INSTRUMENT_DIR = SCRIPT_DIR / "instrument" +DEFAULT_SUBSTRATE = ( + "jxxghp/moviepilot-v3@" + "sha256:925de1fdf1bb0312144bc818bc8ebaa999a9a159c6d14f1b48b0ff05edb7f720" +) +DEFAULT_BROWSER_SOURCE_VOLUME = "mp-perf-v3-browser-seed" +CAMPAIGN_LABEL = "org.moviepilot.perf.campaign" +ROLE_LABEL = "org.moviepilot.perf.role" +SOURCE_LABEL = "org.moviepilot.perf.source-commit" +SUBSTRATE_LABEL = "org.moviepilot.perf.substrate" +CRITICAL_SUBSTRATE_PATHS = ( + "requirements.in", + "docker/Dockerfile", + "scripts/uv-pip-compat.sh", +) +SEED_COMPATIBILITY_PATHS = ("database/versions",) +MODULE_PREFIXES = ( + "lark_oapi", + "slack_bolt", + "slack_sdk", + "discord", + "plexapi", + "telebot", + "langgraph", + "langchain", + "app.agent", + "app.agent.orchestrator", + "app.agent.tools", + "app.modules", +) +BALANCED_RUN_ORDER = ( + ("before", 1), + ("after", 1), + ("after", 2), + ("before", 2), + ("before", 3), + ("after", 3), +) +LAB_API_TOKEN = "moviepilot-perf-lab-token-00000001" +LAB_PASSWORD = "MoviePilot-Perf-Lab-Only-00000001!" +LAB_SECRET_KEY = "moviepilot-perf-secret-key-lab-only-00000001" +LAB_RESOURCE_SECRET_KEY = "moviepilot-perf-resource-key-lab-only-00000001" + + +OVERLAY_DOCKERFILE = r""" +ARG MP_SUBSTRATE +FROM ${MP_SUBSTRATE} AS frozen + +RUN set -eux; \ + mkdir -p /frozen/plugins /frozen/site; \ + cp -a /app/app/plugins/. /frozen/plugins/; \ + rm -f /frozen/plugins/__init__.py; \ + rm -rf /frozen/plugins/__pycache__; \ + find /app/app/application/site -maxdepth 1 -type f \ + \( -name 'sites.*.so' -o -name 'user.sites.v3.bin' \) \ + -exec cp -a '{}' /frozen/site/ \; + +FROM ${MP_SUBSTRATE} +ARG MP_SOURCE_COMMIT +ARG MP_CAMPAIGN + +USER root +RUN rm -rf /app && mkdir -p /app/app/plugins /app/app/application/site +COPY source/ /app/ +COPY --from=frozen /frozen/plugins/ /app/app/plugins/ +COPY --from=frozen /frozen/site/ /app/app/application/site/ + +RUN cp -f /app/docker/nginx.common.conf /etc/nginx/common.conf \ + && cp -f /app/docker/nginx.template.conf /etc/nginx/nginx.template.conf \ + && cp -f /app/docker/update.sh /usr/local/bin/mp_update.sh \ + && cp -f /app/docker/entrypoint.sh /entrypoint.sh \ + && cp -f /app/docker/docker_http_proxy.conf /etc/nginx/docker_http_proxy.conf \ + && chmod +x /entrypoint.sh /usr/local/bin/mp_update.sh + +LABEL org.moviepilot.perf.campaign="${MP_CAMPAIGN}" \ + org.moviepilot.perf.source-commit="${MP_SOURCE_COMMIT}" \ + org.moviepilot.perf.substrate="${MP_SUBSTRATE}" +""".lstrip() + + +IMAGE_FINGERPRINT_SCRIPT = r""" +import hashlib +import importlib.metadata +import json +import platform +from pathlib import Path + + +def tree_fingerprint(root, selected_names=None): + root_path = Path(root) + digest = hashlib.sha256() + count = 0 + total = 0 + if not root_path.exists(): + return {"files": 0, "bytes": 0, "sha256": digest.hexdigest()} + for path in sorted(item for item in root_path.rglob("*") if item.is_file()): + if selected_names and not any(path.match(pattern) for pattern in selected_names): + continue + relative = path.relative_to(root_path).as_posix() + size = path.stat().st_size + count += 1 + total += size + digest.update(relative.encode("utf-8", errors="surrogateescape")) + digest.update(b"\0") + digest.update(str(size).encode("ascii")) + digest.update(b"\0") + with path.open("rb") as input_file: + for chunk in iter(lambda: input_file.read(1024 * 1024), b""): + digest.update(chunk) + return {"files": count, "bytes": total, "sha256": digest.hexdigest()} + + +packages = sorted( + f"{distribution.metadata.get('Name', '')}=={distribution.version}" + for distribution in importlib.metadata.distributions() +) +package_digest = hashlib.sha256("\n".join(packages).encode("utf-8")).hexdigest() +print(json.dumps({ + "python": platform.python_version(), + "packages": {"count": len(packages), "sha256": package_digest}, + "public": tree_fingerprint("/public"), + "plugins": tree_fingerprint("/app/app/plugins"), + "site_resources": tree_fingerprint( + "/app/app/application/site", + ("sites.*.so", "user.sites.v3.bin"), + ), +}, sort_keys=True)) +""" + + +VOLUME_FINGERPRINT_SCRIPT = r""" +import hashlib +import json +from pathlib import Path + +root = Path("/volume") +digest = hashlib.sha256() +count = 0 +total = 0 +if root.exists(): + for path in sorted(item for item in root.rglob("*") if item.is_file()): + relative = path.relative_to(root).as_posix() + size = path.stat().st_size + count += 1 + total += size + digest.update(relative.encode("utf-8", errors="surrogateescape")) + digest.update(b"\0") + digest.update(str(size).encode("ascii")) + digest.update(b"\n") +print(json.dumps({"files": count, "bytes": total, "layout_sha256": digest.hexdigest()})) +""" + + +class HarnessError(RuntimeError): + """表示测量合同无法继续成立。""" + + +def utc_now() -> str: + """返回适合写入 JSON 的 UTC 时间。""" + return datetime.now(timezone.utc).isoformat() + + +def atomic_write_json(path: Path, payload: Any) -> None: + """原子写入 JSON，避免长时间测量中断后留下半个结果文件。""" + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path = path.with_suffix(f"{path.suffix}.tmp") + temporary_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + temporary_path.replace(path) + + +def write_text(path: Path, content: str) -> None: + """创建父目录并写入 UTF-8 文本。""" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + + +def run_command( + command: list[str], + *, + cwd: Optional[Path] = None, + log_path: Optional[Path] = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + """不经过 shell 执行命令，并在需要时保存完整输出。""" + result = subprocess.run( + command, + cwd=cwd, + text=True, + capture_output=True, + check=False, + ) + if log_path: + write_text(log_path, result.stdout + result.stderr) + if check and result.returncode != 0: + detail = (result.stderr or result.stdout).strip().splitlines() + tail = "\n".join(detail[-20:]) + raise HarnessError( + f"命令失败（exit={result.returncode}）：{' '.join(command)}\n{tail}" + ) + return result + + +def require_docker_client(): + """连接 Docker Engine，并在依赖或 daemon 不可用时给出明确错误。""" + if docker is None: + raise HarnessError( + "缺少 docker Python SDK，请使用 MoviePilot 工作区运行环境执行" + ) + try: + client = docker.from_env() + client.ping() + return client + except Exception as default_error: + context = run_command( + ["docker", "context", "inspect", "--format", "{{.Endpoints.docker.Host}}"], + check=False, + ) + context_host = context.stdout.strip() + if not context_host: + raise HarnessError( + f"无法连接 Docker Engine：{default_error}" + ) from default_error + try: + client = docker.DockerClient(base_url=context_host) + client.ping() + return client + except Exception as context_error: + raise HarnessError( + f"无法通过当前 Docker context 连接 Engine：{context_error}" + ) from context_error + + +def normalize_campaign(value: str) -> str: + """限制 campaign 名称，确保 Docker 资源名和标签可安全复用。""" + normalized = value.strip().lower() + if not re.fullmatch(r"[a-z0-9][a-z0-9_.-]{0,39}", normalized): + raise argparse.ArgumentTypeError( + "campaign 只能包含小写字母、数字、点、下划线和短横线，最长 40 字符" + ) + return normalized + + +def parse_points(value: str) -> list[float]: + """解析以分钟为单位的升序采样点。""" + try: + points = sorted({float(item.strip()) for item in value.split(",")}) + except ValueError as error: + raise argparse.ArgumentTypeError("采样点必须是逗号分隔的分钟数") from error + if not points or any(point < 0 for point in points): + raise argparse.ArgumentTypeError("采样点不得为空或小于 0") + return points + + +def campaign_directory(args: argparse.Namespace) -> Path: + """返回当前 campaign 的本地结果目录。""" + return args.output_dir.expanduser().resolve() / args.campaign + + +def resource_prefix(args: argparse.Namespace) -> str: + """生成本工具拥有的 Docker 资源名前缀。""" + return f"mpperf-{args.campaign}" + + +def image_tag(args: argparse.Namespace, variant: str) -> str: + """返回 Before 或 After 派生镜像标签。""" + return f"moviepilot-perf:{args.campaign}-{variant}" + + +def labels(args: argparse.Namespace, role: str) -> dict[str, str]: + """给 Docker 资源附加可审计的精确所有权标签。""" + return {CAMPAIGN_LABEL: args.campaign, ROLE_LABEL: role} + + +def resolve_platform(client, requested: str) -> str: + """将 auto 解析为 Docker daemon 的原生 Linux 架构。""" + if requested != "auto": + return requested + architecture = str(client.info().get("Architecture") or "").lower() + aliases = { + "aarch64": "arm64", + "arm64": "arm64", + "x86_64": "amd64", + "amd64": "amd64", + } + if architecture not in aliases: + raise HarnessError(f"无法把 Docker 架构 {architecture!r} 映射为目标平台") + return f"linux/{aliases[architecture]}" + + +def git_output(repo: Path, *arguments: str) -> str: + """执行只读 Git 命令并返回去除尾部换行的输出。""" + result = run_command(["git", *arguments], cwd=repo) + return result.stdout.strip() + + +def resolve_git_ref(repo: Path, ref: str) -> str: + """把用户给出的 ref 固定为 commit SHA。""" + return git_output(repo, "rev-parse", "--verify", f"{ref}^{{commit}}") + + +def git_path_changed(repo: Path, before: str, after: str, paths: Iterable[str]) -> bool: + """判断两个 commit 在指定运行时 substrate 输入上是否有差异。""" + result = run_command( + ["git", "diff", "--quiet", f"{before}..{after}", "--", *paths], + cwd=repo, + check=False, + ) + if result.returncode not in (0, 1): + raise HarnessError(result.stderr.strip() or "git diff 执行失败") + return result.returncode == 1 + + +def assert_commit_order(repo: Path, before: str, after: str) -> None: + """要求 After 位于 Before 之后，避免比较两条无关历史。""" + result = run_command( + ["git", "merge-base", "--is-ancestor", before, after], + cwd=repo, + check=False, + ) + if result.returncode != 0: + raise HarnessError("After commit 必须是 Before commit 的后代") + + +def ensure_substrate(client, args: argparse.Namespace, pull: bool): + """取得冻结 substrate，只有显式要求时才访问镜像仓库。""" + if pull: + run_command( + ["docker", "pull", "--platform", args.platform, args.substrate], + log_path=campaign_directory(args) / "substrate-pull.log", + ) + try: + return client.images.get(args.substrate) + except Exception as error: + raise HarnessError( + f"本地不存在 substrate {args.substrate}；如需下载请添加 --pull-substrate" + ) from error + + +def image_fingerprint(client, image: str) -> dict[str, Any]: + """在不启动主程序和网络的临时容器中计算运行时资产指纹。""" + try: + output = client.containers.run( + image, + command=["-c", IMAGE_FINGERPRINT_SCRIPT], + entrypoint="python3", + network_disabled=True, + remove=True, + stdout=True, + stderr=True, + ) + return json.loads(output.decode("utf-8")) + except Exception as error: + raise HarnessError(f"无法计算镜像 {image} 的资产指纹：{error}") from error + + +def build_overlay_image( + args: argparse.Namespace, + variant: str, + commit: str, +) -> dict[str, Any]: + """从冻结 substrate 构建仅替换指定 Git commit 源码的派生镜像。""" + campaign_dir = campaign_directory(args) + with tempfile.TemporaryDirectory( + prefix=f"mpperf-{args.campaign}-{variant}-" + ) as temp: + context = Path(temp) + source_dir = context / "source" + source_dir.mkdir() + archive_path = context / "source.tar" + run_command( + ["git", "archive", "--format=tar", "--output", str(archive_path), commit], + cwd=args.repo, + ) + with tarfile.open(archive_path, "r") as archive: + try: + archive.extractall(source_dir, filter="data") + except TypeError: # pragma: no cover - Python 3.11 早期补丁版本兼容 + archive.extractall(source_dir) + archive_path.unlink() + write_text(context / "Dockerfile", OVERLAY_DOCKERFILE) + + tag = image_tag(args, variant) + build_log = campaign_dir / f"build-{variant}.log" + run_command( + [ + "docker", + "build", + "--pull=false", + "--platform", + args.platform, + "--build-arg", + f"MP_SUBSTRATE={args.substrate}", + "--build-arg", + f"MP_SOURCE_COMMIT={commit}", + "--build-arg", + f"MP_CAMPAIGN={args.campaign}", + "--tag", + tag, + "--file", + str(context / "Dockerfile"), + str(context), + ], + log_path=build_log, + ) + + client = require_docker_client() + image = client.images.get(tag) + image_labels = image.attrs.get("Config", {}).get("Labels") or {} + if image_labels.get(SOURCE_LABEL) != commit: + raise HarnessError(f"镜像 {tag} 的 source commit 标签校验失败") + return { + "variant": variant, + "tag": tag, + "image_id": image.id, + "source_commit": commit, + "fingerprint": image_fingerprint(client, tag), + } + + +def command_build(args: argparse.Namespace) -> dict[str, Any]: + """构建 Before/After 派生镜像并记录冻结输入。""" + client = require_docker_client() + args.platform = resolve_platform(client, args.platform) + args.repo = args.repo.expanduser().resolve() + if not (args.repo / ".git").exists(): + raise HarnessError(f"不是 MoviePilot Git 仓库：{args.repo}") + + campaign_dir = campaign_directory(args) + campaign_dir.mkdir(parents=True, exist_ok=True) + substrate = ensure_substrate(client, args, args.pull_substrate) + substrate_labels = substrate.attrs.get("Config", {}).get("Labels") or {} + substrate_revision = substrate_labels.get("org.opencontainers.image.revision") + if not substrate_revision: + raise HarnessError("substrate 缺少 org.opencontainers.image.revision 标签") + + before_commit = resolve_git_ref(args.repo, args.before_ref) + after_commit = resolve_git_ref(args.repo, args.after_ref) + resolve_git_ref(args.repo, substrate_revision) + assert_commit_order(args.repo, before_commit, after_commit) + + if git_path_changed( + args.repo, + substrate_revision, + before_commit, + CRITICAL_SUBSTRATE_PATHS, + ): + raise HarnessError( + "substrate revision 到 Before commit 的依赖或 Docker substrate 输入已变化，" + "禁止使用源码 overlay A/B" + ) + if git_path_changed( + args.repo, + before_commit, + after_commit, + CRITICAL_SUBSTRATE_PATHS, + ): + raise HarnessError( + "Before/After 的依赖或 Docker substrate 输入已变化，禁止使用源码 overlay A/B" + ) + if git_path_changed( + args.repo, + before_commit, + after_commit, + SEED_COMPATIBILITY_PATHS, + ): + raise HarnessError( + "Before/After 的数据库迁移集合不同，禁止复用同一个迁移后 SQLite seed" + ) + + substrate_fingerprint = image_fingerprint(client, args.substrate) + before_image = build_overlay_image(args, "before", before_commit) + after_image = build_overlay_image(args, "after", after_commit) + for key in ("python", "packages", "public", "plugins", "site_resources"): + if before_image["fingerprint"].get(key) != after_image["fingerprint"].get(key): + raise HarnessError(f"Before/After 冻结资产指纹不一致：{key}") + + manifest = { + "schema_version": 1, + "campaign": args.campaign, + "generated_at": utc_now(), + "platform": args.platform, + "before_ref": args.before_ref, + "after_ref": args.after_ref, + "before_commit": before_commit, + "after_commit": after_commit, + "critical_substrate_paths": list(CRITICAL_SUBSTRATE_PATHS), + "seed_compatibility_paths": list(SEED_COMPATIBILITY_PATHS), + "substrate": { + "reference": args.substrate, + "image_id": substrate.id, + "source_revision": substrate_revision, + "repo_digests": substrate.attrs.get("RepoDigests") or [], + "fingerprint": substrate_fingerprint, + }, + "images": {"before": before_image, "after": after_image}, + } + atomic_write_json(campaign_dir / "build.json", manifest) + print(f"Build manifest: {campaign_dir / 'build.json'}") + return manifest + + +def load_build_manifest(args: argparse.Namespace) -> dict[str, Any]: + """读取并校验当前 campaign 的构建结果。""" + path = campaign_directory(args) / "build.json" + if not path.exists(): + raise HarnessError("缺少 build.json，请先执行 build") + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("campaign") != args.campaign: + raise HarnessError("build.json 的 campaign 不匹配") + return payload + + +def fixed_environment(args: argparse.Namespace, instrument: bool) -> dict[str, str]: + """返回不依赖用户 app.env、外部服务或真实凭据的固定空载配置。""" + environment = { + "TZ": "Asia/Shanghai", + "PUID": "0", + "PGID": "0", + "UMASK": "000", + "PORT": "3001", + "NGINX_PORT": "3000", + "API_WORKERS": "1", + "DB_TYPE": "sqlite", + "CACHE_BACKEND_TYPE": "cachetools", + "AI_AGENT_ENABLE": "false", + "DEV": "false", + "DEBUG": "false", + "MOVIEPILOT_SAFE_MODE": "false", + "MOVIEPILOT_AUTO_UPDATE": "false", + "MOVIEPILOT_DOCKER_KEEPALIVE_ON_FAILURE": "false", + "MOVIEPILOT_BACKEND_READY_TIMEOUT": str(args.ready_timeout), + "AUTO_UPDATE_RESOURCE": "false", + "PLUGIN_MARKET": "", + "PLUGIN_AUTO_RELOAD": "false", + "PLUGIN_LOCAL_REPO_PATHS": "", + "PLUGIN_STATISTIC_SHARE": "false", + "SUBSCRIBE_STATISTIC_SHARE": "false", + "USAGE_STATISTIC_SHARE": "false", + "WORKFLOW_STATISTIC_SHARE": "false", + "MEDIA_RECOGNIZE_SHARE": "false", + "MP_SERVER_HOST": "", + "GITHUB_TOKEN": "", + "REPO_GITHUB_TOKEN": "", + "AUTH_SITE": "", + "SKILL_MARKET": "", + "BROWSER_EMULATION": "cloakbrowser", + "FANART_ENABLE": "false", + "API_TOKEN": LAB_API_TOKEN, + "SUPERUSER": "admin", + "SUPERUSER_PASSWORD": LAB_PASSWORD, + "SECRET_KEY": LAB_SECRET_KEY, + "RESOURCE_SECRET_KEY": LAB_RESOURCE_SECRET_KEY, + "LOG_LEVEL": "INFO", + } + if instrument: + environment.update( + { + "PYTHONPATH": "/opt/moviepilot-perf/instrument", + "MP_PERF_OUTPUT_DIR": "/opt/moviepilot-perf/out/modules", + } + ) + return environment + + +def get_volume(client, name: str): + """返回命名 volume；不存在时返回 None。""" + try: + return client.volumes.get(name) + except docker.errors.NotFound: + return None + + +def assert_owned(resource, args: argparse.Namespace, kind: str) -> None: + """删除或复用资源前验证 campaign 标签，避免误伤用户资源。""" + resource_labels = resource.attrs.get("Labels") or {} + if resource_labels.get(CAMPAIGN_LABEL) != args.campaign: + raise HarnessError(f"拒绝操作非本 campaign 的 {kind}：{resource.name}") + + +def prepare_volume( + client, + args: argparse.Namespace, + name: str, + role: str, + replace: bool, +): + """创建工具拥有的命名 volume，并按显式 replace 处理同名旧资源。""" + existing = get_volume(client, name) + if existing: + assert_owned(existing, args, "volume") + if not replace: + raise HarnessError(f"volume 已存在：{name}；如需重建请使用 --replace") + existing.remove(force=True) + return client.volumes.create(name=name, labels=labels(args, role)) + + +def clone_volume(client, image: str, source: str, target: str) -> None: + """在无网络、无主程序的短容器中复制命名 volume。""" + try: + client.containers.run( + image, + command=["-c", "cp -a /source/. /target/"], + entrypoint="/bin/sh", + network_disabled=True, + remove=True, + volumes={ + source: {"bind": "/source", "mode": "ro"}, + target: {"bind": "/target", "mode": "rw"}, + }, + ) + except Exception as error: + raise HarnessError(f"复制 volume {source} → {target} 失败：{error}") from error + + +def volume_fingerprint(client, image: str, volume_name: str) -> dict[str, Any]: + """记录 volume 文件数量、总大小和路径布局哈希，不读取配置内容。""" + try: + output = client.containers.run( + image, + command=["-c", VOLUME_FINGERPRINT_SCRIPT], + entrypoint="python3", + network_disabled=True, + remove=True, + volumes={volume_name: {"bind": "/volume", "mode": "ro"}}, + ) + return json.loads(output.decode("utf-8")) + except Exception as error: + raise HarnessError(f"无法计算 volume {volume_name} 指纹：{error}") from error + + +def ensure_internal_network(client, args: argparse.Namespace): + """创建或复用当前 campaign 的无外网 Docker network。""" + name = f"{resource_prefix(args)}-internal" + try: + network = client.networks.get(name) + assert_owned(network, args, "network") + if not network.attrs.get("Internal"): + raise HarnessError(f"network {name} 不是 internal network") + return network + except docker.errors.NotFound: + return client.networks.create( + name, + driver="bridge", + internal=True, + labels=labels(args, "measurement-network"), + ) + + +def remove_owned_container(client, args: argparse.Namespace, name: str) -> None: + """移除同 campaign 遗留容器，绝不按模糊前缀删除。""" + try: + container = client.containers.get(name) + except docker.errors.NotFound: + return + assert_owned(container, args, "container") + container.remove(force=True, v=False) + + +def create_app_container( + client, + args: argparse.Namespace, + *, + image: str, + name: str, + role: str, + config_volume: str, + browser_volume: str, + network_name: str, + output_dir: Optional[Path], +): + """按固定资源和挂载合同创建 MoviePilot 容器。""" + remove_owned_container(client, args, name) + volume_mounts: dict[str, dict[str, str]] = { + config_volume: {"bind": "/config", "mode": "rw"}, + browser_volume: {"bind": "/moviepilot/.cloakbrowser", "mode": "rw"}, + } + instrument = output_dir is not None + if output_dir: + output_dir.mkdir(parents=True, exist_ok=True) + volume_mounts[str(INSTRUMENT_DIR)] = { + "bind": "/opt/moviepilot-perf/instrument", + "mode": "ro", + } + volume_mounts[str(output_dir.resolve())] = { + "bind": "/opt/moviepilot-perf/out", + "mode": "rw", + } + return client.containers.create( + image, + name=name, + detach=True, + environment=fixed_environment(args, instrument=instrument), + volumes=volume_mounts, + network=network_name, + nano_cpus=int(args.cpus * 1_000_000_000), + mem_limit=args.memory, + memswap_limit=args.memory, + pids_limit=2048, + shm_size="256m", + labels=labels(args, role), + ) + + +def container_running(container) -> bool: + """刷新并返回容器是否仍在运行。""" + container.reload() + return container.status == "running" + + +def wait_for_exec_success( + container, + command: list[str], + timeout: float, + description: str, +) -> float: + """轮询容器内的无副作用探针并返回耗时秒数。""" + started = time.monotonic() + deadline = started + timeout + while time.monotonic() < deadline: + if not container_running(container): + raise HarnessError(f"等待{description}时容器提前退出") + result = container.exec_run(command) + if result.exit_code == 0: + return time.monotonic() - started + time.sleep(0.5) + raise HarnessError(f"等待{description}超时（{timeout:.0f}s）") + + +def wait_for_ready(container, started_at: float, timeout: float) -> float: + """等待公开 health endpoint 完成完整同步启动阶段。""" + deadline = started_at + timeout + command = [ + "curl", + "-fsS", + "--max-time", + "2", + "http://127.0.0.1:3001/api/v1/system/global?token=moviepilot", + ] + while time.monotonic() < deadline: + if not container_running(container): + raise HarnessError("等待 health ready 时容器提前退出") + result = container.exec_run(command) + if result.exit_code == 0: + return time.monotonic() - started_at + time.sleep(0.5) + raise HarnessError(f"等待 health ready 超时（{timeout:.0f}s）") + + +def assert_no_app_env(container) -> None: + """只检查 app.env 不存在；绝不读取文件内容。""" + result = container.exec_run(["/bin/sh", "-c", "test ! -e /config/app.env"]) + if result.exit_code != 0: + raise HarnessError("测量 config volume 出现 app.env，已停止以避免读取用户配置") + + +def capture_engine_stats(container) -> dict[str, Any]: + """从 Docker Engine API 读取原始 cgroup 和网络累计值。""" + try: + stats = container.stats(stream=False, one_shot=True) + except TypeError: # pragma: no cover - 旧 Docker SDK 兼容 + stats = container.stats(stream=False) + memory_stats = stats.get("memory_stats") or {} + memory_detail = memory_stats.get("stats") or {} + memory_current = int(memory_stats.get("usage") or 0) + inactive_file = int( + memory_detail.get("inactive_file") + or memory_detail.get("total_inactive_file") + or 0 + ) + networks = stats.get("networks") or {} + rx_bytes = sum(int(item.get("rx_bytes") or 0) for item in networks.values()) + tx_bytes = sum(int(item.get("tx_bytes") or 0) for item in networks.values()) + return { + "memory_current_bytes": memory_current, + "inactive_file_bytes": inactive_file, + "working_set_bytes": max(memory_current - inactive_file, 0), + "network_rx_bytes": rx_bytes, + "network_tx_bytes": tx_bytes, + } + + +def capture_processes(container) -> dict[str, Any]: + """读取采样开始时已有进程的 PSS/USS/RSS 与线程数。""" + result = container.exec_run( + ["/bin/bash", "/opt/moviepilot-perf/instrument/collect_proc.sh"] + ) + if result.exit_code != 0: + raise HarnessError( + "进程采样失败：" + result.output.decode("utf-8", errors="replace") + ) + lines = result.output.decode("utf-8", errors="replace").splitlines() + processes: list[dict[str, Any]] = [] + for line in lines[1:]: + fields = line.split("\t", 8) + if len(fields) != 9: + continue + pid, ppid, threads, rss, pss, uss, comm, executable, command_line = fields + processes.append( + { + "pid": int(pid), + "ppid": int(ppid), + "threads": int(threads), + "rss_kib": int(rss), + "pss_kib": int(pss), + "uss_kib": int(uss), + "comm": comm, + "executable": executable, + "cmdline": command_line, + } + ) + if not processes: + raise HarnessError("进程采样结果为空") + python_processes = [ + process + for process in processes + if "python" in Path(process["executable"]).name.lower() + ] + main_python = max(python_processes, key=lambda item: item["pss_kib"], default=None) + xvfb_processes = [ + process + for process in processes + if "xvfb" in f"{process['comm']} {process['cmdline']}".lower() + ] + return { + "items": sorted(processes, key=lambda item: item["pid"]), + "totals": { + "rss_kib": sum(item["rss_kib"] for item in processes), + "pss_kib": sum(item["pss_kib"] for item in processes), + "uss_kib": sum(item["uss_kib"] for item in processes), + "threads": sum(item["threads"] for item in processes), + }, + "main_python": main_python, + "xvfb": { + "count": len(xvfb_processes), + "pss_kib": sum(item["pss_kib"] for item in xvfb_processes), + }, + } + + +def capture_modules( + container, + output_dir: Path, + main_python: Optional[dict[str, Any]], +) -> dict[str, Any]: + """通过已注入的 SIGUSR1 handler 获取目标进程自身的 sys.modules。""" + if not main_python: + raise HarnessError("未找到主 Python 进程，无法采集 sys.modules") + modules_dir = output_dir / "modules" + modules_dir.mkdir(parents=True, exist_ok=True) + existing = set(modules_dir.glob("modules-*.txt")) + result = container.exec_run(["kill", "-USR1", str(main_python["pid"])]) + if result.exit_code != 0: + raise HarnessError("向主 Python 进程发送 SIGUSR1 失败") + deadline = time.monotonic() + 5 + snapshot_path: Optional[Path] = None + while time.monotonic() < deadline: + candidates = set(modules_dir.glob("modules-*.txt")) - existing + if candidates: + snapshot_path = max(candidates, key=lambda path: path.stat().st_mtime_ns) + break + time.sleep(0.05) + if snapshot_path is None: + raise HarnessError("主 Python 进程没有写出 sys.modules 快照") + content = snapshot_path.read_bytes() + names = [line for line in content.decode("utf-8").splitlines() if line] + prefix_counts = { + prefix: sum( + 1 for name in names if name == prefix or name.startswith(f"{prefix}.") + ) + for prefix in MODULE_PREFIXES + } + return { + "count": len(names), + "sha256": hashlib.sha256(content).hexdigest(), + "prefix_counts": prefix_counts, + "raw_file": snapshot_path.relative_to(output_dir).as_posix(), + } + + +def capture_measurement( + container, + output_dir: Path, + minute: float, + settled_at: float, +) -> dict[str, Any]: + """按低干扰顺序采集 Engine、进程和模块三层数据。""" + captured_at = time.monotonic() + engine = capture_engine_stats(container) + processes = capture_processes(container) + modules = capture_modules(container, output_dir, processes["main_python"]) + return { + "target_minute": minute, + "elapsed_seconds": captured_at - settled_at, + "captured_at": utc_now(), + "engine": engine, + "processes": processes, + "modules": modules, + } + + +def redact_logs(content: str) -> str: + """移除实验室凭据值和启动生成的本地实例标识。""" + redacted = content + for secret_value in ( + LAB_API_TOKEN, + LAB_PASSWORD, + LAB_SECRET_KEY, + LAB_RESOURCE_SECRET_KEY, + ): + redacted = redacted.replace(secret_value, "") + redacted = re.sub( + r"(当前用户UUID[:：]\s*)[^\s]+", + r"\1", + redacted, + ) + return redacted + + +def save_container_logs(container, path: Path) -> None: + """保存脱敏后的完整容器日志。""" + try: + raw = container.logs(stdout=True, stderr=True, timestamps=True) + write_text(path, redact_logs(raw.decode("utf-8", errors="replace"))) + except Exception as error: + write_text(path, f"无法读取容器日志：{error}\n") + + +def stop_and_remove_container(container, timeout: int) -> dict[str, Any]: + """优雅停止工具拥有的容器，并在超时后限定到该容器强制清理。""" + outcome: dict[str, Any] = {"requested_at": utc_now()} + try: + if container_running(container): + started = time.monotonic() + container.stop(timeout=timeout) + outcome["elapsed_seconds"] = time.monotonic() - started + container.reload() + outcome["exit_code"] = container.attrs.get("State", {}).get("ExitCode") + except Exception as error: + outcome["error"] = str(error) + finally: + try: + container.remove(force=True, v=False) + except docker.errors.NotFound: + pass + return outcome + + +def seed_volume_names(args: argparse.Namespace) -> tuple[str, str]: + """返回迁移后 SQLite 和预热浏览器两个 seed volume 名称。""" + prefix = resource_prefix(args) + return f"{prefix}-config-seed", f"{prefix}-browser-seed" + + +def command_seed(args: argparse.Namespace) -> dict[str, Any]: + """生成不含 app.env 的迁移后 SQLite 和预热浏览器 seed。""" + client = require_docker_client() + build = load_build_manifest(args) + before_image = build["images"]["before"]["tag"] + config_seed_name, browser_seed_name = seed_volume_names(args) + if not args.browser_source_volume and not args.allow_browser_download: + raise HarnessError( + "seed 需要 --browser-source-volume，或显式 --allow-browser-download 执行一次预热" + ) + if args.browser_source_volume: + if get_volume(client, args.browser_source_volume) is None: + raise HarnessError( + f"浏览器来源 volume 不存在：{args.browser_source_volume}" + ) + config_seed = prepare_volume( + client, args, config_seed_name, "config-seed", args.replace + ) + browser_seed = prepare_volume( + client, args, browser_seed_name, "browser-seed", args.replace + ) + if args.browser_source_volume: + clone_volume( + client, + before_image, + args.browser_source_volume, + browser_seed.name, + ) + before_browser = volume_fingerprint(client, before_image, browser_seed.name) + if before_browser["files"] == 0 and not args.allow_browser_download: + raise HarnessError("浏览器 seed 为空，且未允许一次性下载") + + network = ensure_internal_network(client, args) + network_name = "bridge" if args.allow_browser_download else network.name + container_name = f"{resource_prefix(args)}-seed" + seed_dir = campaign_directory(args) / "seed" + seed_dir.mkdir(parents=True, exist_ok=True) + container = create_app_container( + client, + args, + image=before_image, + name=container_name, + role="seed", + config_volume=config_seed.name, + browser_volume=browser_seed.name, + network_name=network_name, + output_dir=None, + ) + started_at = time.monotonic() + result: dict[str, Any] = { + "schema_version": 1, + "campaign": args.campaign, + "generated_at": utc_now(), + "image": before_image, + "browser_source": ( + "named-volume" if args.browser_source_volume else "one-time-download" + ), + "browser_before": before_browser, + } + success = False + try: + container.start() + ready_seconds = wait_for_ready(container, started_at, args.ready_timeout) + settled_wait = wait_for_exec_success( + container, + ["/bin/sh", "-c", "test -e /var/log/nginx/__moviepilot__"], + args.settle_timeout, + "后台初始化完成标志", + ) + assert_no_app_env(container) + result.update( + { + "http_ready_seconds": ready_seconds, + "settled_wait_seconds_after_ready": settled_wait, + "engine_at_settled": capture_engine_stats(container), + } + ) + success = True + except Exception as error: + result["error"] = str(error) + raise + finally: + save_container_logs(container, seed_dir / "container.log") + result["shutdown"] = stop_and_remove_container(container, args.stop_timeout) + if success: + result["browser_after"] = volume_fingerprint( + client, before_image, browser_seed.name + ) + atomic_write_json(campaign_directory(args) / "seed.json", result) + if not success: + for volume in (config_seed, browser_seed): + try: + volume.remove(force=True) + except Exception: + pass + print(f"Seed manifest: {campaign_directory(args) / 'seed.json'}") + return result + + +def require_seed_volumes(client, args: argparse.Namespace) -> tuple[str, str]: + """确认 seed 已完成且两个命名 volume 仍然存在。""" + seed_path = campaign_directory(args) / "seed.json" + if not seed_path.exists(): + raise HarnessError("缺少 seed.json，请先执行 seed") + seed = json.loads(seed_path.read_text(encoding="utf-8")) + if seed.get("error"): + raise HarnessError("seed.json 记录了失败，必须重新生成 seed") + names = seed_volume_names(args) + for name in names: + volume = get_volume(client, name) + if volume is None: + raise HarnessError(f"seed volume 不存在：{name}") + assert_owned(volume, args, "seed volume") + return names + + +def sample_volume_names( + args: argparse.Namespace, + variant: str, + index: int, +) -> tuple[str, str]: + """返回单个样本的隔离配置和浏览器卷名称。""" + prefix = f"{resource_prefix(args)}-{variant}-{index}" + return f"{prefix}-config", f"{prefix}-browser" + + +def sample_result_directory( + args: argparse.Namespace, + variant: str, + index: int, +) -> Path: + """返回单个样本的原始结果目录。""" + return campaign_directory(args) / "samples" / f"{variant}-{index}" + + +def command_sample(args: argparse.Namespace) -> dict[str, Any]: + """执行一个隔离样本并在约定时间点采集完整指标。""" + client = require_docker_client() + build = load_build_manifest(args) + config_seed, browser_seed = require_seed_volumes(client, args) + image = build["images"][args.variant]["tag"] + output_dir = sample_result_directory(args, args.variant, args.index) + if output_dir.exists(): + if not args.replace: + raise HarnessError( + f"样本结果已存在：{output_dir}；如需重测请使用 --replace" + ) + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) + + config_volume_name, browser_volume_name = sample_volume_names( + args, args.variant, args.index + ) + config_volume = prepare_volume( + client, args, config_volume_name, "sample-config", replace=True + ) + browser_volume = prepare_volume( + client, args, browser_volume_name, "sample-browser", replace=True + ) + clone_volume(client, image, config_seed, config_volume.name) + clone_volume(client, image, browser_seed, browser_volume.name) + browser_before = volume_fingerprint(client, image, browser_volume.name) + network = ensure_internal_network(client, args) + container_name = f"{resource_prefix(args)}-{args.variant}-{args.index}" + container = create_app_container( + client, + args, + image=image, + name=container_name, + role=f"sample-{args.variant}-{args.index}", + config_volume=config_volume.name, + browser_volume=browser_volume.name, + network_name=network.name, + output_dir=output_dir, + ) + result: dict[str, Any] = { + "schema_version": 1, + "campaign": args.campaign, + "variant": args.variant, + "sample_index": args.index, + "source_commit": build[f"{args.variant}_commit"], + "image": image, + "started_at": utc_now(), + "points_minutes": args.points, + "resources": { + "cpus": args.cpus, + "memory": args.memory, + "network": "internal", + "database": "sqlite-seed-clone", + "browser": "prewarmed-seed-clone", + }, + "browser_before": browser_before, + "measurements": [], + } + started_at = time.monotonic() + try: + container.start() + ready_seconds = wait_for_ready(container, started_at, args.ready_timeout) + ready_at = time.monotonic() + wait_for_exec_success( + container, + ["/bin/sh", "-c", "test -e /var/log/nginx/__moviepilot__"], + args.settle_timeout, + "后台初始化完成标志", + ) + settled_at = time.monotonic() + assert_no_app_env(container) + result["http_ready_seconds"] = ready_seconds + result["settled_seconds"] = settled_at - started_at + result["settled_wait_seconds_after_ready"] = settled_at - ready_at + + for point in args.points: + deadline = settled_at + point * 60 + remaining = deadline - time.monotonic() + if remaining > 0: + time.sleep(remaining) + if not container_running(container): + raise HarnessError(f"容器在 {point:g}m 采样前退出") + print(f"[{args.variant}-{args.index}] sampling {point:g}m") + result["measurements"].append( + capture_measurement(container, output_dir, point, settled_at) + ) + atomic_write_json(output_dir / "result.partial.json", result) + assert_no_app_env(container) + except Exception as error: + result["error"] = str(error) + raise + finally: + save_container_logs(container, output_dir / "container.log") + result["shutdown"] = stop_and_remove_container(container, args.stop_timeout) + try: + result["browser_after"] = volume_fingerprint( + client, image, browser_volume.name + ) + except Exception as error: + result["browser_after_error"] = str(error) + result["completed_at"] = utc_now() + atomic_write_json(output_dir / "result.json", result) + partial = output_dir / "result.partial.json" + partial.unlink(missing_ok=True) + for volume in (config_volume, browser_volume): + try: + volume.remove(force=True) + except Exception: + pass + update_aggregate_results(args) + return result + + +def load_sample_results(args: argparse.Namespace) -> list[dict[str, Any]]: + """读取当前 campaign 已完成或失败的所有样本结果。""" + sample_root = campaign_directory(args) / "samples" + results = [] + if not sample_root.exists(): + return results + for path in sorted(sample_root.glob("*/result.json")): + results.append(json.loads(path.read_text(encoding="utf-8"))) + return results + + +def median(values: Iterable[float]) -> Optional[float]: + """空序列返回 None，否则返回浮点中位数。""" + items = list(values) + return statistics.median(items) if items else None + + +def measurement_at(result: dict[str, Any], minute: float) -> Optional[dict[str, Any]]: + """按浮点容差返回目标采样点。""" + for measurement in result.get("measurements") or []: + if abs(float(measurement["target_minute"]) - minute) < 1e-9: + return measurement + return None + + +def format_mib(value: Optional[float]) -> str: + """把字节数格式化为 MiB。""" + if value is None: + return "—" + return f"{value / 1024 / 1024:.1f}" + + +def format_kib_as_mib(value: Optional[float]) -> str: + """把 KiB 数格式化为 MiB。""" + if value is None: + return "—" + return f"{value / 1024:.1f}" + + +def format_bytes_as_kib(value: Optional[float]) -> str: + """把字节数格式化为 KiB。""" + if value is None: + return "—" + return f"{value / 1024:.1f}" + + +def build_markdown_report( + build: dict[str, Any], + seed: Optional[dict[str, Any]], + samples: list[dict[str, Any]], +) -> str: + """生成不含本机路径和凭据的 Markdown 汇总。""" + points = sorted( + { + float(measurement["target_minute"]) + for sample in samples + for measurement in sample.get("measurements") or [] + } + ) + lines = [ + "# MoviePilot Docker A/B 测量结果", + "", + f"- Campaign：`{build['campaign']}`", + f"- Platform：`{build['platform']}`", + f"- Before：`{build['before_commit']}`", + f"- After：`{build['after_commit']}`", + f"- Substrate：`{build['substrate']['reference']}`", + "- working set：`memory.current - inactive_file`", + "- 配置：迁移后 SQLite seed、空插件配置、Agent 关闭、浏览器缓存预热、internal network", + "", + ] + if seed: + lines.extend( + [ + "## Seed", + "", + f"- 浏览器来源：`{seed.get('browser_source', 'unknown')}`", + f"- HTTP ready：{seed.get('http_ready_seconds', 0):.2f}s", + f"- 浏览器文件数：{seed.get('browser_after', {}).get('files', 0)}", + "", + ] + ) + + headers = ( + ["版本", "样本", "HTTP ready(s)"] + + [f"{point:g}m WS(MiB)" for point in points] + + [ + "末次 Python PSS(MiB)", + "末次 Python USS(MiB)", + "Python Threads", + "末次 Xvfb PSS(MiB)", + "RX(KiB)", + "TX(KiB)", + "sys.modules", + "状态", + ] + ) + lines.extend(["## 原始样本", "", "| " + " | ".join(headers) + " |"]) + lines.append("| " + " | ".join(["---"] * len(headers)) + " |") + variant_order = {"before": 0, "after": 1} + for sample in sorted( + samples, + key=lambda item: ( + variant_order.get(item["variant"], 99), + item["sample_index"], + ), + ): + row = [ + sample["variant"], + str(sample["sample_index"]), + f"{sample.get('http_ready_seconds', 0):.2f}" + if "http_ready_seconds" in sample + else "—", + ] + for point in points: + target = measurement_at(sample, point) + row.append( + format_mib(target["engine"]["working_set_bytes"] if target else None) + ) + final = ( + sample.get("measurements", [])[-1] if sample.get("measurements") else None + ) + python_pss = ( + final.get("processes", {}).get("main_python", {}).get("pss_kib") + if final and final.get("processes", {}).get("main_python") + else None + ) + python_uss = ( + final.get("processes", {}).get("main_python", {}).get("uss_kib") + if final and final.get("processes", {}).get("main_python") + else None + ) + python_threads = ( + final.get("processes", {}).get("main_python", {}).get("threads") + if final and final.get("processes", {}).get("main_python") + else None + ) + xvfb_pss = ( + final.get("processes", {}).get("xvfb", {}).get("pss_kib") if final else None + ) + row.extend( + [ + format_kib_as_mib(python_pss), + format_kib_as_mib(python_uss), + str(python_threads) if python_threads is not None else "—", + format_kib_as_mib(xvfb_pss), + format_bytes_as_kib( + final.get("engine", {}).get("network_rx_bytes") if final else None + ), + format_bytes_as_kib( + final.get("engine", {}).get("network_tx_bytes") if final else None + ), + str(final.get("modules", {}).get("count")) if final else "—", + "失败" if sample.get("error") else "完成", + ] + ) + lines.append("| " + " | ".join(row) + " |") + + lines.extend(["", "## 中位数对照", ""]) + if points: + lines.append("| 时间点 | Before(MiB) | After(MiB) | 净差(MiB) | 变化 |") + lines.append("| --- | ---: | ---: | ---: | ---: |") + for point in points: + before_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in samples + if sample["variant"] == "before" and measurement_at(sample, point) + ] + after_values = [ + measurement_at(sample, point)["engine"]["working_set_bytes"] + for sample in samples + if sample["variant"] == "after" and measurement_at(sample, point) + ] + before_median = median(before_values) + after_median = median(after_values) + if before_median is None or after_median is None: + lines.append(f"| {point:g}m | — | — | — | — |") + continue + delta = after_median - before_median + percent = delta / before_median * 100 if before_median else 0 + lines.append( + f"| {point:g}m | {format_mib(before_median)} | " + f"{format_mib(after_median)} | {delta / 1024 / 1024:.1f} | {percent:.1f}% |" + ) + ready_before = median( + sample["http_ready_seconds"] + for sample in samples + if sample["variant"] == "before" and "http_ready_seconds" in sample + ) + ready_after = median( + sample["http_ready_seconds"] + for sample in samples + if sample["variant"] == "after" and "http_ready_seconds" in sample + ) + lines.extend(["", "## 启动时间", ""]) + if ready_before is not None and ready_after is not None: + startup_change = ( + (ready_after - ready_before) / ready_before * 100 if ready_before else 0 + ) + lines.append( + f"Before 中位数 {ready_before:.2f}s，After 中位数 {ready_after:.2f}s，" + f"变化 {startup_change:.1f}%。" + ) + else: + lines.append("样本尚不完整。") + lines.extend( + [ + "", + "## 说明", + "", + "- 完整 Engine、进程、线程、网络和模块前缀数据见 `results.json`。", + "- 每个 `sys.modules` 完整名称清单位于对应样本的 `modules/` 目录。", + "- 报告不包含 app.env、真实 token、密码或本机挂载路径。", + "", + ] + ) + return "\n".join(lines) + + +def update_aggregate_results(args: argparse.Namespace) -> None: + """汇总当前 campaign 的 build、seed 和所有样本。""" + campaign_dir = campaign_directory(args) + build_path = campaign_dir / "build.json" + if not build_path.exists(): + return + build = json.loads(build_path.read_text(encoding="utf-8")) + seed_path = campaign_dir / "seed.json" + seed = ( + json.loads(seed_path.read_text(encoding="utf-8")) + if seed_path.exists() + else None + ) + samples = load_sample_results(args) + aggregate = { + "schema_version": 1, + "generated_at": utc_now(), + "build": build, + "seed": seed, + "samples": samples, + } + atomic_write_json(campaign_dir / "results.json", aggregate) + write_text( + campaign_dir / "report.md", + build_markdown_report(build, seed, samples), + ) + + +def cleanup_runtime_resources(client, args: argparse.Namespace) -> dict[str, list[str]]: + """仅按 campaign 标签清理容器、volume 和 internal network。""" + removed: dict[str, list[str]] = {"containers": [], "volumes": [], "networks": []} + label_filter = f"{CAMPAIGN_LABEL}={args.campaign}" + for container in client.containers.list(all=True, filters={"label": label_filter}): + name = container.name + container.remove(force=True, v=False) + removed["containers"].append(name) + for volume in client.volumes.list(filters={"label": label_filter}): + name = volume.name + volume.remove(force=True) + removed["volumes"].append(name) + for network in client.networks.list(filters={"label": label_filter}): + name = network.name + try: + network.remove() + removed["networks"].append(name) + except Exception as error: + raise HarnessError(f"清理 network {name} 失败：{error}") from error + return removed + + +def command_cleanup(args: argparse.Namespace) -> dict[str, Any]: + """清理当前 campaign 的 Docker 资源，保留本地结果文件。""" + client = require_docker_client() + removed: dict[str, Any] = cleanup_runtime_resources(client, args) + removed["images"] = [] + if args.images: + build_path = campaign_directory(args) / "build.json" + if build_path.exists(): + build = json.loads(build_path.read_text(encoding="utf-8")) + for variant in ("before", "after"): + tag = build.get("images", {}).get(variant, {}).get("tag") + if not tag: + continue + try: + image = client.images.get(tag) + except docker.errors.ImageNotFound: + continue + image_labels = image.attrs.get("Config", {}).get("Labels") or {} + if image_labels.get(CAMPAIGN_LABEL) != args.campaign: + raise HarnessError(f"拒绝删除非本 campaign 镜像：{tag}") + client.images.remove(tag, force=False, noprune=True) + removed["images"].append(tag) + atomic_write_json(campaign_directory(args) / "cleanup.json", removed) + print(json.dumps(removed, ensure_ascii=False, indent=2)) + return removed + + +def command_run(args: argparse.Namespace) -> None: + """构建、生成 seed，并按平衡顺序串行执行三组 Before/After。""" + command_build(args) + command_seed(args) + completed = False + try: + for variant, index in BALANCED_RUN_ORDER: + sample_args = argparse.Namespace(**vars(args)) + sample_args.variant = variant + sample_args.index = index + sample_args.replace = args.replace + command_sample(sample_args) + completed = True + finally: + update_aggregate_results(args) + if not args.keep_resources: + client = require_docker_client() + cleanup_runtime_resources(client, args) + if completed: + print(f"Results: {campaign_directory(args) / 'results.json'}") + print(f"Report: {campaign_directory(args) / 'report.md'}") + + +def add_common_build_arguments(parser: argparse.ArgumentParser) -> None: + """添加 build 与 run 共用的 ref 和 substrate 参数。""" + parser.add_argument("--before-ref", default="upstream/v3", help="Before Git ref") + parser.add_argument("--after-ref", default="HEAD", help="After Git ref") + parser.add_argument( + "--pull-substrate", + action="store_true", + help="显式从镜像仓库拉取冻结 substrate", + ) + + +def add_seed_arguments(parser: argparse.ArgumentParser) -> None: + """添加 seed 与 run 共用的浏览器缓存参数。""" + browser_group = parser.add_mutually_exclusive_group(required=False) + browser_group.add_argument( + "--browser-source-volume", + help=( + "从已有命名 volume 复制 CloakBrowser 缓存，不访问外网；" + f"默认 {DEFAULT_BROWSER_SOURCE_VOLUME}" + ), + ) + browser_group.add_argument( + "--allow-browser-download", + action="store_true", + help="允许 seed 阶段一次性联网下载 CloakBrowser；样本仍使用 internal network", + ) + parser.add_argument( + "--replace", + action="store_true", + help="替换同 campaign 的 seed 或样本结果", + ) + + +def build_parser() -> argparse.ArgumentParser: + """构建命令行解析器。""" + parser = argparse.ArgumentParser( + description="MoviePilot V3 reproducible Docker A/B measurement harness" + ) + parser.add_argument("--campaign", type=normalize_campaign, required=True) + parser.add_argument("--repo", type=Path, default=PROJECT_ROOT) + parser.add_argument( + "--output-dir", + type=Path, + default=Path(tempfile.gettempdir()) / "moviepilot-perf-results", + help="结果根目录，默认位于系统临时目录", + ) + parser.add_argument("--substrate", default=DEFAULT_SUBSTRATE) + parser.add_argument( + "--platform", + default="auto", + help="Docker 平台，默认使用 daemon 原生架构；正式数据禁止 QEMU 跨架构", + ) + parser.add_argument("--cpus", type=float, default=4.0) + parser.add_argument("--memory", default="2g") + parser.add_argument("--ready-timeout", type=int, default=300) + parser.add_argument("--settle-timeout", type=int, default=300) + parser.add_argument("--stop-timeout", type=int, default=120) + + subparsers = parser.add_subparsers(dest="command", required=True) + build = subparsers.add_parser( + "build", help="构建冻结 substrate 的 Before/After overlay 镜像" + ) + add_common_build_arguments(build) + + seed = subparsers.add_parser( + "seed", help="创建迁移后 SQLite 和预热浏览器 seed volume" + ) + add_seed_arguments(seed) + + sample = subparsers.add_parser("sample", help="执行一个 Before 或 After 样本") + sample.add_argument("--variant", choices=("before", "after"), required=True) + sample.add_argument("--index", type=int, choices=(1, 2, 3), required=True) + sample.add_argument( + "--points", type=parse_points, default=parse_points("1,5,10,30") + ) + sample.add_argument("--replace", action="store_true") + + run = subparsers.add_parser("run", help="完整执行 build、seed 和三组平衡 A/B") + add_common_build_arguments(run) + add_seed_arguments(run) + run.add_argument("--points", type=parse_points, default=parse_points("1,5,10,30")) + run.add_argument( + "--keep-resources", + action="store_true", + help="完成或失败后保留 seed volume 和 internal network 供诊断", + ) + + cleanup = subparsers.add_parser("cleanup", help="清理当前 campaign 的 Docker 资源") + cleanup.add_argument( + "--images", action="store_true", help="同时移除 Before/After 派生镜像" + ) + return parser + + +def main(argv: Optional[list[str]] = None) -> int: + """解析参数并执行选定阶段。""" + parser = build_parser() + args = parser.parse_args(argv) + if args.command in {"seed", "run"}: + if not args.browser_source_volume and not args.allow_browser_download: + args.browser_source_volume = DEFAULT_BROWSER_SOURCE_VOLUME + if args.cpus <= 0: + parser.error("--cpus 必须大于 0") + if args.ready_timeout <= 0 or args.settle_timeout <= 0 or args.stop_timeout <= 0: + parser.error("timeout 必须大于 0") + try: + if args.command == "build": + command_build(args) + elif args.command == "seed": + command_seed(args) + elif args.command == "sample": + command_sample(args) + elif args.command == "run": + command_run(args) + elif args.command == "cleanup": + command_cleanup(args) + else: # pragma: no cover - argparse 已保证不可达 + parser.error(f"未知命令：{args.command}") + except HarnessError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_capability_registry.py b/tests/test_capability_registry.py new file mode 100644 index 000000000..82adc2586 --- /dev/null +++ b/tests/test_capability_registry.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from app.runtime.capabilities.errors import CapabilityManifestError +from app.runtime.capabilities.model import ( + ActivationPolicy, + SelectorSchema, +) +from app.runtime.capabilities.registry import CapabilityRegistry + + +_BASE_MANIFEST = """ +schema_version = 1 +id = "sample.capability" +kind = "sample" +entrypoint = "sample_implementation:SampleCapability" +depends_on = [] + +[metadata] +name = "Sample capability" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["sample.config"] + +[activation.selector] +kind = "configured" +key = "sample.config" +enabled = true +""" + + +def _write_manifest(root: Path, content: str = _BASE_MANIFEST, name: str = "sample") -> Path: + manifest_dir = root / name + manifest_dir.mkdir(parents=True) + manifest_path = manifest_dir / "capability.toml" + manifest_path.write_text(content.strip() + "\n", encoding="utf-8") + return manifest_path + + +def _discover(root: Path) -> CapabilityRegistry: + return CapabilityRegistry.discover( + roots=[root], + kinds={"sample"}, + selector_schemas={ + "configured": SelectorSchema( + required_fields=frozenset({"key", "enabled"}), + ) + }, + ) + + +def test_discovery_reads_toml_without_importing_entrypoint(tmp_path: Path) -> None: + """能力发现只能读取声明，不能执行 entrypoint 对应的 Python 模块。""" + _write_manifest(tmp_path) + (tmp_path / "sample_implementation.py").write_text( + "raise AssertionError('entrypoint must not be imported during discovery')\n", + encoding="utf-8", + ) + sys.modules.pop("sample_implementation", None) + + registry = _discover(tmp_path) + + spec = registry.get_spec("sample.capability") + assert spec is not None + assert spec.activation is ActivationPolicy.WHEN_CONFIGURED + assert spec.selector is not None + assert spec.selector.kind == "configured" + assert spec.selector.config == {"key": "sample.config", "enabled": True} + assert spec.watch == ("sample.config",) + assert spec.depends_on == () + assert "sample_implementation" not in sys.modules + + +def test_discovered_specs_are_recursively_immutable(tmp_path: Path) -> None: + """Registry 暴露的声明及嵌套 metadata/selector 都不能被调用方改写。""" + _write_manifest(tmp_path) + spec = _discover(tmp_path).require_spec("sample.capability") + + with pytest.raises(TypeError): + spec.metadata["name"] = "changed" + with pytest.raises(TypeError): + spec.selector.config["key"] = "changed" # type: ignore[union-attr] + with pytest.raises(AttributeError): + spec.watch.append("changed") # type: ignore[attr-defined] + + +@pytest.mark.parametrize( + ("replacement", "match"), + [ + ("schema_version = 1", "缺少字段"), + (_BASE_MANIFEST.replace("schema_version = 1", "schema_version = 2"), "schema_version"), + (_BASE_MANIFEST.replace("schema_version = 1", "schema_version = 1.0"), "schema_version"), + (_BASE_MANIFEST.replace('id = "sample.capability"', 'id = "bad id"'), "id"), + (_BASE_MANIFEST.replace('kind = "sample"', 'kind = "unknown"'), "kind"), + ( + _BASE_MANIFEST.replace( + 'entrypoint = "sample_implementation:SampleCapability"', + 'entrypoint = "sample_implementation.SampleCapability"', + ), + "entrypoint", + ), + (_BASE_MANIFEST.replace('kind = "configured"', 'kind = "unknown"'), "selector"), + (_BASE_MANIFEST.replace('enabled = true', 'extra = true'), "selector"), + (_BASE_MANIFEST.replace("depends_on = []", 'depends_on = ["other"]'), "depends_on"), + (_BASE_MANIFEST.replace("watch =", "unknown_field = true\nwatch ="), "activation"), + ], +) +def test_registry_fails_closed_for_invalid_manifest( + tmp_path: Path, + replacement: str, + match: str, +) -> None: + """未知或不完整声明必须阻止 Registry 构建，不能静默丢失能力。""" + _write_manifest(tmp_path, replacement) + + with pytest.raises(CapabilityManifestError, match=match): + _discover(tmp_path) + + +def test_selector_presence_must_match_activation_policy(tmp_path: Path) -> None: + """只有 when_configured 声明可以且必须携带配置 selector。""" + bootstrap = _BASE_MANIFEST.replace( + 'policy = "when_configured"', 'policy = "bootstrap"' + ) + _write_manifest(tmp_path, bootstrap) + + with pytest.raises(CapabilityManifestError, match="selector"): + _discover(tmp_path) + + +def test_registry_rejects_duplicate_ids_across_roots(tmp_path: Path) -> None: + """多个声明根出现相同 capability ID 时必须 fail closed。""" + first_root = tmp_path / "first" + second_root = tmp_path / "second" + _write_manifest(first_root) + _write_manifest(second_root) + + with pytest.raises(CapabilityManifestError, match="重复"): + CapabilityRegistry.discover( + roots=[first_root, second_root], + kinds={"sample"}, + selector_schemas={ + "configured": SelectorSchema( + required_fields=frozenset({"key", "enabled"}), + ) + }, + ) + + +def test_registry_rejects_root_without_manifest(tmp_path: Path) -> None: + """注册了声明根却没有任何 manifest 时应直接失败。""" + with pytest.raises(CapabilityManifestError, match="capability.toml"): + _discover(tmp_path) + + +def test_current_host_module_manifests_follow_the_strict_nested_schema() -> None: + """仓内 Host Module 声明必须全部通过同一套嵌套 schema。""" + modules_root = Path(__file__).parents[1] / "app" / "modules" + imported_before = set(sys.modules) + registry = CapabilityRegistry.discover( + roots=[modules_root], + kinds={"host_module"}, + selector_schemas={ + "system_config_item": SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + ), + "setting_truthy": SelectorSchema( + required_fields=frozenset({"key"}), + ), + }, + ) + + specs = registry.list_specs() + declared_directories = {spec.source.parent for spec in specs} + module_directories = { + path + for path in modules_root.iterdir() + if path.is_dir() and (path / "__init__.py").is_file() + } + entrypoint_modules = {spec.entrypoint.split(":", maxsplit=1)[0] for spec in specs} + + assert declared_directories == module_directories + assert all(spec.kind == "host_module" for spec in specs) + assert not ((set(sys.modules) - imported_before) & entrypoint_modules) diff --git a/tests/test_capability_runtime.py b/tests/test_capability_runtime.py new file mode 100644 index 000000000..3d0641141 --- /dev/null +++ b/tests/test_capability_runtime.py @@ -0,0 +1,834 @@ +from __future__ import annotations + +import asyncio +import sys +import threading +import types +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional +from unittest.mock import patch + +import pytest + +from app.runtime.capabilities.errors import ( + CapabilityAdapterModeError, + CapabilityOperationError, + CapabilityRuntimeClosedError, +) +from app.runtime.capabilities.model import ( + AdapterExecutionMode, + CapabilityLifecycleState, + CapabilityMaterializationState, +) +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.capabilities.runtime import CapabilityRuntime + + +_MANIFEST = """ +schema_version = 1 +id = "sample.capability" +kind = "sample" +entrypoint = "sample_implementation:SampleCapability" +depends_on = [] + +[metadata] +name = "Sample capability" + +[activation] +policy = "on_first_use" +watch = [] +""" + + +def _registry(tmp_path: Path) -> CapabilityRegistry: + manifest_dir = tmp_path / "sample" + manifest_dir.mkdir(parents=True) + (manifest_dir / "capability.toml").write_text( + _MANIFEST.strip() + "\n", + encoding="utf-8", + ) + return CapabilityRegistry.discover( + roots=[tmp_path], + kinds={"sample"}, + selector_schemas={}, + ) + + +@dataclass +class _Candidate: + generation: int + started: bool = False + stopped: bool = False + + +class _SyncAdapter: + execution_mode = AdapterExecutionMode.SYNC + + def __init__(self) -> None: + self.materialize_calls = 0 + self.create_calls = 0 + self.start_calls = 0 + self.stop_calls = 0 + self.stop_instances = [] + self.cleanup_calls = 0 + self.fail_materialize = False + self.fail_start = False + self.fail_stop = False + self.start_entered: Optional[threading.Event] = None + self.start_release: Optional[threading.Event] = None + self.stop_entered: Optional[threading.Event] = None + self.stop_release: Optional[threading.Event] = None + + def materialize(self, spec) -> object: + self.materialize_calls += 1 + if self.fail_materialize: + raise RuntimeError("materialize failed") + return object() + + def create(self, spec, implementation: object, generation: int, previous: Any = None) -> _Candidate: + self.create_calls += 1 + return _Candidate(generation=generation) + + def start(self, spec, candidate: _Candidate, generation: int) -> None: + self.start_calls += 1 + if self.start_entered: + self.start_entered.set() + if self.start_release: + assert self.start_release.wait(timeout=5) + candidate.started = True + if self.fail_start: + raise RuntimeError("start failed") + + def stop(self, spec, instance: _Candidate, generation: int) -> None: + self.stop_calls += 1 + self.stop_instances.append(instance) + if self.stop_entered: + self.stop_entered.set() + if self.stop_release: + assert self.stop_release.wait(timeout=5) + instance.stopped = True + if self.fail_stop: + raise RuntimeError("stop failed") + + def cleanup(self, spec, candidate: _Candidate, generation: int, error: BaseException) -> None: + self.cleanup_calls += 1 + candidate.stopped = True + + +class _AsyncAdapter: + execution_mode = AdapterExecutionMode.ASYNC + + def __init__(self) -> None: + self.materialize_calls = 0 + self.create_calls = 0 + self.start_calls = 0 + self.stop_calls = 0 + self.stop_instances = [] + self.cleanup_calls = 0 + self.fail_materialize = False + self.fail_start = False + self.fail_stop = False + self.start_entered = asyncio.Event() + self.start_release = asyncio.Event() + self.stop_entered: Optional[asyncio.Event] = None + self.stop_release: Optional[asyncio.Event] = None + + async def materialize(self, spec) -> object: + self.materialize_calls += 1 + await asyncio.sleep(0) + if self.fail_materialize: + raise RuntimeError("async materialize failed") + return object() + + async def create(self, spec, implementation: object, generation: int, previous: Any = None) -> _Candidate: + self.create_calls += 1 + await asyncio.sleep(0) + return _Candidate(generation=generation) + + async def start(self, spec, candidate: _Candidate, generation: int) -> None: + self.start_calls += 1 + self.start_entered.set() + await self.start_release.wait() + candidate.started = True + if self.fail_start: + raise RuntimeError("async start failed") + + async def stop(self, spec, instance: _Candidate, generation: int) -> None: + self.stop_calls += 1 + self.stop_instances.append(instance) + if self.stop_entered: + self.stop_entered.set() + if self.stop_release: + await self.stop_release.wait() + await asyncio.sleep(0) + instance.stopped = True + if self.fail_stop: + raise RuntimeError("async stop failed") + + async def cleanup(self, spec, candidate: _Candidate, generation: int, error: BaseException) -> None: + self.cleanup_calls += 1 + await asyncio.sleep(0) + candidate.stopped = True + + +def test_materialize_and_start_have_independent_state_axes(tmp_path: Path) -> None: + """兼容查询只物化代码，资源必须等显式 activate 成功后才对外可见。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + implementation = runtime.materialize("sample.capability", reason="compat_lookup") + materialized = runtime.snapshot("sample.capability") + + assert implementation is not None + assert materialized.materialization is CapabilityMaterializationState.RESOLVED + assert materialized.lifecycle is CapabilityLifecycleState.DISCOVERED + assert materialized.visible is False + assert adapter.start_calls == 0 + + instance = runtime.activate("sample.capability", reason="first_use") + running = runtime.snapshot("sample.capability") + + assert runtime.get_running("sample.capability") is instance + assert running.lifecycle is CapabilityLifecycleState.RUNNING + assert running.visible is True + assert running.generation == 2 + + +def test_materialize_failure_does_not_claim_resource_lifecycle_failure(tmp_path: Path) -> None: + """仅解析代码失败时，资源轴尚未启动，必须保持 DISCOVERED。""" + adapter = _SyncAdapter() + adapter.fail_materialize = True + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityOperationError, match="materialize failed"): + runtime.materialize("sample.capability", reason="compat_lookup") + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.materialization is CapabilityMaterializationState.FAILED + assert snapshot.lifecycle is CapabilityLifecycleState.DISCOVERED + assert snapshot.visible is False + + +def test_state_read_calibrates_consumer_import_without_importing_new_module(tmp_path: Path) -> None: + """显式旧导入存在时应复用 sys.modules 中的 canonical symbol。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + module = types.ModuleType("sample_implementation") + canonical = type("SampleCapability", (), {}) + module.SampleCapability = canonical + + with patch.dict(sys.modules, {"sample_implementation": module}): + snapshot = runtime.snapshot("sample.capability") + implementation = runtime.materialize( + "sample.capability", + reason="compat_lookup", + ) + + assert snapshot.materialization is CapabilityMaterializationState.RESOLVED + assert snapshot.lifecycle is CapabilityLifecycleState.DISCOVERED + assert snapshot.generation == 0 + assert implementation is canonical + assert adapter.materialize_calls == 0 + + +def test_state_read_does_not_invoke_module_level_lazy_export(tmp_path: Path) -> None: + """sys.modules 校准只能读模块字典，不能触发模块级 __getattr__。""" + runtime = CapabilityRuntime( + _registry(tmp_path), + adapters={"sample": _SyncAdapter()}, + ) + module = types.ModuleType("sample_implementation") + lazy_reads = [] + + def resolve(name: str) -> object: + lazy_reads.append(name) + raise AssertionError("state read must not resolve lazy exports") + + module.__getattr__ = resolve + with patch.dict(sys.modules, {"sample_implementation": module}): + snapshot = runtime.snapshot("sample.capability") + + assert snapshot.materialization is CapabilityMaterializationState.UNRESOLVED + assert lazy_reads == [] + + +def test_sync_activate_is_single_flight_and_publishes_only_after_start(tmp_path: Path) -> None: + """并发首启只能创建一个候选实例，start 返回前普通查询不可见。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + results = [] + errors = [] + + def activate() -> None: + try: + results.append(runtime.activate("sample.capability", reason="concurrent")) + except BaseException as error: # pragma: no cover - diagnostic collection + errors.append(error) + + first = threading.Thread(target=activate) + second = threading.Thread(target=activate) + first.start() + assert adapter.start_entered.wait(timeout=5) + second.start() + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STARTING + adapter.start_release.set() + first.join(timeout=5) + second.join(timeout=5) + + assert not errors + assert len(results) == 2 + assert results[0] is results[1] + assert adapter.materialize_calls == 1 + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + assert runtime.snapshot("sample.capability").generation == 1 + + +def test_failed_start_cleans_candidate_and_requires_explicit_retry(tmp_path: Path) -> None: + """半初始化候选必须清理；FAILED 不得被普通 activate 隐式重试。""" + adapter = _SyncAdapter() + adapter.fail_start = True + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityOperationError, match="start failed"): + runtime.activate("sample.capability", reason="first_attempt") + + failed = runtime.snapshot("sample.capability") + assert failed.materialization is CapabilityMaterializationState.RESOLVED + assert failed.lifecycle is CapabilityLifecycleState.FAILED + assert failed.visible is False + assert adapter.cleanup_calls == 1 + + with pytest.raises(CapabilityOperationError, match="显式 retry"): + runtime.activate("sample.capability", reason="implicit_retry") + assert adapter.start_calls == 1 + + adapter.fail_start = False + instance = runtime.activate("sample.capability", reason="explicit_retry", retry=True) + + assert instance.started is True + assert runtime.snapshot("sample.capability").generation == 2 + assert adapter.start_calls == 2 + + +def test_adapter_must_return_candidate_before_start(tmp_path: Path) -> None: + """create 没有候选对象时不得进入 start 或伪造 RUNNING 可见性。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with patch.object(adapter, "create", return_value=None), pytest.raises( + CapabilityOperationError, + match="candidate", + ): + runtime.activate("sample.capability", reason="invalid_candidate") + + assert adapter.start_calls == 0 + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert runtime.get_running("sample.capability") is None + + +def test_stop_withdraws_visibility_before_adapter_callback(tmp_path: Path) -> None: + """释放外部资源可能阻塞，但运行实例必须在 stop 回调前撤销发布。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + instance = runtime.activate("sample.capability", reason="start") + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + + stopper = threading.Thread( + target=lambda: runtime.stop("sample.capability", reason="configuration_removed") + ) + stopper.start() + assert adapter.stop_entered.wait(timeout=5) + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + adapter.stop_release.set() + stopper.join(timeout=5) + + assert instance.stopped is True + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + + +def test_stop_failure_retains_ownership_until_same_instance_stops(tmp_path: Path) -> None: + """stop 失败后的隐藏资源必须保留所有权，禁止用 retry 绕过清理。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + instance = runtime.activate("sample.capability", reason="start") + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.stop("sample.capability", reason="configuration_removed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.activate("sample.capability", reason="unsafe_retry", retry=True) + assert adapter.create_calls == 1 + + adapter.fail_stop = False + runtime.stop("sample.capability", reason="stop_retry") + + assert adapter.stop_instances == [instance, instance, instance] + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + replacement = runtime.activate("sample.capability", reason="after_release") + assert replacement is not instance + assert adapter.create_calls == 2 + + +def test_reload_withdraws_old_instance_and_publishes_one_new_generation(tmp_path: Path) -> None: + """同步 reload 在 stop/start 回调期间不暴露旧实例或半初始化候选。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + old_instance = runtime.activate("sample.capability", reason="initial") + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + results = [] + + reloader = threading.Thread( + target=lambda: results.append(runtime.reload("sample.capability", reason="config_changed")) + ) + reloader.start() + assert adapter.stop_entered.wait(timeout=5) + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.RELOADING + adapter.stop_release.set() + reloader.join(timeout=5) + + assert len(results) == 1 + assert results[0] is not old_instance + assert runtime.get_running("sample.capability") is results[0] + assert runtime.snapshot("sample.capability").generation == 2 + + +def test_failed_reload_cleans_candidate_and_keeps_instance_invisible(tmp_path: Path) -> None: + """reload 新 generation 启动失败时不得恢复旧实例或发布候选。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + runtime.activate("sample.capability", reason="initial") + adapter.fail_start = True + + with pytest.raises(CapabilityOperationError, match="start failed"): + runtime.reload("sample.capability", reason="config_changed") + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.lifecycle is CapabilityLifecycleState.FAILED + assert snapshot.visible is False + assert adapter.cleanup_calls == 1 + + +def test_reload_stop_failure_does_not_create_or_reuse_live_previous(tmp_path: Path) -> None: + """reload 未释放旧资源时必须失败关闭，不能创建或复用同一活对象。""" + adapter = _SyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + old_instance = runtime.activate("sample.capability", reason="initial") + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="stop failed"): + runtime.reload("sample.capability", reason="config_changed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert adapter.stop_instances == [old_instance] + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + with pytest.raises(CapabilityOperationError, match="重试 stop"): + runtime.activate("sample.capability", reason="implicit_retry") + + adapter.fail_stop = False + adapter.stop_entered = threading.Event() + adapter.stop_release = threading.Event() + recovered = [] + recovery = threading.Thread( + target=lambda: recovered.append( + runtime.activate( + "sample.capability", + reason="recover_after_reload", + retry=True, + ) + ) + ) + recovery.start() + assert adapter.stop_entered.wait(timeout=5) + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + assert runtime.get_running("sample.capability") is None + assert adapter.create_calls == 1 + adapter.stop_release.set() + recovery.join(timeout=5) + + assert len(recovered) == 1 + replacement = recovered[0] + assert adapter.stop_instances == [old_instance, old_instance] + assert replacement is not old_instance + assert adapter.create_calls == 2 + + +def test_shutdown_prevents_inflight_start_from_resurrecting_instance(tmp_path: Path) -> None: + """shutdown 与首启竞争时，候选只能清理，不能在关闭开始后重新发布。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + activate_errors = [] + + def activate() -> None: + try: + runtime.activate("sample.capability", reason="racing_start") + except BaseException as error: + activate_errors.append(error) + + starter = threading.Thread(target=activate) + starter.start() + assert adapter.start_entered.wait(timeout=5) + closer = threading.Thread(target=lambda: runtime.shutdown(reason="application_shutdown")) + closer.start() + adapter.start_release.set() + starter.join(timeout=5) + closer.join(timeout=5) + + assert len(activate_errors) == 1 + assert isinstance(activate_errors[0], CapabilityRuntimeClosedError) + assert adapter.cleanup_calls == 1 + assert runtime.get_running("sample.capability") is None + assert runtime.is_shutdown is True + with pytest.raises(CapabilityRuntimeClosedError): + runtime.activate("sample.capability", reason="late_start") + + +def test_shutdown_cannot_return_between_open_check_and_sync_claim(tmp_path: Path) -> None: + """open check 与 inflight claim 必须共享 barrier，关闭扫描不能漏过首启。""" + adapter = _SyncAdapter() + adapter.start_entered = threading.Event() + adapter.start_release = threading.Event() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + check_entered = threading.Event() + check_release = threading.Event() + shutdown_returned = threading.Event() + activate_errors = [] + original_ensure_open = runtime._ensure_open + first_check = True + check_lock = threading.Lock() + + def gated_ensure_open() -> None: + nonlocal first_check + original_ensure_open() + with check_lock: + should_wait = first_check + first_check = False + if should_wait: + check_entered.set() + assert check_release.wait(timeout=5) + + def activate() -> None: + try: + runtime.activate("sample.capability", reason="preclaim_race") + except BaseException as error: + activate_errors.append(error) + + def shutdown() -> None: + runtime.shutdown(reason="application_shutdown") + shutdown_returned.set() + + with patch.object(runtime, "_ensure_open", side_effect=gated_ensure_open): + starter = threading.Thread(target=activate) + starter.start() + assert check_entered.wait(timeout=5) + closer = threading.Thread(target=shutdown) + closer.start() + + assert not shutdown_returned.wait(timeout=0.1) + check_release.set() + assert adapter.start_entered.wait(timeout=5) + assert not shutdown_returned.is_set() + adapter.start_release.set() + starter.join(timeout=5) + closer.join(timeout=5) + + assert shutdown_returned.is_set() + assert len(activate_errors) <= 1 + assert not activate_errors or isinstance( + activate_errors[0], + CapabilityRuntimeClosedError, + ) + assert runtime.get_running("sample.capability") is None + + +@pytest.mark.asyncio +async def test_shutdown_cannot_return_between_open_check_and_async_claim( + tmp_path: Path, +) -> None: + """异步 activate 的同步 claim 区间也必须受同一关闭 barrier 保护。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + check_entered = threading.Event() + check_release = threading.Event() + shutdown_returned = threading.Event() + returned_before_release = [] + closer_threads = [] + original_ensure_open = runtime._ensure_open + first_check = True + check_lock = threading.Lock() + + def gated_ensure_open() -> None: + nonlocal first_check + original_ensure_open() + with check_lock: + should_wait = first_check + first_check = False + if should_wait: + check_entered.set() + assert check_release.wait(timeout=5) + + def shutdown() -> None: + asyncio.run(runtime.shutdown_async(reason="application_shutdown")) + shutdown_returned.set() + + def coordinate_shutdown() -> None: + assert check_entered.wait(timeout=5) + closer = threading.Thread(target=shutdown) + closer_threads.append(closer) + closer.start() + returned_before_release.append(shutdown_returned.wait(timeout=0.1)) + check_release.set() + + coordinator = threading.Thread(target=coordinate_shutdown) + coordinator.start() + with patch.object(runtime, "_ensure_open", side_effect=gated_ensure_open): + activate_task = asyncio.create_task( + runtime.activate_async("sample.capability", reason="preclaim_race") + ) + await adapter.start_entered.wait() + await asyncio.to_thread(coordinator.join, 5) + assert returned_before_release == [False] + assert not shutdown_returned.is_set() + adapter.start_release.set() + try: + await activate_task + except CapabilityRuntimeClosedError: + pass + await asyncio.to_thread(closer_threads[0].join, 5) + + assert shutdown_returned.is_set() + assert runtime.get_running("sample.capability") is None + + +@pytest.mark.asyncio +async def test_async_adapter_uses_same_single_flight_state_machine(tmp_path: Path) -> None: + """异步回调等待不能阻塞事件循环，并发调用共享同一 generation。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + first = asyncio.create_task(runtime.activate_async("sample.capability", reason="first")) + await adapter.start_entered.wait() + second = asyncio.create_task(runtime.activate_async("sample.capability", reason="second")) + await asyncio.sleep(0) + + assert runtime.get_running("sample.capability") is None + adapter.start_release.set() + first_instance, second_instance = await asyncio.gather(first, second) + + assert first_instance is second_instance + assert adapter.materialize_calls == 1 + assert adapter.start_calls == 1 + assert runtime.snapshot("sample.capability").generation == 1 + + +@pytest.mark.asyncio +async def test_stop_async_failure_retains_ownership_for_explicit_retry(tmp_path: Path) -> None: + """异步 stop 失败后只能重试释放同一实例，不能直接启动新实例。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + instance = await initial + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.stop_async("sample.capability", reason="configuration_removed") + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.activate_async("sample.capability", reason="unsafe_retry", retry=True) + assert adapter.create_calls == 1 + + adapter.fail_stop = False + await runtime.stop_async("sample.capability", reason="stop_retry") + + assert adapter.stop_instances == [instance, instance, instance] + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPED + + +@pytest.mark.asyncio +async def test_async_reload_uses_reloading_state_and_hides_candidate(tmp_path: Path) -> None: + """异步 reload 与同步入口遵守相同状态和发布边界。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + old_instance = await initial + + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + reload_task = asyncio.create_task( + runtime.reload_async("sample.capability", reason="config_changed") + ) + await adapter.start_entered.wait() + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.RELOADING + adapter.start_release.set() + new_instance = await reload_task + + assert new_instance is not old_instance + assert runtime.get_running("sample.capability") is new_instance + assert runtime.snapshot("sample.capability").generation == 2 + + +@pytest.mark.asyncio +async def test_failed_async_reload_cleans_candidate_and_enters_failed(tmp_path: Path) -> None: + """异步 reload 失败与同步入口一致，不发布半初始化候选。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + await initial + + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + adapter.fail_start = True + reload_task = asyncio.create_task( + runtime.reload_async("sample.capability", reason="config_changed") + ) + await adapter.start_entered.wait() + adapter.start_release.set() + + with pytest.raises(CapabilityOperationError, match="async start failed"): + await reload_task + + snapshot = runtime.snapshot("sample.capability") + assert snapshot.lifecycle is CapabilityLifecycleState.FAILED + assert snapshot.visible is False + assert adapter.cleanup_calls == 1 + + +@pytest.mark.asyncio +async def test_async_reload_stop_failure_retains_previous_without_new_create( + tmp_path: Path, +) -> None: + """异步 reload 也必须保留未释放旧实例并禁止创建第二份资源。""" + adapter = _AsyncAdapter() + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + initial = asyncio.create_task(runtime.activate_async("sample.capability", reason="initial")) + await adapter.start_entered.wait() + adapter.start_release.set() + old_instance = await initial + + adapter.fail_stop = True + + with pytest.raises(CapabilityOperationError, match="async stop failed"): + await runtime.reload_async("sample.capability", reason="config_changed") + + assert runtime.get_running("sample.capability") is None + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.FAILED + assert adapter.stop_instances == [old_instance] + assert adapter.create_calls == 1 + assert adapter.start_calls == 1 + with pytest.raises(CapabilityOperationError, match="重试 stop"): + await runtime.activate_async("sample.capability", reason="implicit_retry") + adapter.fail_stop = False + adapter.start_entered = asyncio.Event() + adapter.start_release = asyncio.Event() + adapter.stop_entered = asyncio.Event() + adapter.stop_release = asyncio.Event() + recovery = asyncio.create_task( + runtime.activate_async( + "sample.capability", + reason="recover_after_reload", + retry=True, + ) + ) + await adapter.stop_entered.wait() + assert runtime.snapshot("sample.capability").lifecycle is CapabilityLifecycleState.STOPPING + assert runtime.get_running("sample.capability") is None + assert adapter.create_calls == 1 + adapter.stop_release.set() + await adapter.start_entered.wait() + adapter.start_release.set() + replacement = await recovery + + assert adapter.stop_instances == [old_instance, old_instance] + assert replacement is not old_instance + assert adapter.create_calls == 2 + + +def test_one_failed_capability_does_not_remove_specs_or_block_other_capabilities( + tmp_path: Path, +) -> None: + """单项失败只改变自身状态，Registry 中的其它声明仍可继续运行。""" + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + (first_dir / "capability.toml").write_text(_MANIFEST.strip() + "\n", encoding="utf-8") + (second_dir / "capability.toml").write_text( + _MANIFEST.replace("sample.capability", "other.capability") + .replace("sample_implementation", "other_implementation") + .strip() + + "\n", + encoding="utf-8", + ) + registry = CapabilityRegistry.discover( + roots=[tmp_path], + kinds={"sample"}, + selector_schemas={}, + ) + adapter = _SyncAdapter() + runtime = CapabilityRuntime(registry, adapters={"sample": adapter}) + adapter.fail_start = True + + with pytest.raises(CapabilityOperationError): + runtime.activate("sample.capability", reason="fail") + adapter.fail_start = False + other = runtime.activate("other.capability", reason="continue") + + assert other.started is True + assert {spec.id for spec in runtime.list_specs()} == { + "sample.capability", + "other.capability", + } + assert runtime.snapshot("sample.capability").error == "start failed" + + +@pytest.mark.asyncio +async def test_sync_and_async_entrypoints_reject_wrong_adapter_mode(tmp_path: Path) -> None: + """入口与 adapter 执行模型不匹配时应在执行回调前失败。""" + async_runtime = CapabilityRuntime( + _registry(tmp_path / "async"), + adapters={"sample": _AsyncAdapter()}, + ) + with pytest.raises(CapabilityAdapterModeError): + async_runtime.activate("sample.capability", reason="wrong_mode") + + sync_runtime = CapabilityRuntime( + _registry(tmp_path / "sync"), + adapters={"sample": _SyncAdapter()}, + ) + with pytest.raises(CapabilityAdapterModeError): + await sync_runtime.activate_async("sample.capability", reason="wrong_mode") + + +def test_adapter_mode_requires_declared_enum_member(tmp_path: Path) -> None: + """并发模型必须显式声明 enum，不能依赖字符串相等的偶然兼容。""" + adapter = _SyncAdapter() + adapter.execution_mode = "sync" + runtime = CapabilityRuntime(_registry(tmp_path), adapters={"sample": adapter}) + + with pytest.raises(CapabilityAdapterModeError): + runtime.activate("sample.capability", reason="invalid_mode") diff --git a/tests/test_config_reload_handler.py b/tests/test_config_reload_handler.py index 3d1367a34..9789b0cd0 100644 --- a/tests/test_config_reload_handler.py +++ b/tests/test_config_reload_handler.py @@ -126,3 +126,23 @@ async def test_resolve_falls_back_to_qualname_when_name_mismatched(monkeypatch): assert wrapper.__name__ == "wrapper" assert instance.reload_count == 1 + + +def test_externally_managed_reload_class_does_not_register_listener(monkeypatch): + """外部统一管理配置生命周期时，Mixin 保留重载能力但不重复绑定事件。""" + registrations = [] + monkeypatch.setattr( + eventmanager, + "add_event_listener", + lambda *args, **kwargs: registrations.append((args, kwargs)), + ) + + class _ExternallyManagedReloadRecorder(ConfigReloadMixin): + CONFIG_RELOAD_MANAGED_EXTERNALLY = True + CONFIG_WATCH = {"TEST_RELOAD_KEY"} + + def on_config_changed(self): + pass + + assert registrations == [] + assert "handle_config_changed" not in _ExternallyManagedReloadRecorder.__dict__ diff --git a/tests/test_event_dispatch_snapshot.py b/tests/test_event_dispatch_snapshot.py new file mode 100644 index 000000000..ee425919f --- /dev/null +++ b/tests/test_event_dispatch_snapshot.py @@ -0,0 +1,160 @@ +"""事件调度订阅快照的并发回归测试。""" + +import pytest + +from app.runtime.events import Event, eventmanager +from app.schemas.types import ChainEventType, EventType + + +class _ImmediateExecutor: + """在当前线程执行广播 handler，使订阅变更精确发生在调度迭代期间。""" + + @staticmethod + def submit(func, *args, **kwargs): + return func(*args, **kwargs) + + +@pytest.fixture +def isolated_eventmanager(monkeypatch): + """隔离全局事件总线的订阅表和广播执行器。""" + monkeypatch.setattr( + eventmanager, + "_EventManager__broadcast_subscribers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__chain_subscribers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__handler_instance_resolvers", + {}, + ) + monkeypatch.setattr( + eventmanager, + "_EventManager__executor", + _ImmediateExecutor(), + ) + return eventmanager + + +def test_broadcast_dispatch_uses_subscription_snapshot(isolated_eventmanager): + """广播事件中新增或移除的 handler 从下一个事件开始生效。""" + calls = [] + + def late_handler(_event): + calls.append("late") + + def removed_handler(_event): + calls.append("removed") + + def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + EventType.ConfigChanged, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + EventType.ConfigChanged, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_broadcast_event + dispatch(Event(EventType.ConfigChanged, {})) + assert calls == ["mutating", "removed"] + + calls.clear() + dispatch(Event(EventType.ConfigChanged, {})) + assert calls == ["mutating", "late"] + + +def test_sync_chain_dispatch_uses_subscription_snapshot(isolated_eventmanager): + """同步链式事件中的订阅变更不影响当前处理器序列。""" + calls = [] + + def late_handler(_event): + calls.append("late") + + def removed_handler(_event): + calls.append("removed") + + def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_chain_event + assert dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "removed"] + + calls.clear() + assert dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "late"] + + +@pytest.mark.asyncio +async def test_async_chain_dispatch_uses_subscription_snapshot( + isolated_eventmanager, +): + """异步链式事件中的订阅变更不影响当前处理器序列。""" + calls = [] + + async def late_handler(_event): + calls.append("late") + + async def removed_handler(_event): + calls.append("removed") + + async def mutating_handler(_event): + calls.append("mutating") + isolated_eventmanager.remove_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + late_handler, + ) + + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + mutating_handler, + ) + isolated_eventmanager.add_event_listener( + ChainEventType.NameRecognize, + removed_handler, + ) + + dispatch = isolated_eventmanager._EventManager__dispatch_chain_event_async + assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "removed"] + + calls.clear() + assert await dispatch(Event(ChainEventType.NameRecognize, {})) is True + assert calls == ["mutating", "late"] diff --git a/tests/test_lifecycle_shutdown.py b/tests/test_lifecycle_shutdown.py index 309d6d520..6feb4d568 100644 --- a/tests/test_lifecycle_shutdown.py +++ b/tests/test_lifecycle_shutdown.py @@ -364,7 +364,7 @@ def _patch_module_shutdown_dependencies(monkeypatch) -> dict: """替换 stop_modules 的资源所有者，避免测试启动真实后台服务""" dependencies = {} for name, method_name in ( - ("ModuleManager", "stop"), + ("ModuleManager", "shutdown"), ("EventManager", "stop"), ("DisplayHelper", "stop"), ("DohHelper", "shutdown"), diff --git a/tests/test_module_manager_capability_adapter.py b/tests/test_module_manager_capability_adapter.py new file mode 100644 index 000000000..b9e78418a --- /dev/null +++ b/tests/test_module_manager_capability_adapter.py @@ -0,0 +1,930 @@ +"""Host Module Adapter 对 Capability Runtime 的兼容合同测试。""" + +from __future__ import annotations + +import importlib +import os +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Iterator +from unittest.mock import Mock + +import pytest + +from app.db.oper.systemconfig import SystemConfigOper +from app.foundation.singleton import Singleton +from app.runtime.capabilities.errors import CapabilityRuntimeClosedError +from app.runtime.capabilities.model import SelectorSchema +from app.runtime.capabilities.registry import CapabilityRegistry +from app.runtime.events import Event, EventHandlerBinding, eventmanager +from app.runtime.extensions import module_manager as module_manager_extension +from app.runtime.extensions.module_manager import ModuleManager +from app.schemas import ConfigChangeEventData +from app.schemas.types import EventType + + +_SAMPLE_MANIFEST = """ +schema_version = 1 +id = "SampleModule" +kind = "host_module" +entrypoint = "fixture_sample_module:SampleModule" +depends_on = [] + +[metadata] +name = "Sample" +type = "notification" +subtype = "Telegram" +priority = 10 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "sample" +enabled_field = "enabled" +""" + +_OTHER_MANIFEST = """ +schema_version = 1 +id = "OtherModule" +kind = "host_module" +entrypoint = "fixture_other_module:OtherModule" +depends_on = [] + +[metadata] +name = "Other" +type = "notification" +subtype = "Telegram" +priority = 20 + +[activation] +policy = "when_configured" +watch = ["Notifications"] + +[activation.selector] +kind = "system_config_item" +key = "Notifications" +match_field = "type" +match_value = "other" +enabled_field = "enabled" +""" + +_MODULE_SOURCE = """ +class {class_name}: + instances = [] + + def __init__(self): + self.events = ["create"] + type(self).instances.append(self) + + def init_module(self): + self.events.append("start") + + def stop(self): + self.events.append("stop") + + def test(self): + return True, "ok" + + def capability_method(self): + return "handled" + + @staticmethod + def get_name(): + return "{name}" + + @staticmethod + def get_type(): + return "notification" + + @staticmethod + def get_subtype(): + return "Telegram" + + @staticmethod + def get_priority(): + return {priority} +""" + + +def _write_capability(root: Path, directory: str, manifest: str) -> None: + """写入一个合成 Host Module 声明。""" + capability_dir = root / directory + capability_dir.mkdir(parents=True) + (capability_dir / "capability.toml").write_text( + manifest.strip() + "\n", + encoding="utf-8", + ) + + +def _build_registry(root: Path) -> CapabilityRegistry: + """用生产 schema 构造只包含两个合成模块的 Registry。""" + _write_capability(root, "sample", _SAMPLE_MANIFEST) + _write_capability(root, "other", _OTHER_MANIFEST) + return CapabilityRegistry.discover( + roots=[root], + kinds={"host_module"}, + selector_schemas={ + "system_config_item": SelectorSchema( + required_fields=frozenset({ + "key", + "match_field", + "match_value", + "enabled_field", + }), + ), + "setting_truthy": SelectorSchema( + required_fields=frozenset({"key"}), + ), + }, + ) + + +def _config_changed_listeners() -> dict: + """读取 ConfigChanged 监听快照，用于验证全局测试状态完整恢复。""" + subscribers = getattr(eventmanager, "_EventManager__broadcast_subscribers") + return dict(subscribers.get(EventType.ConfigChanged, {})) + + +def _run_real_host_module_check(tmp_path: Path, body: str) -> None: + """在隔离后端和进程内网络守卫下执行真实 Host Module 合同检查。""" + project_root = Path(__file__).parents[1] + prelude = r""" +import ipaddress +import socket +import sys + +network_attempts = [] +allowed_hosts = {"127.0.0.1", "::1", "localhost", "0.0.0.0", "::", ""} +real_getaddrinfo = socket.getaddrinfo +real_connect = socket.socket.connect + +def is_allowed_host(host): + normalized = host.decode() if isinstance(host, (bytes, bytearray)) else host + if normalized is None or normalized in allowed_hosts: + return True + try: + address = ipaddress.ip_address(str(normalized).split("%", 1)[0]) + return address.is_loopback or address.is_unspecified + except ValueError: + return False + +def block_network(operation, host): + network_attempts.append((operation, host)) + raise AssertionError(f"Host Module 合同测试禁止真实出站：{operation} {host!r}") + +def guarded_getaddrinfo(host, *args, **kwargs): + if not is_allowed_host(host): + block_network("DNS", host) + return real_getaddrinfo(host, *args, **kwargs) + +def guarded_connect(sock, address): + if isinstance(address, tuple) and address and not is_allowed_host(address[0]): + block_network("socket", address[0]) + return real_connect(sock, address) + +socket.getaddrinfo = guarded_getaddrinfo +socket.socket.connect = guarded_connect + +from app.testing.bootstrap import prepare_backend +prepare_backend() +""" + code = f"{prelude}\n{body}\nassert network_attempts == [], network_attempts\n" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"真实 Host Module 合同检查失败：\nstdout:\n{result.stdout[-4000:]}\n" + f"stderr:\n{result.stderr[-8000:]}" + ) + + +@pytest.fixture +def module_manager_harness( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> Iterator[SimpleNamespace]: + """用合成声明和内存配置隔离 ModuleManager 单例。""" + source_root = tmp_path / "source" + source_root.mkdir() + (source_root / "fixture_sample_module.py").write_text( + _MODULE_SOURCE.format(class_name="SampleModule", name="Sample", priority=10), + encoding="utf-8", + ) + (source_root / "fixture_other_module.py").write_text( + _MODULE_SOURCE.format(class_name="OtherModule", name="Other", priority=20), + encoding="utf-8", + ) + monkeypatch.syspath_prepend(str(source_root)) + + registry = _build_registry(tmp_path / "capabilities") + monkeypatch.setattr( + module_manager_extension, + "build_host_module_registry", + lambda: registry, + ) + + config_values = {"Notifications": []} + + def get_config(_self, key=None): + key_value = getattr(key, "value", key) + if key_value is None: + return dict(config_values) + return config_values.get(key_value) + + monkeypatch.setattr(SystemConfigOper, "get", get_config) + + singleton_key = (ModuleManager, (), frozenset()) + previous_manager = Singleton._instances.pop(singleton_key, None) + resolver_attr = "_EventManager__handler_instance_resolvers" + previous_resolvers = dict(getattr(eventmanager, resolver_attr)) + previous_config_changed_listeners = _config_changed_listeners() + for module_name in ("fixture_sample_module", "fixture_other_module"): + sys.modules.pop(module_name, None) + + manager = ModuleManager() + restored = False + + def restore() -> None: + """撤销 Manager 构造写入的单例、resolver 和事件监听器。""" + nonlocal restored + if restored: + return + try: + manager.shutdown() + except (AttributeError, CapabilityRuntimeClosedError): + pass + Singleton._instances.pop(singleton_key, None) + if previous_manager is not None: + Singleton._instances[singleton_key] = previous_manager + setattr(eventmanager, resolver_attr, previous_resolvers) + subscribers = getattr(eventmanager, "_EventManager__broadcast_subscribers") + if previous_config_changed_listeners: + subscribers[EventType.ConfigChanged] = dict( + previous_config_changed_listeners + ) + else: + subscribers.pop(EventType.ConfigChanged, None) + for module_name in ("fixture_sample_module", "fixture_other_module"): + sys.modules.pop(module_name, None) + restored = True + + try: + yield SimpleNamespace( + manager=manager, + config_values=config_values, + previous_config_changed_listeners=previous_config_changed_listeners, + restore=restore, + ) + finally: + restore() + + +def _enable_sample(config_values: dict) -> None: + """写入可通过 sample selector 的最小合法通知配置。""" + config_values["Notifications"] = [ + { + "name": "sample", + "type": "sample", + "config": {}, + "switchs": [], + "enabled": True, + } + ] + + +def test_harness_restores_module_manager_config_listener( + module_manager_harness, +) -> None: + """Fixture teardown 不能把临时 Manager 的 bound listener 留在全局事件总线。""" + manager = module_manager_harness.manager + current_listeners = _config_changed_listeners() + + assert current_listeners != ( + module_manager_harness.previous_config_changed_listeners + ) + assert any( + getattr(listener, "__self__", None) is manager + for listener in current_listeners.values() + ) + + module_manager_harness.restore() + + assert _config_changed_listeners() == ( + module_manager_harness.previous_config_changed_listeners + ) + + +def test_specs_are_lightweight_and_do_not_materialize_modules( + module_manager_harness, +) -> None: + """ModuleManager 的声明视图不能解析任何 Host Module 实现。""" + manager = module_manager_harness.manager + + specs = manager.list_specs() + + assert manager.get_specs() == specs + assert [spec.id for spec in specs] == ["OtherModule", "SampleModule"] + assert [spec.metadata["name"] for spec in specs] == ["Other", "Sample"] + assert "fixture_sample_module" not in sys.modules + assert "fixture_other_module" not in sys.modules + assert manager.get_running_module("SampleModule") is None + + +def test_get_module_materializes_one_canonical_class_without_starting_it( + module_manager_harness, +) -> None: + """兼容查询返回 canonical class，但不创建或启动资源。""" + manager = module_manager_harness.manager + + module_class = manager.get_module("SampleModule") + canonical_class = importlib.import_module( + "fixture_sample_module" + ).SampleModule + + assert module_class is canonical_class + assert manager.get_module("SampleModule") is canonical_class + assert canonical_class.instances == [] + assert manager.get_running_module("SampleModule") is None + assert "fixture_other_module" not in sys.modules + + +def test_get_modules_materializes_all_real_classes_without_starting_them( + module_manager_harness, +) -> None: + """旧 get_modules 合同保留真实 class 字典，不返回代理或隐式激活。""" + manager = module_manager_harness.manager + + modules = manager.get_modules() + + sample_module = importlib.import_module("fixture_sample_module") + other_module = importlib.import_module("fixture_other_module") + assert modules == { + "OtherModule": other_module.OtherModule, + "SampleModule": sample_module.SampleModule, + } + assert sample_module.SampleModule.instances == [] + assert other_module.OtherModule.instances == [] + assert manager.get_running_module("SampleModule") is None + assert manager.get_running_module("OtherModule") is None + + +def test_config_reconcile_reload_and_stop_preserve_manager_contract( + module_manager_harness, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """配置激活、全量 reload 与可重启 stop 保持同步可观察顺序。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + + manager.load_modules() + first = manager.get_running_module("SampleModule") + assert first is not None + assert first.events == ["create", "start"] + + send_event = Mock() + monkeypatch.setattr(eventmanager, "send_event", send_event) + manager.reload() + second = manager.get_running_module("SampleModule") + + assert second is not None + assert second is not first + assert first.events == ["create", "start", "stop"] + assert second.events == ["create", "start"] + send_event.assert_called_once_with(etype=EventType.ModuleReload, data={}) + + manager.stop() + assert manager.get_running_module("SampleModule") is None + assert second.events == ["create", "start", "stop"] + + manager.load_modules() + restarted = manager.get_running_module("SampleModule") + assert restarted is not None + assert restarted is not second + assert restarted.events == ["create", "start"] + + module_manager_harness.config_values["Notifications"] = [] + manager.load_modules() + assert manager.get_running_module("SampleModule") is None + assert restarted.events == ["create", "start", "stop"] + + +def test_config_event_reloads_same_instance_and_tracks_selector_changes( + module_manager_harness, +) -> None: + """配置事件由 Host Adapter 唯一协调，并保留模块实例内的重载状态。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + running = manager.get_running_module("SampleModule") + + manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key="Notifications"), + ) + ) + + assert manager.get_running_module("SampleModule") is running + assert running.events == ["create", "start", "stop", "start"] + + module_manager_harness.config_values["Notifications"] = [] + manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key="Notifications"), + ) + ) + assert manager.get_running_module("SampleModule") is None + assert running.events == ["create", "start", "stop", "start", "stop"] + + +def test_shutdown_is_irreversible(module_manager_harness) -> None: + """shutdown 撤销全部可见实例，并拒绝通过 load_modules 再次启动。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + running = manager.get_running_module("SampleModule") + assert running is not None + + manager.shutdown() + + assert manager.get_running_module("SampleModule") is None + assert running.events == ["create", "start", "stop"] + manager.load_modules() + assert manager.get_running_module("SampleModule") is None + assert type(running).instances == [running] + + +def test_all_real_host_modules_zero_arg_construct_without_starting_resources( + tmp_path: Path, +) -> None: + """每份真实 manifest 都必须能解析 canonical class 并零参数构造且不启动资源。""" + body = r""" +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 + +adapter = HostModuleAdapter() +lifecycle_events = [] +instances = {} + +def make_recorder(operation, capability_id): + def record(instance): + lifecycle_events.append((operation, capability_id, id(instance))) + return record + +for spec in specs: + implementation = adapter.materialize(spec) + module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1) + assert implementation is getattr(sys.modules[module_name], symbol_name) + implementation.init_module = make_recorder("start", spec.id) + implementation.stop = make_recorder("stop", spec.id) + + instance = adapter.create(spec, implementation, generation=1) + assert type(instance) is implementation + instances[spec.id] = instance + +assert set(instances) == {spec.id for spec in specs} +assert lifecycle_events == [] +""" + _run_real_host_module_check(tmp_path, body) + + +def test_real_manifest_inventory_drives_full_module_manager_lifecycle( + tmp_path: Path, +) -> None: + """真实声明自动驱动全量激活、原实例重载、禁用停止和不可逆关闭门禁。""" + body = r""" +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.capabilities.model import ActivationPolicy +from app.runtime.config import settings +from app.runtime.events import Event +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) +from app.schemas import ConfigChangeEventData +from app.schemas.types import EventType + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 +spec_by_id = {spec.id: spec for spec in specs} + +events = {spec.id: [] for spec in specs} +adapter = HostModuleAdapter() + +def make_recorder(operation, capability_id): + def record(instance): + events[capability_id].append((operation, id(instance))) + return record + +for spec in specs: + implementation = adapter.materialize(spec) + implementation.init_module = make_recorder("start", spec.id) + implementation.stop = make_recorder("stop", spec.id) + +config_values = {} +enabled_service_values = {} +selector_keys = set() +configured_ids = set() +for spec in specs: + if spec.activation is not ActivationPolicy.WHEN_CONFIGURED: + continue + configured_ids.add(spec.id) + selector = spec.selector + assert selector is not None + key = str(selector.config["key"]) + selector_keys.add(key) + if selector.kind == "setting_truthy": + setattr(settings, key, f"enabled:{spec.id}") + elif selector.kind == "system_config_item": + enabled_service_values.setdefault(key, []).append({ + "name": f"contract-{spec.id}", + "type": selector.config["match_value"], + "config": {}, + "enabled": True, + }) + else: + raise AssertionError(f"未覆盖的 Host Module selector：{selector.kind}") +config_values.update({key: list(value) for key, value in enabled_service_values.items()}) + +def get_config(_self, key=None): + key_value = getattr(key, "value", key) + if key_value is None: + return dict(config_values) + return config_values.get(key_value) + +SystemConfigOper.get = get_config + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +bootstrap_ids = { + spec.id + for spec in specs + if spec.activation is ActivationPolicy.BOOTSTRAP +} +initial_ids = bootstrap_ids | configured_ids +assert bootstrap_ids +assert configured_ids +assert initial_ids == {spec.id for spec in specs} +initial_instances = { + capability_id: manager.get_running_module(capability_id) + for capability_id in initial_ids +} +assert all(initial_instances.values()) +assert { + capability_id: [operation for operation, _instance_id in events[capability_id]] + for capability_id in initial_ids +} == {capability_id: ["start"] for capability_id in initial_ids} + +watch_keys = {key for spec in specs for key in spec.watch} +watched_ids = { + spec.id + for spec in specs + if spec.id in initial_ids and watch_keys.intersection(spec.watch) +} +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=watch_keys), + ) +) + +for capability_id, initial_instance in initial_instances.items(): + assert manager.get_running_module(capability_id) is initial_instance + operations = [operation for operation, _instance_id in events[capability_id]] + expected = ["start", "stop", "start"] if capability_id in watched_ids else ["start"] + assert operations == expected, (capability_id, operations) + assert { + instance_id for _operation, instance_id in events[capability_id] + } == {id(initial_instance)} + +for spec in specs: + if spec.id not in configured_ids: + continue + selector = spec.selector + key = str(selector.config["key"]) + if selector.kind == "setting_truthy": + setattr(settings, key, False) +for key in enabled_service_values: + config_values[key] = [] + +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=selector_keys), + ) +) +for capability_id in configured_ids: + assert manager.get_running_module(capability_id) is None + assert [operation for operation, _instance_id in events[capability_id]] == [ + "start", + "stop", + "start", + "stop", + ] +for capability_id in bootstrap_ids: + assert manager.get_running_module(capability_id) is initial_instances[capability_id] + +manager.shutdown() +assert all(manager.get_running_module(spec.id) is None for spec in specs) +events_after_shutdown = { + capability_id: list(capability_events) + for capability_id, capability_events in events.items() +} + +for spec in specs: + if spec.id not in configured_ids: + continue + selector = spec.selector + key = str(selector.config["key"]) + if selector.kind == "setting_truthy": + setattr(settings, key, f"re-enabled:{spec.id}") +for key, value in enabled_service_values.items(): + config_values[key] = list(value) + +manager.load_modules() +manager.handle_config_changed( + Event( + EventType.ConfigChanged, + ConfigChangeEventData(key=watch_keys), + ) +) +assert all(manager.get_running_module(spec.id) is None for spec in specs) +assert events == events_after_shutdown +assert set(spec_by_id) == set(events) +""" + _run_real_host_module_check(tmp_path, body) + + +def test_default_config_keeps_every_manifest_configured_entrypoint_unimported( + tmp_path: Path, +) -> None: + """默认配置惰性边界由全部 when-configured manifest 自动生成。""" + body = r""" +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.capabilities.model import ActivationPolicy +from app.runtime.config import settings +from app.runtime.extensions.host_module_adapter import ( + HostModuleAdapter, + build_host_module_registry, +) + +registry = build_host_module_registry() +specs = registry.list_specs() +assert len(specs) == 37 +configured_specs = tuple( + spec for spec in specs + if spec.activation is ActivationPolicy.WHEN_CONFIGURED +) +configured_modules = { + spec.entrypoint.split(":", maxsplit=1)[0] + for spec in configured_specs +} +assert configured_modules +assert configured_modules.isdisjoint(sys.modules) + +for spec in configured_specs: + selector = spec.selector + assert selector is not None + if selector.kind == "setting_truthy": + setattr(settings, str(selector.config["key"]), False) + +SystemConfigOper.get = lambda _self, key=None: {} if key is None else [] + +adapter = HostModuleAdapter() +for spec in specs: + if spec.activation is not ActivationPolicy.BOOTSTRAP: + continue + implementation = adapter.materialize(spec) + implementation.init_module = lambda _self: None + implementation.stop = lambda _self: None + +assert configured_modules.isdisjoint(sys.modules) + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +assert manager.get_specs() == manager.list_specs() +assert {spec.id for spec in manager.list_specs()} == {spec.id for spec in specs} +assert all(manager.get_running_module(spec.id) is None for spec in configured_specs) +assert configured_modules.isdisjoint(sys.modules) +manager.shutdown() +assert configured_modules.isdisjoint(sys.modules) +""" + _run_real_host_module_check(tmp_path, body) + + +def test_event_resolver_uses_exact_class_and_blocks_stopped_owner_fallback( + module_manager_harness, +) -> None: + """同名 class 不能冒充 owner；已停止 owner 必须返回 Binding(None)。""" + manager = module_manager_harness.manager + _enable_sample(module_manager_harness.config_values) + manager.load_modules() + module_class = manager.get_module("SampleModule") + running = manager.get_running_module("SampleModule") + + active_binding = manager.resolve_event_handler_instance(module_class) + assert active_binding == EventHandlerBinding( + instance=running, + owner_name="Sample", + ) + + impostor = type("SampleModule", (), {}) + impostor.__module__ = module_class.__module__ + assert manager.resolve_event_handler_instance(impostor) is None + + manager.stop() + stopped_binding = manager.resolve_event_handler_instance(module_class) + assert stopped_binding == EventHandlerBinding( + instance=None, + owner_name="Sample", + ) + + +def test_default_modulelist_does_not_import_unconfigured_provider_sdks( + tmp_path: Path, +) -> None: + """默认配置下构造 Manager 和查询模块列表都不能拉起重量 provider SDK。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +from app.db.oper.systemconfig import SystemConfigOper +from app.runtime.config import settings + +def empty_config(self, key=None): + return {} if key is None else [] + +SystemConfigOper.get = empty_config +settings.ACOUSTID_API_KEY = None +settings.FANART_API_KEY = None + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +assert len(manager.list_specs()) == 37 +assert manager.get_specs() == manager.list_specs() + +from app.api.endpoints.system import modulelist +response = modulelist(None) +assert len(response.data["modules"]) == 37 + +heavy_prefixes = ( + "lark_oapi", + "slack_bolt", + "slack_sdk", + "discord", + "plexapi", + "telebot", +) +loaded = sorted( + name + for name in sys.modules + if any(name == prefix or name.startswith(prefix + ".") for prefix in heavy_prefixes) +) +assert loaded == [], loaded +manager.shutdown() +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", "import sys\n" + code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"子进程模块发现失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) + + +def test_lazy_boundary_annotations_are_reflectable_without_provider_sdks( + tmp_path: Path, +) -> None: + """宿主公共注解可被反射，且反射过程不加载可选 provider SDK。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +import sys +from typing import Any, Optional, get_type_hints + +provider_prefixes = ("qbittorrentapi", "transmission_rpc", "pywebpush") + +def loaded_provider_modules(): + return sorted( + name + for name in sys.modules + if any( + name == prefix or name.startswith(prefix + ".") + for prefix in provider_prefixes + ) + ) + +assert loaded_provider_modules() == [] + +from app.chain import ChainBase +from app.api.endpoints.message import WebPushError, is_webpush_subscription_gone + +assert get_type_hints(ChainBase.torrent_files)["return"] == Optional[Any] +assert get_type_hints(is_webpush_subscription_gone)["error"] is WebPushError +assert loaded_provider_modules() == [] +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"轻量注解反射失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) + + +def test_manifest_metadata_matches_legacy_module_class_contract(tmp_path: Path) -> None: + """manifest 投影必须与插件仍可调用的模块类 metadata 完全一致。""" + project_root = Path(__file__).parents[1] + code = """ +from app.testing.bootstrap import prepare_backend +prepare_backend() + +from app.db.oper.systemconfig import SystemConfigOper + +SystemConfigOper.get = lambda self, key=None: {} if key is None else [] + +from app.runtime.config import settings +settings.ACOUSTID_API_KEY = None +settings.FANART_API_KEY = None + +from app.runtime.extensions.module_manager import ModuleManager + +manager = ModuleManager() +modules = manager.get_modules() +assert len(modules) == len(manager.list_specs()) == 37 +for spec in manager.list_specs(): + implementation = modules[spec.id] + assert implementation.get_name() == spec.metadata["name"] + assert implementation.get_type().value == spec.metadata["type"] + assert implementation.get_subtype().name == spec.metadata["subtype"] + assert implementation.get_priority() == spec.metadata["priority"] +manager.shutdown() +""" + env = os.environ.copy() + env["CONFIG_DIR"] = str(tmp_path / "config") + env["PYTHONPATH"] = str(project_root) + result = subprocess.run( + [sys.executable, "-c", "import sys\n" + code], + cwd=project_root, + env=env, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + + assert result.returncode == 0, ( + f"模块 metadata 兼容检查失败：\nstdout:\n{result.stdout[-2000:]}\n" + f"stderr:\n{result.stderr[-4000:]}" + ) diff --git a/tests/test_navidrome_module.py b/tests/test_navidrome_module.py index 6147e2f71..e6c0ea572 100644 --- a/tests/test_navidrome_module.py +++ b/tests/test_navidrome_module.py @@ -21,9 +21,12 @@ def test_navidrome_module_has_no_system_switch(): assert NavidromeModule().init_setting() is None -def test_navidrome_module_is_loaded_by_module_manager(): - """模块管理器应能加载 Navidrome，否则媒体服务器列表里不会出现该类型。""" - assert "NavidromeModule" in ModuleManager()._running_modules +def test_navidrome_module_is_discovered_without_unconfigured_activation(): + """Navidrome 始终可发现，但没有启用配置时不应创建服务资源。""" + manager = ModuleManager() + + assert "NavidromeModule" in manager.get_module_ids() + assert manager.get_running_module("NavidromeModule") is None def test_navidrome_module_ignores_non_music_media(): diff --git a/tests/test_singleton_thread_safety.py b/tests/test_singleton_thread_safety.py new file mode 100644 index 000000000..1d5ef2074 --- /dev/null +++ b/tests/test_singleton_thread_safety.py @@ -0,0 +1,41 @@ +import threading + +from app.foundation.singleton import Singleton, SingletonClass + + +def _construct_concurrently(singleton_type, count: int = 16): + """让多个线程同时越过起跑线，放大首次构造竞态。""" + barrier = threading.Barrier(count) + instances = [] + + def construct() -> None: + barrier.wait() + instances.append(singleton_type()) + + threads = [threading.Thread(target=construct) for _ in range(count)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=5) + assert not thread.is_alive() + return instances + + +def test_parameterized_singleton_first_construction_is_single_flight(): + """同一参数的并发首次构造只能发布一个完整实例。""" + class _ParameterizedSingleton(metaclass=Singleton): + pass + + instances = _construct_concurrently(_ParameterizedSingleton) + + assert len({id(instance) for instance in instances}) == 1 + + +def test_class_singleton_first_construction_is_single_flight(): + """按类单例的并发首次构造只能发布一个完整实例。""" + class _ClassSingleton(metaclass=SingletonClass): + pass + + instances = _construct_concurrently(_ClassSingleton) + + assert len({id(instance) for instance in instances}) == 1 diff --git a/tests/test_system_i18n.py b/tests/test_system_i18n.py index b3b7db8fc..2f4237356 100644 --- a/tests/test_system_i18n.py +++ b/tests/test_system_i18n.py @@ -1,24 +1,18 @@ from unittest.mock import patch +from types import SimpleNamespace from app.api.endpoints import system as system_endpoint from app.runtime.localization import LocaleHelper -class _FakeDoubanModule: - """构造带中文名称的模块类，模拟真实 DoubanModule。""" - - @staticmethod - def get_name() -> str: - """获取模块中文名称""" - return "豆瓣" - - class _FakeModuleManager: """提供 system 模块接口测试所需的最小模块管理器。""" - def get_modules(self) -> dict: - """返回模块字典""" - return {"DoubanModule": _FakeDoubanModule} + def list_specs(self) -> tuple: + """返回 manifest 元数据视图。""" + return ( + SimpleNamespace(id="DoubanModule", metadata={"name": "豆瓣"}), + ) def test(self, moduleid: str) -> tuple[bool, str]: """返回模块测试结果"""