GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-04 23:17:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: close plugin shutdown mutation races

Browse Source
This commit is contained in:
jxxghp
2026-08-23 21:55:22 +08:00
parent eb36e6be91
commit 820582ab12
16 changed files with 904 additions and 399 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/fixtures/architecture/dependency-baseline.json
Vendored
+13 -2
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6514,
"edge_sha256": "eda9bc500158baa80dc5013eb4e409aa57cb70446e59a52b7e782cb9c58f9c34",
"edge_count": 6525,
"edge_sha256": "72b9416b5309bb51768b95b3c9da4b167242c3bc3f27d4e561a3f097267ca6a0",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -281,6 +281,7 @@
"app.agent.middleware.skills -> app.agent.middleware",
"app.agent.middleware.skills -> app.agent.middleware.utils",
"app.agent.middleware.skills -> app.agent.policy",
"app.agent.middleware.skills -> app.agent.policy.sanitizer",
"app.agent.middleware.skills -> app.agent.skills",
"app.agent.middleware.skills -> app.agent.skills.metadata",
"app.agent.middleware.skills -> app.agent.tools",
@@ -309,6 +310,7 @@
"app.agent.middleware.summarization -> app.runtime.log",
"app.agent.middleware.tool_selection -> app.agent",
"app.agent.middleware.tool_selection -> app.agent.llm",
"app.agent.middleware.tool_selection -> app.agent.llm.helper",
"app.agent.middleware.tool_selection -> app.agent.tools",
"app.agent.middleware.tool_selection -> app.agent.tools.tags",
"app.agent.middleware.tool_selection -> app.runtime",
@@ -435,6 +437,7 @@
"app.agent.tools.base -> app.schemas.types",
"app.agent.tools.catalog -> app.agent",
"app.agent.tools.catalog -> app.agent.policy",
"app.agent.tools.catalog -> app.agent.policy.contracts",
"app.agent.tools.factory -> app.agent",
"app.agent.tools.factory -> app.agent.llm",
"app.agent.tools.factory -> app.agent.llm.capability",
@@ -883,6 +886,7 @@
"app.agent.tools.impl.query_doctor_report -> app.agent.tools.base",
"app.agent.tools.impl.query_doctor_report -> app.agent.tools.tags",
"app.agent.tools.impl.query_doctor_report -> app.doctor",
"app.agent.tools.impl.query_doctor_report -> app.doctor.runner",
"app.agent.tools.impl.query_doctor_report -> app.runtime",
"app.agent.tools.impl.query_doctor_report -> app.runtime.log",
"app.agent.tools.impl.query_download_tasks -> app.agent",
@@ -1471,6 +1475,8 @@
"app.agent.tools.impl.update_system_settings -> app.agent.tools.tags",
"app.agent.tools.impl.update_system_settings -> app.application",
"app.agent.tools.impl.update_system_settings -> app.application.configuration",
"app.agent.tools.impl.update_system_settings -> app.application.plugin",
"app.agent.tools.impl.update_system_settings -> app.application.plugin.runtime",
"app.agent.tools.impl.update_system_settings -> app.runtime",
"app.agent.tools.impl.update_system_settings -> app.runtime.events",
"app.agent.tools.impl.update_system_settings -> app.runtime.log",
@@ -2261,6 +2267,8 @@
"app.api.endpoints.system -> app.application.messaging.message",
"app.api.endpoints.system -> app.application.module",
"app.api.endpoints.system -> app.application.network",
"app.api.endpoints.system -> app.application.plugin",
"app.api.endpoints.system -> app.application.plugin.runtime",
"app.api.endpoints.system -> app.application.rules",
"app.api.endpoints.system -> app.application.scheduling",
"app.api.endpoints.system -> app.application.security",
@@ -2287,6 +2295,7 @@
"app.api.endpoints.system -> app.schemas",
"app.api.endpoints.system -> app.schemas.common",
"app.api.endpoints.system -> app.schemas.event",
"app.api.endpoints.system -> app.schemas.exception",
"app.api.endpoints.system -> app.schemas.response",
"app.api.endpoints.system -> app.schemas.system",
"app.api.endpoints.system -> app.schemas.token",
@@ -2688,6 +2697,8 @@
"app.application.plugin.install -> app.runtime.log",
"app.application.plugin.install -> app.schemas",
"app.application.plugin.install -> app.schemas.exception",
"app.application.plugin.runtime -> app.schemas",
"app.application.plugin.runtime -> app.schemas.types",
"app.application.recognition -> app.application",
"app.application.recognition -> app.application.configuration",
"app.application.recognition -> app.schemas",
tests/fixtures/architecture/startup-performance-baseline.json
Vendored
+118 -116
View File
@@ -1,41 +1,41 @@
{
"schema_version": 2,
"generated_at": "2026-08-23T11:06:42.858188+00:00",
"generated_at": "2026-08-23T13:47:08.442528+00:00",
"platform": "macOS-26.5.2-arm64-arm-64bit-Mach-O",
"python": "3.14.3",
"repeat": 3,
"targets": {
"app.startup.lifecycle": {
"loaded_app_module_count": 362,
"max_ms": 1022.231,
"median_ms": 976.39,
"min_ms": 967.395,
"loaded_app_module_count": 367,
"max_ms": 1591.175,
"median_ms": 1049.37,
"min_ms": 1022.351,
"samples_ms": [
1022.231,
967.395,
976.39
1591.175,
1049.37,
1022.351
]
},
"app.factory": {
"loaded_app_module_count": 374,
"max_ms": 989.21,
"median_ms": 980.605,
"min_ms": 951.564,
"loaded_app_module_count": 379,
"max_ms": 1198.166,
"median_ms": 1083.805,
"min_ms": 1009.309,
"samples_ms": [
980.605,
989.21,
951.564
1009.309,
1198.166,
1083.805
]
},
"app.main": {
"loaded_app_module_count": 376,
"max_ms": 1114.04,
"median_ms": 1088.251,
"min_ms": 1080.26,
"loaded_app_module_count": 381,
"max_ms": 1229.477,
"median_ms": 1179.859,
"min_ms": 1129.055,
"samples_ms": [
1080.26,
1088.251,
1114.04
1129.055,
1179.859,
1229.477
]
}
},
@@ -46,26 +46,26 @@
"samples": [
{
"mode": "normal",
"enabled_component_count": 21,
"startup_ms": 0.619,
"full_lifespan_ms": 0.806,
"enabled_component_count": 23,
"startup_ms": 0.64,
"full_lifespan_ms": 0.805,
"stage_ms": {
"后台任务登记器": 0.075,
"数据库准备": 0.037,
"HTTP 基础能力": 0.031,
"后台任务登记器": 0.079,
"数据库准备": 0.039,
"HTTP 基础能力": 0.029,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.025,
"路由": 0.022,
"模块服务": 0.024,
"插件备份恢复": 0.021,
"插件": 0.02,
"定时器": 0.023,
"监控器": 0.021,
"待处理整理回放": 0.019,
"命令服务": 0.022,
"工作流": 0.022,
"插件同步与启动收尾": 0.034
"数据库连接预算": 0.023,
"路由": 0.023,
"模块服务": 0.021,
"插件备份恢复": 0.024,
"插件": 0.021,
"定时器": 0.026,
"监控器": 0.022,
"待处理整理回放": 0.025,
"命令服务": 0.025,
"工作流": 0.021,
"插件同步与启动收尾": 0.037
},
"threads_before": 2,
"threads_started": 2,
@@ -77,26 +77,26 @@
},
{
"mode": "normal",
"enabled_component_count": 21,
"startup_ms": 0.628,
"enabled_component_count": 23,
"startup_ms": 0.652,
"full_lifespan_ms": 0.808,
"stage_ms": {
"后台任务登记器": 0.073,
"数据库准备": 0.037,
"HTTP 基础能力": 0.039,
"领域依赖装配": 0.027,
"数据库引擎预热": 0.025,
"数据库连接预算": 0.025,
"后台任务登记器": 0.079,
"数据库准备": 0.039,
"HTTP 基础能力": 0.033,
"领域依赖装配": 0.029,
"数据库引擎预热": 0.026,
"数据库连接预算": 0.021,
"路由": 0.022,
"模块服务": 0.022,
"模块服务": 0.023,
"插件备份恢复": 0.021,
"插件": 0.02,
"定时器": 0.024,
"监控器": 0.022,
"待处理整理回放": 0.029,
"命令服务": 0.02,
"工作流": 0.021,
"插件同步与启动收尾": 0.032
"插件": 0.022,
"定时器": 0.026,
"监控器": 0.023,
"待处理整理回放": 0.024,
"命令服务": 0.023,
"工作流": 0.025,
"插件同步与启动收尾": 0.035
},
"threads_before": 2,
"threads_started": 2,
@@ -108,26 +108,26 @@
},
{
"mode": "normal",
"enabled_component_count": 21,
"startup_ms": 0.664,
"full_lifespan_ms": 0.863,
"enabled_component_count": 23,
"startup_ms": 0.739,
"full_lifespan_ms": 0.944,
"stage_ms": {
"后台任务登记器": 0.083,
"数据库准备": 0.047,
"HTTP 基础能力": 0.036,
"后台任务登记器": 0.087,
"数据库准备": 0.037,
"HTTP 基础能力": 0.029,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.029,
"数据库连接预算": 0.025,
"路由": 0.026,
"模块服务": 0.025,
"插件备份恢复": 0.025,
"插件": 0.028,
"定时器": 0.026,
"监控器": 0.021,
"待处理整理回放": 0.021,
"命令服务": 0.024,
"工作流": 0.021,
"插件同步与启动收尾": 0.036
"数据库引擎预热": 0.024,
"数据库连接预算": 0.021,
"路由": 0.02,
"模块服务": 0.02,
"插件备份恢复": 0.024,
"插件": 0.019,
"定时器": 0.025,
"监控器": 0.054,
"待处理整理回放": 0.041,
"命令服务": 0.034,
"工作流": 0.025,
"插件同步与启动收尾": 0.037
},
"threads_before": 2,
"threads_started": 2,
@@ -138,9 +138,9 @@
"database_connections_started": 0
}
],
"median_startup_ms": 0.628,
"median_startup_ms": 0.652,
"median_full_lifespan_ms": 0.808,
"enabled_component_count": 21,
"enabled_component_count": 23,
"enabled_components": [
"后台任务登记器",
"数据库准备",
@@ -157,6 +157,8 @@
"监控器",
"整理后台服务",
"AI智能体会话",
"插件事件入口",
"事件尾任务结算",
"插件后台服务",
"事件投递屏障",
"待处理整理回放",
@@ -170,42 +172,18 @@
{
"mode": "safe",
"enabled_component_count": 11,
"startup_ms": 0.464,
"full_lifespan_ms": 0.637,
"startup_ms": 0.475,
"full_lifespan_ms": 0.619,
"stage_ms": {
"后台任务登记器": 0.074,
"后台任务登记器": 0.079,
"数据库准备": 0.038,
"HTTP 基础能力": 0.03,
"领域依赖装配": 0.029,
"数据库引擎预热": 0.026,
"数据库连接预算": 0.025,
"路由": 0.024,
"模块服务": 0.024,
"插件同步与启动收尾": 0.033
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "safe",
"enabled_component_count": 11,
"startup_ms": 0.451,
"full_lifespan_ms": 0.624,
"stage_ms": {
"后台任务登记器": 0.073,
"数据库准备": 0.035,
"HTTP 基础能力": 0.028,
"领域依赖装配": 0.027,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.024,
"数据库连接预算": 0.023,
"路由": 0.023,
"模块服务": 0.02,
"插件同步与启动收尾": 0.031
"模块服务": 0.022,
"插件同步与启动收尾": 0.036
},
"threads_before": 2,
"threads_started": 2,
@@ -218,18 +196,42 @@
{
"mode": "safe",
"enabled_component_count": 11,
"startup_ms": 0.477,
"full_lifespan_ms": 0.661,
"startup_ms": 0.493,
"full_lifespan_ms": 0.676,
"stage_ms": {
"后台任务登记器": 0.081,
"数据库准备": 0.05,
"后台任务登记器": 0.089,
"数据库准备": 0.037,
"HTTP 基础能力": 0.035,
"领域依赖装配": 0.03,
"数据库引擎预热": 0.026,
"数据库连接预算": 0.024,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.025,
"数据库连接预算": 0.021,
"路由": 0.025,
"模块服务": 0.023,
"插件同步与启动收尾": 0.062
},
"threads_before": 2,
"threads_started": 2,
"threads_after": 2,
"tasks_before": 1,
"tasks_started": 2,
"tasks_after": 1,
"database_connections_started": 0
},
{
"mode": "safe",
"enabled_component_count": 11,
"startup_ms": 0.464,
"full_lifespan_ms": 0.638,
"stage_ms": {
"后台任务登记器": 0.075,
"数据库准备": 0.036,
"HTTP 基础能力": 0.028,
"领域依赖装配": 0.028,
"数据库引擎预热": 0.023,
"数据库连接预算": 0.022,
"路由": 0.024,
"模块服务": 0.02,
"插件同步与启动收尾": 0.026
"模块服务": 0.021,
"插件同步与启动收尾": 0.057
},
"threads_before": 2,
"threads_started": 2,
@@ -240,8 +242,8 @@
"database_connections_started": 0
}
],
"median_startup_ms": 0.464,
"median_full_lifespan_ms": 0.637,
"median_startup_ms": 0.475,
"median_full_lifespan_ms": 0.638,
"enabled_component_count": 11,
"enabled_components": [
"后台任务登记器",
tests/test_lifecycle_shutdown.py
+108 -154
View File
@@ -69,6 +69,8 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
"stop_agent": AsyncMock(return_value=True),
"stop_transfer": AsyncMock(return_value=True),
"quiesce_plugins": AsyncMock(return_value=True),
"settle_events": AsyncMock(return_value=True),
"quiesce_plugin_services": AsyncMock(return_value=True),
"drain_events": AsyncMock(return_value=True),
"finalize_plugins": MagicMock(return_value=True),
"stop_modules": AsyncMock(),
@@ -94,6 +96,16 @@ def _patch_lifespan(monkeypatch, *, failing_step: str | None = None) -> dict:
"quiesce_plugins",
shutdown_steps["quiesce_plugins"],
)
monkeypatch.setattr(
lifecycle,
"settle_events",
shutdown_steps["settle_events"],
)
monkeypatch.setattr(
lifecycle,
"quiesce_plugin_services",
shutdown_steps["quiesce_plugin_services"],
)
monkeypatch.setattr(lifecycle, "drain_events", shutdown_steps["drain_events"])
monkeypatch.setattr(lifecycle, "stop_modules", shutdown_steps["stop_modules"])
monkeypatch.setattr(
@@ -191,166 +203,83 @@ def test_lifespan_validation_failure_does_not_clear_outer_loop_owner(monkeypatch
lifecycle.global_vars.clear_loop.assert_not_called()
def test_lifespan_settles_plugin_handlers_before_legacy_hooks(monkeypatch) -> None:
"""整理尾事件必须在 handler 停用后结算，并先于旧插件停机 hook。"""
shutdown_steps = _patch_lifespan(monkeypatch)
order: list[str] = []
for name in (
"stop_transfer",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
):
shutdown_steps[name].side_effect = (
lambda current=name: order.append(current) or True
)
async def run_lifespan() -> None:
"""运行一个完整的隔离生命周期。"""
async with lifecycle.lifespan(FastAPI()):
pass
asyncio.run(run_lifespan())
assert order == [
"stop_transfer",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
]
_ORDERED_SHUTDOWN_STEPS = (
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"stop_transfer",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
)
@pytest.mark.parametrize(
("failing_step", "completed_steps", "blocked_steps"),
[
(
"stop_plugin_monitor",
("stop_plugin_monitor",),
(
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_monitor",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
),
(
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_scheduler",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
),
(
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_agent",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
),
(
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"quiesce_plugins",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
),
(
"stop_transfer",
"drain_events",
"finalize_plugins",
"stop_modules",
"close_http",
),
),
(
"stop_transfer",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
),
("drain_events", "finalize_plugins", "stop_modules", "close_http"),
),
(
"drain_events",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
),
("finalize_plugins", "stop_modules", "close_http"),
),
(
"finalize_plugins",
(
"stop_plugin_monitor",
"backup_plugins",
"stop_workflow",
"stop_command",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"drain_events",
"finalize_plugins",
),
("stop_modules", "close_http"),
),
],
"failing_step",
(
"stop_plugin_monitor",
"stop_monitor",
"stop_scheduler",
"stop_agent",
"stop_transfer",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
),
)
def test_lifespan_stops_releasing_dependencies_when_owner_does_not_converge(
monkeypatch,
failing_step,
completed_steps,
blocked_steps,
):
"""关键 owner 未收敛时不得关闭仍被活任务使用的后续依赖。"""
shutdown_steps = _patch_lifespan(monkeypatch)
shutdown_steps[failing_step].return_value = False
failed_index = _ORDERED_SHUTDOWN_STEPS.index(failing_step)
completed_steps = _ORDERED_SHUTDOWN_STEPS[: failed_index + 1]
blocked_steps = _ORDERED_SHUTDOWN_STEPS[failed_index + 1 :]
async def run_lifespan():
"""启动并关闭隔离后的应用生命周期。"""
@@ -520,6 +449,8 @@ def test_lifespan_safe_mode_skips_optional_runtime(monkeypatch):
"stop_scheduler",
"stop_plugin_monitor",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"finalize_plugins",
):
shutdown_steps[name].assert_not_called()
@@ -543,6 +474,19 @@ async def test_event_drain_does_not_materialize_manager(monkeypatch) -> None:
event_manager_type.assert_not_called()
@pytest.mark.asyncio
async def test_event_settlement_keeps_tail_event_admission_open(monkeypatch) -> None:
"""中间结算只等待在途 handler，不得提前封死旧 hook 的尾事件。"""
event_manager = MagicMock()
event_manager.drain_async = AsyncMock(return_value=True)
event_manager_type = MagicMock()
event_manager_type.get_existing_instance.return_value = event_manager
monkeypatch.setattr(modules_initializer, "EventManager", event_manager_type)
assert await modules_initializer.settle_events() is True
event_manager.drain_async.assert_awaited_once_with(seal=False)
def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"""组件清单应显式冻结依赖、模式、启动/关闭顺序和超时预算。"""
app = FastAPI()
@@ -591,8 +535,10 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"监控器",
"定时器",
"AI智能体会话",
"插件后台服务",
"整理后台服务",
"插件事件入口",
"事件尾任务结算",
"插件后台服务",
"事件投递屏障",
"插件",
"模块服务",
@@ -623,6 +569,8 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"定时器",
"AI智能体会话",
"整理后台服务",
"插件事件入口",
"事件尾任务结算",
"插件后台服务",
"事件投递屏障",
"插件",
@@ -638,6 +586,8 @@ def test_lifecycle_manifest_declares_normal_and_safe_mode_order() -> None:
"定时器",
"AI智能体会话",
"整理后台服务",
"插件事件入口",
"事件尾任务结算",
"插件后台服务",
"事件投递屏障",
"插件",
@@ -869,8 +819,10 @@ def test_lifespan_cleans_started_owners_after_late_startup_failure(monkeypatch):
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
"stop_modules",
@@ -905,11 +857,13 @@ def test_startup_failure_cleanup_honors_transfer_fail_fast(monkeypatch):
"stop_monitor",
"stop_scheduler",
"stop_agent",
"quiesce_plugins",
"stop_transfer",
):
_assert_completed_once(shutdown_steps[name])
for name in (
"quiesce_plugins",
"settle_events",
"quiesce_plugin_services",
"drain_events",
"finalize_plugins",
"stop_modules",
tests/test_plugin_install_command.py
+97
View File
@@ -477,6 +477,103 @@ async def test_cancelled_install_waits_for_rollback_before_releasing_lifecycle()
rollback.assert_awaited_once()
@pytest.mark.asyncio
async def test_repeated_checkpoint_cancellation_retains_mutation_owner() -> None:
"""快照等待被连续取消时，lease 必须保留到快照子任务终态。"""
admission = PluginMutationAdmission()
checkpoint_started = asyncio.Event()
checkpoint_release = asyncio.Event()
checkpoint_finished = asyncio.Event()
async def checkpoint(_plugin_id: str) -> object:
"""阻塞快照创建，直到测试确认 owner 仍被持有。"""
checkpoint_started.set()
await checkpoint_release.wait()
checkpoint_finished.set()
return object()
task = asyncio.create_task(
_command(
checkpointer=checkpoint,
mutation=admission.hold,
).execute(
plugin_id="DemoPlugin",
repo_url="https://github.com/demo/plugins",
)
)
await checkpoint_started.wait()
task.cancel()
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0)
assert task.done() is False
assert admission.seal() == 1
idle_waiter = asyncio.create_task(asyncio.to_thread(admission.wait_until_idle))
await asyncio.sleep(0.02)
assert idle_waiter.done() is False
checkpoint_release.set()
with pytest.raises(asyncio.CancelledError):
await task
await idle_waiter
assert checkpoint_finished.is_set()
assert admission.active_count == 0
@pytest.mark.asyncio
async def test_repeated_rollback_cancellation_retains_mutation_owner() -> None:
"""补偿等待被再次连续取消时，lease 必须保留到补偿子任务终态。"""
admission = PluginMutationAdmission()
install_started = asyncio.Event()
rollback_started = asyncio.Event()
rollback_release = asyncio.Event()
rollback_finished = asyncio.Event()
async def install(*_args) -> tuple[bool, str]:
"""阻塞包安装，使首次取消进入补偿路径。"""
install_started.set()
await asyncio.Event().wait()
return True, "ok"
async def rollback(_checkpoint: object) -> None:
"""阻塞文件补偿，直到测试确认 owner 仍被持有。"""
rollback_started.set()
await rollback_release.wait()
rollback_finished.set()
task = asyncio.create_task(
_command(
installer=install,
rollback=rollback,
mutation=admission.hold,
).execute(
plugin_id="DemoPlugin",
repo_url="https://github.com/demo/plugins",
)
)
await install_started.wait()
task.cancel()
await rollback_started.wait()
assert admission.seal() == 1
idle_waiter = asyncio.create_task(asyncio.to_thread(admission.wait_until_idle))
task.cancel()
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0.02)
assert task.done() is False
assert idle_waiter.done() is False
assert admission.active_count == 1
rollback_release.set()
with pytest.raises(asyncio.CancelledError):
await task
await idle_waiter
assert rollback_finished.is_set()
assert admission.active_count == 0
@pytest.mark.asyncio
async def test_cancelled_persisted_list_is_restored_conservatively() -> None:
"""清单写入已产生副作用但尚未返回时取消，也必须恢复原清单。"""
tests/test_plugin_lifecycle_status.py
+48 -5
View File
@@ -100,8 +100,8 @@ def test_lifecycle_records_load_failure_when_loader_returns_no_class():
assert statuses["DemoPlugin"] is PluginRuntimeStatus.LOAD_FAILED
def test_quiesce_keeps_instance_and_events_until_finalize():
"""第一阶段仅停止插件生产者，事件 handler 和实例留到屏障后释放。"""
def test_phased_quiesce_disables_events_before_hooks_and_keeps_instance():
"""宿主先停用 handler，再在事件结算后执行旧 hook 并保留实例。"""
order: list[str] = []
class DemoPlugin:
@@ -129,14 +129,19 @@ def test_quiesce_keeps_instance_and_events_until_finalize():
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
lifecycle._disable_events.reset_mock()
lifecycle._disable_events.side_effect = (
lambda _plugin_type: order.append("disable_events")
)
lifecycle._clear_modules.reset_mock()
lifecycle._clear_tools.reset_mock()
assert lifecycle.quiesce() is True
assert order == ["close", "stop_service"]
assert lifecycle.quiesce_handlers() is True
assert order == ["disable_events"]
assert lifecycle.quiesce_services() is True
assert order == ["disable_events", "close", "stop_service"]
assert classes["DemoPlugin"] is DemoPlugin
assert isinstance(running["DemoPlugin"], DemoPlugin)
lifecycle._disable_events.assert_not_called()
lifecycle._disable_events.assert_called_once_with(DemoPlugin)
lifecycle._clear_modules.assert_not_called()
lifecycle._clear_tools.assert_not_called()
@@ -194,6 +199,44 @@ def test_quiesce_runs_stop_service_after_close_failure_and_retries_missing_hook(
assert running == {}
def test_quiesce_does_not_close_resources_until_all_handlers_are_disabled():
"""任一 handler 停用失败时不得调用可能破坏共享资源的旧停机 hook。"""
close = MagicMock()
class DemoPlugin:
"""提供可观察 close hook 的测试插件。"""
plugin_name = "演示插件"
plugin_version = "1.0.0"
def init_plugin(self, _config):
"""接受宿主初始化配置。"""
@staticmethod
def get_state():
"""保持插件事件 handler 启用。"""
return True
def close(self):
"""记录资源关闭调用。"""
close()
lifecycle, classes, running, _statuses = _lifecycle(plugins=[DemoPlugin])
lifecycle.start("DemoPlugin")
lifecycle._disable_events.side_effect = RuntimeError("disable failed")
assert lifecycle.quiesce() is False
close.assert_not_called()
assert lifecycle.finalize() is False
assert "DemoPlugin" in classes
assert "DemoPlugin" in running
lifecycle._disable_events.side_effect = None
assert lifecycle.quiesce() is True
close.assert_called_once_with()
assert lifecycle.finalize() is True
def test_legacy_stop_entry_remains_idempotent():
"""旧 stop 先解绑事件、保持 None 返回，且重复调用不重复执行 hook。"""
order: list[str] = []
tests/test_plugin_monitor_lifecycle.py
+11 -6
View File
@@ -582,11 +582,12 @@ def test_stop_plugin_monitor_returns_existing_manager_result(monkeypatch) -> Non
@pytest.mark.asyncio
async def test_two_phase_plugin_shutdown_does_not_materialize_manager() -> None:
"""两阶段入口在插件管理器尚未创建时都直接视为已收敛。"""
async def test_phased_plugin_shutdown_does_not_materialize_manager() -> None:
"""三个停机入口在插件管理器尚未创建时都直接视为已收敛。"""
_reset_plugin_manager()
assert await plugins_initializer.quiesce_plugins(timeout=0) is True
assert await plugins_initializer.quiesce_plugin_services(timeout=0) is True
assert plugins_initializer.finalize_plugins() is True
assert PluginManager.get_existing_instance() is None
@@ -616,7 +617,10 @@ async def test_quiesce_timeout_retains_future_owner_until_worker_finishes(
release.wait(timeout=2)
return True
manager._plugin_lifecycle.quiesce = MagicMock(side_effect=blocking_quiesce)
manager._plugin_lifecycle.quiesce_handlers = MagicMock(return_value=True)
manager._plugin_lifecycle.quiesce_services = MagicMock(
side_effect=blocking_quiesce,
)
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
try:
@@ -627,9 +631,10 @@ async def test_quiesce_timeout_retains_future_owner_until_worker_finishes(
lambda: thread_helper,
)
assert await manager.quiesce_plugins(timeout=0.01) is False
assert await manager.quiesce_plugins(timeout=1) is True
assert await manager.quiesce_plugin_services(timeout=0.01) is False
assert started.is_set()
owner = manager._plugin_quiesce_future
owner = manager._plugin_service_quiesce_future
assert owner is not None
assert owner.done() is False
assert manager.finalize_plugins() is False
@@ -659,7 +664,7 @@ async def test_quiesce_seals_runtime_until_new_lifespan_reopens(monkeypatch) ->
),
)
manager = PluginManager()
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
manager._plugin_lifecycle.quiesce_handlers = MagicMock(return_value=True)
manager._plugin_lifecycle.start = MagicMock(
return_value={"DemoPlugin": PluginRuntimeStatus.ACTIVE}
)
tests/test_plugin_mutation_admission.py
+78 -1
View File
@@ -1,6 +1,7 @@
"""插件可变事务停机准入的确定性测试。"""
import asyncio
import threading
from collections.abc import Iterator
from concurrent.futures import ThreadPoolExecutor
from contextvars import copy_context
@@ -20,6 +21,42 @@ from app.schemas.plugin import PluginRuntimeStatus
from app.schemas.types import EventType
class _GatedCondition:
"""让指定测试线程在取得真实 Condition 前停住。"""
def __init__(self, blocked_thread_prefix: str) -> None:
"""初始化内部 Condition 和可控的竞态闸门。"""
self._condition = threading.Condition()
self._blocked_thread_prefix = blocked_thread_prefix
self._blocked = False
self.attempted = threading.Event()
self.release = threading.Event()
def __enter__(self):
"""在目标线程首次进入时等待测试放行。"""
if (
threading.current_thread().name.startswith(self._blocked_thread_prefix)
and not self._blocked
):
self._blocked = True
self.attempted.set()
if not self.release.wait(timeout=1):
raise TimeoutError("测试未及时放行 Condition")
return self._condition.__enter__()
def __exit__(self, exc_type, exc_value, traceback):
"""把上下文退出委托给内部 Condition。"""
return self._condition.__exit__(exc_type, exc_value, traceback)
def wait(self) -> bool:
"""等待 admission 活动计数变化。"""
return self._condition.wait()
def notify_all(self) -> None:
"""唤醒全部等待 admission 空闲的线程。"""
self._condition.notify_all()
@pytest.fixture
def plugin_manager() -> Iterator[PluginManager]:
"""构造隔离的插件管理器，并在用例结束后清除单例状态。"""
@@ -60,6 +97,46 @@ def test_seal_rejects_new_root_but_allows_propagated_nested_lease() -> None:
assert admission.reopen() is True
def test_stale_propagated_context_is_rechecked_atomically_after_seal() -> None:
"""外层退出后才取得 Condition 的复制上下文不得冒充嵌套事务。"""
admission = PluginMutationAdmission()
calls: list[str] = []
outer = admission.hold("外层事务")
outer.__enter__()
propagated = copy_context()
condition = _GatedCondition("stale-admission")
admission._condition = condition
outer_closed = False
def mutate() -> None:
"""使用复制上下文尝试执行延迟到封口后的写入。"""
with admission.hold("延迟嵌套事务"):
calls.append("mutated")
try:
with ThreadPoolExecutor(
max_workers=1,
thread_name_prefix="stale-admission",
) as executor:
future = executor.submit(propagated.run, mutate)
assert condition.attempted.wait(timeout=1)
outer.__exit__(None, None, None)
outer_closed = True
assert admission.seal() == 0
admission.wait_until_idle()
condition.release.set()
with pytest.raises(PluginMutationRejectedError):
future.result(timeout=1)
finally:
condition.release.set()
if not outer_closed:
outer.__exit__(None, None, None)
assert calls == []
assert admission.active_count == 0
@pytest.mark.asyncio
async def test_quiesce_timeout_retains_admitted_owner_and_nested_reload(
plugin_manager: PluginManager,
@@ -72,7 +149,7 @@ async def test_quiesce_timeout_retains_admitted_owner_and_nested_reload(
manager._plugin_lifecycle.reload = MagicMock(
return_value=PluginRuntimeStatus.ACTIVE
)
manager._plugin_lifecycle.quiesce = MagicMock(return_value=True)
manager._plugin_lifecycle.quiesce_handlers = MagicMock(return_value=True)
manager._plugin_lifecycle.finalize = MagicMock(return_value=True)
async def mutate() -> PluginRuntimeStatus:
tests/test_plugin_system_setting_admission.py
+263
View File
@@ -0,0 +1,263 @@
"""通用系统设置入口的插件 mutation 准入测试。"""
import asyncio
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.agent.tools.impl.update_system_settings import UpdateSystemSettingsTool
from app.api.endpoints import system as system_endpoint
from app.application.plugin import runtime as plugin_runtime
from app.runtime.extensions.plugin.admission import PluginMutationAdmission
from app.schemas.types import SystemConfigKey
PLUGIN_RUNTIME_KEYS = (
SystemConfigKey.UserInstalledPlugins,
SystemConfigKey.PluginInstances,
SystemConfigKey.PluginFolders,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("config_key", PLUGIN_RUNTIME_KEYS)
async def test_system_http_rejects_plugin_keys_after_admission_seal(
config_key: SystemConfigKey,
monkeypatch,
) -> None:
"""HTTP 通用设置入口在封口后不得写入插件运行态配置。"""
admission = PluginMutationAdmission()
admission.seal()
config = MagicMock()
config.async_set = AsyncMock()
monkeypatch.setattr(
plugin_runtime,
"get_plugin_manager",
lambda: SimpleNamespace(mutation=admission.hold),
)
monkeypatch.setattr(
system_endpoint,
"get_runtime_settings",
lambda: SimpleNamespace(contains=lambda _key: False),
)
monkeypatch.setattr(
system_endpoint,
"get_configured_system_config",
lambda: config,
)
response = await system_endpoint.set_setting(config_key.value, {}, None)
assert response.success is False
assert "停机阶段" in response.message
config.async_set.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("config_key", PLUGIN_RUNTIME_KEYS)
async def test_agent_rejects_plugin_keys_after_admission_seal(
config_key: SystemConfigKey,
monkeypatch,
) -> None:
"""Agent 通用设置入口在封口后不得读取或写入插件运行态配置。"""
admission = PluginMutationAdmission()
admission.seal()
config = MagicMock()
config.async_set = AsyncMock()
monkeypatch.setattr(
plugin_runtime,
"get_plugin_manager",
lambda: SimpleNamespace(mutation=admission.hold),
)
tool = UpdateSystemSettingsTool(
session_id="session-1",
user_id="10001",
system_config=config,
)
payload = json.loads(await tool.run(setting_key=config_key.value, value={}))
assert payload["success"] is False
assert "停机阶段" in payload["message"]
config.get.assert_not_called()
config.async_set.assert_not_awaited()
@pytest.mark.asyncio
async def test_system_http_plugin_write_remains_owned_until_settled(
monkeypatch,
) -> None:
"""HTTP 在途插件配置写入完成前，封口等待不得误判为空闲。"""
admission = PluginMutationAdmission()
write_started = asyncio.Event()
write_release = asyncio.Event()
async def async_set(_key: str, _value: object) -> bool:
"""阻塞配置写入，暴露 quiesce 等待窗口。"""
write_started.set()
await write_release.wait()
return True
config = MagicMock()
config.async_set = AsyncMock(side_effect=async_set)
monkeypatch.setattr(
plugin_runtime,
"get_plugin_manager",
lambda: SimpleNamespace(mutation=admission.hold),
)
monkeypatch.setattr(
system_endpoint,
"get_runtime_settings",
lambda: SimpleNamespace(contains=lambda _key: False),
)
monkeypatch.setattr(
system_endpoint,
"get_configured_system_config",
lambda: config,
)
monkeypatch.setattr(system_endpoint.eventmanager, "async_send_event", AsyncMock())
task = asyncio.create_task(
system_endpoint.set_setting(
SystemConfigKey.UserInstalledPlugins.value,
["DemoPlugin"],
None,
)
)
await write_started.wait()
assert admission.seal() == 1
idle_waiter = asyncio.create_task(asyncio.to_thread(admission.wait_until_idle))
await asyncio.sleep(0.02)
assert idle_waiter.done() is False
write_release.set()
response = await task
await idle_waiter
assert response.success is True
assert admission.active_count == 0
@pytest.mark.asyncio
async def test_agent_plugin_write_remains_owned_until_saved_value_read(
monkeypatch,
) -> None:
"""Agent 插件配置事务在写入和结果读取完成前持续持有 lease。"""
admission = PluginMutationAdmission()
write_started = asyncio.Event()
write_release = asyncio.Event()
async def async_set(_key: SystemConfigKey, _value: object) -> bool:
"""阻塞 Agent 配置写入，暴露 quiesce 等待窗口。"""
write_started.set()
await write_release.wait()
return True
config = MagicMock()
config.get.side_effect = [[], ["DemoPlugin"]]
config.async_set = AsyncMock(side_effect=async_set)
monkeypatch.setattr(
plugin_runtime,
"get_plugin_manager",
lambda: SimpleNamespace(mutation=admission.hold),
)
monkeypatch.setattr(
"app.agent.tools.impl.update_system_settings.eventmanager.async_send_event",
AsyncMock(),
)
tool = UpdateSystemSettingsTool(
session_id="session-1",
user_id="10001",
system_config=config,
)
task = asyncio.create_task(
tool.run(
setting_key=SystemConfigKey.UserInstalledPlugins.value,
value=["DemoPlugin"],
)
)
await write_started.wait()
assert admission.seal() == 1
idle_waiter = asyncio.create_task(asyncio.to_thread(admission.wait_until_idle))
await asyncio.sleep(0.02)
assert idle_waiter.done() is False
write_release.set()
payload = json.loads(await task)
await idle_waiter
assert payload["success"] is True
assert payload["saved_value"] == ["DemoPlugin"]
assert admission.active_count == 0
@pytest.mark.asyncio
async def test_non_plugin_system_config_does_not_resolve_plugin_runtime(
monkeypatch,
) -> None:
"""非插件 SystemConfig 写入保持原路径且不依赖插件组合根。"""
config = MagicMock()
config.async_set = AsyncMock(return_value=True)
def fail_runtime_resolution():
"""若非插件配置误取插件运行时则立即暴露回归。"""
raise AssertionError("非插件配置不应解析插件运行时")
monkeypatch.setattr(plugin_runtime, "get_plugin_manager", fail_runtime_resolution)
monkeypatch.setattr(
system_endpoint,
"get_runtime_settings",
lambda: SimpleNamespace(contains=lambda _key: False),
)
monkeypatch.setattr(
system_endpoint,
"get_configured_system_config",
lambda: config,
)
monkeypatch.setattr(system_endpoint.eventmanager, "async_send_event", AsyncMock())
response = await system_endpoint.set_setting(
SystemConfigKey.Directories.value,
[],
None,
)
assert response.success is True
config.async_set.assert_awaited_once_with(SystemConfigKey.Directories.value, None)
@pytest.mark.asyncio
async def test_agent_non_plugin_config_does_not_resolve_plugin_runtime(
monkeypatch,
) -> None:
"""Agent 非插件配置写入保持既有读写和事件语义。"""
config = MagicMock()
config.get.side_effect = [{}, {"chatgpt": {"enabled": True}}]
config.async_set = AsyncMock(return_value=True)
def fail_runtime_resolution():
"""若 Agent 非插件配置误取插件运行时则立即暴露回归。"""
raise AssertionError("非插件配置不应解析插件运行时")
monkeypatch.setattr(plugin_runtime, "get_plugin_manager", fail_runtime_resolution)
monkeypatch.setattr(
"app.agent.tools.impl.update_system_settings.eventmanager.async_send_event",
AsyncMock(),
)
tool = UpdateSystemSettingsTool(
session_id="session-1",
user_id="10001",
system_config=config,
)
payload = json.loads(
await tool.run(
setting_key=SystemConfigKey.AIAgentConfig.value,
value={"chatgpt": {"enabled": True}},
)
)
assert payload["success"] is True
assert payload["changed"] is True
config.async_set.assert_awaited_once()
tests/test_transfer_worker_lifecycle.py
+2
View File
@@ -297,6 +297,8 @@ def test_failed_retry_schedule_registers_future_observer(monkeypatch) -> None:
chain.retry_scheduler = MagicMock(schedule_retry=schedule_retry)
future = MagicMock(spec=Future)
event_loop = MagicMock()
event_loop.is_running.return_value = True
event_loop.is_closed.return_value = False
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", event_loop)
def submit(coroutine, loop):