Merge remote-tracking branch 'origin/v3' into v3

# Conflicts:
#	tests/fixtures/architecture/dependency-baseline.json
This commit is contained in:
jxxghp
2026-08-22 20:08:35 +08:00
14 changed files with 1266 additions and 186 deletions
+12 -1
View File
@@ -1 +1,12 @@
{}
{
"app/agent/tools/impl/_plugin_tool_utils.py:uninstall_plugin_runtime:shutil.rmtree": 1,
"app/agent/tools/impl/scrape_metadata.py:ScrapeMetadataTool.run:Path.exists": 1,
"app/agent/tools/impl/scrape_metadata.py:ScrapeMetadataTool.run:Path.is_dir": 1,
"app/chain/media.py:MediaChain._async_music_album_dir_fallback:Path.exists": 1,
"app/chain/media.py:MediaChain._async_music_album_dir_fallback:Path.is_file": 1,
"app/chain/media.py:MediaChain.async_recognize_music_album_directory:Path.is_dir": 1,
"app/modules/acoustid/__init__.py:AcoustIdModule.async_identify_music_by_fingerprint:Path.is_file": 1,
"app/modules/discord/discord.py:Discord._send_file:Path.exists": 1,
"app/modules/discord/discord.py:Discord._send_file:Path.is_file": 1,
"app/scheduler.py:Scheduler.execute_agent_task:AgentTaskOper.get": 1
}
@@ -9,7 +9,7 @@
"root": "app"
},
"settings_imports": {
"count": 136,
"count": 135,
"files": [
"app/adapters/cache/backends.py",
"app/adapters/cache/redis.py",
@@ -47,7 +47,6 @@
"app/agent/tools/impl/send_voice_message.py",
"app/agent/tools/impl/update_agent_task.py",
"app/agent/tools/impl/update_system_settings.py",
"app/application/formatting.py",
"app/application/maintenance.py",
"app/application/rss.py",
"app/application/security/auth.py",
+7 -3
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6393,
"edge_sha256": "ea375f19071a37a9c72bd01ce9fa4a070e64ad7f14a87d2e5013b5c7440680cd",
"edge_count": 6397,
"edge_sha256": "801b80e33c626aecf70e5320a2c1c81db401549faa652e718be7386bac593f43",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -2494,12 +2494,13 @@
"app.application.downloader -> app.schemas",
"app.application.downloader -> app.schemas.system",
"app.application.downloader -> app.schemas.types",
"app.application.formatting -> app.application",
"app.application.formatting -> app.application.configuration",
"app.application.formatting -> app.domain",
"app.application.formatting -> app.domain.meta",
"app.application.formatting -> app.domain.meta.metabase",
"app.application.formatting -> app.domain.metainfo",
"app.application.formatting -> app.runtime",
"app.application.formatting -> app.runtime.config",
"app.application.formatting -> app.runtime.log",
"app.application.formatting -> app.schemas",
"app.application.formatting -> app.schemas.transfer",
@@ -3663,6 +3664,9 @@
"app.db.session -> app.runtime.observability",
"app.doctor.checks -> app.adapters",
"app.doctor.checks -> app.adapters.system",
"app.doctor.checks -> app.adapters.system.backup",
"app.doctor.checks -> app.adapters.system.backup.database",
"app.doctor.checks -> app.adapters.system.backup.files",
"app.doctor.checks -> app.adapters.system.host",
"app.doctor.checks -> app.doctor",
"app.doctor.checks -> app.doctor.models",
+294 -1
View File
@@ -1,10 +1,33 @@
"""async 阻塞调用 ratchet 与 debug 模式测试。"""
import asyncio
import textwrap
from pathlib import Path
import pytest
from scripts.architecture.async_blocking import SCAN_ROOTS, compare_async_blocking
from scripts.architecture.async_blocking import (
SCAN_ROOTS,
collect_async_blocking,
compare_async_blocking,
)
def _scan_source(
tmp_path: Path,
source: str,
*,
oper_sources: dict[str, str] | None = None,
) -> dict[str, int]:
"""构造最小仓库并通过公开扫描入口验证源码,而非测试内部 AST 细节。"""
api_root = tmp_path / "app/api"
api_root.mkdir(parents=True)
(api_root / "sample.py").write_text(textwrap.dedent(source), encoding="utf-8")
for filename, oper_source in (oper_sources or {}).items():
oper_path = tmp_path / "app/db/oper" / filename
oper_path.parent.mkdir(parents=True, exist_ok=True)
oper_path.write_text(textwrap.dedent(oper_source), encoding="utf-8")
return collect_async_blocking(tmp_path, scan_roots=("app/api",))
def test_async_blocking_scan_covers_runtime_entrypoints() -> None:
@@ -51,6 +74,276 @@ def test_async_blocking_ratchet_allows_removal_and_rejects_growth() -> None:
assert any("新增" in problem for problem in problems)
@pytest.mark.parametrize(
("source", "expected"),
[
(
"""
from app.adapters.network.http import RequestUtils as RU
async def load():
client = RU()
alias = client
return alias.get_res("https://example.com")
""",
{"app/api/sample.py:load:RequestUtils.get_res": 1},
),
(
"""
import app.adapters.network.http as http
async def submit():
return http.RequestUtils().post_res("https://example.com")
""",
{"app/api/sample.py:submit:RequestUtils.post_res": 1},
),
(
"""
from pathlib import Path as P
async def inspect_file():
path = P("payload") / "item.json"
return path.exists()
""",
{"app/api/sample.py:inspect_file:Path.exists": 1},
),
(
"""
import shutil as files
from subprocess import run as run_process
async def cleanup():
files.rmtree("payload")
run_process(["true"])
""",
{
"app/api/sample.py:cleanup:shutil.rmtree": 1,
"app/api/sample.py:cleanup:subprocess.run": 1,
},
),
(
"""
import os as operating_system
import time as clock
from requests import Session as HttpSession
async def legacy_io():
open("payload")
operating_system.listdir(".")
clock.sleep(0.1)
HttpSession().get("https://example.com")
""",
{
"app/api/sample.py:legacy_io:Session.get": 1,
"app/api/sample.py:legacy_io:open": 1,
"app/api/sample.py:legacy_io:os.listdir": 1,
"app/api/sample.py:legacy_io:time.sleep": 1,
},
),
(
"""
from pathlib import Path
async def outer():
async def inner():
return Path("payload").read_text()
return await inner()
""",
{"app/api/sample.py:outer.inner:Path.read_text": 1},
),
(
"""
from pathlib import Path
async def inspect_files(files: list[Path]):
for file in files:
if file.is_file():
return file
return None
""",
{"app/api/sample.py:inspect_files:Path.is_file": 1},
),
],
)
def test_async_blocking_scan_resolves_imports_aliases_and_nested_async(
tmp_path: Path,
source: str,
expected: dict[str, int],
) -> None:
"""别名、简单局部传播和嵌套 async 都必须进入真实扫描结果。"""
assert _scan_source(tmp_path, source) == expected
def test_async_blocking_scan_uses_oper_source_method_kinds(tmp_path: Path) -> None:
"""Oper 仅按源码中真实同步方法报告,不依赖方法名前缀猜测。"""
source = """
from app.db.oper import SiteOper
from app.db.oper.site import SiteOper as SO
async def load(oper: SO):
oper.list()
return await oper.get_by_id(1)
async def load_from_facade():
return SiteOper().list()
"""
oper_sources = {
"site.py": """
class SiteOper:
def list(self):
return []
async def get_by_id(self, site_id):
return None
""",
}
assert _scan_source(tmp_path, source, oper_sources=oper_sources) == {
"app/api/sample.py:load:SiteOper.list": 1,
"app/api/sample.py:load_from_facade:SiteOper.list": 1,
}
def test_async_blocking_scan_exempts_async_apis_and_memory_reads(
tmp_path: Path,
) -> None:
"""异步实现、受控 worker 与内存配置读取不得成为阻塞债务。"""
source = """
import asyncio
import anyio
import subprocess
from anyio import Path as AsyncPath
from app.agent.tools.base import run_agent_blocking
from app.adapters.network.http import AsyncRequestUtils
from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.execution import run_in_threadpool as runtime_worker
from fastapi.concurrency import run_in_threadpool
async def load(config: SystemConfigOper):
await AsyncRequestUtils().get_res("https://example.com")
await AsyncPath("payload").exists()
await run_in_threadpool(lambda: subprocess.run(["true"]))
await runtime_worker(lambda: subprocess.run(["true"]))
await run_agent_blocking("plugin", lambda: subprocess.run(["true"]))
await asyncio.to_thread(lambda: subprocess.run(["true"]))
await anyio.to_thread.run_sync(lambda: subprocess.run(["true"]))
deferred = lambda: subprocess.run(["true"])
assert deferred
return config.get("key")
def ordinary_sync():
subprocess.run(["true"])
"""
oper_sources = {
"systemconfig.py": """
class SystemConfigOper:
def get(self, key):
return key
""",
}
assert _scan_source(tmp_path, source, oper_sources=oper_sources) == {}
def test_async_blocking_scan_checks_worker_arguments_evaluated_on_loop(
tmp_path: Path,
) -> None:
"""worker 调用前同步求值的普通参数仍在事件循环中执行。"""
source = """
import asyncio
from pathlib import Path
async def load():
await asyncio.to_thread(print, Path("payload").read_text())
"""
assert _scan_source(tmp_path, source) == {
"app/api/sample.py:load:Path.read_text": 1,
}
def test_async_blocking_scan_merges_branch_bindings_conservatively(
tmp_path: Path,
) -> None:
"""任一互斥分支可能产生同步对象时,合流调用仍属于阻塞风险。"""
source = """
from anyio import Path as AsyncPath
from pathlib import Path
async def load(use_sync: bool):
if use_sync:
target = Path("payload")
else:
target = AsyncPath("payload")
return target.read_text()
"""
assert _scan_source(tmp_path, source) == {
"app/api/sample.py:load:Path.read_text": 1,
}
def test_nested_async_inherits_bindings_from_definition_scope(tmp_path: Path) -> None:
"""嵌套 async 使用定义点已有的局部 import 和别名。"""
source = """
async def outer():
from pathlib import Path as LocalPath
alias = LocalPath
async def inner():
return alias("payload").read_text()
return await inner()
"""
assert _scan_source(tmp_path, source) == {
"app/api/sample.py:outer.inner:Path.read_text": 1,
}
def test_definition_time_expressions_remain_in_async_execution_body(
tmp_path: Path,
) -> None:
"""延迟函数体不扫描,但默认值和 decorator 在定义时立即求值。"""
source = """
import asyncio
from pathlib import Path
def register(value):
return lambda function: function
async def outer():
await asyncio.to_thread(
lambda value=Path("lambda").read_text(): value
)
@register(Path("decorator").read_text())
async def inner(value=Path("default").read_text()):
return value
return await inner()
"""
assert _scan_source(tmp_path, source) == {
"app/api/sample.py:outer:Path.read_text": 3,
}
def test_local_shadowing_does_not_reuse_import_or_builtin_bindings(
tmp_path: Path,
) -> None:
"""参数和 comprehension target 会遮蔽同名 builtin 或导入符号。"""
source = """
from pathlib import Path
async def invoke(open, items):
open()
return [Path.exists() for Path in items]
"""
assert _scan_source(tmp_path, source) == {}
@pytest.mark.asyncio
async def test_asyncio_debug_is_enabled_for_async_tests() -> None:
"""专项异步测试必须启用慢 callback 和阻塞诊断所需的 debug 模式。"""
+65
View File
@@ -6,12 +6,14 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from app.adapters.system.backup.database import (
PostgreSQLBackupBackend,
SQLiteBackupBackend,
verify_database_backup,
)
@@ -34,6 +36,15 @@ def test_sqlite_backup_includes_committed_wal_data(tmp_path: Path) -> None:
engine.dispose()
def test_sqlite_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""Doctor 等离线入口不应为校验备份而构造活动数据库引擎。"""
artifact = tmp_path / "backup.db"
with sqlite3.connect(artifact) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
assert verify_database_backup(artifact, db_type="sqlite").valid is True
def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None:
source = tmp_path / "user.db"
backup = tmp_path / "backup.db"
@@ -66,6 +77,11 @@ class _Runner:
return subprocess.CompletedProcess(command, 0, stdout, "")
class _FailedRunner:
def __call__(self, command, **_kwargs):
return subprocess.CompletedProcess(command, 1, "", "invalid archive")
def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend:
engine = SimpleNamespace(
url=make_url(
@@ -114,6 +130,55 @@ def test_postgresql_verify_and_restore_use_pg_restore(tmp_path: Path) -> None:
assert restore_kwargs["env"]["PGPASSWORD"] == "secret"
def test_postgresql_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""PostgreSQL 归档校验只依赖 pg_restore,不连接活动数据库。"""
runner = _Runner()
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=runner,
tool_resolver=lambda executable: executable,
)
assert result.valid is True
command, kwargs = runner.calls[0]
assert command == ["pg_restore", "--list", str(artifact)]
assert "PGPASSWORD" not in kwargs["env"]
assert "PGSSLMODE" not in kwargs["env"]
def test_postgresql_offline_verify_rejects_invalid_archive(tmp_path: Path) -> None:
"""pg_restore 无法读取归档目录时备份必须判定为无效。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"invalid")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=_FailedRunner(),
tool_resolver=lambda executable: executable,
)
assert result.valid is False
assert result.detail == "pg_restore 退出码 1"
def test_postgresql_offline_verify_reports_missing_client(tmp_path: Path) -> None:
"""缺少 pg_restore 时离线校验应给出可执行的安装提示。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
with pytest.raises(RuntimeError, match="PostgreSQL client"):
verify_database_backup(
artifact,
db_type="postgresql",
tool_resolver=lambda _executable: None,
)
def test_postgresql_source_install_reports_missing_native_client() -> None:
runner = _Runner()
engine = SimpleNamespace(
+95 -2
View File
@@ -1,13 +1,14 @@
from __future__ import annotations
from datetime import datetime, timedelta
import sqlite3
from types import SimpleNamespace
from app.runtime.config import settings
from app.doctor import checks, run_doctor
from app.doctor import checks
from app.doctor.formatters import format_json_report, format_text_report
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorSeverity
from app.doctor.runner import DoctorRunner
from app.doctor.runner import DoctorRunner, run_doctor
def _current_log_timestamp() -> str:
@@ -33,6 +34,98 @@ def test_doctor_report_has_stable_json_shape(tmp_path, monkeypatch):
assert any(item["id"] == "runtime.paths" for item in payload["findings"])
def test_doctor_reports_valid_backup_when_sqlite_database_is_corrupt(
tmp_path,
monkeypatch,
):
"""主数据库损坏时 Doctor 仍应离线校验备份并给出还原命令。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
(tmp_path / "user.db").write_bytes(b"not a sqlite database")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
with sqlite3.connect(backup) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
(backup_dir / "sqlite_20260822_040000.db").write_bytes(b"invalid newer backup")
runner = DoctorRunner()
checks._check_database(runner)
assert runner.report.find("database.sqlite_open_failed") is not None
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Ok
assert finding.context["backups"][0]["valid"] is False
assert finding.context["backups"][1]["valid"] is True
assert finding.context["restore_command"] == (
"moviepilot database restore sqlite_20260822_030000.db --confirm"
)
assert finding.context["restore_command"] in finding.recommendation
def test_doctor_distinguishes_missing_and_mismatched_backups(tmp_path, monkeypatch):
"""无备份与仅存在其他数据库类型备份应生成不同诊断结论。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
runner = DoctorRunner()
checks._check_database_backups(runner)
missing = runner.report.find("database.backup_recovery")
assert missing is not None
assert missing.status == DoctorFindingStatus.Skipped
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
runner = DoctorRunner()
checks._check_database_backups(runner)
mismatched = runner.report.find("database.backup_recovery")
assert mismatched is not None
assert mismatched.status == DoctorFindingStatus.Degraded
assert mismatched.context["mismatched"] == ["postgresql_20260822_030000.dump"]
def test_doctor_reports_invalid_backup_without_modifying_it(tmp_path, monkeypatch):
"""Doctor --fix 也只校验备份,不覆盖或删除无效文件。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
original = b"invalid sqlite backup"
backup.write_bytes(original)
runner = DoctorRunner(fix=True)
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert finding.affects_report_status is False
assert finding.context["backups"][0]["valid"] is False
assert backup.read_bytes() == original
assert runner.report.status.value == "healthy"
def test_doctor_exposes_missing_pg_restore_in_text_finding(tmp_path, monkeypatch):
"""PostgreSQL 离线校验工具缺失时应直接告诉用户如何补齐。"""
def missing_pg_restore(*_args, **_kwargs):
raise RuntimeError("未找到 pg_restore,请安装 PostgreSQL client 并加入 PATH")
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
monkeypatch.setattr(settings, "DB_TYPE", "postgresql")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
monkeypatch.setattr(checks, "verify_database_backup", missing_pg_restore)
runner = DoctorRunner()
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert "未找到 pg_restore" in finding.detail
assert "PostgreSQL client" in format_text_report(runner.report)
def test_doctor_formatters_include_status_and_finding(tmp_path, monkeypatch):
"""doctor 文本和 JSON 格式化应展示状态与诊断项。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
+7 -2
View File
@@ -1,4 +1,5 @@
from pathlib import Path
from types import SimpleNamespace
import pytest
@@ -22,8 +23,12 @@ def _make_file(name: str, size: int = 150 * 1024 * 1024) -> FileItem:
@pytest.fixture(autouse=True)
def _patch_media_exts(monkeypatch):
monkeypatch.setattr(
"app.application.formatting.settings.RMT_MEDIAEXT",
[".mkv", ".mp4"],
"app.application.formatting.get_chain_runtime_config_snapshot",
lambda: SimpleNamespace(
video_extensions=(".mkv", ".mp4"),
subtitle_extensions=(".srt", ".ass"),
audio_extensions=(".flac", ".mp3"),
),
)
+14
View File
@@ -1,5 +1,6 @@
"""正式镜像发布的供应链门禁合同。"""
from datetime import date
from pathlib import Path
from ruamel.yaml import YAML
@@ -8,6 +9,7 @@ from ruamel.yaml import YAML
ROOT = Path(__file__).resolve().parents[1]
DOCKERFILE = ROOT / "docker" / "Dockerfile"
RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "build-v3.yml"
TRIVY_IGNORE = ROOT / ".trivyignore.yaml"
def _load_workflow() -> dict:
@@ -94,6 +96,18 @@ def test_release_scans_both_architectures_before_registry_login_and_publish() ->
assert last_scan < names.index("Publish multi-architecture image")
def test_vulnerability_ignores_are_scoped_justified_and_time_bounded() -> None:
"""漏洞豁免必须限定制品范围,并保留复查期限和接受理由。"""
yaml = YAML(typ="safe")
vulnerabilities = yaml.load(TRIVY_IGNORE.read_text(encoding="utf-8"))["vulnerabilities"]
for vulnerability in vulnerabilities:
assert vulnerability["paths"]
assert vulnerability["purls"]
assert vulnerability["statement"]
assert isinstance(vulnerability["expired_at"], date)
def test_publish_reuses_scanned_architecture_caches_without_refreshing_base() -> None:
"""发布构建复用已扫描候选缓存,不得在扫描后重新拉取未审计基础镜像。"""
workflow = _load_workflow()