From 23bff6b2bbc3f586c203d5ea4d3bcee23e45f404 Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 27 Aug 2026 13:14:57 +0800 Subject: [PATCH] refactor: add durable transfer planning checkpoints --- app/application/chain/context.py | 3 +- app/application/transfer.py | 645 ++++++- app/chain/__init__.py | 89 +- app/chain/storage.py | 17 +- app/chain/transfer.py | 964 +++++++++- app/db/adapters/transfer.py | 165 +- app/db/models/transferpending.py | 198 ++- app/db/oper/transferpending.py | 94 +- app/modules/filemanager/module.py | 246 ++- app/modules/filemanager/transhandler.py | 1366 +++++++------- app/runtime/extensions/module/contracts.py | 23 + app/runtime/extensions/module/dispatcher.py | 265 ++- app/startup/initializers/modules.py | 10 +- database/versions/c2f8a4d6e1b3_3_0_14.py | 185 ++ docs/architecture-optimization-checklist.md | 40 +- docs/architecture-overview.md | 14 +- docs/architecture-refactor-roadmap.md | 51 +- docs/rules/05-architecture.md | 29 +- .../architecture/coverage-baseline.json | 10 +- .../architecture/dependency-baseline.json | 9 +- .../fixtures/architecture/mypy-baseline.json | 25 +- .../fixtures/architecture/ruff-baseline.json | 25 - .../runtime-contract-baseline.json | 132 +- tests/test_architecture_contract_baseline.py | 8 +- tests/test_architecture_event_facts.py | 8 +- tests/test_filemanager_planning.py | 577 ++++++ tests/test_module_invocation_dispatcher.py | 253 ++- tests/test_module_method_contracts.py | 5 +- tests/test_music_transfer.py | 14 +- tests/test_storage_chain_strict.py | 43 + tests/test_transfer_admission_migration.py | 14 +- tests/test_transfer_job_manager.py | 78 +- tests/test_transfer_pending_replay.py | 58 +- tests/test_transfer_planning_checkpoint.py | 1563 +++++++++++++++++ tests/test_transfer_planning_compat.py | 285 +++ tests/test_transfer_planning_migration.py | 302 ++++ tests/test_transfer_planning_persistence.py | 555 ++++++ tests/test_transfer_preview.py | 38 +- tests/test_transfer_sync_extra_files.py | 31 +- tests/test_transfer_worker_lifecycle.py | 9 +- tests/test_transhandler_special_extra.py | 53 +- 41 files changed, 7401 insertions(+), 1098 deletions(-) create mode 100644 database/versions/c2f8a4d6e1b3_3_0_14.py create mode 100644 tests/test_filemanager_planning.py create mode 100644 tests/test_storage_chain_strict.py create mode 100644 tests/test_transfer_planning_checkpoint.py create mode 100644 tests/test_transfer_planning_compat.py create mode 100644 tests/test_transfer_planning_migration.py create mode 100644 tests/test_transfer_planning_persistence.py diff --git a/app/application/chain/context.py b/app/application/chain/context.py index 280abcca7..3f8e45986 100644 --- a/app/application/chain/context.py +++ b/app/application/chain/context.py @@ -11,9 +11,9 @@ from app.application.chain.durable_events import ChainDurableEventWriter from app.application.configuration import ChainRuntimeConfig from app.runtime.stop import StopState, runtime_stop_state - MessageQueueFactory = Callable[[Callable[..., Any]], Any] ModuleDispatcherFactory = Callable[..., Any] +LegacyTransferCommand = Callable[..., Any] ChainRuntimeContextProvider = Callable[[], "ChainRuntimeContext"] @@ -30,6 +30,7 @@ class ChainRuntimeContext: async_file_cache: Any message_queue_factory: MessageQueueFactory module_dispatcher_factory: ModuleDispatcherFactory + legacy_transfer_command: Optional[LegacyTransferCommand] = None data_ports: Optional[ChainDataPorts] = None durable_event_writer: Optional[ChainDurableEventWriter] = None configuration: ChainRuntimeConfig = field( diff --git a/app/application/transfer.py b/app/application/transfer.py index ab93a5ba3..01550cd89 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -14,12 +14,24 @@ TransferJob / TransferJobTask，那两个用 app.schemas 的同名 DTO——一 视图，分开表达之后两边都不必再迁就对方。 """ import asyncio +import hashlib +import json import threading from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from time import monotonic -from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + List, + Optional, + Protocol, + Tuple, + TypeAlias, + Union, +) from pydantic import BaseModel, ConfigDict, PrivateAttr @@ -49,6 +61,571 @@ from app.schemas.types import ( ) from app.schemas.workflow import MediaInfo as _SchemaMediaInfo +JSONValue: TypeAlias = Union[None, bool, int, float, str, list["JSONValue"], dict[str, "JSONValue"]] + +TRANSFER_ADMISSION_ACCEPTED = "accepted" +TRANSFER_ADMISSION_PROVIDER_PENDING = "provider_pending" +TRANSFER_ADMISSION_PLANNED = "planned" +TRANSFER_PLANNING_INPUT_VERSION = 1 +TRANSFER_PLAN_CHECKPOINT_VERSION = 1 +TRANSFER_PROVIDER_INVOCATION_VERSION = 1 + + +def _copy_json_mapping(value: Optional[dict[str, JSONValue]]) -> Optional[dict[str, JSONValue]]: + if value is None: + return None + return deepcopy(value) + + +def _read_json_mapping(payload: dict[str, Any], key: str) -> Optional[dict[str, JSONValue]]: + value = payload.get(key) + if value is None: + return None + if not isinstance(value, dict): + raise ValueError(f"整理计划字段 {key} 必须是 JSON 对象") + return deepcopy(value) + + +def _read_json_tuple(payload: dict[str, Any], key: str) -> tuple[dict[str, JSONValue], ...]: + value = payload.get(key, []) + if not isinstance(value, list) or not all(isinstance(item, dict) for item in value): + raise ValueError(f"整理计划字段 {key} 必须是 JSON 对象数组") + return tuple(deepcopy(item) for item in value) + + +def _canonical_json(payload: dict[str, JSONValue]) -> str: + try: + return json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + except (TypeError, ValueError) as error: + raise ValueError("整理计划只能包含有限 JSON 值") from error + + +@dataclass(frozen=True, slots=True) +class TransferPlanningInput: + """保存可跨重启重放的版本化整理规划输入。""" + + source_fileitem: dict[str, JSONValue] + meta: Optional[dict[str, JSONValue]] = None + mediainfo: Optional[dict[str, JSONValue]] = None + target_directory: Optional[dict[str, JSONValue]] = None + target_storage: Optional[str] = None + target_path: Optional[str] = None + requested_transfer_type: Optional[str] = None + media_source: Optional[str] = None + media_id: Optional[str] = None + media_type: Optional[str] = None + need_scrape: bool = False + need_rename: bool = True + need_notify: bool = True + overwrite_mode: Optional[str] = None + episodes_info: tuple[dict[str, JSONValue], ...] = () + preview: bool = False + options: dict[str, JSONValue] = field(default_factory=dict) + schema_version: int = TRANSFER_PLANNING_INPUT_VERSION + + def __post_init__(self) -> None: + """拒绝不可恢复的输入版本或缺少源文件身份的快照。""" + if self.schema_version != TRANSFER_PLANNING_INPUT_VERSION: + raise ValueError(f"不支持的整理规划输入版本: {self.schema_version}") + object.__setattr__(self, "source_fileitem", deepcopy(self.source_fileitem)) + object.__setattr__(self, "meta", _copy_json_mapping(self.meta)) + object.__setattr__(self, "mediainfo", _copy_json_mapping(self.mediainfo)) + object.__setattr__( + self, + "target_directory", + _copy_json_mapping(self.target_directory), + ) + object.__setattr__( + self, + "episodes_info", + tuple(deepcopy(item) for item in self.episodes_info), + ) + object.__setattr__(self, "options", deepcopy(self.options)) + storage = self.source_fileitem.get("storage") + path = self.source_fileitem.get("path") + if not isinstance(storage, str) or not storage or not isinstance(path, str) or not path: + raise ValueError("整理规划输入缺少源文件存储或路径") + _canonical_json(self.to_payload()) + + @classmethod + def legacy(cls, *, storage: str, src_path: str) -> "TransferPlanningInput": + """为升级前只有存储与路径的登记构造保守重规划输入。""" + return cls( + source_fileitem={"storage": storage, "path": src_path}, + options={"legacy_replan": True}, + ) + + @classmethod + def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanningInput": + """从受版本约束的 JSON 对象恢复规划输入。""" + if not isinstance(payload, dict): + raise ValueError("整理规划输入必须是 JSON 对象") + source_fileitem = _read_json_mapping(payload, "source_fileitem") + if source_fileitem is None: + raise ValueError("整理规划输入缺少 source_fileitem") + return cls( + source_fileitem=source_fileitem, + meta=_read_json_mapping(payload, "meta"), + mediainfo=_read_json_mapping(payload, "mediainfo"), + target_directory=_read_json_mapping(payload, "target_directory"), + target_storage=payload.get("target_storage"), + target_path=payload.get("target_path"), + requested_transfer_type=payload.get("requested_transfer_type"), + media_source=payload.get("media_source"), + media_id=payload.get("media_id"), + media_type=payload.get("media_type"), + need_scrape=payload.get("need_scrape", False), + need_rename=payload.get("need_rename", True), + need_notify=payload.get("need_notify", True), + overwrite_mode=payload.get("overwrite_mode"), + episodes_info=_read_json_tuple(payload, "episodes_info"), + preview=payload.get("preview", False), + options=_read_json_mapping(payload, "options") or {}, + schema_version=payload.get("schema_version", 0), + ) + + def to_payload(self) -> dict[str, JSONValue]: + """生成仅含 JSON 值且字段稳定的规划输入投影。""" + return { + "schema_version": self.schema_version, + "source_fileitem": deepcopy(self.source_fileitem), + "meta": _copy_json_mapping(self.meta), + "mediainfo": _copy_json_mapping(self.mediainfo), + "target_directory": _copy_json_mapping(self.target_directory), + "target_storage": self.target_storage, + "target_path": self.target_path, + "requested_transfer_type": self.requested_transfer_type, + "media_source": self.media_source, + "media_id": self.media_id, + "media_type": self.media_type, + "need_scrape": self.need_scrape, + "need_rename": self.need_rename, + "need_notify": self.need_notify, + "overwrite_mode": self.overwrite_mode, + "episodes_info": [deepcopy(item) for item in self.episodes_info], + "preview": self.preview, + "options": deepcopy(self.options), + } + + @property + def fingerprint(self) -> str: + """返回规范 JSON 的稳定 SHA-256 指纹。""" + return hashlib.sha256(_canonical_json(self.to_payload()).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True, slots=True) +class TransferPlanItem: + """描述规划后按序执行的一条叶子文件操作。""" + + sequence: int + source_fileitem: dict[str, JSONValue] + target_storage: str + target_path: str + action: str = "transfer" + + def __post_init__(self) -> None: + """拒绝无法定位源文件或目标文件的计划项。""" + object.__setattr__(self, "source_fileitem", deepcopy(self.source_fileitem)) + if self.sequence < 0: + raise ValueError("整理计划项序号不能小于零") + if not self.target_storage or not self.target_path or not self.action: + raise ValueError("整理计划项缺少目标身份或动作") + if not self.source_fileitem.get("storage") or not self.source_fileitem.get("path"): + raise ValueError("整理计划项缺少源文件身份") + _canonical_json(self.to_payload()) + + @classmethod + def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanItem": + """从 JSON 对象恢复单条叶子文件操作。""" + source_fileitem = _read_json_mapping(payload, "source_fileitem") + if source_fileitem is None: + raise ValueError("整理计划项缺少 source_fileitem") + return cls( + sequence=payload.get("sequence", -1), + source_fileitem=source_fileitem, + target_storage=payload.get("target_storage", ""), + target_path=payload.get("target_path", ""), + action=payload.get("action", "transfer"), + ) + + def to_payload(self) -> dict[str, JSONValue]: + """生成单条叶子文件操作的 JSON 投影。""" + return { + "sequence": self.sequence, + "source_fileitem": deepcopy(self.source_fileitem), + "target_storage": self.target_storage, + "target_path": self.target_path, + "action": self.action, + } + + +@dataclass(frozen=True, slots=True) +class TransferProviderReference: + """保存无需依赖运行时 dispatcher 即可持久化的旧 transfer provider 引用。""" + + plugin_id: str + plugin_name: str + method: str = "transfer" + + def __post_init__(self) -> None: + """拒绝无法稳定定位插件或指向非 transfer 方法的引用。""" + if not isinstance(self.plugin_id, str) or not self.plugin_id.strip(): + raise ValueError("旧 transfer provider 的 plugin_id 必须是非空字符串") + if not isinstance(self.plugin_name, str) or not self.plugin_name.strip(): + raise ValueError("旧 transfer provider 的 plugin_name 必须是非空字符串") + if self.method != "transfer": + raise ValueError("旧 transfer provider 的 method 必须是 transfer") + _canonical_json(self.to_payload()) + + @classmethod + def from_payload(cls, payload: dict[str, Any]) -> "TransferProviderReference": + """从 JSON 对象恢复旧 transfer provider 引用。""" + if not isinstance(payload, dict): + raise ValueError("旧 transfer provider 引用必须是 JSON 对象") + return cls( + plugin_id=payload.get("plugin_id", ""), + plugin_name=payload.get("plugin_name", ""), + method=payload.get("method", "transfer"), + ) + + def to_payload(self) -> dict[str, JSONValue]: + """生成仅包含稳定插件身份与方法名的 JSON 投影。""" + return { + "plugin_id": self.plugin_id, + "plugin_name": self.plugin_name, + "method": self.method, + } + + +@dataclass(frozen=True, slots=True) +class TransferProviderInvocationSnapshot: + """冻结旧 transfer ABI 中可跨重启恢复的精确参数值。""" + + fileitem: dict[str, JSONValue] + meta: Optional[dict[str, JSONValue]] + meta_kind: Optional[str] + mediainfo: Optional[dict[str, JSONValue]] + mediainfo_kind: Optional[str] + target_directory: Optional[dict[str, JSONValue]] = None + target_storage: Optional[str] = None + target_path: Optional[str] = None + transfer_type: Optional[str] = None + scrape: Optional[bool] = None + library_type_folder: Optional[bool] = None + library_category_folder: Optional[bool] = None + episodes_info: tuple[dict[str, JSONValue], ...] = () + preview: bool = False + schema_version: int = TRANSFER_PROVIDER_INVOCATION_VERSION + + def __post_init__(self) -> None: + """拒绝缺少源身份、类型不稳定或版本未知的 provider 快照。""" + if self.schema_version != TRANSFER_PROVIDER_INVOCATION_VERSION: + raise ValueError( + f"不支持的旧 transfer provider 调用快照版本: {self.schema_version}" + ) + object.__setattr__(self, "fileitem", deepcopy(self.fileitem)) + object.__setattr__(self, "meta", _copy_json_mapping(self.meta)) + object.__setattr__(self, "mediainfo", _copy_json_mapping(self.mediainfo)) + object.__setattr__( + self, + "target_directory", + _copy_json_mapping(self.target_directory), + ) + object.__setattr__( + self, + "episodes_info", + tuple(deepcopy(item) for item in self.episodes_info), + ) + if not self.fileitem.get("storage") or not self.fileitem.get("path"): + raise ValueError("旧 transfer provider 调用快照缺少源文件身份") + for kind in (self.meta_kind, self.mediainfo_kind): + if kind is not None and (not isinstance(kind, str) or not kind): + raise ValueError("旧 transfer provider 调用快照类型必须是非空字符串") + for value in ( + self.target_storage, + self.target_path, + self.transfer_type, + ): + if value is not None and not isinstance(value, str): + raise ValueError("旧 transfer provider 可选文本参数必须是字符串或 None") + for bool_value in ( + self.scrape, + self.library_type_folder, + self.library_category_folder, + ): + if bool_value is not None and not isinstance(bool_value, bool): + raise ValueError("旧 transfer provider 可选布尔参数必须是 bool 或 None") + if not isinstance(self.preview, bool): + raise ValueError("旧 transfer provider preview 参数必须是 bool") + _canonical_json(self.to_payload()) + + @classmethod + def from_payload( + cls, + payload: dict[str, Any], + ) -> "TransferProviderInvocationSnapshot": + """从版本化 JSON 对象严格恢复旧 transfer ABI 调用快照。""" + if not isinstance(payload, dict): + raise ValueError("旧 transfer provider 调用快照必须是 JSON 对象") + fileitem = _read_json_mapping(payload, "fileitem") + if fileitem is None: + raise ValueError("旧 transfer provider 调用快照缺少 fileitem") + return cls( + fileitem=fileitem, + meta=_read_json_mapping(payload, "meta"), + meta_kind=payload.get("meta_kind"), + mediainfo=_read_json_mapping(payload, "mediainfo"), + mediainfo_kind=payload.get("mediainfo_kind"), + target_directory=_read_json_mapping(payload, "target_directory"), + target_storage=payload.get("target_storage"), + target_path=payload.get("target_path"), + transfer_type=payload.get("transfer_type"), + scrape=payload.get("scrape"), + library_type_folder=payload.get("library_type_folder"), + library_category_folder=payload.get("library_category_folder"), + episodes_info=_read_json_tuple(payload, "episodes_info"), + preview=payload.get("preview", False), + schema_version=payload.get("schema_version", 0), + ) + + def to_payload(self) -> dict[str, JSONValue]: + """生成仅含 JSON 值且保留 None 与 False 差异的调用投影。""" + return { + "schema_version": self.schema_version, + "fileitem": deepcopy(self.fileitem), + "meta": _copy_json_mapping(self.meta), + "meta_kind": self.meta_kind, + "mediainfo": _copy_json_mapping(self.mediainfo), + "mediainfo_kind": self.mediainfo_kind, + "target_directory": _copy_json_mapping(self.target_directory), + "target_storage": self.target_storage, + "target_path": self.target_path, + "transfer_type": self.transfer_type, + "scrape": self.scrape, + "library_type_folder": self.library_type_folder, + "library_category_folder": self.library_category_folder, + "episodes_info": [deepcopy(item) for item in self.episodes_info], + "preview": self.preview, + } + + +@dataclass(frozen=True, slots=True) +class TransferPlanCheckpoint: + """保存无需重触发识别或重命名即可执行的完整有序计划。""" + + planning_input: TransferPlanningInput + target_storage: str + root_target_path: str + final_target_path: str + resolved_transfer_type: str + items: tuple[TransferPlanItem, ...] + resolved_meta: Optional[dict[str, JSONValue]] = None + resolved_meta_kind: Optional[str] = None + resolved_mediainfo: Optional[dict[str, JSONValue]] = None + resolved_mediainfo_kind: Optional[str] = None + resolved_episodes_info: tuple[dict[str, JSONValue], ...] = () + legacy_transfer_providers: tuple[TransferProviderReference, ...] = () + provider_invocation: Optional[TransferProviderInvocationSnapshot] = None + pre_execution_cleanup_completed: bool = False + need_scrape: bool = False + need_rename: bool = False + need_notify: bool = True + overwrite_mode: Optional[str] = None + preview: bool = False + skip_reason: Optional[str] = None + schema_version: int = TRANSFER_PLAN_CHECKPOINT_VERSION + + def __post_init__(self) -> None: + """验证版本、目标身份和计划项顺序组成完整检查点。""" + if self.schema_version != TRANSFER_PLAN_CHECKPOINT_VERSION: + raise ValueError(f"不支持的整理计划检查点版本: {self.schema_version}") + if not isinstance(self.pre_execution_cleanup_completed, bool): + raise ValueError("整理计划预执行 cleanup 完成标记必须是 bool") + object.__setattr__(self, "resolved_meta", _copy_json_mapping(self.resolved_meta)) + object.__setattr__( + self, + "resolved_mediainfo", + _copy_json_mapping(self.resolved_mediainfo), + ) + object.__setattr__( + self, + "resolved_episodes_info", + tuple(deepcopy(item) for item in self.resolved_episodes_info), + ) + if not isinstance(self.legacy_transfer_providers, tuple) or any( + not isinstance(provider, TransferProviderReference) + for provider in self.legacy_transfer_providers + ): + raise ValueError("整理计划的旧 transfer provider 引用必须是类型化元组") + provider_ids = tuple( + provider.plugin_id for provider in self.legacy_transfer_providers + ) + if len(set(provider_ids)) != len(provider_ids): + raise ValueError("整理计划的旧 transfer provider plugin_id 不能重复") + for resolved_kind in ( + self.resolved_meta_kind, + self.resolved_mediainfo_kind, + ): + if resolved_kind is not None and ( + not isinstance(resolved_kind, str) or not resolved_kind + ): + raise ValueError("整理计划的已解析上下文类型必须是非空字符串") + if self.provider_invocation is not None: + if not self.legacy_transfer_providers: + raise ValueError("provider_pending 检查点缺少冻结 provider") + if self.pre_execution_cleanup_completed: + raise ValueError("provider_pending 提交时 cleanup 尚未执行") + if ( + not self.provider_invocation.meta + or not self.provider_invocation.meta_kind + or not self.provider_invocation.mediainfo + or not self.provider_invocation.mediainfo_kind + ): + raise ValueError("provider_pending 检查点缺少可重放的媒体上下文") + if ( + self.target_storage + or self.root_target_path + or self.final_target_path + or self.resolved_transfer_type + or self.items + or self.skip_reason + ): + raise ValueError("provider_pending 检查点不得包含宿主执行计划") + else: + if not self.target_storage or not self.root_target_path or not self.final_target_path: + raise ValueError("整理计划检查点缺少目标身份") + if not self.resolved_transfer_type: + raise ValueError("整理计划检查点缺少已解析的整理方式") + if tuple(item.sequence for item in self.items) != tuple(range(len(self.items))): + raise ValueError("整理计划项必须按从零开始的连续序号保存") + if ( + self.provider_invocation is None + and not self.items + and not self.preview + and not self.skip_reason + ): + raise ValueError("非预览空计划必须记录合法跳过原因") + _canonical_json(self.to_payload()) + + @property + def is_provider_pending(self) -> bool: + """返回该检查点是否只冻结 provider 调用、尚未生成宿主计划。""" + return self.provider_invocation is not None + + @classmethod + def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanCheckpoint": + """从受版本约束的 JSON 对象恢复完整执行检查点。""" + if not isinstance(payload, dict): + raise ValueError("整理计划检查点必须是 JSON 对象") + planning_input_payload = _read_json_mapping(payload, "planning_input") + if planning_input_payload is None: + raise ValueError("整理计划检查点缺少 planning_input") + item_payloads = payload.get("items", []) + if not isinstance(item_payloads, list) or not all( + isinstance(item, dict) for item in item_payloads + ): + raise ValueError("整理计划检查点 items 必须是 JSON 对象数组") + legacy_provider_payloads = payload.get("legacy_transfer_providers", []) + if not isinstance(legacy_provider_payloads, list): + raise ValueError( + "整理计划检查点 legacy_transfer_providers 必须是 JSON 对象数组" + ) + provider_invocation_payload = payload.get("provider_invocation") + if provider_invocation_payload is not None and not isinstance( + provider_invocation_payload, + dict, + ): + raise ValueError("整理计划检查点 provider_invocation 必须是 JSON 对象") + return cls( + planning_input=TransferPlanningInput.from_payload(planning_input_payload), + target_storage=payload.get("target_storage", ""), + root_target_path=payload.get("root_target_path", ""), + final_target_path=payload.get("final_target_path", ""), + resolved_transfer_type=payload.get("resolved_transfer_type", ""), + items=tuple(TransferPlanItem.from_payload(item) for item in item_payloads), + resolved_meta=_read_json_mapping(payload, "resolved_meta"), + resolved_meta_kind=payload.get("resolved_meta_kind"), + resolved_mediainfo=_read_json_mapping(payload, "resolved_mediainfo"), + resolved_mediainfo_kind=payload.get("resolved_mediainfo_kind"), + resolved_episodes_info=_read_json_tuple( + payload, + "resolved_episodes_info", + ), + legacy_transfer_providers=tuple( + TransferProviderReference.from_payload(provider) + for provider in legacy_provider_payloads + ), + provider_invocation=( + TransferProviderInvocationSnapshot.from_payload( + provider_invocation_payload + ) + if provider_invocation_payload is not None + else None + ), + pre_execution_cleanup_completed=payload.get( + "pre_execution_cleanup_completed", + False, + ), + need_scrape=payload.get("need_scrape", False), + need_rename=payload.get("need_rename", False), + need_notify=payload.get("need_notify", True), + overwrite_mode=payload.get("overwrite_mode"), + preview=payload.get("preview", False), + skip_reason=payload.get("skip_reason"), + schema_version=payload.get("schema_version", 0), + ) + + def to_payload(self) -> dict[str, JSONValue]: + """生成可原子落库的完整版本化 JSON 检查点。""" + return { + "schema_version": self.schema_version, + "planning_input": self.planning_input.to_payload(), + "target_storage": self.target_storage, + "root_target_path": self.root_target_path, + "final_target_path": self.final_target_path, + "resolved_transfer_type": self.resolved_transfer_type, + "items": [item.to_payload() for item in self.items], + "resolved_meta": _copy_json_mapping(self.resolved_meta), + "resolved_meta_kind": self.resolved_meta_kind, + "resolved_mediainfo": _copy_json_mapping(self.resolved_mediainfo), + "resolved_mediainfo_kind": self.resolved_mediainfo_kind, + "resolved_episodes_info": [ + deepcopy(item) for item in self.resolved_episodes_info + ], + "legacy_transfer_providers": [ + provider.to_payload() + for provider in self.legacy_transfer_providers + ], + "provider_invocation": ( + self.provider_invocation.to_payload() + if self.provider_invocation + else None + ), + "pre_execution_cleanup_completed": ( + self.pre_execution_cleanup_completed + ), + "need_scrape": self.need_scrape, + "need_rename": self.need_rename, + "need_notify": self.need_notify, + "overwrite_mode": self.overwrite_mode, + "preview": self.preview, + "skip_reason": self.skip_reason, + } + + +class TransferAdmissionConflictError(ValueError): + """同一源文件以不同规划输入重复准入时抛出的冲突错误。""" + + +class TransferPlanningStateError(RuntimeError): + """计划检查点无法从当前持久状态推进时抛出的状态错误。""" + class TransferTask(OptionalMediaIdentityMixin, BaseModel): """ @@ -81,6 +658,9 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): background: Optional[bool] = True preview: Optional[bool] = False _admission_task_id: Optional[str] = PrivateAttr(default=None) + _planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None) + _plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None) + _planning_context_restored: bool = PrivateAttr(default=False) @property def admission_task_id(self) -> Optional[str]: @@ -91,6 +671,33 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): """绑定持久准入生成的稳定身份，不改变插件可见序列化字段。""" self._admission_task_id = task_id + @property + def planning_input(self) -> Optional[TransferPlanningInput]: + """返回宿主恢复规划使用的内部输入快照。""" + return self._planning_input + + @property + def plan_checkpoint(self) -> Optional[TransferPlanCheckpoint]: + """返回宿主直接执行已规划任务使用的内部检查点。""" + return self._plan_checkpoint + + def bind_planning_input(self, planning_input: TransferPlanningInput) -> None: + """绑定持久规划输入，不改变插件可见序列化字段。""" + self._planning_input = planning_input + + def bind_plan_checkpoint(self, checkpoint: TransferPlanCheckpoint) -> None: + """绑定持久执行检查点，不改变插件可见序列化字段。""" + self._plan_checkpoint = checkpoint + + @property + def planning_context_restored(self) -> bool: + """返回当前领域上下文是否来自持久快照。""" + return self._planning_context_restored + + def mark_planning_context_restored(self) -> None: + """标记领域上下文已离线恢复，禁止旧流程再次在线补充。""" + self._planning_context_restored = True + def to_dict(self): """ 返回字典。 @@ -122,9 +729,6 @@ class TransferQueue(BaseModel): result: Optional[TransferInfo] = None -TRANSFER_ADMISSION_ACCEPTED = "accepted" - - @dataclass(frozen=True, slots=True) class TransferAdmission: """描述已经持久化、可在进程退出后恢复的整理任务准入事实。""" @@ -136,13 +740,22 @@ class TransferAdmission: created_at: str updated_at: str last_error: Optional[str] = None + input_fingerprint: Optional[str] = None + planning_input: Optional[TransferPlanningInput] = None + checkpoint: Optional[TransferPlanCheckpoint] = None class TransferAdmissionRepository(Protocol): """整理任务 durable admission 所需的类型化持久化端口。""" - def admit(self, *, storage: str, src_path: str) -> TransferAdmission: - """幂等登记源文件并返回稳定任务身份。""" + def admit( + self, + *, + storage: str, + src_path: str, + planning_input: Optional[TransferPlanningInput] = None, + ) -> TransferAdmission: + """按规划输入幂等登记源文件并返回稳定任务身份。""" ... def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]: @@ -153,6 +766,24 @@ class TransferAdmissionRepository(Protocol): """记录内存队列接收失败，保留任务供后续恢复。""" ... + def checkpoint_plan( + self, + *, + task_id: str, + input_fingerprint: str, + checkpoint: TransferPlanCheckpoint, + ) -> TransferAdmission: + """原子保存完整计划并将匹配输入的任务推进到已规划。""" + ... + + def record_planning_failure(self, *, task_id: str, error: str) -> None: + """记录规划失败但保留接纳状态供下次恢复重试。""" + ... + + def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]: + """按登记顺序返回接纳或已规划的可恢复任务。""" + ... + def discard_task(self, *, task_id: str) -> int: """按稳定任务身份删除已经到达终态的登记。""" ... diff --git a/app/chain/__init__.py b/app/chain/__init__.py index 0dfa1229c..b2161c652 100644 --- a/app/chain/__init__.py +++ b/app/chain/__init__.py @@ -5,7 +5,7 @@ import traceback from abc import ABCMeta from collections.abc import Callable from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union, cast from app.application.chain.context import ChainRuntimeContext, get_chain_runtime_context from app.application.chain.data import get_chain_data_ports @@ -15,7 +15,7 @@ from app.application.configuration import ( ) from app.chain._messaging import MessageProcessingMixin, NotificationMixin from app.chain._recognition import RecognitionMixin -from app.domain.context import Context, MediaInfo, SubtitleInfo, TorrentInfo +from app.domain.context import Context, MediaInfo, MusicInfo, SubtitleInfo, TorrentInfo from app.domain.meta.metabase import MetaBase from app.runtime.log import logger from app.schemas.category import CategoryConfig @@ -35,6 +35,9 @@ from app.schemas.types import ( ) from app.schemas.workflow import FileItem +if TYPE_CHECKING: + from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput + class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, metaclass=ABCMeta): @@ -65,6 +68,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, system_error_handler=self.__handle_system_error, rate_limit_handler=self.__handle_rate_limit_error, ) + self._legacy_transfer_command = context.legacy_transfer_command self.messagequeue = context.message_queue_factory(self.run_module) @property @@ -216,6 +220,15 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, """ return self._module_dispatcher.dispatch(method, *args, **kwargs) + def run_module_strict( + self, + method: str, + *args: Any, + **kwargs: Any, + ) -> Any: + """运行模块并传播 provider 异常，供必须区分空结果与查询失败的能力使用。""" + return self._module_dispatcher.dispatch_strict(method, *args, **kwargs) + async def async_run_module( self, method: str, @@ -854,7 +867,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, self, fileitem: FileItem, meta: MetaBase, - mediainfo: MediaInfo, + mediainfo: Union[MediaInfo, MusicInfo], target_directory: TransferDirectoryConf = None, target_storage: Optional[str] = None, target_path: Path = None, @@ -868,7 +881,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, preview: bool = False, ) -> Optional[TransferInfo]: """ - 文件转移 + 经启动组合根注入的 canonical durable command 执行旧整理 ABI。 :param fileitem: 文件信息 :param meta: 预识别的元数据 :param mediainfo: 识别的媒体信息 @@ -885,8 +898,9 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, :param preview: 是否仅预览，不执行实际转移 :return: {path, target_path, message} """ - return self.run_module( - "transfer", + if self._legacy_transfer_command is None: + raise RuntimeError("旧整理兼容命令尚未由启动组合根配置") + return self._legacy_transfer_command( fileitem=fileitem, meta=meta, mediainfo=mediainfo, @@ -903,6 +917,69 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin, preview=preview, ) + def plan_transfer( + self, + fileitem: FileItem, + meta: MetaBase, + mediainfo: Union[MediaInfo, MusicInfo], + target_directory: Optional[TransferDirectoryConf] = None, + target_storage: Optional[str] = None, + target_path: Optional[Path] = None, + transfer_type: Optional[str] = None, + scrape: Optional[bool] = None, + library_type_folder: Optional[bool] = None, + library_category_folder: Optional[bool] = None, + episodes_info: Optional[List[TmdbEpisode]] = None, + source_oper: Any = None, + preview: bool = False, + planning_input: Optional[TransferPlanningInput] = None, + ) -> Optional[TransferPlanCheckpoint]: + """调用文件管理模块生成无文件写入的冻结整理计划。""" + return cast( + "Optional[TransferPlanCheckpoint]", + self.run_module( + "plan_transfer", + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_directory=target_directory, + target_path=target_path, + target_storage=target_storage, + transfer_type=transfer_type, + scrape=scrape, + library_type_folder=library_type_folder, + library_category_folder=library_category_folder, + episodes_info=episodes_info, + source_oper=source_oper, + preview=preview, + planning_input=planning_input, + ), + ) + + def execute_transfer_plan( + self, + checkpoint: TransferPlanCheckpoint, + *, + meta: MetaBase, + mediainfo: Union[MediaInfo, MusicInfo], + source_oper: Any = None, + target_oper: Any = None, + cleanup_media_file: Optional[Callable[[FileItem], bool]] = None, + ) -> Optional[TransferInfo]: + """调用文件管理模块执行已冻结计划，并注入统一安全删除能力。""" + return cast( + Optional[TransferInfo], + self.run_module( + "execute_transfer_plan", + checkpoint=checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_oper=target_oper, + cleanup_media_file=cleanup_media_file, + ), + ) + def transfer_completed(self, hashs: str, downloader: Optional[str] = None) -> None: """ 下载器转移完成后的处理 diff --git a/app/chain/storage.py b/app/chain/storage.py index 7cbe09fd9..a6dadc01b 100644 --- a/app/chain/storage.py +++ b/app/chain/storage.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, cast from app.application.directory import DirectoryHelper from app.chain import ChainBase @@ -99,6 +99,21 @@ class StorageChain(ChainBase): """ return self.run_module("get_file_item", storage=storage, path=path) + def get_file_item_strict( + self, + storage: str, + path: Path, + ) -> Optional[_SchemaFileItem]: + """严格查询文件项：确认不存在返回空，provider 或 I/O 失败直接抛出。""" + return cast( + Optional[_SchemaFileItem], + self.run_module_strict( + "get_file_item", + storage=storage, + path=path, + ), + ) + def get_parent_item(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]: """ 获取上级目录项 diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 1d1c0da4a..df64c1630 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -9,26 +9,17 @@ from collections import Counter from concurrent.futures import CancelledError as FutureCancelledError from concurrent.futures import Future from copy import deepcopy +from dataclasses import replace from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast + +from pydantic_core import to_jsonable_python from app.application.chain.data import ( get_chain_download_history_port, get_chain_transfer_history_port, get_chain_transfer_pending_port, ) -from app.chain import ChainBase -from app.chain.media import MediaChain -from app.chain.storage import StorageChain -from app.chain.tmdb import TmdbChain -from app.domain.context import MediaInfo, MusicInfo, TorrentInfo -from app.domain.meta.metabase import MetaBase -from app.domain.meta.metamusic import MetaMusic -from app.domain.metainfo import MetaInfoPath -from app.runtime.config import global_vars -from app.runtime.stop import runtime_stop_state - -DownloadHistory = Any from app.application.configuration import get_configured_system_config from app.application.directory import DirectoryHelper from app.application.formatting import FormatParser @@ -55,12 +46,18 @@ from app.application.transfer import ( TransferAdmission, TransferFailureNotification, TransferFailureNotificationAggregator, + TransferPlanCheckpoint, + TransferPlanningInput, + TransferPlanningStateError, + TransferProviderInvocationSnapshot, + TransferProviderReference, TransferQueue, TransferQueueService, TransferTask, build_transfer_failure_group_key, job_lock, ) +from app.chain import ChainBase from app.chain._transfer import ( EpisodeFormatMixin, FailedRetryMixin, @@ -70,11 +67,22 @@ from app.chain._transfer import ( ManualHistoryMixin, ScrapeBatchMixin, ) +from app.chain.media import MediaChain +from app.chain.storage import StorageChain +from app.chain.tmdb import TmdbChain from app.domain import episode as episode_rules +from app.domain.context import MediaInfo, MusicInfo, TorrentInfo +from app.domain.meta.metaanime import MetaAnime +from app.domain.meta.metabase import MetaBase +from app.domain.meta.metamusic import MetaMusic +from app.domain.meta.metavideo import MetaVideo +from app.domain.metainfo import MetaInfoPath from app.foundation.singleton import Singleton +from app.runtime.config import global_vars from app.runtime.log import logger from app.runtime.progress import ProgressHelper from app.runtime.reload import ConfigReloadMixin +from app.runtime.stop import runtime_stop_state from app.schemas.event import StorageOperSelectionEventData from app.schemas.exception import OperationInterrupted from app.schemas.media import resolve_media_identity @@ -96,6 +104,8 @@ from app.schemas.types import ( ) from app.schemas.workflow import FileItem +DownloadHistory = Any + # 下载器锁 downloader_lock = threading.Lock() # 任务锁 @@ -115,7 +125,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo return (_transfer_mixin.MediaChain or MediaChain)() @classmethod - def _transfer_storage_chain(cls): + def _transfer_storage_chain(cls) -> StorageChain: """为整理 mixin 提供可替换的存储构造点。""" from app.chain import _transfer as _transfer_mixin return (_transfer_mixin.StorageChain or StorageChain)() @@ -931,15 +941,15 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo """ 把落盘登记的待整理文件重新送回整理入口。 - 只回放「存储 + 源路径」这一最小事实，重新走完整的识别与整理流程， - 已经整理完成的由整理历史查重挡掉，因此不存在重复整理的问题。 + accepted 记录按原始请求重新规划；planned 记录绑定已提交检查点，重新识别 + 领域对象后只执行冻结路径，不再次触发重命名或目标计算。 :param stop_event: 宿主关闭信号；只阻止处理下一条登记，不取消运行中的同步 I/O """ stop_event = stop_event or threading.Event() if stop_event.is_set(): return try: - pendings = self._transfer_admissions.list_accepted() + pendings = self._transfer_admissions.list_recoverable() except Exception as err: logger.error(f"读取待整理文件登记失败：{err}") return @@ -953,7 +963,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo storage = admission.storage src_path = admission.src_path try: - fileitem, should_discard = self.__build_replay_fileitem(storage, src_path) + fileitem, should_discard = self.__build_replay_fileitem( + storage, + src_path, + admission.planning_input, + ) # stat 等同步 I/O 返回后重新检查，关闭期间不得注销尚未完成的登记。 if stop_event.is_set(): break @@ -964,7 +978,25 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo task_id=admission.task_id ) continue - self.do_transfer(fileitem=fileitem) + if admission.checkpoint: + if self.__queue_planned_replay(fileitem, admission): + replayed += 1 + continue + planning_input = admission.planning_input + if planning_input and ( + planning_input.meta + or planning_input.mediainfo + or planning_input.episodes_info + ): + if self.__queue_accepted_replay(fileitem, admission): + replayed += 1 + continue + replay_kwargs = self.__build_replay_kwargs(planning_input) + self._execute_transfer( + fileitem=fileitem, + recovery_admission=admission, + **replay_kwargs, + ) replayed += 1 except Exception as err: logger.error(f"回放待整理文件失败：{storage}:{src_path} - {err}") @@ -977,7 +1009,176 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo logger.info(f"✓ 待整理文件回放完成，{replayed} 个文件已重新送入整理链") @staticmethod - def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]: + def __build_replay_kwargs( + planning_input: Optional[TransferPlanningInput], + ) -> dict[str, Any]: + """从持久请求快照恢复公开整理入口可接受的稳定参数。""" + if planning_input is None: + return {} + options = planning_input.options + target_directory_payload = planning_input.target_directory + cleanup_payload = options.get("cleanup_dest_fileitem") + try: + media_source = ( + MediaSource(planning_input.media_source) + if planning_input.media_source + else None + ) + except ValueError: + media_source = None + try: + media_type = ( + MediaType(planning_input.media_type) + if planning_input.media_type + else None + ) + except ValueError: + media_type = None + return { + "mtype": media_type, + "media_source": media_source, + "media_id": planning_input.media_id, + "target_directory": ( + TransferDirectoryConf.model_validate(target_directory_payload) + if target_directory_payload + else None + ), + "target_storage": planning_input.target_storage, + "target_path": ( + Path(planning_input.target_path) + if planning_input.target_path + else None + ), + "transfer_type": planning_input.requested_transfer_type, + "scrape": options.get("scrape"), + "library_type_folder": options.get("library_type_folder"), + "library_category_folder": options.get("library_category_folder"), + "downloader": options.get("downloader"), + "download_hash": options.get("download_hash"), + "background": True, + "manual": bool(options.get("manual")), + "preview": False, + "cleanup_dest_fileitem": ( + FileItem.model_validate(cleanup_payload) + if isinstance(cleanup_payload, dict) + else None + ), + } + + def __queue_accepted_replay( + self, + fileitem: FileItem, + admission: TransferAdmission, + ) -> bool: + """从 accepted 输入离线恢复显式领域上下文并送入单任务队列。""" + planning_input = admission.planning_input + if planning_input is None: + raise TransferPlanningStateError("accepted 回放缺少整理规划输入") + options = planning_input.options + replay_kwargs = self.__build_replay_kwargs(planning_input) + task = TransferTask( + fileitem=fileitem, + meta=( + self.__restore_meta_snapshot( + planning_input.meta, + options.get("_meta_kind"), + ) + if planning_input.meta + else None + ), + mediainfo=( + self.__restore_mediainfo_snapshot( + planning_input.mediainfo, + options.get("_mediainfo_kind"), + ) + if planning_input.mediainfo + else None + ), + mtype=replay_kwargs["mtype"], + media_source=replay_kwargs["media_source"], + media_id=replay_kwargs["media_id"], + target_directory=replay_kwargs["target_directory"], + target_storage=replay_kwargs["target_storage"], + target_path=replay_kwargs["target_path"], + transfer_type=replay_kwargs["transfer_type"], + scrape=replay_kwargs["scrape"], + library_type_folder=replay_kwargs["library_type_folder"], + library_category_folder=replay_kwargs["library_category_folder"], + episodes_info=[ + TmdbEpisode.model_validate(item) + for item in planning_input.episodes_info + ], + username=options.get("username"), + downloader=replay_kwargs["downloader"], + download_hash=replay_kwargs["download_hash"], + manual=replay_kwargs["manual"], + background=True, + preview=False, + ) + task.bind_admission_task_id(admission.task_id) + task.bind_planning_input(planning_input) + if planning_input.mediainfo: + task.mark_planning_context_restored() + return self.put_to_queue(task) + + def __queue_planned_replay( + self, + fileitem: FileItem, + admission: TransferAdmission, + ) -> bool: + """把已提交检查点直接恢复为单个任务，避免重新扫描、识别和规划。""" + checkpoint = admission.checkpoint + if checkpoint is None: + raise TransferPlanningStateError("planned 回放缺少整理计划检查点") + options = checkpoint.planning_input.options + invocation = checkpoint.provider_invocation + task = TransferTask( + fileitem=fileitem, + target_storage=( + invocation.target_storage if invocation else checkpoint.target_storage + ), + target_path=( + Path(invocation.target_path) + if invocation and invocation.target_path + else Path(checkpoint.root_target_path) + if checkpoint.root_target_path + else None + ), + transfer_type=( + invocation.transfer_type + if invocation + else checkpoint.resolved_transfer_type + ), + scrape=invocation.scrape if invocation else checkpoint.need_scrape, + library_type_folder=( + invocation.library_type_folder + if invocation + else bool(options.get("library_type_folder")) + ), + library_category_folder=( + invocation.library_category_folder + if invocation + else bool(options.get("library_category_folder")) + ), + username=options.get("username"), + downloader=options.get("downloader"), + download_hash=options.get("download_hash"), + manual=bool(options.get("manual")), + background=True, + preview=False, + ) + task.bind_admission_task_id(admission.task_id) + task.bind_planning_input(checkpoint.planning_input) + task.bind_plan_checkpoint(checkpoint) + self.__restore_planned_task(task) + return self.put_to_queue(task) + + @staticmethod + def __build_replay_fileitem( + storage: str, + src_path: str, + planning_input: Optional[TransferPlanningInput] = None, + ) -> Tuple[Optional[FileItem], bool]: """ 为回放构造文件项。 @@ -987,6 +1188,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ——那等于在故障期间主动丢件，正是本表要防的事。 :param storage: 存储 :param src_path: 源文件路径，以 / 结尾表示蓝光原盘目录 + :param planning_input: 准入时保存的完整源文件身份快照 :return: (文件项, 是否应注销登记)。文件项为 None 表示本次不回放； 只有确认源文件已经消失时才注销登记 """ @@ -1005,27 +1207,599 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # 挂载未就绪或无响应属于暂时性故障，保留登记等下次启动再回放 logger.warn(f"读取待整理文件失败，保留登记等待下次回放：{src_path} - {err}") return None, False - return FileItem( - storage=storage, - path=src_path if is_dir else path.as_posix(), - type="dir" if is_dir else "file", - name=path.name, - basename=path.stem, - extension=path.suffix[1:] if not is_dir else None, - size=size, - modify_time=modify_time, - ), False + snapshot = ( + planning_input.source_fileitem + if planning_input and planning_input.source_fileitem + else {} + ) + fileitem = FileItem.model_validate({ + **snapshot, + "storage": storage, + "path": src_path if is_dir else path.as_posix(), + "type": "dir" if is_dir else snapshot.get("type") or "file", + "name": snapshot.get("name") or path.name, + "basename": snapshot.get("basename") or path.stem, + "extension": ( + snapshot.get("extension") + if snapshot.get("extension") is not None + else path.suffix[1:] if not is_dir else None + ), + "size": size if size is not None else snapshot.get("size"), + "modify_time": ( + modify_time + if modify_time is not None + else snapshot.get("modify_time") + ), + }) + return fileitem, False def __admit_transfer(self, task: TransferTask) -> TransferAdmission: """在内存入队前持久化源文件并返回稳定任务身份。""" fileitem = task.fileitem if task else None if not fileitem or not fileitem.storage or not fileitem.path: raise ValueError("整理任务缺少源文件身份") + planning_input = task.planning_input or self.__build_planning_input(task) + task.bind_planning_input(planning_input) return self._transfer_admissions.admit( storage=fileitem.storage, src_path=fileitem.path, + planning_input=planning_input, ) + @staticmethod + def __json_snapshot(value: Any) -> Any: + """把受控领域对象投影为可持久化 JSON 值。""" + if value is None: + return None + return to_jsonable_python(value, serialize_unknown=True) + + def __build_planning_input( + self, + task: TransferTask, + *, + cleanup_dest_fileitem: Optional[FileItem] = None, + ) -> TransferPlanningInput: + """冻结准入时已知的请求参数，供 accepted 任务跨重启重新规划。""" + target_directory = task.target_directory + options = { + "scrape": task.scrape, + "library_type_folder": task.library_type_folder, + "library_category_folder": task.library_category_folder, + "manual": task.manual, + "background": task.background, + "username": task.username, + "downloader": task.downloader, + "download_hash": task.download_hash, + "cleanup_dest_fileitem": self.__json_snapshot(cleanup_dest_fileitem), + "_meta_kind": type(task.meta).__name__ if task.meta else None, + "_mediainfo_kind": ( + type(task.mediainfo).__name__ if task.mediainfo else None + ), + } + return TransferPlanningInput( + source_fileitem=self.__json_snapshot(task.fileitem), + meta=self.__json_snapshot(task.meta), + mediainfo=self.__json_snapshot(task.mediainfo), + target_directory=self.__json_snapshot(target_directory), + target_storage=task.target_storage, + target_path=task.target_path.as_posix() if task.target_path else None, + requested_transfer_type=task.transfer_type, + media_source=task.media_source.value if task.media_source else None, + media_id=task.media_id, + media_type=task.mtype.value if task.mtype else None, + need_scrape=bool(task.scrape), + need_rename=bool(target_directory.renaming) if target_directory else True, + need_notify=bool(target_directory.notify) if target_directory else False, + overwrite_mode=(target_directory.overwrite_mode if target_directory else None), + episodes_info=tuple( + self.__json_snapshot(episode) for episode in (task.episodes_info or []) + ), + preview=bool(task.preview), + options=options, + ) + + def __build_provider_invocation_snapshot( + self, + task: TransferTask, + ) -> TransferProviderInvocationSnapshot: + """冻结目录解析后的旧 ABI 参数，并保留 None 与 False 的原值差异。""" + return TransferProviderInvocationSnapshot( + fileitem=self.__json_snapshot(task.fileitem), + meta=self.__json_snapshot(task.meta), + meta_kind=type(task.meta).__name__ if task.meta else None, + mediainfo=self.__json_snapshot(task.mediainfo), + mediainfo_kind=( + type(task.mediainfo).__name__ if task.mediainfo else None + ), + target_directory=self.__json_snapshot(task.target_directory), + target_storage=task.target_storage, + target_path=task.target_path.as_posix() if task.target_path else None, + transfer_type=task.transfer_type, + scrape=task.scrape, + library_type_folder=task.library_type_folder, + library_category_folder=task.library_category_folder, + episodes_info=tuple( + self.__json_snapshot(episode) for episode in (task.episodes_info or []) + ), + preview=bool(task.preview), + ) + + @staticmethod + def __restore_meta_snapshot( + payload: Optional[dict[str, Any]], + kind: object, + ) -> MetaBase: + """从持久快照恢复元数据对象，不重新运行文件名解析。""" + if not payload: + raise TransferPlanningStateError("整理持久快照缺少已解析元数据") + if kind == "MetaMusic" or payload.get("type") == MediaType.MUSIC.value: + return MetaMusic.from_dict(payload) + meta_class = ( + MetaAnime + if kind == "MetaAnime" + else MetaVideo + ) + meta = meta_class("") + for key, value in payload.items(): + descriptor = getattr(meta_class, key, None) + if isinstance(descriptor, property) and descriptor.fset is None: + continue + if key == "type" and isinstance(value, str): + value = MediaType(value) + elif key == "media_source" and isinstance(value, str): + value = MediaSource(value) + setattr(meta, key, value) + return meta + + @staticmethod + def __restore_mediainfo_snapshot( + payload: Optional[dict[str, Any]], + kind: object, + ) -> Union[MediaInfo, MusicInfo]: + """从持久快照恢复媒体信息，不访问任何在线数据源。""" + if not payload: + raise TransferPlanningStateError("整理持久快照缺少已识别媒体信息") + if kind == "MusicInfo" or payload.get("type") == MediaType.MUSIC.value: + return MusicInfo.from_dict(payload) + mediainfo = MediaInfo() + mediainfo.from_dict(payload) + return mediainfo + + def __restore_planned_task(self, task: TransferTask) -> None: + """用冻结检查点覆盖易受配置和在线识别变化影响的任务字段。""" + checkpoint = task.plan_checkpoint + if checkpoint is None: + raise TransferPlanningStateError("planned 任务缺少整理计划检查点") + if checkpoint.provider_invocation is not None: + invocation = checkpoint.provider_invocation + task.fileitem = FileItem.model_validate(invocation.fileitem) + task.meta = self.__restore_meta_snapshot( + invocation.meta, + invocation.meta_kind, + ) + task.mediainfo = self.__restore_mediainfo_snapshot( + invocation.mediainfo, + invocation.mediainfo_kind, + ) + task.target_directory = ( + TransferDirectoryConf.model_validate(invocation.target_directory) + if invocation.target_directory + else None + ) + task.target_storage = invocation.target_storage + task.target_path = ( + Path(invocation.target_path) if invocation.target_path else None + ) + task.transfer_type = invocation.transfer_type + task.scrape = invocation.scrape + task.library_type_folder = invocation.library_type_folder + task.library_category_folder = invocation.library_category_folder + task.episodes_info = [ + TmdbEpisode.model_validate(item) + for item in invocation.episodes_info + ] + task.mark_planning_context_restored() + return + if checkpoint.resolved_meta: + task.meta = self.__restore_meta_snapshot( + checkpoint.resolved_meta, + checkpoint.resolved_meta_kind, + ) + elif task.meta is None: + raise TransferPlanningStateError("整理计划检查点缺少已解析元数据") + if checkpoint.resolved_mediainfo: + task.mediainfo = self.__restore_mediainfo_snapshot( + checkpoint.resolved_mediainfo, + checkpoint.resolved_mediainfo_kind, + ) + elif task.mediainfo is None: + raise TransferPlanningStateError("整理计划检查点缺少已识别媒体信息") + task.episodes_info = [ + TmdbEpisode.model_validate(item) + for item in checkpoint.resolved_episodes_info + ] + task.target_storage = checkpoint.target_storage + task.target_path = Path(checkpoint.root_target_path) + task.transfer_type = checkpoint.resolved_transfer_type + task.scrape = checkpoint.need_scrape + task.mark_planning_context_restored() + + def __select_storage_oper(self, storage: Optional[str]) -> Any: + """按冻结存储标识请求插件存储适配器，未接管时返回空值。""" + event_data = StorageOperSelectionEventData(storage=storage) + event = self.eventmanager.send_event( + ChainEventType.StorageOperSelection, + event_data, + ) + if event and event.event_data: + resolved = event.event_data + return resolved.storage_oper + return None + + def __record_uncheckpointed_failure( + self, + task: TransferTask, + error: object, + ) -> None: + """记录 checkpoint 前失败，保留 accepted 任务供后续重新规划。""" + if task.preview or task.plan_checkpoint or not task.admission_task_id: + return + try: + self._transfer_admissions.record_planning_failure( + task_id=task.admission_task_id, + error=str(error), + ) + except Exception as record_error: + logger.error(f"记录整理规划前失败原因出错：{record_error}") + + def __handle_planned_transfer( + self, + task: TransferTask, + callback: Optional[Callable[[TransferTask, TransferInfo], Tuple[bool, str]]], + ) -> Tuple[bool, str]: + """直接执行已提交检查点，不再触发识别、分类、选目录或重命名。""" + self.__restore_planned_task(task) + self.jobview.running_task(task) + checkpoint = task.plan_checkpoint + assert checkpoint is not None + target_storage = ( + checkpoint.provider_invocation.target_storage + if checkpoint.provider_invocation + else checkpoint.target_storage + ) + transferinfo = self._plan_checkpoint_and_execute( + task, + source_oper=self.__select_storage_oper(task.fileitem.storage), + target_oper=self.__select_storage_oper(target_storage), + ) + if not transferinfo: + raise RuntimeError("文件整理模块未返回检查点执行结果") + if callback: + return callback(task, transferinfo) + return transferinfo.success, transferinfo.message or "" + + def _plan_checkpoint_and_execute( + self, + task: TransferTask, + *, + source_oper: Any = None, + target_oper: Any = None, + ) -> TransferInfo: + """先提交冻结 provider 调用，空结果时再提交并执行宿主计划。""" + planning_input = task.planning_input or self.__build_planning_input(task) + task.bind_planning_input(planning_input) + if not task.preview and not task.admission_task_id: + admission = self.__admit_transfer(task) + task.bind_admission_task_id(admission.task_id) + + checkpoint = task.plan_checkpoint + if checkpoint is None: + try: + frozen_providers = tuple( + TransferProviderReference( + plugin_id=provider.plugin_id, + plugin_name=provider.plugin_name, + method=provider.method, + ) + for provider in self._module_dispatcher.freeze_plugin_providers( + "transfer" + ) + ) + if frozen_providers: + invocation = self.__build_provider_invocation_snapshot(task) + checkpoint = TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="", + root_target_path="", + final_target_path="", + resolved_transfer_type="", + items=(), + resolved_meta=invocation.meta, + resolved_meta_kind=invocation.meta_kind, + resolved_mediainfo=invocation.mediainfo, + resolved_mediainfo_kind=invocation.mediainfo_kind, + resolved_episodes_info=invocation.episodes_info, + legacy_transfer_providers=frozen_providers, + provider_invocation=invocation, + preview=invocation.preview, + ) + else: + checkpoint = self.__plan_host_transfer( + task, + planning_input=planning_input, + source_oper=source_oper, + ) + checkpoint = self.__persist_transfer_checkpoint( + task, + planning_input=planning_input, + checkpoint=checkpoint, + ) + task.bind_plan_checkpoint(checkpoint) + except Exception as error: + self.__record_checkpoint_failure(task, error) + raise + + self.__restore_planned_task(task) + + legacy_result = self.__execute_legacy_transfer_providers( + task, + checkpoint=checkpoint, + source_oper=source_oper, + target_oper=target_oper, + ) + if legacy_result is not None: + return legacy_result + + if checkpoint.is_provider_pending: + try: + checkpoint = replace( + self.__plan_host_transfer( + task, + planning_input=planning_input, + source_oper=source_oper, + ), + pre_execution_cleanup_completed=True, + ) + checkpoint = self.__persist_transfer_checkpoint( + task, + planning_input=planning_input, + checkpoint=checkpoint, + ) + task.bind_plan_checkpoint(checkpoint) + self.__restore_planned_task(task) + except Exception as error: + self.__record_checkpoint_failure(task, error) + raise + + result = self.execute_transfer_plan( + checkpoint, + meta=task.meta, + mediainfo=task.mediainfo, + source_oper=source_oper, + target_oper=target_oper, + cleanup_media_file=self.__cleanup_transfer_destination, + ) + if result is None: + raise RuntimeError("文件整理模块未返回检查点执行结果") + return result + + def __plan_host_transfer( + self, + task: TransferTask, + *, + planning_input: TransferPlanningInput, + source_oper: Any, + ) -> TransferPlanCheckpoint: + """在 provider 未接管时生成纯宿主计划，不执行任何文件写入。""" + checkpoint = self.plan_transfer( + fileitem=task.fileitem, + meta=cast(MetaBase, task.meta), + mediainfo=cast(Union[MediaInfo, MusicInfo], task.mediainfo), + target_directory=task.target_directory, + target_storage=task.target_storage, + target_path=task.target_path, + transfer_type=task.transfer_type, + episodes_info=task.episodes_info, + scrape=task.scrape, + library_type_folder=task.library_type_folder, + library_category_folder=task.library_category_folder, + source_oper=source_oper, + preview=bool(task.preview), + planning_input=planning_input, + ) + if checkpoint is None: + raise RuntimeError("文件整理模块未返回规划检查点") + return checkpoint + + def __persist_transfer_checkpoint( + self, + task: TransferTask, + *, + planning_input: TransferPlanningInput, + checkpoint: TransferPlanCheckpoint, + ) -> TransferPlanCheckpoint: + """提交当前阶段检查点，并只消费仓储回读的持久投影。""" + if task.preview: + return checkpoint + if not task.admission_task_id: + raise RuntimeError("整理计划提交前缺少持久任务身份") + persisted = self._transfer_admissions.checkpoint_plan( + task_id=task.admission_task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + if persisted.checkpoint is None: + raise TransferPlanningStateError("持久投影缺少整理检查点") + return persisted.checkpoint + + def __record_checkpoint_failure( + self, + task: TransferTask, + error: Exception, + ) -> None: + """记录当前 checkpoint 阶段失败并保留原状态供重启恢复。""" + if task.preview or not task.admission_task_id: + return + try: + self._transfer_admissions.record_planning_failure( + task_id=task.admission_task_id, + error=str(error), + ) + setattr(error, "_transfer_planning_failure_recorded", True) + except Exception as record_error: + logger.error(f"记录整理规划失败原因出错：{record_error}") + + def __execute_legacy_transfer_providers( + self, + task: TransferTask, + *, + checkpoint: TransferPlanCheckpoint, + source_oper: Any, + target_oper: Any, + ) -> Optional[TransferInfo]: + """在 checkpoint 提交后按冻结身份执行旧插件 transfer provider。""" + providers = checkpoint.legacy_transfer_providers + if not providers: + return None + invocation = checkpoint.provider_invocation + if invocation is None: + raise TransferPlanningStateError( + "冻结旧 transfer provider 缺少调用快照" + ) + target_directory = ( + TransferDirectoryConf.model_validate(invocation.target_directory) + if invocation.target_directory + else None + ) + provider_meta = ( + self.__restore_meta_snapshot(invocation.meta, invocation.meta_kind) + if invocation.meta + else task.meta + ) + provider_mediainfo = ( + self.__restore_mediainfo_snapshot( + invocation.mediainfo, + invocation.mediainfo_kind, + ) + if invocation.mediainfo + else task.mediainfo + ) + provider_episodes = [ + TmdbEpisode.model_validate(item) + for item in invocation.episodes_info + ] + result = self._module_dispatcher.execute_frozen_plugin_providers( + "transfer", + providers, + before_invoke=lambda: self.__prepare_legacy_provider_execution( + checkpoint=checkpoint + ), + fileitem=FileItem.model_validate(invocation.fileitem), + meta=provider_meta, + mediainfo=provider_mediainfo, + target_directory=target_directory, + target_storage=invocation.target_storage, + target_path=(Path(invocation.target_path) if invocation.target_path else None), + transfer_type=invocation.transfer_type, + scrape=invocation.scrape, + library_type_folder=invocation.library_type_folder, + library_category_folder=invocation.library_category_folder, + episodes_info=provider_episodes, + source_oper=source_oper, + target_oper=target_oper, + preview=invocation.preview, + ) + if result is None or isinstance(result, TransferInfo): + return result + raise TypeError("旧插件 transfer provider 返回了不支持的结果类型") + + def __prepare_legacy_provider_execution( + self, + *, + checkpoint: TransferPlanCheckpoint, + ) -> None: + """按旧批次语义在 provider 副作用前执行冻结 cleanup intent。""" + invocation = checkpoint.provider_invocation + if invocation is None: + raise TransferPlanningStateError("旧 provider 执行缺少调用快照") + if invocation.preview: + return + + cleanup_payload = checkpoint.planning_input.options.get( + "cleanup_dest_fileitem" + ) + if isinstance(cleanup_payload, dict): + cleanup_fileitem = FileItem.model_validate(cleanup_payload) + if not self.__cleanup_transfer_destination(cleanup_fileitem): + raise RuntimeError( + f"{cleanup_fileitem.path} 删除失败，整理计划保留待重试" + ) + + def __cleanup_transfer_destination(self, fileitem: FileItem) -> bool: + """经插件兼容的 StorageChain 幂等删除旧目标并治理父空目录。""" + storage_chain = self._transfer_storage_chain() + if not fileitem.path: + return False + current_item = storage_chain.get_file_item_strict( + storage=fileitem.storage, + path=Path(fileitem.path), + ) + if current_item is None: + return True + return bool(storage_chain.delete_media_file(current_item)) + + def execute_legacy_transfer_command( + self, + fileitem: FileItem, + meta: MetaBase, + mediainfo: Union[MediaInfo, MusicInfo], + target_directory: Optional[TransferDirectoryConf] = None, + target_storage: Optional[str] = None, + target_path: Optional[Path] = None, + transfer_type: Optional[str] = None, + scrape: Optional[bool] = None, + library_type_folder: Optional[bool] = None, + library_category_folder: Optional[bool] = None, + episodes_info: Optional[List[TmdbEpisode]] = None, + source_oper: Any = None, + target_oper: Any = None, + preview: bool = False, + ) -> TransferInfo: + """把旧同步 ABI 适配为准入、检查点和终态结算共用的唯一命令。""" + task = TransferTask( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_directory=target_directory, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + scrape=scrape, + library_type_folder=library_type_folder, + library_category_folder=library_category_folder, + episodes_info=episodes_info, + manual=True, + background=False, + preview=preview, + ) + try: + result = self._plan_checkpoint_and_execute( + task, + source_oper=source_oper, + target_oper=target_oper, + ) + except Exception as error: + logger.error(f"旧整理兼容命令执行失败：{error}") + return TransferInfo( + success=False, + message=str(error), + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + ) + self.__discard_pending(task) + return result + def __record_enqueue_failure( self, task: TransferTask, @@ -1131,13 +1905,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo if marker: marker(task) - def __finish_job_execution(self, task: TransferTask): - """在作业视图支持执行租约时标记主程序任务结束执行。""" + def __finish_job_execution(self, task: TransferTask, *, terminal: bool = True): + """结束内存执行；仅确定到达业务终态时注销 durable 记录。""" marker = getattr(self.jobview, "finish_execution", None) if marker: marker(task) - # 任务已到终态，落盘登记到此作废（未登记过的实时整理路径为无害空操作） - self.__discard_pending(task) + if terminal: + self.__discard_pending(task) def __expire_stale_transfer_tasks(self): """清理外部接管后失去状态心跳的运行中整理任务。""" @@ -1255,6 +2029,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # 增加运行中的任务数 self._active_tasks += 1 + terminal = False try: self.__start_job_execution(task) # 更新进度 @@ -1271,6 +2046,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo state, err_msg = self.__handle_transfer( task=task, callback=item.callback ) + terminal = task.plan_checkpoint is not None with task_lock: if not state: @@ -1295,7 +2071,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._processed_num += 1 self._fail_num += 1 finally: - self.__finish_job_execution(task) + self.__finish_job_execution(task, terminal=terminal) self._queue.task_done() with task_lock: # 减少运行中的任务数 @@ -1312,11 +2088,27 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo def __handle_transfer( self, task: TransferTask, callback: Optional[Callable] = None + ) -> Optional[Tuple[bool, str]]: + """执行整理并统一记录 checkpoint 前的返回失败或异常。""" + try: + result = self.__perform_transfer(task, callback) + except Exception as error: + if not getattr(error, "_transfer_planning_failure_recorded", False): + self.__record_uncheckpointed_failure(task, error) + raise + if result and not result[0]: + self.__record_uncheckpointed_failure(task, result[1]) + return result + + def __perform_transfer( + self, task: TransferTask, callback: Optional[Callable] = None ) -> Optional[Tuple[bool, str]]: """ 处理整理任务 """ try: + if task.plan_checkpoint is not None: + return self.__handle_planned_transfer(task, callback) # 识别 transferhis = get_chain_transfer_history_port() # 显式标注联合：下面既会赋回音乐识别结果（MusicInfo），也会赋回影视识别 @@ -1446,8 +2238,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo mediainfo_changed = True - # TMDB 仅作为辅助信息合并，不能改变原识别源的主身份和标题。 - mediainfo = MediaChain().supplement_tmdb_info(mediainfo, task.meta) + # accepted/planned 恢复使用持久快照；在线补充会让首次执行与重放产生漂移。 + if not task.planning_context_restored: + mediainfo = MediaChain().supplement_tmdb_info(mediainfo, task.meta) task.mediainfo = mediainfo # 只有 TMDB 主源沿用历史 TMDB 标题，避免辅助 ID 改写其它识别源标题。 @@ -1477,6 +2270,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo task.mediainfo.type == MediaType.TV and task.mediainfo.tmdb_id and not task.episodes_info + and not task.planning_context_restored ): # 判断注意season为0的情况 season_num = task.mediainfo.season @@ -1534,54 +2328,15 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # 正在处理 self.jobview.running_task(task) - # 广播事件，请示额外的源存储支持 - source_oper = None - source_event_data = StorageOperSelectionEventData( - storage=task.fileitem.storage, - ) - source_event = self.eventmanager.send_event( - ChainEventType.StorageOperSelection, source_event_data - ) - # 使用事件返回的上下文数据 - if source_event and source_event.event_data: - source_event_data: StorageOperSelectionEventData = ( - source_event.event_data - ) - if source_event_data.storage_oper: - source_oper = source_event_data.storage_oper + # 广播事件，请示额外的源、目标存储支持。 + source_oper = self.__select_storage_oper(task.fileitem.storage) + target_oper = self.__select_storage_oper(task.target_storage) - # 广播事件，请示额外的目标存储支持 - target_oper = None - target_event_data = StorageOperSelectionEventData( - storage=task.target_storage, - ) - target_event = self.eventmanager.send_event( - ChainEventType.StorageOperSelection, target_event_data - ) - # 使用事件返回的上下文数据 - if target_event and target_event.event_data: - target_event_data: StorageOperSelectionEventData = ( - target_event.event_data - ) - if target_event_data.storage_oper: - target_oper = target_event_data.storage_oper - - # 执行整理 - transferinfo: TransferInfo = self.transfer( - fileitem=task.fileitem, - meta=task.meta, - mediainfo=task.mediainfo, - target_directory=task.target_directory, - target_storage=task.target_storage, - target_path=task.target_path, - transfer_type=task.transfer_type, - episodes_info=task.episodes_info, - scrape=task.scrape, - library_type_folder=task.library_type_folder, - library_category_folder=task.library_category_folder, + # 纯规划先提交 durable checkpoint，任何文件副作用只能发生在提交之后。 + transferinfo = self._plan_checkpoint_and_execute( + task, source_oper=source_oper, target_oper=target_oper, - preview=task.preview, ) if not transferinfo: logger.error("文件整理模块运行失败") @@ -2144,6 +2899,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo cleanup_dest_fileitem: Optional[FileItem] = None, continue_callback: Callable = None, reorganize: Optional[bool] = False, + recovery_admission: Optional[TransferAdmission] = None, ) -> Tuple[bool, Union[str, dict]]: """ 执行一个复杂目录的整理操作 @@ -2173,6 +2929,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo :param sync_extra_files: 是否在整理主视频文件时同步整理同媒体附加文件 :param cleanup_dest_fileitem: 确认存在待整理任务后需要清理的旧目标文件 :param continue_callback: 继续处理回调 + :param recovery_admission: 内部恢复调用绑定的既有 durable 记录 返回：成功标识，错误信息 """ mediainfo, media_source, media_id, identity_error = ( @@ -2601,10 +3358,6 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo file_items, inherited_meta_map = _plan_file_items(file_items) planned_file_count = len(file_items) - if cleanup_dest_fileitem and planned_file_count and not preview: - state = StorageChain().delete_media_file(cleanup_dest_fileitem) - if not state: - return False, f"{cleanup_dest_fileitem.path} 删除失败" if preview: logger.info(f"正在预览 {planned_file_count} 个文件的整理路径...") @@ -2615,6 +3368,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo transfer_tasks: List[TransferTask] = [] skipped_history_count = 0 skipped_torrents = set() + cleanup_intent_assigned = False try: for file_item, bluray_dir in file_items: if runtime_stop_state.is_system_stopped: @@ -2777,6 +3531,31 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo background=background, preview=preview, ) + cleanup_intent = ( + cleanup_dest_fileitem + if not preview and not cleanup_intent_assigned + else None + ) + transfer_task.bind_planning_input( + self.__build_planning_input( + transfer_task, + cleanup_dest_fileitem=cleanup_intent, + ) + ) + if ( + recovery_admission + and file_item.storage == recovery_admission.storage + and file_item.path == recovery_admission.src_path + ): + transfer_task.bind_admission_task_id(recovery_admission.task_id) + if recovery_admission.planning_input: + transfer_task.bind_planning_input( + recovery_admission.planning_input + ) + if recovery_admission.checkpoint: + transfer_task.bind_plan_checkpoint( + recovery_admission.checkpoint + ) if background: try: queued = self.put_to_queue(task=transfer_task) @@ -2787,6 +3566,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo logger.error(message) continue if queued: + if cleanup_intent: + cleanup_intent_assigned = True logger.info(f"{file_path.name} 已添加到整理队列") else: logger.debug(f"{file_path.name} 已在整理队列中，跳过") @@ -2795,6 +3576,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo if self.__put_to_jobview(transfer_task): self._register_scrape_batch_task(transfer_task) transfer_tasks.append(transfer_task) + if cleanup_intent: + cleanup_intent_assigned = True else: logger.debug(f"{file_path.name} 已在整理列表中，跳过") except OperationInterrupted: @@ -2805,7 +3588,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._close_scrape_batch(transfer_batch_id) # 实时整理 - preview_items: List[dict] = [] + preview_items: List[dict[str, Any]] = [] def _preview_callback(task: TransferTask, transferinfo: TransferInfo) -> Tuple[bool, str]: item_meta = task.meta @@ -2867,12 +3650,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo "finished": finished_files, }, ) + terminal = False try: self.__start_job_execution(transfer_task) state, err_msg = self.__handle_transfer( task=transfer_task, callback=_preview_callback if preview else self.__default_callback, ) + terminal = bool(preview or transfer_task.plan_checkpoint is not None) except Exception as e: logger.error( f"{transfer_task.fileitem.name} 整理任务处理出现错误：" @@ -2882,7 +3667,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__fail_transfer_task(transfer_task) state, err_msg = False, str(e) finally: - self.__finish_job_execution(transfer_task) + self.__finish_job_execution( + transfer_task, + terminal=terminal, + ) if not state: all_success = False logger.warn(f"{transfer_task.fileitem.name} {err_msg}") diff --git a/app/db/adapters/transfer.py b/app/db/adapters/transfer.py index b89163d39..fe29fd8af 100644 --- a/app/db/adapters/transfer.py +++ b/app/db/adapters/transfer.py @@ -2,6 +2,7 @@ from collections.abc import Callable from datetime import datetime +from typing import Optional from uuid import uuid4 from sqlalchemy.exc import IntegrityError @@ -9,7 +10,13 @@ from sqlalchemy.orm import Session from app.application.transfer import ( TRANSFER_ADMISSION_ACCEPTED, + TRANSFER_ADMISSION_PLANNED, + TRANSFER_ADMISSION_PROVIDER_PENDING, TransferAdmission, + TransferAdmissionConflictError, + TransferPlanCheckpoint, + TransferPlanningInput, + TransferPlanningStateError, ) from app.db.models.transferpending import TransferPending from app.db.oper.transferpending import TransferPendingOper @@ -32,6 +39,40 @@ class TransactionalTransferAdmissionRepository: def _project(pending: TransferPending) -> TransferAdmission: """在 Session 有效期内把 ORM 行冻结为应用层 DTO。""" created_at = pending.created_at or pending.updated_at + planning_input = TransferPlanningInput.from_payload(pending.planning_input) + if pending.input_version != planning_input.schema_version: + raise TransferPlanningStateError("整理规划输入列版本与 JSON 版本不一致") + if pending.input_fingerprint != planning_input.fingerprint: + raise TransferAdmissionConflictError("整理规划输入 JSON 与持久指纹不一致") + checkpoint = ( + TransferPlanCheckpoint.from_payload(pending.checkpoint_payload) + if pending.checkpoint_payload is not None + else None + ) + if checkpoint is not None: + if pending.checkpoint_version != checkpoint.schema_version: + raise TransferPlanningStateError("整理检查点列版本与 JSON 版本不一致") + if checkpoint.planning_input.fingerprint != pending.input_fingerprint: + raise TransferAdmissionConflictError("整理检查点内嵌输入与准入指纹不一致") + if pending.state in { + TRANSFER_ADMISSION_PROVIDER_PENDING, + TRANSFER_ADMISSION_PLANNED, + } and checkpoint is None: + raise TransferPlanningStateError("待执行任务缺少完整检查点") + if pending.state == TRANSFER_ADMISSION_ACCEPTED and checkpoint is not None: + raise TransferPlanningStateError("接纳态任务不能携带计划检查点") + if ( + pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING + and checkpoint is not None + and not checkpoint.is_provider_pending + ): + raise TransferPlanningStateError("provider_pending 状态缺少 provider 调用快照") + if ( + pending.state == TRANSFER_ADMISSION_PLANNED + and checkpoint is not None + and checkpoint.is_provider_pending + ): + raise TransferPlanningStateError("planned 状态不能携带 provider-only 检查点") return TransferAdmission( task_id=pending.task_id, storage=pending.storage, @@ -40,10 +81,39 @@ class TransactionalTransferAdmissionRepository: created_at=created_at, updated_at=pending.updated_at, last_error=pending.last_error, + input_fingerprint=pending.input_fingerprint, + planning_input=planning_input, + checkpoint=checkpoint, ) - def admit(self, *, storage: str, src_path: str) -> TransferAdmission: - """幂等持久化准入事实，并返回跨重启稳定的任务标识。""" + @staticmethod + def _assert_input_match( + pending: TransferPending, + planning_input: TransferPlanningInput, + ) -> None: + """拒绝同一源文件以不同规划输入复用既有任务身份。""" + if pending.input_fingerprint != planning_input.fingerprint: + raise TransferAdmissionConflictError( + f"整理源文件已按不同输入准入: {pending.storage}:{pending.src_path}" + ) + + def admit( + self, + *, + storage: str, + src_path: str, + planning_input: Optional[TransferPlanningInput] = None, + ) -> TransferAdmission: + """按输入指纹幂等持久化准入事实，并返回跨重启稳定身份。""" + effective_input = planning_input or TransferPlanningInput.legacy( + storage=storage, + src_path=src_path, + ) + if ( + effective_input.source_fileitem.get("storage") != storage + or effective_input.source_fileitem.get("path") != src_path + ): + raise ValueError("整理规划输入的源文件身份与准入参数不一致") now_time = self._now() try: with self._session_factory() as session: @@ -55,10 +125,14 @@ class TransactionalTransferAdmissionRepository: src_path=src_path, state=TRANSFER_ADMISSION_ACCEPTED, now_time=now_time, + input_version=effective_input.schema_version, + planning_input=effective_input.to_payload(), + input_fingerprint=effective_input.fingerprint, ) if pending is None: raise ValueError("整理任务的存储与源路径不能为空") session.flush() + self._assert_input_match(pending, effective_input) admission = self._project(pending) transaction.commit() return admission @@ -74,6 +148,7 @@ class TransactionalTransferAdmissionRepository: ) if pending is None: raise RuntimeError("并发准入冲突后未找到已提交记录") from error + self._assert_input_match(pending, effective_input) return self._project(pending) def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]: @@ -85,6 +160,19 @@ class TransactionalTransferAdmissionRepository: ) return [self._project(pending) for pending in pending_items] + def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]: + """投影接纳、provider 待执行或已规划的全部可恢复任务。""" + with self._session_factory() as session: + pending_items = TransferPendingOper(db=session).list_by_states( + states=( + TRANSFER_ADMISSION_ACCEPTED, + TRANSFER_ADMISSION_PROVIDER_PENDING, + TRANSFER_ADMISSION_PLANNED, + ), + limit=limit, + ) + return [self._project(pending) for pending in pending_items] + def record_enqueue_failure(self, *, task_id: str, error: str) -> None: """独立提交最近一次入队失败，保留准入记录供后续恢复。""" with self._session_factory() as session: @@ -100,6 +188,79 @@ class TransactionalTransferAdmissionRepository: transaction.rollback() raise + def checkpoint_plan( + self, + *, + task_id: str, + input_fingerprint: str, + checkpoint: TransferPlanCheckpoint, + ) -> TransferAdmission: + """以输入指纹 CAS 保存 provider 调用快照或升级宿主计划。""" + if checkpoint.planning_input.fingerprint != input_fingerprint: + raise TransferAdmissionConflictError("检查点输入与准入输入指纹不一致") + checkpoint_payload = checkpoint.to_payload() + target_state = ( + TRANSFER_ADMISSION_PROVIDER_PENDING + if checkpoint.is_provider_pending + else TRANSFER_ADMISSION_PLANNED + ) + source_states = ( + (TRANSFER_ADMISSION_ACCEPTED,) + if checkpoint.is_provider_pending + else ( + TRANSFER_ADMISSION_ACCEPTED, + TRANSFER_ADMISSION_PROVIDER_PENDING, + ) + ) + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferPendingOper(db=session) + updated = oper.stage_checkpoint_plan( + task_id=task_id, + input_fingerprint=input_fingerprint, + checkpoint_version=checkpoint.schema_version, + checkpoint_payload=checkpoint_payload, + source_states=source_states, + target_state=target_state, + now_time=self._now(), + ) + session.flush() + session.expire_all() + pending = oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferPlanningStateError(f"未找到整理任务: {task_id}") + if pending.input_fingerprint != input_fingerprint: + raise TransferAdmissionConflictError("整理任务输入指纹已经改变") + if not updated and not ( + pending.state == target_state + and pending.checkpoint_payload == checkpoint_payload + ): + raise TransferPlanningStateError( + f"整理任务不能从状态 {pending.state} 保存 {target_state} 检查点" + ) + admission = self._project(pending) + transaction.commit() + return admission + except Exception: + transaction.rollback() + raise + + def record_planning_failure(self, *, task_id: str, error: str) -> None: + """独立提交规划错误并保持任务处于接纳态供恢复重试。""" + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + TransferPendingOper(db=session).stage_record_planning_failure( + task_id=task_id, + error=error, + now_time=self._now(), + ) + transaction.commit() + except Exception: + transaction.rollback() + raise + def discard_task(self, *, task_id: str) -> int: """在独立事务中按稳定任务标识删除已到终态的准入记录。""" with self._session_factory() as session: diff --git a/app/db/models/transferpending.py b/app/db/models/transferpending.py index bfa870623..dc87e5f1c 100644 --- a/app/db/models/transferpending.py +++ b/app/db/models/transferpending.py @@ -1,13 +1,66 @@ +import hashlib +import json from datetime import datetime -from typing import List, Optional, cast +from typing import Any, List, Optional, cast from uuid import uuid4 -from sqlalchemy import Index, String, Text, UniqueConstraint, delete, select, update +from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update from sqlalchemy.orm import Mapped, Session, mapped_column from app.db.base import Base, execute_dml, get_id_column +def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]: + return { + "schema_version": 1, + "source_fileitem": {"storage": storage, "path": src_path}, + "meta": None, + "mediainfo": None, + "target_directory": None, + "target_storage": None, + "target_path": None, + "requested_transfer_type": None, + "media_source": None, + "media_id": None, + "media_type": None, + "need_scrape": False, + "need_rename": True, + "need_notify": True, + "overwrite_mode": None, + "episodes_info": [], + "preview": False, + "options": {"legacy_replan": True}, + } + + +def _planning_fingerprint(payload: dict[str, Any]) -> str: + canonical = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _default_planning_payload(context: Any) -> dict[str, Any]: + params = context.get_current_parameters() + return _legacy_planning_payload( + params.get("storage", ""), + params.get("src_path", ""), + ) + + +def _default_planning_fingerprint(context: Any) -> str: + params = context.get_current_parameters() + payload = params.get("planning_input") or _legacy_planning_payload( + params.get("storage", ""), + params.get("src_path", ""), + ) + return _planning_fingerprint(payload) + + class TransferPending(Base): """ 待整理文件登记。 @@ -17,9 +70,9 @@ class TransferPending(Base): 蒸发。而已经稳定落地的文件不会再产生任何监控事件，也不会有新的补偿扫描起点 ——结果就是永久漏件，只能靠人工比对补整理。 - 这里只落盘恢复所需的最小事实：稳定任务身份、存储、源文件路径、准入状态和 - 最近入队错误。重启后重新走一遍整理入口，由整理历史查重挡掉已经完成的， - 因此不需要序列化 meta/mediainfo 这些重对象，也不存在识别结果陈旧的问题。 + 准入时保存版本化规划输入和指纹；纯规划完成后以同一行原子保存完整有序计划并 + 推进到 planned。重启恢复可直接消费已规划路径，避免再次触发 rename 等插件事件。 + 旧路径登记接口仍生成最小 legacy_replan 输入，供插件兼容调用方继续使用。 """ id = get_id_column() @@ -42,6 +95,22 @@ class TransferPending(Base): ) # 最近一次入队失败原因 last_error: Mapped[Optional[str]] = mapped_column(Text) + # 规划输入格式版本 + input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + # 版本化规划输入 JSON + planning_input: Mapped[dict[str, Any]] = mapped_column( + JSON, nullable=False, default=_default_planning_payload + ) + # 规划输入规范 JSON 的 SHA-256 指纹 + input_fingerprint: Mapped[str] = mapped_column( + String(64), nullable=False, default=_default_planning_fingerprint + ) + # 完整计划格式版本，尚未规划时为空 + checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer) + # 完整有序计划 JSON，尚未规划时为空 + checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON) + # 规划完成时间 + planned_at: Mapped[Optional[str]] = mapped_column(String(40)) __table_args__ = ( # 同一个文件重复入队只保留一条，回放时不会重复送入整理链 @@ -74,12 +143,16 @@ class TransferPending(Base): ).scalars().first() if pending: return cast("TransferPending", pending) + planning_input = _legacy_planning_payload(storage, src_path) pending = cls( storage=storage, src_path=src_path, state="accepted", created_at=now_time, updated_at=now_time, + input_version=1, + planning_input=planning_input, + input_fingerprint=_planning_fingerprint(planning_input), ) db.add(pending) return pending @@ -87,7 +160,9 @@ class TransferPending(Base): @classmethod def stage_admit(cls, db: Session, *, task_id: str, storage: str, src_path: str, state: str, - now_time: str) -> Optional["TransferPending"]: + now_time: str, input_version: int = 1, + planning_input: Optional[dict[str, Any]] = None, + input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]: """ 在调用方会话中暂存一条持久接纳记录。 @@ -98,6 +173,9 @@ class TransferPending(Base): :param src_path: 源文件路径 :param state: 持久状态 :param now_time: 当前时间 + :param input_version: 规划输入格式版本 + :param planning_input: 版本化规划输入 JSON + :param input_fingerprint: 规划输入规范 JSON 指纹 :return: 接纳记录 """ if not task_id or not storage or not src_path or not state: @@ -107,6 +185,8 @@ class TransferPending(Base): ).scalars().first() if pending: return cast("TransferPending", pending) + effective_input = planning_input or _legacy_planning_payload(storage, src_path) + effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input) pending = cls( task_id=task_id, storage=storage, @@ -114,6 +194,9 @@ class TransferPending(Base): state=state, created_at=now_time, updated_at=now_time, + input_version=input_version, + planning_input=effective_input, + input_fingerprint=effective_fingerprint, ) db.add(pending) return pending @@ -137,6 +220,25 @@ class TransferPending(Base): .limit(limit) ).scalars().all()) + @classmethod + def list_by_states(cls, db: Session, *, states: tuple[str, ...], + limit: Optional[int] = 5000) -> List["TransferPending"]: + """ + 按登记顺序列出多个可恢复持久状态的记录。 + :param db: 数据库会话 + :param states: 允许恢复的状态集合 + :param limit: 单次读取上限 + :return: 接纳记录列表 + """ + if not states: + return [] + return list(db.execute( + select(cls) + .where(cls.state.in_(states)) + .order_by(cls.created_at.asc(), cls.id.asc()) + .limit(limit) + ).scalars().all()) + @classmethod def get_by_identity(cls, db: Session, *, storage: str, src_path: str) -> Optional["TransferPending"]: @@ -159,6 +261,90 @@ class TransferPending(Base): ).scalars().first(), ) + @classmethod + def get_by_task_id(cls, db: Session, *, task_id: str) -> Optional["TransferPending"]: + """ + 按稳定任务标识查询一条持久登记。 + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :return: 接纳记录 + """ + if not task_id: + return None + return cast( + Optional["TransferPending"], + db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(), + ) + + @classmethod + def checkpoint_plan(cls, db: Session, *, task_id: str, + input_fingerprint: str, checkpoint_version: int, + checkpoint_payload: dict[str, Any], + source_states: tuple[str, ...], target_state: str, + now_time: str) -> int: + """ + 以输入指纹为 CAS 条件原子保存计划并推进到已规划。 + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param input_fingerprint: 规划输入规范 JSON 指纹 + :param checkpoint_version: 检查点格式版本 + :param checkpoint_payload: 完整有序计划 JSON + :param source_states: 允许推进检查点的起始状态 + :param target_state: 检查点提交后的目标状态 + :param now_time: 当前时间 + :return: 更新的记录数 + """ + if ( + not task_id + or not input_fingerprint + or not checkpoint_payload + or not source_states + or not target_state + ): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state.in_(source_states), + cls.input_fingerprint == input_fingerprint, + ) + .values( + state=target_state, + checkpoint_version=checkpoint_version, + checkpoint_payload=checkpoint_payload, + planned_at=now_time, + last_error=None, + updated_at=now_time, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def record_planning_failure(cls, db: Session, *, task_id: str, + error: str, now_time: str) -> int: + """ + 为接纳态或 provider 待执行任务记录规划失败，不改变其恢复状态。 + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param error: 失败原因 + :param now_time: 当前时间 + :return: 更新的记录数 + """ + if not task_id: + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state.in_(("accepted", "provider_pending")), + ) + .values(last_error=error, updated_at=now_time), + execution_options={"synchronize_session": False}, + ) + @classmethod def record_enqueue_failure(cls, db: Session, *, task_id: str, error: str, now_time: str) -> int: diff --git a/app/db/oper/transferpending.py b/app/db/oper/transferpending.py index 204cffdcb..531b6ad41 100644 --- a/app/db/oper/transferpending.py +++ b/app/db/oper/transferpending.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple from app.db.base import DbOper from app.db.models.transferpending import TransferPending @@ -32,7 +32,9 @@ class TransferPendingOper(DbOper): ) def stage_admit(self, *, task_id: str, storage: str, src_path: str, - state: str, now_time: str) -> Optional[TransferPending]: + state: str, now_time: str, input_version: int = 1, + planning_input: Optional[dict[str, Any]] = None, + input_fingerprint: Optional[str] = None) -> Optional[TransferPending]: """ 在当前会话中暂存一条持久接纳记录。 @@ -42,6 +44,9 @@ class TransferPendingOper(DbOper): :param src_path: 源文件路径 :param state: 持久状态 :param now_time: 当前时间 + :param input_version: 规划输入格式版本 + :param planning_input: 版本化规划输入 JSON + :param input_fingerprint: 规划输入规范 JSON 指纹 :return: 接纳记录 """ return self._execute_sync_write( @@ -52,6 +57,9 @@ class TransferPendingOper(DbOper): src_path=src_path, state=state, now_time=now_time, + input_version=input_version, + planning_input=planning_input, + input_fingerprint=input_fingerprint, ) ) @@ -71,6 +79,22 @@ class TransferPendingOper(DbOper): ) ) or [] + def list_by_states(self, *, states: tuple[str, ...], + limit: Optional[int] = 5000) -> List[TransferPending]: + """ + 使用当前会话列出多个可恢复状态的记录。 + :param states: 允许恢复的状态集合 + :param limit: 单次读取上限 + :return: ORM 接纳记录列表 + """ + return self._execute_sync_query( + lambda session: TransferPending.list_by_states( + session, + states=states, + limit=limit, + ) + ) or [] + def get_by_identity(self, *, storage: str, src_path: str) -> Optional[TransferPending]: """ @@ -87,6 +111,72 @@ class TransferPendingOper(DbOper): ) ) + def get_by_task_id(self, *, task_id: str) -> Optional[TransferPending]: + """ + 使用当前会话按稳定任务标识查询接纳记录。 + :param task_id: 稳定任务标识 + :return: 接纳记录 + """ + return self._execute_sync_query( + lambda session: TransferPending.get_by_task_id( + session, + task_id=task_id, + ) + ) + + def stage_checkpoint_plan( + self, + *, + task_id: str, + input_fingerprint: str, + checkpoint_version: int, + checkpoint_payload: dict[str, Any], + source_states: tuple[str, ...], + target_state: str, + now_time: str, + ) -> int: + """ + 在当前会话中以输入指纹为条件暂存完整计划检查点。 + :param task_id: 稳定任务标识 + :param input_fingerprint: 规划输入规范 JSON 指纹 + :param checkpoint_version: 检查点格式版本 + :param checkpoint_payload: 完整有序计划 JSON + :param source_states: 允许执行 CAS 的起始状态 + :param target_state: 检查点提交后的目标状态 + :param now_time: 当前时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.checkpoint_plan( + session, + task_id=task_id, + input_fingerprint=input_fingerprint, + checkpoint_version=checkpoint_version, + checkpoint_payload=checkpoint_payload, + source_states=source_states, + target_state=target_state, + now_time=now_time, + ) + ) + + def stage_record_planning_failure(self, *, task_id: str, error: str, + now_time: str) -> int: + """ + 在当前会话中记录规划失败并保持任务处于接纳态。 + :param task_id: 稳定任务标识 + :param error: 失败原因 + :param now_time: 当前时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.record_planning_failure( + session, + task_id=task_id, + error=error, + now_time=now_time, + ) + ) + def stage_record_enqueue_failure(self, *, task_id: str, error: str, now_time: str) -> int: """ diff --git a/app/modules/filemanager/module.py b/app/modules/filemanager/module.py index 153697d9d..dfe77a3af 100644 --- a/app/modules/filemanager/module.py +++ b/app/modules/filemanager/module.py @@ -1,28 +1,28 @@ from pathlib import Path -from typing import Any, Optional, List, Tuple, Union, Dict, Callable - -from app.runtime.settings import get_runtime_setting +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from app.adapters.system.host import SystemUtils +from app.application.directory import DirectoryHelper +from app.application.messaging.message import MessageHelper +from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput from app.domain.context import MediaInfo, MusicInfo from app.domain.meta.metabase import MetaBase from app.domain.meta.metamusic import MetaMusic from app.domain.metainfo import MetaInfo -from app.application.directory import DirectoryHelper -from app.application.messaging.message import MessageHelper +from app.foundation import text as text_tools from app.foundation.reflection import ModuleHelper -from app.runtime.log import logger from app.modules import _ModuleBase from app.modules.filemanager.storages import StorageBase from app.modules.filemanager.transhandler import TransHandler -from app.schemas.transfer import TransferInfo -from app.schemas.mediaserver import ExistMediaInfo -from app.schemas.tmdb import TmdbEpisode -from app.schemas.system import TransferDirectoryConf -from app.schemas.workflow import FileItem +from app.runtime.log import logger +from app.runtime.settings import get_runtime_setting from app.schemas.file import StorageUsage +from app.schemas.mediaserver import ExistMediaInfo +from app.schemas.system import TransferDirectoryConf +from app.schemas.tmdb import TmdbEpisode +from app.schemas.transfer import TransferInfo from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaType, ModuleType, OtherModulesType, StorageAction -from app.adapters.system.host import SystemUtils -from app.foundation import text as text_tools +from app.schemas.workflow import FileItem class FileManagerModule(_ModuleBase): @@ -345,7 +345,7 @@ class FileManagerModule(_ModuleBase): def get_file_item(self, storage: str, path: Path) -> Optional[FileItem]: """ - 根据路径获取文件项 + 根据路径严格获取文件项；普通兼容调度仍会隔离异常并投影为空结果。 """ if storage not in self._support_storages: return None @@ -353,7 +353,7 @@ class FileManagerModule(_ModuleBase): if not storage_oper: logger.error(f"不支持 {storage} 的文件获取") return None - return storage_oper.get_item(path) + return storage_oper.get_item_strict(path) def get_parent_item(self, fileitem: FileItem) -> Optional[FileItem]: """ @@ -391,16 +391,24 @@ class FileManagerModule(_ModuleBase): previous_snapshot=previous_snapshot ) - def transfer(self, fileitem: FileItem, meta: MetaBase, mediainfo: MediaInfo, - target_directory: TransferDirectoryConf = None, - target_storage: Optional[str] = None, target_path: Path = None, - transfer_type: Optional[str] = None, scrape: Optional[bool] = None, - library_type_folder: Optional[bool] = None, library_category_folder: Optional[bool] = None, - episodes_info: List[TmdbEpisode] = None, - source_oper: Callable = None, target_oper: Callable = None, - preview: Optional[bool] = False) -> TransferInfo: + def plan_transfer( + self, + fileitem: FileItem, + meta: MetaBase, + mediainfo: Union[MediaInfo, MusicInfo], + target_directory: Optional[TransferDirectoryConf] = None, + target_storage: Optional[str] = None, + target_path: Optional[Path] = None, + transfer_type: Optional[str] = None, scrape: Optional[bool] = None, + library_type_folder: Optional[bool] = None, + library_category_folder: Optional[bool] = None, + episodes_info: Optional[List[TmdbEpisode]] = None, + source_oper: Optional[StorageBase] = None, + preview: Optional[bool] = False, + planning_input: Optional[TransferPlanningInput] = None, + ) -> TransferPlanCheckpoint: """ - 文件整理 + 解析整理策略并生成零写副作用的冻结计划。 :param fileitem: 文件信息 :param meta: 预识别的元数据 :param mediainfo: 识别的媒体信息 @@ -413,29 +421,26 @@ class FileManagerModule(_ModuleBase): :param library_category_folder: 是否按媒体类别创建目录 :param episodes_info: 当前季的全部集信息 :param source_oper: 源存储操作对象 - :param target_oper: 目标存储操作对象 - :return: {path, target_path, message} + :param planning_input: admission 阶段冻结的原始请求，传入时不得改写 + :return: 可持久化的整理计划检查点 """ handler = TransHandler() # 检查目录路径 - if fileitem.storage == "local" and not Path(fileitem.path).exists(): - return TransferInfo(success=False, - fileitem=fileitem, - message=f"{fileitem.path} 不存在") + if ( + fileitem.storage == "local" + and (not fileitem.path or not Path(fileitem.path).exists()) + ): + raise ValueError(f"{fileitem.path} 不存在") # 目标路径不能是文件 if target_path and target_path.is_file(): logger.error(f"整理目标路径 {target_path} 是一个文件") - return TransferInfo(success=False, - fileitem=fileitem, - message=f"{target_path} 不是有效目录") + raise ValueError(f"{target_path} 不是有效目录") # 获取目标路径 if target_directory: # 目标媒体库目录未设置 if not target_directory.library_path: logger.error(f"目标媒体库目录未设置，无法整理文件，源路径：{fileitem.path}") - return TransferInfo(success=False, - fileitem=fileitem, - message="目标媒体库目录未设置") + raise ValueError("目标媒体库目录未设置") # 整理方式 if not transfer_type: transfer_type = target_directory.transfer_type @@ -467,56 +472,141 @@ class FileManagerModule(_ModuleBase): # 未找到有效的媒体库目录 logger.error( f"{mediainfo.type.value if mediainfo.type else '未知类型'} {mediainfo.title_year} 未找到有效的媒体库目录，无法整理文件，源路径：{fileitem.path}") - return TransferInfo(success=False, - fileitem=fileitem, - message="未找到有效的媒体库目录") + raise ValueError("未找到有效的媒体库目录") # 整理方式 if not transfer_type: - logger.error(f"{target_directory.name} 未设置整理方式") - return TransferInfo(success=False, - fileitem=fileitem, - message=f"{target_directory.name} 未设置整理方式") + directory_name = target_directory.name if target_directory else "目标目录" + logger.error(f"{directory_name} 未设置整理方式") + raise ValueError(f"{directory_name} 未设置整理方式") + if target_path is None: + raise ValueError("整理规划缺少目标路径") # 源操作对象 + source_storage = fileitem.storage or "local" if not source_oper: - source_oper = self.__get_storage_oper(fileitem.storage) + source_oper = self.__get_storage_oper(source_storage) if not source_oper: - return TransferInfo(success=False, - message=f"不支持的存储类型：{fileitem.storage}", - fileitem=fileitem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify - ) - # 目的操作对象 - if not target_oper: - if not target_storage: - target_storage = fileitem.storage - target_oper = self.__get_storage_oper(target_storage) - if not target_oper: - return TransferInfo(success=False, - message=f"不支持的存储类型：{target_storage}", - fileitem=fileitem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify) + raise ValueError(f"不支持的存储类型：{source_storage}") + + if not target_storage: + target_storage = source_storage + if planning_input is None: + planning_input = TransferPlanningInput( + source_fileitem=fileitem.model_dump(mode="json"), + meta=self._serialize_transfer_model(meta) if meta else None, + mediainfo=( + self._serialize_transfer_model(mediainfo) if mediainfo else None + ), + target_directory=( + target_directory.model_dump(mode="json") + if target_directory + else None + ), + target_storage=target_storage, + target_path=target_path.as_posix(), + requested_transfer_type=transfer_type, + media_type=mediainfo.type.value if mediainfo.type else None, + need_scrape=bool(need_scrape), + need_rename=bool(need_rename), + need_notify=bool(need_notify), + overwrite_mode=overwrite_mode, + episodes_info=tuple( + episode.model_dump(mode="json") for episode in episodes_info or [] + ), + preview=bool(preview), + ) - # 整理 logger.info(f"获取整理目标路径：【{target_storage}】{target_path}") - return handler.transfer_media(fileitem=fileitem, - in_meta=meta, - mediainfo=mediainfo, - target_storage=target_storage, - target_path=target_path, - transfer_type=transfer_type, - need_scrape=need_scrape, - need_rename=need_rename, - need_notify=need_notify, - overwrite_mode=overwrite_mode, - episodes_info=episodes_info, - preview=preview, - source_oper=source_oper, - target_oper=target_oper) + return handler.plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + need_scrape=bool(need_scrape), + need_rename=bool(need_rename), + need_notify=bool(need_notify), + overwrite_mode=overwrite_mode, + episodes_info=episodes_info, + preview=bool(preview), + ) + + def execute_transfer_plan( + self, + checkpoint: TransferPlanCheckpoint, + *, + meta: MetaBase, + mediainfo: Union[MediaInfo, MusicInfo], + source_oper: Optional[StorageBase] = None, + target_oper: Optional[StorageBase] = None, + cleanup_media_file: Optional[Callable[[FileItem], bool]] = None, + ) -> TransferInfo: + """解析存储适配器并通过统一删除能力执行已冻结计划。""" + source_fileitem = FileItem(**checkpoint.planning_input.source_fileitem) + cleanup_before_transfer = None + cleanup_payload = checkpoint.planning_input.options.get( + "cleanup_dest_fileitem" + ) + if ( + isinstance(cleanup_payload, dict) + and not checkpoint.preview + and not checkpoint.pre_execution_cleanup_completed + ): + cleanup_fileitem = FileItem.model_validate(cleanup_payload) + if not cleanup_media_file: + raise RuntimeError("整理计划缺少统一媒体删除兼容能力") + + def cleanup_before_transfer() -> None: + """拦截通过后委托统一能力治理旧目标及父空目录。""" + if not cleanup_media_file(cleanup_fileitem): + raise RuntimeError( + f"{cleanup_fileitem.path} 删除失败，整理计划保留待重试" + ) + + source_storage = source_fileitem.storage or "local" + if not source_oper: + source_oper = self.__get_storage_oper(source_storage) + if not source_oper: + return TransferInfo( + success=False, + message=f"不支持的存储类型：{source_storage}", + fileitem=source_fileitem, + fail_list=[source_fileitem.path], + transfer_type=checkpoint.resolved_transfer_type, + need_notify=checkpoint.need_notify, + ) + if not target_oper: + target_oper = self.__get_storage_oper(checkpoint.target_storage) + if not target_oper: + return TransferInfo( + success=False, + message=f"不支持的存储类型：{checkpoint.target_storage}", + fileitem=source_fileitem, + fail_list=[source_fileitem.path], + transfer_type=checkpoint.resolved_transfer_type, + need_notify=checkpoint.need_notify, + ) + return TransHandler().execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_oper=target_oper, + cleanup_before_transfer=cleanup_before_transfer, + ) + + @staticmethod + def _serialize_transfer_model(value: object) -> dict[str, Any]: + """校验旧领域对象的动态序列化结果，避免 Any 穿透规划边界。""" + serializer = getattr(value, "to_dict", None) + if not callable(serializer): + raise TypeError(f"{type(value).__name__} 不支持整理快照序列化") + payload = serializer() + if not isinstance(payload, dict): + raise TypeError(f"{type(value).__name__} 返回了无效整理快照") + return payload @staticmethod def _build_library_lookup_meta( diff --git a/app/modules/filemanager/transhandler.py b/app/modules/filemanager/transhandler.py index ed8b1c2ce..5120a4dd3 100644 --- a/app/modules/filemanager/transhandler.py +++ b/app/modules/filemanager/transhandler.py @@ -1,32 +1,39 @@ import re +from copy import deepcopy from pathlib import Path -from typing import Optional, List, Tuple +from typing import Any, Callable, List, Optional, Tuple, cast from jinja2 import Template -from app.runtime.settings import get_runtime_setting - -from app.domain.context import MediaInfo, MusicInfo -from app.runtime.events import eventmanager -from app.domain.meta.metabase import MetaBase -from app.domain.meta.metamusic import MetaMusic -from app.domain.metainfo import MetaInfoPath +from app.adapters.system.host import SystemUtils from app.application.audio import AudioMetadataHelper from app.application.directory import DirectoryHelper from app.application.messaging.message import TemplateHelper -from app.runtime.log import logger +from app.application.transfer import ( + TransferPlanCheckpoint, + TransferPlanItem, + TransferPlanningInput, +) +from app.domain.context import MediaInfo, MusicInfo +from app.domain.meta.metabase import MetaBase +from app.domain.meta.metamusic import MetaMusic +from app.domain.metainfo import MetaInfoPath from app.modules.filemanager.storages import StorageBase -from app.schemas.transfer import TransferInfo -from app.schemas.tmdb import TmdbEpisode -from app.schemas.system import TransferDirectoryConf -from app.schemas.workflow import FileItem -from app.schemas.event import TransferInterceptEventData -from app.schemas.event import TransferOverwriteCheckEventData -from app.schemas.event import TransferRenameBuildEventData -from app.schemas.event import TransferRenameEventData +from app.runtime.events import eventmanager +from app.runtime.log import logger +from app.runtime.settings import get_runtime_setting +from app.schemas.event import ( + TransferInterceptEventData, + TransferOverwriteCheckEventData, + TransferRenameBuildEventData, + TransferRenameEventData, +) from app.schemas.exception import StorageQueryError -from app.schemas.types import MediaType, ChainEventType -from app.adapters.system.host import SystemUtils +from app.schemas.system import TransferDirectoryConf +from app.schemas.tmdb import TmdbEpisode +from app.schemas.transfer import TransferInfo +from app.schemas.types import ChainEventType, MediaType +from app.schemas.workflow import FileItem class TransHandler: @@ -34,7 +41,8 @@ class TransHandler: 文件转移整理类 """ - def __init__(self): + def __init__(self) -> None: + """初始化无状态整理执行器。""" pass @staticmethod @@ -159,539 +167,706 @@ class TransHandler: return None return source_score > target_score - def transfer_media( + @staticmethod + def __serialize_fileitem(fileitem: FileItem) -> dict[str, Any]: + """生成可持久化且不再引用调用方对象的文件快照。""" + return cast(dict[str, Any], fileitem.model_dump(mode="json")) + + @staticmethod + def __serialize_transfer_model(value: object) -> dict[str, Any]: + """校验旧领域对象的动态序列化结果，阻断 Any 向检查点扩散。""" + serializer = getattr(value, "to_dict", None) + if not callable(serializer): + raise TypeError(f"{type(value).__name__} 不支持整理快照序列化") + payload = serializer() + if not isinstance(payload, dict): + raise TypeError(f"{type(value).__name__} 返回了无效整理快照") + return payload + + @staticmethod + def __is_subtitle_file(fileitem: FileItem) -> bool: + """判断文件是否为配置支持的字幕附件。""" + return bool( + fileitem.extension + and f".{fileitem.extension.lower()}" in get_runtime_setting("RMT_SUBEXT") + ) + + @staticmethod + def __is_extra_file(fileitem: FileItem, mediainfo: MediaInfo | MusicInfo) -> bool: + """判断文件是否为需要无条件覆盖的媒体附件。""" + if not fileitem.extension: + return False + extension = f".{fileitem.extension.lower()}" + path = str(fileitem.path or fileitem.name or "").casefold() + if extension in get_runtime_setting("RMT_SUBEXT"): + return True + if mediainfo.type == MediaType.MUSIC and path.endswith( + (".lrc", ".txt", ".lyricsfile.yaml") + ): + return True + return ( + mediainfo.type != MediaType.MUSIC + and extension in get_runtime_setting("RMT_AUDIOEXT") + ) + + @staticmethod + def __is_special_extra_file(fileitem: FileItem) -> bool: + """识别没有季集号但允许合法跳过的特典视频。""" + return bool( + re.search( + r"(?:^|[\s_.\-\[【(])(NC(?:OP|ED)|NCOP|NCED|OP|ED|MENU|PV|CM|TRAILER|" + r"TV\s*SPOT|SP|OVA|OAD|EVENT|IV|INTERVIEW|LOGO|PRODUCER\s*LOGO|" + r"BEHIND\s*THE\s*SCENES|FEATURETTE" + r")(?:\d*|[\s_.\-\]】)]|$)", + fileitem.name or "", + re.IGNORECASE, + ) + ) + + def __plan_directory_items( self, + *, fileitem: FileItem, - in_meta: MetaBase, + target_storage: str, + target_path: Path, + source_oper: StorageBase, + ) -> tuple[TransferPlanItem, ...]: + """只读遍历源目录并冻结叶子文件的稳定执行顺序。""" + items: list[TransferPlanItem] = [] + + def collect(source_dir: FileItem, destination_dir: Path) -> None: + """按存储返回顺序深度优先收集叶子文件。""" + for source_item in source_oper.list(source_dir) or []: + destination = destination_dir / source_item.name + if source_item.type == "dir": + collect(source_item, destination) + continue + items.append( + TransferPlanItem( + sequence=len(items), + source_fileitem=self.__serialize_fileitem(source_item), + target_storage=target_storage, + target_path=destination.as_posix(), + ) + ) + + collect(fileitem, target_path) + return tuple(items) + + def plan_transfer( + self, + planning_input: TransferPlanningInput, + *, + meta: MetaBase, + mediainfo: MediaInfo | MusicInfo, + source_oper: StorageBase, + target_storage: str, + target_path: Path, + transfer_type: str, + need_scrape: bool, + need_rename: bool, + need_notify: bool, + overwrite_mode: Optional[str], + episodes_info: Optional[List[TmdbEpisode]], + preview: bool, + ) -> TransferPlanCheckpoint: + """只计算最终目标和有序操作，不触发任何文件或目录写副作用。""" + fileitem = FileItem(**planning_input.source_fileitem) + rename_format = get_runtime_setting("RENAME_FORMAT")(mediainfo.type) + planning_meta = deepcopy(meta) + resolved_mediainfo = self.__serialize_transfer_model(mediainfo) + resolved_episodes_info = tuple( + episode.model_dump(mode="json") for episode in episodes_info or [] + ) + + if fileitem.type == "dir": + if need_rename: + rendered_path = self.get_rename_path( + path=target_path, + template_string=rename_format, + rename_dict=self.get_naming_dict( + meta=planning_meta, + mediainfo=mediainfo, + ), + source_path=fileitem.path, + source_item=fileitem, + ) + if mediainfo.type == MediaType.TV: + final_target = self.__get_tv_bluray_dir_path( + rendered_path=rendered_path, + source_item=fileitem, + meta=planning_meta, + ) + else: + final_target = DirectoryHelper.get_media_root_path( + rename_format, + rename_path=rendered_path, + media_type=mediainfo.type, + ) + if not final_target: + raise ValueError("重命名格式无效") + else: + final_target = target_path / fileitem.name + items = ( + () + if preview + else self.__plan_directory_items( + fileitem=fileitem, + target_storage=target_storage, + target_path=final_target, + source_oper=source_oper, + ) + ) + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage=target_storage, + root_target_path=target_path.as_posix(), + final_target_path=final_target.as_posix(), + resolved_transfer_type=transfer_type, + items=items, + resolved_meta=self.__serialize_transfer_model(planning_meta), + resolved_meta_kind=( + type(planning_meta).__name__ if planning_meta else None + ), + resolved_mediainfo=resolved_mediainfo, + resolved_mediainfo_kind=( + type(mediainfo).__name__ if mediainfo else None + ), + resolved_episodes_info=resolved_episodes_info, + need_scrape=need_scrape, + need_rename=need_rename, + need_notify=need_notify, + overwrite_mode=overwrite_mode, + preview=preview, + skip_reason=( + "源目录中没有可整理文件" + if not preview and not items + else None + ), + ) + + if mediainfo.type == MediaType.TV: + if planning_meta.begin_episode is None: + if self.__is_special_extra_file(fileitem): + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage=target_storage, + root_target_path=target_path.as_posix(), + final_target_path=target_path.as_posix(), + resolved_transfer_type=transfer_type, + items=(), + resolved_meta=( + self.__serialize_transfer_model(planning_meta) + ), + resolved_meta_kind=( + type(planning_meta).__name__ if planning_meta else None + ), + resolved_mediainfo=resolved_mediainfo, + resolved_mediainfo_kind=( + type(mediainfo).__name__ if mediainfo else None + ), + resolved_episodes_info=resolved_episodes_info, + need_scrape=need_scrape, + need_rename=need_rename, + need_notify=False, + overwrite_mode=overwrite_mode, + preview=preview, + skip_reason="未识别到文件集数，识别为特典/附加视频文件", + ) + raise ValueError("未识别到文件集数") + planning_meta.end_season = None + if planning_meta.total_season: + planning_meta.total_season = 1 + if planning_meta.total_episode > 2: + planning_meta.total_episode = 1 + planning_meta.end_episode = None + + if need_rename: + file_extension = ( + ".lyricsfile.yaml" + if str(fileitem.path or "").casefold().endswith(".lyricsfile.yaml") + else f".{fileitem.extension}" + ) + final_target = self.get_rename_path( + path=target_path, + template_string=rename_format, + rename_dict=self.get_naming_dict( + meta=planning_meta, + mediainfo=mediainfo, + episodes_info=episodes_info, + file_ext=file_extension, + ), + source_path=fileitem.path, + source_item=fileitem, + ) + if self.__is_subtitle_file(fileitem): + final_target = self.__rename_subtitles(fileitem, final_target) + target_directory_path = DirectoryHelper.get_media_root_path( + rename_format, + rename_path=final_target, + media_type=mediainfo.type, + ) + if not target_directory_path: + raise ValueError("重命名格式无效") + else: + final_target = target_path / fileitem.name + + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage=target_storage, + root_target_path=target_path.as_posix(), + final_target_path=final_target.as_posix(), + resolved_transfer_type=transfer_type, + items=( + TransferPlanItem( + sequence=0, + source_fileitem=self.__serialize_fileitem(fileitem), + target_storage=target_storage, + target_path=final_target.as_posix(), + ), + ), + resolved_meta=self.__serialize_transfer_model(planning_meta), + resolved_meta_kind=( + type(planning_meta).__name__ if planning_meta else None + ), + resolved_mediainfo=resolved_mediainfo, + resolved_mediainfo_kind=( + type(mediainfo).__name__ if mediainfo else None + ), + resolved_episodes_info=resolved_episodes_info, + need_scrape=need_scrape, + need_rename=need_rename, + need_notify=need_notify, + overwrite_mode=overwrite_mode, + preview=preview, + ) + + @staticmethod + def __intercept_transfer( + *, + fileitem: FileItem, + meta: Optional[MetaBase], mediainfo: MediaInfo | MusicInfo, target_storage: str, target_path: Path, transfer_type: str, + over_flag: Optional[bool] = None, + ) -> tuple[bool, str]: + """在宿主写副作用前执行插件拦截并返回取消原因。""" + if over_flag is None: + event_data = TransferInterceptEventData( + fileitem=fileitem, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + ) + else: + event_data = TransferInterceptEventData( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + options={"over_flag": over_flag}, + ) + event = eventmanager.send_event(ChainEventType.TransferIntercept, event_data) + if ( + event + and isinstance(event.event_data, TransferInterceptEventData) + and event.event_data.cancel + ): + canceled = event.event_data + logger.debug( + f"Transfer canceled by event: {canceled.source},Reason: {canceled.reason}" + ) + return False, canceled.reason + return True, "" + + def __resolve_overwrite( + self, + *, + fileitem: FileItem, + meta: MetaBase, + mediainfo: MediaInfo | MusicInfo, + target_oper: StorageBase, + target_storage: str, + target_file: Path, + transfer_type: str, + overwrite_mode: Optional[str], + need_notify: bool, + ) -> tuple[bool, bool, Optional[TransferInfo]]: + """在执行期完成覆盖事件与策略判断，不改变冻结目标。""" + if self.__is_extra_file(fileitem, mediainfo): + return True, False, None + try: + target_item = target_oper.get_item_strict(target_file) + except StorageQueryError as query_error: + message = ( + f"无法确认目标文件状态，已跳过整理以避免误覆盖：" + f"{target_file} - {query_error}" + ) + logger.warn(message) + return False, False, TransferInfo( + success=False, + message=message, + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=need_notify, + ) + if not target_item: + return False, overwrite_mode == "latest", None + + checked_target = target_file + over_flag = False + if target_storage == "local" and target_file.is_symlink(): + checked_target = target_file.readlink() + if not checked_target.exists(): + over_flag = True + if over_flag: + return True, False, None + + logger.info( + f"目的文件系统中已经存在同名文件 {checked_target}，" + f"当前整理覆盖模式设置为 {overwrite_mode}" + ) + event_data = TransferOverwriteCheckEventData( + fileitem=fileitem, + target_item=target_item, + target_storage=target_storage, + target_path=target_file, + overwrite_mode=overwrite_mode or "", + transfer_type=transfer_type, + ) + event = eventmanager.send_event( + ChainEventType.TransferOverwriteCheck, + event_data, + ) + plugin_overwrite: Optional[bool] = None + plugin_source_size: Optional[int] = None + plugin_target_size: Optional[int] = None + plugin_reason: Optional[str] = None + if event and isinstance( + event.event_data, + TransferOverwriteCheckEventData, + ): + plugin_event_data = event.event_data + plugin_overwrite = plugin_event_data.overwrite + plugin_source_size = plugin_event_data.source_size + plugin_target_size = plugin_event_data.target_size + plugin_reason = plugin_event_data.reason + if plugin_overwrite is True: + return True, False, None + if plugin_overwrite is False: + return False, False, TransferInfo( + success=False, + message=plugin_reason or "插件决定不覆盖已有文件", + fileitem=fileitem, + target_item=target_item, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=need_notify, + overwrite_skipped=True, + ) + if overwrite_mode in {"always", "latest"}: + return True, False, None + if overwrite_mode == "never": + return False, False, TransferInfo( + success=False, + message="媒体库存在同名文件，当前覆盖模式为不覆盖", + fileitem=fileitem, + target_item=target_item, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=need_notify, + overwrite_skipped=True, + ) + if overwrite_mode == "size": + music_overwrite = self.__music_quality_overwrite_decision( + meta=meta, + mediainfo=mediainfo, + target_item=target_item, + ) + if music_overwrite is True: + return True, False, None + if music_overwrite is False: + return False, False, TransferInfo( + success=False, + message="媒体库存在同名音乐文件，且目标音质更好", + fileitem=fileitem, + target_item=target_item, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=need_notify, + overwrite_skipped=True, + ) + source_size = ( + plugin_source_size + if plugin_source_size is not None + else fileitem.size + ) + target_size = ( + plugin_target_size + if plugin_target_size is not None + else target_item.size + ) + if target_size < source_size: + return True, False, None + return False, False, TransferInfo( + success=False, + message="媒体库存在同名文件，且质量更好", + fileitem=fileitem, + target_item=target_item, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=need_notify, + ) + return False, False, None + + def execute_transfer_plan( + self, + checkpoint: TransferPlanCheckpoint, + *, + meta: MetaBase, + mediainfo: MediaInfo | MusicInfo, source_oper: StorageBase, target_oper: StorageBase, - need_scrape: Optional[bool] = False, - need_rename: Optional[bool] = True, - need_notify: Optional[bool] = True, - overwrite_mode: Optional[str] = None, - episodes_info: List[TmdbEpisode] = None, - preview: Optional[bool] = False, + cleanup_before_transfer: Optional[Callable[[], None]] = None, ) -> TransferInfo: - """ - 识别并整理一个文件或者一个目录下的所有文件 - :param fileitem: 整理的文件对象，可能是一个文件也可以是一个目录 - :param in_meta：预识别元数据 - :param mediainfo: 媒体信息 - :param target_storage: 目标存储 - :param target_path: 目标路径 - :param transfer_type: 文件整理方式 - :param source_oper: 源存储操作对象 - :param target_oper: 目标存储操作对象 - :param need_scrape: 是否需要刮削 - :param need_rename: 是否需要重命名 - :param need_notify: 是否需要通知 - :param overwrite_mode: 覆盖模式 - :param episodes_info: 当前季的全部集信息 - :param preview: 是否仅预览 - :return: TransferInfo、错误信息 - """ - - def __is_subtitle_file(_fileitem: FileItem) -> bool: - """ - 判断是否为字幕文件 - :param _fileitem: 文件项 - :return: True/False - """ - if not _fileitem.extension: - return False - if f".{_fileitem.extension.lower()}" in get_runtime_setting('RMT_SUBEXT'): - return True - return False - - def __is_music_lyrics_file(_fileitem: FileItem) -> bool: - """判断是否为音乐音轨的歌词附件。""" - path = str(_fileitem.path or _fileitem.name or "").casefold() - return mediainfo.type == MediaType.MUSIC and path.endswith( - (".lrc", ".txt", ".lyricsfile.yaml") - ) - - def __is_extra_file(_fileitem: FileItem) -> bool: - """ - 判断是否为附加文件 - :param _fileitem: 文件项 - :return: True/False - """ - if not _fileitem.extension: - return False - extension = f".{_fileitem.extension.lower()}" - if extension in get_runtime_setting('RMT_SUBEXT'): - return True - if __is_music_lyrics_file(_fileitem): - return True - if mediainfo.type != MediaType.MUSIC and extension in get_runtime_setting('RMT_AUDIOEXT'): - return True - return False - - def __is_special_extra_file(_fileitem: FileItem) -> bool: - """ - 判断是否为特典/附加视频文件（如 NCOP/NCED/Menu/CM/PV/Event/Logo 等无集数编号的视频/样本） - """ - file_name = _fileitem.name or "" - return bool( - re.search( - r"(?:^|[\s_.\-\[【(])(" - r"NC(?:OP|ED)|NCOP|NCED|OP|ED|MENU|PV|CM|TRAILER|TV\s*SPOT|SP|OVA|OAD|EVENT|IV|INTERVIEW|LOGO|PRODUCER\s*LOGO|BEHIND\s*THE\s*SCENES|FEATURETTE" - r")(?:\d*|[\s_.\-\]】)]|$)", - file_name, - re.IGNORECASE, - ) - ) - - # 整理结果 + """只消费冻结目标和有序操作，并在执行期处理覆盖与插件拦截。""" + fileitem = FileItem(**checkpoint.planning_input.source_fileitem) + target_storage = checkpoint.target_storage + target_path = Path(checkpoint.final_target_path) + transfer_type = checkpoint.resolved_transfer_type result = TransferInfo() - try: - # 重命名格式 - rename_format = get_runtime_setting('RENAME_FORMAT')(mediainfo.type) + if checkpoint.skip_reason: + logger.info(f"文件 {fileitem.path} 跳过整理：{checkpoint.skip_reason}") + return TransferInfo( + success=True, + message=checkpoint.skip_reason, + fileitem=fileitem, + transfer_type=transfer_type, + need_notify=False, + ) - # 判断是否为文件夹 - if fileitem.type == "dir": - # 整理整个目录，一般为蓝光原盘 - if need_rename: - rendered_path = self.get_rename_path( - path=target_path, - template_string=rename_format, - rename_dict=self.get_naming_dict( - meta=in_meta, mediainfo=mediainfo - ), - source_path=fileitem.path, - source_item=fileitem, - ) - if mediainfo.type == MediaType.TV: - new_path = self.__get_tv_bluray_dir_path( - rendered_path=rendered_path, - source_item=fileitem, - meta=in_meta, - ) - else: - new_path = DirectoryHelper.get_media_root_path( - rename_format, - rename_path=rendered_path, - media_type=mediainfo.type, - ) - if not new_path: - self.__update_result( - result=result, - success=False, - message="重命名格式无效", - fileitem=fileitem, - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - else: - new_path = target_path / fileitem.name - if preview: - preview_diritem = self.__build_preview_item( - storage=target_storage, - path=new_path, - item_type="dir", - ) - self.__update_result( - result=result, - success=True, - fileitem=fileitem, - target_item=preview_diritem, - target_diritem=preview_diritem, - file_list=[fileitem.path], - file_list_new=[new_path.as_posix()], - need_scrape=need_scrape, - need_notify=False, - transfer_type=transfer_type, - ) - return result - # 原盘大小只计算STREAM目录内的文件大小 - if stream_fileitem := source_oper.get_item( - Path(fileitem.path) / "BDMV" / "STREAM" + if checkpoint.preview: + item_type = "dir" if fileitem.type == "dir" else "file" + preview_target_item = self.__build_preview_item( + storage=target_storage, + path=target_path, + item_type=item_type, + size=fileitem.size, + ) + preview_target_diritem = ( + preview_target_item + if item_type == "dir" + else self.__build_preview_item( + storage=target_storage, + path=target_path.parent, + item_type="dir", + ) + ) + return TransferInfo( + success=True, + fileitem=fileitem, + target_item=preview_target_item, + target_diritem=preview_target_diritem, + file_list=[fileitem.path], + file_list_new=[target_path.as_posix()], + file_count=0 if item_type == "dir" else 1, + total_size=0 if item_type == "dir" else fileitem.size or 0, + need_scrape=checkpoint.need_scrape, + transfer_type=transfer_type, + need_notify=False, + ) + + if fileitem.type == "dir": + stream_path = Path(fileitem.path) / "BDMV" / "STREAM" + stream_sizes = [ + FileItem(**item.source_fileitem).size or 0 + for item in checkpoint.items + if Path(str(item.source_fileitem.get("path", ""))).parent + == stream_path + ] + if stream_sizes: + fileitem.size = sum(stream_sizes) + allowed, reason = self.__intercept_transfer( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + ) + if not allowed: + return TransferInfo( + success=False, + message=reason, + fileitem=fileitem, + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + if cleanup_before_transfer: + cleanup_before_transfer() + target_diritem = target_oper.get_folder(target_path) + if not target_diritem: + return TransferInfo( + success=False, + message=f"获取目标目录失败：{target_path}", + fileitem=fileitem, + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + for planned_item in checkpoint.items: + if ( + planned_item.action != "transfer" + or planned_item.target_storage != target_storage ): - fileitem.size = sum( - file.size for file in source_oper.list(stream_fileitem) or [] + return TransferInfo( + success=False, + message="整理计划包含不支持的操作或目标存储", + fileitem=fileitem, + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, ) - # 整理目录 - new_diritem, errmsg = self.__transfer_dir( - fileitem=fileitem, - mediainfo=mediainfo, + source_item = FileItem(**planned_item.source_fileitem) + new_item, error = self.__transfer_command( + fileitem=source_item, + target_storage=planned_item.target_storage, source_oper=source_oper, target_oper=target_oper, - target_storage=target_storage, - target_path=new_path, + target_file=Path(planned_item.target_path), transfer_type=transfer_type, - result=result, - ) - if not new_diritem: - logger.error(f"文件夹 {fileitem.path} 整理失败：{errmsg}") - self.__update_result( - result=result, - success=False, - message=errmsg, - fileitem=fileitem, - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - - logger.info(f"文件夹 {fileitem.path} 整理成功") - # 返回整理后的路径 - self.__update_result( - result=result, - success=True, - fileitem=fileitem, - target_item=new_diritem, - target_diritem=new_diritem, - need_scrape=need_scrape, - need_notify=need_notify, - transfer_type=transfer_type, - ) - return result - else: - # 整理单个文件 - if mediainfo.type == MediaType.TV: - # 电视剧 - if in_meta.begin_episode is None: - if __is_special_extra_file(fileitem): - logger.info(f"文件 {fileitem.path} 未识别到文件集数，识别为特典/附加视频文件，跳过正片集数整理") - self.__update_result( - result=result, - success=True, - fileitem=fileitem, - transfer_type=transfer_type, - need_notify=False, - ) - return result - - logger.warn(f"文件 {fileitem.path} 整理失败：未识别到文件集数") - self.__update_result( - result=result, - success=False, - message="未识别到文件集数", - fileitem=fileitem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - - # 文件结束季为空 - in_meta.end_season = None - # 文件总季数为1 - if in_meta.total_season: - in_meta.total_season = 1 - # 文件不可能超过2集 - if in_meta.total_episode > 2: - in_meta.total_episode = 1 - in_meta.end_episode = None - - # 目的文件名 - if need_rename: - file_extension = ( - ".lyricsfile.yaml" - if str(fileitem.path or "").casefold().endswith(".lyricsfile.yaml") - else f".{fileitem.extension}" - ) - new_file = self.get_rename_path( - path=target_path, - template_string=rename_format, - rename_dict=self.get_naming_dict( - meta=in_meta, - mediainfo=mediainfo, - episodes_info=episodes_info, - file_ext=file_extension, - ), - source_path=fileitem.path, - source_item=fileitem, - ) - - # 针对字幕文件，文件名中补充额外标识信息 - if __is_subtitle_file(fileitem): - new_file = self.__rename_subtitles(fileitem, new_file) - - # 文件目录 - folder_path = DirectoryHelper.get_media_root_path( - rename_format, - rename_path=new_file, - media_type=mediainfo.type, - ) - if not folder_path: - self.__update_result( - result=result, - success=False, - message="重命名格式无效", - fileitem=fileitem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - else: - new_file = target_path / fileitem.name - folder_path = target_path - - # 目标目录 - if preview: - # 预览只做路径推算，不检查目录或同名文件冲突，避免目标存储探测触发真实整理。 - target_diritem = self.__build_preview_item( - storage=target_storage, - path=folder_path, - item_type="dir", - ) - target_item = self.__build_preview_item( - storage=target_storage, - path=new_file, - item_type="file", - size=fileitem.size, - ) - self.__update_result( - result=result, - success=True, - fileitem=fileitem, - target_item=target_item, - target_diritem=target_diritem, - file_list=[fileitem.path], - file_list_new=[new_file.as_posix()], - file_count=1, - total_size=fileitem.size or 0, - need_scrape=need_scrape, - transfer_type=transfer_type, - need_notify=False, - ) - return result - - target_diritem = target_oper.get_folder(folder_path) - if not target_diritem: - logger.error(f"目标目录 {folder_path} 获取失败") - self.__update_result( - result=result, - success=False, - message=f"目标目录 {folder_path} 获取失败", - fileitem=fileitem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - - # 判断是否要覆盖，附加文件强制覆盖 - overflag = False - if not __is_extra_file(fileitem): - # 目标文件（严格查询：无法确认状态时拒绝覆盖，避免已有文件被误覆盖） - try: - target_item = target_oper.get_item_strict(new_file) - except StorageQueryError as query_err: - errmsg = f"无法确认目标文件状态，已跳过整理以避免误覆盖：{new_file} - {query_err}" - logger.warn(errmsg) - self.__update_result( - result=result, - success=False, - message=errmsg, - fileitem=fileitem, - target_diritem=target_diritem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - if target_item: - # 目标文件已存在 - target_file = new_file - if target_storage == "local" and new_file.is_symlink(): - target_file = new_file.readlink() - if not target_file.exists(): - overflag = True - if not overflag: - # 目标文件已存在 - logger.info( - f"目的文件系统中已经存在同名文件 {target_file}，当前整理覆盖模式设置为 {overwrite_mode}" - ) - # 触发覆盖检查事件，允许插件提供源/目标文件真实大小 - # 或直接给出覆盖决策（例如 .strm 文件指向网盘原始文件） - overwrite_event_data = TransferOverwriteCheckEventData( - fileitem=fileitem, - target_item=target_item, - target_storage=target_storage, - target_path=new_file, - overwrite_mode=overwrite_mode or "", - transfer_type=transfer_type, - ) - overwrite_event = eventmanager.send_event( - ChainEventType.TransferOverwriteCheck, - overwrite_event_data, - ) - plugin_overwrite: Optional[bool] = None - plugin_source_size: Optional[int] = None - plugin_target_size: Optional[int] = None - if overwrite_event and overwrite_event.event_data: - overwrite_event_data = overwrite_event.event_data - plugin_overwrite = overwrite_event_data.overwrite - plugin_source_size = overwrite_event_data.source_size - plugin_target_size = overwrite_event_data.target_size - if ( - plugin_overwrite is not None - or plugin_source_size is not None - or plugin_target_size is not None - ): - logger.info( - f"覆盖检查事件由 {overwrite_event_data.source} 处理：" - f"overwrite={plugin_overwrite}, " - f"source_size={plugin_source_size}, " - f"target_size={plugin_target_size}, " - f"reason={overwrite_event_data.reason}" - ) - if plugin_overwrite is True: - overflag = True - elif plugin_overwrite is False: - self.__update_result( - result=result, - success=False, - message=overwrite_event_data.reason - or "插件决定不覆盖已有文件", - fileitem=fileitem, - target_item=target_item, - target_diritem=target_diritem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - overwrite_skipped=True, - ) - return result - elif overwrite_mode == "always": - # 总是覆盖同名文件 - overflag = True - elif overwrite_mode == "size": - # 音乐先比较真实音质，无法判断时再沿用文件大小策略 - music_overwrite = self.__music_quality_overwrite_decision( - meta=in_meta, - mediainfo=mediainfo, - target_item=target_item, - ) - if music_overwrite is True: - logger.info(f"目标音乐音质较低，将覆盖：{new_file}") - overflag = True - elif music_overwrite is False: - self.__update_result( - result=result, - success=False, - message="媒体库存在同名音乐文件，且目标音质更好", - fileitem=fileitem, - target_item=target_item, - target_diritem=target_diritem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - overwrite_skipped=True, - ) - return result - else: - source_size = ( - plugin_source_size - if plugin_source_size is not None - else fileitem.size - ) - target_size = ( - plugin_target_size - if plugin_target_size is not None - else target_item.size - ) - if target_size < source_size: - logger.info( - f"目标文件文件大小更小，将覆盖：{new_file}" - ) - overflag = True - else: - self.__update_result( - result=result, - success=False, - message=f"媒体库存在同名文件，且质量更好", - fileitem=fileitem, - target_item=target_item, - target_diritem=target_diritem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - ) - return result - elif overwrite_mode == "never": - # 存在不覆盖 - self.__update_result( - result=result, - success=False, - message=f"媒体库存在同名文件，当前覆盖模式为不覆盖", - fileitem=fileitem, - target_item=target_item, - target_diritem=target_diritem, - fail_list=[fileitem.path], - transfer_type=transfer_type, - need_notify=need_notify, - overwrite_skipped=True, - ) - return result - elif overwrite_mode == "latest": - # 仅保留最新版本 - logger.info( - f"当前整理覆盖模式设置为仅保留最新版本，将覆盖：{new_file}" - ) - overflag = True - else: - if overwrite_mode == "latest": - # 文件不存在，但仅保留最新版本 - logger.info( - f"当前整理覆盖模式设置为 {overwrite_mode}，仅保留最新版本，正在删除已有版本文件 ..." - ) - self.__delete_version_files(target_oper, new_file) - else: - # 附加文件 总是需要覆盖 - overflag = True - - # 整理文件 - new_item, err_msg = self.__transfer_file( - fileitem=fileitem, - meta=in_meta, - mediainfo=mediainfo, - target_storage=target_storage, - target_file=new_file, - transfer_type=transfer_type, - over_flag=overflag, - source_oper=source_oper, - target_oper=target_oper, - result=result, ) if not new_item: - err_msg = err_msg or f"{fileitem.path} 整理后未获取到目标文件信息" - logger.error(f"文件 {fileitem.path} 整理失败：{err_msg}") - self.__update_result( - result=result, + return TransferInfo( success=False, - message=err_msg, + message=error, fileitem=fileitem, - fail_list=[fileitem.path], transfer_type=transfer_type, - need_notify=need_notify, + need_notify=checkpoint.need_notify, ) - return result - - logger.info(f"文件 {fileitem.path} 整理成功") self.__update_result( result=result, - success=True, - fileitem=fileitem, - target_item=new_item, - target_diritem=target_diritem, - need_scrape=need_scrape, - transfer_type=transfer_type, - need_notify=need_notify, + file_list=[source_item.path], + file_list_new=[new_item.path], ) - return result - except Exception as e: - logger.error(f"媒体整理出错：{e}") - return TransferInfo(success=False, message=str(e)) + self.__update_result( + result=result, + success=True, + fileitem=fileitem, + target_item=target_diritem, + target_diritem=target_diritem, + need_scrape=checkpoint.need_scrape, + need_notify=checkpoint.need_notify, + transfer_type=transfer_type, + ) + return result + + if len(checkpoint.items) != 1: + return TransferInfo( + success=False, + message="单文件整理计划必须且只能包含一个操作", + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + planned_item = checkpoint.items[0] + if ( + planned_item.action != "transfer" + or planned_item.target_storage != target_storage + or planned_item.target_path != checkpoint.final_target_path + ): + return TransferInfo( + success=False, + message="单文件整理计划与冻结目标不一致", + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + target_file = Path(planned_item.target_path) + over_flag, delete_versions, overwrite_failure = self.__resolve_overwrite( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_oper=target_oper, + target_storage=target_storage, + target_file=target_file, + transfer_type=transfer_type, + overwrite_mode=checkpoint.overwrite_mode, + need_notify=checkpoint.need_notify, + ) + if overwrite_failure: + return overwrite_failure + allowed, reason = self.__intercept_transfer( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_file, + transfer_type=transfer_type, + over_flag=over_flag, + ) + if not allowed: + return TransferInfo( + success=False, + message=reason, + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + + if cleanup_before_transfer: + cleanup_before_transfer() + target_diritem = target_oper.get_folder(target_file.parent) + if not target_diritem: + return TransferInfo( + success=False, + message=f"目标目录 {target_file.parent} 获取失败", + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + if delete_versions: + self.__delete_version_files(target_oper, target_file) + new_item, error = self.__transfer_file( + fileitem=fileitem, + target_storage=target_storage, + target_file=target_file, + transfer_type=transfer_type, + over_flag=over_flag, + source_oper=source_oper, + target_oper=target_oper, + result=result, + ) + if not new_item: + error = error or f"{fileitem.path} 整理后未获取到目标文件信息" + return TransferInfo( + success=False, + message=error, + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + self.__update_result( + result=result, + success=True, + fileitem=fileitem, + target_item=new_item, + target_diritem=target_diritem, + need_scrape=checkpoint.need_scrape, + transfer_type=transfer_type, + need_notify=checkpoint.need_notify, + ) + return result @staticmethod def __transfer_command( @@ -963,125 +1138,9 @@ class TransHandler: return new_file.with_name(new_file.stem + new_sub_tag + file_ext) - def __transfer_dir( - self, - fileitem: FileItem, - mediainfo: MediaInfo, - source_oper: StorageBase, - target_oper: StorageBase, - transfer_type: str, - target_storage: str, - target_path: Path, - result: TransferInfo, - ) -> Tuple[Optional[FileItem], str]: - """ - 整理整个文件夹 - :param fileitem: 源文件 - :param mediainfo: 媒体信息 - :param source_oper: 源存储操作对象 - :param target_oper: 目标存储操作对象 - :param transfer_type: 整理方式 - :param target_storage: 目标存储 - :param target_path: 目标路径 - """ - logger.info(f"正在整理目录：{fileitem.path} 到 {target_path}") - target_item = target_oper.get_folder(target_path) - if not target_item: - return None, f"获取目标目录失败：{target_path}" - event_data = TransferInterceptEventData( - fileitem=fileitem, - mediainfo=mediainfo, - target_storage=target_storage, - target_path=target_path, - transfer_type=transfer_type, - ) - event = eventmanager.send_event(ChainEventType.TransferIntercept, event_data) - if event and event.event_data: - event_data = event.event_data - # 如果事件被取消，跳过文件整理 - if event_data.cancel: - logger.debug( - f"Transfer dir canceled by event: {event_data.source}," - f"Reason: {event_data.reason}" - ) - return None, event_data.reason - # 处理所有文件 - state, errmsg = self.__transfer_dir_files( - fileitem=fileitem, - target_storage=target_storage, - source_oper=source_oper, - target_oper=target_oper, - target_path=target_path, - transfer_type=transfer_type, - result=result, - ) - if state: - return target_item, errmsg - else: - return None, errmsg - - def __transfer_dir_files( - self, - fileitem: FileItem, - target_storage: str, - source_oper: StorageBase, - target_oper: StorageBase, - transfer_type: str, - target_path: Path, - result: TransferInfo, - ) -> Tuple[bool, str]: - """ - 按目录结构整理目录下所有文件 - :param fileitem: 源文件 - :param target_storage: 目标存储 - :param source_oper: 源存储操作对象 - :param target_oper: 目标存储操作对象 - :param target_path: 目标路径 - :param transfer_type: 整理方式 - """ - file_list: List[FileItem] = source_oper.list(fileitem) - # 整理文件 - for item in file_list: - if item.type == "dir": - # 递归整理目录 - new_path = target_path / item.name - state, errmsg = self.__transfer_dir_files( - fileitem=item, - target_storage=target_storage, - source_oper=source_oper, - target_oper=target_oper, - transfer_type=transfer_type, - target_path=new_path, - result=result, - ) - if not state: - return False, errmsg - else: - # 整理文件 - new_file = target_path / item.name - new_item, errmsg = self.__transfer_command( - fileitem=item, - target_storage=target_storage, - source_oper=source_oper, - target_oper=target_oper, - target_file=new_file, - transfer_type=transfer_type, - ) - if not new_item: - return False, errmsg - self.__update_result( - result=result, - file_list=[item.path], - file_list_new=[new_item.path], - ) - # 返回成功 - return True, "" - def __transfer_file( self, fileitem: FileItem, - meta: Optional[MetaBase], - mediainfo: MediaInfo, source_oper: StorageBase, target_oper: StorageBase, target_storage: str, @@ -1093,8 +1152,6 @@ class TransHandler: """ 整理一个文件，同时处理其他相关文件 :param fileitem: 原文件 - :param meta: 元数据 - :param mediainfo: 媒体信息 :param source_oper: 源存储操作对象 :param target_oper: 目标存储操作对象 :param target_storage: 目标存储 @@ -1108,25 +1165,6 @@ class TransHandler: f"正在整理文件：【{fileitem.storage}】{fileitem.path} 到 【{target_storage}】{target_file}，" f"操作类型：{transfer_type}" ) - event_data = TransferInterceptEventData( - fileitem=fileitem, - meta=meta, - mediainfo=mediainfo, - target_storage=target_storage, - target_path=target_file, - transfer_type=transfer_type, - options={"over_flag": over_flag}, - ) - event = eventmanager.send_event(ChainEventType.TransferIntercept, event_data) - if event and event.event_data: - event_data = event.event_data - # 如果事件被取消，跳过文件整理 - if event_data.cancel: - logger.debug( - f"Transfer file canceled by event: {event_data.source}," - f"Reason: {event_data.reason}" - ) - return None, event_data.reason if target_storage == "local" and ( target_file.exists() or target_file.is_symlink() ): diff --git a/app/runtime/extensions/module/contracts.py b/app/runtime/extensions/module/contracts.py index 57b1d01de..06809cb65 100644 --- a/app/runtime/extensions/module/contracts.py +++ b/app/runtime/extensions/module/contracts.py @@ -124,6 +124,29 @@ _METHOD_CONTRACTS = { "rename_file": ModuleMethodContract(family="storage", input_contract="StorageRenameRequest", result_contract="bool | FileItem", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("fileitem", "name")), "storage_manage": ModuleMethodContract(family="storage", input_contract="StorageManageRequest", result_contract="StorageProviderResult", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("storage", "action")), "snapshot_storage": ModuleMethodContract(family="storage", input_contract="StorageSnapshotRequest", result_contract="dict[str, dict] | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("storage", "path", "last_snapshot_time", "max_depth", "previous_snapshot")), + "plan_transfer": ModuleMethodContract( + family="storage", + input_contract="TransferPlanningInput", + result_contract="TransferPlanCheckpoint | None", + aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, + required_parameters=( + "fileitem", "meta", "mediainfo", "target_directory", "target_storage", + "target_path", "transfer_type", "scrape", "library_type_folder", + "library_category_folder", "episodes_info", "source_oper", "preview", + "planning_input", + ), + public_to_plugins=False, + ), + "execute_transfer_plan": ModuleMethodContract( + family="storage", + input_contract="TransferPlanCheckpoint", + result_contract="TransferInfo | None", + aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, + required_parameters=( + "checkpoint", "meta", "mediainfo", "source_oper", "target_oper", + ), + public_to_plugins=False, + ), "transfer": ModuleMethodContract(family="storage", input_contract="TransferRequest", result_contract="TransferInfo | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("fileitem", "meta", "mediainfo", "target_directory", "target_storage", "target_path", "transfer_type", "scrape", "library_type_folder", "library_category_folder", "episodes_info", "source_oper", "target_oper", "preview")), "load_category_config": ModuleMethodContract(family="category", input_contract="CategoryConfigReadRequest", result_contract="CategoryConfig | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY), "clear_cache": ModuleMethodContract(family="category", input_contract="CacheClearRequest", result_contract="None", aggregation=ModuleResultAggregation.FAN_OUT, plugin_short_circuit=False), diff --git a/app/runtime/extensions/module/dispatcher.py b/app/runtime/extensions/module/dispatcher.py index 704b6e88d..a0a5dde39 100644 --- a/app/runtime/extensions/module/dispatcher.py +++ b/app/runtime/extensions/module/dispatcher.py @@ -4,13 +4,12 @@ from __future__ import annotations import inspect from collections.abc import Callable, Mapping +from dataclasses import dataclass from enum import StrEnum from typing import Any, Protocol, cast from app.foundation.reflection import ObjectUtils from app.runtime.execution import run_in_threadpool_to_completion -from app.runtime.log import logger -from app.runtime.observability import observe_duration, record_metric from app.runtime.extensions.module.contracts import ( ModuleResultAggregation, diagnose_module_callable, @@ -18,6 +17,8 @@ from app.runtime.extensions.module.contracts import ( get_module_method_contract, is_explicit_module_method, ) +from app.runtime.log import logger +from app.runtime.observability import observe_duration, record_metric from app.schemas.exception import RateLimitExceededException @@ -41,6 +42,50 @@ ModuleErrorHandler = Callable[..., None] AsyncFunctionRunner = Callable[..., Any] +class FrozenModuleProviderMissingError(LookupError): + """表示冻结插件 provider 已无法在当前精确目录中解析。""" + + +@dataclass(frozen=True, slots=True) +class FrozenPluginProviderRef: + """保存可持久化的插件 provider 身份及其冻结方法。""" + + plugin_id: str + plugin_name: str + method: str + + def __post_init__(self) -> None: + """拒绝无法精确解析的空 provider 身份。""" + if not self.plugin_id or not self.plugin_name or not self.method: + raise ValueError("冻结插件 provider 缺少 id、名称或方法") + + def to_payload(self) -> dict[str, str]: + """生成可直接写入 JSON 的稳定 provider 引用。""" + return { + "plugin_id": self.plugin_id, + "plugin_name": self.plugin_name, + "method": self.method, + } + + @classmethod + def from_payload(cls, payload: Mapping[str, Any]) -> "FrozenPluginProviderRef": + """从持久化映射恢复并校验 provider 引用。""" + return cls( + plugin_id=str(payload.get("plugin_id") or ""), + plugin_name=str(payload.get("plugin_name") or ""), + method=str(payload.get("method") or ""), + ) + + +@dataclass(frozen=True, slots=True) +class _PluginProvider: + """绑定一次目录解析得到的插件身份和可调用对象。""" + + plugin_id: str + plugin_name: str + func: Callable[..., Any] + + class _ProviderCallMode(StrEnum): """描述当前 provider 应采用的兼容调用方式。""" @@ -77,14 +122,72 @@ class ModuleInvocationDispatcher: return all(value is None for value in result) return result is None + def freeze_plugin_providers( + self, + method: str, + ) -> tuple[FrozenPluginProviderRef, ...]: + """按当前插件目录顺序冻结实现指定方法的精确 provider 引用。""" + return tuple( + FrozenPluginProviderRef( + plugin_id=provider.plugin_id, + plugin_name=provider.plugin_name, + method=method, + ) + for provider in self._collect_plugin_providers(method, error_kwargs={}) + ) + + def execute_frozen_plugin_providers( + self, + method: str, + providers: tuple[FrozenPluginProviderRef, ...], + *args: Any, + initial_result: Any = None, + before_invoke: Callable[[], None] | None = None, + **kwargs: Any, + ) -> Any: + """解析全部冻结引用后运行前置钩子，并严格执行原顺序 provider。""" + resolved = self._resolve_frozen_plugin_providers(method, providers) + if resolved and before_invoke is not None: + before_invoke() + return self._execute_plugin_provider_sequence( + method, + initial_result, + resolved, + *args, + strict_errors=True, + **kwargs, + ) + def dispatch(self, method: str, *args: Any, **kwargs: Any) -> Any: """先执行插件模块，再按优先级执行宿主模块。""" + return self._dispatch(method, *args, strict_errors=False, **kwargs) + + def dispatch_strict(self, method: str, *args: Any, **kwargs: Any) -> Any: + """执行模块并传播 provider 异常，使空结果与查询失败保持可区分。""" + return self._dispatch(method, *args, strict_errors=True, **kwargs) + + def _dispatch( + self, + method: str, + *args: Any, + strict_errors: bool, + **kwargs: Any, + ) -> Any: + """按统一聚合规则调度，并由调用方选择是否隔离 provider 异常。""" contract = get_module_method_contract(method) logger.debug("模块方法契约：%s -> %s", method, contract.family) - with observe_duration( - "module.provider.duration", method=method, provider_type="plugin" - ): - result = self.execute_plugin_modules(method, None, *args, **kwargs) + result = None + if contract.public_to_plugins: + with observe_duration( + "module.provider.duration", method=method, provider_type="plugin" + ): + result = self.execute_plugin_modules( + method, + None, + *args, + strict_errors=strict_errors, + **kwargs, + ) if ( contract.plugin_short_circuit and not self.is_valid_empty(result) @@ -94,21 +197,29 @@ class ModuleInvocationDispatcher: with observe_duration( "module.provider.duration", method=method, provider_type="system" ): - return self.execute_system_modules(method, result, *args, **kwargs) + return self.execute_system_modules( + method, + result, + *args, + strict_errors=strict_errors, + **kwargs, + ) async def async_dispatch(self, method: str, *args: Any, **kwargs: Any) -> Any: """以与同步路径相同的聚合规则执行同步或异步模块方法。""" contract = get_module_method_contract(method) logger.debug("异步模块方法契约：%s -> %s", method, contract.family) - with observe_duration( - "module.provider.duration", method=method, provider_type="plugin" - ): - result = await self.async_execute_plugin_modules( - method, - None, - *args, - **kwargs, - ) + result = None + if contract.public_to_plugins: + with observe_duration( + "module.provider.duration", method=method, provider_type="plugin" + ): + result = await self.async_execute_plugin_modules( + method, + None, + *args, + **kwargs, + ) if ( contract.plugin_short_circuit and not self.is_valid_empty(result) @@ -130,37 +241,130 @@ class ModuleInvocationDispatcher: method: str, result: Any, *args: Any, + strict_errors: bool = False, **kwargs: Any, ) -> Any: """同步执行插件方法，保留插件顺序、短路和列表合并语义。""" - aggregation = get_module_method_contract(method).aggregation + providers = self._collect_plugin_providers( + method, + error_kwargs=kwargs, + strict_errors=strict_errors, + ) + return self._execute_plugin_provider_sequence( + method, + result, + providers, + *args, + strict_errors=strict_errors, + **kwargs, + ) + + def _collect_plugin_providers( + self, + method: str, + *, + error_kwargs: Mapping[str, Any], + strict_errors: bool = False, + ) -> tuple[_PluginProvider, ...]: + """从同一插件目录快照收集 provider，并隔离损坏的方法表。""" + providers = [] for plugin, module_dict in self._plugin_catalog.get_plugin_modules().items(): plugin_id, plugin_name = plugin try: - # 防御坏插件把方法表声明成非映射类型，避免击穿整个模块调度 if not isinstance(module_dict, Mapping): raise TypeError( - f"插件 {plugin_id} 的模块声明必须是映射，实际是 {type(module_dict).__name__}" + f"插件 {plugin_id} 的模块声明必须是映射，实际是 " + f"{type(module_dict).__name__}" ) func = module_dict.get(method) if not func: continue + providers.append( + _PluginProvider( + plugin_id=plugin_id, + plugin_name=plugin_name, + func=func, + ) + ) + except Exception as err: + self._record_timeout(method, "plugin", err) + self._plugin_error_handler( + err, + plugin_id, + plugin_name, + method, + **error_kwargs, + ) + if strict_errors: + raise + return tuple(providers) + + def _resolve_frozen_plugin_providers( + self, + method: str, + providers: tuple[FrozenPluginProviderRef, ...], + ) -> tuple[_PluginProvider, ...]: + """一次性精确解析全部冻结引用，缺失时在执行任何副作用前失败。""" + module_catalog = self._plugin_catalog.get_plugin_modules() + resolved = [] + for provider in providers: + if provider.method != method: + raise FrozenModuleProviderMissingError( + f"冻结插件 provider 方法不匹配：" + f"{provider.plugin_id}/{provider.plugin_name} " + f"冻结为 {provider.method}，请求执行 {method}" + ) + module_dict = module_catalog.get( + (provider.plugin_id, provider.plugin_name) + ) + func = module_dict.get(method) if isinstance(module_dict, Mapping) else None + if not callable(func): + raise FrozenModuleProviderMissingError( + f"冻结插件 provider 已缺失：" + f"{provider.plugin_id}/{provider.plugin_name}.{method}" + ) + resolved.append( + _PluginProvider( + plugin_id=provider.plugin_id, + plugin_name=provider.plugin_name, + func=func, + ) + ) + return tuple(resolved) + + def _execute_plugin_provider_sequence( + self, + method: str, + result: Any, + providers: tuple[_PluginProvider, ...], + *args: Any, + strict_errors: bool = False, + **kwargs: Any, + ) -> Any: + """按统一契约执行已解析插件序列，并按调用模式处理 provider 故障。""" + aggregation = get_module_method_contract(method).aggregation + for provider in providers: + try: self._record_legacy_hit( method, caller_type="plugin", abi_source="third_party_plugin", ) - self._diagnose_callable(method, func, f"插件 {plugin_id}") - logger.info("请求插件 %s 执行：%s ...", plugin_name, method) + self._diagnose_callable( + method, + provider.func, + f"插件 {provider.plugin_id}", + ) + logger.info("请求插件 %s 执行：%s ...", provider.plugin_name, method) call_mode = self._provider_call_mode( aggregation, result, - func, + provider.func, allow_relay=False, ) if call_mode is _ProviderCallMode.STOP: break - provider_result = func(*args, **kwargs) + provider_result = provider.func(*args, **kwargs) self._diagnose_result(method, provider_result, "plugin") result = self._aggregate_provider_result( result, @@ -172,19 +376,23 @@ class ModuleInvocationDispatcher: self._rate_limit_handler( err, "插件", - plugin_id, + provider.plugin_id, method, **kwargs, ) + if strict_errors: + raise except Exception as err: self._record_timeout(method, "plugin", err) self._plugin_error_handler( err, - plugin_id, - plugin_name, + provider.plugin_id, + provider.plugin_name, method, **kwargs, ) + if strict_errors: + raise return result async def async_execute_plugin_modules( @@ -254,6 +462,7 @@ class ModuleInvocationDispatcher: method: str, result: Any, *args: Any, + strict_errors: bool = False, **kwargs: Any, ) -> Any: """同步执行按优先级排序的宿主模块，并支持签名接力。""" @@ -301,6 +510,8 @@ class ModuleInvocationDispatcher: method, **kwargs, ) + if strict_errors: + raise except Exception as err: self._record_timeout(method, "system", err) self._system_error_handler( @@ -310,6 +521,8 @@ class ModuleInvocationDispatcher: method, **kwargs, ) + if strict_errors: + raise return result async def async_execute_system_modules( diff --git a/app/startup/initializers/modules.py b/app/startup/initializers/modules.py index 9a43cf897..d51fa49a1 100644 --- a/app/startup/initializers/modules.py +++ b/app/startup/initializers/modules.py @@ -1,7 +1,7 @@ import asyncio import inspect import sys -from typing import Callable +from typing import Any, Callable from app.adapters.cache.redis import AsyncRedisHelper, RedisHelper from app.application.plugin.transaction import ( @@ -241,6 +241,13 @@ async def _async_get_workflow(workflow_id: int): return await WorkflowOper().async_get(workflow_id) +def _execute_legacy_transfer_command(**kwargs: Any) -> Any: + """把旧 Chain ABI 延迟转入唯一 TransferChain durable command。""" + from app.chain.transfer import TransferChain + + return TransferChain().execute_legacy_transfer_command(**kwargs) + + def _build_chain_runtime_context() -> ChainRuntimeContext: """在启动组合根创建 Chain 所需的运行时对象和数据端口。""" return ChainRuntimeContext( @@ -255,6 +262,7 @@ def _build_chain_runtime_context() -> ChainRuntimeContext: send_callback=callback ), module_dispatcher_factory=ModuleInvocationDispatcher, + legacy_transfer_command=_execute_legacy_transfer_command, configuration=build_chain_runtime_config(legacy_settings), data_ports=get_chain_data_ports(), durable_event_writer=TransactionalChainDurableEventWriter(SessionFactory), diff --git a/database/versions/c2f8a4d6e1b3_3_0_14.py b/database/versions/c2f8a4d6e1b3_3_0_14.py new file mode 100644 index 000000000..e13fbcb14 --- /dev/null +++ b/database/versions/c2f8a4d6e1b3_3_0_14.py @@ -0,0 +1,185 @@ +"""3.0.14 为整理任务增加版本化规划输入与原子计划检查点。 + +Revision ID: c2f8a4d6e1b3 +Revises: b1e7d3f5a9c2 +Create Date: 2026-08-27 +""" + +import hashlib +import json + +import sqlalchemy as sa +from alembic import op + +revision = "c2f8a4d6e1b3" +down_revision = "b1e7d3f5a9c2" +branch_labels = None +depends_on = None + +_TABLE_NAME = "transferpending" +_NEW_COLUMNS = { + "input_version", + "planning_input", + "input_fingerprint", + "checkpoint_version", + "checkpoint_payload", + "planned_at", +} + + +def _column_names() -> set[str]: + """返回当前待整理登记表的字段集合。""" + inspector = sa.inspect(op.get_bind()) + if _TABLE_NAME not in inspector.get_table_names(): + return set() + return { + column["name"] + for column in inspector.get_columns(_TABLE_NAME) + } + + +def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, object]: + """构造与 Application 规划输入第一版一致的保守重规划 JSON。""" + return { + "schema_version": 1, + "source_fileitem": {"storage": storage, "path": src_path}, + "meta": None, + "mediainfo": None, + "target_directory": None, + "target_storage": None, + "target_path": None, + "requested_transfer_type": None, + "media_source": None, + "media_id": None, + "media_type": None, + "need_scrape": False, + "need_rename": True, + "need_notify": True, + "overwrite_mode": None, + "episodes_info": [], + "preview": False, + "options": {"legacy_replan": True}, + } + + +def _fingerprint(payload: dict[str, object]) -> str: + """按稳定 JSON 编码计算与 Application 一致的输入指纹。""" + canonical = json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _backfill_planning_input() -> None: + """为 3.0.13 登记生成可辨识且必须重新规划的最小输入。""" + pending = sa.table( + _TABLE_NAME, + sa.column("id", sa.Integer()), + sa.column("storage", sa.String()), + sa.column("src_path", sa.String()), + sa.column("input_version", sa.Integer()), + sa.column("planning_input", sa.JSON()), + sa.column("input_fingerprint", sa.String()), + ) + connection = op.get_bind() + rows = connection.execute( + sa.select( + pending.c.id, + pending.c.storage, + pending.c.src_path, + pending.c.input_version, + pending.c.planning_input, + pending.c.input_fingerprint, + ) + ).mappings().all() + for row in rows: + if ( + row["input_version"] is not None + and row["planning_input"] is not None + and row["input_fingerprint"] + ): + continue + payload = row["planning_input"] + if not isinstance(payload, dict): + payload = _legacy_planning_payload(row["storage"], row["src_path"]) + payload_version = payload.get("schema_version") + input_version = row["input_version"] + if input_version is None: + input_version = payload_version if isinstance(payload_version, int) else 1 + input_fingerprint = row["input_fingerprint"] or _fingerprint(payload) + connection.execute( + pending.update() + .where(pending.c.id == row["id"]) + .values( + input_version=input_version, + planning_input=payload, + input_fingerprint=input_fingerprint, + ) + ) + + +def upgrade() -> None: + """增加规划输入和完整检查点字段并保守回填旧登记。""" + columns = _column_names() + if not columns: + return + additions = ( + ("input_version", sa.Column("input_version", sa.Integer(), nullable=True)), + ("planning_input", sa.Column("planning_input", sa.JSON(), nullable=True)), + ( + "input_fingerprint", + sa.Column("input_fingerprint", sa.String(length=64), nullable=True), + ), + ("checkpoint_version", sa.Column("checkpoint_version", sa.Integer(), nullable=True)), + ("checkpoint_payload", sa.Column("checkpoint_payload", sa.JSON(), nullable=True)), + ("planned_at", sa.Column("planned_at", sa.String(length=40), nullable=True)), + ) + for column_name, column in additions: + if column_name not in columns: + op.add_column(_TABLE_NAME, column) + + _backfill_planning_input() + with op.batch_alter_table(_TABLE_NAME) as batch_op: + batch_op.alter_column( + "input_version", existing_type=sa.Integer(), nullable=False + ) + batch_op.alter_column( + "planning_input", existing_type=sa.JSON(), nullable=False + ) + batch_op.alter_column( + "input_fingerprint", + existing_type=sa.String(length=64), + nullable=False, + ) + + +def downgrade() -> None: + """移除规划字段并把旧版本无法识别的状态保守恢复为接纳态。""" + columns = _column_names() + if not columns or not (_NEW_COLUMNS & columns): + return + if "state" in columns: + pending = sa.table( + _TABLE_NAME, + sa.column("state", sa.String()), + ) + op.get_bind().execute( + pending.update() + .where(pending.c.state.in_(("planned", "provider_pending"))) + .values(state="accepted") + ) + with op.batch_alter_table(_TABLE_NAME) as batch_op: + for column_name in ( + "planned_at", + "checkpoint_payload", + "checkpoint_version", + "input_fingerprint", + "planning_input", + "input_version", + ): + if column_name in columns: + batch_op.drop_column(column_name) diff --git a/docs/architecture-optimization-checklist.md b/docs/architecture-optimization-checklist.md index 6d9760da8..284325246 100644 --- a/docs/architecture-optimization-checklist.md +++ b/docs/architecture-optimization-checklist.md @@ -69,17 +69,17 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` | 指标 | 当前值 | 解释 | |---|---:|---| -| 宿主 Python 模块 / 内部依赖边 | 836 / 6,827 | `dependency-baseline.json` 当前快照 | +| 宿主 Python 模块 / 内部依赖边 | 836 / 6,832 | `dependency-baseline.json` 当前快照 | | 非平凡 SCC | 2 | 新增 Chain 包根环；另一个是隔离的 29 模块 TMDB 移植包环 | | 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 | | Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 | -| Module Contract | 215 specs / 214 methods / 264 calls | 动态方法名为 0；仍有 50 个结果形状为 `ANY` | +| Module Contract | 217 specs / 215 methods / 265 calls | 动态方法名为 0；内部 planning 合同不进入插件调度，旧 transfer 只保留 provider ABI | | Event Contract | 53 | 均已有 payload model，但当前全部是 diagnostic enforcement | | Python 源码量 | 约 271,400 行 | 60 个文件超过 1,000 行，14 个超过 2,000 行 | | 长方法 | 281 个超过 80 行 | 67 个超过 150 行，23 个超过 250 行；大量是私有方法 | | 全量 mypy 历史债务 | 11,983 / 601 文件 | strict frontier 当前只覆盖 41 个文件，且 ratchet 已新增 2 个错误 | -| Ruff 历史诊断 | 967 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` | -| 覆盖率低水位 | Application 77.85%，Domain 79.24% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 | +| Ruff 历史诊断 | 937 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` | +| 覆盖率低水位 | Application 78.06%，Domain 79.29% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 | ### 3.3 热点文件 @@ -160,7 +160,8 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` - 架构总览此前仍记录 811 模块、6,572 条边和 1 个 SCC，已经落后于当前基线。 - Event consumer 扫描曾把任意同名 `.register()` 调用当成事件注册；S0-L2.5 已改为证明 canonical EventManager receiver，10 个动态误报归零并保留唯一 workflow 动态注册。 -- S0-L2.6 已将 producer/consumer 合并为逐调用事实源：99 个 producer（98 静态、1 动态）与 +- S0-L2.6 已将 producer/consumer 合并为逐调用事实源；本轮统一 Transfer 事件发送点后为 + 97 个 producer（96 静态、1 动态）与 17 个 consumer（16 静态、1 动态）；consumer 由不可自动写入的精确人工 policy 管理。 **目标与步骤** @@ -188,7 +189,9 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` - `S1-L1.1 Durable admission`：`VERIFIED`。已交付 persist-before-enqueue、Application-owned typed Port、 DB adapter 与可逆 migration，宿主退出 raw/`Any` `TransferPendingOper` admission 路径。 -- `S1-L1.2 Planning checkpoint`：`PLANNED`。持久化稳定任务身份、整理模式、规划状态和目标 checkpoint。 +- `S1-L1.2 Planning checkpoint`：`VERIFIED`。版本化请求与指纹先准入；无 legacy provider 时通过 + `accepted -> planned` CAS 提交完整目标和有序操作，有 provider 时先提交 `provider_pending`，全部 + 返回空后再以第二次 CAS 提交 `planned`；planned 重放只消费冻结上下文和目标。 - `S1-L1.3 Lease 与恢复调度`：`PLANNED`。交付 claim/lease/heartbeat/attempt、过期接管与唯一恢复入口。 - `S1-L1.4 幂等执行与终态结算`：`PLANNED`。交付文件/历史幂等、唯一 retry owner 和 `manual_review` 语义。 @@ -201,28 +204,35 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` 异常均不会伪装成重复任务成功，失败记录可供恢复。 - 宿主 canonical Chain 只取得类型化 `TransferAdmissionRepository`；旧 Oper API 仅保留给统一兼容层， 插件公开 `TransferTask.to_dict()` 字段未增加内部任务标识。 -- worker 未知异常最终也会在 `app/chain/transfer.py:1107-1113,1262-1272` 删除 pending。 -- `app/db/models/transferpending.py:9-34` 只有 `storage/src_path/created_at`，没有目标、模式、 - step、lease、attempt、last_error，无法判定“文件已移动、历史未提交”等中间态。 +- `S1-L1.2` 已消除 checkpoint 前的文件副作用：目标路径、操作顺序及 resolved 识别上下文原子落库后， + 执行器才允许触发 cleanup、建目录和复制/移动；规划失败保留 `accepted` 并记录 `last_error`。 +- 旧插件 `transfer` provider 的身份、顺序和原始 ABI 参数先冻结为 `provider_pending`；提交后才精确 + 解析并严格执行，缺失或异常不 fallback。全部返回空后才生成宿主计划，并以第二次 CAS 提升为 + `planned` 后执行。旧 caller 只经 `ChainBase.transfer` 注入式兼容门面进入同一 durable command， + 宿主 FileManager/TransHandler 的旧执行入口已删除。 +- `TransferPending` 现在可区分 `accepted/provider_pending/planned`，但尚无 + claim/lease/heartbeat/attempt、逐步骤执行结果和 `manual_review`，仍无法判定“文件已移动、历史未提交” + 等后续中间态。 - 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、 lease/heartbeat 和人工恢复要求不一致。 **目标与步骤** -- [ ] 先在独立持久事务中 commit pending，再尝试放入内存队列；数据库事务不能与 `queue.Queue` +- [x] 先在独立持久事务中 commit pending，再尝试放入内存队列；数据库事务不能与 `queue.Queue` 原子提交，入队失败时必须保留 pending 供重放。 -- [ ] 初始登记保存稳定源身份、模式、状态和 attempt/lease；目标在规划完成后以 planning checkpoint - 更新，不能要求任务刚入队时已经具备尚未计算的目标路径。 +- [x] 初始登记保存稳定源身份、版本化请求和状态；目标与有序操作在纯规划完成后以 planning + checkpoint 原子更新，任何文件副作用不得早于该提交。 +- [ ] 增加 claim/lease/heartbeat/attempt 与过期接管，同一任务同时只能有一个 worker owner。 - [ ] 设计幂等文件操作和历史提交；只有所有必要步骤达到持久终态后才能删除记录。 - [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner，定义旧记录迁移和兼容规则。 - [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界，不直接套用 E2 Outbox 的 dead-letter 语义；禁止按年龄通用清理 pending。 -- [ ] 数据模型变更必须配套 Alembic migration，并验证升级与降级路径。 +- [x] 当前 admission/planning 数据模型变更均配套 Alembic migration，并验证升级、降级和中断重跑。 **故障注入验收** -- [ ] 登记后、内存入队前崩溃，重启可继续。 -- [ ] 持久登记成功但内存入队失败，重启可继续。 +- [x] 登记后、内存入队前崩溃，重启可继续。 +- [x] 持久登记成功但内存入队失败，重启可继续。 - [ ] 文件移动后、历史提交前崩溃，在支持稳定身份/幂等操作的存储上不重复移动且可补齐历史。 - [ ] worker 未知异常和 lease 超时后保留可诊断状态。 - [ ] 重复回放、重复消息和人工重试都保持幂等。 diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index f175fc9b7..ba9a57eee 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -369,8 +369,8 @@ collector 只接受 canonical `eventmanager`、`EventManager()` 及其有限别 `register`/`add_event_listener`；当前宿主有 16 个静态注册点，另保留 1 个由工作流配置驱动的 真实动态注册。`app/plugins/**` 插件副本不进入宿主事实。 -生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 99 个 -生产调用，其中 98 个静态解析为 100 个事件引用，只有 `Command.send_plugin_event` 的插件事件类型 +生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 97 个 +生产调用，其中 96 个静态解析为 98 个事件引用，只有 `Command.send_plugin_event` 的插件事件类型 保持动态；17 个消费注册中 16 个静态、1 个动态。生成的 `runtime-contract-baseline.json` 保存 line-free 事实、数量和枚举索引；人工维护的 `runtime-contract-policy.json` 只批准 consumer 的精确 fingerprint、owner 和理由，任何新增、替换、 @@ -690,7 +690,7 @@ flowchart LR stream/vendor/diagnostic/control-plane 事实是精确 containment。每条初始边的指纹由测试独立冻结， bindings/uses 变化、分类互换、通配导入和初始边增长都会失败；债务删除时同步删除冻结项以禁止恢复， `--write-host` 不会改写人工 policy 或冻结上界。 -- `event_facts` 是生产者/消费者唯一收集源；运行快照记录 99 个生产调用和 17 个消费注册， +- `event_facts` 是生产者/消费者唯一收集源；运行快照记录 97 个生产调用和 17 个消费注册， consumer 的 17 个唯一 fingerprint 另由只读人工 policy 精确准入。CI 将语义 policy 与生成快照 分成独立步骤，前者不能通过刷新后者绕过。 - 任何所有权迁移必须同步更新：canonical 导入、`app/runtime/compat/manifest.py`、 @@ -704,13 +704,13 @@ flowchart LR | 指标 | 当前值 | |---|---:| -| Python 模块 | 835 | -| 内部导入边 | 6,827 | +| Python 模块 | 836 | +| 内部导入边 | 6,832 | | 非平凡 SCC | 2（`ARCH-107` 临时 Chain 包根环；精确 containment 的 TMDB 移植包环） | | Direct egress | 66（12 条待迁移债务，54 条精确 containment） | -| Module Contract V2 spec | 215（其中 214 个进入 `run_module` 观察面） | +| Module Contract V2 spec | 217（其中 215 个进入 `run_module` 观察面） | | Event Contract | 53 | -| Event producer / consumer | 99（98 静态、1 动态）/ 17（16 静态、1 动态） | +| Event producer / consumer | 97（96 静态、1 动态）/ 17（16 静态、1 动态） | | Model/Oper 自动事务与自建 Session | 0 | | 组合根外 `SystemConfigOper()` | 0 | diff --git a/docs/architecture-refactor-roadmap.md b/docs/architecture-refactor-roadmap.md index 8e9af3375..2a863184f 100644 --- a/docs/architecture-refactor-roadmap.md +++ b/docs/architecture-refactor-roadmap.md @@ -93,7 +93,7 @@ G-ARCH 只有在以下条件全部满足后才可完成： | Leaf | 状态 | 依赖 | 完成定义 | |---|---|---|---| | S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地；先持久 commit 再入队，入队失败保留可恢复记录；宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission | -| S1-L1.2 Planning checkpoint | `PLANNED` | S1-L1.1 | 稳定任务身份、整理模式和 planning 状态持久化；目标路径只在规划完成后写入 checkpoint，任何文件副作用前已有可判定状态 | +| S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化；无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划，有 provider 时先提交 `provider_pending`，全部返回空后再以第二次 CAS 提交 `planned`；重放只执行冻结目标，所有文件副作用晚于对应 checkpoint commit | | S1-L1.3 Lease 与恢复调度 | `PLANNED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地；启动回放和同进程恢复共用唯一调度入口，同一任务同时只有一个 worker owner | | S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放；唯一 retry owner 生效，未知外部结果进入 `manual_review`，仅完整终态删除 pending | | S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整；旧 fail-open、重复状态与兼容层外旧入口删除，ARCH-102 债务归零 | @@ -144,7 +144,7 @@ G-ARCH 只有在以下条件全部满足后才可完成： | S4-L2 Event strict contract | `PLANNED` | S0-L2.6,S1-L6 | 宿主事件输入/输出按风险 strict，诊断例外只属于第三方插件兼容 | | S4-L3 Complexity v2 | `PLANNED` | S3 | 私有方法、class/file、圈复杂度进入门禁；所有超限通过职责拆分归零 | | S4-L4 全量 mypy 清零 | `PLANNED` | S3,S4-L1,S4-L2 | `mypy-baseline.json` 归零并删除债务接受路径，全宿主 strict 类型通过 | -| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 967 条诊断归零，规则集扩展经过独立审查且新增诊断为零 | +| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 937 条诊断归零，规则集扩展经过独立审查且新增诊断为零 | | S4-L6 Coverage/并发/质量证据 | `PLANNED` | S3,S4-L1,S4-L2 | 高风险包纳入 coverage；raw concurrency 分类清零；Module Quality 有真实 evidence test | ### S5：Plugin、Agent、Domain、Startup 与最终收口 @@ -227,3 +227,50 @@ git diff --check `git diff --check` 全部通过。 - failure injection 已覆盖 admission 失败不入队、batch/enqueue 失败保留记录、批次返回失败、 queue -> worker -> terminal discard 稳定身份，以及 Legacy TransferTask 序列化字段不变。 + +### S1-L1.2 Planning checkpoint + +**Status:** `VERIFIED` + +**Outcome** + +把 durable admission 推进为可独立恢复的 `accepted -> provider_pending -> planned` 状态：准入时冻结 +版本化请求 JSON 和 SHA-256 指纹；存在旧插件 provider 时先 CAS 提交精确身份、顺序和原始 ABI 参数， +全部返回空后才由 FileManager 只读规划目标及有序叶操作，并通过第二次 CAS 提交宿主 checkpoint。 +任何对应 checkpoint 提交前都不允许进入其文件副作用；`provider_pending` 重放只消费冻结调用， +`planned` 重放只消费冻结 resolved 上下文、目标和操作，不重新访问在线识别、目录选择或重命名配置。 + +**Ownership and compatibility** + +- `app/application/transfer.py` 拥有 planning input、plan item、checkpoint 和状态错误合同；JSON 版本、 + 指纹及 resolved 上下文均可跨进程 round-trip。 +- `app/modules/filemanager/transhandler.py` 是唯一目标规划与文件执行实现；`FileManagerModule.transfer` + 与 `TransHandler.transfer_media` 已删除，不保留第二套重命名、覆盖或目录递归逻辑。 +- `app/db/adapters/transfer.py` 通过短 Session/UoW 提交 checkpoint；Oper 只负责带状态和指纹条件的 + stage，3.0.14 migration 可升级、降级并在中断后重跑。 +- cleanup intent 随准入输入冻结。宿主路径由 FileManager 在 `TransferIntercept` 放行后、任何文件写入前 + 执行；legacy provider 路径为保持旧 ABI 顺序，在全部冻结引用解析成功后、调用 provider 前执行。 + strict 查询确认目标不存在才视为幂等成功，查询或删除失败抛错并保留对应 checkpoint 供重试；provider + 全空后提升的宿主 checkpoint 会记录 cleanup 已完成，禁止二次查询或删除。 +- 插件公开 Transfer 方法签名、事件类型和 payload 不变；旧 provider 身份和顺序随 checkpoint + 冻结，提交后由统一 dispatcher 精确解析并严格执行，缺失或异常时明确失败而不静默换路；全部返回空 + 才生成宿主 plan，并以第二次 CAS 提交 `planned` checkpoint 后执行。`ChainBase.transfer` 仅委托启动 + 组合根注入的 canonical durable + command，内部 plan/execute 合同不向插件调度；新 DTO 不从包根重复导出，`app/plugins/**` 插件 + 副本不参与改造。 + +**Excluded** + +- 本叶不引入 claim、lease、heartbeat、attempt、执行步骤幂等或 `manual_review`；这些由 + `S1-L1.3` 和 `S1-L1.4` 交付。 +- 文件操作成功后到历史结算前的未知结果仍未达到 E3，ARCH-102 父项继续保持执行中。 + +**Local verification (2026-08-27)** + +- planning、持久化、迁移、兼容、replay 和 worker 聚焦回归：`224 passed, 2 skipped`；跳过项仅为 + 本机未配置隔离 PostgreSQL，SQLite upgrade/downgrade/re-upgrade 已覆盖。 +- 完整本地套件：`6,578 passed, 8 skipped`；架构回归：`174 passed`；scoped Pylint `10.00/10`； + host baseline、Ruff/mypy ratchet 与 `git diff --check` 通过。 +- failure injection 覆盖 commit 前零文件副作用、commit 后崩溃重放、离线 resolved context 恢复、 + 配置漂移仍使用冻结 target storage、规划失败留痕、旧 provider 提交后短路、严格异常、空结果两阶段 + fallback、缺失引用零 cleanup，以及 cleanup 顺序/幂等/瞬时失败。 diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index 9ce8f1142..c0dfc7bcd 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -467,10 +467,25 @@ Durable post-commit side effects have a separate boundary: Transfer durable admission follows the same ownership direction without using the Outbox as an execution queue: `app/application/transfer.py` owns the typed -admission contract and persist-before-enqueue orchestration, while -`app/db/adapters/transfer.py` commits it in a short Session/UoW. Canonical host -chains never obtain `TransferPendingOper`; its no-Session API remains only for -the exact legacy plugin import contract. +admission and versioned planning-checkpoint contracts, while +`app/db/adapters/transfer.py` commits admission and the +`accepted -> provider_pending -> planned` compare-and-set transitions in short +Session/UoW scopes. `app/modules/filemanager/` owns the +single pure-plan and checkpoint-execution implementation: all file writes occur +after checkpoint commit, and planned recovery consumes frozen resolved context, +target storage and ordered operations without online recognition or renaming. +Legacy plugin `transfer` providers are frozen by exact identity, order, and ABI +arguments in a provider-only checkpoint, then executed by the unified module +dispatcher only after commit. The dispatcher resolves every frozen reference +before the compatibility cleanup hook and propagates provider failures. Missing +or failing providers therefore remain `provider_pending`; only an all-empty +result permits host planning and a second CAS to `planned`. Host-only `plan_transfer` and +`execute_transfer_plan` contracts never dispatch to plugins. `ChainBase.transfer` +is the sole legacy caller facade and delegates the startup-injected durable +command; `FileManagerModule.transfer` and `TransHandler.transfer_media` must not +be recreated. +Canonical host chains never obtain `TransferPendingOper`; its no-Session API +remains only for the exact legacy plugin import contract. ## Composition and Compatibility Boundaries @@ -604,8 +619,8 @@ driven workflow registration. | `app/application/subscription/write.py` | Subscription media translation and sync/async write-port orchestration | | `app/application/outbox.py` | Durable intent, topic handler and Outbox repository contracts | | `app/db/adapters/outbox.py` | SQLAlchemy Outbox persistence, claim/lease and retry state adapter | -| `app/application/transfer.py` | Transfer task, durable admission contract and persist-before-enqueue use case | -| `app/db/adapters/transfer.py` | SQLAlchemy durable admission persistence and detached snapshot adapter | +| `app/application/transfer.py` | Transfer task, durable admission, versioned planning input/checkpoint contracts and queue use case | +| `app/db/adapters/transfer.py` | SQLAlchemy admission/checkpoint persistence, CAS state transition and detached snapshot adapter | | `app/application/scheduling.py` | Runtime scheduler facade for Agent tools and endpoints; `Scheduler` class registered by `app/startup/initializers/scheduler.py` | | `app/application/commands.py` | Command registry facade for Agent tools and endpoints; `Command` class registered by `app/startup/initializers/command.py` | | `app/application/workflow.py` | Workflow use cases plus the runtime port consumed by API and Chain; `WorkFlowManager` is registered by `app/startup/initializers/workflow.py` | @@ -626,7 +641,7 @@ driven workflow registration. | `app/runtime/event/snapshot.py` | Read-only typed payload snapshots for the plugin SDK; never mutates or replaces the event ABI | | `app/runtime/extensions/module/dispatcher.py` | Plugin-first invocation, short-circuit, list merge, signature relay and sync/async execution | | `app/runtime/extensions/module/contracts.py` | High-frequency method families and frozen legacy fallback contract | -| `app/application/chain/context.py` | Injectable Chain dependencies and no-argument compatibility provider | +| `app/application/chain/context.py` | Injectable Chain dependencies, no-argument compatibility provider and legacy Transfer command Port | | `app/startup/lifecycle/components.py` | Declarative normal/safe-mode lifecycle manifest, ordering and timeout budgets | | `app/runtime/extensions/module_manager.py` | Module discovery and lifecycle | | `app/runtime/extensions/plugin_manager.py` | Plugin discovery and lifecycle | diff --git a/tests/fixtures/architecture/coverage-baseline.json b/tests/fixtures/architecture/coverage-baseline.json index 200d85109..c3cffed4e 100644 --- a/tests/fixtures/architecture/coverage-baseline.json +++ b/tests/fixtures/architecture/coverage-baseline.json @@ -1,12 +1,12 @@ { "application": { - "covered_lines": 9341, - "percent": 77.85, - "statements": 11999 + "covered_lines": 9572, + "percent": 78.06, + "statements": 12263 }, "domain": { - "covered_lines": 3390, - "percent": 79.24, + "covered_lines": 3392, + "percent": 79.29, "statements": 4278 } } diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index e0128f404..9fcfdfee5 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -1441,8 +1441,8 @@ "runtime_only": true } }, - "edge_count": 6827, - "edge_sha256": "34e2be621e40f0f07ff04065655446c4ad0062883701f9e1e2235c29359900f7", + "edge_count": 6832, + "edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39", "edges": [ "app -> app.runtime", "app -> app.runtime.compat", @@ -4998,8 +4998,10 @@ "app.chain.transfer -> app.domain.context", "app.chain.transfer -> app.domain.episode", "app.chain.transfer -> app.domain.meta", + "app.chain.transfer -> app.domain.meta.metaanime", "app.chain.transfer -> app.domain.meta.metabase", "app.chain.transfer -> app.domain.meta.metamusic", + "app.chain.transfer -> app.domain.meta.metavideo", "app.chain.transfer -> app.domain.metainfo", "app.chain.transfer -> app.foundation", "app.chain.transfer -> app.foundation.singleton", @@ -5807,6 +5809,7 @@ "app.modules.filemanager.module -> app.application.directory", "app.modules.filemanager.module -> app.application.messaging", "app.modules.filemanager.module -> app.application.messaging.message", + "app.modules.filemanager.module -> app.application.transfer", "app.modules.filemanager.module -> app.domain", "app.modules.filemanager.module -> app.domain.context", "app.modules.filemanager.module -> app.domain.meta", @@ -5958,6 +5961,7 @@ "app.modules.filemanager.transhandler -> app.application.directory", "app.modules.filemanager.transhandler -> app.application.messaging", "app.modules.filemanager.transhandler -> app.application.messaging.message", + "app.modules.filemanager.transhandler -> app.application.transfer", "app.modules.filemanager.transhandler -> app.domain", "app.modules.filemanager.transhandler -> app.domain.context", "app.modules.filemanager.transhandler -> app.domain.meta", @@ -7903,6 +7907,7 @@ "app.startup.initializers.modules -> app.chain.site", "app.startup.initializers.modules -> app.chain.subscribe", "app.startup.initializers.modules -> app.chain.tmdb", + "app.startup.initializers.modules -> app.chain.transfer", "app.startup.initializers.modules -> app.chain.workflow", "app.startup.initializers.modules -> app.command", "app.startup.initializers.modules -> app.db", diff --git a/tests/fixtures/architecture/mypy-baseline.json b/tests/fixtures/architecture/mypy-baseline.json index 42b173ffb..46d470b53 100644 --- a/tests/fixtures/architecture/mypy-baseline.json +++ b/tests/fixtures/architecture/mypy-baseline.json @@ -1587,15 +1587,14 @@ "union-attr": 2 }, "app/chain/transfer.py": { - "arg-type": 65, - "assignment": 26, + "arg-type": 57, + "assignment": 25, "attr-defined": 4, "func-returns-value": 1, "misc": 2, "no-any-return": 3, - "no-redef": 2, "no-untyped-call": 8, - "no-untyped-def": 15, + "no-untyped-def": 14, "operator": 3, "return-value": 2, "truthy-function": 5, @@ -2105,15 +2104,14 @@ "union-attr": 8 }, "app/modules/filemanager/module.py": { - "arg-type": 21, - "assignment": 9, + "arg-type": 19, + "assignment": 4, "attr-defined": 1, "empty-body": 1, "no-any-return": 2, - "no-untyped-call": 6, + "no-untyped-call": 3, "no-untyped-def": 6, - "truthy-function": 4, - "type-arg": 4, + "type-arg": 3, "var-annotated": 3 }, "app/modules/filemanager/storages/__init__.py": { @@ -2203,15 +2201,14 @@ "union-attr": 32 }, "app/modules/filemanager/transhandler.py": { - "arg-type": 19, - "assignment": 8, + "arg-type": 18, + "assignment": 2, "attr-defined": 1, - "misc": 1, "no-any-return": 1, "no-redef": 1, "no-untyped-call": 1, - "no-untyped-def": 4, - "operator": 7, + "no-untyped-def": 3, + "operator": 6, "type-arg": 2, "union-attr": 2 }, diff --git a/tests/fixtures/architecture/ruff-baseline.json b/tests/fixtures/architecture/ruff-baseline.json index 49c77abf7..0b5fdd7d8 100644 --- a/tests/fixtures/architecture/ruff-baseline.json +++ b/tests/fixtures/architecture/ruff-baseline.json @@ -306,9 +306,6 @@ "app/application/agenttask.py": { "I001": 1 }, - "app/application/chain/context.py": { - "I001": 1 - }, "app/application/chain/data.py": { "I001": 1 }, @@ -414,9 +411,6 @@ "app/chain/media.py": { "E731": 2 }, - "app/chain/transfer.py": { - "E402": 21 - }, "app/command.py": { "I001": 1 }, @@ -676,13 +670,6 @@ "app/modules/filemanager/__init__.py": { "I001": 1 }, - "app/modules/filemanager/module.py": { - "I001": 1 - }, - "app/modules/filemanager/transhandler.py": { - "F541": 2, - "I001": 1 - }, "app/modules/filter/__init__.py": { "I001": 1 }, @@ -996,9 +983,6 @@ "app/runtime/extensions/managed_resource_adapter.py": { "I001": 1 }, - "app/runtime/extensions/module/dispatcher.py": { - "I001": 1 - }, "app/runtime/extensions/module_manager.py": { "I001": 1 }, @@ -1607,9 +1591,6 @@ "tests/test_music_torrents.py": { "I001": 1 }, - "tests/test_music_transfer.py": { - "I001": 1 - }, "tests/test_music_workflows.py": { "F401": 1 }, @@ -1855,9 +1836,6 @@ "tests/test_transfer_history_write_path.py": { "I001": 1 }, - "tests/test_transfer_job_manager.py": { - "I001": 1 - }, "tests/test_transfer_mark_torrent_completed.py": { "I001": 1 }, @@ -1867,9 +1845,6 @@ "tests/test_transfer_movie_collection.py": { "I001": 1 }, - "tests/test_transfer_preview.py": { - "I001": 1 - }, "tests/test_transfer_queue_count.py": { "I001": 1 }, diff --git a/tests/fixtures/architecture/runtime-contract-baseline.json b/tests/fixtures/architecture/runtime-contract-baseline.json index 516f846bd..377f06328 100644 --- a/tests/fixtures/architecture/runtime-contract-baseline.json +++ b/tests/fixtures/architecture/runtime-contract-baseline.json @@ -1518,8 +1518,7 @@ "ChainEventType.StorageOperSelection": { "consumer_fingerprints": [], "producer_fingerprints": [ - "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476", - "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476" + "76a388b1caf87a51cacf9d7534f64576f7c98e0af6b604d30114dab893066d96" ] }, "ChainEventType.SubscribeCompletionCheck": { @@ -1538,14 +1537,13 @@ "ChainEventType.TransferIntercept": { "consumer_fingerprints": [], "producer_fingerprints": [ - "144f9ffac815b2f58ade53881df0ce43381a7b1adff62f53a019794b801a0cd7", - "d7ee7c450874ea4bf6ea97c903b9cd3ebeab7266834c22e1a699d067205e8912" + "410f9308657c3c228e5357acf2e9a083c4fe6476c3b4ddaf240cab385d8bd4af" ] }, "ChainEventType.TransferOverwriteCheck": { "consumer_fingerprints": [], "producer_fingerprints": [ - "e761a75952ab6835e67b2c43c671f6361f1977690cfd7724f79439229408af40" + "be3334d20e9aa1fd641fe67f0bc3abd3ef48433728ddb76538cfbb475b4dc92b" ] }, "ChainEventType.TransferRename": { @@ -1809,11 +1807,11 @@ ] } }, - "fact_count": 116, + "fact_count": 114, "invalid_consumer_count": 0, "invalid_producer_count": 0, - "producer_call_count": 99, - "producer_event_reference_count": 100, + "producer_call_count": 97, + "producer_event_reference_count": 98, "producers": [ { "caller": "app.agent.orchestrator", @@ -2495,22 +2493,10 @@ "events": [ "ChainEventType.StorageOperSelection" ], - "fingerprint": "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476", + "fingerprint": "76a388b1caf87a51cacf9d7534f64576f7c98e0af6b604d30114dab893066d96", "invalid": false, "method": "send_event", - "qualname": "TransferChain.__handle_transfer", - "receiver_kind": "injected_event_manager" - }, - { - "caller": "app.chain.transfer", - "dynamic": false, - "events": [ - "ChainEventType.StorageOperSelection" - ], - "fingerprint": "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476", - "invalid": false, - "method": "send_event", - "qualname": "TransferChain.__handle_transfer", + "qualname": "TransferChain.__select_storage_oper", "receiver_kind": "injected_event_manager" }, { @@ -2673,22 +2659,22 @@ "events": [ "ChainEventType.TransferIntercept" ], - "fingerprint": "d7ee7c450874ea4bf6ea97c903b9cd3ebeab7266834c22e1a699d067205e8912", + "fingerprint": "410f9308657c3c228e5357acf2e9a083c4fe6476c3b4ddaf240cab385d8bd4af", "invalid": false, "method": "send_event", - "qualname": "TransHandler.__transfer_dir", + "qualname": "TransHandler.__intercept_transfer", "receiver_kind": "canonical_singleton" }, { "caller": "app.modules.filemanager.transhandler", "dynamic": false, "events": [ - "ChainEventType.TransferIntercept" + "ChainEventType.TransferOverwriteCheck" ], - "fingerprint": "144f9ffac815b2f58ade53881df0ce43381a7b1adff62f53a019794b801a0cd7", + "fingerprint": "be3334d20e9aa1fd641fe67f0bc3abd3ef48433728ddb76538cfbb475b4dc92b", "invalid": false, "method": "send_event", - "qualname": "TransHandler.__transfer_file", + "qualname": "TransHandler.__resolve_overwrite", "receiver_kind": "canonical_singleton" }, { @@ -2715,18 +2701,6 @@ "qualname": "TransHandler.get_rename_path", "receiver_kind": "canonical_singleton" }, - { - "caller": "app.modules.filemanager.transhandler", - "dynamic": false, - "events": [ - "ChainEventType.TransferOverwriteCheck" - ], - "fingerprint": "e761a75952ab6835e67b2c43c671f6361f1977690cfd7724f79439229408af40", - "invalid": false, - "method": "send_event", - "qualname": "TransHandler.transfer_media", - "receiver_kind": "canonical_singleton" - }, { "caller": "app.modules.navidrome", "dynamic": false, @@ -3005,7 +2979,7 @@ } ], "static_consumer_count": 16, - "static_producer_call_count": 98 + "static_producer_call_count": 96 }, "event_specs": { "ChainEventType.AgentLLMProvider": { @@ -5825,6 +5799,28 @@ "timeout_policy": "caller_budget", "version": 1 }, + "execute_transfer_plan": { + "aggregation": "first_non_empty", + "error_policy": "isolate_provider", + "execution": "sync_or_async", + "family": "storage", + "input_contract": "TransferPlanCheckpoint", + "plugin_short_circuit": true, + "public_to_plugins": false, + "required_parameters": [ + "checkpoint", + "mediainfo", + "meta", + "source_oper", + "target_oper" + ], + "result_contract": "TransferInfo | None", + "result_shape": "any", + "supports_async": true, + "supports_sync": true, + "timeout_policy": "caller_budget", + "version": 1 + }, "filter_torrents": { "aggregation": "ordered_list_merge", "error_policy": "isolate_provider", @@ -6864,6 +6860,37 @@ "timeout_policy": "caller_budget", "version": 1 }, + "plan_transfer": { + "aggregation": "first_non_empty", + "error_policy": "isolate_provider", + "execution": "sync_or_async", + "family": "storage", + "input_contract": "TransferPlanningInput", + "plugin_short_circuit": true, + "public_to_plugins": false, + "required_parameters": [ + "episodes_info", + "fileitem", + "library_category_folder", + "library_type_folder", + "mediainfo", + "meta", + "planning_input", + "preview", + "scrape", + "source_oper", + "target_directory", + "target_path", + "target_storage", + "transfer_type" + ], + "result_contract": "TransferPlanCheckpoint | None", + "result_shape": "any", + "supports_async": true, + "supports_sync": true, + "timeout_policy": "caller_budget", + "version": 1 + }, "recognize_media": { "aggregation": "first_non_empty", "error_policy": "isolate_provider", @@ -7947,10 +7974,10 @@ } }, "run_module": { - "call_count": 264, + "call_count": 265, "dynamic_call_count": 0, "dynamic_calls": [], - "method_count": 214, + "method_count": 215, "methods": { "anilist_credits": [ { @@ -8769,6 +8796,13 @@ "mode": "sync" } ], + "execute_transfer_plan": [ + { + "caller": "app.chain", + "count": 1, + "mode": "sync" + } + ], "filter_torrents": [ { "caller": "app.chain", @@ -9190,6 +9224,13 @@ "mode": "sync" } ], + "plan_transfer": [ + { + "caller": "app.chain", + "count": 1, + "mode": "sync" + } + ], "recognize_media": [ { "caller": "app.chain._recognition", @@ -9514,13 +9555,6 @@ "mode": "sync" } ], - "transfer": [ - { - "caller": "app.chain", - "count": 1, - "mode": "sync" - } - ], "transfer_completed": [ { "caller": "app.chain", diff --git a/tests/test_architecture_contract_baseline.py b/tests/test_architecture_contract_baseline.py index 2f6199f24..f9458809f 100644 --- a/tests/test_architecture_contract_baseline.py +++ b/tests/test_architecture_contract_baseline.py @@ -435,17 +435,17 @@ def test_event_contract_baseline_covers_every_public_event_enum() -> None: assert set(events["event_index"]) == expected assert events["event_count"] == len(expected) - assert events["producer_call_count"] == 99 - assert events["static_producer_call_count"] == 98 + assert events["producer_call_count"] == 97 + assert events["static_producer_call_count"] == 96 assert events["dynamic_producer_count"] == 1 assert events["invalid_producer_count"] == 0 - assert events["producer_event_reference_count"] == 100 + assert events["producer_event_reference_count"] == 98 assert events["consumer_registration_count"] == 17 assert events["static_consumer_count"] == 16 assert events["dynamic_consumer_count"] == 1 assert events["invalid_consumer_count"] == 0 assert events["consumer_event_reference_count"] == 16 - assert events["fact_count"] == 116 + assert events["fact_count"] == 114 assert len({fact["fingerprint"] for fact in events["consumers"]}) == 17 assert all( not fact["caller"].startswith("app.plugins") diff --git a/tests/test_architecture_event_facts.py b/tests/test_architecture_event_facts.py index aded0b695..09f9d47c1 100644 --- a/tests/test_architecture_event_facts.py +++ b/tests/test_architecture_event_facts.py @@ -946,7 +946,7 @@ def handler(event): def test_collect_event_facts_matches_current_host_inventory() -> None: - """统一事实覆盖当前 99 个 producer 调用与 17 个 consumer 调用。""" + """统一事实覆盖当前 97 个 producer 调用与 17 个 consumer 调用。""" from scripts.architecture.baseline import ( _event_enum_members, discover_modules, @@ -962,11 +962,11 @@ def test_collect_event_facts_matches_current_host_inventory() -> None: producers = facts["producers"] consumers = facts["consumers"] - assert len(producers) == 99 - assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 98 + assert len(producers) == 97 + assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 96 assert sum(fact["dynamic"] for fact in producers) == 1 assert sum(fact["invalid"] for fact in producers) == 0 - assert sum(len(fact["events"]) for fact in producers) == 100 + assert sum(len(fact["events"]) for fact in producers) == 98 assert len(consumers) == 17 assert sum(not fact["dynamic"] and not fact["invalid"] for fact in consumers) == 16 assert sum(fact["dynamic"] for fact in consumers) == 1 diff --git a/tests/test_filemanager_planning.py b/tests/test_filemanager_planning.py new file mode 100644 index 000000000..bdd610281 --- /dev/null +++ b/tests/test_filemanager_planning.py @@ -0,0 +1,577 @@ +from dataclasses import replace +from pathlib import Path + +import pytest + +from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput +from app.domain.context import MediaInfo +from app.domain.meta.metabase import MetaBase +from app.modules.filemanager import transhandler as transhandler_module +from app.modules.filemanager.module import FileManagerModule +from app.modules.filemanager.transhandler import TransHandler +from app.runtime.events import eventmanager +from app.schemas.system import TransferDirectoryConf +from app.schemas.types import ChainEventType, MediaType +from app.schemas.workflow import FileItem + + +class ReadOnlyTreeStorage: + """提供目录只读遍历，并让任何写接口调用立即失败。""" + + def __init__(self, children: dict[str, list[FileItem]]): + """保存按源目录路径索引的测试树。""" + self.children = children + self.reads: list[str] = [] + + def list(self, fileitem: FileItem) -> list[FileItem]: + """返回目录孩子并记录只读访问。""" + self.reads.append(fileitem.path) + return self.children.get(fileitem.path, []) + + def __getattr__(self, name: str): + """拒绝规划期意外访问的存储接口。""" + raise AssertionError(f"规划阶段不应访问存储接口：{name}") + + +class RecordingStorage: + """记录执行期存储调用顺序并模拟同一网盘复制。""" + + def __init__(self, calls: list[str], existing: FileItem = None): + """保存调用日志与可选同名目标。""" + self.calls = calls + self.existing = existing + + def get_item_strict(self, path: Path): + """记录严格目标查询。""" + self.calls.append("strict") + return self.existing + + def get_item(self, path: Path): + """记录普通目标查询。""" + self.calls.append("get_item") + return self.existing + + def get_folder(self, path: Path) -> FileItem: + """记录可能创建目录的接口。""" + self.calls.append("get_folder") + return FileItem( + storage="alist", + path=path.as_posix(), + name=path.name, + type="dir", + ) + + def delete(self, fileitem: FileItem) -> bool: + """记录删除副作用。""" + self.calls.append("delete") + self.existing = None + return True + + def is_support_transtype(self, transfer_type: str) -> bool: + """声明测试存储支持复制。""" + return transfer_type == "copy" + + def copy(self, fileitem: FileItem, path: Path, name: str) -> bool: + """记录复制副作用。""" + self.calls.append(f"copy:{(path / name).as_posix()}") + return True + + +def _build_media() -> tuple[MetaBase, MediaInfo]: + """构造文件规划所需的最小电视剧领域对象。""" + meta = MetaBase("Test.Show.S01E01.mkv") + meta.type = MediaType.TV + meta.name = "Test Show" + meta.year = "2026" + meta.begin_season = 1 + meta.begin_episode = 1 + mediainfo = MediaInfo( + type=MediaType.TV, + title="Test Show", + year="2026", + tmdb_id=12345, + ) + return meta, mediainfo + + +def _build_fileitem() -> FileItem: + """构造无需访问宿主文件系统的网盘源文件。""" + return FileItem( + storage="alist", + path="/downloads/Test.Show.S01E01.mkv", + name="Test.Show.S01E01.mkv", + basename="Test.Show.S01E01", + extension="mkv", + type="file", + size=1024, + ) + + +def _build_input(fileitem: FileItem, **overrides) -> TransferPlanningInput: + """构造可持久化规划输入。""" + values = { + "source_fileitem": fileitem.model_dump(mode="json"), + "target_storage": "alist", + "target_path": "/library", + "requested_transfer_type": "copy", + "need_rename": True, + "overwrite_mode": "always", + } + values.update(overrides) + return TransferPlanningInput(**values) + + +def _plan_file( + handler: TransHandler, + planning_input: TransferPlanningInput, + meta: MetaBase, + mediainfo: MediaInfo, + source_oper, +): + """使用测试固定策略规划单个文件。""" + return handler.plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_storage="alist", + target_path=Path("/library"), + transfer_type="copy", + need_scrape=False, + need_rename=True, + need_notify=True, + overwrite_mode="always", + episodes_info=None, + preview=False, + ) + + +def test_directory_planning_only_reads_source_and_freezes_ordered_leaf_operations(): + root = FileItem(storage="alist", path="/source/disc", name="disc", type="dir") + nested = FileItem(storage="alist", path="/source/disc/BDMV", name="BDMV", type="dir") + first = FileItem( + storage="alist", + path="/source/disc/BDMV/index.bdmv", + name="index.bdmv", + type="file", + extension="bdmv", + ) + second = FileItem( + storage="alist", + path="/source/disc/MovieObject.bdmv", + name="MovieObject.bdmv", + type="file", + extension="bdmv", + ) + storage = ReadOnlyTreeStorage( + { + root.path: [nested, second], + nested.path: [first], + } + ) + meta, mediainfo = _build_media() + planning_input = _build_input(root, need_rename=False) + + checkpoint = TransHandler().plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=storage, + target_storage="alist", + target_path=Path("/library"), + transfer_type="copy", + need_scrape=False, + need_rename=False, + need_notify=True, + overwrite_mode="never", + episodes_info=None, + preview=False, + ) + + assert storage.reads == [root.path, nested.path] + assert [item.sequence for item in checkpoint.items] == [0, 1] + assert [item.source_fileitem["path"] for item in checkpoint.items] == [ + first.path, + second.path, + ] + assert [item.target_path for item in checkpoint.items] == [ + "/library/disc/BDMV/index.bdmv", + "/library/disc/MovieObject.bdmv", + ] + + +def test_execute_uses_frozen_target_and_intercepts_before_directory_or_delete( + monkeypatch, +): + calls: list[str] = [] + fileitem = _build_fileitem() + meta, mediainfo = _build_media() + + def record_event(event_type, event_data): + """记录规划和执行事件顺序。""" + calls.append(event_type.value) + return None + + monkeypatch.setattr(eventmanager, "send_event", record_event) + handler = TransHandler() + checkpoint = _plan_file( + handler, + _build_input(fileitem), + meta, + mediainfo, + ReadOnlyTreeStorage({}), + ) + frozen_target = checkpoint.final_target_path + assert calls == [ + ChainEventType.TransferRenameBuild.value, + ChainEventType.TransferRename.value, + ] + calls.clear() + original_get_runtime_setting = transhandler_module.get_runtime_setting + + def drifted_runtime_setting(name: str): + """让执行期重新读取重命名配置时立即失败。""" + if name == "RENAME_FORMAT": + raise AssertionError("执行冻结计划不应重新读取重命名配置") + return original_get_runtime_setting(name) + + monkeypatch.setattr( + transhandler_module, + "get_runtime_setting", + drifted_runtime_setting, + ) + existing = FileItem( + storage="alist", + path=frozen_target, + name=Path(frozen_target).name, + type="file", + extension="mkv", + size=100, + ) + storage = RecordingStorage(calls, existing=existing) + + result = handler.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=storage, + target_oper=storage, + ) + + assert result.success is True + assert ChainEventType.TransferRenameBuild.value not in calls + assert ChainEventType.TransferRename.value not in calls + assert ChainEventType.TransferOverwriteCheck.value in calls + intercept_index = calls.index(ChainEventType.TransferIntercept.value) + assert intercept_index < calls.index("get_folder") + assert intercept_index < calls.index("delete") + assert f"copy:{frozen_target}" in calls + + +def test_module_preserves_admission_input_while_freezing_resolved_directory(): + fileitem = _build_fileitem() + meta, mediainfo = _build_media() + admission_input = _build_input( + fileitem, + target_storage=None, + target_path=None, + requested_transfer_type=None, + target_directory={"name": "library"}, + ) + fingerprint = admission_input.fingerprint + target_directory = TransferDirectoryConf( + name="library", + transfer_type="copy", + overwrite_mode="never", + library_path="/resolved-library", + library_storage="alist", + renaming=True, + scraping=False, + notify=True, + ) + + checkpoint = FileManagerModule().plan_transfer( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_directory=target_directory, + source_oper=ReadOnlyTreeStorage({}), + planning_input=admission_input, + ) + + assert checkpoint.planning_input is admission_input + assert checkpoint.planning_input.fingerprint == fingerprint + assert checkpoint.root_target_path == "/resolved-library" + assert checkpoint.final_target_path.startswith("/resolved-library/") + assert checkpoint.resolved_meta_kind == "MetaBase" + assert checkpoint.resolved_meta["begin_episode"] == 1 + assert checkpoint.resolved_mediainfo_kind == "MediaInfo" + assert checkpoint.resolved_mediainfo["title"] == "Test Show" + + +def _build_cleanup_checkpoint(monkeypatch, calls: list[str]): + """构造带冻结旧目标清理意图的单文件检查点。""" + fileitem = _build_fileitem() + meta, mediainfo = _build_media() + cleanup_item = FileItem( + storage="cleanup", + path="/old-library/old.mkv", + name="old.mkv", + type="file", + extension="mkv", + ) + planning_input = _build_input( + fileitem, + options={"cleanup_dest_fileitem": cleanup_item.model_dump(mode="json")}, + ) + + def record_event(event_type, event_data): + """记录执行事件顺序。""" + calls.append(event_type.value) + return None + + monkeypatch.setattr(eventmanager, "send_event", record_event) + checkpoint = _plan_file( + TransHandler(), + planning_input, + meta, + mediainfo, + ReadOnlyTreeStorage({}), + ) + calls.clear() + return checkpoint, meta, mediainfo, cleanup_item + + +def test_cleanup_runs_after_intercept_and_before_transfer_side_effects(monkeypatch): + calls: list[str] = [] + checkpoint, meta, mediainfo, cleanup_item = _build_cleanup_checkpoint( + monkeypatch, + calls, + ) + transfer_storage = RecordingStorage(calls) + module = FileManagerModule() + cleaned_items: list[FileItem] = [] + + def cleanup_media_file(fileitem: FileItem) -> bool: + """模拟含目录保护与插件路由的统一删除兼容能力。""" + calls.append("cleanup_compat") + cleaned_items.append(fileitem) + return True + + result = module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=transfer_storage, + target_oper=transfer_storage, + cleanup_media_file=cleanup_media_file, + ) + + assert result.success is True + intercept_index = calls.index(ChainEventType.TransferIntercept.value) + assert intercept_index < calls.index("cleanup_compat") + assert calls.index("cleanup_compat") < calls.index("get_folder") + assert calls.index("cleanup_compat") < next( + index for index, call in enumerate(calls) if call.startswith("copy:") + ) + assert cleaned_items == [cleanup_item] + + +def test_cleanup_compatibility_capability_can_report_idempotent_success(monkeypatch): + calls: list[str] = [] + checkpoint, meta, mediainfo, _ = _build_cleanup_checkpoint(monkeypatch, calls) + transfer_storage = RecordingStorage(calls) + module = FileManagerModule() + + def cleanup_media_file(fileitem: FileItem) -> bool: + """统一能力将目标不存在归一为幂等成功。""" + calls.append("cleanup_compat_missing") + return True + + result = module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=transfer_storage, + target_oper=transfer_storage, + cleanup_media_file=cleanup_media_file, + ) + + assert result.success is True + assert "cleanup_compat_missing" in calls + + +def test_replayed_checkpoint_skips_cleanup_already_completed_before_provider( + monkeypatch, +): + """provider 前已完成 cleanup 的持久检查点在宿主重放时不得再次删除。""" + calls: list[str] = [] + checkpoint, meta, mediainfo, _ = _build_cleanup_checkpoint(monkeypatch, calls) + persisted_checkpoint = TransferPlanCheckpoint.from_payload( + replace( + checkpoint, + pre_execution_cleanup_completed=True, + ).to_payload() + ) + transfer_storage = RecordingStorage(calls) + + def unexpected_cleanup(_fileitem: FileItem) -> bool: + """若持久完成事实未被消费则立即暴露重复清理。""" + raise AssertionError("已完成的 provider 前 cleanup 不应在宿主重放时重复执行") + + result = FileManagerModule().execute_transfer_plan( + persisted_checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=transfer_storage, + target_oper=transfer_storage, + cleanup_media_file=unexpected_cleanup, + ) + + assert result.success is True + assert ChainEventType.TransferIntercept.value in calls + assert any(call.startswith("copy:") for call in calls) + + +def test_cleanup_failure_raises_before_directory_or_copy(monkeypatch): + calls: list[str] = [] + checkpoint, meta, mediainfo, cleanup_item = _build_cleanup_checkpoint( + monkeypatch, + calls, + ) + transfer_storage = RecordingStorage(calls) + module = FileManagerModule() + + def cleanup_media_file(fileitem: FileItem) -> bool: + """模拟统一删除能力执行保护治理后的失败结果。""" + calls.append("cleanup_compat_failed") + return False + + with pytest.raises(RuntimeError, match="整理计划保留待重试"): + module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=transfer_storage, + target_oper=transfer_storage, + cleanup_media_file=cleanup_media_file, + ) + + assert ChainEventType.TransferIntercept.value in calls + assert "cleanup_compat_failed" in calls + assert "get_folder" not in calls + assert not any(call.startswith("copy:") for call in calls) + + +def test_empty_directory_plan_is_zero_operation_without_intercept_or_cleanup( + monkeypatch, +): + calls: list[str] = [] + root = FileItem( + storage="alist", + path="/downloads/empty", + name="empty", + type="dir", + ) + cleanup_item = FileItem( + storage="alist", + path="/library/old.mkv", + name="old.mkv", + type="file", + ) + planning_input = _build_input( + root, + need_rename=False, + options={"cleanup_dest_fileitem": cleanup_item.model_dump(mode="json")}, + ) + meta, mediainfo = _build_media() + checkpoint = TransHandler().plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=ReadOnlyTreeStorage({root.path: []}), + target_storage="alist", + target_path=Path("/library"), + transfer_type="copy", + need_scrape=False, + need_rename=False, + need_notify=True, + overwrite_mode="never", + episodes_info=None, + preview=False, + ) + assert checkpoint.items == () + + def unexpected_event(event_type, event_data): + """空计划若触发事件则立即失败。""" + raise AssertionError(f"空计划不应触发事件：{event_type}") + + monkeypatch.setattr(eventmanager, "send_event", unexpected_event) + result = FileManagerModule().execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=RecordingStorage(calls), + target_oper=RecordingStorage(calls), + cleanup_media_file=lambda _fileitem: calls.append("cleanup") or True, + ) + + assert result.success is True + assert calls == [] + + +def test_directory_intercept_preserves_legacy_payload(monkeypatch): + calls: list[str] = [] + root = FileItem( + storage="alist", + path="/downloads/disc", + name="disc", + type="dir", + ) + child = FileItem( + storage="alist", + path="/downloads/disc/index.bdmv", + name="index.bdmv", + type="file", + extension="bdmv", + ) + meta, mediainfo = _build_media() + checkpoint = TransHandler().plan_transfer( + _build_input(root, need_rename=False), + meta=meta, + mediainfo=mediainfo, + source_oper=ReadOnlyTreeStorage({root.path: [child]}), + target_storage="alist", + target_path=Path("/library"), + transfer_type="copy", + need_scrape=False, + need_rename=False, + need_notify=True, + overwrite_mode="never", + episodes_info=None, + preview=False, + ) + intercept_payloads = [] + + def capture_event(event_type, event_data): + """捕获目录根拦截事件。""" + if event_type == ChainEventType.TransferIntercept: + intercept_payloads.append(event_data) + return None + + monkeypatch.setattr(eventmanager, "send_event", capture_event) + result = TransHandler().execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=RecordingStorage(calls), + target_oper=RecordingStorage(calls), + ) + + assert result.success is True + assert len(intercept_payloads) == 1 + payload = intercept_payloads[0] + assert payload.meta is None + assert payload.options is None + assert "meta" not in payload.model_fields_set + assert "options" not in payload.model_fields_set diff --git a/tests/test_module_invocation_dispatcher.py b/tests/test_module_invocation_dispatcher.py index 8251c7b9b..309c994ca 100644 --- a/tests/test_module_invocation_dispatcher.py +++ b/tests/test_module_invocation_dispatcher.py @@ -7,7 +7,11 @@ from unittest.mock import Mock import pytest -from app.runtime.extensions.module.dispatcher import ModuleInvocationDispatcher +from app.runtime.extensions.module.dispatcher import ( + FrozenModuleProviderMissingError, + FrozenPluginProviderRef, + ModuleInvocationDispatcher, +) class _PluginCatalog: @@ -94,6 +98,253 @@ def test_plugin_scalar_short_circuits_system_modules() -> None: system_call.assert_not_called() +def test_strict_dispatch_propagates_plugin_failure_before_host_fallback() -> None: + """严格查询不得把插件 provider 异常吞成空结果后继续宿主 fallback。""" + system_call = Mock(return_value=None) + module = _Module("系统", 10, system_call) + setattr(module, "get_file_item", module.execute) + + def failed_provider(**_kwargs): + """模拟插件存储查询发生网络或 I/O 故障。""" + raise RuntimeError("provider lookup failed") + + dispatcher, plugin_error, _, _ = _dispatcher( + plugins={("P1", "插件一"): {"get_file_item": failed_provider}}, + modules=[module], + ) + + with pytest.raises(RuntimeError, match="provider lookup failed"): + dispatcher.dispatch_strict( + "get_file_item", + storage="plugin", + path="/library/old.mkv", + ) + + plugin_error.assert_called_once() + system_call.assert_not_called() + + +def test_strict_dispatch_preserves_confirmed_absence() -> None: + """全部 provider 正常返回空值时，严格查询仍以 None 表示确认不存在。""" + module = _Module("系统", 10, lambda **_kwargs: None) + setattr(module, "get_file_item", module.execute) + dispatcher, _, _, _ = _dispatcher( + plugins={("P1", "插件一"): {"get_file_item": lambda **_kwargs: None}}, + modules=[module], + ) + + assert ( + dispatcher.dispatch_strict( + "get_file_item", + storage="plugin", + path="/library/missing.mkv", + ) + is None + ) + + +def test_strict_dispatch_propagates_host_io_failure() -> None: + """严格查询也必须传播宿主存储适配器的 I/O 故障。""" + def failed_host(**_kwargs): + """模拟宿主存储 stat 或远端请求失败。""" + raise RuntimeError("host io failed") + + module = _Module("系统", 10, failed_host) + setattr(module, "get_file_item", module.execute) + dispatcher, _, system_error, _ = _dispatcher(modules=[module]) + + with pytest.raises(RuntimeError, match="host io failed"): + dispatcher.dispatch_strict( + "get_file_item", + storage="local", + path="/library/old.mkv", + ) + + system_error.assert_called_once() + + +def test_frozen_plugin_providers_preserve_original_order_after_catalog_reorder() -> None: + """冻结执行必须采用持久化顺序，不受当前插件目录重排影响。""" + calls = [] + plugins = { + ("P1", "插件一"): {"transfer": lambda: calls.append("P1")}, + ("P2", "插件二"): {"transfer": lambda: calls.append("P2")}, + } + dispatcher, _, _, _ = _dispatcher(plugins=plugins) + + providers = dispatcher.freeze_plugin_providers("transfer") + payloads = [provider.to_payload() for provider in providers] + restored = tuple(FrozenPluginProviderRef.from_payload(item) for item in payloads) + plugins.clear() + plugins.update( + { + ("P2", "插件二"): {"transfer": lambda: calls.append("new-P2")}, + ("P1", "插件一"): {"transfer": lambda: calls.append("new-P1")}, + } + ) + + assert dispatcher.execute_frozen_plugin_providers("transfer", restored) is None + assert payloads == [ + {"plugin_id": "P1", "plugin_name": "插件一", "method": "transfer"}, + {"plugin_id": "P2", "plugin_name": "插件二", "method": "transfer"}, + ] + assert calls == ["new-P1", "new-P2"] + + +def test_frozen_plugin_provider_propagates_failure_and_stops() -> None: + """冻结序列中的插件异常必须向上抛出，不能伪装成空结果。""" + calls = [] + + def fail() -> None: + """记录调用后模拟旧插件执行失败。""" + calls.append("P1") + raise RuntimeError("provider failed") + + dispatcher, plugin_error, _, _ = _dispatcher( + plugins={ + ("P1", "插件一"): {"transfer": fail}, + ("P2", "插件二"): { + "transfer": lambda: calls.append("P2") or "success" + }, + } + ) + providers = dispatcher.freeze_plugin_providers("transfer") + + with pytest.raises(RuntimeError, match="provider failed"): + dispatcher.execute_frozen_plugin_providers("transfer", providers) + + assert calls == ["P1"] + plugin_error.assert_called_once() + assert plugin_error.call_args.args[1:4] == ("P1", "插件一", "transfer") + + +def test_frozen_plugin_provider_first_non_empty_short_circuits() -> None: + """冻结 transfer 序列仍应在首个非空结果后停止。""" + second_provider = Mock(return_value="second") + dispatcher, _, _, _ = _dispatcher( + plugins={ + ("P1", "插件一"): {"transfer": lambda: "first"}, + ("P2", "插件二"): {"transfer": second_provider}, + } + ) + providers = dispatcher.freeze_plugin_providers("transfer") + + assert ( + dispatcher.execute_frozen_plugin_providers("transfer", providers) == "first" + ) + second_provider.assert_not_called() + + +def test_frozen_plugin_provider_missing_fails_before_any_execution() -> None: + """任一冻结 provider 缺失时必须显式失败且不得产生部分执行。""" + first_provider = Mock(return_value=None) + plugins = { + ("P1", "插件一"): {"transfer": first_provider}, + ("P2", "插件二"): {"transfer": Mock(return_value=None)}, + } + dispatcher, _, _, _ = _dispatcher(plugins=plugins) + providers = dispatcher.freeze_plugin_providers("transfer") + plugins.pop(("P2", "插件二")) + + with pytest.raises( + FrozenModuleProviderMissingError, + match=r"P2/插件二\.transfer", + ): + dispatcher.execute_frozen_plugin_providers("transfer", providers) + + first_provider.assert_not_called() + + +def test_frozen_plugin_provider_missing_fails_before_pre_invoke_hook() -> None: + """全部冻结引用解析成功前不得触发 cleanup 等前置副作用。""" + first_provider = Mock(return_value=None) + before_invoke = Mock() + plugins = { + ("P1", "插件一"): {"transfer": first_provider}, + ("P2", "插件二"): {"transfer": Mock(return_value=None)}, + } + dispatcher, _, _, _ = _dispatcher(plugins=plugins) + providers = dispatcher.freeze_plugin_providers("transfer") + plugins.pop(("P2", "插件二")) + + with pytest.raises(FrozenModuleProviderMissingError): + dispatcher.execute_frozen_plugin_providers( + "transfer", + providers, + before_invoke=before_invoke, + ) + + before_invoke.assert_not_called() + first_provider.assert_not_called() + + +def test_empty_frozen_plugin_provider_sequence_skips_pre_invoke_hook() -> None: + """没有冻结 provider 时不得执行仅服务于 provider 的前置副作用。""" + before_invoke = Mock() + dispatcher, _, _, _ = _dispatcher() + + assert ( + dispatcher.execute_frozen_plugin_providers( + "transfer", + (), + before_invoke=before_invoke, + ) + is None + ) + before_invoke.assert_not_called() + + +def test_ordinary_transfer_dispatch_remains_dynamic_and_compatible() -> None: + """普通 transfer 调度仍按当前目录动态发现并采用既有短路语义。""" + system_call = Mock(return_value="system") + module = _Module("系统", 10, system_call) + setattr(module, "transfer", module.execute) + dispatcher, _, _, _ = _dispatcher( + plugins={("P1", "插件一"): {"transfer": lambda: "plugin"}}, + modules=[module], + ) + + assert dispatcher.dispatch("transfer") == "plugin" + system_call.assert_not_called() + + +def test_host_internal_contract_skips_plugin_provider() -> None: + """宿主内部两阶段协议不得暴露给同名第三方 provider。""" + plugin_call = Mock(return_value="plugin") + system_call = Mock(return_value="system") + module = _Module("系统", 10, system_call) + setattr(module, "plan_transfer", module.execute) + dispatcher, _, _, _ = _dispatcher( + plugins={ + ("P1", "插件一"): {"plan_transfer": plugin_call}, + }, + modules=[module], + ) + + assert dispatcher.dispatch("plan_transfer") == "system" + plugin_call.assert_not_called() + system_call.assert_called_once() + + +@pytest.mark.asyncio +async def test_async_host_internal_contract_skips_plugin_provider() -> None: + """异步调度同样只允许宿主执行内部检查点协议。""" + plugin_call = Mock(return_value="plugin") + system_call = Mock(return_value="system") + module = _Module("系统", 10, system_call) + setattr(module, "execute_transfer_plan", module.execute) + dispatcher, _, _, _ = _dispatcher( + plugins={ + ("P1", "插件一"): {"execute_transfer_plan": plugin_call}, + }, + modules=[module], + ) + + assert await dispatcher.async_dispatch("execute_transfer_plan") == "system" + plugin_call.assert_not_called() + system_call.assert_called_once() + + def test_fan_out_contract_runs_every_provider_and_ignores_results() -> None: """副作用广播应执行全部插件和宿主 provider，并稳定返回 None。""" calls = [] diff --git a/tests/test_module_method_contracts.py b/tests/test_module_method_contracts.py index 732a46fc4..53dda23be 100644 --- a/tests/test_module_method_contracts.py +++ b/tests/test_module_method_contracts.py @@ -69,14 +69,15 @@ def test_contract_v2_freezes_every_observed_host_method() -> None: contracts = list_explicit_module_contracts() assert len(contracts) >= 211 - for contract in contracts.values(): + host_internal_methods = {"plan_transfer", "execute_transfer_plan"} + for method_name, contract in contracts.items(): assert contract.version == 1 assert contract.input_contract != "legacy_args" assert contract.result_contract assert contract.execution is ModuleExecutionMode.SYNC_OR_ASYNC assert contract.timeout_policy == "caller_budget" assert contract.error_policy is ModuleErrorPolicy.ISOLATE_PROVIDER - assert contract.public_to_plugins is True + assert contract.public_to_plugins is (method_name not in host_internal_methods) def test_signature_diagnostics_do_not_reject_legacy_callable() -> None: diff --git a/tests/test_music_transfer.py b/tests/test_music_transfer.py index 521372351..98fafde71 100644 --- a/tests/test_music_transfer.py +++ b/tests/test_music_transfer.py @@ -4,16 +4,16 @@ from unittest.mock import Mock from jinja2 import Template +from app.application.messaging.message import TemplateHelper +from app.application.transfer import TransferTask from app.chain.media import MediaChain from app.chain.transfer import JobManager, TransferChain -from app.runtime.config import settings -from app.domain.meta.metamusic import MetaMusic from app.domain.context import MusicInfo -from app.application.messaging.message import TemplateHelper +from app.domain.meta.metamusic import MetaMusic +from app.runtime.config import settings from app.schemas.file import FileItem from app.schemas.system import TransferDirectoryConf from app.schemas.transfer import TransferInfo, TransferTorrent -from app.application.transfer import TransferTask from app.schemas.types import EventType, MediaType @@ -601,7 +601,7 @@ def test_automatic_audio_transfer_runs_music_recognition(tmp_path, monkeypatch): monkeypatch.setattr(chain, "_resolve_download_history", Mock(return_value=None)) monkeypatch.setattr( chain, - "transfer", + "_plan_checkpoint_and_execute", Mock( return_value=TransferInfo( success=True, @@ -704,7 +704,7 @@ def test_explicit_music_batch_excludes_video_from_mixed_directory(tmp_path, monk monkeypatch.setattr(MediaChain, "recognize_by_meta", Mock(return_value=recognized)) monkeypatch.setattr( chain, - "transfer", + "_plan_checkpoint_and_execute", Mock( return_value=TransferInfo( success=True, @@ -727,7 +727,7 @@ def test_explicit_music_batch_excludes_video_from_mixed_directory(tmp_path, monk assert state is True assert [item["source"] for item in preview["items"]] == [audio_item.path] - assert chain.transfer.call_count == 1 + assert chain._plan_checkpoint_and_execute.call_count == 1 def test_downloader_process_forwards_music_history_type(tmp_path, monkeypatch): diff --git a/tests/test_storage_chain_strict.py b/tests/test_storage_chain_strict.py new file mode 100644 index 000000000..83a1539e3 --- /dev/null +++ b/tests/test_storage_chain_strict.py @@ -0,0 +1,43 @@ +"""StorageChain 严格查询的三态传播合同。""" + +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from app.chain.storage import StorageChain +from app.modules.filemanager.module import FileManagerModule +from app.schemas.exception import StorageQueryError + + +def test_storage_chain_strict_query_distinguishes_absent_and_failure() -> None: + """严格查询用 None 表示确认不存在，并原样传播 provider 查询失败。""" + chain = object.__new__(StorageChain) + chain._module_dispatcher = Mock() + chain._module_dispatcher.dispatch_strict.return_value = None + + assert chain.get_file_item_strict("plugin", Path("/missing.mkv")) is None + + chain._module_dispatcher.dispatch_strict.side_effect = StorageQueryError( + "provider lookup failed" + ) + with pytest.raises(StorageQueryError, match="provider lookup failed"): + chain.get_file_item_strict("plugin", Path("/unknown.mkv")) + + +def test_filemanager_storage_query_uses_strict_storage_adapter(monkeypatch) -> None: + """宿主存储 provider 必须调用 get_item_strict，不能回退会吞错的 get_item。""" + module = object.__new__(FileManagerModule) + module._support_storages = ["local"] + storage = Mock() + storage.get_item_strict.side_effect = StorageQueryError("local stat failed") + monkeypatch.setattr( + module, + "_FileManagerModule__get_storage_oper", + lambda _storage: storage, + ) + + with pytest.raises(StorageQueryError, match="local stat failed"): + module.get_file_item("local", Path("/library/old.mkv")) + + storage.get_item.assert_not_called() diff --git a/tests/test_transfer_admission_migration.py b/tests/test_transfer_admission_migration.py index 060ba02ec..d7668cb35 100644 --- a/tests/test_transfer_admission_migration.py +++ b/tests/test_transfer_admission_migration.py @@ -9,8 +9,6 @@ import sqlalchemy as sa from alembic.migration import MigrationContext from alembic.operations import Operations -from app.db.models.transferpending import TransferPending - try: import psycopg2 as postgres_driver from psycopg2 import sql @@ -23,6 +21,16 @@ except ModuleNotFoundError: POSTGRESQL_DIALECT = "postgresql+psycopg" MIGRATION = "database.versions.b1e7d3f5a9c2_3_0_13" +ADMISSION_COLUMNS = { + "id", + "task_id", + "storage", + "src_path", + "created_at", + "state", + "updated_at", + "last_error", +} def _bind_migration(monkeypatch, connection): @@ -105,7 +113,7 @@ def test_transfer_admission_upgrade_downgrade_reupgrade( assert { column["name"] for column in inspector.get_columns("transferpending") - } == {column.name for column in TransferPending.__table__.columns} + } == ADMISSION_COLUMNS constraints = { constraint["name"] for constraint in inspector.get_unique_constraints("transferpending") diff --git a/tests/test_transfer_job_manager.py b/tests/test_transfer_job_manager.py index 2c99a7741..c6fd2b1db 100644 --- a/tests/test_transfer_job_manager.py +++ b/tests/test_transfer_job_manager.py @@ -1,21 +1,21 @@ import unittest from pathlib import Path from types import SimpleNamespace -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch -from app.runtime.config import settings -from app.domain.context import MediaInfo -from app.domain.meta.metabase import MetaBase -from app.domain.meta.metavideo import MetaVideo -from app.chain.transfer import JobManager, TransferChain from app.application.history import ( clear_transfer_failures, failed_retry_count, record_transfer_failure, ) +from app.application.transfer import TransferPlanningInput, TransferTask +from app.chain.transfer import JobManager, TransferChain +from app.domain.context import MediaInfo +from app.domain.meta.metabase import MetaBase +from app.domain.meta.metavideo import MetaVideo from app.modules.filemanager.transhandler import TransHandler +from app.runtime.config import settings from app.schemas import EpisodeFormat, FileItem, TransferInfo -from app.application.transfer import TransferTask from app.schemas.types import EventType, MediaSource, MediaType @@ -179,6 +179,58 @@ def make_fileitem(path: str, size: int = 1024) -> FileItem: ) +def execute_transfer_plan( + handler: TransHandler, + *, + fileitem: FileItem, + meta: MetaBase, + mediainfo: MediaInfo, + target_storage: str, + target_path: Path, + transfer_type: str, + source_oper: object, + target_oper: object, + need_scrape: bool = False, + need_notify: bool = True, +) -> TransferInfo: + """在测试中显式规划并执行单文件整理。""" + planning_input = TransferPlanningInput( + source_fileitem=fileitem.model_dump(mode="json"), + meta=meta.to_dict(), + mediainfo=mediainfo.to_dict(), + target_storage=target_storage, + target_path=target_path.as_posix(), + requested_transfer_type=transfer_type, + media_type=mediainfo.type.value if mediainfo.type else None, + need_scrape=need_scrape, + need_rename=True, + need_notify=need_notify, + preview=False, + ) + checkpoint = handler.plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + need_scrape=need_scrape, + need_rename=True, + need_notify=need_notify, + overwrite_mode=None, + episodes_info=None, + preview=False, + ) + return handler.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_oper=target_oper, + ) + + def migrate_to_media_job(jobview: JobManager, task: TransferTask): task.mediainfo = FakeMedia() jobview.migrate_task(task) @@ -238,7 +290,7 @@ class TransferJobManagerTest(unittest.TestCase): self.assertEqual("file", new_item.type) self.assertEqual(1024, new_item.size) - def test_transfer_media_uses_target_folder_returned_by_storage(self): + def test_transfer_plan_uses_target_folder_returned_by_storage(self): """ 整理成功时直接使用存储层返回的目标目录项，回调和事件不再二次拼装。 """ @@ -294,9 +346,10 @@ class TransferJobManagerTest(unittest.TestCase): return_value=(target_item, ""), ), patch("app.modules.filemanager.transhandler.eventmanager") as eventmanager_mock: eventmanager_mock.send_event.return_value = None - transferinfo = handler.transfer_media( + transferinfo = execute_transfer_plan( + handler, fileitem=source_item, - in_meta=MetaVideo("Test.Show.S01E01"), + meta=MetaVideo("Test.Show.S01E01"), mediainfo=make_media_info(), target_storage="alist", target_path=target_path, @@ -368,9 +421,10 @@ class TransferJobManagerTest(unittest.TestCase): "app.modules.filemanager.transhandler.eventmanager.send_event", return_value=None, ) as send_event: - transferinfo = handler.transfer_media( + transferinfo = execute_transfer_plan( + handler, fileitem=source_item, - in_meta=in_meta, + meta=in_meta, mediainfo=make_media_info(), target_storage="alist", target_path=target_path, diff --git a/tests/test_transfer_pending_replay.py b/tests/test_transfer_pending_replay.py index 0db82e83d..5664429ca 100644 --- a/tests/test_transfer_pending_replay.py +++ b/tests/test_transfer_pending_replay.py @@ -11,7 +11,7 @@ import threading from pathlib import Path from unittest.mock import MagicMock -from app.application.transfer import TransferAdmission, TransferTask +from app.application.transfer import TransferAdmission, TransferPlanningInput, TransferTask from app.chain.transfer import TransferChain from app.schemas.file import FileItem @@ -71,9 +71,11 @@ def test_admit_transfer_records_storage_and_path(): _task("/mnt/cd2/downloads/Movie.2024.mkv") ) - admissions.admit.assert_called_once_with( - storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv" - ) + call = admissions.admit.call_args.kwargs + assert call["storage"] == "local" + assert call["src_path"] == "/mnt/cd2/downloads/Movie.2024.mkv" + assert isinstance(call["planning_input"], TransferPlanningInput) + assert call["planning_input"].source_fileitem["path"] == call["src_path"] assert result.task_id == "task-1" @@ -99,11 +101,15 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch): media.write_bytes(b"x" * 10) admissions = MagicMock() - admissions.list_accepted.return_value = [_admission(str(media))] + admissions.list_recoverable.return_value = [_admission(str(media))] chain = _build_chain(admissions) transferred = [] - monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"])) + monkeypatch.setattr( + chain, + "_execute_transfer", + lambda **kw: transferred.append(kw["fileitem"]), + ) chain._TransferChain__replay_pending() @@ -122,13 +128,13 @@ def test_replay_discards_vanished_files(tmp_path): """ admissions = MagicMock() missing = tmp_path / "gone.mkv" - admissions.list_accepted.return_value = [_admission(str(missing))] + admissions.list_recoverable.return_value = [_admission(str(missing))] chain = _build_chain(admissions) - chain.do_transfer = MagicMock() + chain._execute_transfer = MagicMock() chain._TransferChain__replay_pending() - chain.do_transfer.assert_not_called() + chain._execute_transfer.assert_not_called() admissions.discard_task.assert_called_once_with(task_id="task-1") @@ -142,9 +148,9 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch): media.write_bytes(b"x") admissions = MagicMock() - admissions.list_accepted.return_value = [_admission(str(media))] + admissions.list_recoverable.return_value = [_admission(str(media))] chain = _build_chain(admissions) - chain.do_transfer = MagicMock() + chain._execute_transfer = MagicMock() def unreadable(self, *_args, **_kwargs): """ @@ -156,7 +162,7 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch): chain._TransferChain__replay_pending() - chain.do_transfer.assert_not_called() + chain._execute_transfer.assert_not_called() admissions.discard_task.assert_not_called() @@ -169,11 +175,15 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch): src_path = f"{bluray.as_posix()}/" admissions = MagicMock() - admissions.list_accepted.return_value = [_admission(src_path)] + admissions.list_recoverable.return_value = [_admission(src_path)] chain = _build_chain(admissions) transferred = [] - monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"])) + monkeypatch.setattr( + chain, + "_execute_transfer", + lambda **kw: transferred.append(kw["fileitem"]), + ) chain._TransferChain__replay_pending() @@ -187,13 +197,13 @@ def test_replay_is_noop_without_registrations(): 没有登记时回放不应触碰整理链。 """ admissions = MagicMock() - admissions.list_accepted.return_value = [] + admissions.list_recoverable.return_value = [] chain = _build_chain(admissions) - chain.do_transfer = MagicMock() + chain._execute_transfer = MagicMock() chain._TransferChain__replay_pending() - chain.do_transfer.assert_not_called() + chain._execute_transfer.assert_not_called() def test_replay_survives_db_failure(): @@ -201,13 +211,13 @@ def test_replay_survives_db_failure(): 读取登记失败不能让启动流程报错。 """ admissions = MagicMock() - admissions.list_accepted.side_effect = RuntimeError("db gone") + admissions.list_recoverable.side_effect = RuntimeError("db gone") chain = _build_chain(admissions) - chain.do_transfer = MagicMock() + chain._execute_transfer = MagicMock() chain._TransferChain__replay_pending() - chain.do_transfer.assert_not_called() + chain._execute_transfer.assert_not_called() def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch): @@ -220,7 +230,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch): item.write_bytes(b"x") admissions = MagicMock() - admissions.list_accepted.return_value = [ + admissions.list_recoverable.return_value = [ _admission(str(first), "task-1"), _admission(str(second), "task-2"), ] @@ -236,7 +246,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch): raise RuntimeError("boom") handled.append(kw["fileitem"].name) - monkeypatch.setattr(chain, "do_transfer", flaky) + monkeypatch.setattr(chain, "_execute_transfer", flaky) chain._TransferChain__replay_pending() @@ -251,7 +261,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch): first.write_bytes(b"x") missing_second = tmp_path / "gone.mkv" admissions = MagicMock() - admissions.list_accepted.return_value = [ + admissions.list_recoverable.return_value = [ _admission(str(first), "task-1"), _admission(str(missing_second), "task-2"), ] @@ -264,7 +274,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch): transferred.append(kwargs["fileitem"].path) stop_event.set() - monkeypatch.setattr(chain, "do_transfer", transfer_first) + monkeypatch.setattr(chain, "_execute_transfer", transfer_first) chain._TransferChain__replay_pending(stop_event) diff --git a/tests/test_transfer_planning_checkpoint.py b/tests/test_transfer_planning_checkpoint.py new file mode 100644 index 000000000..d89eba76a --- /dev/null +++ b/tests/test_transfer_planning_checkpoint.py @@ -0,0 +1,1563 @@ +"""整理规划检查点的事务顺序、恢复与无副作用边界合同。""" + +import ast +import inspect +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.application import transfer as transfer_application +from app.application.transfer import TransferTask +from app.chain.transfer import TransferChain +from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferpending import TransferPending +from app.domain.context import MediaInfo +from app.domain.meta.metabase import MetaBase +from app.modules.filemanager.module import FileManagerModule +from app.modules.filemanager.transhandler import TransHandler +from app.runtime.extensions.module.dispatcher import ( + FrozenModuleProviderMissingError, + ModuleInvocationDispatcher, +) +from app.schemas.exception import StorageQueryError +from app.schemas.file import FileItem +from app.schemas.system import TransferDirectoryConf +from app.schemas.transfer import TransferInfo + + +def _planning_contracts(): + """延迟解析新合同，使未落地时每个 pytest 用例给出精确缺口。""" + return ( + getattr(transfer_application, "TransferPlanningInput"), + getattr(transfer_application, "TransferPlanItem"), + getattr(transfer_application, "TransferPlanCheckpoint"), + ) + + +def _source_snapshot() -> dict[str, object]: + """返回可独立跨重启还原的源文件快照。""" + return { + "storage": "local", + "path": "/downloads/Movie.2026.mkv", + "type": "file", + "name": "Movie.2026.mkv", + "basename": "Movie.2026", + "extension": "mkv", + "size": 1024, + "modify_time": 1770000000, + "fileid": "source-1", + } + + +def _planning_input(*, preview: bool = False, target_path: str = "/library"): + """构造包含旧调用意图、媒体提示和目标配置的规划输入。""" + TransferPlanningInput, _TransferPlanItem, _TransferPlanCheckpoint = ( + _planning_contracts() + ) + return TransferPlanningInput( + source_fileitem=_source_snapshot(), + meta={"title": "Movie.2026", "year": "2026"}, + mediainfo={ + "media_source": "themoviedb", + "media_id": "123", + "type": "电影", + }, + target_directory={"library_path": target_path}, + target_storage="local", + target_path=target_path, + requested_transfer_type="copy", + media_source="themoviedb", + media_id="123", + media_type="电影", + need_scrape=True, + need_rename=True, + need_notify=True, + overwrite_mode="always", + episodes_info=(), + preview=preview, + options={ + "manual": True, + "background": False, + "cleanup_dest_fileitem": None, + }, + ) + + +def _checkpoint(*, preview: bool = False, target_path: str = "/library/Movie (2026)/Movie.mkv"): + """构造带完整输入和有序执行项的冻结计划。""" + _TransferPlanningInput, TransferPlanItem, TransferPlanCheckpoint = ( + _planning_contracts() + ) + planning_input = _planning_input(preview=preview) + item = TransferPlanItem( + sequence=0, + source_fileitem=_source_snapshot(), + target_storage="local", + target_path=target_path, + action="transfer", + ) + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="local", + root_target_path="/library", + final_target_path=target_path, + resolved_transfer_type="copy", + items=(item,), + resolved_meta={ + "org_string": "Movie.2026.mkv", + "title": "Movie 2026", + "type": "电影", + }, + resolved_meta_kind="MetaVideo", + resolved_mediainfo={ + "title": "Movie", + "year": "2026", + "type": "电影", + "media_source": "themoviedb", + "media_id": "123", + }, + resolved_mediainfo_kind="MediaInfo", + need_scrape=True, + need_rename=True, + need_notify=True, + overwrite_mode="always", + preview=preview, + ) + + +def _resolved_checkpoint(): + """构造包含在线识别完成后领域快照的 planned checkpoint。""" + return replace( + _checkpoint(), + resolved_meta={ + "org_string": "Frozen.Show.S01E01.mkv", + "title": "Frozen Show S01E01", + "name": "Frozen Show", + "year": "2026", + "type": "电视剧", + "begin_season": 1, + "begin_episode": 1, + "episode_list": [1], + }, + resolved_mediainfo={ + "title": "Frozen Show", + "year": "2026", + "type": "电视剧", + "media_source": "themoviedb", + "media_id": "987", + "tmdb_id": 987, + }, + resolved_episodes_info=( + { + "episode_number": 1, + "season_number": 1, + "name": "Frozen Episode", + }, + ), + ) + + +def _task(*, preview: bool = False) -> TransferTask: + """构造不触碰真实文件系统的规划任务。""" + return TransferTask( + fileitem=FileItem(**_source_snapshot()), + target_storage="local", + target_path=Path("/library"), + transfer_type="copy", + scrape=True, + manual=True, + background=False, + preview=preview, + ) + + +def _bind_planning_input(task: TransferTask, planning_input) -> None: + """通过正式绑定接口保存私有输入，禁止测试写入 Pydantic 内部字典。""" + binder = getattr(task, "bind_planning_input") + binder(planning_input) + + +def _bind_checkpoint(task: TransferTask, checkpoint) -> None: + """通过正式绑定接口模拟 planned 任务的跨重启恢复。""" + binder = getattr(task, "bind_plan_checkpoint") + binder(checkpoint) + + +def _planned_admission(task: TransferTask, checkpoint): + """构造仓储 checkpoint 提交后返回的冻结投影。""" + return SimpleNamespace( + task_id=task.admission_task_id, + state="planned", + planning_input=task.planning_input, + checkpoint=checkpoint, + ) + + +def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain: + """构造只保留规划编排依赖的 TransferChain 骨架。""" + chain = object.__new__(TransferChain) + chain._transfer_admissions = repository or Mock() + chain._module_dispatcher = Mock() + chain._module_dispatcher.freeze_plugin_providers.return_value = () + chain.eventmanager = Mock() + chain.eventmanager.send_event.return_value = None + chain.plan_transfer = Mock(return_value=checkpoint or _checkpoint()) + chain.execute_transfer_plan = Mock( + return_value=result or TransferInfo( + success=True, + fileitem=_task().fileitem, + transfer_type="copy", + ) + ) + return chain + + +def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher: + """构造使用真实冻结解析与执行内核的内存插件调度器。""" + plugin_catalog = Mock() + plugin_catalog.get_plugin_modules.return_value = plugins + return ModuleInvocationDispatcher( + module_catalog=Mock(), + plugin_catalog=plugin_catalog, + plugin_error_handler=Mock(), + system_error_handler=Mock(), + rate_limit_handler=Mock(), + ) + + +def test_legacy_provider_runs_only_after_checkpoint_commit_and_short_circuits_host(): + """旧插件 provider 必须随计划冻结，并在 CAS 提交后才能接管执行。""" + calls = [] + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.bind_admission_task_id("task-plugin-short-circuit") + _bind_planning_input(task, _planning_input()) + checkpoint = _checkpoint() + provider = SimpleNamespace( + plugin_id="LegacyTransfer", + plugin_name="旧整理插件", + method="transfer", + ) + plugin_result = TransferInfo( + success=True, + fileitem=task.fileitem, + transfer_type="copy", + ) + repository = Mock() + + def checkpoint_plan(**kwargs): + """记录 CAS 顺序并回读包含 provider 路由的持久检查点。""" + calls.append("checkpoint") + persisted_checkpoint = kwargs["checkpoint"] + assert persisted_checkpoint.legacy_transfer_providers[0].plugin_id == ( + "LegacyTransfer" + ) + return _planned_admission(task, persisted_checkpoint) + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository, checkpoint=checkpoint) + chain._module_dispatcher.freeze_plugin_providers.return_value = (provider,) + chain._module_dispatcher.execute_frozen_plugin_providers.side_effect = ( + lambda *_args, **_kwargs: calls.append("plugin") or plugin_result + ) + + returned = chain._plan_checkpoint_and_execute(task) + + assert returned is plugin_result + assert calls == ["checkpoint", "plugin"] + chain.plan_transfer.assert_not_called() + chain.execute_transfer_plan.assert_not_called() + frozen_call = chain._module_dispatcher.execute_frozen_plugin_providers.call_args + assert frozen_call.args[0] == "transfer" + assert frozen_call.args[1][0].plugin_id == "LegacyTransfer" + assert frozen_call.kwargs["target_path"] == Path("/library") + assert frozen_call.kwargs["preview"] is False + + +def test_legacy_provider_cleanup_runs_before_provider_without_duplicate_intercept(): + """旧 provider 前只做 cleanup，不重复发送插件自行负责的真实目标拦截事件。""" + calls = [] + cleanup_item = FileItem( + storage="plugin-storage", + path="/old-library/old.mkv", + name="old.mkv", + type="file", + extension="mkv", + ) + resolved = _resolved_checkpoint() + planning_input = replace( + resolved.planning_input, + options={ + **resolved.planning_input.options, + "cleanup_dest_fileitem": cleanup_item.model_dump(mode="json"), + }, + ) + invocation = transfer_application.TransferProviderInvocationSnapshot( + fileitem=_source_snapshot(), + meta=resolved.resolved_meta, + meta_kind=resolved.resolved_meta_kind, + mediainfo=resolved.resolved_mediainfo, + mediainfo_kind=resolved.resolved_mediainfo_kind, + target_directory={ + "library_storage": "local", + "library_path": "/library", + "transfer_type": "copy", + }, + target_storage=None, + target_path=None, + transfer_type=None, + scrape=True, + library_type_folder=False, + library_category_folder=False, + episodes_info=resolved.resolved_episodes_info, + preview=False, + ) + provider = transfer_application.TransferProviderReference( + plugin_id="LegacyTransfer", + plugin_name="旧整理插件", + ) + checkpoint = transfer_application.TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="", + root_target_path="", + final_target_path="", + resolved_transfer_type="", + items=(), + legacy_transfer_providers=(provider,), + provider_invocation=invocation, + need_notify=True, + ) + task = _task() + task.meta = MetaBase("Drifted.mkv") + task.mediainfo = MediaInfo(title="Drifted") + chain = object.__new__(TransferChain) + chain.eventmanager = Mock() + storage_chain = Mock() + storage_chain.get_file_item_strict.side_effect = ( + lambda **_kwargs: calls.append("lookup") or cleanup_item + ) + storage_chain.delete_media_file.side_effect = ( + lambda _fileitem: calls.append("cleanup") or True + ) + chain._transfer_storage_chain = Mock(return_value=storage_chain) + provider_result = TransferInfo( + success=True, + fileitem=task.fileitem, + transfer_type="copy", + ) + + def execute_provider(*_args, **kwargs): + """记录 provider 文件副作用入口并核对旧 ABI 的原始 None 语义。""" + calls.append("provider") + assert kwargs["target_storage"] is None + assert kwargs["target_path"] is None + assert kwargs["transfer_type"] is None + return provider_result + + chain._module_dispatcher = _real_dispatcher( + { + ("LegacyTransfer", "旧整理插件"): { + "transfer": execute_provider, + } + } + ) + + returned = chain._TransferChain__execute_legacy_transfer_providers( + task, + checkpoint=checkpoint, + source_oper=None, + target_oper=None, + ) + + assert returned is provider_result + assert calls == ["lookup", "cleanup", "provider"] + chain.eventmanager.send_event.assert_not_called() + + +def test_empty_frozen_legacy_providers_fall_back_to_host_checkpoint_executor(): + """provider 空结果在已完成 cleanup 后持久提升宿主计划且只清理一次。""" + calls = [] + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.bind_admission_task_id("task-plugin-fallback") + cleanup_item = FileItem( + storage="local", + path="/old-library/old.mkv", + name="old.mkv", + type="file", + extension="mkv", + ) + planning_input = replace( + _planning_input(), + options={ + **_planning_input().options, + "cleanup_dest_fileitem": cleanup_item.model_dump(mode="json"), + }, + ) + _bind_planning_input(task, planning_input) + checkpoint = _checkpoint() + provider = SimpleNamespace( + plugin_id="EmptyTransfer", + plugin_name="空整理插件", + method="transfer", + ) + repository = Mock() + + def checkpoint_plan(**kwargs): + """记录宿主 fallback 前的 CAS 提交。""" + calls.append("checkpoint") + if not kwargs["checkpoint"].is_provider_pending: + assert kwargs["checkpoint"].pre_execution_cleanup_completed is True + return _planned_admission(task, kwargs["checkpoint"]) + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository, checkpoint=checkpoint) + chain._module_dispatcher = _real_dispatcher( + { + (provider.plugin_id, provider.plugin_name): { + "transfer": lambda **_kwargs: calls.append("plugin") + } + } + ) + storage_chain = Mock() + storage_chain.get_file_item_strict.side_effect = ( + lambda **_kwargs: calls.append("lookup") or cleanup_item + ) + storage_chain.delete_media_file.side_effect = ( + lambda _fileitem: calls.append("cleanup") or True + ) + chain._transfer_storage_chain = Mock(return_value=storage_chain) + chain.execute_transfer_plan.side_effect = ( + lambda *_args, **_kwargs: calls.append("host") + or TransferInfo( + success=True, + fileitem=task.fileitem, + transfer_type="copy", + ) + ) + + returned = chain._plan_checkpoint_and_execute(task) + + assert returned.success is True + assert calls == [ + "checkpoint", + "lookup", + "cleanup", + "plugin", + "checkpoint", + "host", + ] + chain.execute_transfer_plan.assert_called_once() + executed_checkpoint = chain.execute_transfer_plan.call_args.args[0] + assert executed_checkpoint.pre_execution_cleanup_completed is True + storage_chain.get_file_item_strict.assert_called_once() + storage_chain.delete_media_file.assert_called_once_with(cleanup_item) + + +def test_missing_frozen_provider_keeps_pending_and_skips_cleanup() -> None: + """提交后任一冻结引用缺失时应保留 pending 且不得提前 cleanup。""" + calls = [] + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.bind_admission_task_id("task-provider-missing") + cleanup_item = FileItem( + storage="local", + path="/old-library/old.mkv", + name="old.mkv", + type="file", + extension="mkv", + ) + planning_input = replace( + _planning_input(), + options={ + **_planning_input().options, + "cleanup_dest_fileitem": cleanup_item.model_dump(mode="json"), + }, + ) + _bind_planning_input(task, planning_input) + first_provider = Mock(return_value=None) + plugins = { + ("ProviderOne", "插件一"): {"transfer": first_provider}, + ("ProviderTwo", "插件二"): {"transfer": Mock(return_value=None)}, + } + repository = Mock() + + def checkpoint_plan(**kwargs): + """模拟提交后第二个 provider 被卸载的崩溃恢复窗口。""" + calls.append("checkpoint") + plugins.pop(("ProviderTwo", "插件二")) + return _planned_admission(task, kwargs["checkpoint"]) + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository) + chain._module_dispatcher = _real_dispatcher(plugins) + storage_chain = Mock() + chain._transfer_storage_chain = Mock(return_value=storage_chain) + + with pytest.raises( + FrozenModuleProviderMissingError, + match=r"ProviderTwo/插件二\.transfer", + ): + chain._plan_checkpoint_and_execute(task) + + assert calls == ["checkpoint"] + assert task.plan_checkpoint is not None + assert task.plan_checkpoint.is_provider_pending is True + repository.checkpoint_plan.assert_called_once() + repository.record_planning_failure.assert_not_called() + storage_chain.get_file_item_strict.assert_not_called() + storage_chain.delete_media_file.assert_not_called() + first_provider.assert_not_called() + chain.plan_transfer.assert_not_called() + chain.execute_transfer_plan.assert_not_called() + + +def test_frozen_provider_failure_keeps_pending_without_host_fallback() -> None: + """真实冻结调度器必须传播 provider 异常并保留可重放检查点。""" + calls = [] + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.bind_admission_task_id("task-provider-failed") + cleanup_item = FileItem( + storage="local", + path="/old-library/old.mkv", + name="old.mkv", + type="file", + extension="mkv", + ) + planning_input = replace( + _planning_input(), + options={ + **_planning_input().options, + "cleanup_dest_fileitem": cleanup_item.model_dump(mode="json"), + }, + ) + _bind_planning_input(task, planning_input) + repository = Mock() + + def checkpoint_plan(**kwargs): + """记录 provider_pending 已在插件副作用前完成提交。""" + calls.append("checkpoint") + return _planned_admission(task, kwargs["checkpoint"]) + + def failed_provider(**_kwargs): + """模拟冻结旧插件在 cleanup 完成后的执行失败。""" + calls.append("provider") + raise RuntimeError("legacy provider failed") + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository) + chain._module_dispatcher = _real_dispatcher( + { + ("FailedProvider", "失败插件"): { + "transfer": failed_provider, + } + } + ) + storage_chain = Mock() + storage_chain.get_file_item_strict.side_effect = ( + lambda **_kwargs: calls.append("lookup") or cleanup_item + ) + storage_chain.delete_media_file.side_effect = ( + lambda _fileitem: calls.append("cleanup") or True + ) + chain._transfer_storage_chain = Mock(return_value=storage_chain) + + with pytest.raises(RuntimeError, match="legacy provider failed"): + chain._plan_checkpoint_and_execute(task) + + assert calls == ["checkpoint", "lookup", "cleanup", "provider"] + assert task.plan_checkpoint is not None + assert task.plan_checkpoint.is_provider_pending is True + repository.checkpoint_plan.assert_called_once() + repository.record_planning_failure.assert_not_called() + chain.plan_transfer.assert_not_called() + chain.execute_transfer_plan.assert_not_called() + + +def test_auto_directory_provider_empty_preserves_legacy_none_values_for_fallback(): + """自动目录 provider 与空结果后的宿主规划都必须保留旧 ABI 的可空原值。""" + calls = [] + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.target_directory = TransferDirectoryConf( + name="auto-library", + transfer_type="copy", + library_path="/library/Movies", + library_storage="alist", + renaming=True, + scraping=True, + notify=True, + ) + task.target_storage = "alist" + task.target_path = None + task.transfer_type = None + task.scrape = None + task.library_type_folder = False + task.library_category_folder = None + task.bind_admission_task_id("task-auto-provider-fallback") + planning_input = replace( + _planning_input(), + target_directory=None, + target_storage=None, + target_path=None, + requested_transfer_type=None, + ) + _bind_planning_input(task, planning_input) + host_checkpoint = replace(_checkpoint(), planning_input=planning_input) + provider = SimpleNamespace( + plugin_id="AutoDirectoryProvider", + plugin_name="自动目录插件", + method="transfer", + ) + repository = Mock() + + def checkpoint_plan(**kwargs): + """回读每个阶段检查点并记录 provider_pending 到 planned 的顺序。""" + checkpoint = kwargs["checkpoint"] + calls.append( + "commit-provider" if checkpoint.is_provider_pending else "commit-host" + ) + return _planned_admission(task, checkpoint) + + def execute_provider(*_args, **kwargs): + """核对自动目录只冻结解析目录，不改写旧 ABI 的可空参数。""" + calls.append("provider") + assert kwargs["target_directory"].name == "auto-library" + assert kwargs["target_storage"] == "alist" + assert kwargs["target_path"] is None + assert kwargs["transfer_type"] is None + assert kwargs["scrape"] is None + assert kwargs["library_type_folder"] is False + assert kwargs["library_category_folder"] is None + return None + + def plan_host(**kwargs): + """核对 provider 重放恢复未用空宿主字段污染 fallback 参数。""" + calls.append("plan-host") + assert kwargs["target_directory"].name == "auto-library" + assert kwargs["target_storage"] == "alist" + assert kwargs["target_path"] is None + assert kwargs["transfer_type"] is None + assert kwargs["scrape"] is None + assert kwargs["library_type_folder"] is False + assert kwargs["library_category_folder"] is None + return host_checkpoint + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository, checkpoint=host_checkpoint) + chain._module_dispatcher.freeze_plugin_providers.return_value = (provider,) + chain._module_dispatcher.execute_frozen_plugin_providers.side_effect = ( + execute_provider + ) + chain.plan_transfer.side_effect = plan_host + chain.execute_transfer_plan.side_effect = ( + lambda *_args, **_kwargs: calls.append("execute-host") + or TransferInfo( + success=True, + fileitem=task.fileitem, + transfer_type="copy", + ) + ) + + returned = chain._plan_checkpoint_and_execute(task) + + assert returned.success is True + assert calls == [ + "commit-provider", + "provider", + "plan-host", + "commit-host", + "execute-host", + ] + promoted = repository.checkpoint_plan.call_args_list[1].kwargs["checkpoint"] + assert promoted.pre_execution_cleanup_completed is True + + +def test_provider_empty_fallback_cas_failure_blocks_host_execution(): + """provider 空结果后的 host checkpoint 升级失败时不得执行宿主副作用。""" + task = _task() + task.meta = MetaBase("Movie.2026.mkv") + task.mediainfo = MediaInfo(title="Movie") + task.bind_admission_task_id("task-provider-cas-failure") + planning_input = _planning_input() + _bind_planning_input(task, planning_input) + host_checkpoint = replace(_checkpoint(), planning_input=planning_input) + provider = SimpleNamespace( + plugin_id="EmptyProvider", + plugin_name="空结果插件", + method="transfer", + ) + repository = Mock() + checkpoint_calls = [] + + def checkpoint_plan(**kwargs): + """只允许 provider_pending 提交，模拟 host promotion CAS 竞争失败。""" + checkpoint_calls.append(kwargs["checkpoint"]) + if len(checkpoint_calls) == 1: + return _planned_admission(task, kwargs["checkpoint"]) + raise transfer_application.TransferPlanningStateError("CAS failed") + + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository, checkpoint=host_checkpoint) + chain._module_dispatcher.freeze_plugin_providers.return_value = (provider,) + chain._module_dispatcher.execute_frozen_plugin_providers.return_value = None + + with pytest.raises( + transfer_application.TransferPlanningStateError, + match="CAS failed", + ): + chain._plan_checkpoint_and_execute(task) + + assert checkpoint_calls[0].is_provider_pending is True + assert checkpoint_calls[1].is_provider_pending is False + chain.plan_transfer.assert_called_once() + chain.execute_transfer_plan.assert_not_called() + repository.record_planning_failure.assert_called_once_with( + task_id="task-provider-cas-failure", + error="CAS failed", + ) + + +def test_provider_pending_crash_replay_executes_snapshot_without_host_planning(): + """provider 提交后崩溃重放必须直接执行冻结调用，不重新冻结或宿主规划。""" + planning_input = _planning_input() + provider = SimpleNamespace( + plugin_id="CrashProvider", + plugin_name="崩溃恢复插件", + method="transfer", + ) + first_task = _task() + first_task.meta = MetaBase("Movie.2026.mkv") + first_task.mediainfo = MediaInfo(title="Movie") + first_task.bind_admission_task_id("task-provider-replay") + _bind_planning_input(first_task, planning_input) + first_repository = Mock() + first_repository.checkpoint_plan.side_effect = lambda **kwargs: ( + _planned_admission(first_task, kwargs["checkpoint"]) + ) + first_chain = _chain(repository=first_repository) + first_chain._module_dispatcher.freeze_plugin_providers.return_value = (provider,) + first_chain._module_dispatcher.execute_frozen_plugin_providers.side_effect = ( + RuntimeError("process crashed") + ) + + with pytest.raises(RuntimeError, match="process crashed"): + first_chain._plan_checkpoint_and_execute(first_task) + + provider_checkpoint = first_task.plan_checkpoint + assert provider_checkpoint is not None + assert provider_checkpoint.is_provider_pending is True + first_chain.plan_transfer.assert_not_called() + + recovered_task = _task() + recovered_task.meta = MetaBase("Drifted.mkv") + recovered_task.mediainfo = MediaInfo(title="Drifted") + recovered_task.bind_admission_task_id("task-provider-replay") + _bind_planning_input(recovered_task, planning_input) + _bind_checkpoint(recovered_task, provider_checkpoint) + recovered_result = TransferInfo( + success=True, + fileitem=recovered_task.fileitem, + transfer_type="copy", + ) + recovered_chain = _chain(repository=Mock()) + recovered_chain._module_dispatcher.execute_frozen_plugin_providers.return_value = ( + recovered_result + ) + + returned = recovered_chain._plan_checkpoint_and_execute(recovered_task) + + assert returned is recovered_result + recovered_chain._module_dispatcher.freeze_plugin_providers.assert_not_called() + recovered_chain.plan_transfer.assert_not_called() + recovered_chain._transfer_admissions.checkpoint_plan.assert_not_called() + provider_call = ( + recovered_chain._module_dispatcher.execute_frozen_plugin_providers.call_args + ) + assert provider_call.kwargs["meta"].org_string == "Movie.2026.mkv" + assert provider_call.kwargs["mediainfo"].title == "Movie" + + +def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): + """旧同步调用必须经过 admission、checkpoint、执行和终态注销。""" + calls = [] + repository = Mock() + repository.admit.side_effect = ( + lambda **_kwargs: calls.append("admit") + or SimpleNamespace(task_id="task-legacy-command") + ) + + def checkpoint_plan(**kwargs): + """回读刚提交的检查点并记录顺序。""" + calls.append("checkpoint") + return SimpleNamespace(checkpoint=kwargs["checkpoint"]) + + repository.checkpoint_plan.side_effect = checkpoint_plan + repository.discard_task.side_effect = ( + lambda **_kwargs: calls.append("discard") or 1 + ) + result = TransferInfo( + success=True, + fileitem=_task().fileitem, + transfer_type="copy", + ) + chain = _chain(repository=repository, checkpoint=_checkpoint(), result=result) + chain.execute_transfer_plan.side_effect = ( + lambda *_args, **_kwargs: calls.append("execute") or result + ) + meta = MetaBase("Movie.2026.mkv") + mediainfo = MediaInfo() + + returned = chain.execute_legacy_transfer_command( + fileitem=_task().fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage="local", + target_path=Path("/library"), + transfer_type="copy", + ) + + assert returned is result + assert calls == ["admit", "checkpoint", "execute", "discard"] + repository.discard_task.assert_called_once_with( + task_id="task-legacy-command" + ) + + +def test_cleanup_destination_is_idempotent_and_uses_storage_safety_policy(): + """cleanup 不存在时成功，存在时必须委托 StorageChain 安全删除。""" + cleanup_item = FileItem(**_source_snapshot()) + chain = object.__new__(TransferChain) + storage_chain = Mock() + chain._transfer_storage_chain = Mock(return_value=storage_chain) + storage_chain.get_file_item_strict.return_value = None + + assert chain._TransferChain__cleanup_transfer_destination(cleanup_item) is True + storage_chain.delete_media_file.assert_not_called() + + current_item = cleanup_item.model_copy(update={"fileid": "current"}) + storage_chain.get_file_item_strict.return_value = current_item + storage_chain.delete_media_file.return_value = True + + assert chain._TransferChain__cleanup_transfer_destination(cleanup_item) is True + storage_chain.delete_media_file.assert_called_once_with(current_item) + + +def test_cleanup_destination_propagates_strict_lookup_failure() -> None: + """cleanup 查询失败必须保留 planned，不能被投影成目标不存在。""" + cleanup_item = FileItem(**_source_snapshot()) + chain = object.__new__(TransferChain) + storage_chain = Mock() + chain._transfer_storage_chain = Mock(return_value=storage_chain) + storage_chain.get_file_item_strict.side_effect = StorageQueryError( + "provider lookup failed" + ) + + with pytest.raises(StorageQueryError, match="provider lookup failed"): + chain._TransferChain__cleanup_transfer_destination(cleanup_item) + + storage_chain.delete_media_file.assert_not_called() + + +def test_planning_payload_round_trip_is_self_contained_and_stable(): + """planned 行仅凭 checkpoint payload 即可还原输入、目标与有序动作。""" + TransferPlanningInput, _TransferPlanItem, TransferPlanCheckpoint = ( + _planning_contracts() + ) + checkpoint = _checkpoint() + + restored = TransferPlanCheckpoint.from_payload(checkpoint.to_payload()) + + assert restored == checkpoint + assert isinstance(restored.planning_input, TransferPlanningInput) + assert restored.planning_input.source_fileitem == _source_snapshot() + assert restored.planning_input.fingerprint == checkpoint.planning_input.fingerprint + assert tuple(item.sequence for item in restored.items) == (0,) + assert restored.final_target_path == "/library/Movie (2026)/Movie.mkv" + + +def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_path): + """checkpoint 内嵌输入与 accepted 指纹不一致时必须拒绝状态跃迁。""" + engine = create_engine(f"sqlite:///{tmp_path / 'fingerprint.db'}") + TransferPending.__table__.create(engine) + repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) + accepted_input = _planning_input(target_path="/library/A") + admission = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=accepted_input, + ) + mismatched = _checkpoint(target_path="/library/B/Movie.mkv") + + with pytest.raises(ValueError, match="指纹|fingerprint|规划输入"): + repository.checkpoint_plan( + task_id=admission.task_id, + input_fingerprint=accepted_input.fingerprint, + checkpoint=mismatched, + ) + + recovered = repository.list_recoverable() + assert len(recovered) == 1 + assert recovered[0].state == "accepted" + assert recovered[0].checkpoint is None + engine.dispose() + + +def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path): + """仓储必须同时恢复 accepted 输入和 planned 自包含 checkpoint。""" + engine = create_engine(f"sqlite:///{tmp_path / 'recoverable.db'}") + TransferPending.__table__.create(engine) + repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) + planning_input = _planning_input() + admission = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + + accepted = repository.list_recoverable() + assert len(accepted) == 1 + assert accepted[0].state == "accepted" + assert accepted[0].planning_input == planning_input + assert accepted[0].checkpoint is None + + repository.record_planning_failure( + task_id=admission.task_id, + error="rename unavailable", + ) + retryable = repository.list_recoverable()[0] + assert retryable.state == "accepted" + assert retryable.last_error == "rename unavailable" + + checkpoint = _checkpoint() + planned = repository.checkpoint_plan( + task_id=admission.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + + assert planned.state == "planned" + assert planned.checkpoint == checkpoint + assert planned.last_error is None + assert repository.list_recoverable() == [planned] + engine.dispose() + + +def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted(): + """同步非 preview 任务也必须按 admit、plan、commit、execute 顺序运行。""" + order = [] + repository = Mock() + task = _task() + planning_input = _planning_input() + _bind_planning_input(task, planning_input) + checkpoint = _checkpoint() + result = TransferInfo(success=True, fileitem=task.fileitem, transfer_type="copy") + + def admit(**_kwargs): + """记录同步任务首次持久准入。""" + order.append("admit") + return SimpleNamespace( + task_id="task-sync", + state="accepted", + planning_input=planning_input, + checkpoint=None, + ) + + def checkpoint_plan(**_kwargs): + """记录 durable checkpoint 已经提交并返回 planned 投影。""" + order.append("commit-checkpoint") + return _planned_admission(task, checkpoint) + + repository.admit.side_effect = admit + repository.checkpoint_plan.side_effect = checkpoint_plan + chain = _chain(repository=repository, checkpoint=checkpoint, result=result) + chain.plan_transfer.side_effect = lambda *_args, **_kwargs: ( + order.append("plan") or checkpoint + ) + chain.execute_transfer_plan.side_effect = lambda *args, **_kwargs: ( + order.append("execute") or result + ) + + returned = chain._plan_checkpoint_and_execute(task) + + assert returned is result + assert order == ["admit", "plan", "commit-checkpoint", "execute"] + assert task.admission_task_id == "task-sync" + repository.admit.assert_called_once() + assert repository.admit.call_args.kwargs["planning_input"] is planning_input + repository.checkpoint_plan.assert_called_once_with( + task_id="task-sync", + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + chain.execute_transfer_plan.assert_called_once() + + +def test_checkpoint_commit_failure_blocks_executor_and_keeps_accepted(): + """checkpoint 提交失败时不得产生文件副作用，错误留在 accepted 记录。""" + repository = Mock() + task = _task() + planning_input = _planning_input() + checkpoint = _checkpoint() + task.bind_admission_task_id("task-accepted") + _bind_planning_input(task, planning_input) + repository.checkpoint_plan.side_effect = RuntimeError("commit failed") + chain = _chain(repository=repository, checkpoint=checkpoint) + + with pytest.raises(RuntimeError, match="commit failed"): + chain._plan_checkpoint_and_execute(task) + + chain.plan_transfer.assert_called_once() + chain.execute_transfer_plan.assert_not_called() + repository.record_planning_failure.assert_called_once() + assert repository.record_planning_failure.call_args.kwargs["task_id"] == ( + "task-accepted" + ) + assert "commit failed" in repository.record_planning_failure.call_args.kwargs["error"] + + +def test_post_commit_crash_replays_frozen_plan_without_replanning(): + """commit 后执行崩溃仍保留 planned checkpoint，重启后跳过 rename 规划。""" + repository = Mock() + planning_input = _planning_input() + checkpoint = _checkpoint() + first_task = _task() + first_task.bind_admission_task_id("task-planned") + _bind_planning_input(first_task, planning_input) + repository.checkpoint_plan.side_effect = lambda **_kwargs: _planned_admission( + first_task, checkpoint + ) + first_chain = _chain(repository=repository, checkpoint=checkpoint) + first_chain.execute_transfer_plan.side_effect = RuntimeError("process crashed") + + with pytest.raises(RuntimeError, match="process crashed"): + first_chain._plan_checkpoint_and_execute(first_task) + + assert first_task.plan_checkpoint == checkpoint + repository.record_planning_failure.assert_not_called() + + recovered_task = _task() + recovered_task.bind_admission_task_id("task-planned") + _bind_planning_input(recovered_task, planning_input) + _bind_checkpoint(recovered_task, checkpoint) + recovered_chain = _chain(repository=Mock(), checkpoint=checkpoint) + + recovered_chain._plan_checkpoint_and_execute(recovered_task) + + recovered_chain.plan_transfer.assert_not_called() + recovered_chain._transfer_admissions.checkpoint_plan.assert_not_called() + recovered_chain.execute_transfer_plan.assert_called_once() + execute_call = recovered_chain.execute_transfer_plan.call_args + executed_checkpoint = ( + execute_call.args[0] + if execute_call.args + else execute_call.kwargs["checkpoint"] + ) + assert executed_checkpoint is checkpoint + + +def test_first_execution_and_planned_replay_consume_identical_persisted_context(): + """首次提交后执行与 planned 重放必须消费同一份持久领域上下文。""" + _TransferPlanningInput, _TransferPlanItem, TransferPlanCheckpoint = ( + _planning_contracts() + ) + planner_checkpoint = _resolved_checkpoint() + persisted_checkpoint = TransferPlanCheckpoint.from_payload( + planner_checkpoint.to_payload() + ) + planning_input = planner_checkpoint.planning_input + + first_task = _task() + first_task.meta = MetaBase("Drifted.Before.Commit.mkv") + first_task.bind_admission_task_id("task-first-context") + _bind_planning_input(first_task, planning_input) + first_repository = Mock() + first_repository.checkpoint_plan.return_value = SimpleNamespace( + task_id="task-first-context", + state="planned", + planning_input=planning_input, + checkpoint=persisted_checkpoint, + ) + first_chain = _chain( + repository=first_repository, + checkpoint=planner_checkpoint, + ) + + first_chain._plan_checkpoint_and_execute(first_task) + + replay_task = _task() + replay_task.bind_admission_task_id("task-first-context") + _bind_planning_input(replay_task, planning_input) + _bind_checkpoint(replay_task, persisted_checkpoint) + replay_chain = _chain(repository=Mock(), checkpoint=persisted_checkpoint) + + replay_chain._plan_checkpoint_and_execute(replay_task) + + first_call = first_chain.execute_transfer_plan.call_args + replay_call = replay_chain.execute_transfer_plan.call_args + first_meta = first_call.kwargs["meta"] + replay_meta = replay_call.kwargs["meta"] + first_media = first_call.kwargs["mediainfo"] + replay_media = replay_call.kwargs["mediainfo"] + assert first_task.plan_checkpoint is persisted_checkpoint + assert first_meta.to_dict() == replay_meta.to_dict() + assert first_media.to_dict() == replay_media.to_dict() + assert [item.model_dump() for item in first_task.episodes_info] == [ + item.model_dump() for item in replay_task.episodes_info + ] + assert first_meta.title == "Frozen Show S01E01" + assert first_media.media_id == "987" + + +def test_accepted_recovery_replans_but_planned_recovery_does_not(): + """accepted 可重新触发 rename 规划，planned 必须只读冻结计划。""" + planning_input = _planning_input() + checkpoint = _checkpoint() + + accepted_task = _task() + accepted_task.bind_admission_task_id("task-accepted") + _bind_planning_input(accepted_task, planning_input) + accepted_repository = Mock() + accepted_repository.checkpoint_plan.side_effect = lambda **_kwargs: ( + _planned_admission(accepted_task, checkpoint) + ) + accepted_chain = _chain( + repository=accepted_repository, + checkpoint=checkpoint, + ) + accepted_chain._plan_checkpoint_and_execute(accepted_task) + + accepted_chain.plan_transfer.assert_called_once() + accepted_chain.execute_transfer_plan.assert_called_once() + + planned_task = _task() + planned_task.bind_admission_task_id("task-planned") + _bind_planning_input(planned_task, planning_input) + _bind_checkpoint(planned_task, checkpoint) + planned_chain = _chain(repository=Mock(), checkpoint=checkpoint) + planned_chain._plan_checkpoint_and_execute(planned_task) + + planned_chain.plan_transfer.assert_not_called() + planned_chain._transfer_admissions.checkpoint_plan.assert_not_called() + planned_chain.execute_transfer_plan.assert_called_once() + + +def test_accepted_replay_restores_explicit_context_without_online_lookup( + tmp_path, + monkeypatch, +): + """accepted 重放必须恢复显式 meta/media/episodes，并可离线完成重新规划。""" + media_path = tmp_path / "Frozen.Show.S01E01.mkv" + media_path.write_bytes(b"frozen") + planning_input = replace( + _resolved_checkpoint().planning_input, + source_fileitem={ + **_source_snapshot(), + "path": media_path.as_posix(), + "name": media_path.name, + "basename": media_path.stem, + }, + meta=_resolved_checkpoint().resolved_meta, + mediainfo=_resolved_checkpoint().resolved_mediainfo, + target_directory={ + "library_path": "/library", + "library_storage": "local", + "renaming": True, + "notify": True, + }, + episodes_info=_resolved_checkpoint().resolved_episodes_info, + options={ + **_resolved_checkpoint().planning_input.options, + "_meta_kind": "MetaVideo", + "_mediainfo_kind": "MediaInfo", + }, + ) + admission = transfer_application.TransferAdmission( + task_id="task-accepted-offline", + storage="local", + src_path=media_path.as_posix(), + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + input_fingerprint=planning_input.fingerprint, + planning_input=planning_input, + ) + replay_repository = Mock() + replay_repository.list_recoverable.return_value = [admission] + replay_chain = object.__new__(TransferChain) + replay_chain._transfer_admissions = replay_repository + replay_chain._execute_transfer = Mock() + queued_tasks = [] + replay_chain.put_to_queue = Mock( + side_effect=lambda task: queued_tasks.append(task) or True + ) + + replay_chain._TransferChain__replay_pending() + + assert len(queued_tasks) == 1 + restored_task = queued_tasks[0] + assert restored_task.admission_task_id == "task-accepted-offline" + assert restored_task.planning_input is planning_input + assert restored_task.planning_context_restored is True + assert restored_task.meta.title == "Frozen Show S01E01" + assert restored_task.mediainfo.media_id == "987" + assert restored_task.episodes_info[0].name == "Frozen Episode" + replay_chain._execute_transfer.assert_not_called() + + checkpoint = replace( + _resolved_checkpoint(), + planning_input=planning_input, + ) + execution_repository = Mock() + execution_repository.checkpoint_plan.return_value = SimpleNamespace( + task_id=admission.task_id, + state="planned", + planning_input=planning_input, + checkpoint=checkpoint, + ) + execution_chain = _chain( + repository=execution_repository, + checkpoint=checkpoint, + ) + execution_chain.jobview = Mock() + execution_chain.eventmanager = Mock() + execution_chain.eventmanager.send_event.return_value = None + execution_chain.runtime_config = SimpleNamespace(scrape_follow_tmdb=True) + execution_chain._TransferChain__finish_scrape_batch_task = Mock() + monkeypatch.setattr( + "app.chain.transfer.get_chain_transfer_history_port", + lambda: Mock(), + ) + monkeypatch.setattr( + "app.chain.transfer.MediaChain", + Mock(side_effect=AssertionError("accepted replay 不应重新在线识别")), + ) + monkeypatch.setattr( + "app.chain.transfer.TmdbChain", + Mock(side_effect=AssertionError("accepted replay 不应重新在线获取剧集")), + ) + + result = execution_chain._TransferChain__handle_transfer(restored_task) + + assert result[0] is True + execution_chain.plan_transfer.assert_called_once() + execution_chain.execute_transfer_plan.assert_called_once() + + +def test_accepted_replay_with_explicit_empty_episodes_stays_offline( + tmp_path, + monkeypatch, +): + """显式媒体快照即使没有剧集明细，也不得在 accepted 重放时在线补查。""" + media_path = tmp_path / "Frozen.Show.S01.mkv" + media_path.write_bytes(b"frozen") + resolved = _resolved_checkpoint() + planning_input = replace( + resolved.planning_input, + source_fileitem={ + **_source_snapshot(), + "path": media_path.as_posix(), + "name": media_path.name, + "basename": media_path.stem, + }, + meta=resolved.resolved_meta, + mediainfo=resolved.resolved_mediainfo, + target_directory={ + "library_path": "/library", + "library_storage": "local", + }, + episodes_info=(), + options={ + **resolved.planning_input.options, + "_meta_kind": "MetaVideo", + "_mediainfo_kind": "MediaInfo", + }, + ) + admission = transfer_application.TransferAdmission( + task_id="task-accepted-empty-episodes", + storage="local", + src_path=media_path.as_posix(), + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + planning_input=planning_input, + ) + replay_repository = Mock() + replay_repository.list_recoverable.return_value = [admission] + replay_chain = object.__new__(TransferChain) + replay_chain._transfer_admissions = replay_repository + queued_tasks = [] + replay_chain.put_to_queue = Mock( + side_effect=lambda task: queued_tasks.append(task) or True + ) + + replay_chain._TransferChain__replay_pending() + + restored_task = queued_tasks[0] + checkpoint = replace( + resolved, + planning_input=planning_input, + resolved_episodes_info=(), + ) + execution_repository = Mock() + execution_repository.checkpoint_plan.return_value = SimpleNamespace( + task_id=admission.task_id, + checkpoint=checkpoint, + ) + execution_chain = _chain( + repository=execution_repository, + checkpoint=checkpoint, + ) + execution_chain.jobview = Mock() + execution_chain.eventmanager = Mock() + execution_chain.eventmanager.send_event.return_value = None + execution_chain.runtime_config = SimpleNamespace(scrape_follow_tmdb=True) + execution_chain._TransferChain__finish_scrape_batch_task = Mock() + monkeypatch.setattr( + "app.chain.transfer.get_chain_transfer_history_port", + lambda: Mock(), + ) + monkeypatch.setattr( + "app.chain.transfer.MediaChain", + Mock(side_effect=AssertionError("accepted replay 不应重新在线识别")), + ) + monkeypatch.setattr( + "app.chain.transfer.TmdbChain", + Mock(side_effect=AssertionError("accepted replay 不应重新在线获取剧集")), + ) + + result = execution_chain._TransferChain__handle_transfer(restored_task) + + assert result[0] is True + assert restored_task.episodes_info == [] + + +def test_planned_execution_restores_resolved_context_without_online_recognition(): + """planned 恢复必须从 checkpoint 还原 meta/media/episodes，不再访问识别服务。""" + checkpoint = _resolved_checkpoint() + task = _task() + task.bind_admission_task_id("task-planned-context") + _bind_planning_input(task, checkpoint.planning_input) + _bind_checkpoint(task, checkpoint) + chain = _chain(repository=Mock(), checkpoint=checkpoint) + chain.jobview = Mock() + chain.eventmanager = Mock() + chain.eventmanager.send_event.return_value = None + + with patch( + "app.chain.transfer.MediaChain", + side_effect=AssertionError("planned replay 不应重新在线识别"), + ): + result = chain._TransferChain__handle_transfer(task) + + assert result[0] is True + chain.plan_transfer.assert_not_called() + chain.execute_transfer_plan.assert_called_once() + execute_call = chain.execute_transfer_plan.call_args + restored_meta = execute_call.kwargs["meta"] + restored_media = execute_call.kwargs["mediainfo"] + assert isinstance(restored_meta, MetaBase) + assert restored_meta.title == "Frozen Show S01E01" + assert restored_meta.begin_episode == 1 + assert restored_media.title == "Frozen Show" + assert restored_media.media_id == "987" + assert task.meta is restored_meta + assert task.mediainfo is restored_media + assert len(task.episodes_info) == 1 + assert task.episodes_info[0].episode_number == 1 + assert task.episodes_info[0].name == "Frozen Episode" + + +def test_filemanager_resolves_drifted_target_from_checkpoint(monkeypatch): + """运行时目录配置漂移后，executor 仍只按 checkpoint.target_storage 选适配器。""" + checkpoint = _checkpoint() + frozen_target = object() + drifted_target = object() + source_oper = object() + requested_storages = [] + module = object.__new__(FileManagerModule) + + def get_storage(storage: str): + """记录执行期选择的存储并为漂移配置提供错误适配器。""" + requested_storages.append(storage) + return frozen_target if storage == checkpoint.target_storage else drifted_target + + monkeypatch.setattr( + module, + "_FileManagerModule__get_storage_oper", + get_storage, + ) + execute = Mock( + return_value=TransferInfo( + success=True, + fileitem=_task().fileitem, + transfer_type="copy", + ) + ) + monkeypatch.setattr(TransHandler, "execute_transfer_plan", execute) + + FileManagerModule.execute_transfer_plan( + module, + checkpoint, + meta=None, + mediainfo=None, + source_oper=source_oper, + ) + + assert requested_storages == [checkpoint.target_storage] + assert execute.call_args.kwargs["target_oper"] is frozen_target + assert drifted_target is not frozen_target + + +def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch): + """准入后、checkpoint 前的业务失败必须写 last_error 并保持 accepted。""" + task = _task() + task.meta = MetaBase("Unrecognized.Movie.2026.mkv") + task.bind_admission_task_id("task-before-checkpoint") + chain = object.__new__(TransferChain) + chain._transfer_admissions = Mock() + chain.jobview = Mock() + chain.queue_failed_transfer_notification = Mock() + chain.runtime_config = SimpleNamespace( + ai_agent_enable=False, + ai_agent_retry_transfer=False, + ) + chain._TransferChain__mark_torrent_completed_if_done = Mock() + media_chain = Mock() + media_chain.recognize_by_meta.return_value = None + monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain) + monkeypatch.setattr( + "app.chain.transfer.get_chain_transfer_history_port", + lambda: SimpleNamespace(), + ) + monkeypatch.setattr("app.chain.transfer.record_transfer_failure", Mock()) + monkeypatch.setattr("app.chain.transfer.add_transfer_fail", lambda **_kwargs: None) + + result = chain._TransferChain__handle_transfer(task) + + assert result == (False, "未识别到媒体信息") + chain._transfer_admissions.record_planning_failure.assert_called_once_with( + task_id="task-before-checkpoint", + error="未识别到媒体信息", + ) + + +def test_preview_plans_without_persistence_or_file_side_effects(): + """preview 只生成和投影计划，不得触碰准入仓储或存储执行器。""" + repository = Mock() + task = _task(preview=True) + planning_input = _planning_input(preview=True) + checkpoint = _checkpoint(preview=True) + _bind_planning_input(task, planning_input) + chain = _chain(repository=repository, checkpoint=checkpoint) + + chain._plan_checkpoint_and_execute(task) + + chain.plan_transfer.assert_called_once() + repository.admit.assert_not_called() + repository.checkpoint_plan.assert_not_called() + repository.record_planning_failure.assert_not_called() + chain.execute_transfer_plan.assert_called_once() + execute_call = chain.execute_transfer_plan.call_args + executed_checkpoint = ( + execute_call.args[0] + if execute_call.args + else execute_call.kwargs["checkpoint"] + ) + assert executed_checkpoint.preview is True + + +def test_preview_executor_projects_result_without_touching_storage(): + """FileManager 的 preview executor 不得查询、创建、复制或删除存储对象。""" + checkpoint = _checkpoint(preview=True) + source_oper = Mock() + target_oper = Mock() + + result = TransHandler().execute_transfer_plan( + checkpoint, + meta=None, + mediainfo=None, + source_oper=source_oper, + target_oper=target_oper, + ) + + assert result.success is True + assert result.target_item.path == checkpoint.final_target_path + assert result.file_list_new == [checkpoint.final_target_path] + assert source_oper.mock_calls == [] + assert target_oper.mock_calls == [] + + +def test_filemanager_explicit_plan_then_execute_uses_same_checkpoint(): + """FileManager 的调用方必须显式把冻结计划交给唯一执行入口。""" + module = object.__new__(FileManagerModule) + checkpoint = _checkpoint() + result = TransferInfo(success=True, fileitem=_task().fileitem, transfer_type="copy") + module.plan_transfer = Mock(return_value=checkpoint) + module.execute_transfer_plan = Mock(return_value=result) + fileitem = _task().fileitem + + planned = module.plan_transfer( + fileitem=fileitem, + meta=None, + mediainfo=None, + target_storage="local", + target_path=Path("/library"), + transfer_type="copy", + preview=False, + ) + returned = module.execute_transfer_plan( + planned, + meta=None, + mediainfo=None, + ) + + assert returned is result + module.plan_transfer.assert_called_once() + module.execute_transfer_plan.assert_called_once() + execute_call = module.execute_transfer_plan.call_args + assert execute_call.args[0] is checkpoint + + +def test_sync_and_worker_paths_share_checkpoint_orchestrator(): + """同步直跑和后台 worker 必须汇入同一个检查点编排，不得旁路旧 transfer。""" + module = ast.parse(inspect.getsource(inspect.getmodule(TransferChain))) + functions = { + node.name: node + for node in ast.walk(module) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + handler = functions["__handle_transfer"] + handler_calls = { + node.func.attr + for node in ast.walk(handler) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + performer = functions["__perform_transfer"] + performer_calls = { + node.func.attr + for node in ast.walk(performer) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + execute_batch = functions["_execute_transfer"] + execute_calls = { + node.func.attr + for node in ast.walk(execute_batch) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + } + + assert "__perform_transfer" in handler_calls + assert "_plan_checkpoint_and_execute" in performer_calls + assert "transfer" not in handler_calls + assert "__handle_transfer" in execute_calls diff --git a/tests/test_transfer_planning_compat.py b/tests/test_transfer_planning_compat.py new file mode 100644 index 000000000..87ef64cbe --- /dev/null +++ b/tests/test_transfer_planning_compat.py @@ -0,0 +1,285 @@ +"""整理规划重构必须保持的旧调用与事件兼容合同。""" + +from inspect import Parameter, signature +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from app.application.transfer import TransferTask +from app.chain import ChainBase +from app.chain.transfer import TransferChain +from app.modules.filemanager.module import FileManagerModule +from app.modules.filemanager.transhandler import TransHandler +from app.schemas.file import FileItem +from app.schemas.transfer import TransferInfo +from app.schemas.types import EventType + + +def _assert_signature(callable_object, *parameters: tuple[str, object]) -> None: + """断言公开参数顺序和默认值，不把测试辅助哨兵泄漏到失败输出。""" + actual = signature(callable_object).parameters + assert tuple(actual) == tuple(name for name, _default in parameters) + for name, expected_default in parameters: + parameter = actual[name] + if expected_default is ...: + assert parameter.default is Parameter.empty + else: + assert parameter.default == expected_default + + +def _fileitem() -> FileItem: + """构造无需文件系统 I/O 的最小旧整理源对象。""" + return FileItem( + storage="local", + path="/downloads/Movie.2026.mkv", + type="file", + name="Movie.2026.mkv", + basename="Movie.2026", + extension="mkv", + size=1024, + modify_time=1770000000, + fileid="source-1", + ) + + +def test_transfer_task_to_dict_keeps_exact_legacy_fields(): + """内部准入和规划快照不得进入插件可见的旧任务字典。""" + task = TransferTask( + fileitem=_fileitem(), + target_storage="local", + target_path=Path("/library/Movie (2026)"), + transfer_type="copy", + scrape=True, + manual=True, + background=False, + ) + task.bind_admission_task_id("task-stable") + + values = task.to_dict() + + assert set(values) == { + "fileitem", + "meta", + "mediainfo", + "media_source", + "media_id", + "mtype", + "target_directory", + "target_storage", + "target_path", + "transfer_type", + "scrape", + "library_type_folder", + "library_category_folder", + "episodes_info", + "username", + "downloader", + "download_hash", + "download_history", + "transfer_batch_id", + "manual", + "background", + "preview", + } + assert values["fileitem"] == task.fileitem.model_dump() + assert values["target_path"] == Path("/library/Movie (2026)") + assert "admission_task_id" not in values + assert "planning_input" not in values + assert "plan_checkpoint" not in values + + +def test_transfer_chain_do_transfer_keeps_legacy_signature(): + """公开整理入口必须继续接受原调用方的全部关键字参数。""" + _assert_signature( + TransferChain.do_transfer, + ("self", ...), + ("fileitem", ...), + ("meta", None), + ("mediainfo", None), + ("mtype", None), + ("media_source", None), + ("media_id", None), + ("target_directory", None), + ("target_storage", None), + ("target_path", None), + ("transfer_type", None), + ("scrape", None), + ("library_type_folder", None), + ("library_category_folder", None), + ("season", None), + ("epformat", None), + ("min_filesize", 0), + ("downloader", None), + ("download_hash", None), + ("force", False), + ("background", True), + ("manual", False), + ("preview", False), + ("sync_extra_files", False), + ("cleanup_dest_fileitem", None), + ("continue_callback", None), + ("reorganize", False), + ) + + +def test_chain_base_transfer_keeps_legacy_signature(): + """仅 Chain 对外兼容层保留旧整理参数，宿主模块不再重复导出。""" + _assert_signature( + ChainBase.transfer, + ("self", ...), + ("fileitem", ...), + ("meta", ...), + ("mediainfo", ...), + ("target_directory", None), + ("target_storage", None), + ("target_path", None), + ("transfer_type", None), + ("scrape", None), + ("library_type_folder", None), + ("library_category_folder", None), + ("episodes_info", None), + ("source_oper", None), + ("target_oper", None), + ("preview", False), + ) + + +def test_chain_base_transfer_delegates_exactly_to_injected_command(): + """旧 ABI 必须原样调用注入命令，不得再次进入动态模块调度。""" + chain = object.__new__(ChainBase) + result = TransferInfo(success=True, fileitem=_fileitem(), transfer_type="copy") + command = Mock(return_value=result) + chain._legacy_transfer_command = command + chain.run_module = Mock(side_effect=AssertionError("不得调用 run_module")) + meta = Mock(name="meta") + mediainfo = Mock(name="mediainfo") + target_directory = Mock(name="target_directory") + source_oper = Mock(name="source_oper") + target_oper = Mock(name="target_oper") + episodes_info = [Mock(name="episode")] + + returned = chain.transfer( + fileitem=result.fileitem, + meta=meta, + mediainfo=mediainfo, + target_directory=target_directory, + target_storage="alist", + target_path=Path("/library/Movie (2026)"), + transfer_type="copy", + scrape=True, + library_type_folder=False, + library_category_folder=True, + episodes_info=episodes_info, + source_oper=source_oper, + target_oper=target_oper, + preview=True, + ) + + assert returned is result + command.assert_called_once_with( + fileitem=result.fileitem, + meta=meta, + mediainfo=mediainfo, + target_directory=target_directory, + target_path=Path("/library/Movie (2026)"), + target_storage="alist", + transfer_type="copy", + scrape=True, + library_type_folder=False, + library_category_folder=True, + episodes_info=episodes_info, + source_oper=source_oper, + target_oper=target_oper, + preview=True, + ) + chain.run_module.assert_not_called() + + +def test_chain_base_plan_transfer_keeps_internal_dto_type_only(): + """规划入口运行时只做内部调度，不要求导入或重复导出 DTO。""" + chain = object.__new__(ChainBase) + checkpoint = Mock(name="checkpoint") + chain.run_module = Mock(return_value=checkpoint) + meta = Mock(name="meta") + mediainfo = Mock(name="mediainfo") + + returned = chain.plan_transfer( + fileitem=_fileitem(), + meta=meta, + mediainfo=mediainfo, + ) + + assert returned is checkpoint + chain.run_module.assert_called_once_with( + "plan_transfer", + fileitem=_fileitem(), + meta=meta, + mediainfo=mediainfo, + target_directory=None, + target_path=None, + target_storage=None, + transfer_type=None, + scrape=None, + library_type_folder=None, + library_category_folder=None, + episodes_info=None, + source_oper=None, + preview=False, + planning_input=None, + ) + + +def test_filemanager_module_has_no_legacy_transfer_provider(): + """FileManager 宿主只暴露规划与执行阶段，不再注册旧 transfer provider。""" + assert not hasattr(FileManagerModule, "transfer") + assert not hasattr(TransHandler, "transfer_media") + assert callable(FileManagerModule.plan_transfer) + assert callable(FileManagerModule.execute_transfer_plan) + + +@pytest.mark.parametrize( + ("event_type", "success"), + [ + (EventType.TransferComplete, True), + (EventType.TransferFailed, False), + ], +) +def test_transfer_result_event_keeps_single_exact_legacy_payload(event_type, success): + """规划与回放不得重复发送结果事件或改变插件读取的 payload。""" + chain = object.__new__(TransferChain) + chain.eventmanager = Mock() + task = TransferTask( + fileitem=_fileitem(), + downloader="qbittorrent", + download_hash="download-1", + ) + transferinfo = TransferInfo( + success=success, + fileitem=task.fileitem, + transfer_type="copy", + message="" if success else "copy failed", + ) + payload = chain._transfer_result_payload( + task, + transferinfo, + history_id=42, + ) + + chain._publish_transfer_result(event_type, payload) + + chain.eventmanager.send_event.assert_called_once_with(event_type, payload) + assert set(payload) == { + "fileitem", + "meta", + "mediainfo", + "transferinfo", + "downloader", + "download_hash", + "transfer_history_id", + } + assert payload["fileitem"] is task.fileitem + assert payload["transferinfo"] is transferinfo + assert payload["downloader"] == "qbittorrent" + assert payload["download_hash"] == "download-1" + assert payload["transfer_history_id"] == 42 diff --git a/tests/test_transfer_planning_migration.py b/tests/test_transfer_planning_migration.py new file mode 100644 index 000000000..413dedde9 --- /dev/null +++ b/tests/test_transfer_planning_migration.py @@ -0,0 +1,302 @@ +"""整理规划输入与检查点字段的 Alembic 迁移测试。""" + +import importlib +import json +import os +import uuid + +import pytest +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from app.application.transfer import TransferPlanningInput +from app.db.models.transferpending import TransferPending + +try: + import psycopg2 as postgres_driver + from psycopg2 import sql + + POSTGRESQL_DIALECT = "postgresql+psycopg2" +except ModuleNotFoundError: + import psycopg as postgres_driver + from psycopg import sql + + POSTGRESQL_DIALECT = "postgresql+psycopg" + +MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14" + + +def _bind_migration(monkeypatch, connection): + """把迁移绑定到隔离数据库连接。""" + migration = importlib.import_module(MIGRATION) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + return migration + + +def _create_admission_table(connection) -> None: + """创建 3.0.13 时代的持久准入表。""" + metadata = sa.MetaData() + table = sa.Table( + "transferpending", + metadata, + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("storage", sa.String(), nullable=False), + sa.Column("src_path", sa.String(), nullable=False), + sa.Column("created_at", sa.String(), nullable=True), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("updated_at", sa.String(40), nullable=False), + sa.Column("last_error", sa.Text(), nullable=True), + sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"), + ) + sa.Index( + "ux_transferpending_storage_path", + table.c.storage, + table.c.src_path, + unique=True, + ) + sa.Index( + "ix_transferpending_state_created", + table.c.state, + table.c.created_at, + table.c.id, + ) + metadata.create_all(connection) + connection.execute(table.insert(), { + "id": 1, + "task_id": "stable-task", + "storage": "local", + "src_path": "/downloads/Movie.mkv", + "created_at": "2026-08-27 10:00:00", + "state": "accepted", + "updated_at": "2026-08-27 10:00:00", + "last_error": "previous enqueue failure", + }) + + +def _planning_row(connection) -> dict[str, object]: + """读取单条规划持久字段快照。""" + return dict(connection.execute(sa.text( + "SELECT task_id, state, input_version, planning_input, " + "input_fingerprint, checkpoint_version, checkpoint_payload, planned_at " + "FROM transferpending WHERE id = 1" + )).mappings().one()) + + +def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: + """断言规划迁移在当前隔离连接上的完整可逆生命周期。""" + _create_admission_table(connection) + migration = _bind_migration(monkeypatch, connection) + + migration.upgrade() + migration.upgrade() + + inspector = sa.inspect(connection) + assert { + column["name"] + for column in inspector.get_columns("transferpending") + } == {column.name for column in TransferPending.__table__.columns} + upgraded = _planning_row(connection) + planning_payload = upgraded["planning_input"] + if isinstance(planning_payload, str): + planning_payload = json.loads(planning_payload) + planning_input = TransferPlanningInput.from_payload(planning_payload) + assert planning_input == TransferPlanningInput.legacy( + storage="local", + src_path="/downloads/Movie.mkv", + ) + assert upgraded["input_version"] == 1 + assert upgraded["input_fingerprint"] == planning_input.fingerprint + assert upgraded["checkpoint_payload"] is None + assert upgraded["state"] == "accepted" + + pending = sa.table( + "transferpending", + sa.column("id", sa.Integer()), + sa.column("state", sa.String()), + sa.column("checkpoint_version", sa.Integer()), + sa.column("checkpoint_payload", sa.JSON()), + sa.column("planned_at", sa.String()), + ) + connection.execute( + pending.update() + .where(pending.c.id == 1) + .values( + state="planned", + checkpoint_version=1, + checkpoint_payload={"schema_version": 1}, + planned_at="2026-08-27 11:00:00", + ) + ) + migration.downgrade() + + downgraded = sa.inspect(connection) + assert { + column["name"] + for column in downgraded.get_columns("transferpending") + } == { + "id", "task_id", "storage", "src_path", "created_at", + "state", "updated_at", "last_error", + } + legacy = connection.execute(sa.text( + "SELECT task_id, state FROM transferpending WHERE id = 1" + )).mappings().one() + assert dict(legacy) == {"task_id": "stable-task", "state": "accepted"} + assert { + index["name"] + for index in downgraded.get_indexes("transferpending") + } == {"ix_transferpending_state_created", "ux_transferpending_storage_path"} + + migration.upgrade() + reupgraded = _planning_row(connection) + assert reupgraded["task_id"] == "stable-task" + assert reupgraded["state"] == "accepted" + assert reupgraded["checkpoint_payload"] is None + assert reupgraded["input_fingerprint"] == planning_input.fingerprint + + +def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None: + """SQLite 应支持规划字段重复升级、降级和再次升级。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) + + +def test_provider_pending_downgrade_restores_accepted_on_sqlite(monkeypatch) -> None: + """旧版本无法解释 provider 快照，降级时必须保守恢复为 accepted。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + _create_admission_table(connection) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + connection.execute(sa.text( + "UPDATE transferpending " + "SET state = 'provider_pending', checkpoint_version = 1, " + "checkpoint_payload = '{\"schema_version\": 1}' " + "WHERE id = 1" + )) + + migration.downgrade() + + assert connection.execute(sa.text( + "SELECT state FROM transferpending WHERE id = 1" + )).scalar_one() == "accepted" + assert { + column["name"] + for column in sa.inspect(connection).get_columns("transferpending") + } == { + "id", "task_id", "storage", "src_path", "created_at", + "state", "updated_at", "last_error", + } + + +def test_partial_upgrade_preserves_existing_planning_json(monkeypatch) -> None: + """迁移中断后重跑应补齐版本和指纹，不得覆盖已经写入的完整输入。""" + engine = sa.create_engine("sqlite://") + planning_input = TransferPlanningInput( + source_fileitem={ + "storage": "local", + "path": "/downloads/Movie.mkv", + "type": "file", + }, + target_storage="local", + target_path="/library/Movies", + requested_transfer_type="copy", + options={"manual": True}, + ) + with engine.begin() as connection: + _create_admission_table(connection) + connection.execute(sa.text( + "ALTER TABLE transferpending ADD COLUMN planning_input JSON" + )) + pending = sa.table( + "transferpending", + sa.column("id", sa.Integer()), + sa.column("planning_input", sa.JSON()), + ) + connection.execute( + pending.update() + .where(pending.c.id == 1) + .values(planning_input=planning_input.to_payload()) + ) + + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + upgraded = _planning_row(connection) + payload = upgraded["planning_input"] + if isinstance(payload, str): + payload = json.loads(payload) + + assert TransferPlanningInput.from_payload(payload) == planning_input + assert upgraded["input_version"] == planning_input.schema_version + assert upgraded["input_fingerprint"] == planning_input.fingerprint + + connection.execute(sa.text( + "UPDATE transferpending SET state = 'future-state' WHERE id = 1" + )) + migration.downgrade() + assert connection.execute(sa.text( + "SELECT state FROM transferpending WHERE id = 1" + )).scalar_one() == "future-state" + + +def test_transfer_planning_migration_runs_on_postgresql(monkeypatch) -> None: + """配置隔离 PostgreSQL 时真实验证规划字段的完整可逆迁移。""" + prefix = "MOVIEPILOT_TEST_POSTGRESQL_" + host = os.getenv(f"{prefix}HOST") + database = os.getenv(f"{prefix}DATABASE") + username = os.getenv(f"{prefix}USERNAME") + if not host or not database or not username: + pytest.skip("未配置隔离 PostgreSQL migration 测试库") + + port = os.getenv(f"{prefix}PORT", "5432") + password = os.getenv(f"{prefix}PASSWORD", "") + schema = f"transfer_planning_{uuid.uuid4().hex}" + with postgres_driver.connect( + host=host, + port=port, + dbname=database, + user=username, + password=password, + ) as connection: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema))) + + engine = None + try: + engine = sa.create_engine( + sa.URL.create( + POSTGRESQL_DIALECT, + username=username, + password=password, + host=host, + port=int(port), + database=database, + ), + connect_args={"options": f"-csearch_path={schema}"}, + ) + with engine.begin() as connection: + _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) + finally: + if engine is not None: + engine.dispose() + with postgres_driver.connect( + host=host, + port=port, + dbname=database, + user=username, + password=password, + ) as connection: + connection.autocommit = True + with connection.cursor() as cursor: + cursor.execute( + sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format( + sql.Identifier(schema) + ) + ) diff --git a/tests/test_transfer_planning_persistence.py b/tests/test_transfer_planning_persistence.py new file mode 100644 index 000000000..8a391fa49 --- /dev/null +++ b/tests/test_transfer_planning_persistence.py @@ -0,0 +1,555 @@ +"""整理规划输入与原子检查点持久化测试。""" + +from dataclasses import replace + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from app.application.transfer import ( + TRANSFER_ADMISSION_ACCEPTED, + TRANSFER_ADMISSION_PLANNED, + TRANSFER_ADMISSION_PROVIDER_PENDING, + TransferAdmissionConflictError, + TransferPlanCheckpoint, + TransferPlanItem, + TransferPlanningInput, + TransferPlanningStateError, + TransferProviderInvocationSnapshot, + TransferProviderReference, +) +from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferpending import TransferPending + + +@pytest.fixture +def repository(tmp_path): + """创建只服务单个测试的 SQLite 整理计划仓储。""" + engine = create_engine(f"sqlite:///{tmp_path / 'transfer-planning.db'}") + TransferPending.__table__.create(engine) + return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) + + +def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput: + """构造包含恢复所需媒体上下文的完整规划输入。""" + return TransferPlanningInput( + source_fileitem={ + "storage": "local", + "path": "/downloads/Movie.2026.mkv", + "type": "file", + "size": 1024, + }, + meta={"name": "Movie", "year": 2026}, + mediainfo={"title": "Movie", "tmdb_id": 42}, + target_directory={"storage": "local", "path": "/library"}, + target_storage="local", + target_path=target_path, + requested_transfer_type="copy", + media_source="themoviedb", + media_id="42", + media_type="电影", + need_scrape=True, + need_rename=True, + need_notify=True, + overwrite_mode="always", + episodes_info=({"season_number": 1, "episode_number": 1},), + options={"username": "admin", "download_hash": "hash-1"}, + ) + + +def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint: + """构造可直接执行且不会再次触发 rename 的计划检查点。""" + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="local", + root_target_path="/library", + final_target_path="/library/Movies/Movie (2026)/Movie.mkv", + resolved_transfer_type="copy", + items=( + TransferPlanItem( + sequence=0, + source_fileitem=planning_input.source_fileitem, + target_storage="local", + target_path="/library/Movies/Movie (2026)/Movie.mkv", + ), + ), + resolved_meta=planning_input.meta, + resolved_meta_kind="MetaVideo", + resolved_mediainfo=planning_input.mediainfo, + resolved_mediainfo_kind="MediaInfo", + resolved_episodes_info=planning_input.episodes_info, + legacy_transfer_providers=( + TransferProviderReference( + plugin_id="builtin-filemanager", + plugin_name="FileManager", + ), + TransferProviderReference( + plugin_id="plugin-provider-a", + plugin_name="Provider A", + ), + ), + need_scrape=True, + need_rename=False, + need_notify=True, + overwrite_mode="always", + ) + + +def _provider_checkpoint( + planning_input: TransferPlanningInput, +) -> TransferPlanCheckpoint: + """构造只冻结旧 ABI、尚未生成宿主文件计划的检查点。""" + invocation = TransferProviderInvocationSnapshot( + fileitem=planning_input.source_fileitem, + meta=planning_input.meta, + meta_kind="MetaVideo", + mediainfo=planning_input.mediainfo, + mediainfo_kind="MediaInfo", + target_directory={ + "library_storage": "local", + "library_path": "/library/Movies", + "transfer_type": "copy", + }, + target_storage="local", + target_path=None, + transfer_type=None, + scrape=None, + library_type_folder=False, + library_category_folder=None, + episodes_info=planning_input.episodes_info, + preview=False, + ) + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="", + root_target_path="", + final_target_path="", + resolved_transfer_type="", + items=(), + resolved_meta=invocation.meta, + resolved_meta_kind=invocation.meta_kind, + resolved_mediainfo=invocation.mediainfo, + resolved_mediainfo_kind=invocation.mediainfo_kind, + resolved_episodes_info=invocation.episodes_info, + legacy_transfer_providers=( + TransferProviderReference( + plugin_id="plugin-provider-a", + plugin_name="Provider A", + ), + ), + provider_invocation=invocation, + ) + + +def test_planning_dtos_round_trip_versioned_json() -> None: + """输入和检查点应完整往返 JSON，并保留有序叶操作。""" + planning_input = _planning_input() + checkpoint = _checkpoint(planning_input) + + restored_input = TransferPlanningInput.from_payload(planning_input.to_payload()) + restored_checkpoint = TransferPlanCheckpoint.from_payload(checkpoint.to_payload()) + + assert restored_input == planning_input + assert restored_input.fingerprint == planning_input.fingerprint + assert restored_checkpoint == checkpoint + assert [item.sequence for item in restored_checkpoint.items] == [0] + assert restored_checkpoint.planning_input == planning_input + assert restored_checkpoint.resolved_mediainfo["tmdb_id"] == 42 + assert restored_checkpoint.resolved_meta_kind == "MetaVideo" + assert restored_checkpoint.resolved_mediainfo_kind == "MediaInfo" + assert restored_checkpoint.resolved_episodes_info == planning_input.episodes_info + assert restored_checkpoint.legacy_transfer_providers == ( + TransferProviderReference("builtin-filemanager", "FileManager"), + TransferProviderReference("plugin-provider-a", "Provider A"), + ) + assert restored_checkpoint.legacy_transfer_providers[0].method == "transfer" + + +def test_provider_invocation_snapshot_round_trip_preserves_optional_values() -> None: + """旧 ABI 快照往返 JSON 后必须保留 None、False 和自动目录原始值。""" + checkpoint = _provider_checkpoint(_planning_input()) + + restored = TransferPlanCheckpoint.from_payload(checkpoint.to_payload()) + + assert restored == checkpoint + assert restored.is_provider_pending is True + assert restored.provider_invocation.target_path is None + assert restored.provider_invocation.transfer_type is None + assert restored.provider_invocation.scrape is None + assert restored.provider_invocation.library_type_folder is False + assert restored.provider_invocation.library_category_folder is None + + invalid_payload = checkpoint.to_payload() + invalid_payload["provider_invocation"]["schema_version"] = 0 + with pytest.raises(ValueError, match="调用快照版本"): + TransferPlanCheckpoint.from_payload(invalid_payload) + + +def test_legacy_checkpoint_payload_defaults_resolved_context() -> None: + """旧检查点缺少 resolved 字段时仍应恢复为兼容空快照。""" + payload = _checkpoint(_planning_input()).to_payload() + for key in ( + "resolved_meta", + "resolved_meta_kind", + "resolved_mediainfo", + "resolved_mediainfo_kind", + "resolved_episodes_info", + "legacy_transfer_providers", + ): + payload.pop(key) + + restored = TransferPlanCheckpoint.from_payload(payload) + + assert restored.resolved_meta is None + assert restored.resolved_meta_kind is None + assert restored.resolved_mediainfo is None + assert restored.resolved_mediainfo_kind is None + assert restored.resolved_episodes_info == () + assert restored.legacy_transfer_providers == () + + +@pytest.mark.parametrize( + ("plugin_id", "plugin_name", "method"), + [ + ("", "Provider A", "transfer"), + (" ", "Provider A", "transfer"), + ("provider-a", "", "transfer"), + ("provider-a", " ", "transfer"), + ("provider-a", "Provider A", "delete"), + ], +) +def test_transfer_provider_reference_rejects_invalid_fields( + plugin_id, + plugin_name, + method, +) -> None: + """旧 provider 引用必须具有稳定插件身份且只能指向 transfer 方法。""" + with pytest.raises(ValueError, match="provider"): + TransferProviderReference( + plugin_id=plugin_id, + plugin_name=plugin_name, + method=method, + ) + + +def test_checkpoint_rejects_duplicate_legacy_provider_plugin_id() -> None: + """同一 checkpoint 不得以不同名称重复冻结同一插件身份。""" + with pytest.raises(ValueError, match="plugin_id.*重复"): + replace( + _checkpoint(_planning_input()), + legacy_transfer_providers=( + TransferProviderReference("provider-a", "Provider A"), + TransferProviderReference("provider-a", "Provider A Renamed"), + ), + ) + + +def test_checkpoint_rejects_non_array_legacy_provider_payload() -> None: + """JSON 恢复边界不得把单个对象等非数组值当成 provider 序列。""" + payload = _checkpoint(_planning_input()).to_payload() + payload["legacy_transfer_providers"] = { + "plugin_id": "provider-a", + "plugin_name": "Provider A", + } + + with pytest.raises(ValueError, match="legacy_transfer_providers"): + TransferPlanCheckpoint.from_payload(payload) + + +def test_resolved_context_does_not_change_admission_fingerprint(repository) -> None: + """规划后上下文属于 checkpoint，不得反向改变已提交的准入指纹。""" + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + checkpoint = replace( + _checkpoint(planning_input), + resolved_meta={"name": "Resolved Movie", "year": 2026}, + resolved_meta_kind="MetaAnime", + resolved_mediainfo={"title": "Resolved Movie", "tmdb_id": 84}, + resolved_mediainfo_kind="MediaInfo", + resolved_episodes_info=({"season_number": 2, "episode_number": 3},), + ) + + planned = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + + assert planned.input_fingerprint == planning_input.fingerprint + assert planned.planning_input == planning_input + assert planned.checkpoint.resolved_meta_kind == "MetaAnime" + assert planned.checkpoint.resolved_mediainfo["tmdb_id"] == 84 + + with pytest.raises(TransferPlanningStateError): + repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=replace( + checkpoint, + resolved_mediainfo={"title": "Different", "tmdb_id": 85}, + ), + ) + + +def test_admit_reuses_identical_input_and_rejects_conflict(repository) -> None: + """同一源文件只允许复用完全相同的规划输入。""" + planning_input = _planning_input() + first = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + repeated = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=TransferPlanningInput.from_payload(planning_input.to_payload()), + ) + + assert repeated == first + with pytest.raises(TransferAdmissionConflictError): + repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=_planning_input(target_path="/other-library"), + ) + + +def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None: + """完整计划和 planned 状态应同事务提交且允许相同检查点重试。""" + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + checkpoint = _checkpoint(planning_input) + + planned = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + repeated = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + + assert planned.state == TRANSFER_ADMISSION_PLANNED + assert planned.checkpoint == checkpoint + assert planned.checkpoint.items[0].target_path.endswith("Movie.mkv") + assert tuple( + provider.plugin_id + for provider in planned.checkpoint.legacy_transfer_providers + ) == ( + "builtin-filemanager", + "plugin-provider-a", + ) + assert repeated == planned + assert repository.list_accepted() == [] + assert repository.list_recoverable() == [planned] + + +def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None: + """崩溃可恢复的 provider 快照只能经 CAS 升级为宿主 planned 计划。""" + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + provider_checkpoint = _provider_checkpoint(planning_input) + + provider_pending = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=provider_checkpoint, + ) + + assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING + assert provider_pending.checkpoint == provider_checkpoint + assert repository.list_recoverable() == [provider_pending] + + repository.record_planning_failure( + task_id=admitted.task_id, + error="host planning unavailable", + ) + failed = repository.list_recoverable()[0] + assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING + assert failed.checkpoint == provider_checkpoint + assert failed.last_error == "host planning unavailable" + + host_checkpoint = _checkpoint(planning_input) + planned = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=host_checkpoint, + ) + repeated = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=host_checkpoint, + ) + + assert planned.state == TRANSFER_ADMISSION_PLANNED + assert planned.checkpoint == host_checkpoint + assert planned.last_error is None + assert repeated == planned + with pytest.raises(TransferPlanningStateError): + repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=provider_checkpoint, + ) + + +def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> None: + """错误输入指纹不能写入部分计划或改变 accepted 状态。""" + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + + with pytest.raises(TransferAdmissionConflictError): + repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint="0" * 64, + checkpoint=_checkpoint(planning_input), + ) + + recovered = repository.list_recoverable() + assert len(recovered) == 1 + assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED + assert recovered[0].checkpoint is None + + +def test_planning_failure_stays_accepted_until_success(repository) -> None: + """规划失败只留痕，后续成功规划应清错并原子推进状态。""" + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + + repository.record_planning_failure(task_id=admitted.task_id, error="rename failed") + failed = repository.list_recoverable()[0] + assert failed.state == TRANSFER_ADMISSION_ACCEPTED + assert failed.last_error == "rename failed" + assert failed.checkpoint is None + + planned = repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + assert planned.state == TRANSFER_ADMISSION_PLANNED + assert planned.last_error is None + + +def test_checkpoint_rejects_missing_task(repository) -> None: + """不存在的稳定任务身份不能凭空创建已规划记录。""" + planning_input = _planning_input() + with pytest.raises(TransferPlanningStateError): + repository.checkpoint_plan( + task_id="missing", + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + + +def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None: + """兼容直接构造 ORM 行时也必须生成匹配路径的版本化输入与指纹。""" + engine = create_engine(f"sqlite:///{tmp_path / 'orm-defaults.db'}") + factory = sessionmaker(bind=engine) + TransferPending.__table__.create(engine) + with factory() as session: + session.add(TransferPending( + storage="local", + src_path="/downloads/legacy.mkv", + state=TRANSFER_ADMISSION_ACCEPTED, + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + )) + session.commit() + + admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0] + + assert admitted.planning_input == TransferPlanningInput.legacy( + storage="local", + src_path="/downloads/legacy.mkv", + ) + assert admitted.input_fingerprint == admitted.planning_input.fingerprint + engine.dispose() + + +def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -> None: + """列版本、JSON 和指纹任一不一致时都不得返回伪冻结 DTO。""" + engine = create_engine(f"sqlite:///{tmp_path / 'input-corruption.db'}") + factory = sessionmaker(bind=engine) + TransferPending.__table__.create(engine) + repository = TransactionalTransferAdmissionRepository(factory) + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + with factory() as session: + row = session.execute( + select(TransferPending).where(TransferPending.task_id == admitted.task_id) + ).scalar_one() + row.input_version = 2 + session.commit() + + with pytest.raises(TransferPlanningStateError, match="版本"): + repository.list_accepted() + + with factory() as session: + row = session.execute( + select(TransferPending).where(TransferPending.task_id == admitted.task_id) + ).scalar_one() + row.input_version = 1 + corrupted = planning_input.to_payload() + corrupted["media_id"] = "different" + row.planning_input = corrupted + session.commit() + + with pytest.raises(TransferAdmissionConflictError, match="指纹"): + repository.list_accepted() + engine.dispose() + + +def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None: + """planned 行的列版本与自包含 checkpoint JSON 必须严格一致。""" + engine = create_engine(f"sqlite:///{tmp_path / 'checkpoint-corruption.db'}") + factory = sessionmaker(bind=engine) + TransferPending.__table__.create(engine) + repository = TransactionalTransferAdmissionRepository(factory) + planning_input = _planning_input() + admitted = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=planning_input, + ) + repository.checkpoint_plan( + task_id=admitted.task_id, + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + with factory() as session: + row = session.execute( + select(TransferPending).where(TransferPending.task_id == admitted.task_id) + ).scalar_one() + row.checkpoint_version = 2 + session.commit() + + with pytest.raises(TransferPlanningStateError, match="版本"): + repository.list_recoverable() + engine.dispose() diff --git a/tests/test_transfer_preview.py b/tests/test_transfer_preview.py index 17855b1f3..6da4f12b7 100644 --- a/tests/test_transfer_preview.py +++ b/tests/test_transfer_preview.py @@ -1,6 +1,5 @@ from pathlib import Path - from app.domain.context import MediaInfo from app.domain.meta.metabase import MetaBase from app.modules.filemanager import FileManagerModule @@ -78,15 +77,22 @@ def test_cloud_storage_preview_only_calculates_target_path(): ) guarded_storage = GuardedStorage() - transferinfo = FileManagerModule().transfer( + module = FileManagerModule() + checkpoint = module.plan_transfer( fileitem=fileitem, meta=meta, mediainfo=mediainfo, target_directory=target_directory, source_oper=guarded_storage, - target_oper=guarded_storage, preview=True, ) + transferinfo = module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=guarded_storage, + target_oper=guarded_storage, + ) assert transferinfo.success is True assert transferinfo.need_notify is False @@ -131,15 +137,22 @@ def test_local_storage_preview_skips_target_conflict_checks(tmp_path): ) guarded_storage = GuardedStorage() - transferinfo = FileManagerModule().transfer( + module = FileManagerModule() + checkpoint = module.plan_transfer( fileitem=fileitem, meta=meta, mediainfo=mediainfo, target_directory=target_directory, source_oper=guarded_storage, - target_oper=guarded_storage, preview=True, ) + transferinfo = module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=guarded_storage, + target_oper=guarded_storage, + ) assert transferinfo.success is True assert transferinfo.need_notify is False @@ -188,15 +201,24 @@ def _build_bluray_dir_preview( notify=True, ) - return FileManagerModule().transfer( + module = FileManagerModule() + source_oper = GuardedStorage() + target_oper = GuardedStorage() + checkpoint = module.plan_transfer( fileitem=fileitem, meta=meta, mediainfo=mediainfo, target_directory=target_directory, - source_oper=GuardedStorage(), - target_oper=GuardedStorage(), + source_oper=source_oper, preview=True, ) + return module.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_oper=target_oper, + ) def test_tv_bluray_dir_preview_preserves_disk_folder_from_meta_part(): diff --git a/tests/test_transfer_sync_extra_files.py b/tests/test_transfer_sync_extra_files.py index 612733c57..b41265cdf 100644 --- a/tests/test_transfer_sync_extra_files.py +++ b/tests/test_transfer_sync_extra_files.py @@ -688,12 +688,11 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat assert planned == [(subtitle_fileitem.path, 2)] -def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeypatch): +def test_cleanup_dest_fileitem_is_checkpointed_only_after_allowed_items_exist(monkeypatch): """ - 旧目标文件只应在模板筛选后确实存在待整理任务时清理。 + 旧目标清理意图只应在模板筛选后确实存在待整理任务时进入规划输入。 """ chain = make_transfer_chain() - delete_calls = [] planned = [] main_fileitem = make_fileitem( "/downloads/Test Show (2026)/Show - 01.mkv" @@ -721,9 +720,14 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp def fake_handle_transfer(task, callback=None): """ - 记录旧目标清理后的整理任务。 + 记录实际任务携带的冻结 cleanup intent。 """ - planned.append(task.fileitem.path) + planned.append( + ( + task.fileitem.path, + task.planning_input.options.get("cleanup_dest_fileitem"), + ) + ) return True, "" monkeypatch.setattr(chain, "_TransferChain__handle_transfer", fake_handle_transfer) @@ -752,15 +756,6 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp lambda: SimpleNamespace(get=lambda key: None), ) monkeypatch.setattr("app.chain._transfer.get_configured_system_config", lambda: SimpleNamespace(get=lambda key: None)) - monkeypatch.setattr( - "app.chain.transfer.StorageChain", - lambda: SimpleNamespace( - delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, - ), - ) - monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace( - delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True, - )) monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1)) state, errmsg = TransferChain.do_transfer( @@ -773,8 +768,12 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp assert state is True assert errmsg == "" - assert delete_calls == [old_dest_fileitem.path] - assert planned == [main_fileitem.path] + assert planned == [ + ( + main_fileitem.path, + old_dest_fileitem.model_dump(mode="json"), + ) + ] def test_cleanup_dest_fileitem_is_kept_when_episode_format_matches_nothing(monkeypatch): diff --git a/tests/test_transfer_worker_lifecycle.py b/tests/test_transfer_worker_lifecycle.py index ccbe5f654..a4188c575 100644 --- a/tests/test_transfer_worker_lifecycle.py +++ b/tests/test_transfer_worker_lifecycle.py @@ -447,7 +447,14 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch) chain._processed_num = 0 chain._fail_num = 0 chain._total_num = 0 - chain._TransferChain__handle_transfer = MagicMock(return_value=(True, "")) + def complete_with_checkpoint(*, task, callback): + """模拟真实 worker 只有提交 checkpoint 后才返回终态成功。""" + task.bind_plan_checkpoint(MagicMock()) + return True, "" + + chain._TransferChain__handle_transfer = MagicMock( + side_effect=complete_with_checkpoint + ) monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) assert chain.put_to_queue(task) is True diff --git a/tests/test_transhandler_special_extra.py b/tests/test_transhandler_special_extra.py index 1b759a1b7..b92071cec 100644 --- a/tests/test_transhandler_special_extra.py +++ b/tests/test_transhandler_special_extra.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import pytest +from app.application.transfer import TransferPlanningInput from app.domain.context import MediaInfo from app.domain.meta.metavideo import MetaVideo from app.modules.filemanager.transhandler import TransHandler @@ -28,17 +29,53 @@ def _transfer_without_episode(file_name: str) -> tuple[TransferInfo, MagicMock, source_oper = MagicMock() target_oper = MagicMock() - result = TransHandler().transfer_media( - fileitem=fileitem, - in_meta=meta, - mediainfo=mediainfo, + handler = TransHandler() + planning_input = TransferPlanningInput( + source_fileitem=fileitem.model_dump(mode="json"), + meta=meta.to_dict(), + mediainfo=mediainfo.to_dict(), target_storage="local", - target_path=Path("/library"), - transfer_type="copy", - source_oper=source_oper, - target_oper=target_oper, + target_path="/library", + requested_transfer_type="copy", + media_type=MediaType.TV.value, + need_scrape=False, + need_rename=True, need_notify=True, + preview=False, ) + try: + checkpoint = handler.plan_transfer( + planning_input, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_storage="local", + target_path=Path("/library"), + transfer_type="copy", + need_scrape=False, + need_rename=True, + need_notify=True, + overwrite_mode=None, + episodes_info=None, + preview=False, + ) + except ValueError as error: + result = TransferInfo( + success=False, + message=str(error), + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type="copy", + need_notify=True, + ) + else: + result = handler.execute_transfer_plan( + checkpoint, + meta=meta, + mediainfo=mediainfo, + source_oper=source_oper, + target_oper=target_oper, + ) return result, source_oper, target_oper