refactor: add durable transfer planning checkpoints

This commit is contained in:
jxxghp
2026-08-27 13:16:33 +08:00
parent 49d30e5fdf
commit 23bff6b2bb
41 changed files with 7401 additions and 1098 deletions
+2 -1
View File
@@ -11,9 +11,9 @@ from app.application.chain.durable_events import ChainDurableEventWriter
from app.application.configuration import ChainRuntimeConfig
from app.runtime.stop import StopState, runtime_stop_state
MessageQueueFactory = Callable[[Callable[..., Any]], Any]
ModuleDispatcherFactory = Callable[..., Any]
LegacyTransferCommand = Callable[..., Any]
ChainRuntimeContextProvider = Callable[[], "ChainRuntimeContext"]
@@ -30,6 +30,7 @@ class ChainRuntimeContext:
async_file_cache: Any
message_queue_factory: MessageQueueFactory
module_dispatcher_factory: ModuleDispatcherFactory
legacy_transfer_command: Optional[LegacyTransferCommand] = None
data_ports: Optional[ChainDataPorts] = None
durable_event_writer: Optional[ChainDurableEventWriter] = None
configuration: ChainRuntimeConfig = field(
+638 -7
View File
@@ -14,12 +14,24 @@ TransferJob / TransferJobTask,那两个用 app.schemas 的同名 DTO——一
视图,分开表达之后两边都不必再迁就对方。
"""
import asyncio
import hashlib
import json
import threading
from copy import deepcopy
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path
from time import monotonic
from typing import Callable, Dict, List, Optional, Protocol, Tuple, Union
from typing import (
Any,
Callable,
Dict,
List,
Optional,
Protocol,
Tuple,
TypeAlias,
Union,
)
from pydantic import BaseModel, ConfigDict, PrivateAttr
@@ -49,6 +61,571 @@ from app.schemas.types import (
)
from app.schemas.workflow import MediaInfo as _SchemaMediaInfo
JSONValue: TypeAlias = Union[None, bool, int, float, str, list["JSONValue"], dict[str, "JSONValue"]]
TRANSFER_ADMISSION_ACCEPTED = "accepted"
TRANSFER_ADMISSION_PROVIDER_PENDING = "provider_pending"
TRANSFER_ADMISSION_PLANNED = "planned"
TRANSFER_PLANNING_INPUT_VERSION = 1
TRANSFER_PLAN_CHECKPOINT_VERSION = 1
TRANSFER_PROVIDER_INVOCATION_VERSION = 1
def _copy_json_mapping(value: Optional[dict[str, JSONValue]]) -> Optional[dict[str, JSONValue]]:
if value is None:
return None
return deepcopy(value)
def _read_json_mapping(payload: dict[str, Any], key: str) -> Optional[dict[str, JSONValue]]:
value = payload.get(key)
if value is None:
return None
if not isinstance(value, dict):
raise ValueError(f"整理计划字段 {key} 必须是 JSON 对象")
return deepcopy(value)
def _read_json_tuple(payload: dict[str, Any], key: str) -> tuple[dict[str, JSONValue], ...]:
value = payload.get(key, [])
if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
raise ValueError(f"整理计划字段 {key} 必须是 JSON 对象数组")
return tuple(deepcopy(item) for item in value)
def _canonical_json(payload: dict[str, JSONValue]) -> str:
try:
return json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
except (TypeError, ValueError) as error:
raise ValueError("整理计划只能包含有限 JSON 值") from error
@dataclass(frozen=True, slots=True)
class TransferPlanningInput:
"""保存可跨重启重放的版本化整理规划输入。"""
source_fileitem: dict[str, JSONValue]
meta: Optional[dict[str, JSONValue]] = None
mediainfo: Optional[dict[str, JSONValue]] = None
target_directory: Optional[dict[str, JSONValue]] = None
target_storage: Optional[str] = None
target_path: Optional[str] = None
requested_transfer_type: Optional[str] = None
media_source: Optional[str] = None
media_id: Optional[str] = None
media_type: Optional[str] = None
need_scrape: bool = False
need_rename: bool = True
need_notify: bool = True
overwrite_mode: Optional[str] = None
episodes_info: tuple[dict[str, JSONValue], ...] = ()
preview: bool = False
options: dict[str, JSONValue] = field(default_factory=dict)
schema_version: int = TRANSFER_PLANNING_INPUT_VERSION
def __post_init__(self) -> None:
"""拒绝不可恢复的输入版本或缺少源文件身份的快照。"""
if self.schema_version != TRANSFER_PLANNING_INPUT_VERSION:
raise ValueError(f"不支持的整理规划输入版本: {self.schema_version}")
object.__setattr__(self, "source_fileitem", deepcopy(self.source_fileitem))
object.__setattr__(self, "meta", _copy_json_mapping(self.meta))
object.__setattr__(self, "mediainfo", _copy_json_mapping(self.mediainfo))
object.__setattr__(
self,
"target_directory",
_copy_json_mapping(self.target_directory),
)
object.__setattr__(
self,
"episodes_info",
tuple(deepcopy(item) for item in self.episodes_info),
)
object.__setattr__(self, "options", deepcopy(self.options))
storage = self.source_fileitem.get("storage")
path = self.source_fileitem.get("path")
if not isinstance(storage, str) or not storage or not isinstance(path, str) or not path:
raise ValueError("整理规划输入缺少源文件存储或路径")
_canonical_json(self.to_payload())
@classmethod
def legacy(cls, *, storage: str, src_path: str) -> "TransferPlanningInput":
"""为升级前只有存储与路径的登记构造保守重规划输入。"""
return cls(
source_fileitem={"storage": storage, "path": src_path},
options={"legacy_replan": True},
)
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanningInput":
"""从受版本约束的 JSON 对象恢复规划输入。"""
if not isinstance(payload, dict):
raise ValueError("整理规划输入必须是 JSON 对象")
source_fileitem = _read_json_mapping(payload, "source_fileitem")
if source_fileitem is None:
raise ValueError("整理规划输入缺少 source_fileitem")
return cls(
source_fileitem=source_fileitem,
meta=_read_json_mapping(payload, "meta"),
mediainfo=_read_json_mapping(payload, "mediainfo"),
target_directory=_read_json_mapping(payload, "target_directory"),
target_storage=payload.get("target_storage"),
target_path=payload.get("target_path"),
requested_transfer_type=payload.get("requested_transfer_type"),
media_source=payload.get("media_source"),
media_id=payload.get("media_id"),
media_type=payload.get("media_type"),
need_scrape=payload.get("need_scrape", False),
need_rename=payload.get("need_rename", True),
need_notify=payload.get("need_notify", True),
overwrite_mode=payload.get("overwrite_mode"),
episodes_info=_read_json_tuple(payload, "episodes_info"),
preview=payload.get("preview", False),
options=_read_json_mapping(payload, "options") or {},
schema_version=payload.get("schema_version", 0),
)
def to_payload(self) -> dict[str, JSONValue]:
"""生成仅含 JSON 值且字段稳定的规划输入投影。"""
return {
"schema_version": self.schema_version,
"source_fileitem": deepcopy(self.source_fileitem),
"meta": _copy_json_mapping(self.meta),
"mediainfo": _copy_json_mapping(self.mediainfo),
"target_directory": _copy_json_mapping(self.target_directory),
"target_storage": self.target_storage,
"target_path": self.target_path,
"requested_transfer_type": self.requested_transfer_type,
"media_source": self.media_source,
"media_id": self.media_id,
"media_type": self.media_type,
"need_scrape": self.need_scrape,
"need_rename": self.need_rename,
"need_notify": self.need_notify,
"overwrite_mode": self.overwrite_mode,
"episodes_info": [deepcopy(item) for item in self.episodes_info],
"preview": self.preview,
"options": deepcopy(self.options),
}
@property
def fingerprint(self) -> str:
"""返回规范 JSON 的稳定 SHA-256 指纹。"""
return hashlib.sha256(_canonical_json(self.to_payload()).encode("utf-8")).hexdigest()
@dataclass(frozen=True, slots=True)
class TransferPlanItem:
"""描述规划后按序执行的一条叶子文件操作。"""
sequence: int
source_fileitem: dict[str, JSONValue]
target_storage: str
target_path: str
action: str = "transfer"
def __post_init__(self) -> None:
"""拒绝无法定位源文件或目标文件的计划项。"""
object.__setattr__(self, "source_fileitem", deepcopy(self.source_fileitem))
if self.sequence < 0:
raise ValueError("整理计划项序号不能小于零")
if not self.target_storage or not self.target_path or not self.action:
raise ValueError("整理计划项缺少目标身份或动作")
if not self.source_fileitem.get("storage") or not self.source_fileitem.get("path"):
raise ValueError("整理计划项缺少源文件身份")
_canonical_json(self.to_payload())
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanItem":
"""从 JSON 对象恢复单条叶子文件操作。"""
source_fileitem = _read_json_mapping(payload, "source_fileitem")
if source_fileitem is None:
raise ValueError("整理计划项缺少 source_fileitem")
return cls(
sequence=payload.get("sequence", -1),
source_fileitem=source_fileitem,
target_storage=payload.get("target_storage", ""),
target_path=payload.get("target_path", ""),
action=payload.get("action", "transfer"),
)
def to_payload(self) -> dict[str, JSONValue]:
"""生成单条叶子文件操作的 JSON 投影。"""
return {
"sequence": self.sequence,
"source_fileitem": deepcopy(self.source_fileitem),
"target_storage": self.target_storage,
"target_path": self.target_path,
"action": self.action,
}
@dataclass(frozen=True, slots=True)
class TransferProviderReference:
"""保存无需依赖运行时 dispatcher 即可持久化的旧 transfer provider 引用。"""
plugin_id: str
plugin_name: str
method: str = "transfer"
def __post_init__(self) -> None:
"""拒绝无法稳定定位插件或指向非 transfer 方法的引用。"""
if not isinstance(self.plugin_id, str) or not self.plugin_id.strip():
raise ValueError("旧 transfer provider 的 plugin_id 必须是非空字符串")
if not isinstance(self.plugin_name, str) or not self.plugin_name.strip():
raise ValueError("旧 transfer provider 的 plugin_name 必须是非空字符串")
if self.method != "transfer":
raise ValueError("旧 transfer provider 的 method 必须是 transfer")
_canonical_json(self.to_payload())
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferProviderReference":
"""从 JSON 对象恢复旧 transfer provider 引用。"""
if not isinstance(payload, dict):
raise ValueError("旧 transfer provider 引用必须是 JSON 对象")
return cls(
plugin_id=payload.get("plugin_id", ""),
plugin_name=payload.get("plugin_name", ""),
method=payload.get("method", "transfer"),
)
def to_payload(self) -> dict[str, JSONValue]:
"""生成仅包含稳定插件身份与方法名的 JSON 投影。"""
return {
"plugin_id": self.plugin_id,
"plugin_name": self.plugin_name,
"method": self.method,
}
@dataclass(frozen=True, slots=True)
class TransferProviderInvocationSnapshot:
"""冻结旧 transfer ABI 中可跨重启恢复的精确参数值。"""
fileitem: dict[str, JSONValue]
meta: Optional[dict[str, JSONValue]]
meta_kind: Optional[str]
mediainfo: Optional[dict[str, JSONValue]]
mediainfo_kind: Optional[str]
target_directory: Optional[dict[str, JSONValue]] = None
target_storage: Optional[str] = None
target_path: Optional[str] = None
transfer_type: Optional[str] = None
scrape: Optional[bool] = None
library_type_folder: Optional[bool] = None
library_category_folder: Optional[bool] = None
episodes_info: tuple[dict[str, JSONValue], ...] = ()
preview: bool = False
schema_version: int = TRANSFER_PROVIDER_INVOCATION_VERSION
def __post_init__(self) -> None:
"""拒绝缺少源身份、类型不稳定或版本未知的 provider 快照。"""
if self.schema_version != TRANSFER_PROVIDER_INVOCATION_VERSION:
raise ValueError(
f"不支持的旧 transfer provider 调用快照版本: {self.schema_version}"
)
object.__setattr__(self, "fileitem", deepcopy(self.fileitem))
object.__setattr__(self, "meta", _copy_json_mapping(self.meta))
object.__setattr__(self, "mediainfo", _copy_json_mapping(self.mediainfo))
object.__setattr__(
self,
"target_directory",
_copy_json_mapping(self.target_directory),
)
object.__setattr__(
self,
"episodes_info",
tuple(deepcopy(item) for item in self.episodes_info),
)
if not self.fileitem.get("storage") or not self.fileitem.get("path"):
raise ValueError("旧 transfer provider 调用快照缺少源文件身份")
for kind in (self.meta_kind, self.mediainfo_kind):
if kind is not None and (not isinstance(kind, str) or not kind):
raise ValueError("旧 transfer provider 调用快照类型必须是非空字符串")
for value in (
self.target_storage,
self.target_path,
self.transfer_type,
):
if value is not None and not isinstance(value, str):
raise ValueError("旧 transfer provider 可选文本参数必须是字符串或 None")
for bool_value in (
self.scrape,
self.library_type_folder,
self.library_category_folder,
):
if bool_value is not None and not isinstance(bool_value, bool):
raise ValueError("旧 transfer provider 可选布尔参数必须是 bool 或 None")
if not isinstance(self.preview, bool):
raise ValueError("旧 transfer provider preview 参数必须是 bool")
_canonical_json(self.to_payload())
@classmethod
def from_payload(
cls,
payload: dict[str, Any],
) -> "TransferProviderInvocationSnapshot":
"""从版本化 JSON 对象严格恢复旧 transfer ABI 调用快照。"""
if not isinstance(payload, dict):
raise ValueError("旧 transfer provider 调用快照必须是 JSON 对象")
fileitem = _read_json_mapping(payload, "fileitem")
if fileitem is None:
raise ValueError("旧 transfer provider 调用快照缺少 fileitem")
return cls(
fileitem=fileitem,
meta=_read_json_mapping(payload, "meta"),
meta_kind=payload.get("meta_kind"),
mediainfo=_read_json_mapping(payload, "mediainfo"),
mediainfo_kind=payload.get("mediainfo_kind"),
target_directory=_read_json_mapping(payload, "target_directory"),
target_storage=payload.get("target_storage"),
target_path=payload.get("target_path"),
transfer_type=payload.get("transfer_type"),
scrape=payload.get("scrape"),
library_type_folder=payload.get("library_type_folder"),
library_category_folder=payload.get("library_category_folder"),
episodes_info=_read_json_tuple(payload, "episodes_info"),
preview=payload.get("preview", False),
schema_version=payload.get("schema_version", 0),
)
def to_payload(self) -> dict[str, JSONValue]:
"""生成仅含 JSON 值且保留 None 与 False 差异的调用投影。"""
return {
"schema_version": self.schema_version,
"fileitem": deepcopy(self.fileitem),
"meta": _copy_json_mapping(self.meta),
"meta_kind": self.meta_kind,
"mediainfo": _copy_json_mapping(self.mediainfo),
"mediainfo_kind": self.mediainfo_kind,
"target_directory": _copy_json_mapping(self.target_directory),
"target_storage": self.target_storage,
"target_path": self.target_path,
"transfer_type": self.transfer_type,
"scrape": self.scrape,
"library_type_folder": self.library_type_folder,
"library_category_folder": self.library_category_folder,
"episodes_info": [deepcopy(item) for item in self.episodes_info],
"preview": self.preview,
}
@dataclass(frozen=True, slots=True)
class TransferPlanCheckpoint:
"""保存无需重触发识别或重命名即可执行的完整有序计划。"""
planning_input: TransferPlanningInput
target_storage: str
root_target_path: str
final_target_path: str
resolved_transfer_type: str
items: tuple[TransferPlanItem, ...]
resolved_meta: Optional[dict[str, JSONValue]] = None
resolved_meta_kind: Optional[str] = None
resolved_mediainfo: Optional[dict[str, JSONValue]] = None
resolved_mediainfo_kind: Optional[str] = None
resolved_episodes_info: tuple[dict[str, JSONValue], ...] = ()
legacy_transfer_providers: tuple[TransferProviderReference, ...] = ()
provider_invocation: Optional[TransferProviderInvocationSnapshot] = None
pre_execution_cleanup_completed: bool = False
need_scrape: bool = False
need_rename: bool = False
need_notify: bool = True
overwrite_mode: Optional[str] = None
preview: bool = False
skip_reason: Optional[str] = None
schema_version: int = TRANSFER_PLAN_CHECKPOINT_VERSION
def __post_init__(self) -> None:
"""验证版本、目标身份和计划项顺序组成完整检查点。"""
if self.schema_version != TRANSFER_PLAN_CHECKPOINT_VERSION:
raise ValueError(f"不支持的整理计划检查点版本: {self.schema_version}")
if not isinstance(self.pre_execution_cleanup_completed, bool):
raise ValueError("整理计划预执行 cleanup 完成标记必须是 bool")
object.__setattr__(self, "resolved_meta", _copy_json_mapping(self.resolved_meta))
object.__setattr__(
self,
"resolved_mediainfo",
_copy_json_mapping(self.resolved_mediainfo),
)
object.__setattr__(
self,
"resolved_episodes_info",
tuple(deepcopy(item) for item in self.resolved_episodes_info),
)
if not isinstance(self.legacy_transfer_providers, tuple) or any(
not isinstance(provider, TransferProviderReference)
for provider in self.legacy_transfer_providers
):
raise ValueError("整理计划的旧 transfer provider 引用必须是类型化元组")
provider_ids = tuple(
provider.plugin_id for provider in self.legacy_transfer_providers
)
if len(set(provider_ids)) != len(provider_ids):
raise ValueError("整理计划的旧 transfer provider plugin_id 不能重复")
for resolved_kind in (
self.resolved_meta_kind,
self.resolved_mediainfo_kind,
):
if resolved_kind is not None and (
not isinstance(resolved_kind, str) or not resolved_kind
):
raise ValueError("整理计划的已解析上下文类型必须是非空字符串")
if self.provider_invocation is not None:
if not self.legacy_transfer_providers:
raise ValueError("provider_pending 检查点缺少冻结 provider")
if self.pre_execution_cleanup_completed:
raise ValueError("provider_pending 提交时 cleanup 尚未执行")
if (
not self.provider_invocation.meta
or not self.provider_invocation.meta_kind
or not self.provider_invocation.mediainfo
or not self.provider_invocation.mediainfo_kind
):
raise ValueError("provider_pending 检查点缺少可重放的媒体上下文")
if (
self.target_storage
or self.root_target_path
or self.final_target_path
or self.resolved_transfer_type
or self.items
or self.skip_reason
):
raise ValueError("provider_pending 检查点不得包含宿主执行计划")
else:
if not self.target_storage or not self.root_target_path or not self.final_target_path:
raise ValueError("整理计划检查点缺少目标身份")
if not self.resolved_transfer_type:
raise ValueError("整理计划检查点缺少已解析的整理方式")
if tuple(item.sequence for item in self.items) != tuple(range(len(self.items))):
raise ValueError("整理计划项必须按从零开始的连续序号保存")
if (
self.provider_invocation is None
and not self.items
and not self.preview
and not self.skip_reason
):
raise ValueError("非预览空计划必须记录合法跳过原因")
_canonical_json(self.to_payload())
@property
def is_provider_pending(self) -> bool:
"""返回该检查点是否只冻结 provider 调用、尚未生成宿主计划。"""
return self.provider_invocation is not None
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanCheckpoint":
"""从受版本约束的 JSON 对象恢复完整执行检查点。"""
if not isinstance(payload, dict):
raise ValueError("整理计划检查点必须是 JSON 对象")
planning_input_payload = _read_json_mapping(payload, "planning_input")
if planning_input_payload is None:
raise ValueError("整理计划检查点缺少 planning_input")
item_payloads = payload.get("items", [])
if not isinstance(item_payloads, list) or not all(
isinstance(item, dict) for item in item_payloads
):
raise ValueError("整理计划检查点 items 必须是 JSON 对象数组")
legacy_provider_payloads = payload.get("legacy_transfer_providers", [])
if not isinstance(legacy_provider_payloads, list):
raise ValueError(
"整理计划检查点 legacy_transfer_providers 必须是 JSON 对象数组"
)
provider_invocation_payload = payload.get("provider_invocation")
if provider_invocation_payload is not None and not isinstance(
provider_invocation_payload,
dict,
):
raise ValueError("整理计划检查点 provider_invocation 必须是 JSON 对象")
return cls(
planning_input=TransferPlanningInput.from_payload(planning_input_payload),
target_storage=payload.get("target_storage", ""),
root_target_path=payload.get("root_target_path", ""),
final_target_path=payload.get("final_target_path", ""),
resolved_transfer_type=payload.get("resolved_transfer_type", ""),
items=tuple(TransferPlanItem.from_payload(item) for item in item_payloads),
resolved_meta=_read_json_mapping(payload, "resolved_meta"),
resolved_meta_kind=payload.get("resolved_meta_kind"),
resolved_mediainfo=_read_json_mapping(payload, "resolved_mediainfo"),
resolved_mediainfo_kind=payload.get("resolved_mediainfo_kind"),
resolved_episodes_info=_read_json_tuple(
payload,
"resolved_episodes_info",
),
legacy_transfer_providers=tuple(
TransferProviderReference.from_payload(provider)
for provider in legacy_provider_payloads
),
provider_invocation=(
TransferProviderInvocationSnapshot.from_payload(
provider_invocation_payload
)
if provider_invocation_payload is not None
else None
),
pre_execution_cleanup_completed=payload.get(
"pre_execution_cleanup_completed",
False,
),
need_scrape=payload.get("need_scrape", False),
need_rename=payload.get("need_rename", False),
need_notify=payload.get("need_notify", True),
overwrite_mode=payload.get("overwrite_mode"),
preview=payload.get("preview", False),
skip_reason=payload.get("skip_reason"),
schema_version=payload.get("schema_version", 0),
)
def to_payload(self) -> dict[str, JSONValue]:
"""生成可原子落库的完整版本化 JSON 检查点。"""
return {
"schema_version": self.schema_version,
"planning_input": self.planning_input.to_payload(),
"target_storage": self.target_storage,
"root_target_path": self.root_target_path,
"final_target_path": self.final_target_path,
"resolved_transfer_type": self.resolved_transfer_type,
"items": [item.to_payload() for item in self.items],
"resolved_meta": _copy_json_mapping(self.resolved_meta),
"resolved_meta_kind": self.resolved_meta_kind,
"resolved_mediainfo": _copy_json_mapping(self.resolved_mediainfo),
"resolved_mediainfo_kind": self.resolved_mediainfo_kind,
"resolved_episodes_info": [
deepcopy(item) for item in self.resolved_episodes_info
],
"legacy_transfer_providers": [
provider.to_payload()
for provider in self.legacy_transfer_providers
],
"provider_invocation": (
self.provider_invocation.to_payload()
if self.provider_invocation
else None
),
"pre_execution_cleanup_completed": (
self.pre_execution_cleanup_completed
),
"need_scrape": self.need_scrape,
"need_rename": self.need_rename,
"need_notify": self.need_notify,
"overwrite_mode": self.overwrite_mode,
"preview": self.preview,
"skip_reason": self.skip_reason,
}
class TransferAdmissionConflictError(ValueError):
"""同一源文件以不同规划输入重复准入时抛出的冲突错误。"""
class TransferPlanningStateError(RuntimeError):
"""计划检查点无法从当前持久状态推进时抛出的状态错误。"""
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""
@@ -81,6 +658,9 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
background: Optional[bool] = True
preview: Optional[bool] = False
_admission_task_id: Optional[str] = PrivateAttr(default=None)
_planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None)
_plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None)
_planning_context_restored: bool = PrivateAttr(default=False)
@property
def admission_task_id(self) -> Optional[str]:
@@ -91,6 +671,33 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""绑定持久准入生成的稳定身份,不改变插件可见序列化字段。"""
self._admission_task_id = task_id
@property
def planning_input(self) -> Optional[TransferPlanningInput]:
"""返回宿主恢复规划使用的内部输入快照。"""
return self._planning_input
@property
def plan_checkpoint(self) -> Optional[TransferPlanCheckpoint]:
"""返回宿主直接执行已规划任务使用的内部检查点。"""
return self._plan_checkpoint
def bind_planning_input(self, planning_input: TransferPlanningInput) -> None:
"""绑定持久规划输入,不改变插件可见序列化字段。"""
self._planning_input = planning_input
def bind_plan_checkpoint(self, checkpoint: TransferPlanCheckpoint) -> None:
"""绑定持久执行检查点,不改变插件可见序列化字段。"""
self._plan_checkpoint = checkpoint
@property
def planning_context_restored(self) -> bool:
"""返回当前领域上下文是否来自持久快照。"""
return self._planning_context_restored
def mark_planning_context_restored(self) -> None:
"""标记领域上下文已离线恢复,禁止旧流程再次在线补充。"""
self._planning_context_restored = True
def to_dict(self):
"""
返回字典。
@@ -122,9 +729,6 @@ class TransferQueue(BaseModel):
result: Optional[TransferInfo] = None
TRANSFER_ADMISSION_ACCEPTED = "accepted"
@dataclass(frozen=True, slots=True)
class TransferAdmission:
"""描述已经持久化、可在进程退出后恢复的整理任务准入事实。"""
@@ -136,13 +740,22 @@ class TransferAdmission:
created_at: str
updated_at: str
last_error: Optional[str] = None
input_fingerprint: Optional[str] = None
planning_input: Optional[TransferPlanningInput] = None
checkpoint: Optional[TransferPlanCheckpoint] = None
class TransferAdmissionRepository(Protocol):
"""整理任务 durable admission 所需的类型化持久化端口。"""
def admit(self, *, storage: str, src_path: str) -> TransferAdmission:
"""幂等登记源文件并返回稳定任务身份。"""
def admit(
self,
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
) -> TransferAdmission:
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
...
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
@@ -153,6 +766,24 @@ class TransferAdmissionRepository(Protocol):
"""记录内存队列接收失败,保留任务供后续恢复。"""
...
def checkpoint_plan(
self,
*,
task_id: str,
input_fingerprint: str,
checkpoint: TransferPlanCheckpoint,
) -> TransferAdmission:
"""原子保存完整计划并将匹配输入的任务推进到已规划。"""
...
def record_planning_failure(self, *, task_id: str, error: str) -> None:
"""记录规划失败但保留接纳状态供下次恢复重试。"""
...
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
"""按登记顺序返回接纳或已规划的可恢复任务。"""
...
def discard_task(self, *, task_id: str) -> int:
"""按稳定任务身份删除已经到达终态的登记。"""
...
+83 -6
View File
@@ -5,7 +5,7 @@ import traceback
from abc import ABCMeta
from collections.abc import Callable
from pathlib import Path
from typing import Any, Dict, List, Optional, Set, Tuple, Union
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union, cast
from app.application.chain.context import ChainRuntimeContext, get_chain_runtime_context
from app.application.chain.data import get_chain_data_ports
@@ -15,7 +15,7 @@ from app.application.configuration import (
)
from app.chain._messaging import MessageProcessingMixin, NotificationMixin
from app.chain._recognition import RecognitionMixin
from app.domain.context import Context, MediaInfo, SubtitleInfo, TorrentInfo
from app.domain.context import Context, MediaInfo, MusicInfo, SubtitleInfo, TorrentInfo
from app.domain.meta.metabase import MetaBase
from app.runtime.log import logger
from app.schemas.category import CategoryConfig
@@ -35,6 +35,9 @@ from app.schemas.types import (
)
from app.schemas.workflow import FileItem
if TYPE_CHECKING:
from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput
class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
metaclass=ABCMeta):
@@ -65,6 +68,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
system_error_handler=self.__handle_system_error,
rate_limit_handler=self.__handle_rate_limit_error,
)
self._legacy_transfer_command = context.legacy_transfer_command
self.messagequeue = context.message_queue_factory(self.run_module)
@property
@@ -216,6 +220,15 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
"""
return self._module_dispatcher.dispatch(method, *args, **kwargs)
def run_module_strict(
self,
method: str,
*args: Any,
**kwargs: Any,
) -> Any:
"""运行模块并传播 provider 异常,供必须区分空结果与查询失败的能力使用。"""
return self._module_dispatcher.dispatch_strict(method, *args, **kwargs)
async def async_run_module(
self,
method: str,
@@ -854,7 +867,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
self,
fileitem: FileItem,
meta: MetaBase,
mediainfo: MediaInfo,
mediainfo: Union[MediaInfo, MusicInfo],
target_directory: TransferDirectoryConf = None,
target_storage: Optional[str] = None,
target_path: Path = None,
@@ -868,7 +881,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
preview: bool = False,
) -> Optional[TransferInfo]:
"""
文件转移
经启动组合根注入的 canonical durable command 执行旧整理 ABI。
:param fileitem: 文件信息
:param meta: 预识别的元数据
:param mediainfo: 识别的媒体信息
@@ -885,8 +898,9 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
:param preview: 是否仅预览,不执行实际转移
:return: {path, target_path, message}
"""
return self.run_module(
"transfer",
if self._legacy_transfer_command is None:
raise RuntimeError("旧整理兼容命令尚未由启动组合根配置")
return self._legacy_transfer_command(
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
@@ -903,6 +917,69 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
preview=preview,
)
def plan_transfer(
self,
fileitem: FileItem,
meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo],
target_directory: Optional[TransferDirectoryConf] = None,
target_storage: Optional[str] = None,
target_path: Optional[Path] = None,
transfer_type: Optional[str] = None,
scrape: Optional[bool] = None,
library_type_folder: Optional[bool] = None,
library_category_folder: Optional[bool] = None,
episodes_info: Optional[List[TmdbEpisode]] = None,
source_oper: Any = None,
preview: bool = False,
planning_input: Optional[TransferPlanningInput] = None,
) -> Optional[TransferPlanCheckpoint]:
"""调用文件管理模块生成无文件写入的冻结整理计划。"""
return cast(
"Optional[TransferPlanCheckpoint]",
self.run_module(
"plan_transfer",
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
target_path=target_path,
target_storage=target_storage,
transfer_type=transfer_type,
scrape=scrape,
library_type_folder=library_type_folder,
library_category_folder=library_category_folder,
episodes_info=episodes_info,
source_oper=source_oper,
preview=preview,
planning_input=planning_input,
),
)
def execute_transfer_plan(
self,
checkpoint: TransferPlanCheckpoint,
*,
meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo],
source_oper: Any = None,
target_oper: Any = None,
cleanup_media_file: Optional[Callable[[FileItem], bool]] = None,
) -> Optional[TransferInfo]:
"""调用文件管理模块执行已冻结计划,并注入统一安全删除能力。"""
return cast(
Optional[TransferInfo],
self.run_module(
"execute_transfer_plan",
checkpoint=checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_oper=target_oper,
cleanup_media_file=cleanup_media_file,
),
)
def transfer_completed(self, hashs: str, downloader: Optional[str] = None) -> None:
"""
下载器转移完成后的处理
+16 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, cast
from app.application.directory import DirectoryHelper
from app.chain import ChainBase
@@ -99,6 +99,21 @@ class StorageChain(ChainBase):
"""
return self.run_module("get_file_item", storage=storage, path=path)
def get_file_item_strict(
self,
storage: str,
path: Path,
) -> Optional[_SchemaFileItem]:
"""严格查询文件项:确认不存在返回空,provider 或 I/O 失败直接抛出。"""
return cast(
Optional[_SchemaFileItem],
self.run_module_strict(
"get_file_item",
storage=storage,
path=path,
),
)
def get_parent_item(self, fileitem: _SchemaFileItem) -> Optional[_SchemaFileItem]:
"""
获取上级目录项
+876 -88
View File
File diff suppressed because it is too large Load Diff
+163 -2
View File
@@ -2,6 +2,7 @@
from collections.abc import Callable
from datetime import datetime
from typing import Optional
from uuid import uuid4
from sqlalchemy.exc import IntegrityError
@@ -9,7 +10,13 @@ from sqlalchemy.orm import Session
from app.application.transfer import (
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferAdmission,
TransferAdmissionConflictError,
TransferPlanCheckpoint,
TransferPlanningInput,
TransferPlanningStateError,
)
from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
@@ -32,6 +39,40 @@ class TransactionalTransferAdmissionRepository:
def _project(pending: TransferPending) -> TransferAdmission:
"""在 Session 有效期内把 ORM 行冻结为应用层 DTO。"""
created_at = pending.created_at or pending.updated_at
planning_input = TransferPlanningInput.from_payload(pending.planning_input)
if pending.input_version != planning_input.schema_version:
raise TransferPlanningStateError("整理规划输入列版本与 JSON 版本不一致")
if pending.input_fingerprint != planning_input.fingerprint:
raise TransferAdmissionConflictError("整理规划输入 JSON 与持久指纹不一致")
checkpoint = (
TransferPlanCheckpoint.from_payload(pending.checkpoint_payload)
if pending.checkpoint_payload is not None
else None
)
if checkpoint is not None:
if pending.checkpoint_version != checkpoint.schema_version:
raise TransferPlanningStateError("整理检查点列版本与 JSON 版本不一致")
if checkpoint.planning_input.fingerprint != pending.input_fingerprint:
raise TransferAdmissionConflictError("整理检查点内嵌输入与准入指纹不一致")
if pending.state in {
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
} and checkpoint is None:
raise TransferPlanningStateError("待执行任务缺少完整检查点")
if pending.state == TRANSFER_ADMISSION_ACCEPTED and checkpoint is not None:
raise TransferPlanningStateError("接纳态任务不能携带计划检查点")
if (
pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING
and checkpoint is not None
and not checkpoint.is_provider_pending
):
raise TransferPlanningStateError("provider_pending 状态缺少 provider 调用快照")
if (
pending.state == TRANSFER_ADMISSION_PLANNED
and checkpoint is not None
and checkpoint.is_provider_pending
):
raise TransferPlanningStateError("planned 状态不能携带 provider-only 检查点")
return TransferAdmission(
task_id=pending.task_id,
storage=pending.storage,
@@ -40,10 +81,39 @@ class TransactionalTransferAdmissionRepository:
created_at=created_at,
updated_at=pending.updated_at,
last_error=pending.last_error,
input_fingerprint=pending.input_fingerprint,
planning_input=planning_input,
checkpoint=checkpoint,
)
def admit(self, *, storage: str, src_path: str) -> TransferAdmission:
"""幂等持久化准入事实,并返回跨重启稳定的任务标识。"""
@staticmethod
def _assert_input_match(
pending: TransferPending,
planning_input: TransferPlanningInput,
) -> None:
"""拒绝同一源文件以不同规划输入复用既有任务身份。"""
if pending.input_fingerprint != planning_input.fingerprint:
raise TransferAdmissionConflictError(
f"整理源文件已按不同输入准入: {pending.storage}:{pending.src_path}"
)
def admit(
self,
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
) -> TransferAdmission:
"""按输入指纹幂等持久化准入事实,并返回跨重启稳定身份。"""
effective_input = planning_input or TransferPlanningInput.legacy(
storage=storage,
src_path=src_path,
)
if (
effective_input.source_fileitem.get("storage") != storage
or effective_input.source_fileitem.get("path") != src_path
):
raise ValueError("整理规划输入的源文件身份与准入参数不一致")
now_time = self._now()
try:
with self._session_factory() as session:
@@ -55,10 +125,14 @@ class TransactionalTransferAdmissionRepository:
src_path=src_path,
state=TRANSFER_ADMISSION_ACCEPTED,
now_time=now_time,
input_version=effective_input.schema_version,
planning_input=effective_input.to_payload(),
input_fingerprint=effective_input.fingerprint,
)
if pending is None:
raise ValueError("整理任务的存储与源路径不能为空")
session.flush()
self._assert_input_match(pending, effective_input)
admission = self._project(pending)
transaction.commit()
return admission
@@ -74,6 +148,7 @@ class TransactionalTransferAdmissionRepository:
)
if pending is None:
raise RuntimeError("并发准入冲突后未找到已提交记录") from error
self._assert_input_match(pending, effective_input)
return self._project(pending)
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
@@ -85,6 +160,19 @@ class TransactionalTransferAdmissionRepository:
)
return [self._project(pending) for pending in pending_items]
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
"""投影接纳、provider 待执行或已规划的全部可恢复任务。"""
with self._session_factory() as session:
pending_items = TransferPendingOper(db=session).list_by_states(
states=(
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
),
limit=limit,
)
return [self._project(pending) for pending in pending_items]
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
"""独立提交最近一次入队失败,保留准入记录供后续恢复。"""
with self._session_factory() as session:
@@ -100,6 +188,79 @@ class TransactionalTransferAdmissionRepository:
transaction.rollback()
raise
def checkpoint_plan(
self,
*,
task_id: str,
input_fingerprint: str,
checkpoint: TransferPlanCheckpoint,
) -> TransferAdmission:
"""以输入指纹 CAS 保存 provider 调用快照或升级宿主计划。"""
if checkpoint.planning_input.fingerprint != input_fingerprint:
raise TransferAdmissionConflictError("检查点输入与准入输入指纹不一致")
checkpoint_payload = checkpoint.to_payload()
target_state = (
TRANSFER_ADMISSION_PROVIDER_PENDING
if checkpoint.is_provider_pending
else TRANSFER_ADMISSION_PLANNED
)
source_states = (
(TRANSFER_ADMISSION_ACCEPTED,)
if checkpoint.is_provider_pending
else (
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
)
)
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
oper = TransferPendingOper(db=session)
updated = oper.stage_checkpoint_plan(
task_id=task_id,
input_fingerprint=input_fingerprint,
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
now_time=self._now(),
)
session.flush()
session.expire_all()
pending = oper.get_by_task_id(task_id=task_id)
if pending is None:
raise TransferPlanningStateError(f"未找到整理任务: {task_id}")
if pending.input_fingerprint != input_fingerprint:
raise TransferAdmissionConflictError("整理任务输入指纹已经改变")
if not updated and not (
pending.state == target_state
and pending.checkpoint_payload == checkpoint_payload
):
raise TransferPlanningStateError(
f"整理任务不能从状态 {pending.state} 保存 {target_state} 检查点"
)
admission = self._project(pending)
transaction.commit()
return admission
except Exception:
transaction.rollback()
raise
def record_planning_failure(self, *, task_id: str, error: str) -> None:
"""独立提交规划错误并保持任务处于接纳态供恢复重试。"""
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
TransferPendingOper(db=session).stage_record_planning_failure(
task_id=task_id,
error=error,
now_time=self._now(),
)
transaction.commit()
except Exception:
transaction.rollback()
raise
def discard_task(self, *, task_id: str) -> int:
"""在独立事务中按稳定任务标识删除已到终态的准入记录。"""
with self._session_factory() as session:
+192 -6
View File
@@ -1,13 +1,66 @@
import hashlib
import json
from datetime import datetime
from typing import List, Optional, cast
from typing import Any, List, Optional, cast
from uuid import uuid4
from sqlalchemy import Index, String, Text, UniqueConstraint, delete, select, update
from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import Base, execute_dml, get_id_column
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _planning_fingerprint(payload: dict[str, Any]) -> str:
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _default_planning_payload(context: Any) -> dict[str, Any]:
params = context.get_current_parameters()
return _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
def _default_planning_fingerprint(context: Any) -> str:
params = context.get_current_parameters()
payload = params.get("planning_input") or _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
return _planning_fingerprint(payload)
class TransferPending(Base):
"""
待整理文件登记。
@@ -17,9 +70,9 @@ class TransferPending(Base):
蒸发。而已经稳定落地的文件不会再产生任何监控事件,也不会有新的补偿扫描起点
——结果就是永久漏件,只能靠人工比对补整理。
这里只落盘恢复所需的最小事实:稳定任务身份、存储、源文件路径、准入状态和
最近入队错误。重启后重新走一遍整理入口,由整理历史查重挡掉已经完成的,
因此不需要序列化 meta/mediainfo 这些重对象,也不存在识别结果陈旧的问题
准入时保存版本化规划输入和指纹;纯规划完成后以同一行原子保存完整有序计划并
推进到 planned。重启恢复可直接消费已规划路径,避免再次触发 rename 等插件事件。
旧路径登记接口仍生成最小 legacy_replan 输入,供插件兼容调用方继续使用
"""
id = get_id_column()
@@ -42,6 +95,22 @@ class TransferPending(Base):
)
# 最近一次入队失败原因
last_error: Mapped[Optional[str]] = mapped_column(Text)
# 规划输入格式版本
input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
# 版本化规划输入 JSON
planning_input: Mapped[dict[str, Any]] = mapped_column(
JSON, nullable=False, default=_default_planning_payload
)
# 规划输入规范 JSON 的 SHA-256 指纹
input_fingerprint: Mapped[str] = mapped_column(
String(64), nullable=False, default=_default_planning_fingerprint
)
# 完整计划格式版本,尚未规划时为空
checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer)
# 完整有序计划 JSON,尚未规划时为空
checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
# 规划完成时间
planned_at: Mapped[Optional[str]] = mapped_column(String(40))
__table_args__ = (
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
@@ -74,12 +143,16 @@ class TransferPending(Base):
).scalars().first()
if pending:
return cast("TransferPending", pending)
planning_input = _legacy_planning_payload(storage, src_path)
pending = cls(
storage=storage,
src_path=src_path,
state="accepted",
created_at=now_time,
updated_at=now_time,
input_version=1,
planning_input=planning_input,
input_fingerprint=_planning_fingerprint(planning_input),
)
db.add(pending)
return pending
@@ -87,7 +160,9 @@ class TransferPending(Base):
@classmethod
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
src_path: str, state: str,
now_time: str) -> Optional["TransferPending"]:
now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]:
"""
在调用方会话中暂存一条持久接纳记录。
@@ -98,6 +173,9 @@ class TransferPending(Base):
:param src_path: 源文件路径
:param state: 持久状态
:param now_time: 当前时间
:param input_version: 规划输入格式版本
:param planning_input: 版本化规划输入 JSON
:param input_fingerprint: 规划输入规范 JSON 指纹
:return: 接纳记录
"""
if not task_id or not storage or not src_path or not state:
@@ -107,6 +185,8 @@ class TransferPending(Base):
).scalars().first()
if pending:
return cast("TransferPending", pending)
effective_input = planning_input or _legacy_planning_payload(storage, src_path)
effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input)
pending = cls(
task_id=task_id,
storage=storage,
@@ -114,6 +194,9 @@ class TransferPending(Base):
state=state,
created_at=now_time,
updated_at=now_time,
input_version=input_version,
planning_input=effective_input,
input_fingerprint=effective_fingerprint,
)
db.add(pending)
return pending
@@ -137,6 +220,25 @@ class TransferPending(Base):
.limit(limit)
).scalars().all())
@classmethod
def list_by_states(cls, db: Session, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
按登记顺序列出多个可恢复持久状态的记录。
:param db: 数据库会话
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: 接纳记录列表
"""
if not states:
return []
return list(db.execute(
select(cls)
.where(cls.state.in_(states))
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def get_by_identity(cls, db: Session, *, storage: str,
src_path: str) -> Optional["TransferPending"]:
@@ -159,6 +261,90 @@ class TransferPending(Base):
).scalars().first(),
)
@classmethod
def get_by_task_id(cls, db: Session, *, task_id: str) -> Optional["TransferPending"]:
"""
按稳定任务标识查询一条持久登记。
:param db: 数据库会话
:param task_id: 稳定任务标识
:return: 接纳记录
"""
if not task_id:
return None
return cast(
Optional["TransferPending"],
db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(),
)
@classmethod
def checkpoint_plan(cls, db: Session, *, task_id: str,
input_fingerprint: str, checkpoint_version: int,
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...], target_state: str,
now_time: str) -> int:
"""
以输入指纹为 CAS 条件原子保存计划并推进到已规划。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param input_fingerprint: 规划输入规范 JSON 指纹
:param checkpoint_version: 检查点格式版本
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许推进检查点的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:return: 更新的记录数
"""
if (
not task_id
or not input_fingerprint
or not checkpoint_payload
or not source_states
or not target_state
):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(source_states),
cls.input_fingerprint == input_fingerprint,
)
.values(
state=target_state,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
planned_at=now_time,
last_error=None,
updated_at=now_time,
),
execution_options={"synchronize_session": False},
)
@classmethod
def record_planning_failure(cls, db: Session, *, task_id: str,
error: str, now_time: str) -> int:
"""
为接纳态或 provider 待执行任务记录规划失败,不改变其恢复状态。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param error: 失败原因
:param now_time: 当前时间
:return: 更新的记录数
"""
if not task_id:
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(("accepted", "provider_pending")),
)
.values(last_error=error, updated_at=now_time),
execution_options={"synchronize_session": False},
)
@classmethod
def record_enqueue_failure(cls, db: Session, *, task_id: str,
error: str, now_time: str) -> int:
+92 -2
View File
@@ -1,5 +1,5 @@
from datetime import datetime
from typing import List, Optional, Tuple
from typing import Any, List, Optional, Tuple
from app.db.base import DbOper
from app.db.models.transferpending import TransferPending
@@ -32,7 +32,9 @@ class TransferPendingOper(DbOper):
)
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
state: str, now_time: str) -> Optional[TransferPending]:
state: str, now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional[TransferPending]:
"""
在当前会话中暂存一条持久接纳记录。
@@ -42,6 +44,9 @@ class TransferPendingOper(DbOper):
:param src_path: 源文件路径
:param state: 持久状态
:param now_time: 当前时间
:param input_version: 规划输入格式版本
:param planning_input: 版本化规划输入 JSON
:param input_fingerprint: 规划输入规范 JSON 指纹
:return: 接纳记录
"""
return self._execute_sync_write(
@@ -52,6 +57,9 @@ class TransferPendingOper(DbOper):
src_path=src_path,
state=state,
now_time=now_time,
input_version=input_version,
planning_input=planning_input,
input_fingerprint=input_fingerprint,
)
)
@@ -71,6 +79,22 @@ class TransferPendingOper(DbOper):
)
) or []
def list_by_states(self, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List[TransferPending]:
"""
使用当前会话列出多个可恢复状态的记录。
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_by_states(
session,
states=states,
limit=limit,
)
) or []
def get_by_identity(self, *, storage: str,
src_path: str) -> Optional[TransferPending]:
"""
@@ -87,6 +111,72 @@ class TransferPendingOper(DbOper):
)
)
def get_by_task_id(self, *, task_id: str) -> Optional[TransferPending]:
"""
使用当前会话按稳定任务标识查询接纳记录。
:param task_id: 稳定任务标识
:return: 接纳记录
"""
return self._execute_sync_query(
lambda session: TransferPending.get_by_task_id(
session,
task_id=task_id,
)
)
def stage_checkpoint_plan(
self,
*,
task_id: str,
input_fingerprint: str,
checkpoint_version: int,
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...],
target_state: str,
now_time: str,
) -> int:
"""
在当前会话中以输入指纹为条件暂存完整计划检查点。
:param task_id: 稳定任务标识
:param input_fingerprint: 规划输入规范 JSON 指纹
:param checkpoint_version: 检查点格式版本
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许执行 CAS 的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.checkpoint_plan(
session,
task_id=task_id,
input_fingerprint=input_fingerprint,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
now_time=now_time,
)
)
def stage_record_planning_failure(self, *, task_id: str, error: str,
now_time: str) -> int:
"""
在当前会话中记录规划失败并保持任务处于接纳态。
:param task_id: 稳定任务标识
:param error: 失败原因
:param now_time: 当前时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.record_planning_failure(
session,
task_id=task_id,
error=error,
now_time=now_time,
)
)
def stage_record_enqueue_failure(self, *, task_id: str, error: str,
now_time: str) -> int:
"""
+168 -78
View File
@@ -1,28 +1,28 @@
from pathlib import Path
from typing import Any, Optional, List, Tuple, Union, Dict, Callable
from app.runtime.settings import get_runtime_setting
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
from app.adapters.system.host import SystemUtils
from app.application.directory import DirectoryHelper
from app.application.messaging.message import MessageHelper
from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput
from app.domain.context import MediaInfo, MusicInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo
from app.application.directory import DirectoryHelper
from app.application.messaging.message import MessageHelper
from app.foundation import text as text_tools
from app.foundation.reflection import ModuleHelper
from app.runtime.log import logger
from app.modules import _ModuleBase
from app.modules.filemanager.storages import StorageBase
from app.modules.filemanager.transhandler import TransHandler
from app.schemas.transfer import TransferInfo
from app.schemas.mediaserver import ExistMediaInfo
from app.schemas.tmdb import TmdbEpisode
from app.schemas.system import TransferDirectoryConf
from app.schemas.workflow import FileItem
from app.runtime.log import logger
from app.runtime.settings import get_runtime_setting
from app.schemas.file import StorageUsage
from app.schemas.mediaserver import ExistMediaInfo
from app.schemas.system import TransferDirectoryConf
from app.schemas.tmdb import TmdbEpisode
from app.schemas.transfer import TransferInfo
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaType, ModuleType, OtherModulesType, StorageAction
from app.adapters.system.host import SystemUtils
from app.foundation import text as text_tools
from app.schemas.workflow import FileItem
class FileManagerModule(_ModuleBase):
@@ -345,7 +345,7 @@ class FileManagerModule(_ModuleBase):
def get_file_item(self, storage: str, path: Path) -> Optional[FileItem]:
"""
根据路径获取文件项
根据路径严格获取文件项;普通兼容调度仍会隔离异常并投影为空结果。
"""
if storage not in self._support_storages:
return None
@@ -353,7 +353,7 @@ class FileManagerModule(_ModuleBase):
if not storage_oper:
logger.error(f"不支持 {storage} 的文件获取")
return None
return storage_oper.get_item(path)
return storage_oper.get_item_strict(path)
def get_parent_item(self, fileitem: FileItem) -> Optional[FileItem]:
"""
@@ -391,16 +391,24 @@ class FileManagerModule(_ModuleBase):
previous_snapshot=previous_snapshot
)
def transfer(self, fileitem: FileItem, meta: MetaBase, mediainfo: MediaInfo,
target_directory: TransferDirectoryConf = None,
target_storage: Optional[str] = None, target_path: Path = None,
transfer_type: Optional[str] = None, scrape: Optional[bool] = None,
library_type_folder: Optional[bool] = None, library_category_folder: Optional[bool] = None,
episodes_info: List[TmdbEpisode] = None,
source_oper: Callable = None, target_oper: Callable = None,
preview: Optional[bool] = False) -> TransferInfo:
def plan_transfer(
self,
fileitem: FileItem,
meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo],
target_directory: Optional[TransferDirectoryConf] = None,
target_storage: Optional[str] = None,
target_path: Optional[Path] = None,
transfer_type: Optional[str] = None, scrape: Optional[bool] = None,
library_type_folder: Optional[bool] = None,
library_category_folder: Optional[bool] = None,
episodes_info: Optional[List[TmdbEpisode]] = None,
source_oper: Optional[StorageBase] = None,
preview: Optional[bool] = False,
planning_input: Optional[TransferPlanningInput] = None,
) -> TransferPlanCheckpoint:
"""
文件整理
解析整理策略并生成零写副作用的冻结计划。
:param fileitem: 文件信息
:param meta: 预识别的元数据
:param mediainfo: 识别的媒体信息
@@ -413,29 +421,26 @@ class FileManagerModule(_ModuleBase):
:param library_category_folder: 是否按媒体类别创建目录
:param episodes_info: 当前季的全部集信息
:param source_oper: 源存储操作对象
:param target_oper: 目标存储操作对象
:return: {path, target_path, message}
:param planning_input: admission 阶段冻结的原始请求,传入时不得改写
:return: 可持久化的整理计划检查点
"""
handler = TransHandler()
# 检查目录路径
if fileitem.storage == "local" and not Path(fileitem.path).exists():
return TransferInfo(success=False,
fileitem=fileitem,
message=f"{fileitem.path} 不存在")
if (
fileitem.storage == "local"
and (not fileitem.path or not Path(fileitem.path).exists())
):
raise ValueError(f"{fileitem.path} 不存在")
# 目标路径不能是文件
if target_path and target_path.is_file():
logger.error(f"整理目标路径 {target_path} 是一个文件")
return TransferInfo(success=False,
fileitem=fileitem,
message=f"{target_path} 不是有效目录")
raise ValueError(f"{target_path} 不是有效目录")
# 获取目标路径
if target_directory:
# 目标媒体库目录未设置
if not target_directory.library_path:
logger.error(f"目标媒体库目录未设置,无法整理文件,源路径:{fileitem.path}")
return TransferInfo(success=False,
fileitem=fileitem,
message="目标媒体库目录未设置")
raise ValueError("目标媒体库目录未设置")
# 整理方式
if not transfer_type:
transfer_type = target_directory.transfer_type
@@ -467,56 +472,141 @@ class FileManagerModule(_ModuleBase):
# 未找到有效的媒体库目录
logger.error(
f"{mediainfo.type.value if mediainfo.type else '未知类型'} {mediainfo.title_year} 未找到有效的媒体库目录,无法整理文件,源路径:{fileitem.path}")
return TransferInfo(success=False,
fileitem=fileitem,
message="未找到有效的媒体库目录")
raise ValueError("未找到有效的媒体库目录")
# 整理方式
if not transfer_type:
logger.error(f"{target_directory.name} 未设置整理方式")
return TransferInfo(success=False,
fileitem=fileitem,
message=f"{target_directory.name} 未设置整理方式")
directory_name = target_directory.name if target_directory else "目标目录"
logger.error(f"{directory_name} 未设置整理方式")
raise ValueError(f"{directory_name} 未设置整理方式")
if target_path is None:
raise ValueError("整理规划缺少目标路径")
# 源操作对象
source_storage = fileitem.storage or "local"
if not source_oper:
source_oper = self.__get_storage_oper(fileitem.storage)
source_oper = self.__get_storage_oper(source_storage)
if not source_oper:
return TransferInfo(success=False,
message=f"不支持的存储类型:{fileitem.storage}",
fileitem=fileitem,
fail_list=[fileitem.path],
transfer_type=transfer_type,
need_notify=need_notify
)
# 目的操作对象
if not target_oper:
if not target_storage:
target_storage = fileitem.storage
target_oper = self.__get_storage_oper(target_storage)
if not target_oper:
return TransferInfo(success=False,
message=f"不支持的存储类型:{target_storage}",
fileitem=fileitem,
fail_list=[fileitem.path],
transfer_type=transfer_type,
need_notify=need_notify)
raise ValueError(f"不支持的存储类型:{source_storage}")
if not target_storage:
target_storage = source_storage
if planning_input is None:
planning_input = TransferPlanningInput(
source_fileitem=fileitem.model_dump(mode="json"),
meta=self._serialize_transfer_model(meta) if meta else None,
mediainfo=(
self._serialize_transfer_model(mediainfo) if mediainfo else None
),
target_directory=(
target_directory.model_dump(mode="json")
if target_directory
else None
),
target_storage=target_storage,
target_path=target_path.as_posix(),
requested_transfer_type=transfer_type,
media_type=mediainfo.type.value if mediainfo.type else None,
need_scrape=bool(need_scrape),
need_rename=bool(need_rename),
need_notify=bool(need_notify),
overwrite_mode=overwrite_mode,
episodes_info=tuple(
episode.model_dump(mode="json") for episode in episodes_info or []
),
preview=bool(preview),
)
# 整理
logger.info(f"获取整理目标路径:【{target_storage}{target_path}")
return handler.transfer_media(fileitem=fileitem,
in_meta=meta,
mediainfo=mediainfo,
target_storage=target_storage,
target_path=target_path,
transfer_type=transfer_type,
need_scrape=need_scrape,
need_rename=need_rename,
need_notify=need_notify,
overwrite_mode=overwrite_mode,
episodes_info=episodes_info,
preview=preview,
source_oper=source_oper,
target_oper=target_oper)
return handler.plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_storage=target_storage,
target_path=target_path,
transfer_type=transfer_type,
need_scrape=bool(need_scrape),
need_rename=bool(need_rename),
need_notify=bool(need_notify),
overwrite_mode=overwrite_mode,
episodes_info=episodes_info,
preview=bool(preview),
)
def execute_transfer_plan(
self,
checkpoint: TransferPlanCheckpoint,
*,
meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo],
source_oper: Optional[StorageBase] = None,
target_oper: Optional[StorageBase] = None,
cleanup_media_file: Optional[Callable[[FileItem], bool]] = None,
) -> TransferInfo:
"""解析存储适配器并通过统一删除能力执行已冻结计划。"""
source_fileitem = FileItem(**checkpoint.planning_input.source_fileitem)
cleanup_before_transfer = None
cleanup_payload = checkpoint.planning_input.options.get(
"cleanup_dest_fileitem"
)
if (
isinstance(cleanup_payload, dict)
and not checkpoint.preview
and not checkpoint.pre_execution_cleanup_completed
):
cleanup_fileitem = FileItem.model_validate(cleanup_payload)
if not cleanup_media_file:
raise RuntimeError("整理计划缺少统一媒体删除兼容能力")
def cleanup_before_transfer() -> None:
"""拦截通过后委托统一能力治理旧目标及父空目录。"""
if not cleanup_media_file(cleanup_fileitem):
raise RuntimeError(
f"{cleanup_fileitem.path} 删除失败,整理计划保留待重试"
)
source_storage = source_fileitem.storage or "local"
if not source_oper:
source_oper = self.__get_storage_oper(source_storage)
if not source_oper:
return TransferInfo(
success=False,
message=f"不支持的存储类型:{source_storage}",
fileitem=source_fileitem,
fail_list=[source_fileitem.path],
transfer_type=checkpoint.resolved_transfer_type,
need_notify=checkpoint.need_notify,
)
if not target_oper:
target_oper = self.__get_storage_oper(checkpoint.target_storage)
if not target_oper:
return TransferInfo(
success=False,
message=f"不支持的存储类型:{checkpoint.target_storage}",
fileitem=source_fileitem,
fail_list=[source_fileitem.path],
transfer_type=checkpoint.resolved_transfer_type,
need_notify=checkpoint.need_notify,
)
return TransHandler().execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_oper=target_oper,
cleanup_before_transfer=cleanup_before_transfer,
)
@staticmethod
def _serialize_transfer_model(value: object) -> dict[str, Any]:
"""校验旧领域对象的动态序列化结果,避免 Any 穿透规划边界。"""
serializer = getattr(value, "to_dict", None)
if not callable(serializer):
raise TypeError(f"{type(value).__name__} 不支持整理快照序列化")
payload = serializer()
if not isinstance(payload, dict):
raise TypeError(f"{type(value).__name__} 返回了无效整理快照")
return payload
@staticmethod
def _build_library_lookup_meta(
File diff suppressed because it is too large Load Diff
@@ -124,6 +124,29 @@ _METHOD_CONTRACTS = {
"rename_file": ModuleMethodContract(family="storage", input_contract="StorageRenameRequest", result_contract="bool | FileItem", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("fileitem", "name")),
"storage_manage": ModuleMethodContract(family="storage", input_contract="StorageManageRequest", result_contract="StorageProviderResult", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("storage", "action")),
"snapshot_storage": ModuleMethodContract(family="storage", input_contract="StorageSnapshotRequest", result_contract="dict[str, dict] | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("storage", "path", "last_snapshot_time", "max_depth", "previous_snapshot")),
"plan_transfer": ModuleMethodContract(
family="storage",
input_contract="TransferPlanningInput",
result_contract="TransferPlanCheckpoint | None",
aggregation=ModuleResultAggregation.FIRST_NON_EMPTY,
required_parameters=(
"fileitem", "meta", "mediainfo", "target_directory", "target_storage",
"target_path", "transfer_type", "scrape", "library_type_folder",
"library_category_folder", "episodes_info", "source_oper", "preview",
"planning_input",
),
public_to_plugins=False,
),
"execute_transfer_plan": ModuleMethodContract(
family="storage",
input_contract="TransferPlanCheckpoint",
result_contract="TransferInfo | None",
aggregation=ModuleResultAggregation.FIRST_NON_EMPTY,
required_parameters=(
"checkpoint", "meta", "mediainfo", "source_oper", "target_oper",
),
public_to_plugins=False,
),
"transfer": ModuleMethodContract(family="storage", input_contract="TransferRequest", result_contract="TransferInfo | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY, required_parameters=("fileitem", "meta", "mediainfo", "target_directory", "target_storage", "target_path", "transfer_type", "scrape", "library_type_folder", "library_category_folder", "episodes_info", "source_oper", "target_oper", "preview")),
"load_category_config": ModuleMethodContract(family="category", input_contract="CategoryConfigReadRequest", result_contract="CategoryConfig | None", aggregation=ModuleResultAggregation.FIRST_NON_EMPTY),
"clear_cache": ModuleMethodContract(family="category", input_contract="CacheClearRequest", result_contract="None", aggregation=ModuleResultAggregation.FAN_OUT, plugin_short_circuit=False),
+239 -26
View File
@@ -4,13 +4,12 @@ from __future__ import annotations
import inspect
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from enum import StrEnum
from typing import Any, Protocol, cast
from app.foundation.reflection import ObjectUtils
from app.runtime.execution import run_in_threadpool_to_completion
from app.runtime.log import logger
from app.runtime.observability import observe_duration, record_metric
from app.runtime.extensions.module.contracts import (
ModuleResultAggregation,
diagnose_module_callable,
@@ -18,6 +17,8 @@ from app.runtime.extensions.module.contracts import (
get_module_method_contract,
is_explicit_module_method,
)
from app.runtime.log import logger
from app.runtime.observability import observe_duration, record_metric
from app.schemas.exception import RateLimitExceededException
@@ -41,6 +42,50 @@ ModuleErrorHandler = Callable[..., None]
AsyncFunctionRunner = Callable[..., Any]
class FrozenModuleProviderMissingError(LookupError):
"""表示冻结插件 provider 已无法在当前精确目录中解析。"""
@dataclass(frozen=True, slots=True)
class FrozenPluginProviderRef:
"""保存可持久化的插件 provider 身份及其冻结方法。"""
plugin_id: str
plugin_name: str
method: str
def __post_init__(self) -> None:
"""拒绝无法精确解析的空 provider 身份。"""
if not self.plugin_id or not self.plugin_name or not self.method:
raise ValueError("冻结插件 provider 缺少 id、名称或方法")
def to_payload(self) -> dict[str, str]:
"""生成可直接写入 JSON 的稳定 provider 引用。"""
return {
"plugin_id": self.plugin_id,
"plugin_name": self.plugin_name,
"method": self.method,
}
@classmethod
def from_payload(cls, payload: Mapping[str, Any]) -> "FrozenPluginProviderRef":
"""从持久化映射恢复并校验 provider 引用。"""
return cls(
plugin_id=str(payload.get("plugin_id") or ""),
plugin_name=str(payload.get("plugin_name") or ""),
method=str(payload.get("method") or ""),
)
@dataclass(frozen=True, slots=True)
class _PluginProvider:
"""绑定一次目录解析得到的插件身份和可调用对象。"""
plugin_id: str
plugin_name: str
func: Callable[..., Any]
class _ProviderCallMode(StrEnum):
"""描述当前 provider 应采用的兼容调用方式。"""
@@ -77,14 +122,72 @@ class ModuleInvocationDispatcher:
return all(value is None for value in result)
return result is None
def freeze_plugin_providers(
self,
method: str,
) -> tuple[FrozenPluginProviderRef, ...]:
"""按当前插件目录顺序冻结实现指定方法的精确 provider 引用。"""
return tuple(
FrozenPluginProviderRef(
plugin_id=provider.plugin_id,
plugin_name=provider.plugin_name,
method=method,
)
for provider in self._collect_plugin_providers(method, error_kwargs={})
)
def execute_frozen_plugin_providers(
self,
method: str,
providers: tuple[FrozenPluginProviderRef, ...],
*args: Any,
initial_result: Any = None,
before_invoke: Callable[[], None] | None = None,
**kwargs: Any,
) -> Any:
"""解析全部冻结引用后运行前置钩子,并严格执行原顺序 provider。"""
resolved = self._resolve_frozen_plugin_providers(method, providers)
if resolved and before_invoke is not None:
before_invoke()
return self._execute_plugin_provider_sequence(
method,
initial_result,
resolved,
*args,
strict_errors=True,
**kwargs,
)
def dispatch(self, method: str, *args: Any, **kwargs: Any) -> Any:
"""先执行插件模块,再按优先级执行宿主模块。"""
return self._dispatch(method, *args, strict_errors=False, **kwargs)
def dispatch_strict(self, method: str, *args: Any, **kwargs: Any) -> Any:
"""执行模块并传播 provider 异常,使空结果与查询失败保持可区分。"""
return self._dispatch(method, *args, strict_errors=True, **kwargs)
def _dispatch(
self,
method: str,
*args: Any,
strict_errors: bool,
**kwargs: Any,
) -> Any:
"""按统一聚合规则调度,并由调用方选择是否隔离 provider 异常。"""
contract = get_module_method_contract(method)
logger.debug("模块方法契约:%s -> %s", method, contract.family)
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = self.execute_plugin_modules(method, None, *args, **kwargs)
result = None
if contract.public_to_plugins:
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = self.execute_plugin_modules(
method,
None,
*args,
strict_errors=strict_errors,
**kwargs,
)
if (
contract.plugin_short_circuit
and not self.is_valid_empty(result)
@@ -94,21 +197,29 @@ class ModuleInvocationDispatcher:
with observe_duration(
"module.provider.duration", method=method, provider_type="system"
):
return self.execute_system_modules(method, result, *args, **kwargs)
return self.execute_system_modules(
method,
result,
*args,
strict_errors=strict_errors,
**kwargs,
)
async def async_dispatch(self, method: str, *args: Any, **kwargs: Any) -> Any:
"""以与同步路径相同的聚合规则执行同步或异步模块方法。"""
contract = get_module_method_contract(method)
logger.debug("异步模块方法契约:%s -> %s", method, contract.family)
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = await self.async_execute_plugin_modules(
method,
None,
*args,
**kwargs,
)
result = None
if contract.public_to_plugins:
with observe_duration(
"module.provider.duration", method=method, provider_type="plugin"
):
result = await self.async_execute_plugin_modules(
method,
None,
*args,
**kwargs,
)
if (
contract.plugin_short_circuit
and not self.is_valid_empty(result)
@@ -130,37 +241,130 @@ class ModuleInvocationDispatcher:
method: str,
result: Any,
*args: Any,
strict_errors: bool = False,
**kwargs: Any,
) -> Any:
"""同步执行插件方法,保留插件顺序、短路和列表合并语义。"""
aggregation = get_module_method_contract(method).aggregation
providers = self._collect_plugin_providers(
method,
error_kwargs=kwargs,
strict_errors=strict_errors,
)
return self._execute_plugin_provider_sequence(
method,
result,
providers,
*args,
strict_errors=strict_errors,
**kwargs,
)
def _collect_plugin_providers(
self,
method: str,
*,
error_kwargs: Mapping[str, Any],
strict_errors: bool = False,
) -> tuple[_PluginProvider, ...]:
"""从同一插件目录快照收集 provider,并隔离损坏的方法表。"""
providers = []
for plugin, module_dict in self._plugin_catalog.get_plugin_modules().items():
plugin_id, plugin_name = plugin
try:
# 防御坏插件把方法表声明成非映射类型,避免击穿整个模块调度
if not isinstance(module_dict, Mapping):
raise TypeError(
f"插件 {plugin_id} 的模块声明必须是映射,实际是 {type(module_dict).__name__}"
f"插件 {plugin_id} 的模块声明必须是映射,实际是 "
f"{type(module_dict).__name__}"
)
func = module_dict.get(method)
if not func:
continue
providers.append(
_PluginProvider(
plugin_id=plugin_id,
plugin_name=plugin_name,
func=func,
)
)
except Exception as err:
self._record_timeout(method, "plugin", err)
self._plugin_error_handler(
err,
plugin_id,
plugin_name,
method,
**error_kwargs,
)
if strict_errors:
raise
return tuple(providers)
def _resolve_frozen_plugin_providers(
self,
method: str,
providers: tuple[FrozenPluginProviderRef, ...],
) -> tuple[_PluginProvider, ...]:
"""一次性精确解析全部冻结引用,缺失时在执行任何副作用前失败。"""
module_catalog = self._plugin_catalog.get_plugin_modules()
resolved = []
for provider in providers:
if provider.method != method:
raise FrozenModuleProviderMissingError(
f"冻结插件 provider 方法不匹配:"
f"{provider.plugin_id}/{provider.plugin_name} "
f"冻结为 {provider.method},请求执行 {method}"
)
module_dict = module_catalog.get(
(provider.plugin_id, provider.plugin_name)
)
func = module_dict.get(method) if isinstance(module_dict, Mapping) else None
if not callable(func):
raise FrozenModuleProviderMissingError(
f"冻结插件 provider 已缺失:"
f"{provider.plugin_id}/{provider.plugin_name}.{method}"
)
resolved.append(
_PluginProvider(
plugin_id=provider.plugin_id,
plugin_name=provider.plugin_name,
func=func,
)
)
return tuple(resolved)
def _execute_plugin_provider_sequence(
self,
method: str,
result: Any,
providers: tuple[_PluginProvider, ...],
*args: Any,
strict_errors: bool = False,
**kwargs: Any,
) -> Any:
"""按统一契约执行已解析插件序列,并按调用模式处理 provider 故障。"""
aggregation = get_module_method_contract(method).aggregation
for provider in providers:
try:
self._record_legacy_hit(
method,
caller_type="plugin",
abi_source="third_party_plugin",
)
self._diagnose_callable(method, func, f"插件 {plugin_id}")
logger.info("请求插件 %s 执行:%s ...", plugin_name, method)
self._diagnose_callable(
method,
provider.func,
f"插件 {provider.plugin_id}",
)
logger.info("请求插件 %s 执行:%s ...", provider.plugin_name, method)
call_mode = self._provider_call_mode(
aggregation,
result,
func,
provider.func,
allow_relay=False,
)
if call_mode is _ProviderCallMode.STOP:
break
provider_result = func(*args, **kwargs)
provider_result = provider.func(*args, **kwargs)
self._diagnose_result(method, provider_result, "plugin")
result = self._aggregate_provider_result(
result,
@@ -172,19 +376,23 @@ class ModuleInvocationDispatcher:
self._rate_limit_handler(
err,
"插件",
plugin_id,
provider.plugin_id,
method,
**kwargs,
)
if strict_errors:
raise
except Exception as err:
self._record_timeout(method, "plugin", err)
self._plugin_error_handler(
err,
plugin_id,
plugin_name,
provider.plugin_id,
provider.plugin_name,
method,
**kwargs,
)
if strict_errors:
raise
return result
async def async_execute_plugin_modules(
@@ -254,6 +462,7 @@ class ModuleInvocationDispatcher:
method: str,
result: Any,
*args: Any,
strict_errors: bool = False,
**kwargs: Any,
) -> Any:
"""同步执行按优先级排序的宿主模块,并支持签名接力。"""
@@ -301,6 +510,8 @@ class ModuleInvocationDispatcher:
method,
**kwargs,
)
if strict_errors:
raise
except Exception as err:
self._record_timeout(method, "system", err)
self._system_error_handler(
@@ -310,6 +521,8 @@ class ModuleInvocationDispatcher:
method,
**kwargs,
)
if strict_errors:
raise
return result
async def async_execute_system_modules(
+9 -1
View File
@@ -1,7 +1,7 @@
import asyncio
import inspect
import sys
from typing import Callable
from typing import Any, Callable
from app.adapters.cache.redis import AsyncRedisHelper, RedisHelper
from app.application.plugin.transaction import (
@@ -241,6 +241,13 @@ async def _async_get_workflow(workflow_id: int):
return await WorkflowOper().async_get(workflow_id)
def _execute_legacy_transfer_command(**kwargs: Any) -> Any:
"""把旧 Chain ABI 延迟转入唯一 TransferChain durable command。"""
from app.chain.transfer import TransferChain
return TransferChain().execute_legacy_transfer_command(**kwargs)
def _build_chain_runtime_context() -> ChainRuntimeContext:
"""在启动组合根创建 Chain 所需的运行时对象和数据端口。"""
return ChainRuntimeContext(
@@ -255,6 +262,7 @@ def _build_chain_runtime_context() -> ChainRuntimeContext:
send_callback=callback
),
module_dispatcher_factory=ModuleInvocationDispatcher,
legacy_transfer_command=_execute_legacy_transfer_command,
configuration=build_chain_runtime_config(legacy_settings),
data_ports=get_chain_data_ports(),
durable_event_writer=TransactionalChainDurableEventWriter(SessionFactory),
+185
View File
@@ -0,0 +1,185 @@
"""3.0.14 为整理任务增加版本化规划输入与原子计划检查点。
Revision ID: c2f8a4d6e1b3
Revises: b1e7d3f5a9c2
Create Date: 2026-08-27
"""
import hashlib
import json
import sqlalchemy as sa
from alembic import op
revision = "c2f8a4d6e1b3"
down_revision = "b1e7d3f5a9c2"
branch_labels = None
depends_on = None
_TABLE_NAME = "transferpending"
_NEW_COLUMNS = {
"input_version",
"planning_input",
"input_fingerprint",
"checkpoint_version",
"checkpoint_payload",
"planned_at",
}
def _column_names() -> set[str]:
"""返回当前待整理登记表的字段集合。"""
inspector = sa.inspect(op.get_bind())
if _TABLE_NAME not in inspector.get_table_names():
return set()
return {
column["name"]
for column in inspector.get_columns(_TABLE_NAME)
}
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, object]:
"""构造与 Application 规划输入第一版一致的保守重规划 JSON。"""
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _fingerprint(payload: dict[str, object]) -> str:
"""按稳定 JSON 编码计算与 Application 一致的输入指纹。"""
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _backfill_planning_input() -> None:
"""为 3.0.13 登记生成可辨识且必须重新规划的最小输入。"""
pending = sa.table(
_TABLE_NAME,
sa.column("id", sa.Integer()),
sa.column("storage", sa.String()),
sa.column("src_path", sa.String()),
sa.column("input_version", sa.Integer()),
sa.column("planning_input", sa.JSON()),
sa.column("input_fingerprint", sa.String()),
)
connection = op.get_bind()
rows = connection.execute(
sa.select(
pending.c.id,
pending.c.storage,
pending.c.src_path,
pending.c.input_version,
pending.c.planning_input,
pending.c.input_fingerprint,
)
).mappings().all()
for row in rows:
if (
row["input_version"] is not None
and row["planning_input"] is not None
and row["input_fingerprint"]
):
continue
payload = row["planning_input"]
if not isinstance(payload, dict):
payload = _legacy_planning_payload(row["storage"], row["src_path"])
payload_version = payload.get("schema_version")
input_version = row["input_version"]
if input_version is None:
input_version = payload_version if isinstance(payload_version, int) else 1
input_fingerprint = row["input_fingerprint"] or _fingerprint(payload)
connection.execute(
pending.update()
.where(pending.c.id == row["id"])
.values(
input_version=input_version,
planning_input=payload,
input_fingerprint=input_fingerprint,
)
)
def upgrade() -> None:
"""增加规划输入和完整检查点字段并保守回填旧登记。"""
columns = _column_names()
if not columns:
return
additions = (
("input_version", sa.Column("input_version", sa.Integer(), nullable=True)),
("planning_input", sa.Column("planning_input", sa.JSON(), nullable=True)),
(
"input_fingerprint",
sa.Column("input_fingerprint", sa.String(length=64), nullable=True),
),
("checkpoint_version", sa.Column("checkpoint_version", sa.Integer(), nullable=True)),
("checkpoint_payload", sa.Column("checkpoint_payload", sa.JSON(), nullable=True)),
("planned_at", sa.Column("planned_at", sa.String(length=40), nullable=True)),
)
for column_name, column in additions:
if column_name not in columns:
op.add_column(_TABLE_NAME, column)
_backfill_planning_input()
with op.batch_alter_table(_TABLE_NAME) as batch_op:
batch_op.alter_column(
"input_version", existing_type=sa.Integer(), nullable=False
)
batch_op.alter_column(
"planning_input", existing_type=sa.JSON(), nullable=False
)
batch_op.alter_column(
"input_fingerprint",
existing_type=sa.String(length=64),
nullable=False,
)
def downgrade() -> None:
"""移除规划字段并把旧版本无法识别的状态保守恢复为接纳态。"""
columns = _column_names()
if not columns or not (_NEW_COLUMNS & columns):
return
if "state" in columns:
pending = sa.table(
_TABLE_NAME,
sa.column("state", sa.String()),
)
op.get_bind().execute(
pending.update()
.where(pending.c.state.in_(("planned", "provider_pending")))
.values(state="accepted")
)
with op.batch_alter_table(_TABLE_NAME) as batch_op:
for column_name in (
"planned_at",
"checkpoint_payload",
"checkpoint_version",
"input_fingerprint",
"planning_input",
"input_version",
):
if column_name in columns:
batch_op.drop_column(column_name)
+25 -15
View File
@@ -69,17 +69,17 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
| 指标 | 当前值 | 解释 |
|---|---:|---|
| 宿主 Python 模块 / 内部依赖边 | 836 / 6,827 | `dependency-baseline.json` 当前快照 |
| 宿主 Python 模块 / 内部依赖边 | 836 / 6,832 | `dependency-baseline.json` 当前快照 |
| 非平凡 SCC | 2 | 新增 Chain 包根环;另一个是隔离的 29 模块 TMDB 移植包环 |
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
| Module Contract | 215 specs / 214 methods / 264 calls | 动态方法名为 0仍有 50 个结果形状为 `ANY` |
| Module Contract | 217 specs / 215 methods / 265 calls | 动态方法名为 0内部 planning 合同不进入插件调度,旧 transfer 只保留 provider ABI |
| Event Contract | 53 | 均已有 payload model,但当前全部是 diagnostic enforcement |
| Python 源码量 | 约 271,400 行 | 60 个文件超过 1,000 行,14 个超过 2,000 行 |
| 长方法 | 281 个超过 80 行 | 67 个超过 150 行,23 个超过 250 行;大量是私有方法 |
| 全量 mypy 历史债务 | 11,983 / 601 文件 | strict frontier 当前只覆盖 41 个文件,且 ratchet 已新增 2 个错误 |
| Ruff 历史诊断 | 967 | 低水位门禁通过,但规则集只覆盖 `E4/E7/E9/F/I` |
| 覆盖率低水位 | Application 77.85%Domain 79.24% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
| Ruff 历史诊断 | 937 | 低水位门禁通过,但规则集只覆盖 `E4/E7/E9/F/I` |
| 覆盖率低水位 | Application 78.06%Domain 79.29% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
### 3.3 热点文件
@@ -160,7 +160,8 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
- 架构总览此前仍记录 811 模块、6,572 条边和 1 个 SCC,已经落后于当前基线。
- Event consumer 扫描曾把任意同名 `.register()` 调用当成事件注册;S0-L2.5 已改为证明
canonical EventManager receiver10 个动态误报归零并保留唯一 workflow 动态注册。
- S0-L2.6 已将 producer/consumer 合并为逐调用事实源99 个 producer98 静态、1 动态)与
- S0-L2.6 已将 producer/consumer 合并为逐调用事实源;本轮统一 Transfer 事件发送点后为
97 个 producer96 静态、1 动态)与
17 个 consumer16 静态、1 动态);consumer 由不可自动写入的精确人工 policy 管理。
**目标与步骤**
@@ -188,7 +189,9 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
- `S1-L1.1 Durable admission``VERIFIED`。已交付 persist-before-enqueue、Application-owned typed Port、
DB adapter 与可逆 migration,宿主退出 raw/`Any` `TransferPendingOper` admission 路径。
- `S1-L1.2 Planning checkpoint``PLANNED`。持久化稳定任务身份、整理模式、规划状态和目标 checkpoint。
- `S1-L1.2 Planning checkpoint``VERIFIED`。版本化请求与指纹先准入;无 legacy provider 时通过
`accepted -> planned` CAS 提交完整目标和有序操作,有 provider 时先提交 `provider_pending`,全部
返回空后再以第二次 CAS 提交 `planned`;planned 重放只消费冻结上下文和目标。
- `S1-L1.3 Lease 与恢复调度``PLANNED`。交付 claim/lease/heartbeat/attempt、过期接管与唯一恢复入口。
- `S1-L1.4 幂等执行与终态结算``PLANNED`。交付文件/历史幂等、唯一 retry owner 和
`manual_review` 语义。
@@ -201,28 +204,35 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
异常均不会伪装成重复任务成功,失败记录可供恢复。
- 宿主 canonical Chain 只取得类型化 `TransferAdmissionRepository`;旧 Oper API 仅保留给统一兼容层,
插件公开 `TransferTask.to_dict()` 字段未增加内部任务标识。
- worker 未知异常最终也会在 `app/chain/transfer.py:1107-1113,1262-1272` 删除 pending。
- `app/db/models/transferpending.py:9-34` 只有 `storage/src_path/created_at`,没有目标、模式、
step、lease、attempt、last_error,无法判定“文件已移动、历史未提交”等中间态。
- `S1-L1.2` 已消除 checkpoint 前的文件副作用:目标路径、操作顺序及 resolved 识别上下文原子落库后,
执行器才允许触发 cleanup、建目录和复制/移动;规划失败保留 `accepted` 并记录 `last_error`
- 旧插件 `transfer` provider 的身份、顺序和原始 ABI 参数先冻结为 `provider_pending`;提交后才精确
解析并严格执行,缺失或异常不 fallback。全部返回空后才生成宿主计划,并以第二次 CAS 提升为
`planned` 后执行。旧 caller 只经 `ChainBase.transfer` 注入式兼容门面进入同一 durable command
宿主 FileManager/TransHandler 的旧执行入口已删除。
- `TransferPending` 现在可区分 `accepted/provider_pending/planned`,但尚无
claim/lease/heartbeat/attempt、逐步骤执行结果和 `manual_review`,仍无法判定“文件已移动、历史未提交”
等后续中间态。
- 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、
lease/heartbeat 和人工恢复要求不一致。
**目标与步骤**
- [ ] 先在独立持久事务中 commit pending,再尝试放入内存队列;数据库事务不能与 `queue.Queue`
- [x] 先在独立持久事务中 commit pending,再尝试放入内存队列;数据库事务不能与 `queue.Queue`
原子提交,入队失败时必须保留 pending 供重放。
- [ ] 初始登记保存稳定源身份、模式、状态和 attempt/lease;目标在规划完成后以 planning checkpoint
更新,不能要求任务刚入队时已经具备尚未计算的目标路径
- [x] 初始登记保存稳定源身份、版本化请求和状态;目标与有序操作在纯规划完成后以 planning
checkpoint 原子更新,任何文件副作用不得早于该提交
- [ ] 增加 claim/lease/heartbeat/attempt 与过期接管,同一任务同时只能有一个 worker owner。
- [ ] 设计幂等文件操作和历史提交;只有所有必要步骤达到持久终态后才能删除记录。
- [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner,定义旧记录迁移和兼容规则。
- [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界,不直接套用 E2
Outbox 的 dead-letter 语义;禁止按年龄通用清理 pending。
- [ ] 数据模型变更必须配套 Alembic migration,并验证升级降级路径
- [x] 当前 admission/planning 数据模型变更配套 Alembic migration,并验证升级降级和中断重跑
**故障注入验收**
- [ ] 登记后、内存入队前崩溃,重启可继续。
- [ ] 持久登记成功但内存入队失败,重启可继续。
- [x] 登记后、内存入队前崩溃,重启可继续。
- [x] 持久登记成功但内存入队失败,重启可继续。
- [ ] 文件移动后、历史提交前崩溃,在支持稳定身份/幂等操作的存储上不重复移动且可补齐历史。
- [ ] worker 未知异常和 lease 超时后保留可诊断状态。
- [ ] 重复回放、重复消息和人工重试都保持幂等。
+7 -7
View File
@@ -369,8 +369,8 @@ collector 只接受 canonical `eventmanager`、`EventManager()` 及其有限别
`register`/`add_event_listener`;当前宿主有 16 个静态注册点,另保留 1 个由工作流配置驱动的
真实动态注册。`app/plugins/**` 插件副本不进入宿主事实。
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 99
生产调用,其中 98 个静态解析为 100 个事件引用,只有 `Command.send_plugin_event` 的插件事件类型
生产者与消费者共用 `scripts/architecture/event_facts.py` 这一份逐调用事实源。当前宿主有 97
生产调用,其中 96 个静态解析为 98 个事件引用,只有 `Command.send_plugin_event` 的插件事件类型
保持动态;17 个消费注册中 16 个静态、1 个动态。生成的
`runtime-contract-baseline.json` 保存 line-free 事实、数量和枚举索引;人工维护的
`runtime-contract-policy.json` 只批准 consumer 的精确 fingerprint、owner 和理由,任何新增、替换、
@@ -690,7 +690,7 @@ flowchart LR
stream/vendor/diagnostic/control-plane 事实是精确 containment。每条初始边的指纹由测试独立冻结,
bindings/uses 变化、分类互换、通配导入和初始边增长都会失败;债务删除时同步删除冻结项以禁止恢复,
`--write-host` 不会改写人工 policy 或冻结上界。
- `event_facts` 是生产者/消费者唯一收集源;运行快照记录 99 个生产调用和 17 个消费注册,
- `event_facts` 是生产者/消费者唯一收集源;运行快照记录 97 个生产调用和 17 个消费注册,
consumer 的 17 个唯一 fingerprint 另由只读人工 policy 精确准入。CI 将语义 policy 与生成快照
分成独立步骤,前者不能通过刷新后者绕过。
- 任何所有权迁移必须同步更新:canonical 导入、`app/runtime/compat/manifest.py`
@@ -704,13 +704,13 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 835 |
| 内部导入边 | 6,827 |
| Python 模块 | 836 |
| 内部导入边 | 6,832 |
| 非平凡 SCC | 2`ARCH-107` 临时 Chain 包根环;精确 containment 的 TMDB 移植包环) |
| Direct egress | 6612 条待迁移债务,54 条精确 containment |
| Module Contract V2 spec | 215(其中 214 个进入 `run_module` 观察面) |
| Module Contract V2 spec | 217(其中 215 个进入 `run_module` 观察面) |
| Event Contract | 53 |
| Event producer / consumer | 9998 静态、1 动态)/ 17(16 静态、1 动态) |
| Event producer / consumer | 9796 静态、1 动态)/ 17(16 静态、1 动态) |
| Model/Oper 自动事务与自建 Session | 0 |
| 组合根外 `SystemConfigOper()` | 0 |
+49 -2
View File
@@ -93,7 +93,7 @@ G-ARCH 只有在以下条件全部满足后才可完成:
| Leaf | 状态 | 依赖 | 完成定义 |
|---|---|---|---|
| S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地;先持久 commit 再入队,入队失败保留可恢复记录;宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission |
| S1-L1.2 Planning checkpoint | `PLANNED` | S1-L1.1 | 稳定任务身份、整理模式和 planning 状态持久化;目标路径只在规划完成后写入 checkpoint,任何文件副作用前已有可判定状态 |
| S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化;无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划,有 provider 时先提交 `provider_pending`,全部返回空后再以第二次 CAS 提交 `planned`;重放只执行冻结目标,所有文件副作用晚于对应 checkpoint commit |
| S1-L1.3 Lease 与恢复调度 | `PLANNED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地;启动回放和同进程恢复共用唯一调度入口,同一任务同时只有一个 worker owner |
| S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放;唯一 retry owner 生效,未知外部结果进入 `manual_review`,仅完整终态删除 pending |
| S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整;旧 fail-open、重复状态与兼容层外旧入口删除,ARCH-102 债务归零 |
@@ -144,7 +144,7 @@ G-ARCH 只有在以下条件全部满足后才可完成:
| S4-L2 Event strict contract | `PLANNED` | S0-L2.6,S1-L6 | 宿主事件输入/输出按风险 strict,诊断例外只属于第三方插件兼容 |
| S4-L3 Complexity v2 | `PLANNED` | S3 | 私有方法、class/file、圈复杂度进入门禁;所有超限通过职责拆分归零 |
| S4-L4 全量 mypy 清零 | `PLANNED` | S3,S4-L1,S4-L2 | `mypy-baseline.json` 归零并删除债务接受路径,全宿主 strict 类型通过 |
| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 967 条诊断归零,规则集扩展经过独立审查且新增诊断为零 |
| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 937 条诊断归零,规则集扩展经过独立审查且新增诊断为零 |
| S4-L6 Coverage/并发/质量证据 | `PLANNED` | S3,S4-L1,S4-L2 | 高风险包纳入 coverageraw concurrency 分类清零;Module Quality 有真实 evidence test |
### S5Plugin、Agent、Domain、Startup 与最终收口
@@ -227,3 +227,50 @@ git diff --check
`git diff --check` 全部通过。
- failure injection 已覆盖 admission 失败不入队、batch/enqueue 失败保留记录、批次返回失败、
queue -> worker -> terminal discard 稳定身份,以及 Legacy TransferTask 序列化字段不变。
### S1-L1.2 Planning checkpoint
**Status:** `VERIFIED`
**Outcome**
把 durable admission 推进为可独立恢复的 `accepted -> provider_pending -> planned` 状态:准入时冻结
版本化请求 JSON 和 SHA-256 指纹;存在旧插件 provider 时先 CAS 提交精确身份、顺序和原始 ABI 参数,
全部返回空后才由 FileManager 只读规划目标及有序叶操作,并通过第二次 CAS 提交宿主 checkpoint。
任何对应 checkpoint 提交前都不允许进入其文件副作用;`provider_pending` 重放只消费冻结调用,
`planned` 重放只消费冻结 resolved 上下文、目标和操作,不重新访问在线识别、目录选择或重命名配置。
**Ownership and compatibility**
- `app/application/transfer.py` 拥有 planning input、plan item、checkpoint 和状态错误合同;JSON 版本、
指纹及 resolved 上下文均可跨进程 round-trip。
- `app/modules/filemanager/transhandler.py` 是唯一目标规划与文件执行实现;`FileManagerModule.transfer`
`TransHandler.transfer_media` 已删除,不保留第二套重命名、覆盖或目录递归逻辑。
- `app/db/adapters/transfer.py` 通过短 Session/UoW 提交 checkpointOper 只负责带状态和指纹条件的
stage3.0.14 migration 可升级、降级并在中断后重跑。
- cleanup intent 随准入输入冻结。宿主路径由 FileManager 在 `TransferIntercept` 放行后、任何文件写入前
执行;legacy provider 路径为保持旧 ABI 顺序,在全部冻结引用解析成功后、调用 provider 前执行。
strict 查询确认目标不存在才视为幂等成功,查询或删除失败抛错并保留对应 checkpoint 供重试;provider
全空后提升的宿主 checkpoint 会记录 cleanup 已完成,禁止二次查询或删除。
- 插件公开 Transfer 方法签名、事件类型和 payload 不变;旧 provider 身份和顺序随 checkpoint
冻结,提交后由统一 dispatcher 精确解析并严格执行,缺失或异常时明确失败而不静默换路;全部返回空
才生成宿主 plan,并以第二次 CAS 提交 `planned` checkpoint 后执行。`ChainBase.transfer` 仅委托启动
组合根注入的 canonical durable
command,内部 plan/execute 合同不向插件调度;新 DTO 不从包根重复导出,`app/plugins/**` 插件
副本不参与改造。
**Excluded**
- 本叶不引入 claim、lease、heartbeat、attempt、执行步骤幂等或 `manual_review`;这些由
`S1-L1.3``S1-L1.4` 交付。
- 文件操作成功后到历史结算前的未知结果仍未达到 E3,ARCH-102 父项继续保持执行中。
**Local verification (2026-08-27)**
- planning、持久化、迁移、兼容、replay 和 worker 聚焦回归:`224 passed, 2 skipped`;跳过项仅为
本机未配置隔离 PostgreSQLSQLite upgrade/downgrade/re-upgrade 已覆盖。
- 完整本地套件:`6,578 passed, 8 skipped`;架构回归:`174 passed`scoped Pylint `10.00/10`
host baseline、Ruff/mypy ratchet 与 `git diff --check` 通过。
- failure injection 覆盖 commit 前零文件副作用、commit 后崩溃重放、离线 resolved context 恢复、
配置漂移仍使用冻结 target storage、规划失败留痕、旧 provider 提交后短路、严格异常、空结果两阶段
fallback、缺失引用零 cleanup,以及 cleanup 顺序/幂等/瞬时失败。
+22 -7
View File
@@ -467,10 +467,25 @@ Durable post-commit side effects have a separate boundary:
Transfer durable admission follows the same ownership direction without using
the Outbox as an execution queue: `app/application/transfer.py` owns the typed
admission contract and persist-before-enqueue orchestration, while
`app/db/adapters/transfer.py` commits it in a short Session/UoW. Canonical host
chains never obtain `TransferPendingOper`; its no-Session API remains only for
the exact legacy plugin import contract.
admission and versioned planning-checkpoint contracts, while
`app/db/adapters/transfer.py` commits admission and the
`accepted -> provider_pending -> planned` compare-and-set transitions in short
Session/UoW scopes. `app/modules/filemanager/` owns the
single pure-plan and checkpoint-execution implementation: all file writes occur
after checkpoint commit, and planned recovery consumes frozen resolved context,
target storage and ordered operations without online recognition or renaming.
Legacy plugin `transfer` providers are frozen by exact identity, order, and ABI
arguments in a provider-only checkpoint, then executed by the unified module
dispatcher only after commit. The dispatcher resolves every frozen reference
before the compatibility cleanup hook and propagates provider failures. Missing
or failing providers therefore remain `provider_pending`; only an all-empty
result permits host planning and a second CAS to `planned`. Host-only `plan_transfer` and
`execute_transfer_plan` contracts never dispatch to plugins. `ChainBase.transfer`
is the sole legacy caller facade and delegates the startup-injected durable
command; `FileManagerModule.transfer` and `TransHandler.transfer_media` must not
be recreated.
Canonical host chains never obtain `TransferPendingOper`; its no-Session API
remains only for the exact legacy plugin import contract.
## Composition and Compatibility Boundaries
@@ -604,8 +619,8 @@ driven workflow registration.
| `app/application/subscription/write.py` | Subscription media translation and sync/async write-port orchestration |
| `app/application/outbox.py` | Durable intent, topic handler and Outbox repository contracts |
| `app/db/adapters/outbox.py` | SQLAlchemy Outbox persistence, claim/lease and retry state adapter |
| `app/application/transfer.py` | Transfer task, durable admission contract and persist-before-enqueue use case |
| `app/db/adapters/transfer.py` | SQLAlchemy durable admission persistence and detached snapshot adapter |
| `app/application/transfer.py` | Transfer task, durable admission, versioned planning input/checkpoint contracts and queue use case |
| `app/db/adapters/transfer.py` | SQLAlchemy admission/checkpoint persistence, CAS state transition and detached snapshot adapter |
| `app/application/scheduling.py` | Runtime scheduler facade for Agent tools and endpoints; `Scheduler` class registered by `app/startup/initializers/scheduler.py` |
| `app/application/commands.py` | Command registry facade for Agent tools and endpoints; `Command` class registered by `app/startup/initializers/command.py` |
| `app/application/workflow.py` | Workflow use cases plus the runtime port consumed by API and Chain; `WorkFlowManager` is registered by `app/startup/initializers/workflow.py` |
@@ -626,7 +641,7 @@ driven workflow registration.
| `app/runtime/event/snapshot.py` | Read-only typed payload snapshots for the plugin SDK; never mutates or replaces the event ABI |
| `app/runtime/extensions/module/dispatcher.py` | Plugin-first invocation, short-circuit, list merge, signature relay and sync/async execution |
| `app/runtime/extensions/module/contracts.py` | High-frequency method families and frozen legacy fallback contract |
| `app/application/chain/context.py` | Injectable Chain dependencies and no-argument compatibility provider |
| `app/application/chain/context.py` | Injectable Chain dependencies, no-argument compatibility provider and legacy Transfer command Port |
| `app/startup/lifecycle/components.py` | Declarative normal/safe-mode lifecycle manifest, ordering and timeout budgets |
| `app/runtime/extensions/module_manager.py` | Module discovery and lifecycle |
| `app/runtime/extensions/plugin_manager.py` | Plugin discovery and lifecycle |
+5 -5
View File
@@ -1,12 +1,12 @@
{
"application": {
"covered_lines": 9341,
"percent": 77.85,
"statements": 11999
"covered_lines": 9572,
"percent": 78.06,
"statements": 12263
},
"domain": {
"covered_lines": 3390,
"percent": 79.24,
"covered_lines": 3392,
"percent": 79.29,
"statements": 4278
}
}
+7 -2
View File
@@ -1441,8 +1441,8 @@
"runtime_only": true
}
},
"edge_count": 6827,
"edge_sha256": "34e2be621e40f0f07ff04065655446c4ad0062883701f9e1e2235c29359900f7",
"edge_count": 6832,
"edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -4998,8 +4998,10 @@
"app.chain.transfer -> app.domain.context",
"app.chain.transfer -> app.domain.episode",
"app.chain.transfer -> app.domain.meta",
"app.chain.transfer -> app.domain.meta.metaanime",
"app.chain.transfer -> app.domain.meta.metabase",
"app.chain.transfer -> app.domain.meta.metamusic",
"app.chain.transfer -> app.domain.meta.metavideo",
"app.chain.transfer -> app.domain.metainfo",
"app.chain.transfer -> app.foundation",
"app.chain.transfer -> app.foundation.singleton",
@@ -5807,6 +5809,7 @@
"app.modules.filemanager.module -> app.application.directory",
"app.modules.filemanager.module -> app.application.messaging",
"app.modules.filemanager.module -> app.application.messaging.message",
"app.modules.filemanager.module -> app.application.transfer",
"app.modules.filemanager.module -> app.domain",
"app.modules.filemanager.module -> app.domain.context",
"app.modules.filemanager.module -> app.domain.meta",
@@ -5958,6 +5961,7 @@
"app.modules.filemanager.transhandler -> app.application.directory",
"app.modules.filemanager.transhandler -> app.application.messaging",
"app.modules.filemanager.transhandler -> app.application.messaging.message",
"app.modules.filemanager.transhandler -> app.application.transfer",
"app.modules.filemanager.transhandler -> app.domain",
"app.modules.filemanager.transhandler -> app.domain.context",
"app.modules.filemanager.transhandler -> app.domain.meta",
@@ -7903,6 +7907,7 @@
"app.startup.initializers.modules -> app.chain.site",
"app.startup.initializers.modules -> app.chain.subscribe",
"app.startup.initializers.modules -> app.chain.tmdb",
"app.startup.initializers.modules -> app.chain.transfer",
"app.startup.initializers.modules -> app.chain.workflow",
"app.startup.initializers.modules -> app.command",
"app.startup.initializers.modules -> app.db",
+11 -14
View File
@@ -1587,15 +1587,14 @@
"union-attr": 2
},
"app/chain/transfer.py": {
"arg-type": 65,
"assignment": 26,
"arg-type": 57,
"assignment": 25,
"attr-defined": 4,
"func-returns-value": 1,
"misc": 2,
"no-any-return": 3,
"no-redef": 2,
"no-untyped-call": 8,
"no-untyped-def": 15,
"no-untyped-def": 14,
"operator": 3,
"return-value": 2,
"truthy-function": 5,
@@ -2105,15 +2104,14 @@
"union-attr": 8
},
"app/modules/filemanager/module.py": {
"arg-type": 21,
"assignment": 9,
"arg-type": 19,
"assignment": 4,
"attr-defined": 1,
"empty-body": 1,
"no-any-return": 2,
"no-untyped-call": 6,
"no-untyped-call": 3,
"no-untyped-def": 6,
"truthy-function": 4,
"type-arg": 4,
"type-arg": 3,
"var-annotated": 3
},
"app/modules/filemanager/storages/__init__.py": {
@@ -2203,15 +2201,14 @@
"union-attr": 32
},
"app/modules/filemanager/transhandler.py": {
"arg-type": 19,
"assignment": 8,
"arg-type": 18,
"assignment": 2,
"attr-defined": 1,
"misc": 1,
"no-any-return": 1,
"no-redef": 1,
"no-untyped-call": 1,
"no-untyped-def": 4,
"operator": 7,
"no-untyped-def": 3,
"operator": 6,
"type-arg": 2,
"union-attr": 2
},
-25
View File
@@ -306,9 +306,6 @@
"app/application/agenttask.py": {
"I001": 1
},
"app/application/chain/context.py": {
"I001": 1
},
"app/application/chain/data.py": {
"I001": 1
},
@@ -414,9 +411,6 @@
"app/chain/media.py": {
"E731": 2
},
"app/chain/transfer.py": {
"E402": 21
},
"app/command.py": {
"I001": 1
},
@@ -676,13 +670,6 @@
"app/modules/filemanager/__init__.py": {
"I001": 1
},
"app/modules/filemanager/module.py": {
"I001": 1
},
"app/modules/filemanager/transhandler.py": {
"F541": 2,
"I001": 1
},
"app/modules/filter/__init__.py": {
"I001": 1
},
@@ -996,9 +983,6 @@
"app/runtime/extensions/managed_resource_adapter.py": {
"I001": 1
},
"app/runtime/extensions/module/dispatcher.py": {
"I001": 1
},
"app/runtime/extensions/module_manager.py": {
"I001": 1
},
@@ -1607,9 +1591,6 @@
"tests/test_music_torrents.py": {
"I001": 1
},
"tests/test_music_transfer.py": {
"I001": 1
},
"tests/test_music_workflows.py": {
"F401": 1
},
@@ -1855,9 +1836,6 @@
"tests/test_transfer_history_write_path.py": {
"I001": 1
},
"tests/test_transfer_job_manager.py": {
"I001": 1
},
"tests/test_transfer_mark_torrent_completed.py": {
"I001": 1
},
@@ -1867,9 +1845,6 @@
"tests/test_transfer_movie_collection.py": {
"I001": 1
},
"tests/test_transfer_preview.py": {
"I001": 1
},
"tests/test_transfer_queue_count.py": {
"I001": 1
},
+83 -49
View File
@@ -1518,8 +1518,7 @@
"ChainEventType.StorageOperSelection": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476",
"e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476"
"76a388b1caf87a51cacf9d7534f64576f7c98e0af6b604d30114dab893066d96"
]
},
"ChainEventType.SubscribeCompletionCheck": {
@@ -1538,14 +1537,13 @@
"ChainEventType.TransferIntercept": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"144f9ffac815b2f58ade53881df0ce43381a7b1adff62f53a019794b801a0cd7",
"d7ee7c450874ea4bf6ea97c903b9cd3ebeab7266834c22e1a699d067205e8912"
"410f9308657c3c228e5357acf2e9a083c4fe6476c3b4ddaf240cab385d8bd4af"
]
},
"ChainEventType.TransferOverwriteCheck": {
"consumer_fingerprints": [],
"producer_fingerprints": [
"e761a75952ab6835e67b2c43c671f6361f1977690cfd7724f79439229408af40"
"be3334d20e9aa1fd641fe67f0bc3abd3ef48433728ddb76538cfbb475b4dc92b"
]
},
"ChainEventType.TransferRename": {
@@ -1809,11 +1807,11 @@
]
}
},
"fact_count": 116,
"fact_count": 114,
"invalid_consumer_count": 0,
"invalid_producer_count": 0,
"producer_call_count": 99,
"producer_event_reference_count": 100,
"producer_call_count": 97,
"producer_event_reference_count": 98,
"producers": [
{
"caller": "app.agent.orchestrator",
@@ -2495,22 +2493,10 @@
"events": [
"ChainEventType.StorageOperSelection"
],
"fingerprint": "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476",
"fingerprint": "76a388b1caf87a51cacf9d7534f64576f7c98e0af6b604d30114dab893066d96",
"invalid": false,
"method": "send_event",
"qualname": "TransferChain.__handle_transfer",
"receiver_kind": "injected_event_manager"
},
{
"caller": "app.chain.transfer",
"dynamic": false,
"events": [
"ChainEventType.StorageOperSelection"
],
"fingerprint": "e5cfe4cabe945282695a3490147ee73acb3c6f8a4889fe510e8c0d3d2044e476",
"invalid": false,
"method": "send_event",
"qualname": "TransferChain.__handle_transfer",
"qualname": "TransferChain.__select_storage_oper",
"receiver_kind": "injected_event_manager"
},
{
@@ -2673,22 +2659,22 @@
"events": [
"ChainEventType.TransferIntercept"
],
"fingerprint": "d7ee7c450874ea4bf6ea97c903b9cd3ebeab7266834c22e1a699d067205e8912",
"fingerprint": "410f9308657c3c228e5357acf2e9a083c4fe6476c3b4ddaf240cab385d8bd4af",
"invalid": false,
"method": "send_event",
"qualname": "TransHandler.__transfer_dir",
"qualname": "TransHandler.__intercept_transfer",
"receiver_kind": "canonical_singleton"
},
{
"caller": "app.modules.filemanager.transhandler",
"dynamic": false,
"events": [
"ChainEventType.TransferIntercept"
"ChainEventType.TransferOverwriteCheck"
],
"fingerprint": "144f9ffac815b2f58ade53881df0ce43381a7b1adff62f53a019794b801a0cd7",
"fingerprint": "be3334d20e9aa1fd641fe67f0bc3abd3ef48433728ddb76538cfbb475b4dc92b",
"invalid": false,
"method": "send_event",
"qualname": "TransHandler.__transfer_file",
"qualname": "TransHandler.__resolve_overwrite",
"receiver_kind": "canonical_singleton"
},
{
@@ -2715,18 +2701,6 @@
"qualname": "TransHandler.get_rename_path",
"receiver_kind": "canonical_singleton"
},
{
"caller": "app.modules.filemanager.transhandler",
"dynamic": false,
"events": [
"ChainEventType.TransferOverwriteCheck"
],
"fingerprint": "e761a75952ab6835e67b2c43c671f6361f1977690cfd7724f79439229408af40",
"invalid": false,
"method": "send_event",
"qualname": "TransHandler.transfer_media",
"receiver_kind": "canonical_singleton"
},
{
"caller": "app.modules.navidrome",
"dynamic": false,
@@ -3005,7 +2979,7 @@
}
],
"static_consumer_count": 16,
"static_producer_call_count": 98
"static_producer_call_count": 96
},
"event_specs": {
"ChainEventType.AgentLLMProvider": {
@@ -5825,6 +5799,28 @@
"timeout_policy": "caller_budget",
"version": 1
},
"execute_transfer_plan": {
"aggregation": "first_non_empty",
"error_policy": "isolate_provider",
"execution": "sync_or_async",
"family": "storage",
"input_contract": "TransferPlanCheckpoint",
"plugin_short_circuit": true,
"public_to_plugins": false,
"required_parameters": [
"checkpoint",
"mediainfo",
"meta",
"source_oper",
"target_oper"
],
"result_contract": "TransferInfo | None",
"result_shape": "any",
"supports_async": true,
"supports_sync": true,
"timeout_policy": "caller_budget",
"version": 1
},
"filter_torrents": {
"aggregation": "ordered_list_merge",
"error_policy": "isolate_provider",
@@ -6864,6 +6860,37 @@
"timeout_policy": "caller_budget",
"version": 1
},
"plan_transfer": {
"aggregation": "first_non_empty",
"error_policy": "isolate_provider",
"execution": "sync_or_async",
"family": "storage",
"input_contract": "TransferPlanningInput",
"plugin_short_circuit": true,
"public_to_plugins": false,
"required_parameters": [
"episodes_info",
"fileitem",
"library_category_folder",
"library_type_folder",
"mediainfo",
"meta",
"planning_input",
"preview",
"scrape",
"source_oper",
"target_directory",
"target_path",
"target_storage",
"transfer_type"
],
"result_contract": "TransferPlanCheckpoint | None",
"result_shape": "any",
"supports_async": true,
"supports_sync": true,
"timeout_policy": "caller_budget",
"version": 1
},
"recognize_media": {
"aggregation": "first_non_empty",
"error_policy": "isolate_provider",
@@ -7947,10 +7974,10 @@
}
},
"run_module": {
"call_count": 264,
"call_count": 265,
"dynamic_call_count": 0,
"dynamic_calls": [],
"method_count": 214,
"method_count": 215,
"methods": {
"anilist_credits": [
{
@@ -8769,6 +8796,13 @@
"mode": "sync"
}
],
"execute_transfer_plan": [
{
"caller": "app.chain",
"count": 1,
"mode": "sync"
}
],
"filter_torrents": [
{
"caller": "app.chain",
@@ -9190,6 +9224,13 @@
"mode": "sync"
}
],
"plan_transfer": [
{
"caller": "app.chain",
"count": 1,
"mode": "sync"
}
],
"recognize_media": [
{
"caller": "app.chain._recognition",
@@ -9514,13 +9555,6 @@
"mode": "sync"
}
],
"transfer": [
{
"caller": "app.chain",
"count": 1,
"mode": "sync"
}
],
"transfer_completed": [
{
"caller": "app.chain",
+4 -4
View File
@@ -435,17 +435,17 @@ def test_event_contract_baseline_covers_every_public_event_enum() -> None:
assert set(events["event_index"]) == expected
assert events["event_count"] == len(expected)
assert events["producer_call_count"] == 99
assert events["static_producer_call_count"] == 98
assert events["producer_call_count"] == 97
assert events["static_producer_call_count"] == 96
assert events["dynamic_producer_count"] == 1
assert events["invalid_producer_count"] == 0
assert events["producer_event_reference_count"] == 100
assert events["producer_event_reference_count"] == 98
assert events["consumer_registration_count"] == 17
assert events["static_consumer_count"] == 16
assert events["dynamic_consumer_count"] == 1
assert events["invalid_consumer_count"] == 0
assert events["consumer_event_reference_count"] == 16
assert events["fact_count"] == 116
assert events["fact_count"] == 114
assert len({fact["fingerprint"] for fact in events["consumers"]}) == 17
assert all(
not fact["caller"].startswith("app.plugins")
+4 -4
View File
@@ -946,7 +946,7 @@ def handler(event):
def test_collect_event_facts_matches_current_host_inventory() -> None:
"""统一事实覆盖当前 99 个 producer 调用与 17 个 consumer 调用。"""
"""统一事实覆盖当前 97 个 producer 调用与 17 个 consumer 调用。"""
from scripts.architecture.baseline import (
_event_enum_members,
discover_modules,
@@ -962,11 +962,11 @@ def test_collect_event_facts_matches_current_host_inventory() -> None:
producers = facts["producers"]
consumers = facts["consumers"]
assert len(producers) == 99
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 98
assert len(producers) == 97
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in producers) == 96
assert sum(fact["dynamic"] for fact in producers) == 1
assert sum(fact["invalid"] for fact in producers) == 0
assert sum(len(fact["events"]) for fact in producers) == 100
assert sum(len(fact["events"]) for fact in producers) == 98
assert len(consumers) == 17
assert sum(not fact["dynamic"] and not fact["invalid"] for fact in consumers) == 16
assert sum(fact["dynamic"] for fact in consumers) == 1
+577
View File
@@ -0,0 +1,577 @@
from dataclasses import replace
from pathlib import Path
import pytest
from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput
from app.domain.context import MediaInfo
from app.domain.meta.metabase import MetaBase
from app.modules.filemanager import transhandler as transhandler_module
from app.modules.filemanager.module import FileManagerModule
from app.modules.filemanager.transhandler import TransHandler
from app.runtime.events import eventmanager
from app.schemas.system import TransferDirectoryConf
from app.schemas.types import ChainEventType, MediaType
from app.schemas.workflow import FileItem
class ReadOnlyTreeStorage:
"""提供目录只读遍历,并让任何写接口调用立即失败。"""
def __init__(self, children: dict[str, list[FileItem]]):
"""保存按源目录路径索引的测试树。"""
self.children = children
self.reads: list[str] = []
def list(self, fileitem: FileItem) -> list[FileItem]:
"""返回目录孩子并记录只读访问。"""
self.reads.append(fileitem.path)
return self.children.get(fileitem.path, [])
def __getattr__(self, name: str):
"""拒绝规划期意外访问的存储接口。"""
raise AssertionError(f"规划阶段不应访问存储接口:{name}")
class RecordingStorage:
"""记录执行期存储调用顺序并模拟同一网盘复制。"""
def __init__(self, calls: list[str], existing: FileItem = None):
"""保存调用日志与可选同名目标。"""
self.calls = calls
self.existing = existing
def get_item_strict(self, path: Path):
"""记录严格目标查询。"""
self.calls.append("strict")
return self.existing
def get_item(self, path: Path):
"""记录普通目标查询。"""
self.calls.append("get_item")
return self.existing
def get_folder(self, path: Path) -> FileItem:
"""记录可能创建目录的接口。"""
self.calls.append("get_folder")
return FileItem(
storage="alist",
path=path.as_posix(),
name=path.name,
type="dir",
)
def delete(self, fileitem: FileItem) -> bool:
"""记录删除副作用。"""
self.calls.append("delete")
self.existing = None
return True
def is_support_transtype(self, transfer_type: str) -> bool:
"""声明测试存储支持复制。"""
return transfer_type == "copy"
def copy(self, fileitem: FileItem, path: Path, name: str) -> bool:
"""记录复制副作用。"""
self.calls.append(f"copy:{(path / name).as_posix()}")
return True
def _build_media() -> tuple[MetaBase, MediaInfo]:
"""构造文件规划所需的最小电视剧领域对象。"""
meta = MetaBase("Test.Show.S01E01.mkv")
meta.type = MediaType.TV
meta.name = "Test Show"
meta.year = "2026"
meta.begin_season = 1
meta.begin_episode = 1
mediainfo = MediaInfo(
type=MediaType.TV,
title="Test Show",
year="2026",
tmdb_id=12345,
)
return meta, mediainfo
def _build_fileitem() -> FileItem:
"""构造无需访问宿主文件系统的网盘源文件。"""
return FileItem(
storage="alist",
path="/downloads/Test.Show.S01E01.mkv",
name="Test.Show.S01E01.mkv",
basename="Test.Show.S01E01",
extension="mkv",
type="file",
size=1024,
)
def _build_input(fileitem: FileItem, **overrides) -> TransferPlanningInput:
"""构造可持久化规划输入。"""
values = {
"source_fileitem": fileitem.model_dump(mode="json"),
"target_storage": "alist",
"target_path": "/library",
"requested_transfer_type": "copy",
"need_rename": True,
"overwrite_mode": "always",
}
values.update(overrides)
return TransferPlanningInput(**values)
def _plan_file(
handler: TransHandler,
planning_input: TransferPlanningInput,
meta: MetaBase,
mediainfo: MediaInfo,
source_oper,
):
"""使用测试固定策略规划单个文件。"""
return handler.plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_storage="alist",
target_path=Path("/library"),
transfer_type="copy",
need_scrape=False,
need_rename=True,
need_notify=True,
overwrite_mode="always",
episodes_info=None,
preview=False,
)
def test_directory_planning_only_reads_source_and_freezes_ordered_leaf_operations():
root = FileItem(storage="alist", path="/source/disc", name="disc", type="dir")
nested = FileItem(storage="alist", path="/source/disc/BDMV", name="BDMV", type="dir")
first = FileItem(
storage="alist",
path="/source/disc/BDMV/index.bdmv",
name="index.bdmv",
type="file",
extension="bdmv",
)
second = FileItem(
storage="alist",
path="/source/disc/MovieObject.bdmv",
name="MovieObject.bdmv",
type="file",
extension="bdmv",
)
storage = ReadOnlyTreeStorage(
{
root.path: [nested, second],
nested.path: [first],
}
)
meta, mediainfo = _build_media()
planning_input = _build_input(root, need_rename=False)
checkpoint = TransHandler().plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=storage,
target_storage="alist",
target_path=Path("/library"),
transfer_type="copy",
need_scrape=False,
need_rename=False,
need_notify=True,
overwrite_mode="never",
episodes_info=None,
preview=False,
)
assert storage.reads == [root.path, nested.path]
assert [item.sequence for item in checkpoint.items] == [0, 1]
assert [item.source_fileitem["path"] for item in checkpoint.items] == [
first.path,
second.path,
]
assert [item.target_path for item in checkpoint.items] == [
"/library/disc/BDMV/index.bdmv",
"/library/disc/MovieObject.bdmv",
]
def test_execute_uses_frozen_target_and_intercepts_before_directory_or_delete(
monkeypatch,
):
calls: list[str] = []
fileitem = _build_fileitem()
meta, mediainfo = _build_media()
def record_event(event_type, event_data):
"""记录规划和执行事件顺序。"""
calls.append(event_type.value)
return None
monkeypatch.setattr(eventmanager, "send_event", record_event)
handler = TransHandler()
checkpoint = _plan_file(
handler,
_build_input(fileitem),
meta,
mediainfo,
ReadOnlyTreeStorage({}),
)
frozen_target = checkpoint.final_target_path
assert calls == [
ChainEventType.TransferRenameBuild.value,
ChainEventType.TransferRename.value,
]
calls.clear()
original_get_runtime_setting = transhandler_module.get_runtime_setting
def drifted_runtime_setting(name: str):
"""让执行期重新读取重命名配置时立即失败。"""
if name == "RENAME_FORMAT":
raise AssertionError("执行冻结计划不应重新读取重命名配置")
return original_get_runtime_setting(name)
monkeypatch.setattr(
transhandler_module,
"get_runtime_setting",
drifted_runtime_setting,
)
existing = FileItem(
storage="alist",
path=frozen_target,
name=Path(frozen_target).name,
type="file",
extension="mkv",
size=100,
)
storage = RecordingStorage(calls, existing=existing)
result = handler.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=storage,
target_oper=storage,
)
assert result.success is True
assert ChainEventType.TransferRenameBuild.value not in calls
assert ChainEventType.TransferRename.value not in calls
assert ChainEventType.TransferOverwriteCheck.value in calls
intercept_index = calls.index(ChainEventType.TransferIntercept.value)
assert intercept_index < calls.index("get_folder")
assert intercept_index < calls.index("delete")
assert f"copy:{frozen_target}" in calls
def test_module_preserves_admission_input_while_freezing_resolved_directory():
fileitem = _build_fileitem()
meta, mediainfo = _build_media()
admission_input = _build_input(
fileitem,
target_storage=None,
target_path=None,
requested_transfer_type=None,
target_directory={"name": "library"},
)
fingerprint = admission_input.fingerprint
target_directory = TransferDirectoryConf(
name="library",
transfer_type="copy",
overwrite_mode="never",
library_path="/resolved-library",
library_storage="alist",
renaming=True,
scraping=False,
notify=True,
)
checkpoint = FileManagerModule().plan_transfer(
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
source_oper=ReadOnlyTreeStorage({}),
planning_input=admission_input,
)
assert checkpoint.planning_input is admission_input
assert checkpoint.planning_input.fingerprint == fingerprint
assert checkpoint.root_target_path == "/resolved-library"
assert checkpoint.final_target_path.startswith("/resolved-library/")
assert checkpoint.resolved_meta_kind == "MetaBase"
assert checkpoint.resolved_meta["begin_episode"] == 1
assert checkpoint.resolved_mediainfo_kind == "MediaInfo"
assert checkpoint.resolved_mediainfo["title"] == "Test Show"
def _build_cleanup_checkpoint(monkeypatch, calls: list[str]):
"""构造带冻结旧目标清理意图的单文件检查点。"""
fileitem = _build_fileitem()
meta, mediainfo = _build_media()
cleanup_item = FileItem(
storage="cleanup",
path="/old-library/old.mkv",
name="old.mkv",
type="file",
extension="mkv",
)
planning_input = _build_input(
fileitem,
options={"cleanup_dest_fileitem": cleanup_item.model_dump(mode="json")},
)
def record_event(event_type, event_data):
"""记录执行事件顺序。"""
calls.append(event_type.value)
return None
monkeypatch.setattr(eventmanager, "send_event", record_event)
checkpoint = _plan_file(
TransHandler(),
planning_input,
meta,
mediainfo,
ReadOnlyTreeStorage({}),
)
calls.clear()
return checkpoint, meta, mediainfo, cleanup_item
def test_cleanup_runs_after_intercept_and_before_transfer_side_effects(monkeypatch):
calls: list[str] = []
checkpoint, meta, mediainfo, cleanup_item = _build_cleanup_checkpoint(
monkeypatch,
calls,
)
transfer_storage = RecordingStorage(calls)
module = FileManagerModule()
cleaned_items: list[FileItem] = []
def cleanup_media_file(fileitem: FileItem) -> bool:
"""模拟含目录保护与插件路由的统一删除兼容能力。"""
calls.append("cleanup_compat")
cleaned_items.append(fileitem)
return True
result = module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=transfer_storage,
target_oper=transfer_storage,
cleanup_media_file=cleanup_media_file,
)
assert result.success is True
intercept_index = calls.index(ChainEventType.TransferIntercept.value)
assert intercept_index < calls.index("cleanup_compat")
assert calls.index("cleanup_compat") < calls.index("get_folder")
assert calls.index("cleanup_compat") < next(
index for index, call in enumerate(calls) if call.startswith("copy:")
)
assert cleaned_items == [cleanup_item]
def test_cleanup_compatibility_capability_can_report_idempotent_success(monkeypatch):
calls: list[str] = []
checkpoint, meta, mediainfo, _ = _build_cleanup_checkpoint(monkeypatch, calls)
transfer_storage = RecordingStorage(calls)
module = FileManagerModule()
def cleanup_media_file(fileitem: FileItem) -> bool:
"""统一能力将目标不存在归一为幂等成功。"""
calls.append("cleanup_compat_missing")
return True
result = module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=transfer_storage,
target_oper=transfer_storage,
cleanup_media_file=cleanup_media_file,
)
assert result.success is True
assert "cleanup_compat_missing" in calls
def test_replayed_checkpoint_skips_cleanup_already_completed_before_provider(
monkeypatch,
):
"""provider 前已完成 cleanup 的持久检查点在宿主重放时不得再次删除。"""
calls: list[str] = []
checkpoint, meta, mediainfo, _ = _build_cleanup_checkpoint(monkeypatch, calls)
persisted_checkpoint = TransferPlanCheckpoint.from_payload(
replace(
checkpoint,
pre_execution_cleanup_completed=True,
).to_payload()
)
transfer_storage = RecordingStorage(calls)
def unexpected_cleanup(_fileitem: FileItem) -> bool:
"""若持久完成事实未被消费则立即暴露重复清理。"""
raise AssertionError("已完成的 provider 前 cleanup 不应在宿主重放时重复执行")
result = FileManagerModule().execute_transfer_plan(
persisted_checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=transfer_storage,
target_oper=transfer_storage,
cleanup_media_file=unexpected_cleanup,
)
assert result.success is True
assert ChainEventType.TransferIntercept.value in calls
assert any(call.startswith("copy:") for call in calls)
def test_cleanup_failure_raises_before_directory_or_copy(monkeypatch):
calls: list[str] = []
checkpoint, meta, mediainfo, cleanup_item = _build_cleanup_checkpoint(
monkeypatch,
calls,
)
transfer_storage = RecordingStorage(calls)
module = FileManagerModule()
def cleanup_media_file(fileitem: FileItem) -> bool:
"""模拟统一删除能力执行保护治理后的失败结果。"""
calls.append("cleanup_compat_failed")
return False
with pytest.raises(RuntimeError, match="整理计划保留待重试"):
module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=transfer_storage,
target_oper=transfer_storage,
cleanup_media_file=cleanup_media_file,
)
assert ChainEventType.TransferIntercept.value in calls
assert "cleanup_compat_failed" in calls
assert "get_folder" not in calls
assert not any(call.startswith("copy:") for call in calls)
def test_empty_directory_plan_is_zero_operation_without_intercept_or_cleanup(
monkeypatch,
):
calls: list[str] = []
root = FileItem(
storage="alist",
path="/downloads/empty",
name="empty",
type="dir",
)
cleanup_item = FileItem(
storage="alist",
path="/library/old.mkv",
name="old.mkv",
type="file",
)
planning_input = _build_input(
root,
need_rename=False,
options={"cleanup_dest_fileitem": cleanup_item.model_dump(mode="json")},
)
meta, mediainfo = _build_media()
checkpoint = TransHandler().plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=ReadOnlyTreeStorage({root.path: []}),
target_storage="alist",
target_path=Path("/library"),
transfer_type="copy",
need_scrape=False,
need_rename=False,
need_notify=True,
overwrite_mode="never",
episodes_info=None,
preview=False,
)
assert checkpoint.items == ()
def unexpected_event(event_type, event_data):
"""空计划若触发事件则立即失败。"""
raise AssertionError(f"空计划不应触发事件:{event_type}")
monkeypatch.setattr(eventmanager, "send_event", unexpected_event)
result = FileManagerModule().execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=RecordingStorage(calls),
target_oper=RecordingStorage(calls),
cleanup_media_file=lambda _fileitem: calls.append("cleanup") or True,
)
assert result.success is True
assert calls == []
def test_directory_intercept_preserves_legacy_payload(monkeypatch):
calls: list[str] = []
root = FileItem(
storage="alist",
path="/downloads/disc",
name="disc",
type="dir",
)
child = FileItem(
storage="alist",
path="/downloads/disc/index.bdmv",
name="index.bdmv",
type="file",
extension="bdmv",
)
meta, mediainfo = _build_media()
checkpoint = TransHandler().plan_transfer(
_build_input(root, need_rename=False),
meta=meta,
mediainfo=mediainfo,
source_oper=ReadOnlyTreeStorage({root.path: [child]}),
target_storage="alist",
target_path=Path("/library"),
transfer_type="copy",
need_scrape=False,
need_rename=False,
need_notify=True,
overwrite_mode="never",
episodes_info=None,
preview=False,
)
intercept_payloads = []
def capture_event(event_type, event_data):
"""捕获目录根拦截事件。"""
if event_type == ChainEventType.TransferIntercept:
intercept_payloads.append(event_data)
return None
monkeypatch.setattr(eventmanager, "send_event", capture_event)
result = TransHandler().execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=RecordingStorage(calls),
target_oper=RecordingStorage(calls),
)
assert result.success is True
assert len(intercept_payloads) == 1
payload = intercept_payloads[0]
assert payload.meta is None
assert payload.options is None
assert "meta" not in payload.model_fields_set
assert "options" not in payload.model_fields_set
+252 -1
View File
@@ -7,7 +7,11 @@ from unittest.mock import Mock
import pytest
from app.runtime.extensions.module.dispatcher import ModuleInvocationDispatcher
from app.runtime.extensions.module.dispatcher import (
FrozenModuleProviderMissingError,
FrozenPluginProviderRef,
ModuleInvocationDispatcher,
)
class _PluginCatalog:
@@ -94,6 +98,253 @@ def test_plugin_scalar_short_circuits_system_modules() -> None:
system_call.assert_not_called()
def test_strict_dispatch_propagates_plugin_failure_before_host_fallback() -> None:
"""严格查询不得把插件 provider 异常吞成空结果后继续宿主 fallback。"""
system_call = Mock(return_value=None)
module = _Module("系统", 10, system_call)
setattr(module, "get_file_item", module.execute)
def failed_provider(**_kwargs):
"""模拟插件存储查询发生网络或 I/O 故障。"""
raise RuntimeError("provider lookup failed")
dispatcher, plugin_error, _, _ = _dispatcher(
plugins={("P1", "插件一"): {"get_file_item": failed_provider}},
modules=[module],
)
with pytest.raises(RuntimeError, match="provider lookup failed"):
dispatcher.dispatch_strict(
"get_file_item",
storage="plugin",
path="/library/old.mkv",
)
plugin_error.assert_called_once()
system_call.assert_not_called()
def test_strict_dispatch_preserves_confirmed_absence() -> None:
"""全部 provider 正常返回空值时,严格查询仍以 None 表示确认不存在。"""
module = _Module("系统", 10, lambda **_kwargs: None)
setattr(module, "get_file_item", module.execute)
dispatcher, _, _, _ = _dispatcher(
plugins={("P1", "插件一"): {"get_file_item": lambda **_kwargs: None}},
modules=[module],
)
assert (
dispatcher.dispatch_strict(
"get_file_item",
storage="plugin",
path="/library/missing.mkv",
)
is None
)
def test_strict_dispatch_propagates_host_io_failure() -> None:
"""严格查询也必须传播宿主存储适配器的 I/O 故障。"""
def failed_host(**_kwargs):
"""模拟宿主存储 stat 或远端请求失败。"""
raise RuntimeError("host io failed")
module = _Module("系统", 10, failed_host)
setattr(module, "get_file_item", module.execute)
dispatcher, _, system_error, _ = _dispatcher(modules=[module])
with pytest.raises(RuntimeError, match="host io failed"):
dispatcher.dispatch_strict(
"get_file_item",
storage="local",
path="/library/old.mkv",
)
system_error.assert_called_once()
def test_frozen_plugin_providers_preserve_original_order_after_catalog_reorder() -> None:
"""冻结执行必须采用持久化顺序,不受当前插件目录重排影响。"""
calls = []
plugins = {
("P1", "插件一"): {"transfer": lambda: calls.append("P1")},
("P2", "插件二"): {"transfer": lambda: calls.append("P2")},
}
dispatcher, _, _, _ = _dispatcher(plugins=plugins)
providers = dispatcher.freeze_plugin_providers("transfer")
payloads = [provider.to_payload() for provider in providers]
restored = tuple(FrozenPluginProviderRef.from_payload(item) for item in payloads)
plugins.clear()
plugins.update(
{
("P2", "插件二"): {"transfer": lambda: calls.append("new-P2")},
("P1", "插件一"): {"transfer": lambda: calls.append("new-P1")},
}
)
assert dispatcher.execute_frozen_plugin_providers("transfer", restored) is None
assert payloads == [
{"plugin_id": "P1", "plugin_name": "插件一", "method": "transfer"},
{"plugin_id": "P2", "plugin_name": "插件二", "method": "transfer"},
]
assert calls == ["new-P1", "new-P2"]
def test_frozen_plugin_provider_propagates_failure_and_stops() -> None:
"""冻结序列中的插件异常必须向上抛出,不能伪装成空结果。"""
calls = []
def fail() -> None:
"""记录调用后模拟旧插件执行失败。"""
calls.append("P1")
raise RuntimeError("provider failed")
dispatcher, plugin_error, _, _ = _dispatcher(
plugins={
("P1", "插件一"): {"transfer": fail},
("P2", "插件二"): {
"transfer": lambda: calls.append("P2") or "success"
},
}
)
providers = dispatcher.freeze_plugin_providers("transfer")
with pytest.raises(RuntimeError, match="provider failed"):
dispatcher.execute_frozen_plugin_providers("transfer", providers)
assert calls == ["P1"]
plugin_error.assert_called_once()
assert plugin_error.call_args.args[1:4] == ("P1", "插件一", "transfer")
def test_frozen_plugin_provider_first_non_empty_short_circuits() -> None:
"""冻结 transfer 序列仍应在首个非空结果后停止。"""
second_provider = Mock(return_value="second")
dispatcher, _, _, _ = _dispatcher(
plugins={
("P1", "插件一"): {"transfer": lambda: "first"},
("P2", "插件二"): {"transfer": second_provider},
}
)
providers = dispatcher.freeze_plugin_providers("transfer")
assert (
dispatcher.execute_frozen_plugin_providers("transfer", providers) == "first"
)
second_provider.assert_not_called()
def test_frozen_plugin_provider_missing_fails_before_any_execution() -> None:
"""任一冻结 provider 缺失时必须显式失败且不得产生部分执行。"""
first_provider = Mock(return_value=None)
plugins = {
("P1", "插件一"): {"transfer": first_provider},
("P2", "插件二"): {"transfer": Mock(return_value=None)},
}
dispatcher, _, _, _ = _dispatcher(plugins=plugins)
providers = dispatcher.freeze_plugin_providers("transfer")
plugins.pop(("P2", "插件二"))
with pytest.raises(
FrozenModuleProviderMissingError,
match=r"P2/插件二\.transfer",
):
dispatcher.execute_frozen_plugin_providers("transfer", providers)
first_provider.assert_not_called()
def test_frozen_plugin_provider_missing_fails_before_pre_invoke_hook() -> None:
"""全部冻结引用解析成功前不得触发 cleanup 等前置副作用。"""
first_provider = Mock(return_value=None)
before_invoke = Mock()
plugins = {
("P1", "插件一"): {"transfer": first_provider},
("P2", "插件二"): {"transfer": Mock(return_value=None)},
}
dispatcher, _, _, _ = _dispatcher(plugins=plugins)
providers = dispatcher.freeze_plugin_providers("transfer")
plugins.pop(("P2", "插件二"))
with pytest.raises(FrozenModuleProviderMissingError):
dispatcher.execute_frozen_plugin_providers(
"transfer",
providers,
before_invoke=before_invoke,
)
before_invoke.assert_not_called()
first_provider.assert_not_called()
def test_empty_frozen_plugin_provider_sequence_skips_pre_invoke_hook() -> None:
"""没有冻结 provider 时不得执行仅服务于 provider 的前置副作用。"""
before_invoke = Mock()
dispatcher, _, _, _ = _dispatcher()
assert (
dispatcher.execute_frozen_plugin_providers(
"transfer",
(),
before_invoke=before_invoke,
)
is None
)
before_invoke.assert_not_called()
def test_ordinary_transfer_dispatch_remains_dynamic_and_compatible() -> None:
"""普通 transfer 调度仍按当前目录动态发现并采用既有短路语义。"""
system_call = Mock(return_value="system")
module = _Module("系统", 10, system_call)
setattr(module, "transfer", module.execute)
dispatcher, _, _, _ = _dispatcher(
plugins={("P1", "插件一"): {"transfer": lambda: "plugin"}},
modules=[module],
)
assert dispatcher.dispatch("transfer") == "plugin"
system_call.assert_not_called()
def test_host_internal_contract_skips_plugin_provider() -> None:
"""宿主内部两阶段协议不得暴露给同名第三方 provider。"""
plugin_call = Mock(return_value="plugin")
system_call = Mock(return_value="system")
module = _Module("系统", 10, system_call)
setattr(module, "plan_transfer", module.execute)
dispatcher, _, _, _ = _dispatcher(
plugins={
("P1", "插件一"): {"plan_transfer": plugin_call},
},
modules=[module],
)
assert dispatcher.dispatch("plan_transfer") == "system"
plugin_call.assert_not_called()
system_call.assert_called_once()
@pytest.mark.asyncio
async def test_async_host_internal_contract_skips_plugin_provider() -> None:
"""异步调度同样只允许宿主执行内部检查点协议。"""
plugin_call = Mock(return_value="plugin")
system_call = Mock(return_value="system")
module = _Module("系统", 10, system_call)
setattr(module, "execute_transfer_plan", module.execute)
dispatcher, _, _, _ = _dispatcher(
plugins={
("P1", "插件一"): {"execute_transfer_plan": plugin_call},
},
modules=[module],
)
assert await dispatcher.async_dispatch("execute_transfer_plan") == "system"
plugin_call.assert_not_called()
system_call.assert_called_once()
def test_fan_out_contract_runs_every_provider_and_ignores_results() -> None:
"""副作用广播应执行全部插件和宿主 provider,并稳定返回 None。"""
calls = []
+3 -2
View File
@@ -69,14 +69,15 @@ def test_contract_v2_freezes_every_observed_host_method() -> None:
contracts = list_explicit_module_contracts()
assert len(contracts) >= 211
for contract in contracts.values():
host_internal_methods = {"plan_transfer", "execute_transfer_plan"}
for method_name, contract in contracts.items():
assert contract.version == 1
assert contract.input_contract != "legacy_args"
assert contract.result_contract
assert contract.execution is ModuleExecutionMode.SYNC_OR_ASYNC
assert contract.timeout_policy == "caller_budget"
assert contract.error_policy is ModuleErrorPolicy.ISOLATE_PROVIDER
assert contract.public_to_plugins is True
assert contract.public_to_plugins is (method_name not in host_internal_methods)
def test_signature_diagnostics_do_not_reject_legacy_callable() -> None:
+7 -7
View File
@@ -4,16 +4,16 @@ from unittest.mock import Mock
from jinja2 import Template
from app.application.messaging.message import TemplateHelper
from app.application.transfer import TransferTask
from app.chain.media import MediaChain
from app.chain.transfer import JobManager, TransferChain
from app.runtime.config import settings
from app.domain.meta.metamusic import MetaMusic
from app.domain.context import MusicInfo
from app.application.messaging.message import TemplateHelper
from app.domain.meta.metamusic import MetaMusic
from app.runtime.config import settings
from app.schemas.file import FileItem
from app.schemas.system import TransferDirectoryConf
from app.schemas.transfer import TransferInfo, TransferTorrent
from app.application.transfer import TransferTask
from app.schemas.types import EventType, MediaType
@@ -601,7 +601,7 @@ def test_automatic_audio_transfer_runs_music_recognition(tmp_path, monkeypatch):
monkeypatch.setattr(chain, "_resolve_download_history", Mock(return_value=None))
monkeypatch.setattr(
chain,
"transfer",
"_plan_checkpoint_and_execute",
Mock(
return_value=TransferInfo(
success=True,
@@ -704,7 +704,7 @@ def test_explicit_music_batch_excludes_video_from_mixed_directory(tmp_path, monk
monkeypatch.setattr(MediaChain, "recognize_by_meta", Mock(return_value=recognized))
monkeypatch.setattr(
chain,
"transfer",
"_plan_checkpoint_and_execute",
Mock(
return_value=TransferInfo(
success=True,
@@ -727,7 +727,7 @@ def test_explicit_music_batch_excludes_video_from_mixed_directory(tmp_path, monk
assert state is True
assert [item["source"] for item in preview["items"]] == [audio_item.path]
assert chain.transfer.call_count == 1
assert chain._plan_checkpoint_and_execute.call_count == 1
def test_downloader_process_forwards_music_history_type(tmp_path, monkeypatch):
+43
View File
@@ -0,0 +1,43 @@
"""StorageChain 严格查询的三态传播合同。"""
from pathlib import Path
from unittest.mock import Mock
import pytest
from app.chain.storage import StorageChain
from app.modules.filemanager.module import FileManagerModule
from app.schemas.exception import StorageQueryError
def test_storage_chain_strict_query_distinguishes_absent_and_failure() -> None:
"""严格查询用 None 表示确认不存在,并原样传播 provider 查询失败。"""
chain = object.__new__(StorageChain)
chain._module_dispatcher = Mock()
chain._module_dispatcher.dispatch_strict.return_value = None
assert chain.get_file_item_strict("plugin", Path("/missing.mkv")) is None
chain._module_dispatcher.dispatch_strict.side_effect = StorageQueryError(
"provider lookup failed"
)
with pytest.raises(StorageQueryError, match="provider lookup failed"):
chain.get_file_item_strict("plugin", Path("/unknown.mkv"))
def test_filemanager_storage_query_uses_strict_storage_adapter(monkeypatch) -> None:
"""宿主存储 provider 必须调用 get_item_strict,不能回退会吞错的 get_item。"""
module = object.__new__(FileManagerModule)
module._support_storages = ["local"]
storage = Mock()
storage.get_item_strict.side_effect = StorageQueryError("local stat failed")
monkeypatch.setattr(
module,
"_FileManagerModule__get_storage_oper",
lambda _storage: storage,
)
with pytest.raises(StorageQueryError, match="local stat failed"):
module.get_file_item("local", Path("/library/old.mkv"))
storage.get_item.assert_not_called()
+11 -3
View File
@@ -9,8 +9,6 @@ import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from app.db.models.transferpending import TransferPending
try:
import psycopg2 as postgres_driver
from psycopg2 import sql
@@ -23,6 +21,16 @@ except ModuleNotFoundError:
POSTGRESQL_DIALECT = "postgresql+psycopg"
MIGRATION = "database.versions.b1e7d3f5a9c2_3_0_13"
ADMISSION_COLUMNS = {
"id",
"task_id",
"storage",
"src_path",
"created_at",
"state",
"updated_at",
"last_error",
}
def _bind_migration(monkeypatch, connection):
@@ -105,7 +113,7 @@ def test_transfer_admission_upgrade_downgrade_reupgrade(
assert {
column["name"]
for column in inspector.get_columns("transferpending")
} == {column.name for column in TransferPending.__table__.columns}
} == ADMISSION_COLUMNS
constraints = {
constraint["name"]
for constraint in inspector.get_unique_constraints("transferpending")
+66 -12
View File
@@ -1,21 +1,21 @@
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch, MagicMock
from unittest.mock import MagicMock, patch
from app.runtime.config import settings
from app.domain.context import MediaInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metavideo import MetaVideo
from app.chain.transfer import JobManager, TransferChain
from app.application.history import (
clear_transfer_failures,
failed_retry_count,
record_transfer_failure,
)
from app.application.transfer import TransferPlanningInput, TransferTask
from app.chain.transfer import JobManager, TransferChain
from app.domain.context import MediaInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metavideo import MetaVideo
from app.modules.filemanager.transhandler import TransHandler
from app.runtime.config import settings
from app.schemas import EpisodeFormat, FileItem, TransferInfo
from app.application.transfer import TransferTask
from app.schemas.types import EventType, MediaSource, MediaType
@@ -179,6 +179,58 @@ def make_fileitem(path: str, size: int = 1024) -> FileItem:
)
def execute_transfer_plan(
handler: TransHandler,
*,
fileitem: FileItem,
meta: MetaBase,
mediainfo: MediaInfo,
target_storage: str,
target_path: Path,
transfer_type: str,
source_oper: object,
target_oper: object,
need_scrape: bool = False,
need_notify: bool = True,
) -> TransferInfo:
"""在测试中显式规划并执行单文件整理。"""
planning_input = TransferPlanningInput(
source_fileitem=fileitem.model_dump(mode="json"),
meta=meta.to_dict(),
mediainfo=mediainfo.to_dict(),
target_storage=target_storage,
target_path=target_path.as_posix(),
requested_transfer_type=transfer_type,
media_type=mediainfo.type.value if mediainfo.type else None,
need_scrape=need_scrape,
need_rename=True,
need_notify=need_notify,
preview=False,
)
checkpoint = handler.plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_storage=target_storage,
target_path=target_path,
transfer_type=transfer_type,
need_scrape=need_scrape,
need_rename=True,
need_notify=need_notify,
overwrite_mode=None,
episodes_info=None,
preview=False,
)
return handler.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_oper=target_oper,
)
def migrate_to_media_job(jobview: JobManager, task: TransferTask):
task.mediainfo = FakeMedia()
jobview.migrate_task(task)
@@ -238,7 +290,7 @@ class TransferJobManagerTest(unittest.TestCase):
self.assertEqual("file", new_item.type)
self.assertEqual(1024, new_item.size)
def test_transfer_media_uses_target_folder_returned_by_storage(self):
def test_transfer_plan_uses_target_folder_returned_by_storage(self):
"""
整理成功时直接使用存储层返回的目标目录项回调和事件不再二次拼装
"""
@@ -294,9 +346,10 @@ class TransferJobManagerTest(unittest.TestCase):
return_value=(target_item, ""),
), patch("app.modules.filemanager.transhandler.eventmanager") as eventmanager_mock:
eventmanager_mock.send_event.return_value = None
transferinfo = handler.transfer_media(
transferinfo = execute_transfer_plan(
handler,
fileitem=source_item,
in_meta=MetaVideo("Test.Show.S01E01"),
meta=MetaVideo("Test.Show.S01E01"),
mediainfo=make_media_info(),
target_storage="alist",
target_path=target_path,
@@ -368,9 +421,10 @@ class TransferJobManagerTest(unittest.TestCase):
"app.modules.filemanager.transhandler.eventmanager.send_event",
return_value=None,
) as send_event:
transferinfo = handler.transfer_media(
transferinfo = execute_transfer_plan(
handler,
fileitem=source_item,
in_meta=in_meta,
meta=in_meta,
mediainfo=make_media_info(),
target_storage="alist",
target_path=target_path,
+34 -24
View File
@@ -11,7 +11,7 @@ import threading
from pathlib import Path
from unittest.mock import MagicMock
from app.application.transfer import TransferAdmission, TransferTask
from app.application.transfer import TransferAdmission, TransferPlanningInput, TransferTask
from app.chain.transfer import TransferChain
from app.schemas.file import FileItem
@@ -71,9 +71,11 @@ def test_admit_transfer_records_storage_and_path():
_task("/mnt/cd2/downloads/Movie.2024.mkv")
)
admissions.admit.assert_called_once_with(
storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv"
)
call = admissions.admit.call_args.kwargs
assert call["storage"] == "local"
assert call["src_path"] == "/mnt/cd2/downloads/Movie.2024.mkv"
assert isinstance(call["planning_input"], TransferPlanningInput)
assert call["planning_input"].source_fileitem["path"] == call["src_path"]
assert result.task_id == "task-1"
@@ -99,11 +101,15 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
media.write_bytes(b"x" * 10)
admissions = MagicMock()
admissions.list_accepted.return_value = [_admission(str(media))]
admissions.list_recoverable.return_value = [_admission(str(media))]
chain = _build_chain(admissions)
transferred = []
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
monkeypatch.setattr(
chain,
"_execute_transfer",
lambda **kw: transferred.append(kw["fileitem"]),
)
chain._TransferChain__replay_pending()
@@ -122,13 +128,13 @@ def test_replay_discards_vanished_files(tmp_path):
"""
admissions = MagicMock()
missing = tmp_path / "gone.mkv"
admissions.list_accepted.return_value = [_admission(str(missing))]
admissions.list_recoverable.return_value = [_admission(str(missing))]
chain = _build_chain(admissions)
chain.do_transfer = MagicMock()
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
chain._execute_transfer.assert_not_called()
admissions.discard_task.assert_called_once_with(task_id="task-1")
@@ -142,9 +148,9 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
media.write_bytes(b"x")
admissions = MagicMock()
admissions.list_accepted.return_value = [_admission(str(media))]
admissions.list_recoverable.return_value = [_admission(str(media))]
chain = _build_chain(admissions)
chain.do_transfer = MagicMock()
chain._execute_transfer = MagicMock()
def unreadable(self, *_args, **_kwargs):
"""
@@ -156,7 +162,7 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
chain._execute_transfer.assert_not_called()
admissions.discard_task.assert_not_called()
@@ -169,11 +175,15 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
src_path = f"{bluray.as_posix()}/"
admissions = MagicMock()
admissions.list_accepted.return_value = [_admission(src_path)]
admissions.list_recoverable.return_value = [_admission(src_path)]
chain = _build_chain(admissions)
transferred = []
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
monkeypatch.setattr(
chain,
"_execute_transfer",
lambda **kw: transferred.append(kw["fileitem"]),
)
chain._TransferChain__replay_pending()
@@ -187,13 +197,13 @@ def test_replay_is_noop_without_registrations():
没有登记时回放不应触碰整理链
"""
admissions = MagicMock()
admissions.list_accepted.return_value = []
admissions.list_recoverable.return_value = []
chain = _build_chain(admissions)
chain.do_transfer = MagicMock()
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
chain._execute_transfer.assert_not_called()
def test_replay_survives_db_failure():
@@ -201,13 +211,13 @@ def test_replay_survives_db_failure():
读取登记失败不能让启动流程报错
"""
admissions = MagicMock()
admissions.list_accepted.side_effect = RuntimeError("db gone")
admissions.list_recoverable.side_effect = RuntimeError("db gone")
chain = _build_chain(admissions)
chain.do_transfer = MagicMock()
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
chain._execute_transfer.assert_not_called()
def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
@@ -220,7 +230,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
item.write_bytes(b"x")
admissions = MagicMock()
admissions.list_accepted.return_value = [
admissions.list_recoverable.return_value = [
_admission(str(first), "task-1"),
_admission(str(second), "task-2"),
]
@@ -236,7 +246,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
raise RuntimeError("boom")
handled.append(kw["fileitem"].name)
monkeypatch.setattr(chain, "do_transfer", flaky)
monkeypatch.setattr(chain, "_execute_transfer", flaky)
chain._TransferChain__replay_pending()
@@ -251,7 +261,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
first.write_bytes(b"x")
missing_second = tmp_path / "gone.mkv"
admissions = MagicMock()
admissions.list_accepted.return_value = [
admissions.list_recoverable.return_value = [
_admission(str(first), "task-1"),
_admission(str(missing_second), "task-2"),
]
@@ -264,7 +274,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
transferred.append(kwargs["fileitem"].path)
stop_event.set()
monkeypatch.setattr(chain, "do_transfer", transfer_first)
monkeypatch.setattr(chain, "_execute_transfer", transfer_first)
chain._TransferChain__replay_pending(stop_event)
File diff suppressed because it is too large Load Diff
+285
View File
@@ -0,0 +1,285 @@
"""整理规划重构必须保持的旧调用与事件兼容合同。"""
from inspect import Parameter, signature
from pathlib import Path
from unittest.mock import Mock
import pytest
from app.application.transfer import TransferTask
from app.chain import ChainBase
from app.chain.transfer import TransferChain
from app.modules.filemanager.module import FileManagerModule
from app.modules.filemanager.transhandler import TransHandler
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
from app.schemas.types import EventType
def _assert_signature(callable_object, *parameters: tuple[str, object]) -> None:
"""断言公开参数顺序和默认值,不把测试辅助哨兵泄漏到失败输出。"""
actual = signature(callable_object).parameters
assert tuple(actual) == tuple(name for name, _default in parameters)
for name, expected_default in parameters:
parameter = actual[name]
if expected_default is ...:
assert parameter.default is Parameter.empty
else:
assert parameter.default == expected_default
def _fileitem() -> FileItem:
"""构造无需文件系统 I/O 的最小旧整理源对象。"""
return FileItem(
storage="local",
path="/downloads/Movie.2026.mkv",
type="file",
name="Movie.2026.mkv",
basename="Movie.2026",
extension="mkv",
size=1024,
modify_time=1770000000,
fileid="source-1",
)
def test_transfer_task_to_dict_keeps_exact_legacy_fields():
"""内部准入和规划快照不得进入插件可见的旧任务字典。"""
task = TransferTask(
fileitem=_fileitem(),
target_storage="local",
target_path=Path("/library/Movie (2026)"),
transfer_type="copy",
scrape=True,
manual=True,
background=False,
)
task.bind_admission_task_id("task-stable")
values = task.to_dict()
assert set(values) == {
"fileitem",
"meta",
"mediainfo",
"media_source",
"media_id",
"mtype",
"target_directory",
"target_storage",
"target_path",
"transfer_type",
"scrape",
"library_type_folder",
"library_category_folder",
"episodes_info",
"username",
"downloader",
"download_hash",
"download_history",
"transfer_batch_id",
"manual",
"background",
"preview",
}
assert values["fileitem"] == task.fileitem.model_dump()
assert values["target_path"] == Path("/library/Movie (2026)")
assert "admission_task_id" not in values
assert "planning_input" not in values
assert "plan_checkpoint" not in values
def test_transfer_chain_do_transfer_keeps_legacy_signature():
"""公开整理入口必须继续接受原调用方的全部关键字参数。"""
_assert_signature(
TransferChain.do_transfer,
("self", ...),
("fileitem", ...),
("meta", None),
("mediainfo", None),
("mtype", None),
("media_source", None),
("media_id", None),
("target_directory", None),
("target_storage", None),
("target_path", None),
("transfer_type", None),
("scrape", None),
("library_type_folder", None),
("library_category_folder", None),
("season", None),
("epformat", None),
("min_filesize", 0),
("downloader", None),
("download_hash", None),
("force", False),
("background", True),
("manual", False),
("preview", False),
("sync_extra_files", False),
("cleanup_dest_fileitem", None),
("continue_callback", None),
("reorganize", False),
)
def test_chain_base_transfer_keeps_legacy_signature():
"""仅 Chain 对外兼容层保留旧整理参数,宿主模块不再重复导出。"""
_assert_signature(
ChainBase.transfer,
("self", ...),
("fileitem", ...),
("meta", ...),
("mediainfo", ...),
("target_directory", None),
("target_storage", None),
("target_path", None),
("transfer_type", None),
("scrape", None),
("library_type_folder", None),
("library_category_folder", None),
("episodes_info", None),
("source_oper", None),
("target_oper", None),
("preview", False),
)
def test_chain_base_transfer_delegates_exactly_to_injected_command():
"""旧 ABI 必须原样调用注入命令,不得再次进入动态模块调度。"""
chain = object.__new__(ChainBase)
result = TransferInfo(success=True, fileitem=_fileitem(), transfer_type="copy")
command = Mock(return_value=result)
chain._legacy_transfer_command = command
chain.run_module = Mock(side_effect=AssertionError("不得调用 run_module"))
meta = Mock(name="meta")
mediainfo = Mock(name="mediainfo")
target_directory = Mock(name="target_directory")
source_oper = Mock(name="source_oper")
target_oper = Mock(name="target_oper")
episodes_info = [Mock(name="episode")]
returned = chain.transfer(
fileitem=result.fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
target_storage="alist",
target_path=Path("/library/Movie (2026)"),
transfer_type="copy",
scrape=True,
library_type_folder=False,
library_category_folder=True,
episodes_info=episodes_info,
source_oper=source_oper,
target_oper=target_oper,
preview=True,
)
assert returned is result
command.assert_called_once_with(
fileitem=result.fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
target_path=Path("/library/Movie (2026)"),
target_storage="alist",
transfer_type="copy",
scrape=True,
library_type_folder=False,
library_category_folder=True,
episodes_info=episodes_info,
source_oper=source_oper,
target_oper=target_oper,
preview=True,
)
chain.run_module.assert_not_called()
def test_chain_base_plan_transfer_keeps_internal_dto_type_only():
"""规划入口运行时只做内部调度,不要求导入或重复导出 DTO。"""
chain = object.__new__(ChainBase)
checkpoint = Mock(name="checkpoint")
chain.run_module = Mock(return_value=checkpoint)
meta = Mock(name="meta")
mediainfo = Mock(name="mediainfo")
returned = chain.plan_transfer(
fileitem=_fileitem(),
meta=meta,
mediainfo=mediainfo,
)
assert returned is checkpoint
chain.run_module.assert_called_once_with(
"plan_transfer",
fileitem=_fileitem(),
meta=meta,
mediainfo=mediainfo,
target_directory=None,
target_path=None,
target_storage=None,
transfer_type=None,
scrape=None,
library_type_folder=None,
library_category_folder=None,
episodes_info=None,
source_oper=None,
preview=False,
planning_input=None,
)
def test_filemanager_module_has_no_legacy_transfer_provider():
"""FileManager 宿主只暴露规划与执行阶段,不再注册旧 transfer provider。"""
assert not hasattr(FileManagerModule, "transfer")
assert not hasattr(TransHandler, "transfer_media")
assert callable(FileManagerModule.plan_transfer)
assert callable(FileManagerModule.execute_transfer_plan)
@pytest.mark.parametrize(
("event_type", "success"),
[
(EventType.TransferComplete, True),
(EventType.TransferFailed, False),
],
)
def test_transfer_result_event_keeps_single_exact_legacy_payload(event_type, success):
"""规划与回放不得重复发送结果事件或改变插件读取的 payload。"""
chain = object.__new__(TransferChain)
chain.eventmanager = Mock()
task = TransferTask(
fileitem=_fileitem(),
downloader="qbittorrent",
download_hash="download-1",
)
transferinfo = TransferInfo(
success=success,
fileitem=task.fileitem,
transfer_type="copy",
message="" if success else "copy failed",
)
payload = chain._transfer_result_payload(
task,
transferinfo,
history_id=42,
)
chain._publish_transfer_result(event_type, payload)
chain.eventmanager.send_event.assert_called_once_with(event_type, payload)
assert set(payload) == {
"fileitem",
"meta",
"mediainfo",
"transferinfo",
"downloader",
"download_hash",
"transfer_history_id",
}
assert payload["fileitem"] is task.fileitem
assert payload["transferinfo"] is transferinfo
assert payload["downloader"] == "qbittorrent"
assert payload["download_hash"] == "download-1"
assert payload["transfer_history_id"] == 42
+302
View File
@@ -0,0 +1,302 @@
"""整理规划输入与检查点字段的 Alembic 迁移测试。"""
import importlib
import json
import os
import uuid
import pytest
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from app.application.transfer import TransferPlanningInput
from app.db.models.transferpending import TransferPending
try:
import psycopg2 as postgres_driver
from psycopg2 import sql
POSTGRESQL_DIALECT = "postgresql+psycopg2"
except ModuleNotFoundError:
import psycopg as postgres_driver
from psycopg import sql
POSTGRESQL_DIALECT = "postgresql+psycopg"
MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
def _bind_migration(monkeypatch, connection):
"""把迁移绑定到隔离数据库连接。"""
migration = importlib.import_module(MIGRATION)
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
return migration
def _create_admission_table(connection) -> None:
"""创建 3.0.13 时代的持久准入表。"""
metadata = sa.MetaData()
table = sa.Table(
"transferpending",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("task_id", sa.String(64), nullable=False),
sa.Column("storage", sa.String(), nullable=False),
sa.Column("src_path", sa.String(), nullable=False),
sa.Column("created_at", sa.String(), nullable=True),
sa.Column("state", sa.String(32), nullable=False),
sa.Column("updated_at", sa.String(40), nullable=False),
sa.Column("last_error", sa.Text(), nullable=True),
sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"),
)
sa.Index(
"ux_transferpending_storage_path",
table.c.storage,
table.c.src_path,
unique=True,
)
sa.Index(
"ix_transferpending_state_created",
table.c.state,
table.c.created_at,
table.c.id,
)
metadata.create_all(connection)
connection.execute(table.insert(), {
"id": 1,
"task_id": "stable-task",
"storage": "local",
"src_path": "/downloads/Movie.mkv",
"created_at": "2026-08-27 10:00:00",
"state": "accepted",
"updated_at": "2026-08-27 10:00:00",
"last_error": "previous enqueue failure",
})
def _planning_row(connection) -> dict[str, object]:
"""读取单条规划持久字段快照。"""
return dict(connection.execute(sa.text(
"SELECT task_id, state, input_version, planning_input, "
"input_fingerprint, checkpoint_version, checkpoint_payload, planned_at "
"FROM transferpending WHERE id = 1"
)).mappings().one())
def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
"""断言规划迁移在当前隔离连接上的完整可逆生命周期。"""
_create_admission_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
migration.upgrade()
inspector = sa.inspect(connection)
assert {
column["name"]
for column in inspector.get_columns("transferpending")
} == {column.name for column in TransferPending.__table__.columns}
upgraded = _planning_row(connection)
planning_payload = upgraded["planning_input"]
if isinstance(planning_payload, str):
planning_payload = json.loads(planning_payload)
planning_input = TransferPlanningInput.from_payload(planning_payload)
assert planning_input == TransferPlanningInput.legacy(
storage="local",
src_path="/downloads/Movie.mkv",
)
assert upgraded["input_version"] == 1
assert upgraded["input_fingerprint"] == planning_input.fingerprint
assert upgraded["checkpoint_payload"] is None
assert upgraded["state"] == "accepted"
pending = sa.table(
"transferpending",
sa.column("id", sa.Integer()),
sa.column("state", sa.String()),
sa.column("checkpoint_version", sa.Integer()),
sa.column("checkpoint_payload", sa.JSON()),
sa.column("planned_at", sa.String()),
)
connection.execute(
pending.update()
.where(pending.c.id == 1)
.values(
state="planned",
checkpoint_version=1,
checkpoint_payload={"schema_version": 1},
planned_at="2026-08-27 11:00:00",
)
)
migration.downgrade()
downgraded = sa.inspect(connection)
assert {
column["name"]
for column in downgraded.get_columns("transferpending")
} == {
"id", "task_id", "storage", "src_path", "created_at",
"state", "updated_at", "last_error",
}
legacy = connection.execute(sa.text(
"SELECT task_id, state FROM transferpending WHERE id = 1"
)).mappings().one()
assert dict(legacy) == {"task_id": "stable-task", "state": "accepted"}
assert {
index["name"]
for index in downgraded.get_indexes("transferpending")
} == {"ix_transferpending_state_created", "ux_transferpending_storage_path"}
migration.upgrade()
reupgraded = _planning_row(connection)
assert reupgraded["task_id"] == "stable-task"
assert reupgraded["state"] == "accepted"
assert reupgraded["checkpoint_payload"] is None
assert reupgraded["input_fingerprint"] == planning_input.fingerprint
def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None:
"""SQLite 应支持规划字段重复升级、降级和再次升级。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_assert_upgrade_downgrade_reupgrade(connection, monkeypatch)
def test_provider_pending_downgrade_restores_accepted_on_sqlite(monkeypatch) -> None:
"""旧版本无法解释 provider 快照,降级时必须保守恢复为 accepted。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_admission_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
connection.execute(sa.text(
"UPDATE transferpending "
"SET state = 'provider_pending', checkpoint_version = 1, "
"checkpoint_payload = '{\"schema_version\": 1}' "
"WHERE id = 1"
))
migration.downgrade()
assert connection.execute(sa.text(
"SELECT state FROM transferpending WHERE id = 1"
)).scalar_one() == "accepted"
assert {
column["name"]
for column in sa.inspect(connection).get_columns("transferpending")
} == {
"id", "task_id", "storage", "src_path", "created_at",
"state", "updated_at", "last_error",
}
def test_partial_upgrade_preserves_existing_planning_json(monkeypatch) -> None:
"""迁移中断后重跑应补齐版本和指纹,不得覆盖已经写入的完整输入。"""
engine = sa.create_engine("sqlite://")
planning_input = TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": "/downloads/Movie.mkv",
"type": "file",
},
target_storage="local",
target_path="/library/Movies",
requested_transfer_type="copy",
options={"manual": True},
)
with engine.begin() as connection:
_create_admission_table(connection)
connection.execute(sa.text(
"ALTER TABLE transferpending ADD COLUMN planning_input JSON"
))
pending = sa.table(
"transferpending",
sa.column("id", sa.Integer()),
sa.column("planning_input", sa.JSON()),
)
connection.execute(
pending.update()
.where(pending.c.id == 1)
.values(planning_input=planning_input.to_payload())
)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
upgraded = _planning_row(connection)
payload = upgraded["planning_input"]
if isinstance(payload, str):
payload = json.loads(payload)
assert TransferPlanningInput.from_payload(payload) == planning_input
assert upgraded["input_version"] == planning_input.schema_version
assert upgraded["input_fingerprint"] == planning_input.fingerprint
connection.execute(sa.text(
"UPDATE transferpending SET state = 'future-state' WHERE id = 1"
))
migration.downgrade()
assert connection.execute(sa.text(
"SELECT state FROM transferpending WHERE id = 1"
)).scalar_one() == "future-state"
def test_transfer_planning_migration_runs_on_postgresql(monkeypatch) -> None:
"""配置隔离 PostgreSQL 时真实验证规划字段的完整可逆迁移。"""
prefix = "MOVIEPILOT_TEST_POSTGRESQL_"
host = os.getenv(f"{prefix}HOST")
database = os.getenv(f"{prefix}DATABASE")
username = os.getenv(f"{prefix}USERNAME")
if not host or not database or not username:
pytest.skip("未配置隔离 PostgreSQL migration 测试库")
port = os.getenv(f"{prefix}PORT", "5432")
password = os.getenv(f"{prefix}PASSWORD", "")
schema = f"transfer_planning_{uuid.uuid4().hex}"
with postgres_driver.connect(
host=host,
port=port,
dbname=database,
user=username,
password=password,
) as connection:
connection.autocommit = True
with connection.cursor() as cursor:
cursor.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
engine = None
try:
engine = sa.create_engine(
sa.URL.create(
POSTGRESQL_DIALECT,
username=username,
password=password,
host=host,
port=int(port),
database=database,
),
connect_args={"options": f"-csearch_path={schema}"},
)
with engine.begin() as connection:
_assert_upgrade_downgrade_reupgrade(connection, monkeypatch)
finally:
if engine is not None:
engine.dispose()
with postgres_driver.connect(
host=host,
port=port,
dbname=database,
user=username,
password=password,
) as connection:
connection.autocommit = True
with connection.cursor() as cursor:
cursor.execute(
sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(
sql.Identifier(schema)
)
)
+555
View File
@@ -0,0 +1,555 @@
"""整理规划输入与原子检查点持久化测试。"""
from dataclasses import replace
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application.transfer import (
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferAdmissionConflictError,
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
TransferPlanningStateError,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.adapters.transfer import TransactionalTransferAdmissionRepository
from app.db.models.transferpending import TransferPending
@pytest.fixture
def repository(tmp_path):
"""创建只服务单个测试的 SQLite 整理计划仓储。"""
engine = create_engine(f"sqlite:///{tmp_path / 'transfer-planning.db'}")
TransferPending.__table__.create(engine)
return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput:
"""构造包含恢复所需媒体上下文的完整规划输入。"""
return TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": "/downloads/Movie.2026.mkv",
"type": "file",
"size": 1024,
},
meta={"name": "Movie", "year": 2026},
mediainfo={"title": "Movie", "tmdb_id": 42},
target_directory={"storage": "local", "path": "/library"},
target_storage="local",
target_path=target_path,
requested_transfer_type="copy",
media_source="themoviedb",
media_id="42",
media_type="电影",
need_scrape=True,
need_rename=True,
need_notify=True,
overwrite_mode="always",
episodes_info=({"season_number": 1, "episode_number": 1},),
options={"username": "admin", "download_hash": "hash-1"},
)
def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
"""构造可直接执行且不会再次触发 rename 的计划检查点。"""
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/library",
final_target_path="/library/Movies/Movie (2026)/Movie.mkv",
resolved_transfer_type="copy",
items=(
TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path="/library/Movies/Movie (2026)/Movie.mkv",
),
),
resolved_meta=planning_input.meta,
resolved_meta_kind="MetaVideo",
resolved_mediainfo=planning_input.mediainfo,
resolved_mediainfo_kind="MediaInfo",
resolved_episodes_info=planning_input.episodes_info,
legacy_transfer_providers=(
TransferProviderReference(
plugin_id="builtin-filemanager",
plugin_name="FileManager",
),
TransferProviderReference(
plugin_id="plugin-provider-a",
plugin_name="Provider A",
),
),
need_scrape=True,
need_rename=False,
need_notify=True,
overwrite_mode="always",
)
def _provider_checkpoint(
planning_input: TransferPlanningInput,
) -> TransferPlanCheckpoint:
"""构造只冻结旧 ABI、尚未生成宿主文件计划的检查点。"""
invocation = TransferProviderInvocationSnapshot(
fileitem=planning_input.source_fileitem,
meta=planning_input.meta,
meta_kind="MetaVideo",
mediainfo=planning_input.mediainfo,
mediainfo_kind="MediaInfo",
target_directory={
"library_storage": "local",
"library_path": "/library/Movies",
"transfer_type": "copy",
},
target_storage="local",
target_path=None,
transfer_type=None,
scrape=None,
library_type_folder=False,
library_category_folder=None,
episodes_info=planning_input.episodes_info,
preview=False,
)
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
resolved_episodes_info=invocation.episodes_info,
legacy_transfer_providers=(
TransferProviderReference(
plugin_id="plugin-provider-a",
plugin_name="Provider A",
),
),
provider_invocation=invocation,
)
def test_planning_dtos_round_trip_versioned_json() -> None:
"""输入和检查点应完整往返 JSON,并保留有序叶操作。"""
planning_input = _planning_input()
checkpoint = _checkpoint(planning_input)
restored_input = TransferPlanningInput.from_payload(planning_input.to_payload())
restored_checkpoint = TransferPlanCheckpoint.from_payload(checkpoint.to_payload())
assert restored_input == planning_input
assert restored_input.fingerprint == planning_input.fingerprint
assert restored_checkpoint == checkpoint
assert [item.sequence for item in restored_checkpoint.items] == [0]
assert restored_checkpoint.planning_input == planning_input
assert restored_checkpoint.resolved_mediainfo["tmdb_id"] == 42
assert restored_checkpoint.resolved_meta_kind == "MetaVideo"
assert restored_checkpoint.resolved_mediainfo_kind == "MediaInfo"
assert restored_checkpoint.resolved_episodes_info == planning_input.episodes_info
assert restored_checkpoint.legacy_transfer_providers == (
TransferProviderReference("builtin-filemanager", "FileManager"),
TransferProviderReference("plugin-provider-a", "Provider A"),
)
assert restored_checkpoint.legacy_transfer_providers[0].method == "transfer"
def test_provider_invocation_snapshot_round_trip_preserves_optional_values() -> None:
"""旧 ABI 快照往返 JSON 后必须保留 None、False 和自动目录原始值。"""
checkpoint = _provider_checkpoint(_planning_input())
restored = TransferPlanCheckpoint.from_payload(checkpoint.to_payload())
assert restored == checkpoint
assert restored.is_provider_pending is True
assert restored.provider_invocation.target_path is None
assert restored.provider_invocation.transfer_type is None
assert restored.provider_invocation.scrape is None
assert restored.provider_invocation.library_type_folder is False
assert restored.provider_invocation.library_category_folder is None
invalid_payload = checkpoint.to_payload()
invalid_payload["provider_invocation"]["schema_version"] = 0
with pytest.raises(ValueError, match="调用快照版本"):
TransferPlanCheckpoint.from_payload(invalid_payload)
def test_legacy_checkpoint_payload_defaults_resolved_context() -> None:
"""旧检查点缺少 resolved 字段时仍应恢复为兼容空快照。"""
payload = _checkpoint(_planning_input()).to_payload()
for key in (
"resolved_meta",
"resolved_meta_kind",
"resolved_mediainfo",
"resolved_mediainfo_kind",
"resolved_episodes_info",
"legacy_transfer_providers",
):
payload.pop(key)
restored = TransferPlanCheckpoint.from_payload(payload)
assert restored.resolved_meta is None
assert restored.resolved_meta_kind is None
assert restored.resolved_mediainfo is None
assert restored.resolved_mediainfo_kind is None
assert restored.resolved_episodes_info == ()
assert restored.legacy_transfer_providers == ()
@pytest.mark.parametrize(
("plugin_id", "plugin_name", "method"),
[
("", "Provider A", "transfer"),
(" ", "Provider A", "transfer"),
("provider-a", "", "transfer"),
("provider-a", " ", "transfer"),
("provider-a", "Provider A", "delete"),
],
)
def test_transfer_provider_reference_rejects_invalid_fields(
plugin_id,
plugin_name,
method,
) -> None:
"""旧 provider 引用必须具有稳定插件身份且只能指向 transfer 方法。"""
with pytest.raises(ValueError, match="provider"):
TransferProviderReference(
plugin_id=plugin_id,
plugin_name=plugin_name,
method=method,
)
def test_checkpoint_rejects_duplicate_legacy_provider_plugin_id() -> None:
"""同一 checkpoint 不得以不同名称重复冻结同一插件身份。"""
with pytest.raises(ValueError, match="plugin_id.*重复"):
replace(
_checkpoint(_planning_input()),
legacy_transfer_providers=(
TransferProviderReference("provider-a", "Provider A"),
TransferProviderReference("provider-a", "Provider A Renamed"),
),
)
def test_checkpoint_rejects_non_array_legacy_provider_payload() -> None:
"""JSON 恢复边界不得把单个对象等非数组值当成 provider 序列。"""
payload = _checkpoint(_planning_input()).to_payload()
payload["legacy_transfer_providers"] = {
"plugin_id": "provider-a",
"plugin_name": "Provider A",
}
with pytest.raises(ValueError, match="legacy_transfer_providers"):
TransferPlanCheckpoint.from_payload(payload)
def test_resolved_context_does_not_change_admission_fingerprint(repository) -> None:
"""规划后上下文属于 checkpoint,不得反向改变已提交的准入指纹。"""
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
checkpoint = replace(
_checkpoint(planning_input),
resolved_meta={"name": "Resolved Movie", "year": 2026},
resolved_meta_kind="MetaAnime",
resolved_mediainfo={"title": "Resolved Movie", "tmdb_id": 84},
resolved_mediainfo_kind="MediaInfo",
resolved_episodes_info=({"season_number": 2, "episode_number": 3},),
)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
assert planned.input_fingerprint == planning_input.fingerprint
assert planned.planning_input == planning_input
assert planned.checkpoint.resolved_meta_kind == "MetaAnime"
assert planned.checkpoint.resolved_mediainfo["tmdb_id"] == 84
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=replace(
checkpoint,
resolved_mediainfo={"title": "Different", "tmdb_id": 85},
),
)
def test_admit_reuses_identical_input_and_rejects_conflict(repository) -> None:
"""同一源文件只允许复用完全相同的规划输入。"""
planning_input = _planning_input()
first = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
repeated = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=TransferPlanningInput.from_payload(planning_input.to_payload()),
)
assert repeated == first
with pytest.raises(TransferAdmissionConflictError):
repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=_planning_input(target_path="/other-library"),
)
def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
"""完整计划和 planned 状态应同事务提交且允许相同检查点重试。"""
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
checkpoint = _checkpoint(planning_input)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
repeated = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
assert planned.state == TRANSFER_ADMISSION_PLANNED
assert planned.checkpoint == checkpoint
assert planned.checkpoint.items[0].target_path.endswith("Movie.mkv")
assert tuple(
provider.plugin_id
for provider in planned.checkpoint.legacy_transfer_providers
) == (
"builtin-filemanager",
"plugin-provider-a",
)
assert repeated == planned
assert repository.list_accepted() == []
assert repository.list_recoverable() == [planned]
def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None:
"""崩溃可恢复的 provider 快照只能经 CAS 升级为宿主 planned 计划。"""
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
provider_checkpoint = _provider_checkpoint(planning_input)
provider_pending = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=provider_checkpoint,
)
assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING
assert provider_pending.checkpoint == provider_checkpoint
assert repository.list_recoverable() == [provider_pending]
repository.record_planning_failure(
task_id=admitted.task_id,
error="host planning unavailable",
)
failed = repository.list_recoverable()[0]
assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING
assert failed.checkpoint == provider_checkpoint
assert failed.last_error == "host planning unavailable"
host_checkpoint = _checkpoint(planning_input)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=host_checkpoint,
)
repeated = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=host_checkpoint,
)
assert planned.state == TRANSFER_ADMISSION_PLANNED
assert planned.checkpoint == host_checkpoint
assert planned.last_error is None
assert repeated == planned
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=provider_checkpoint,
)
def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> None:
"""错误输入指纹不能写入部分计划或改变 accepted 状态。"""
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
with pytest.raises(TransferAdmissionConflictError):
repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint="0" * 64,
checkpoint=_checkpoint(planning_input),
)
recovered = repository.list_recoverable()
assert len(recovered) == 1
assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED
assert recovered[0].checkpoint is None
def test_planning_failure_stays_accepted_until_success(repository) -> None:
"""规划失败只留痕,后续成功规划应清错并原子推进状态。"""
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
repository.record_planning_failure(task_id=admitted.task_id, error="rename failed")
failed = repository.list_recoverable()[0]
assert failed.state == TRANSFER_ADMISSION_ACCEPTED
assert failed.last_error == "rename failed"
assert failed.checkpoint is None
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
assert planned.state == TRANSFER_ADMISSION_PLANNED
assert planned.last_error is None
def test_checkpoint_rejects_missing_task(repository) -> None:
"""不存在的稳定任务身份不能凭空创建已规划记录。"""
planning_input = _planning_input()
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id="missing",
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
"""兼容直接构造 ORM 行时也必须生成匹配路径的版本化输入与指纹。"""
engine = create_engine(f"sqlite:///{tmp_path / 'orm-defaults.db'}")
factory = sessionmaker(bind=engine)
TransferPending.__table__.create(engine)
with factory() as session:
session.add(TransferPending(
storage="local",
src_path="/downloads/legacy.mkv",
state=TRANSFER_ADMISSION_ACCEPTED,
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
))
session.commit()
admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0]
assert admitted.planning_input == TransferPlanningInput.legacy(
storage="local",
src_path="/downloads/legacy.mkv",
)
assert admitted.input_fingerprint == admitted.planning_input.fingerprint
engine.dispose()
def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -> None:
"""列版本、JSON 和指纹任一不一致时都不得返回伪冻结 DTO。"""
engine = create_engine(f"sqlite:///{tmp_path / 'input-corruption.db'}")
factory = sessionmaker(bind=engine)
TransferPending.__table__.create(engine)
repository = TransactionalTransferAdmissionRepository(factory)
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
with factory() as session:
row = session.execute(
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
).scalar_one()
row.input_version = 2
session.commit()
with pytest.raises(TransferPlanningStateError, match="版本"):
repository.list_accepted()
with factory() as session:
row = session.execute(
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
).scalar_one()
row.input_version = 1
corrupted = planning_input.to_payload()
corrupted["media_id"] = "different"
row.planning_input = corrupted
session.commit()
with pytest.raises(TransferAdmissionConflictError, match="指纹"):
repository.list_accepted()
engine.dispose()
def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
"""planned 行的列版本与自包含 checkpoint JSON 必须严格一致。"""
engine = create_engine(f"sqlite:///{tmp_path / 'checkpoint-corruption.db'}")
factory = sessionmaker(bind=engine)
TransferPending.__table__.create(engine)
repository = TransactionalTransferAdmissionRepository(factory)
planning_input = _planning_input()
admitted = repository.admit(
storage="local",
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
repository.checkpoint_plan(
task_id=admitted.task_id,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
with factory() as session:
row = session.execute(
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
).scalar_one()
row.checkpoint_version = 2
session.commit()
with pytest.raises(TransferPlanningStateError, match="版本"):
repository.list_recoverable()
engine.dispose()
+30 -8
View File
@@ -1,6 +1,5 @@
from pathlib import Path
from app.domain.context import MediaInfo
from app.domain.meta.metabase import MetaBase
from app.modules.filemanager import FileManagerModule
@@ -78,15 +77,22 @@ def test_cloud_storage_preview_only_calculates_target_path():
)
guarded_storage = GuardedStorage()
transferinfo = FileManagerModule().transfer(
module = FileManagerModule()
checkpoint = module.plan_transfer(
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
source_oper=guarded_storage,
target_oper=guarded_storage,
preview=True,
)
transferinfo = module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=guarded_storage,
target_oper=guarded_storage,
)
assert transferinfo.success is True
assert transferinfo.need_notify is False
@@ -131,15 +137,22 @@ def test_local_storage_preview_skips_target_conflict_checks(tmp_path):
)
guarded_storage = GuardedStorage()
transferinfo = FileManagerModule().transfer(
module = FileManagerModule()
checkpoint = module.plan_transfer(
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
source_oper=guarded_storage,
target_oper=guarded_storage,
preview=True,
)
transferinfo = module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=guarded_storage,
target_oper=guarded_storage,
)
assert transferinfo.success is True
assert transferinfo.need_notify is False
@@ -188,15 +201,24 @@ def _build_bluray_dir_preview(
notify=True,
)
return FileManagerModule().transfer(
module = FileManagerModule()
source_oper = GuardedStorage()
target_oper = GuardedStorage()
checkpoint = module.plan_transfer(
fileitem=fileitem,
meta=meta,
mediainfo=mediainfo,
target_directory=target_directory,
source_oper=GuardedStorage(),
target_oper=GuardedStorage(),
source_oper=source_oper,
preview=True,
)
return module.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_oper=target_oper,
)
def test_tv_bluray_dir_preview_preserves_disk_folder_from_meta_part():
+15 -16
View File
@@ -688,12 +688,11 @@ def test_single_matching_subtitle_uses_unmatched_video_only_as_context(monkeypat
assert planned == [(subtitle_fileitem.path, 2)]
def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeypatch):
def test_cleanup_dest_fileitem_is_checkpointed_only_after_allowed_items_exist(monkeypatch):
"""
旧目标文件只应在模板筛选后确实存在待整理任务时清理
旧目标清理意图只应在模板筛选后确实存在待整理任务时进入规划输入
"""
chain = make_transfer_chain()
delete_calls = []
planned = []
main_fileitem = make_fileitem(
"/downloads/Test Show (2026)/Show - 01.mkv"
@@ -721,9 +720,14 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp
def fake_handle_transfer(task, callback=None):
"""
记录旧目标清理后的整理任务
记录实际任务携带的冻结 cleanup intent
"""
planned.append(task.fileitem.path)
planned.append(
(
task.fileitem.path,
task.planning_input.options.get("cleanup_dest_fileitem"),
)
)
return True, ""
monkeypatch.setattr(chain, "_TransferChain__handle_transfer", fake_handle_transfer)
@@ -752,15 +756,6 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp
lambda: SimpleNamespace(get=lambda key: None),
)
monkeypatch.setattr("app.chain._transfer.get_configured_system_config", lambda: SimpleNamespace(get=lambda key: None))
monkeypatch.setattr(
"app.chain.transfer.StorageChain",
lambda: SimpleNamespace(
delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True,
),
)
monkeypatch.setattr("app.chain._transfer.StorageChain", lambda: SimpleNamespace(
delete_media_file=lambda fileitem: delete_calls.append(fileitem.path) or True,
))
monkeypatch.setattr("app.chain.transfer.MetaInfoPath", lambda path, custom_words=None, **kwargs: FakeMeta(1))
state, errmsg = TransferChain.do_transfer(
@@ -773,8 +768,12 @@ def test_cleanup_dest_fileitem_is_deleted_only_after_allowed_items_exist(monkeyp
assert state is True
assert errmsg == ""
assert delete_calls == [old_dest_fileitem.path]
assert planned == [main_fileitem.path]
assert planned == [
(
main_fileitem.path,
old_dest_fileitem.model_dump(mode="json"),
)
]
def test_cleanup_dest_fileitem_is_kept_when_episode_format_matches_nothing(monkeypatch):
+8 -1
View File
@@ -447,7 +447,14 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._TransferChain__handle_transfer = MagicMock(return_value=(True, ""))
def complete_with_checkpoint(*, task, callback):
"""模拟真实 worker 只有提交 checkpoint 后才返回终态成功。"""
task.bind_plan_checkpoint(MagicMock())
return True, ""
chain._TransferChain__handle_transfer = MagicMock(
side_effect=complete_with_checkpoint
)
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
assert chain.put_to_queue(task) is True
+45 -8
View File
@@ -3,6 +3,7 @@ from unittest.mock import MagicMock
import pytest
from app.application.transfer import TransferPlanningInput
from app.domain.context import MediaInfo
from app.domain.meta.metavideo import MetaVideo
from app.modules.filemanager.transhandler import TransHandler
@@ -28,17 +29,53 @@ def _transfer_without_episode(file_name: str) -> tuple[TransferInfo, MagicMock,
source_oper = MagicMock()
target_oper = MagicMock()
result = TransHandler().transfer_media(
fileitem=fileitem,
in_meta=meta,
mediainfo=mediainfo,
handler = TransHandler()
planning_input = TransferPlanningInput(
source_fileitem=fileitem.model_dump(mode="json"),
meta=meta.to_dict(),
mediainfo=mediainfo.to_dict(),
target_storage="local",
target_path=Path("/library"),
transfer_type="copy",
source_oper=source_oper,
target_oper=target_oper,
target_path="/library",
requested_transfer_type="copy",
media_type=MediaType.TV.value,
need_scrape=False,
need_rename=True,
need_notify=True,
preview=False,
)
try:
checkpoint = handler.plan_transfer(
planning_input,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_storage="local",
target_path=Path("/library"),
transfer_type="copy",
need_scrape=False,
need_rename=True,
need_notify=True,
overwrite_mode=None,
episodes_info=None,
preview=False,
)
except ValueError as error:
result = TransferInfo(
success=False,
message=str(error),
fileitem=fileitem,
fail_list=[fileitem.path],
transfer_type="copy",
need_notify=True,
)
else:
result = handler.execute_transfer_plan(
checkpoint,
meta=meta,
mediainfo=mediainfo,
source_oper=source_oper,
target_oper=target_oper,
)
return result, source_oper, target_oper