GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-08-31 04:57:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge upstream/v3 into codex/feat/plugin-data-query-sdk-v3

Browse Source
This commit is contained in:
InfinityPacer
2026-08-28 00:43:57 +08:00
parent e31b3f8de7 a2e249f206
commit 7657618f08
49 changed files with 5423 additions and 1143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/application/transfer/execution.py
+74 -7
View File
@@ -43,6 +43,14 @@ class TransferTerminalState(StrEnum):
FAILED = "failed"
class TransferExecutionOutcome(StrEnum):
"""描述执行检查点可裁决的业务结果。"""
SUCCEEDED = "succeeded"
FAILED = "failed"
OVERWRITE_SKIPPED = "overwrite_skipped"
class TransferOperationObservationState(StrEnum):
"""描述重启后对遗留 STARTED 外部操作的严格探测结论。"""
@@ -249,6 +257,29 @@ class TransferExecutionCheckpoint:
raise ValueError("整理执行检查点包含无效操作标识")
if not self.operation_ids and not self.skip_reason:
raise ValueError("零副作用整理执行检查点必须记录 skip_reason")
raw_outcome = self.payload.get("outcome")
try:
outcome = TransferExecutionOutcome(
raw_outcome if isinstance(raw_outcome, str) else ""
)
except (TypeError, ValueError) as error:
raise ValueError("整理执行检查点 outcome 无效") from error
transferinfo = self.payload.get("transferinfo")
if transferinfo is not None:
if not isinstance(transferinfo, dict):
raise ValueError("整理执行检查点 TransferInfo 必须是 JSON 对象")
expected_success = outcome is TransferExecutionOutcome.SUCCEEDED
expected_overwrite_skip = (
outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED
)
if (
bool(transferinfo.get("success")) != expected_success
or bool(transferinfo.get("overwrite_skipped"))
!= expected_overwrite_skip
):
raise ValueError("整理执行 checkpoint outcome 与 TransferInfo 不一致")
elif outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED:
raise ValueError("覆盖跳过执行检查点缺少冻结 TransferInfo")
_canonical_json(self.payload)
if build_transfer_checkpoint_fingerprint(self.to_payload()) != self.fingerprint:
raise ValueError("整理执行检查点内容与指纹不一致")
@@ -285,6 +316,35 @@ class TransferExecutionCheckpoint:
"skip_reason": self.skip_reason,
}
def validate_settlement_outcome(
self,
settlement_outcome: str,
) -> TransferTerminalState:
"""解析执行事实，并校验同事务历史裁决给出的结算方向。"""
raw_outcome = self.payload.get("outcome")
try:
outcome = TransferExecutionOutcome(
raw_outcome if isinstance(raw_outcome, str) else ""
)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理执行检查点 outcome 无效"
) from error
try:
terminal_state = TransferTerminalState(settlement_outcome)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理结算 outcome 无效"
) from error
if (
outcome != TransferExecutionOutcome.OVERWRITE_SKIPPED
and outcome.value != terminal_state.value
):
raise TransferExecutionConflictError(
"整理结算 outcome 与执行检查点不一致"
)
return terminal_state
@classmethod
def from_payload(
cls,
@@ -293,6 +353,8 @@ class TransferExecutionCheckpoint:
fingerprint: str,
) -> "TransferExecutionCheckpoint":
"""解析并校验数据库中的执行检查点版本与稳定指纹。"""
if not isinstance(payload, Mapping):
raise TransferExecutionConflictError("整理执行检查点 JSON 结构无效")
serialized = dict(payload)
if build_transfer_checkpoint_fingerprint(serialized) != fingerprint:
raise TransferExecutionConflictError("整理执行检查点 JSON 与指纹不一致")
@@ -308,13 +370,18 @@ class TransferExecutionCheckpoint:
skip_reason = serialized.get("skip_reason")
if skip_reason is not None and not isinstance(skip_reason, str):
raise TransferExecutionConflictError("整理执行跳过原因类型无效")
return cls(
fingerprint=fingerprint,
payload=result_payload,
operation_ids=tuple(operation_ids),
skip_reason=skip_reason,
version=version,
)
try:
return cls(
fingerprint=fingerprint,
payload=result_payload,
operation_ids=tuple(operation_ids),
skip_reason=skip_reason,
version=version,
)
except ValueError as error:
raise TransferExecutionConflictError(
"整理执行检查点内容无效"
) from error
@dataclass(frozen=True, slots=True)
app/application/transfer/workflow.py
+22 -173
View File
@@ -38,7 +38,6 @@ from typing import (
from pydantic import BaseModel, ConfigDict, PrivateAttr
from app.adapters.system.host import SystemUtils
from app.application.agent import get_prompt_manager, get_running_agent_manager
from app.application.transfer.execution import TransferExecutionCheckpoint
from app.domain.context import MediaInfo, MusicInfo
from app.domain.media import normalize_music_type
@@ -61,7 +60,6 @@ from app.schemas.types import (
MUSIC_ENTITY_RECORDING,
MediaSource,
MediaType,
ReplyMode,
)
if TYPE_CHECKING:
@@ -209,14 +207,6 @@ class TransferPlanningInput:
raise ValueError("整理规划输入缺少源文件存储或路径")
_canonical_json(self.to_payload())
@classmethod
def legacy(cls, *, storage: str, src_path: str) -> "TransferPlanningInput":
"""为升级前只有存储与路径的登记构造保守重规划输入。"""
return cls(
source_fileitem={"storage": storage, "path": src_path},
options={"legacy_replan": True},
)
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanningInput":
"""从受版本约束的 JSON 对象恢复规划输入。"""
@@ -495,6 +485,7 @@ class TransferPlanCheckpoint:
overwrite_mode: Optional[str] = None
preview: bool = False
skip_reason: Optional[str] = None
rejection_error: Optional[str] = None
schema_version: int = TRANSFER_PLAN_CHECKPOINT_VERSION
def __post_init__(self) -> None:
@@ -551,6 +542,7 @@ class TransferPlanCheckpoint:
or self.resolved_transfer_type
or self.items
or self.skip_reason
or self.rejection_error
):
raise ValueError("provider_pending 检查点不得包含宿主执行计划")
else:
@@ -558,6 +550,12 @@ class TransferPlanCheckpoint:
raise ValueError("整理计划检查点缺少目标身份")
if not self.resolved_transfer_type:
raise ValueError("整理计划检查点缺少已解析的整理方式")
if self.rejection_error is not None and (
not isinstance(self.rejection_error, str)
or not self.rejection_error.strip()
or self.items
):
raise ValueError("整理拒绝检查点必须包含非空错误且不得包含文件步骤")
if tuple(item.sequence for item in self.items) != tuple(range(len(self.items))):
raise ValueError("整理计划项必须按从零开始的连续序号保存")
if (
@@ -565,6 +563,7 @@ class TransferPlanCheckpoint:
and not self.items
and not self.preview
and not self.skip_reason
and not self.rejection_error
):
raise ValueError("非预览空计划必须记录合法跳过原因")
_canonical_json(self.to_payload())
@@ -574,6 +573,13 @@ class TransferPlanCheckpoint:
"""返回该检查点是否只冻结 provider 调用、尚未生成宿主计划。"""
return self.provider_invocation is not None
@property
def fingerprint(self) -> str:
"""返回完整冻结计划规范 JSON 的稳定 SHA-256 指纹。"""
return hashlib.sha256(
_canonical_json(self.to_payload()).encode("utf-8")
).hexdigest()
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanCheckpoint":
"""从受版本约束的 JSON 对象恢复完整执行检查点。"""
@@ -634,6 +640,7 @@ class TransferPlanCheckpoint:
overwrite_mode=payload.get("overwrite_mode"),
preview=payload.get("preview", False),
skip_reason=payload.get("skip_reason"),
rejection_error=payload.get("rejection_error"),
schema_version=payload.get("schema_version", 0),
)
@@ -672,6 +679,7 @@ class TransferPlanCheckpoint:
"overwrite_mode": self.overwrite_mode,
"preview": self.preview,
"skip_reason": self.skip_reason,
"rejection_error": self.rejection_error,
}
@@ -847,9 +855,9 @@ class TransferAdmission:
state: str
created_at: str
updated_at: str
planning_input: TransferPlanningInput
last_error: Optional[str] = None
input_fingerprint: Optional[str] = None
planning_input: Optional[TransferPlanningInput] = None
checkpoint: Optional[TransferPlanCheckpoint] = None
lease_owner: Optional[str] = None
lease_token: Optional[str] = None
@@ -866,7 +874,7 @@ class TransferAdmissionRepository(Protocol):
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
planning_input: TransferPlanningInput,
) -> TransferAdmission:
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
...
@@ -932,8 +940,8 @@ class TransferAdmissionRepository(Protocol):
"""按 token 释放当前 claim，并可记录可恢复错误。"""
...
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅允许当前 lease owner 删除已经到达终态的登记。"""
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
"""仅允许当前 lease owner 删除确认未开始且源已消失的登记。"""
...
@@ -1929,162 +1937,3 @@ class JobManager:
with job_lock:
__mediaid__ = self.__get_media_id(media=media, season=season)
return self._season_episodes.get(__mediaid__) or []
class FailedRetryScheduler:
"""
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。
缓冲不提供持久化保证；关闭时会取消尚未触发的记录，由上层 durable
工作流在后续阶段承接需要跨进程保证的重试意图。
"""
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
def __init__(self) -> None:
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
super().__init__()
self._retry_transfer_buffer: dict[str, list[int]] = {}
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
self._retry_transfer_generations: dict[str, int] = {}
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
self._retry_transfer_lock = asyncio.Lock()
self._closed = False
async def close(self) -> None:
"""停止接收重试，取消定时器，并等待活跃 flush 任务完成取消。"""
async with self._retry_transfer_lock:
self._closed = True
timers = list(self._retry_transfer_timers.values())
buffered_count = sum(
len(history_ids)
for history_ids in self._retry_transfer_buffer.values()
)
self._retry_transfer_timers.clear()
self._retry_transfer_generations.clear()
self._retry_transfer_buffer.clear()
tasks = tuple(self._retry_transfer_tasks)
for timer in timers:
timer.cancel()
if buffered_count:
logger.warning(
f"智能体重试整理调度器关闭，取消 {buffered_count} 条未持久化缓冲记录"
)
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
if self._closed:
return
task = asyncio.create_task(
self._flush_retry_transfer(group_key, generation),
name="transfer.failed_retry.flush",
)
self._retry_transfer_tasks.add(task)
task.add_done_callback(self._observe_retry_transfer_task)
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
"""移除已结束任务，并观察未被 flush 逻辑处理的异常。"""
self._retry_transfer_tasks.discard(task)
if task.cancelled():
return
exception = task.exception()
if exception is not None:
logger.error(f"智能体重试整理后台任务异常: {exception}")
@staticmethod
def _build_retry_transfer_template_context(
history_ids: list[int],
) -> tuple[str, dict[str, int | str]]:
"""仅负责把失败重试任务的动态数据映射成模板变量。"""
is_batch = len(history_ids) > 1
task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry"
template_context: dict[str, int | str] = {
"history_ids_csv": ", ".join(str(item) for item in history_ids),
"history_count": len(history_ids),
}
if not is_batch:
template_context["history_id"] = history_ids[0]
return task_type, template_context
def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str:
"""根据失败记录数量构建统一的重试整理后台任务提示词。"""
task_type, template_context = self._build_retry_transfer_template_context(history_ids)
return cast(str, get_prompt_manager().render_system_task_message(
task_type,
template_context=template_context,
))
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
"""
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
"""
if not group_key:
group_key = f"_default_{history_id}"
async with self._retry_transfer_lock:
if self._closed:
raise RuntimeError("智能体重试整理调度器正在关闭，不能再接收任务")
if group_key not in self._retry_transfer_buffer:
self._retry_transfer_buffer[group_key] = []
if history_id not in self._retry_transfer_buffer[group_key]:
self._retry_transfer_buffer[group_key].append(history_id)
logger.info(
f"智能体重试整理：记录 ID={history_id} 已加入缓冲区 "
f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)"
)
if group_key in self._retry_transfer_timers:
self._retry_transfer_timers[group_key].cancel()
loop = asyncio.get_running_loop()
generation = self._retry_transfer_generations.get(group_key, 0) + 1
self._retry_transfer_generations[group_key] = generation
self._retry_transfer_timers[group_key] = loop.call_later(
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
self._start_retry_transfer_task,
group_key,
generation,
)
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
"""
延迟定时器到期后，取出该分组的所有 history_id 并合并为一次 agent 调用。
"""
async with self._retry_transfer_lock:
# callback 到期与真正取得锁之间可能有新记录续期；旧代不能提前取走新批次。
if self._retry_transfer_generations.get(group_key) != generation:
return
history_ids = self._retry_transfer_buffer.pop(group_key, [])
self._retry_transfer_timers.pop(group_key, None)
self._retry_transfer_generations.pop(group_key, None)
if not history_ids:
return
ids_str = ", ".join(str(item) for item in history_ids)
logger.info(
f"智能体重试整理：开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})"
)
try:
manager = get_running_agent_manager()
if manager is None:
logger.warning("智能助手服务未运行，跳过整理失败自动重试")
return
await manager.run_background_prompt(
message=self._build_retry_transfer_prompt(history_ids),
session_prefix="__agent_retry_transfer_batch",
reply_mode=ReplyMode.DISPATCH,
)
logger.info(
f"智能体重试整理：批量处理完成 IDs=[{ids_str}] (group={group_key})"
)
except Exception as err:
logger.error(
f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}"
)
app/chain/transfer.py
+363 -373
View File
File diff suppressed because it is too large Load Diff
app/db/adapters/chain.py
+8
View File
@@ -26,6 +26,7 @@ from app.application.outbox import (
OutboxIntent,
)
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferExecutionConflictError,
TransferExecutionLeaseLostError,
TransferExecutionState,
@@ -403,6 +404,13 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
!= settlement.execution_fingerprint
):
raise TransferExecutionConflictError("整理终态与执行检查点不匹配")
checkpoint = TransferExecutionCheckpoint.from_payload(
pending.execution_payload,
fingerprint=settlement.execution_fingerprint,
)
if pending.execution_version != checkpoint.version:
raise TransferExecutionConflictError("整理执行检查点列版本与内容不一致")
checkpoint.validate_settlement_outcome(settlement.outcome)
return int(pending.settlement_revision)
@staticmethod
app/db/adapters/transfer/admission.py
+11 -15
View File
@@ -142,18 +142,14 @@ class TransactionalTransferAdmissionRepository:
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
planning_input: TransferPlanningInput,
) -> TransferAdmission:
"""按输入指纹幂等持久化准入事实，并返回跨重启稳定身份。"""
if not storage or not src_path:
raise ValueError("整理任务的存储与源路径不能为空")
effective_input = planning_input or TransferPlanningInput.legacy(
storage=storage,
src_path=src_path,
)
if (
effective_input.source_fileitem.get("storage") != storage
or effective_input.source_fileitem.get("path") != src_path
planning_input.source_fileitem.get("storage") != storage
or planning_input.source_fileitem.get("path") != src_path
):
raise ValueError("整理规划输入的源文件身份与准入参数不一致")
now_time = self._now()
@@ -167,16 +163,16 @@ class TransactionalTransferAdmissionRepository:
src_path=src_path,
state=TRANSFER_ADMISSION_ACCEPTED,
now_time=now_time,
input_version=effective_input.schema_version,
planning_input=effective_input.to_payload(),
input_fingerprint=effective_input.fingerprint,
input_version=planning_input.schema_version,
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
)
if pending is None:
raise TransferAdmissionConflictError(
f"整理源文件已有持久终态回执: {storage}:{src_path}"
)
session.flush()
self._assert_input_match(pending, effective_input)
self._assert_input_match(pending, planning_input)
admission = self._project(pending)
transaction.commit()
return admission
@@ -192,7 +188,7 @@ class TransactionalTransferAdmissionRepository:
)
if pending is None:
raise RuntimeError("并发准入冲突后未找到已提交记录") from error
self._assert_input_match(pending, effective_input)
self._assert_input_match(pending, planning_input)
return self._project(pending)
def claim_task(
@@ -525,14 +521,14 @@ class TransactionalTransferAdmissionRepository:
transaction.rollback()
raise
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前未过期 token 删除终态任务，拒绝陈旧 worker 变更。"""
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前 token 删除无执行证据的缺失源任务，拒绝陈旧 worker。"""
if not task_id or not lease_token:
return 0
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
deleted = TransferPendingOper(db=session).stage_discard_claimed(
deleted = TransferPendingOper(db=session).stage_abandon_unstarted(
task_id=task_id,
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
app/db/adapters/transfer/execution.py
+412 -12
View File
@@ -4,7 +4,8 @@ from __future__ import annotations
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Optional
from pathlib import Path
from typing import Any, Optional
from sqlalchemy import and_, func, or_, select
from sqlalchemy.exc import IntegrityError
@@ -27,6 +28,14 @@ from app.application.transfer.execution import (
TransferStepIntent,
TransferStepResult,
TransferStepState,
build_transfer_operation_id,
)
from app.application.transfer.workflow import (
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferPlanCheckpoint,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.models.transferexecutionstep import (
TransferExecutionStep as TransferExecutionStepModel,
@@ -229,14 +238,332 @@ class TransactionalTransferExecutionRepository:
))
@staticmethod
def _raise_fenced_failure(
def _plan_identity(
pending: TransferPending,
) -> tuple[TransferPlanCheckpoint, str]:
"""恢复完整冻结计划，并返回其规范 payload 的稳定指纹。"""
if pending.state not in {
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
}:
raise TransferExecutionConflictError("整理任务尚未进入可执行规划状态")
if (
pending.checkpoint_version is None
or pending.checkpoint_payload is None
or pending.planned_at is None
):
raise TransferExecutionConflictError("整理任务缺少完整计划检查点")
try:
checkpoint = TransferPlanCheckpoint.from_payload(
pending.checkpoint_payload
)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理任务计划检查点无法恢复"
) from error
if pending.checkpoint_version != checkpoint.schema_version:
raise TransferExecutionConflictError("整理任务计划检查点版本不一致")
if checkpoint.planning_input.fingerprint != pending.input_fingerprint:
raise TransferExecutionConflictError("整理任务计划与准入输入指纹不一致")
expected_state = (
TRANSFER_ADMISSION_PROVIDER_PENDING
if checkpoint.is_provider_pending
else TRANSFER_ADMISSION_PLANNED
)
if pending.state != expected_state:
raise TransferExecutionConflictError("整理任务计划类型与准入状态不一致")
return checkpoint, checkpoint.fingerprint
@staticmethod
def _provider_predecessor_checkpoint(
checkpoint: TransferPlanCheckpoint,
step: TransferExecutionStepModel,
) -> Optional[TransferPlanCheckpoint]:
"""重建 provider 回退前的冻结计划，证明序号零步骤的历史归属。"""
if (
not checkpoint.pre_execution_cleanup_completed
or step.ordinal != 0
or step.phase != "provider"
or step.kind != "legacy_transfer_provider_sequence"
):
return None
providers_payload = step.intent_payload.get("providers")
invocation_payload = step.intent_payload.get("invocation")
if (
not isinstance(providers_payload, list)
or not providers_payload
or not all(isinstance(item, dict) for item in providers_payload)
or not isinstance(invocation_payload, dict)
):
return None
try:
providers = tuple(
TransferProviderReference.from_payload(item)
for item in providers_payload
)
invocation = TransferProviderInvocationSnapshot.from_payload(
invocation_payload
)
if invocation.fileitem != checkpoint.planning_input.source_fileitem:
return None
predecessor = TransferPlanCheckpoint(
planning_input=checkpoint.planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
resolved_episodes_info=invocation.episodes_info,
legacy_transfer_providers=providers,
provider_invocation=invocation,
preview=invocation.preview,
)
except (TypeError, ValueError):
return None
return predecessor
@staticmethod
def _intent_belongs_to_checkpoint(
checkpoint: TransferPlanCheckpoint,
*,
phase: str,
kind: str,
payload: dict[str, Any],
previous_steps: list[TransferExecutionStepModel],
) -> bool:
"""校验步骤类型及稳定参数由冻结计划或其已提交发现证据导出。"""
if checkpoint.rejection_error:
return (
phase == "planning"
and kind == "reject"
and payload == {"error": checkpoint.rejection_error}
)
if checkpoint.is_provider_pending:
invocation = checkpoint.provider_invocation
if invocation is None:
return False
return (
phase == "provider"
and kind == "legacy_transfer_provider_sequence"
and payload == {
"providers": [
provider.to_payload()
for provider in checkpoint.legacy_transfer_providers
],
"invocation": invocation.to_payload(),
}
)
plan_items = tuple(checkpoint.items)
if phase == "transfer" and kind == checkpoint.resolved_transfer_type:
return any(payload == {
"source": item.source_fileitem.get("path"),
"target": item.target_path,
} for item in plan_items)
if phase == "transfer" and kind == "materialize_target":
return any(payload == {
"source": item.source_fileitem,
"target_storage": item.target_storage,
"target_path": item.target_path,
"transfer_type": (
"copy"
if (
checkpoint.resolved_transfer_type == "move"
and item.source_fileitem.get("storage")
!= item.target_storage
)
else checkpoint.resolved_transfer_type
),
} for item in plan_items)
if phase == "transfer" and kind == "delete_move_source":
return (
checkpoint.resolved_transfer_type == "move"
and any(
item.source_fileitem.get("storage") != item.target_storage
and payload == {
"source": item.source_fileitem,
"target_storage": item.target_storage,
"target_path": item.target_path,
}
for item in plan_items
)
)
source = checkpoint.planning_input.source_fileitem
if phase == "prepare" and kind == "cleanup_previous_destination":
return payload == {"source_path": source.get("path")}
if phase == "prepare" and kind == "ensure_target_directory":
target_path = Path(checkpoint.final_target_path)
directory_path = (
target_path
if source.get("type") == "dir"
else target_path.parent
)
return payload == {
"storage": checkpoint.target_storage,
"path": directory_path.as_posix(),
}
if phase == "prepare" and kind == "delete_overwrite_target":
return payload == {
"storage": checkpoint.target_storage,
"path": checkpoint.final_target_path,
}
if phase == "decision" and kind == "resolve_overwrite":
return payload == {
"source": source,
"target_storage": checkpoint.target_storage,
"target_path": checkpoint.final_target_path,
"transfer_type": checkpoint.resolved_transfer_type,
"overwrite_mode": checkpoint.overwrite_mode,
"need_notify": checkpoint.need_notify,
}
if phase == "decision" and kind == "plugin_transfer_intercept":
stable_payload = {
"source": source,
"target_storage": checkpoint.target_storage,
"target_path": checkpoint.final_target_path,
"transfer_type": checkpoint.resolved_transfer_type,
}
return payload == stable_payload or (
set(payload) == {*stable_payload, "over_flag"}
and all(payload[key] == value for key, value in stable_payload.items())
and isinstance(payload.get("over_flag"), bool)
)
if phase == "prepare" and kind == "discover_version_targets":
return payload == {
"storage": checkpoint.target_storage,
"path": checkpoint.final_target_path,
}
if phase == "prepare" and kind == "delete_version_target":
candidate = payload.get("item")
if (
set(payload) != {"storage", "item"}
or payload.get("storage") != checkpoint.target_storage
or not isinstance(candidate, dict)
):
return False
return any(
step.phase == "prepare"
and step.kind == "discover_version_targets"
and step.state == TransferStepState.SUCCEEDED.value
and isinstance(step.result_payload, dict)
and candidate in step.result_payload.get("items", [])
for step in previous_steps
)
return False
@classmethod
def _validate_plan_steps(
cls,
*,
task_id: str,
checkpoint: TransferPlanCheckpoint,
checkpoint_fingerprint: str,
steps: list[TransferExecutionStepModel],
) -> None:
"""验证全部持久步骤属于冻结计划演进且身份与全局顺序未被篡改。"""
if tuple(step.ordinal for step in steps) != tuple(range(len(steps))):
raise TransferExecutionConflictError("整理步骤全局序号不连续")
for index, step in enumerate(steps):
if step.task_id != task_id:
raise TransferExecutionConflictError("整理步骤绑定了错误任务")
step_checkpoint = checkpoint
if step.checkpoint_fingerprint != checkpoint_fingerprint:
predecessor = cls._provider_predecessor_checkpoint(
checkpoint,
step,
)
if (
predecessor is None
or step.checkpoint_fingerprint != predecessor.fingerprint
):
raise TransferExecutionConflictError(
"整理步骤不属于当前冻结计划或合法 provider 前驱计划"
)
step_checkpoint = predecessor
if not cls._intent_belongs_to_checkpoint(
step_checkpoint,
phase=step.phase,
kind=step.kind,
payload=step.intent_payload,
previous_steps=steps[:index],
):
raise TransferExecutionConflictError(
"整理步骤类型或参数不能由冻结计划导出"
)
expected_operation_id = build_transfer_operation_id(
task_id=task_id,
checkpoint_fingerprint=step.checkpoint_fingerprint,
ordinal=step.ordinal,
phase=step.phase,
kind=step.kind,
intent_payload=step.intent_payload,
)
if step.operation_id != expected_operation_id:
raise TransferExecutionConflictError("整理步骤 operation ID 与冻结意图不一致")
@classmethod
def _validate_new_intent(
cls,
*,
task_id: str,
checkpoint: TransferPlanCheckpoint,
checkpoint_fingerprint: str,
steps: list[TransferExecutionStepModel],
intent: TransferStepIntent,
) -> None:
"""验证待准备意图属于当前冻结计划，并且只追加或幂等重放既有序号。"""
cls._validate_plan_steps(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
)
if intent.checkpoint_fingerprint != checkpoint_fingerprint:
raise TransferExecutionConflictError("整理步骤意图未绑定当前冻结计划指纹")
if not cls._intent_belongs_to_checkpoint(
checkpoint,
phase=intent.phase,
kind=intent.kind,
payload=intent.payload,
previous_steps=steps,
):
raise TransferExecutionConflictError(
"整理步骤意图类型或参数不能由冻结计划导出"
)
expected_operation_id = build_transfer_operation_id(
task_id=task_id,
checkpoint_fingerprint=intent.checkpoint_fingerprint,
ordinal=intent.ordinal,
phase=intent.phase,
kind=intent.kind,
intent_payload=intent.payload,
)
if intent.operation_id != expected_operation_id:
raise TransferExecutionConflictError("整理步骤意图 operation ID 不可信")
existing = next(
(step for step in steps if step.operation_id == intent.operation_id),
None,
)
if existing is not None:
if not cls._intent_matches(existing, task_id=task_id, intent=intent):
raise TransferExecutionConflictError(
"稳定 operation ID 已绑定不同步骤意图"
)
return
if intent.ordinal != len(steps):
raise TransferExecutionConflictError("整理步骤意图必须按全局序号连续追加")
@staticmethod
def _require_active_lease(
pending: Optional[TransferPending],
*,
lease_token: str,
now_utc: str,
detail: str,
) -> None:
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
) -> TransferPending:
"""返回仍由调用方持有的任务，并优先报告 lease fencing 失败。"""
if (
pending is None
or pending.lease_token != lease_token
@@ -244,6 +571,23 @@ class TransactionalTransferExecutionRepository:
or pending.lease_expires_at <= now_utc
):
raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管")
return pending
@classmethod
def _raise_fenced_failure(
cls,
pending: Optional[TransferPending],
*,
lease_token: str,
now_utc: str,
detail: str,
) -> None:
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
cls._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
raise TransferExecutionConflictError(detail)
def _times(self) -> tuple[str, str]:
@@ -354,9 +698,29 @@ class TransactionalTransferExecutionRepository:
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
oper.list_by_task_id(task_id=task_id)
)
self._validate_new_intent(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
intent=intent,
)
updated = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
@@ -367,7 +731,6 @@ class TransactionalTransferExecutionRepository:
now_utc=now_utc,
detail="整理任务当前状态不能准备外部步骤",
)
oper = TransferExecutionStepOper(session)
step = oper.get_by_operation_id(operation_id=intent.operation_id)
if step is None:
step = oper.stage_prepare(
@@ -498,9 +861,32 @@ class TransactionalTransferExecutionRepository:
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
oper.list_by_task_id(task_id=task_id)
)
self._validate_plan_steps(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
)
if operation_id not in {step.operation_id for step in steps}:
raise TransferExecutionConflictError(
"待恢复步骤不属于冻结计划"
)
pending_updated = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
@@ -511,7 +897,6 @@ class TransactionalTransferExecutionRepository:
now_utc=now_utc,
detail="重试任务未到期或当前状态不能恢复",
)
oper = TransferExecutionStepOper(session)
updated = oper.stage_resume_failed_attempt(
task_id=task_id,
lease_token=lease_token,
@@ -973,24 +1358,39 @@ class TransactionalTransferExecutionRepository:
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
plan_checkpoint, plan_fingerprint = self._plan_identity(pending)
step_oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
step_oper.list_by_task_id(task_id=task_id)
)
step_ids = {step.operation_id for step in steps}
if step_ids != set(checkpoint.operation_ids):
self._validate_plan_steps(
task_id=task_id,
checkpoint=plan_checkpoint,
checkpoint_fingerprint=plan_fingerprint,
steps=steps,
)
step_ids = tuple(step.operation_id for step in steps)
if step_ids != checkpoint.operation_ids:
raise TransferExecutionConflictError(
"执行检查点引用的步骤集合与持久步骤不一致"
"执行检查点引用的步骤顺序与持久步骤不一致"
)
if any(step.state != TransferStepState.SUCCEEDED.value for step in steps):
raise TransferExecutionConflictError(
"存在未成功步骤，不能提交执行检查点"
)
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
running = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=plan_checkpoint.schema_version,
checkpoint_payload=plan_checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
app/db/models/transferpending.py
+76 -69
View File
@@ -1,5 +1,3 @@
import hashlib
import json
from datetime import datetime
from typing import Any, List, Optional, cast
from uuid import uuid4
@@ -12,66 +10,21 @@ from sqlalchemy import (
Text,
UniqueConstraint,
and_,
column,
delete,
exists,
func,
or_,
select,
table,
update,
)
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import Base, execute_dml, get_id_column
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _planning_fingerprint(payload: dict[str, Any]) -> str:
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _default_planning_payload(context: Any) -> dict[str, Any]:
params = context.get_current_parameters()
return _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
def _default_planning_fingerprint(context: Any) -> str:
params = context.get_current_parameters()
payload = params.get("planning_input") or _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
return _planning_fingerprint(payload)
_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
class TransferPending(Base):
@@ -112,11 +65,11 @@ class TransferPending(Base):
input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
# 版本化规划输入 JSON
planning_input: Mapped[dict[str, Any]] = mapped_column(
JSON, nullable=False, default=_default_planning_payload
JSON, nullable=False
)
# 规划输入规范 JSON 的 SHA-256 指纹
input_fingerprint: Mapped[str] = mapped_column(
String(64), nullable=False, default=_default_planning_fingerprint
String(64), nullable=False
)
# 完整计划格式版本，尚未规划时为空
checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer)
@@ -203,9 +156,9 @@ class TransferPending(Base):
@classmethod
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
src_path: str, state: str,
now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]:
now_time: str, input_version: int,
planning_input: dict[str, Any],
input_fingerprint: str) -> Optional["TransferPending"]:
"""
在调用方会话中暂存一条持久接纳记录。
@@ -221,15 +174,20 @@ class TransferPending(Base):
:param input_fingerprint: 规划输入规范 JSON 指纹
:return: 接纳记录
"""
if not task_id or not storage or not src_path or not state:
if (
not task_id
or not storage
or not src_path
or not state
or not planning_input
or not input_fingerprint
):
return None
pending = db.execute(
select(cls).where(cls.storage == storage, cls.src_path == src_path)
).scalars().first()
if pending:
return cast("TransferPending", pending)
effective_input = planning_input or _legacy_planning_payload(storage, src_path)
effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input)
pending = cls(
task_id=task_id,
storage=storage,
@@ -238,8 +196,8 @@ class TransferPending(Base):
created_at=now_time,
updated_at=now_time,
input_version=input_version,
planning_input=effective_input,
input_fingerprint=effective_fingerprint,
planning_input=planning_input,
input_fingerprint=input_fingerprint,
)
db.add(pending)
return pending
@@ -425,17 +383,33 @@ class TransferPending(Base):
*,
task_id: str,
lease_token: str,
admission_state: str,
checkpoint_version: int,
checkpoint_payload: dict[str, Any],
now_utc: str,
updated_at: str,
) -> int:
"""以有效租约把可执行任务推进或保持为 running。"""
if not all((task_id, lease_token, now_utc, updated_at)):
"""仅以有效租约和完整冻结计划把任务推进或保持为 running。"""
if not all((
task_id,
lease_token,
admission_state,
checkpoint_version,
checkpoint_payload,
now_utc,
updated_at,
)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state == admission_state,
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version == checkpoint_version,
cls.checkpoint_payload == checkpoint_payload,
cls.planned_at.is_not(None),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_utc,
@@ -470,9 +444,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -512,6 +487,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state.in_(("not_started", "running", "retry_wait")),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -558,9 +537,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -598,9 +578,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -886,7 +867,7 @@ class TransferPending(Base):
)
@classmethod
def discard_claimed(
def abandon_unstarted(
cls,
db: Session,
*,
@@ -895,7 +876,10 @@ class TransferPending(Base):
now_time: str,
) -> int:
"""
仅以当前且未过期的 token 删除已经到达终态的租约任务。
仅以当前且未过期的 token 删除确认从未执行的缺失源任务。
任何执行状态、聚合检查点或步骤证据都意味着外部结果需要由状态机
判定；即使源文件已经消失，也不能以此推断 move 已经安全完成。
:param db: 数据库会话
:param task_id: 稳定任务标识
@@ -912,6 +896,29 @@ class TransferPending(Base):
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
cls.state == "accepted",
cls.checkpoint_version.is_(None),
cls.checkpoint_payload.is_(None),
cls.planned_at.is_(None),
cls.execution_state == "not_started",
cls.execution_version.is_(None),
cls.execution_payload.is_(None),
cls.execution_fingerprint.is_(None),
cls.retry_generation == 0,
cls.retry_count == 0,
cls.retry_due_at.is_(None),
cls.settlement_revision == 0,
cls.terminal_history_id.is_(None),
~exists(
select(_TRANSFER_EXECUTION_STEP.c.task_id).where(
_TRANSFER_EXECUTION_STEP.c.task_id == cls.task_id
)
),
~exists(
select(_TRANSFER_HISTORY.c.transfer_task_id).where(
_TRANSFER_HISTORY.c.transfer_task_id == cls.task_id
)
),
),
execution_options={"synchronize_session": False},
)
app/db/oper/transferpending.py
+12 -6
View File
@@ -13,9 +13,9 @@ class TransferPendingOper(DbOper):
"""
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
state: str, now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional[TransferPending]:
state: str, now_time: str, input_version: int,
planning_input: dict[str, Any],
input_fingerprint: str) -> Optional[TransferPending]:
"""
在当前会话中暂存一条持久接纳记录。
@@ -287,7 +287,7 @@ class TransferPendingOper(DbOper):
)
)
def stage_discard_claimed(
def stage_abandon_unstarted(
self,
*,
task_id: str,
@@ -295,7 +295,7 @@ class TransferPendingOper(DbOper):
now_time: str,
) -> int:
"""
使用当前会话按当前未过期 token 删除终态任务。
使用当前会话按当前未过期 token 删除确认从未执行的缺失源任务。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
@@ -303,7 +303,7 @@ class TransferPendingOper(DbOper):
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_claimed(
lambda session: TransferPending.abandon_unstarted(
session,
task_id=task_id,
lease_token=lease_token,
@@ -334,6 +334,9 @@ class TransferPendingOper(DbOper):
*,
task_id: str,
lease_token: str,
admission_state: str,
checkpoint_version: int,
checkpoint_payload: dict[str, Any],
now_utc: str,
updated_at: str,
) -> int:
@@ -343,6 +346,9 @@ class TransferPendingOper(DbOper):
session,
task_id=task_id,
lease_token=lease_token,
admission_state=admission_state,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
now_utc=now_utc,
updated_at=updated_at,
)
app/schemas/exports.py
-1
View File
@@ -301,7 +301,6 @@ SCHEMA_EXPORTS = {
'PluginTriggeredEventData': ('app.schemas.event', 'PluginTriggeredEventData'),
'PluginUpdateCandidate': ('app.schemas.plugin', 'PluginUpdateCandidate'),
'PluginWorkflowActionGroup': ('app.schemas.workflow', 'PluginWorkflowActionGroup'),
'PrivateAttr': ('app.schemas.plugin', 'PrivateAttr'),
'ProcessInfo': ('app.schemas.dashboard', 'ProcessInfo'),
'ProgressKeyData': ('app.schemas.common', 'ProgressKeyData'),
'RadarrMovie': ('app.schemas.servarr', 'RadarrMovie'),
app/schemas/plugin.py
+3 -2
View File
@@ -1,7 +1,8 @@
from enum import Enum as _Enum
from typing import Dict, List, Literal, Optional, Union
from pydantic import BaseModel, Field, PrivateAttr, RootModel, field_validator
from pydantic import BaseModel, Field, RootModel, field_validator
from pydantic import PrivateAttr as _PrivateAttr
from app.schemas.common import JsonData
@@ -62,7 +63,7 @@ class Plugin(BaseModel):
"""
插件信息
"""
_package_version: Optional[str] = PrivateAttr(default=None)
_package_version: Optional[str] = _PrivateAttr(default=None)
id: str = None
# 插件名称
app/sdk/_legacy/transferpending.py
+42
View File
@@ -1,5 +1,7 @@
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
import hashlib
import json
from datetime import datetime
from typing import Any, List, Optional, Tuple
from uuid import uuid4
@@ -13,6 +15,42 @@ _TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
"""只在旧登记 ABI 内构造可由新状态机保守重规划的最小输入。"""
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _planning_fingerprint(payload: dict[str, Any]) -> str:
"""生成兼容登记输入与 canonical DTO 一致的规范指纹。"""
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _safe_legacy_delete_predicates() -> tuple[Any, ...]:
"""只允许旧接口删除从未 claim、执行或结算的新鲜登记。"""
return (
@@ -61,6 +99,7 @@ class TransferPendingOper(DbOper):
:return: 登记记录
"""
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
planning_input = _legacy_planning_payload(storage, src_path)
return self._execute_sync_write(
lambda session: _TransferPending.stage_admit(
session,
@@ -69,6 +108,9 @@ class TransferPendingOper(DbOper):
src_path=src_path,
state="accepted",
now_time=now_time,
input_version=1,
planning_input=planning_input,
input_fingerprint=_planning_fingerprint(planning_input),
)
)
database/versions/b1e7d3f5a9c2_3_0_13.py
+55 -29
View File
@@ -33,22 +33,49 @@ def _column_names() -> set[str]:
}
def _has_task_id_constraint() -> bool:
"""判断稳定任务标识唯一约束是否已经存在。"""
inspector = sa.inspect(op.get_bind())
return any(
constraint.get("name") == _TASK_ID_CONSTRAINT
for constraint in inspector.get_unique_constraints(_TABLE_NAME)
)
def _repair_task_id_constraint() -> None:
"""把稳定任务身份约束修复为 task_id 单列唯一约束。"""
constraints = {
constraint.get("name"): tuple(constraint.get("column_names") or ())
for constraint in sa.inspect(op.get_bind()).get_unique_constraints(
_TABLE_NAME
)
if constraint.get("name")
}
current = constraints.get(_TASK_ID_CONSTRAINT)
if current is not None and current != ("task_id",):
with op.batch_alter_table(_TABLE_NAME) as batch_op:
batch_op.drop_constraint(_TASK_ID_CONSTRAINT, type_="unique")
current = None
if current is None:
with op.batch_alter_table(_TABLE_NAME) as batch_op:
batch_op.create_unique_constraint(
_TASK_ID_CONSTRAINT,
["task_id"],
)
def _has_state_created_index() -> bool:
"""判断恢复主查询的复合索引是否已经存在。"""
inspector = sa.inspect(op.get_bind())
return any(
index.get("name") == _STATE_CREATED_INDEX
for index in inspector.get_indexes(_TABLE_NAME)
)
def _repair_state_created_index() -> None:
"""按列顺序和非唯一语义修复恢复扫描索引。"""
indexes = {
index.get("name"): index
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
if index.get("name")
}
current = indexes.get(_STATE_CREATED_INDEX)
if current is not None and (
tuple(current.get("column_names") or ()) != ("state", "created_at", "id")
or bool(current.get("unique"))
):
op.drop_index(_STATE_CREATED_INDEX, table_name=_TABLE_NAME)
current = None
if current is None:
op.create_index(
_STATE_CREATED_INDEX,
_TABLE_NAME,
["state", "created_at", "id"],
unique=False,
)
def _backfill_admission_state() -> None:
@@ -135,19 +162,8 @@ def upgrade() -> None:
batch_op.alter_column(
"updated_at", existing_type=sa.String(length=40), nullable=False
)
if not _has_task_id_constraint():
with op.batch_alter_table(_TABLE_NAME) as batch_op:
batch_op.create_unique_constraint(
_TASK_ID_CONSTRAINT,
["task_id"],
)
if not _has_state_created_index():
op.create_index(
_STATE_CREATED_INDEX,
_TABLE_NAME,
["state", "created_at", "id"],
unique=False,
)
_repair_task_id_constraint()
_repair_state_created_index()
def downgrade() -> None:
@@ -155,10 +171,20 @@ def downgrade() -> None:
columns = _column_names()
if not columns or not (_NEW_COLUMNS & columns):
return
if _has_state_created_index():
index_names = {
index.get("name")
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
}
if _STATE_CREATED_INDEX in index_names:
op.drop_index(_STATE_CREATED_INDEX, table_name=_TABLE_NAME)
constraint_names = {
constraint.get("name")
for constraint in sa.inspect(op.get_bind()).get_unique_constraints(
_TABLE_NAME
)
}
with op.batch_alter_table(_TABLE_NAME) as batch_op:
if "task_id" in columns and _has_task_id_constraint():
if "task_id" in columns and _TASK_ID_CONSTRAINT in constraint_names:
batch_op.drop_constraint(_TASK_ID_CONSTRAINT, type_="unique")
for column_name in ("last_error", "updated_at", "state", "task_id"):
if column_name in columns:
database/versions/c2f8a4d6e1b3_3_0_14.py
+11 -12
View File
@@ -97,27 +97,26 @@ def _backfill_planning_input() -> None:
)
).mappings().all()
for row in rows:
if (
row["input_version"] is not None
and row["planning_input"] is not None
and row["input_fingerprint"]
):
continue
payload = row["planning_input"]
if not isinstance(payload, dict):
payload = _legacy_planning_payload(row["storage"], row["src_path"])
payload_version = payload.get("schema_version")
input_version = row["input_version"]
if input_version is None:
input_version = payload_version if isinstance(payload_version, int) else 1
input_fingerprint = row["input_fingerprint"] or _fingerprint(payload)
if (
not isinstance(payload_version, int)
or isinstance(payload_version, bool)
or payload_version != 1
):
raise RuntimeError(
f"transferpending {row['id']} 的规划输入版本不受支持: "
f"{payload_version}"
)
connection.execute(
pending.update()
.where(pending.c.id == row["id"])
.values(
input_version=input_version,
input_version=payload_version,
planning_input=payload,
input_fingerprint=input_fingerprint,
input_fingerprint=_fingerprint(payload),
)
)
database/versions/d3a9e5f7b2c4_3_0_15.py
+26 -16
View File
@@ -35,16 +35,28 @@ def _column_names() -> set[str]:
}
def _index_names() -> set[str]:
"""返回当前待整理登记表的索引名称集合。"""
inspector = sa.inspect(op.get_bind())
if _TABLE_NAME not in inspector.get_table_names():
return set()
return {
index["name"]
for index in inspector.get_indexes(_TABLE_NAME)
def _repair_lease_index() -> None:
"""按列顺序和非唯一语义修复租约接管索引。"""
indexes = {
index.get("name"): index
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
if index.get("name")
}
current = indexes.get(_LEASE_INDEX)
if current is not None and (
tuple(current.get("column_names") or ())
!= ("state", "lease_expires_at", "created_at", "id")
or bool(current.get("unique"))
):
op.drop_index(_LEASE_INDEX, table_name=_TABLE_NAME)
current = None
if current is None:
op.create_index(
_LEASE_INDEX,
_TABLE_NAME,
["state", "lease_expires_at", "created_at", "id"],
unique=False,
)
def upgrade() -> None:
@@ -96,13 +108,7 @@ def upgrade() -> None:
nullable=False,
)
if _LEASE_INDEX not in _index_names():
op.create_index(
_LEASE_INDEX,
_TABLE_NAME,
["state", "lease_expires_at", "created_at", "id"],
unique=False,
)
_repair_lease_index()
def downgrade() -> None:
@@ -110,7 +116,11 @@ def downgrade() -> None:
columns = _column_names()
if not columns or not (_LEASE_COLUMNS & columns):
return
if _LEASE_INDEX in _index_names():
index_names = {
index.get("name")
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
}
if _LEASE_INDEX in index_names:
op.drop_index(_LEASE_INDEX, table_name=_TABLE_NAME)
with op.batch_alter_table(_TABLE_NAME) as batch_op:
for column_name in (
database/versions/e3d9f4b7c806_3_0_4.py
+98 -17
View File
@@ -10,14 +10,18 @@ Revises: 7f5c1d2e3a4b
Create Date: 2026-08-10
"""
from alembic import op
import sqlalchemy as sa
from alembic import op
revision = "e3d9f4b7c806"
down_revision = "7f5c1d2e3a4b"
branch_labels = None
depends_on = None
_TABLE_NAME = "transferpending"
_INDEX_NAME = "ux_transferpending_storage_path"
_EXPECTED_COLUMNS = {"id", "storage", "src_path", "created_at"}
def _has_table(table_name: str) -> bool:
"""检查数据表是否已存在。"""
@@ -25,33 +29,110 @@ def _has_table(table_name: str) -> bool:
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""
创建待整理文件登记表。
"""
if _has_table("transferpending"):
return
def _table_row_count() -> int:
"""返回待整理表行数，用于判断残缺结构能否无损重建。"""
return op.get_bind().execute(
sa.select(sa.func.count()).select_from(sa.table(_TABLE_NAME))
).scalar_one()
def _create_table() -> None:
"""创建 3.0.4 定义的完整待整理登记表。"""
op.create_table(
"transferpending",
_TABLE_NAME,
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column("storage", sa.String, nullable=False),
sa.Column("src_path", sa.String, nullable=False),
sa.Column("created_at", sa.String),
)
# 同一个文件重复入队只保留一条，回放时不会重复送入整理链
op.create_index(
"ux_transferpending_storage_path",
"transferpending",
["storage", "src_path"],
unique=True,
def _validate_or_recreate_table() -> None:
"""校验中断升级留下的表结构，仅允许空残表自动重建。"""
inspector = sa.inspect(op.get_bind())
columns = inspector.get_columns(_TABLE_NAME)
column_names = {column["name"] for column in columns}
nullable = {
column["name"]
for column in columns
if column.get("nullable", True)
}
primary_key = tuple(
inspector.get_pk_constraint(_TABLE_NAME).get("constrained_columns") or ()
)
column_types = {
column["name"]: column["type"]
for column in columns
}
wrong_types = (
not isinstance(column_types.get("id"), sa.Integer)
or any(
not isinstance(column_types.get(column_name), sa.String)
for column_name in ("storage", "src_path", "created_at")
)
)
malformed = (
column_names != _EXPECTED_COLUMNS
or nullable != {"created_at"}
or primary_key != ("id",)
or wrong_types
)
if not malformed:
return
if _table_row_count() > 0:
raise RuntimeError(
"检测到含数据的不完整 transferpending 表，"
"无法自动恢复 3.0.4 迁移"
)
op.drop_table(_TABLE_NAME)
_create_table()
def _repair_storage_path_index() -> None:
"""把源身份索引修复为指定列上的唯一索引。"""
indexes = {
index["name"]: index
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
if index.get("name")
}
current = indexes.get(_INDEX_NAME)
if current is not None and (
tuple(current.get("column_names") or ()) != ("storage", "src_path")
or not current.get("unique")
):
op.drop_index(_INDEX_NAME, table_name=_TABLE_NAME)
current = None
if current is None:
op.create_index(
_INDEX_NAME,
_TABLE_NAME,
["storage", "src_path"],
unique=True,
)
def upgrade() -> None:
"""
创建待整理文件登记表。
"""
if not _has_table(_TABLE_NAME):
_create_table()
else:
_validate_or_recreate_table()
_repair_storage_path_index()
def downgrade() -> None:
"""
删除待整理文件登记表。
"""
if not _has_table("transferpending"):
if not _has_table(_TABLE_NAME):
return
op.drop_index("ux_transferpending_storage_path", table_name="transferpending")
op.drop_table("transferpending")
index_names = {
index["name"]
for index in sa.inspect(op.get_bind()).get_indexes(_TABLE_NAME)
if index.get("name")
}
if _INDEX_NAME in index_names:
op.drop_index(_INDEX_NAME, table_name=_TABLE_NAME)
op.drop_table(_TABLE_NAME)
database/versions/e5c7a9b1d3f6_3_0_16.py
+89 -23
View File
@@ -5,9 +5,9 @@ Revises: d3a9e5f7b2c4
Create Date: 2026-08-27
"""
from collections.abc import Callable
import hashlib
import json
from collections.abc import Callable
import sqlalchemy as sa
from alembic import op
@@ -21,6 +21,7 @@ _PENDING_TABLE = "transferpending"
_HISTORY_TABLE = "transferhistory"
_STEP_TABLE = "transferexecutionstep"
_RECEIPT_TABLE = "transfersettlementreceipt"
_RECEIPT_ARCHIVE_TABLE = "transfersettlementreceipt_3_0_16_archive"
_PENDING_INDEX = "ix_transferpending_execution_due"
_HISTORY_INDEX = "ux_transferhistory_transfer_task_id"
_STEP_OPERATION_UNIQUE = "uq_transferexecutionstep_operation_id"
@@ -247,6 +248,35 @@ def _repair_indexes(
)
def _repair_owned_index(
*,
table_name: str,
index_name: str,
columns: tuple[str, ...],
unique: bool,
) -> None:
"""按列顺序和唯一性精确修复宿主表上的迁移自有索引。"""
indexes = {
item["name"]: item
for item in sa.inspect(op.get_bind()).get_indexes(table_name)
if item.get("name") and not item.get("duplicates_constraint")
}
current = indexes.get(index_name)
if current is not None and (
tuple(current.get("column_names") or ()) != columns
or bool(current.get("unique")) != unique
):
op.drop_index(index_name, table_name=table_name)
current = None
if current is None:
op.create_index(
index_name,
table_name,
list(columns),
unique=unique,
)
def _column_type_signature(column_type: sa.types.TypeEngine) -> tuple[str, object]:
"""把方言反射类型归一为迁移可稳定比较的类型与长度。"""
if isinstance(column_type, sa.JSON):
@@ -554,6 +584,27 @@ def _create_receipt_table() -> None:
)
def _restore_receipt_archive() -> None:
"""在重升时原子恢复降级保留的 append-only 结算证据。"""
tables = _table_names()
if _RECEIPT_ARCHIVE_TABLE not in tables:
return
if _RECEIPT_TABLE not in tables:
op.rename_table(_RECEIPT_ARCHIVE_TABLE, _RECEIPT_TABLE)
return
archive_count = _table_row_count(_RECEIPT_ARCHIVE_TABLE)
live_count = _table_row_count(_RECEIPT_TABLE)
if archive_count > 0 and live_count > 0:
raise RuntimeError(
"结算回执现表与降级归档同时含数据，无法自动判定恢复顺序"
)
if archive_count > 0:
op.drop_table(_RECEIPT_TABLE)
op.rename_table(_RECEIPT_ARCHIVE_TABLE, _RECEIPT_TABLE)
else:
op.drop_table(_RECEIPT_ARCHIVE_TABLE)
def _create_or_repair_receipt_table() -> None:
"""创建结算回执表，并只对空的残缺表执行无损重建。"""
if _RECEIPT_TABLE not in _table_names():
@@ -704,30 +755,29 @@ def upgrade() -> None:
tables = _table_names()
if _PENDING_TABLE not in tables:
return
_restore_receipt_archive()
_add_pending_columns()
_backfill_pending()
if _PENDING_INDEX not in _index_names(_PENDING_TABLE):
op.create_index(
_PENDING_INDEX,
_PENDING_TABLE,
[
"execution_state",
"retry_due_at",
"state",
"created_at",
"id",
],
unique=False,
)
_repair_owned_index(
table_name=_PENDING_TABLE,
index_name=_PENDING_INDEX,
columns=(
"execution_state",
"retry_due_at",
"state",
"created_at",
"id",
),
unique=False,
)
if _HISTORY_TABLE in tables:
_add_history_columns()
if _HISTORY_INDEX not in _index_names(_HISTORY_TABLE):
op.create_index(
_HISTORY_INDEX,
_HISTORY_TABLE,
["transfer_task_id"],
unique=True,
)
_repair_owned_index(
table_name=_HISTORY_TABLE,
index_name=_HISTORY_INDEX,
columns=("transfer_task_id",),
unique=True,
)
_create_or_repair_receipt_table()
_create_or_repair_step_table()
_backfill_legacy_review_steps()
@@ -817,8 +867,24 @@ def downgrade() -> None:
_mark_downgrade_uncertain()
if _STEP_TABLE in _table_names():
op.drop_table(_STEP_TABLE)
if _RECEIPT_TABLE in _table_names():
op.drop_table(_RECEIPT_TABLE)
tables = _table_names()
if _RECEIPT_TABLE in tables:
live_count = _table_row_count(_RECEIPT_TABLE)
if _RECEIPT_ARCHIVE_TABLE in tables:
archive_count = _table_row_count(_RECEIPT_ARCHIVE_TABLE)
if live_count > 0 and archive_count > 0:
raise RuntimeError(
"结算回执现表与降级归档同时含数据，拒绝覆盖恢复证据"
)
if archive_count > 0:
op.drop_table(_RECEIPT_TABLE)
live_count = 0
else:
op.drop_table(_RECEIPT_ARCHIVE_TABLE)
if live_count > 0:
op.rename_table(_RECEIPT_TABLE, _RECEIPT_ARCHIVE_TABLE)
elif _RECEIPT_TABLE in _table_names():
op.drop_table(_RECEIPT_TABLE)
if _HISTORY_TABLE in _table_names():
history_columns = _column_names(_HISTORY_TABLE)
if _HISTORY_INDEX in _index_names(_HISTORY_TABLE):
database/versions/f6d8b0c2e4a7_3_0_17.py
+703
View File
@@ -0,0 +1,703 @@
"""3.0.17 收口整理规划状态与执行恢复证据。
Revision ID: f6d8b0c2e4a7
Revises: e5c7a9b1d3f6
Create Date: 2026-08-27
"""
import hashlib
import json
from typing import Any
import sqlalchemy as sa
from alembic import op
revision = "f6d8b0c2e4a7"
down_revision = "e5c7a9b1d3f6"
branch_labels = None
depends_on = None
_PENDING_TABLE = "transferpending"
_STEP_TABLE = "transferexecutionstep"
_RECEIPT_TABLE = "transfersettlementreceipt"
_HISTORY_TABLE = "transferhistory"
_ALLOWED_EXECUTION_STATES = {
"not_started",
"running",
"retry_wait",
"settling",
"failed",
"manual_review",
}
_REVIEW_DIAGNOSTIC = "升级检测到不完整执行状态，需人工确认后再处理"
_REVIEW_STEP_KIND = "legacy_execution_review"
_REVIEW_STEP_ORDINAL = 2_147_483_647
_FALLBACK_TIME = "1970-01-01 00:00:00"
def _table_names() -> set[str]:
"""返回当前数据库的表名集合。"""
return set(sa.inspect(op.get_bind()).get_table_names())
def _canonical_json(payload: dict[str, Any]) -> str:
"""生成与运行时一致的稳定 JSON 表达。"""
return json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
def _fingerprint(payload: dict[str, Any]) -> str:
"""计算版本化 JSON 的 SHA-256 指纹。"""
return hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
def _is_nonempty_string(value: object) -> bool:
"""判断值是否为非空文本。"""
return isinstance(value, str) and bool(value)
def _is_source_fileitem(value: object) -> bool:
"""判断 JSON 对象是否包含可恢复的源文件身份。"""
return (
isinstance(value, dict)
and _is_nonempty_string(value.get("storage"))
and _is_nonempty_string(value.get("path"))
)
def _is_optional_mapping(value: object) -> bool:
"""判断值是否为 JSON 对象或空值。"""
return value is None or isinstance(value, dict)
def _is_optional_string(value: object) -> bool:
"""判断值是否为字符串或空值。"""
return value is None or isinstance(value, str)
def _is_planning_input(
value: object,
*,
expected_fingerprint: object,
) -> bool:
"""按第一版运行时契约校验规划输入及其持久指纹。"""
if not isinstance(value, dict) or value.get("schema_version") != 1:
return False
if not _is_source_fileitem(value.get("source_fileitem")):
return False
if not all(
_is_optional_mapping(value.get(field_name))
for field_name in ("meta", "mediainfo", "target_directory")
):
return False
if not all(
_is_optional_string(value.get(field_name))
for field_name in (
"target_storage",
"target_path",
"requested_transfer_type",
"media_source",
"media_id",
"media_type",
"overwrite_mode",
)
):
return False
if not all(
isinstance(value.get(field_name, default), bool)
for field_name, default in (
("need_scrape", False),
("need_rename", True),
("need_notify", True),
("preview", False),
)
):
return False
episodes = value.get("episodes_info", [])
if (
not isinstance(episodes, list)
or not all(isinstance(item, dict) for item in episodes)
or not isinstance(value.get("options", {}), dict)
):
return False
try:
return _fingerprint(value) == expected_fingerprint
except (TypeError, ValueError):
return False
def _is_provider_reference(value: object) -> bool:
"""判断旧 provider 引用是否包含稳定身份与固定方法。"""
return (
isinstance(value, dict)
and _is_nonempty_string(value.get("plugin_id"))
and _is_nonempty_string(value.get("plugin_name"))
and value.get("method", "transfer") == "transfer"
)
def _is_provider_invocation(value: object) -> bool:
"""按第一版旧 ABI 契约校验 provider 调用快照。"""
if not isinstance(value, dict) or value.get("schema_version") != 1:
return False
if not _is_source_fileitem(value.get("fileitem")):
return False
for field_name in ("meta", "mediainfo", "target_directory"):
field_value = value.get(field_name)
if field_value is not None and not isinstance(field_value, dict):
return False
for field_name in ("meta_kind", "mediainfo_kind"):
field_value = value.get(field_name)
if field_value is not None and not _is_nonempty_string(field_value):
return False
for field_name in ("target_storage", "target_path", "transfer_type"):
field_value = value.get(field_name)
if field_value is not None and not isinstance(field_value, str):
return False
for field_name in (
"scrape",
"library_type_folder",
"library_category_folder",
):
field_value = value.get(field_name)
if field_value is not None and not isinstance(field_value, bool):
return False
episodes = value.get("episodes_info", [])
return (
isinstance(value.get("preview", False), bool)
and isinstance(episodes, list)
and all(isinstance(item, dict) for item in episodes)
)
def _is_plan_item(value: object, *, sequence: int) -> bool:
"""判断计划叶子项是否具有连续序号和完整源目标身份。"""
return (
isinstance(value, dict)
and value.get("sequence") == sequence
and _is_source_fileitem(value.get("source_fileitem"))
and _is_nonempty_string(value.get("target_storage"))
and _is_nonempty_string(value.get("target_path"))
and _is_nonempty_string(value.get("action", "transfer"))
)
def _classify_checkpoint(
*,
checkpoint_version: object,
checkpoint_payload: object,
input_fingerprint: object,
) -> str | None:
"""把完整计划检查点分类为宿主计划或 provider 待执行态。"""
if (
checkpoint_version != 1
or not isinstance(checkpoint_payload, dict)
or checkpoint_payload.get("schema_version") != 1
or not _is_planning_input(
checkpoint_payload.get("planning_input"),
expected_fingerprint=input_fingerprint,
)
):
return None
providers = checkpoint_payload.get("legacy_transfer_providers", [])
if (
not isinstance(providers, list)
or not all(_is_provider_reference(item) for item in providers)
):
return None
if not all(
_is_optional_mapping(checkpoint_payload.get(field_name))
for field_name in ("resolved_meta", "resolved_mediainfo")
):
return None
if not all(
value is None or _is_nonempty_string(value)
for value in (
checkpoint_payload.get("resolved_meta_kind"),
checkpoint_payload.get("resolved_mediainfo_kind"),
)
):
return None
resolved_episodes = checkpoint_payload.get("resolved_episodes_info", [])
if (
not isinstance(resolved_episodes, list)
or not all(isinstance(item, dict) for item in resolved_episodes)
or not all(
isinstance(checkpoint_payload.get(field_name, default), bool)
for field_name, default in (
("pre_execution_cleanup_completed", False),
("need_scrape", False),
("need_rename", False),
("need_notify", True),
("preview", False),
)
)
or not _is_optional_string(checkpoint_payload.get("overwrite_mode"))
or not _is_optional_string(checkpoint_payload.get("skip_reason"))
or not _is_optional_string(checkpoint_payload.get("rejection_error"))
):
return None
invocation = checkpoint_payload.get("provider_invocation")
if invocation is not None:
if (
not providers
or not _is_provider_invocation(invocation)
or not invocation.get("meta")
or not _is_nonempty_string(invocation.get("meta_kind"))
or not invocation.get("mediainfo")
or not _is_nonempty_string(invocation.get("mediainfo_kind"))
or not checkpoint_payload.get("resolved_meta")
or not _is_nonempty_string(checkpoint_payload.get("resolved_meta_kind"))
or not checkpoint_payload.get("resolved_mediainfo")
or not _is_nonempty_string(
checkpoint_payload.get("resolved_mediainfo_kind")
)
or checkpoint_payload.get("pre_execution_cleanup_completed", False)
or any(checkpoint_payload.get(field_name) for field_name in (
"target_storage",
"root_target_path",
"final_target_path",
"resolved_transfer_type",
"items",
"skip_reason",
))
):
return None
return "provider_pending"
items = checkpoint_payload.get("items", [])
rejection_error = checkpoint_payload.get("rejection_error")
if (
not isinstance(items, list)
or not all(
_is_plan_item(item, sequence=sequence)
for sequence, item in enumerate(items)
)
or not all(
_is_nonempty_string(checkpoint_payload.get(field_name))
for field_name in (
"target_storage",
"root_target_path",
"final_target_path",
"resolved_transfer_type",
)
)
or (
not items
and not checkpoint_payload.get("preview", False)
and not _is_nonempty_string(checkpoint_payload.get("skip_reason"))
and not _is_nonempty_string(rejection_error)
)
or (
rejection_error is not None
and (
not _is_nonempty_string(rejection_error)
or not rejection_error.strip()
or bool(items)
)
)
):
return None
return "planned"
def _normalize_legacy_planning_states() -> None:
"""把旧 planning manual_review 恢复为运行时可领取的稳定状态。"""
pending = sa.table(
_PENDING_TABLE,
sa.column("id", sa.Integer()),
sa.column("state", sa.String(32)),
sa.column("input_fingerprint", sa.String(64)),
sa.column("checkpoint_version", sa.Integer()),
sa.column("checkpoint_payload", sa.JSON()),
sa.column("planned_at", sa.String(40)),
)
bind = op.get_bind()
rows = bind.execute(
sa.select(
pending.c.id,
pending.c.state,
pending.c.input_fingerprint,
pending.c.checkpoint_version,
pending.c.checkpoint_payload,
).where(pending.c.state.in_((
"accepted",
"planned",
"provider_pending",
"manual_review",
)))
).mappings().all()
for row in rows:
state = _classify_checkpoint(
checkpoint_version=row["checkpoint_version"],
checkpoint_payload=row["checkpoint_payload"],
input_fingerprint=row["input_fingerprint"],
)
target_state = state
values: dict[str, object] = {"state": target_state or "accepted"}
if target_state is None:
values.update({
"checkpoint_version": None,
"checkpoint_payload": sa.null(),
"planned_at": None,
})
bind.execute(
pending.update().where(pending.c.id == row["id"]).values(**values)
)
def _is_execution_checkpoint(row: dict[str, Any]) -> bool:
"""校验执行检查点三元组的完整性、版本和内容指纹。"""
values = (
row["execution_version"],
row["execution_payload"],
row["execution_fingerprint"],
)
if all(value is None for value in values):
return True
if (
row["execution_version"] != 1
or not isinstance(row["execution_payload"], dict)
or not _is_nonempty_string(row["execution_fingerprint"])
):
return False
payload = row["execution_payload"]
operation_ids = payload.get("operation_ids")
skip_reason = payload.get("skip_reason")
execution_result = payload.get("payload")
if (
payload.get("schema_version") != 1
or not isinstance(execution_result, dict)
or not isinstance(operation_ids, list)
or not all(_is_nonempty_string(item) for item in operation_ids)
or len(operation_ids) != len(set(operation_ids))
or (not operation_ids and not _is_nonempty_string(skip_reason))
or (skip_reason is not None and not isinstance(skip_reason, str))
):
return False
outcome = execution_result.get("outcome")
if outcome not in {"succeeded", "failed", "overwrite_skipped"}:
return False
transferinfo = execution_result.get("transferinfo")
if transferinfo is not None:
if not isinstance(transferinfo, dict):
return False
if (
bool(transferinfo.get("success")) != (outcome == "succeeded")
or bool(transferinfo.get("overwrite_skipped"))
!= (outcome == "overwrite_skipped")
):
return False
elif outcome == "overwrite_skipped":
return False
try:
return _fingerprint(payload) == row["execution_fingerprint"]
except (TypeError, ValueError):
return False
def _has_failed_receipt(row: dict[str, Any]) -> bool:
"""判断失败 pending 是否具有匹配历史与 append-only 回执。"""
if (
_RECEIPT_TABLE not in _table_names()
or _HISTORY_TABLE not in _table_names()
or not isinstance(row["terminal_history_id"], int)
or not isinstance(row["settlement_revision"], int)
or row["settlement_revision"] <= 0
):
return False
receipts = sa.table(
_RECEIPT_TABLE,
sa.column("task_id", sa.String(64)),
sa.column("history_id", sa.Integer()),
sa.column("settlement_revision", sa.Integer()),
sa.column("outcome", sa.String(16)),
)
history = sa.table(
_HISTORY_TABLE,
sa.column("id", sa.Integer()),
)
bind = op.get_bind()
receipt_exists = bind.execute(
sa.select(sa.literal(1)).select_from(receipts).where(
receipts.c.task_id == row["task_id"],
receipts.c.history_id == row["terminal_history_id"],
receipts.c.settlement_revision == row["settlement_revision"],
receipts.c.outcome == "failed",
).limit(1)
).first() is not None
history_exists = bind.execute(
sa.select(sa.literal(1)).select_from(history).where(
history.c.id == row["terminal_history_id"]
).limit(1)
).first() is not None
return receipt_exists and history_exists
def _checkpoint_steps_complete(row: dict[str, Any]) -> bool:
"""判断执行检查点引用的每个外部操作是否已有确定结果。"""
payload = row["execution_payload"]
if not isinstance(payload, dict):
return False
operation_ids = payload.get("operation_ids")
if not isinstance(operation_ids, list):
return False
if not operation_ids:
return True
steps = sa.table(
_STEP_TABLE,
sa.column("task_id", sa.String(64)),
sa.column("operation_id", sa.String(64)),
sa.column("state", sa.String(32)),
)
completed = set(op.get_bind().execute(
sa.select(steps.c.operation_id).where(
steps.c.task_id == row["task_id"],
steps.c.operation_id.in_(operation_ids),
steps.c.state.in_(("succeeded", "failed")),
)
).scalars().all())
return completed == set(operation_ids)
def _has_manual_review_step(task_id: str) -> bool:
"""判断人工复核态是否具有运行时可发现的对应步骤证据。"""
steps = sa.table(
_STEP_TABLE,
sa.column("task_id", sa.String(64)),
sa.column("state", sa.String(32)),
)
return op.get_bind().execute(
sa.select(sa.literal(1)).select_from(steps).where(
steps.c.task_id == task_id,
steps.c.state == "manual_review",
).limit(1)
).first() is not None
def _execution_issue(row: dict[str, Any]) -> str | None:
"""返回阻止执行状态安全恢复的持久不变量缺口。"""
state = row["execution_state"]
if state not in _ALLOWED_EXECUTION_STATES:
return f"未知执行状态: {state}"
checkpoint_valid = _is_execution_checkpoint(row)
has_checkpoint = row["execution_payload"] is not None
if not checkpoint_valid:
return "执行检查点三元组不完整或内容无效"
if state != "not_started" and row["state"] not in {
"planned",
"provider_pending",
}:
return "执行态缺少完整 planned 或 provider_pending 计划"
lease_values = (
row["lease_owner"],
row["lease_token"],
row["lease_expires_at"],
)
if state != "manual_review" and any(lease_values) and not all(lease_values):
return "执行租约身份不完整"
if state == "not_started" and has_checkpoint:
return "未开始态错误携带执行检查点"
if state == "running" and has_checkpoint:
return "运行态错误携带终态执行检查点"
if state == "manual_review" and has_checkpoint:
return "人工复核态错误携带终态执行检查点"
if state == "manual_review" and not _has_manual_review_step(row["task_id"]):
return "人工复核态缺少步骤证据"
if state == "retry_wait" and not row["retry_due_at"]:
return "重试等待态缺少到期时间"
if state == "retry_wait" and any(lease_values):
return "重试等待态错误持有执行租约"
if state == "retry_wait" and has_checkpoint and (
not _checkpoint_steps_complete(row) or not _has_failed_receipt(row)
):
return "终态重试缺少匹配步骤、历史或结算回执"
if state == "settling" and (
not has_checkpoint or not _checkpoint_steps_complete(row)
):
return "结算态缺少完整检查点或步骤结果"
if state == "failed" and (
not has_checkpoint
or any(lease_values)
or not _checkpoint_steps_complete(row)
or not _has_failed_receipt(row)
):
return "失败终态缺少匹配步骤、检查点、历史或结算回执"
return None
def _review_operation_id(task_id: str) -> str:
"""生成 3.0.17 数据修复步骤的确定性身份。"""
return hashlib.sha256(
f"moviepilot:3.0.17:execution-review:{task_id}".encode("utf-8")
).hexdigest()
def _append_review_error(last_error: object, issue: str) -> str:
"""保留原始错误并追加可识别的迁移诊断。"""
diagnostic = f"{_REVIEW_DIAGNOSTIC}（{issue}）"
if not _is_nonempty_string(last_error):
return diagnostic
if _REVIEW_DIAGNOSTIC in last_error:
return last_error
return f"{last_error}\n{diagnostic}"
def _next_review_ordinal(task_id: str) -> int:
"""选择不会覆盖既有步骤证据的高位人工复核序号。"""
steps = sa.table(
_STEP_TABLE,
sa.column("task_id", sa.String(64)),
sa.column("ordinal", sa.Integer()),
)
ordinals = set(op.get_bind().execute(
sa.select(steps.c.ordinal).where(steps.c.task_id == task_id)
).scalars().all())
ordinal = _REVIEW_STEP_ORDINAL
while ordinal in ordinals:
ordinal -= 1
return ordinal
def _insert_review_step(row: dict[str, Any], *, issue: str) -> None:
"""把非法执行组合冻结为可人工判定的 synthetic 审计步骤。"""
operation_id = _review_operation_id(row["task_id"])
steps = sa.table(
_STEP_TABLE,
sa.column("task_id", sa.String(64)),
sa.column("operation_id", sa.String(64)),
sa.column("checkpoint_fingerprint", sa.String(64)),
sa.column("ordinal", sa.Integer()),
sa.column("phase", sa.String(32)),
sa.column("kind", sa.String(32)),
sa.column("state", sa.String(32)),
sa.column("attempt_token", sa.String(64)),
sa.column("attempt_count", sa.Integer()),
sa.column("intent_version", sa.Integer()),
sa.column("intent_payload", sa.JSON()),
sa.column("result_version", sa.Integer()),
sa.column("result_payload", sa.JSON()),
sa.column("last_error", sa.Text()),
sa.column("prepared_at", sa.String(40)),
sa.column("started_at", sa.String(40)),
sa.column("completed_at", sa.String(40)),
sa.column("updated_at", sa.String(40)),
)
bind = op.get_bind()
if bind.execute(
sa.select(sa.literal(1)).select_from(steps).where(
steps.c.operation_id == operation_id
).limit(1)
).first() is not None:
return
evidence = {
"schema_version": 1,
"origin": "3.0.17_migration",
"diagnostic": issue,
"legacy_state": row["state"],
"legacy_execution_state": row["execution_state"],
"execution_version": row["execution_version"],
"execution_payload": row["execution_payload"],
"execution_fingerprint": row["execution_fingerprint"],
"settlement_revision": row["settlement_revision"],
"terminal_history_id": row["terminal_history_id"],
"lease_owner": row["lease_owner"],
"lease_token": row["lease_token"],
}
evidence_time = row["updated_at"] or row["created_at"] or _FALLBACK_TIME
bind.execute(steps.insert().values(
task_id=row["task_id"],
operation_id=operation_id,
checkpoint_fingerprint=_fingerprint(evidence),
ordinal=_next_review_ordinal(row["task_id"]),
phase="legacy_upgrade",
kind=_REVIEW_STEP_KIND,
state="manual_review",
attempt_token=None,
attempt_count=row["attempt_count"] or 0,
intent_version=1,
intent_payload=evidence,
result_version=None,
result_payload=None,
last_error=_append_review_error(row["last_error"], issue),
prepared_at=evidence_time,
started_at=None,
completed_at=None,
updated_at=evidence_time,
))
def _reconcile_execution_states() -> None:
"""隔离非法执行组合，并统一清除人工复核态残留租约。"""
pending = sa.table(
_PENDING_TABLE,
sa.column("id", sa.Integer()),
sa.column("task_id", sa.String(64)),
sa.column("state", sa.String(32)),
sa.column("created_at", sa.String(40)),
sa.column("updated_at", sa.String(40)),
sa.column("last_error", sa.Text()),
sa.column("lease_owner", sa.String(128)),
sa.column("lease_token", sa.String(64)),
sa.column("lease_expires_at", sa.String(40)),
sa.column("heartbeat_at", sa.String(40)),
sa.column("attempt_count", sa.Integer()),
sa.column("execution_state", sa.String(32)),
sa.column("execution_version", sa.Integer()),
sa.column("execution_payload", sa.JSON()),
sa.column("execution_fingerprint", sa.String(64)),
sa.column("retry_due_at", sa.String(40)),
sa.column("settlement_revision", sa.Integer()),
sa.column("terminal_history_id", sa.Integer()),
)
bind = op.get_bind()
rows = [
dict(row)
for row in bind.execute(sa.select(pending)).mappings().all()
]
for row in rows:
issue = _execution_issue(row)
if issue is not None:
_insert_review_step(row, issue=issue)
bind.execute(
pending.update().where(pending.c.id == row["id"]).values(
execution_state="manual_review",
execution_version=None,
execution_payload=sa.null(),
execution_fingerprint=None,
lease_owner=None,
lease_token=None,
lease_expires_at=None,
heartbeat_at=None,
retry_due_at=None,
last_error=_append_review_error(row["last_error"], issue),
)
)
elif row["execution_state"] == "manual_review":
bind.execute(
pending.update().where(pending.c.id == row["id"]).values(
lease_owner=None,
lease_token=None,
lease_expires_at=None,
heartbeat_at=None,
)
)
def upgrade() -> None:
"""归一旧人工态并隔离无法安全自动恢复的执行组合。"""
tables = _table_names()
if _PENDING_TABLE not in tables:
return
if _STEP_TABLE not in tables:
raise RuntimeError("缺少 3.0.16 transferexecutionstep 表，拒绝执行 3.0.17")
_normalize_legacy_planning_states()
_reconcile_execution_states()
def downgrade() -> None:
"""保留更安全的状态归一和人工证据，结构由 3.0.16 负责降级。"""
docs/architecture-optimization-checklist.md
+2 -2
View File
@@ -77,9 +77,9 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain`
| Event Contract | 53 | 均已有 payload model，但当前全部是 diagnostic enforcement |
| Python 源码量 | 约 271,400 行 | 60 个文件超过 1,000 行，14 个超过 2,000 行 |
| 长方法 | 281 个超过 80 行 | 67 个超过 150 行，23 个超过 250 行；大量是私有方法 |
| 全量 mypy 历史债务 | 11,827 / 596 文件 | strict frontier 当前覆盖 41 个文件，本批迁移路径的类型债务已清零 |
| 全量 mypy 历史债务 | 11,820 / 596 文件 | strict frontier 当前覆盖 41 个文件，本批迁移路径的类型债务已清零 |
| Ruff 历史诊断 | 885 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` |
| 覆盖率低水位 | Application 78.74%，Domain 79.29% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
| 覆盖率低水位 | Application 78.81%，Domain 79.29% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 |
### 3.3 热点文件
docs/rules/05-architecture.md
+3
View File
@@ -550,6 +550,9 @@ expired claimed task remains exclusively owned by fenced recovery APIs.
- 物理模块仍存在但公开符号已经迁走时（例如 `app.domain.media` 的身份原语、
`app.schemas` 的整理工作项），兼容 Finder 在标准 Loader 执行后叠加白名单符号路由；
canonical 模块不得为兼容而反向 import `app.runtime.compat`。
- 符号级插件 ABI 只保证显式导入和属性访问；兼容符号不加入物理包的 `__all__`，
不支持依赖 `from ... import *` 获得迁移符号。宿主源码不得消费 `SYMBOL_ALIASES`
中的旧符号，必须直接导入 canonical owner，避免包根形成第二份宿主导出面。
- Canonical implementation packages may not import `app/runtime/compat` or
`app/sdk`.
- Host code uses canonical paths. Only `app/plugins/` and compatibility tests
tests/fixtures/architecture/coverage-baseline.json
Vendored
+3 -3
View File
@@ -1,8 +1,8 @@
{
"application": {
"covered_lines": 10098,
"percent": 78.74,
"statements": 12825
"covered_lines": 10068,
"percent": 78.81,
"statements": 12775
},
"domain": {
"covered_lines": 3392,
tests/fixtures/architecture/dependency-baseline.json
Vendored
+2 -2
View File
@@ -1442,7 +1442,7 @@
}
},
"edge_count": 6940,
"edge_sha256": "8ff91be099f1230655ceeb006bd1bf604063054ddb0721b3e887c956ea1251fb",
"edge_sha256": "9e3e8485c94c46a75eb577ccfc24708c80d4296b6422ec66ab9f23ef9964568f",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -4454,7 +4454,6 @@
"app.application.transfer.workflow -> app.adapters.system",
"app.application.transfer.workflow -> app.adapters.system.host",
"app.application.transfer.workflow -> app.application",
"app.application.transfer.workflow -> app.application.agent",
"app.application.transfer.workflow -> app.application.transfer",
"app.application.transfer.workflow -> app.application.transfer.execution",
"app.application.transfer.workflow -> app.domain",
@@ -5201,6 +5200,7 @@
"app.db.adapters.transfer.execution -> app.application",
"app.db.adapters.transfer.execution -> app.application.transfer",
"app.db.adapters.transfer.execution -> app.application.transfer.execution",
"app.db.adapters.transfer.execution -> app.application.transfer.workflow",
"app.db.adapters.transfer.execution -> app.db",
"app.db.adapters.transfer.execution -> app.db.models",
"app.db.adapters.transfer.execution -> app.db.models.transferexecutionstep",
tests/fixtures/architecture/mypy-baseline.json
Vendored
+3 -3
View File
@@ -1568,19 +1568,19 @@
"union-attr": 2
},
"app/chain/transfer.py": {
"arg-type": 57,
"arg-type": 53,
"assignment": 25,
"attr-defined": 4,
"func-returns-value": 1,
"misc": 2,
"no-any-return": 3,
"no-any-return": 2,
"no-untyped-call": 8,
"no-untyped-def": 12,
"operator": 3,
"return-value": 2,
"truthy-function": 5,
"type-arg": 10,
"union-attr": 33,
"union-attr": 31,
"var-annotated": 4
},
"app/chain/user.py": {
tests/test_architecture_dependencies.py
+95
View File
@@ -3,6 +3,7 @@ import json
from functools import lru_cache
from pathlib import Path
from app.runtime.compat.manifest import SYMBOL_ALIASES
from scripts.architecture.baseline import (
collect_current_event_facts as _collect_current_event_facts,
)
@@ -200,6 +201,58 @@ def _legacy_imports(path: Path) -> set[str]:
return imports
def _attribute_parts(node: ast.Attribute) -> list[str]:
"""将静态属性访问还原为从根名称开始的完整路径片段。"""
parts = [node.attr]
value = node.value
while isinstance(value, ast.Attribute):
parts.append(value.attr)
value = value.value
if not isinstance(value, ast.Name):
return []
parts.append(value.id)
return list(reversed(parts))
def _compat_symbol_references(tree: ast.AST) -> set[tuple[int, str]]:
"""收集显式导入或静态属性访问命中的兼容符号。"""
compatibility_symbols = {
(module_name, symbol_name)
for module_name, symbols in SYMBOL_ALIASES.items()
for symbol_name in symbols
}
module_bindings: dict[str, str] = {}
references: set[tuple[int, str]] = set()
for node in ast.walk(tree):
if isinstance(node, ast.Import):
for alias in node.names:
binding = alias.asname or alias.name.split(".", maxsplit=1)[0]
module_bindings[binding] = alias.name if alias.asname else binding
elif isinstance(node, ast.ImportFrom) and node.module:
for alias in node.names:
if (node.module, alias.name) in compatibility_symbols:
references.add((node.lineno, f"{node.module}.{alias.name}"))
continue
binding = alias.asname or alias.name
module_bindings[binding] = f"{node.module}.{alias.name}"
for node in ast.walk(tree):
if not isinstance(node, ast.Attribute):
continue
parts = _attribute_parts(node)
if len(parts) < 2:
continue
resolved_root = module_bindings.get(parts[0], parts[0])
resolved = [*resolved_root.split("."), *parts[1:]]
module_name = ".".join(resolved[:-1])
symbol_name = resolved[-1]
if (module_name, symbol_name) in compatibility_symbols:
references.add((node.lineno, f"{module_name}.{symbol_name}"))
return references
def test_legacy_roots_contain_no_python_sources():
"""旧目录只能作为运行时虚拟包存在，仓库中不得重新出现源码。"""
leftovers = sorted(
@@ -270,6 +323,48 @@ def test_host_code_does_not_import_legacy_roots():
assert violations == {}
def test_compat_symbol_scanner_covers_static_import_shapes() -> None:
"""兼容符号扫描必须覆盖显式导入、模块别名和完整属性链。"""
tree = ast.parse(
"""
from app.schemas import TransferTask
import app.schemas as schema_alias
schema_alias.TransferQueue
import app.schemas
app.schemas.TransferTask
from app.application import transfer as transfer_package
transfer_package.TransferQueue
"""
)
assert _compat_symbol_references(tree) == {
(2, "app.schemas.TransferTask"),
(4, "app.schemas.TransferQueue"),
(6, "app.schemas.TransferTask"),
(8, "app.application.transfer.TransferQueue"),
}
def test_host_code_does_not_use_compat_symbol_aliases() -> None:
"""宿主必须导入 canonical 符号，不得反向消费插件兼容覆盖。"""
violations: list[str] = []
for path in APP_ROOT.rglob("*.py"):
relative = path.relative_to(APP_ROOT)
if (
relative.parts[0] == "plugins"
or relative.parts[:2] == ("runtime", "compat")
or relative.parts[:2] == ("sdk", "_legacy")
):
continue
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
violations.extend(
f"{relative.as_posix()}:{line}:{symbol}"
for line, symbol in sorted(_compat_symbol_references(tree))
)
assert violations == []
def test_host_code_uses_explicit_runtime_facade_getters():
"""宿主消费者必须显式调用 getter，不得把兼容 Facade 当作新代码入口。"""
forbidden_imports = {
tests/test_chain_durable_events.py
+244 -19
View File
@@ -22,8 +22,11 @@ from app.application.chain.events import (
)
from app.application.history import TransferHistoryMutationCommand
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferExecutionConflictError,
TransferExecutionLeaseLostError,
TransferSettlementResult,
build_transfer_checkpoint_fingerprint,
)
from app.db.adapters.chain import TransactionalChainDurableEventWriter
from app.db.base import Base
@@ -81,16 +84,44 @@ def _objects():
return meta, media, context, fileitem, transferinfo
def _execution_checkpoint(
*,
outcome: str,
identity: str = "execution-1",
) -> TransferExecutionCheckpoint:
"""构造 outcome 与完整指纹一致的测试执行检查点。"""
overwrite_skipped = outcome == "overwrite_skipped"
transferinfo = (
TransferInfo(success=False, overwrite_skipped=True).model_dump(mode="json")
if overwrite_skipped
else None
)
return TransferExecutionCheckpoint.create(
payload={
"outcome": outcome,
"test_identity": identity,
**({"transferinfo": transferinfo} if transferinfo else {}),
},
operation_ids=() if overwrite_skipped else ("operation-1",),
skip_reason="overwrite_skipped" if overwrite_skipped else None,
)
def _add_settling_pending(
factory,
*,
task_id: str = "task-1",
lease_token: str = "lease-1",
execution_fingerprint: str = "execution-1",
execution_outcome: str = "succeeded",
execution_identity: str = "execution-1",
settlement_revision: int = 0,
src_path: str | None = None,
) -> None:
) -> TransferExecutionCheckpoint:
"""写入具备有效长租约和执行检查点的待结算任务。"""
checkpoint = _execution_checkpoint(
outcome=execution_outcome,
identity=execution_identity,
)
with factory() as session:
session.add(TransferPending(
task_id=task_id,
@@ -111,14 +142,15 @@ def _add_settling_pending(
heartbeat_at="2026-08-27 01:00:00.000000",
attempt_count=1,
execution_state="settling",
execution_version=1,
execution_payload={"schema_version": 1},
execution_fingerprint=execution_fingerprint,
execution_version=checkpoint.version,
execution_payload=checkpoint.to_payload(),
execution_fingerprint=checkpoint.fingerprint,
retry_generation=0,
retry_count=0,
settlement_revision=settlement_revision,
))
session.commit()
return checkpoint
def _settlement(
@@ -126,13 +158,18 @@ def _settlement(
outcome: str,
task_id: str = "task-1",
lease_token: str = "lease-1",
execution_fingerprint: str = "execution-1",
checkpoint_outcome: str | None = None,
execution_identity: str = "execution-1",
) -> TransferResultSettlement:
"""构造测试使用的稳定终态结算身份。"""
checkpoint = _execution_checkpoint(
outcome=checkpoint_outcome or outcome,
identity=execution_identity,
)
return TransferResultSettlement(
task_id=task_id,
lease_token=lease_token,
execution_fingerprint=execution_fingerprint,
execution_fingerprint=checkpoint.fingerprint,
outcome=outcome,
error="目标文件校验失败" if outcome == "failed" else None,
)
@@ -519,7 +556,9 @@ def test_task_success_settlement_atomically_deletes_pending_and_steps():
assert receipt.task_id == "task-1"
assert receipt.history_id == history.id
assert receipt.outcome == "succeeded"
assert receipt.execution_fingerprint == "execution-1"
assert receipt.execution_fingerprint == _execution_checkpoint(
outcome="succeeded"
).fingerprint
assert receipt.lease_token == "lease-1"
assert receipt.history_status is True
assert receipt.src == "/downloads/task-1.mkv"
@@ -599,7 +638,7 @@ def test_multiple_same_source_tasks_keep_independent_replay_receipts():
factory,
task_id="new-task",
lease_token="lease-2",
execution_fingerprint="execution-2",
execution_identity="execution-2",
src_path=shared_src,
)
@@ -617,7 +656,7 @@ def test_multiple_same_source_tasks_keep_independent_replay_receipts():
outcome="succeeded",
task_id="new-task",
lease_token="lease-2",
execution_fingerprint="execution-2",
execution_identity="execution-2",
),
)
old_replay = writer.transfer_result(
@@ -684,7 +723,11 @@ def test_task_settlement_without_public_topic_commits_no_outbox():
def test_task_settlement_binds_receipt_without_overwriting_success_history():
"""不覆盖裁决只绑定任务回执，保留旧成功历史的全部业务字段。"""
factory = _session_factory()
_add_settling_pending(factory, task_id="declined-task")
_add_settling_pending(
factory,
task_id="declined-task",
execution_outcome="overwrite_skipped",
)
with factory() as session:
session.add(TransferHistory(
src="/downloads/declined-task.mkv",
@@ -699,6 +742,7 @@ def test_task_settlement_binds_receipt_without_overwriting_success_history():
settlement = _settlement(
outcome="succeeded",
task_id="declined-task",
checkpoint_outcome="overwrite_skipped",
)
first = writer.transfer_result(
@@ -738,6 +782,170 @@ def test_task_settlement_binds_receipt_without_overwriting_success_history():
assert history.transfer_settlement_revision is None
def test_task_overwrite_skip_without_success_history_settles_failed():
"""覆盖跳过找不到成功历史时，显式执行事实仍可裁决为失败。"""
factory = _session_factory()
_add_settling_pending(factory, execution_outcome="overwrite_skipped")
writer = TransactionalChainDurableEventWriter(factory)
result = writer.transfer_result(
topic="transfer.failed",
stage_history=lambda repository: _stage_result_history(
repository,
task_id="task-1",
succeeded=False,
),
event_payload={},
publish=None,
settlement=_settlement(
outcome="failed",
checkpoint_outcome="overwrite_skipped",
),
)
assert isinstance(result, TransferSettlementResult)
assert result.pending_deleted is False
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
history = session.execute(select(TransferHistory)).scalar_one()
receipt = session.execute(select(TransferSettlementReceipt)).scalar_one()
assert pending.execution_state == "failed"
assert history.status is False
assert receipt.outcome == "failed"
@pytest.mark.parametrize(
("checkpoint_outcome", "settlement_outcome"),
[
("failed", "succeeded"),
("succeeded", "failed"),
],
)
def test_task_settlement_rejects_checkpoint_outcome_conflicts_before_writes(
checkpoint_outcome,
settlement_outcome,
):
"""执行证据与结算方向冲突或未知时，不得进入历史暂存。"""
factory = _session_factory()
_add_settling_pending(
factory,
execution_outcome=checkpoint_outcome,
)
writer = TransactionalChainDurableEventWriter(factory)
with pytest.raises(TransferExecutionConflictError):
writer.transfer_result(
topic="transfer.completed",
stage_history=lambda _repository: pytest.fail("冲突结算不得写历史"),
event_payload={},
publish=lambda _payload: pytest.fail("冲突结算不得发布"),
settlement=_settlement(
outcome=settlement_outcome,
checkpoint_outcome=checkpoint_outcome,
),
)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
assert session.execute(select(TransferHistory)).scalar_one_or_none() is None
assert session.execute(
select(TransferSettlementReceipt)
).scalar_one_or_none() is None
assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None
assert pending.execution_state == "settling"
assert pending.settlement_revision == 0
def test_task_settlement_rejects_corrupted_checkpoint_outcome_before_writes():
"""持久层出现未知执行 outcome 时必须隔离，不得构造历史或事件。"""
factory = _session_factory()
_add_settling_pending(factory)
corrupted_payload = {
"schema_version": 1,
"payload": {"outcome": "unknown", "test_identity": "execution-1"},
"operation_ids": ["operation-1"],
"skip_reason": None,
}
corrupted_fingerprint = build_transfer_checkpoint_fingerprint(
corrupted_payload
)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
pending.execution_payload = corrupted_payload
pending.execution_fingerprint = corrupted_fingerprint
session.commit()
writer = TransactionalChainDurableEventWriter(factory)
settlement = TransferResultSettlement(
task_id="task-1",
lease_token="lease-1",
execution_fingerprint=corrupted_fingerprint,
outcome="succeeded",
)
with pytest.raises(TransferExecutionConflictError):
writer.transfer_result(
topic="transfer.completed",
stage_history=lambda _repository: pytest.fail("损坏检查点不得写历史"),
event_payload={},
publish=lambda _payload: pytest.fail("损坏检查点不得发布"),
settlement=settlement,
)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
assert session.execute(select(TransferHistory)).scalar_one_or_none() is None
assert session.execute(
select(TransferSettlementReceipt)
).scalar_one_or_none() is None
assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None
assert pending.execution_state == "settling"
assert pending.settlement_revision == 0
def test_task_settlement_rejects_malformed_checkpoint_before_writes():
"""指纹自洽但结构损坏的数据库检查点也不得驱动终态写入。"""
factory = _session_factory()
_add_settling_pending(factory)
malformed_payload = {
"schema_version": 1,
"payload": {"outcome": "succeeded"},
"operation_ids": "operation-1",
"skip_reason": None,
}
fingerprint = build_transfer_checkpoint_fingerprint(malformed_payload)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
pending.execution_payload = malformed_payload
pending.execution_fingerprint = fingerprint
session.commit()
writer = TransactionalChainDurableEventWriter(factory)
settlement = TransferResultSettlement(
task_id="task-1",
lease_token="lease-1",
execution_fingerprint=fingerprint,
outcome="succeeded",
)
with pytest.raises(TransferExecutionConflictError):
writer.transfer_result(
topic="transfer.completed",
stage_history=lambda _repository: pytest.fail("损坏检查点不得写历史"),
event_payload={},
publish=lambda _payload: pytest.fail("损坏检查点不得发布"),
settlement=settlement,
)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
assert session.execute(select(TransferHistory)).scalar_one_or_none() is None
assert session.execute(
select(TransferSettlementReceipt)
).scalar_one_or_none() is None
assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None
assert pending.execution_state == "settling"
assert pending.settlement_revision == 0
@pytest.mark.parametrize("cleanup", ["delete", "truncate"])
def test_receipt_replay_survives_real_history_command_cleanup(cleanup):
"""真实历史删除或清空命令执行后，独立回执仍可重放成功终态。"""
@@ -857,7 +1065,7 @@ def test_success_receipt_allows_expiry_and_legacy_same_source_replace():
def test_task_failure_settlement_is_replayable_and_retry_advances_revision():
"""失败保留终态证据，重复调用幂等，显式重试后才递增修订号。"""
factory = _session_factory()
_add_settling_pending(factory)
_add_settling_pending(factory, execution_outcome="failed")
writer = TransactionalChainDurableEventWriter(factory)
calls = []
first_settlement = _settlement(outcome="failed")
@@ -890,8 +1098,14 @@ def test_task_failure_settlement_is_replayable_and_retry_advances_revision():
assert pending.lease_token is None
assert pending.settlement_revision == 1
assert pending.terminal_history_id == first.history_id
retry_checkpoint = _execution_checkpoint(
outcome="failed",
identity="execution-2",
)
pending.execution_state = "settling"
pending.execution_fingerprint = "execution-2"
pending.execution_version = retry_checkpoint.version
pending.execution_payload = retry_checkpoint.to_payload()
pending.execution_fingerprint = retry_checkpoint.fingerprint
pending.lease_owner = "worker-2"
pending.lease_token = "lease-2"
pending.lease_expires_at = "2099-01-01 00:00:00.000000"
@@ -909,7 +1123,7 @@ def test_task_failure_settlement_is_replayable_and_retry_advances_revision():
settlement=_settlement(
outcome="failed",
lease_token="lease-2",
execution_fingerprint="execution-2",
execution_identity="execution-2",
),
)
@@ -949,10 +1163,15 @@ def test_task_failure_settlement_is_replayable_and_retry_advances_revision():
assert [receipt.settlement_revision for receipt in receipts] == [1, 2]
assert all(receipt.task_id == "task-1" for receipt in receipts)
assert all(receipt.history_id == first.history_id for receipt in receipts)
assert receipts[0].execution_fingerprint == "execution-1"
assert receipts[0].execution_fingerprint == _execution_checkpoint(
outcome="failed"
).fingerprint
assert receipts[0].lease_token == "lease-1"
assert receipts[1].outcome == "failed"
assert receipts[1].execution_fingerprint == "execution-2"
assert receipts[1].execution_fingerprint == _execution_checkpoint(
outcome="failed",
identity="execution-2",
).fingerprint
assert receipts[1].lease_token == "lease-2"
assert receipts[1].pending_deleted is False
assert receipts[1].error == "目标文件校验失败"
@@ -966,7 +1185,7 @@ def test_task_failure_settlement_is_replayable_and_retry_advances_revision():
def test_failed_revision_replays_after_later_success_deleted_pending():
"""后续重试成功删除 pending 后，旧失败修订仍按原执行身份幂等回读。"""
factory = _session_factory()
_add_settling_pending(factory)
_add_settling_pending(factory, execution_outcome="failed")
writer = TransactionalChainDurableEventWriter(factory)
failed_settlement = _settlement(outcome="failed")
failed = writer.transfer_result(
@@ -982,8 +1201,14 @@ def test_failed_revision_replays_after_later_success_deleted_pending():
)
with factory() as session:
pending = session.execute(select(TransferPending)).scalar_one()
retry_checkpoint = _execution_checkpoint(
outcome="succeeded",
identity="execution-2",
)
pending.execution_state = "settling"
pending.execution_fingerprint = "execution-2"
pending.execution_version = retry_checkpoint.version
pending.execution_payload = retry_checkpoint.to_payload()
pending.execution_fingerprint = retry_checkpoint.fingerprint
pending.lease_owner = "worker-2"
pending.lease_token = "lease-2"
pending.lease_expires_at = "2099-01-01 00:00:00.000000"
@@ -991,7 +1216,7 @@ def test_failed_revision_replays_after_later_success_deleted_pending():
succeeded_settlement = _settlement(
outcome="succeeded",
lease_token="lease-2",
execution_fingerprint="execution-2",
execution_identity="execution-2",
)
succeeded = writer.transfer_result(
topic="transfer.completed",
tests/test_db_media_identity_normalizer.py
+10
View File
@@ -11,6 +11,7 @@ flush 前的事件。因此这里断言的是「绕过 Oper 直接建模写库
"""
import pytest
from app.application.transfer.workflow import TransferPlanningInput
from app.db.models.transferhistory import TransferHistory
from app.db.models.transferpending import TransferPending
from app.schemas.types import MediaSource
@@ -158,6 +159,12 @@ def test_tables_without_identity_columns_are_untouched(db):
不带身份列的表不受影响——事件挂在 Mapper 上覆盖全部映射，必须靠列名检查收窄，
否则会去动一张根本没有这两列的表。
"""
planning_input = TransferPlanningInput(source_fileitem={
"storage": "local",
"path": "/mnt/a.mkv",
"type": "file",
"name": "a.mkv",
})
TransferPending.stage_admit(
db.session,
task_id="identity-free-table",
@@ -165,6 +172,9 @@ def test_tables_without_identity_columns_are_untouched(db):
src_path="/mnt/a.mkv",
state="accepted",
now_time="2026-08-14 10:00:00",
input_version=planning_input.schema_version,
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
)
row = TransferPending.get_by_identity(
tests/test_db_transferpending_queries.py
+180 -1
View File
@@ -11,8 +11,10 @@ import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application.transfer.workflow import TransferPlanningInput
from app.db import base as db_base
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.models.transferexecutionstep import TransferExecutionStep
from app.db.models.transferhistory import TransferHistory
from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
@@ -22,6 +24,60 @@ from app.db.oper.transferpending import TransferPendingOper
def _track(db):
"""把待整理表纳入用例级回收。"""
db.watermark(TransferPending)
db.watermark(TransferExecutionStep)
def _leased_pending(
*,
task_id: str,
execution_state: str,
admission_state: str = "accepted",
) -> TransferPending:
"""构造持有有效租约且其余执行证据为空的 pending。"""
return TransferPending(
task_id=task_id,
storage="local",
src_path=f"/mnt/{task_id}.mkv",
state=admission_state,
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
input_version=1,
planning_input={"schema_version": 1},
input_fingerprint="input",
lease_owner="worker",
lease_token=f"lease-{task_id}",
lease_expires_at="2099-01-01 00:00:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
execution_state=execution_state,
retry_generation=0,
retry_count=0,
settlement_revision=0,
)
def _planning_input(path: str) -> TransferPlanningInput:
"""构造准入仓储要求的真实版本化规划输入。"""
return TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": path,
"type": "file",
"name": path.rsplit("/", 1)[-1],
},
meta=None,
mediainfo=None,
)
def _planning_fields(path: str) -> dict[str, object]:
"""返回 direct model/Oper 准入所需的显式版本化字段。"""
planning_input = _planning_input(path)
return {
"input_version": planning_input.schema_version,
"planning_input": planning_input.to_payload(),
"input_fingerprint": planning_input.fingerprint,
}
def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
@@ -33,6 +89,7 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
src_path="/mnt/durable.mkv",
state="accepted",
now_time="2026-08-27 10:00:00",
**_planning_fields("/mnt/durable.mkv"),
)
second = TransferPending.stage_admit(
db.session,
@@ -41,6 +98,7 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
src_path="/mnt/durable.mkv",
state="accepted",
now_time="2026-08-27 11:00:00",
**_planning_fields("/mnt/durable.mkv"),
)
assert first is second
@@ -81,7 +139,9 @@ def test_state_queries_and_failure_record_share_stable_identity(db):
src_path="/mnt/accepted.mkv",
state="accepted",
now_time="2026-08-27 10:00:00",
**_planning_fields("/mnt/accepted.mkv"),
)
other_fields = _planning_fields("/mnt/other.mkv")
db.add(TransferPending(
task_id="task-other",
storage="local",
@@ -89,6 +149,7 @@ def test_state_queries_and_failure_record_share_stable_identity(db):
state="other",
created_at="2026-08-27 10:00:01",
updated_at="2026-08-27 10:00:01",
**other_fields,
))
db.session.flush()
@@ -130,6 +191,7 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
src_path="/mnt/explicit-stage.mkv",
state="accepted",
now_time="2026-08-27 10:00:00",
**_planning_fields("/mnt/explicit-stage.mkv"),
)
assert pending.task_id == "task-explicit"
assert oper.get_by_task_id(task_id="task-explicit").state == "accepted"
@@ -146,16 +208,19 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
engine = create_engine(f"sqlite:///{tmp_path / 'transfer.db'}")
TransferHistory.__table__.create(engine)
TransferPending.__table__.create(engine)
TransferExecutionStep.__table__.create(engine)
factory = sessionmaker(bind=engine)
repository = TransactionalTransferAdmissionRepository(factory)
admitted = repository.admit(
storage="local",
src_path="/mnt/repository.mkv",
planning_input=_planning_input("/mnt/repository.mkv"),
)
repeated = repository.admit(
storage="local",
src_path="/mnt/repository.mkv",
planning_input=_planning_input("/mnt/repository.mkv"),
)
assert repeated == admitted
assert admitted.task_id
@@ -178,7 +243,7 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
lease_seconds=60,
)
assert claimed is not None
assert repository.discard_claimed(
assert repository.abandon_unstarted(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
) == 1
@@ -191,6 +256,120 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
engine.dispose()
@pytest.mark.parametrize(
("execution_state", "expected_deleted"),
[
("not_started", 1),
("running", 0),
("retry_wait", 0),
("settling", 0),
("failed", 0),
("manual_review", 0),
],
)
def test_abandon_unstarted_allows_only_pristine_execution_state(
db,
execution_state,
expected_deleted,
) -> None:
"""缺失源注销必须拒绝所有已开始、待重试、结算和人工状态。"""
task_id = f"abandon-{execution_state}"
db.add(_leased_pending(task_id=task_id, execution_state=execution_state))
db.session.flush()
deleted = TransferPending.abandon_unstarted(
db.session,
task_id=task_id,
lease_token=f"lease-{task_id}",
now_time="2026-08-27 10:01:00.000000",
)
db.session.flush()
assert deleted == expected_deleted
remaining = db.session.execute(
select(TransferPending).where(TransferPending.task_id == task_id)
).scalar_one_or_none()
assert (remaining is None) is bool(expected_deleted)
@pytest.mark.parametrize("admission_state", ["planned", "provider_pending"])
def test_abandon_unstarted_rejects_nonaccepted_admission_state(
db,
admission_state,
) -> None:
"""已进入 provider 或计划态的任务即使无步骤也不得按缺失源删除。"""
task_id = f"abandon-{admission_state}"
db.add(_leased_pending(
task_id=task_id,
execution_state="not_started",
admission_state=admission_state,
))
deleted = TransferPending.abandon_unstarted(
db.session,
task_id=task_id,
lease_token=f"lease-{task_id}",
now_time="2026-08-27 10:01:00.000000",
)
assert deleted == 0
def test_abandon_unstarted_rejects_task_with_any_step_evidence(db) -> None:
"""即使聚合状态尚未推进，已落库步骤也必须阻止删除 pending。"""
task_id = "abandon-with-step"
db.add(_leased_pending(task_id=task_id, execution_state="not_started"))
db.add(TransferExecutionStep(
task_id=task_id,
operation_id="operation-with-step",
checkpoint_fingerprint="plan",
ordinal=0,
phase="transfer",
kind="move",
state="prepared",
attempt_count=0,
intent_version=1,
intent_payload={"source": "/mnt/source.mkv"},
prepared_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
))
db.session.flush()
deleted = TransferPending.abandon_unstarted(
db.session,
task_id=task_id,
lease_token=f"lease-{task_id}",
now_time="2026-08-27 10:01:00.000000",
)
assert deleted == 0
assert db.session.execute(
select(TransferPending).where(TransferPending.task_id == task_id)
).scalar_one_or_none() is not None
def test_abandon_unstarted_rejects_task_with_terminal_history_evidence(db) -> None:
"""已有任务关联历史时不得删除 pending，避免掩盖未闭环结算。"""
task_id = "abandon-with-history"
db.add(_leased_pending(task_id=task_id, execution_state="not_started"))
db.add(TransferHistory(
transfer_task_id=task_id,
transfer_settlement_revision=1,
src="/mnt/abandon-with-history.mkv",
src_storage="local",
status=False,
))
deleted = TransferPending.abandon_unstarted(
db.session,
task_id=task_id,
lease_token=f"lease-{task_id}",
now_time="2026-08-27 10:01:00.000000",
)
assert deleted == 0
def test_transactional_repository_rolls_back_failed_write(monkeypatch):
"""适配器写入异常时必须回滚自身 UoW 并传播原异常。"""
class SessionContext:
tests/test_legacy_import_compat.py
+52
View File
@@ -220,6 +220,58 @@ def test_transfer_package_exposes_plugin_symbols_only_through_overlay() -> None:
reset_legacy_import_diagnostics()
def test_transfer_legacy_symbols_support_all_explicit_imports() -> None:
"""六个旧整理符号显式导入应在隔离进程中共享同一兼容类型。"""
code = """
from app.application.transfer import TransferTask as ApplicationTask
from app.application.transfer import TransferQueue as ApplicationQueue
from app.schemas import TransferTask as SchemaTask
from app.schemas import TransferQueue as SchemaQueue
from app.schemas.transfer import TransferTask as TransferSchemaTask
from app.schemas.transfer import TransferQueue as TransferSchemaQueue
import app.application.transfer as application_package
import app.schemas as schemas_package
import app.schemas.transfer as transfer_schema
from app.application.transfer.workflow import TransferQueue as CanonicalQueue
from app.application.transfer.workflow import TransferTask as CanonicalTask
from app.sdk._legacy.transfer import TransferQueue as LegacyQueue
from app.sdk._legacy.transfer import TransferTask as LegacyTask
class LegacyPayload:
def model_dump(self):
return {"kind": "legacy"}
task_types = (ApplicationTask, SchemaTask, TransferSchemaTask)
queue_types = (ApplicationQueue, SchemaQueue, TransferSchemaQueue)
assert all(task_type is LegacyTask for task_type in task_types)
assert all(queue_type is LegacyQueue for queue_type in queue_types)
assert issubclass(LegacyTask, CanonicalTask)
assert issubclass(LegacyQueue, CanonicalQueue)
task = ApplicationTask(
fileitem={"storage": "local", "path": "/downloads/movie.mkv", "type": "file"},
meta=LegacyPayload(),
)
assert isinstance(task, CanonicalTask)
assert task.to_dict()["meta"] == {"kind": "legacy"}
for queue_type in queue_types:
queue = queue_type(task=task)
assert queue.task is task
for package in (application_package, schemas_package, transfer_schema):
assert "TransferTask" not in package.__all__
assert "TransferQueue" not in package.__all__
"""
subprocess.run(
[sys.executable, "-c", code],
cwd=Path(__file__).parents[1],
check=True,
)
def test_virtual_package_exports_resolve_exact_manifest_symbols():
"""合成旧包仅公开 manifest 声明的符号，并记录 DEBUG 兼容警告。"""
legacy_package = "app.core.meta"
tests/test_music_transfer.py
+31 -2
View File
@@ -5,6 +5,10 @@ from unittest.mock import Mock
from jinja2 import Template
from app.application.messaging.message import TemplateHelper
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferSettlementResult,
)
from app.application.transfer.workflow import TransferTask
from app.chain.media import MediaChain
from app.chain.transfer import JobManager, TransferChain
@@ -494,6 +498,18 @@ def test_success_file_aggregation_is_isolated_between_music_jobs_in_same_directo
chain.eventmanager = Mock()
chain.transfer_completed = Mock()
chain.send_transfer_message = Mock()
def transfer_result(**kwargs):
"""执行测试历史暂存并返回 task-aware 原子结算回执。"""
history = kwargs["stage_history"](SimpleNamespace())
return TransferSettlementResult(
history_id=history.id,
settlement_revision=1,
pending_deleted=True,
)
chain.durable_event_writer = Mock()
chain.durable_event_writer.transfer_result.side_effect = transfer_result
album_infos = [
MusicInfo(
music_type="album",
@@ -544,8 +560,21 @@ def test_success_file_aggregation_is_isolated_between_music_jobs_in_same_directo
lambda **kwargs: SimpleNamespace(id=1),
)
for task in tasks:
chain._TransferChain__default_callback(task, transfer_info(task))
for sequence, task in enumerate(tasks):
result = transfer_info(task)
task.bind_admission_task_id(f"music-terminal-{sequence}")
task.bind_execution_lease(
owner_id="music-test-owner",
lease_token=f"music-lease-{sequence}",
)
task.bind_execution_checkpoint(TransferExecutionCheckpoint.create(
payload={
"outcome": "succeeded",
"transferinfo": result.model_dump(mode="json"),
},
operation_ids=(f"music-operation-{sequence}",),
))
chain._TransferChain__default_callback(task, result)
notified_lists = [
call.kwargs["transferinfo"].file_list_new
tests/test_transfer_admission_migration.py
+40
View File
@@ -170,6 +170,46 @@ def test_transfer_admission_upgrade_downgrade_reupgrade(
}
def test_replayed_upgrade_repairs_named_constraint_and_index(monkeypatch) -> None:
"""同名但列或唯一性错误的准入约束与索引必须精确重建。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_legacy_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
with migration.op.batch_alter_table("transferpending") as batch_op:
batch_op.drop_constraint(
"uq_transferpending_task_id",
type_="unique",
)
batch_op.create_unique_constraint(
"uq_transferpending_task_id",
["src_path"],
)
connection.execute(sa.text(
"DROP INDEX ix_transferpending_state_created"
))
connection.execute(sa.text(
"CREATE UNIQUE INDEX ix_transferpending_state_created "
"ON transferpending (task_id)"
))
migration.upgrade()
inspector = sa.inspect(connection)
constraint = next(
item for item in inspector.get_unique_constraints("transferpending")
if item["name"] == "uq_transferpending_task_id"
)
index = next(
item for item in inspector.get_indexes("transferpending")
if item["name"] == "ix_transferpending_state_created"
)
assert constraint["column_names"] == ["task_id"]
assert index["column_names"] == ["state", "created_at", "id"]
assert index["unique"] == 0
def test_transfer_admission_migration_runs_on_postgresql(monkeypatch) -> None:
"""隔离 PostgreSQL 应真实执行准入字段、约束、索引和可逆回滚。"""
prefix = "MOVIEPILOT_TEST_POSTGRESQL_"
tests/test_transfer_execution_migration.py
+105 -13
View File
@@ -378,6 +378,83 @@ def test_downgrade_marks_step_evidence_then_reupgrade_keeps_manual_review(
engine.dispose()
def test_downgrade_archives_and_reupgrade_restores_settlement_receipts(
monkeypatch,
) -> None:
"""降级必须归档 append-only 终态证据，重复降级和重升均不得丢失。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
pending, _ = _create_legacy_tables(connection)
_insert_legacy_rows(connection, pending)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
connection.execute(sa.text(
"INSERT INTO transfersettlementreceipt ("
"task_id, history_id, settlement_revision, outcome, "
"execution_fingerprint, lease_token, history_status, src, src_storage, "
"pending_deleted, error, created_at, updated_at"
") VALUES ("
"'settled', 42, 1, 'succeeded', 'fingerprint', 'lease', 1, "
"'/source', 'local', 1, NULL, "
"'2026-08-27 12:00:00', '2026-08-27 12:00:00'"
")"
))
migration.downgrade()
migration.downgrade()
tables = sa.inspect(connection).get_table_names()
assert "transfersettlementreceipt" not in tables
assert "transfersettlementreceipt_3_0_16_archive" in tables
migration.upgrade()
migration.upgrade()
receipt = connection.execute(sa.text(
"SELECT task_id, outcome, settlement_revision "
"FROM transfersettlementreceipt"
)).one()
assert receipt == ("settled", "succeeded", 1)
assert "transfersettlementreceipt_3_0_16_archive" not in (
sa.inspect(connection).get_table_names()
)
def test_upgrade_repairs_owned_indexes_by_columns_and_uniqueness(monkeypatch) -> None:
"""关键索引同名但列或唯一性错误时必须按 ORM 契约重建。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
pending, _ = _create_legacy_tables(connection)
_insert_legacy_rows(connection, pending)
migration = _bind_migration(monkeypatch, connection)
migration._add_pending_columns()
migration._backfill_pending()
migration._add_history_columns()
connection.execute(sa.text(
"CREATE UNIQUE INDEX ix_transferpending_execution_due "
"ON transferpending (task_id)"
))
connection.execute(sa.text(
"CREATE INDEX ux_transferhistory_transfer_task_id "
"ON transferhistory (src_storage)"
))
migration.upgrade()
inspector = sa.inspect(connection)
pending_index = next(
item for item in inspector.get_indexes("transferpending")
if item["name"] == "ix_transferpending_execution_due"
)
history_index = next(
item for item in inspector.get_indexes("transferhistory")
if item["name"] == "ux_transferhistory_transfer_task_id"
)
assert pending_index["column_names"] == [
"execution_state", "retry_due_at", "state", "created_at", "id",
]
assert pending_index["unique"] == 0
assert history_index["column_names"] == ["transfer_task_id"]
assert history_index["unique"] == 1
def test_upgrade_without_pending_table_is_a_safe_noop(monkeypatch):
"""全新数据库尚未执行前置迁移时本版本应安全等待迁移链建表。"""
engine = sa.create_engine("sqlite://")
@@ -659,7 +736,7 @@ def test_upgrade_adds_synthetic_review_when_nonmanual_step_already_exists(
def test_migrated_legacy_reviews_are_discoverable_resolvable_and_retryable(
monkeypatch,
) -> None:
"""迁移遗留任务应可分页判定，并在判定后准备真实首步骤。"""
"""迁移遗留任务应可分页判定，判定后仍须重新规划才能准备步骤。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
pending, _ = _create_legacy_tables(connection)
@@ -753,17 +830,32 @@ def test_migrated_legacy_reviews_are_discoverable_resolvable_and_retryable(
snapshot = repository.get_snapshot(task_id=task_id)
assert snapshot is not None
assert snapshot.steps == ()
prepared = command.prepare(
task_id=task_id,
lease_token=f"lease-{task_id}",
intent=TransferStepIntent.create(
with factory() as session:
pending = session.scalar(
sa.select(TransferPending).where(
TransferPending.task_id == task_id
)
)
assert pending is not None
assert pending.state in {"accepted", "planned"}
with pytest.raises(
TransferExecutionConflictError,
match="可执行规划状态|完整计划检查点|无法恢复",
):
command.prepare(
task_id=task_id,
checkpoint_fingerprint="f" * 64,
ordinal=0,
phase="transfer",
kind="copy",
payload={"source": task_id},
),
)
assert prepared.ordinal == 0
lease_token=f"lease-{task_id}",
intent=TransferStepIntent.create(
task_id=task_id,
checkpoint_fingerprint="f" * 64,
ordinal=0,
phase="transfer",
kind="copy",
payload={"source": task_id},
),
)
current = repository.get_snapshot(task_id=task_id)
assert current is not None
assert current.state is TransferExecutionState.RETRY_WAIT
assert current.steps == ()
engine.dispose()
tests/test_transfer_execution_persistence.py
+449 -12
View File
@@ -1,5 +1,6 @@
"""验证整理执行证据、CAS fencing 与终态结算持久化。"""
from dataclasses import replace
from datetime import datetime, timezone
import pytest
@@ -20,6 +21,13 @@ from app.application.transfer.execution import (
build_transfer_checkpoint_fingerprint,
build_transfer_operation_id,
)
from app.application.transfer.workflow import (
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.adapters.transfer.execution import (
TransactionalTransferExecutionRepository,
)
@@ -48,22 +56,70 @@ def execution_store():
engine.dispose()
def _seed_pending(factory, *, task_id: str = "task-1", lease_token: str = "lease-1"):
def _planning_input(task_id: str) -> TransferPlanningInput:
"""构造与测试源文件一致的持久规划输入。"""
return TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": f"/{task_id}.mkv",
"type": "file",
},
target_storage="local",
target_path="/media",
requested_transfer_type="copy",
)
def _plan_checkpoint(task_id: str) -> TransferPlanCheckpoint:
"""构造包含一个叶子文件计划的完整宿主检查点。"""
planning_input = _planning_input(task_id)
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/media",
final_target_path=f"/media/{task_id}.mkv",
resolved_transfer_type="copy",
items=(TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path=f"/media/{task_id}.mkv",
),),
)
def _plan_fingerprint(task_id: str) -> str:
"""返回测试冻结计划使用的 canonical 指纹。"""
return build_transfer_checkpoint_fingerprint(
_plan_checkpoint(task_id).to_payload()
)
def _seed_pending(
factory,
*,
task_id: str = "task-1",
lease_token: str = "lease-1",
state: str = "planned",
with_checkpoint: bool = True,
):
"""写入一条带有效租约与合法 planning checkpoint 的待执行任务。"""
planning_input = _planning_input(task_id)
checkpoint = _plan_checkpoint(task_id) if with_checkpoint else None
with factory() as session:
session.add(TransferPending(
task_id=task_id,
storage="local",
src_path=f"/{task_id}.mkv",
created_at="2026-08-27 09:00:00",
state="planned",
state=state,
updated_at="2026-08-27 09:00:00",
input_version=1,
planning_input={"schema_version": 1, "source": task_id},
input_fingerprint="input-fingerprint",
checkpoint_version=1,
checkpoint_payload={"schema_version": 1, "task_id": task_id},
planned_at="2026-08-27 09:00:00",
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
checkpoint_version=checkpoint.schema_version if checkpoint else None,
checkpoint_payload=checkpoint.to_payload() if checkpoint else None,
planned_at="2026-08-27 09:00:00" if checkpoint else None,
lease_owner="worker-1",
lease_token=lease_token,
lease_expires_at="2099-01-01 00:00:00.000000",
@@ -93,13 +149,19 @@ def _repository(factory, token_values: list[str] | None = None):
def _intent(*, task_id: str = "task-1", ordinal: int = 0) -> TransferStepIntent:
"""构造稳定且可重复计算身份的测试步骤意图。"""
planning_input = _planning_input(task_id)
return TransferStepIntent.create(
task_id=task_id,
checkpoint_fingerprint="plan-fingerprint",
checkpoint_fingerprint=_plan_fingerprint(task_id),
ordinal=ordinal,
phase="transfer",
kind="copy",
payload={"src": f"/{task_id}.mkv", "dest": f"/media/{task_id}.mkv"},
kind="materialize_target",
payload={
"source": planning_input.source_fileitem,
"target_storage": "local",
"target_path": f"/media/{task_id}.mkv",
"transfer_type": "copy",
},
)
@@ -138,6 +200,143 @@ def test_stable_operation_and_checkpoint_identities_are_canonical():
assert intent.payload == {"path": "/original"}
@pytest.mark.parametrize(
("state", "with_checkpoint"),
(("accepted", False), ("planned", False)),
)
def test_prepare_rejects_task_without_executable_plan(
execution_store,
state,
with_checkpoint,
):
"""接纳态或缺失完整 checkpoint 的任务不得进入外部步骤准备。"""
_seed_pending(
execution_store,
state=state,
with_checkpoint=with_checkpoint,
)
_, command = _repository(execution_store)
with pytest.raises(TransferExecutionConflictError):
command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=_intent(),
)
with execution_store() as session:
pending = session.scalar(select(TransferPending))
assert pending is not None
assert pending.execution_state == "not_started"
assert session.scalar(select(TransferExecutionStep)) is None
def test_prepare_rejects_noncanonical_plan_fingerprint(execution_store):
"""步骤 intent 必须精确绑定当前持久计划的 canonical 指纹。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
intent = TransferStepIntent.create(
task_id="task-1",
checkpoint_fingerprint="0" * 64,
ordinal=0,
phase="transfer",
kind="materialize_target",
payload=_intent().payload,
)
with pytest.raises(TransferExecutionConflictError, match="当前冻结计划指纹"):
command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=intent,
)
def test_prepare_rejects_forged_operation_id(execution_store):
"""调用方手工构造的伪 operation ID 不得绕过稳定身份计算。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
valid = _intent()
forged = TransferStepIntent(
operation_id="f" * 64,
checkpoint_fingerprint=valid.checkpoint_fingerprint,
ordinal=valid.ordinal,
phase=valid.phase,
kind=valid.kind,
payload=valid.payload,
)
with pytest.raises(TransferExecutionConflictError, match="operation ID 不可信"):
command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=forged,
)
def test_prepare_rejects_arbitrary_intent_with_known_plan_fingerprint(
execution_store,
):
"""已知计划指纹也不能构造计划未授权的操作类型或参数。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
arbitrary = TransferStepIntent.create(
task_id="task-1",
checkpoint_fingerprint=_plan_fingerprint("task-1"),
ordinal=0,
phase="transfer",
kind="delete_move_source",
payload={
"source": _planning_input("task-1").source_fileitem,
"target_storage": "local",
"target_path": "/media/task-1.mkv",
},
)
with pytest.raises(TransferExecutionConflictError, match="冻结计划导出"):
command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=arbitrary,
)
def test_prepare_rejects_noncontiguous_ordinal(execution_store):
"""新步骤只能在完整既有序列尾部连续追加。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
with pytest.raises(TransferExecutionConflictError, match="连续追加"):
command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=_intent(ordinal=1),
)
def test_stage_execution_running_cas_binds_exact_plan_identity(execution_store):
"""running CAS 必须拒绝读取后已变化的准入状态或 checkpoint payload。"""
_seed_pending(execution_store)
checkpoint = _plan_checkpoint("task-1")
with execution_store() as session:
stale = TransferPending.stage_execution_running(
session,
task_id="task-1",
lease_token="lease-1",
admission_state="planned",
checkpoint_version=checkpoint.schema_version,
checkpoint_payload={"schema_version": checkpoint.schema_version},
now_utc="2026-08-27 01:30:00.000000",
updated_at="2026-08-27 09:30:00",
)
assert stale == 0
current = TransferPending.stage_execution_running(
session,
task_id="task-1",
lease_token="lease-1",
admission_state="planned",
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint.to_payload(),
now_utc="2026-08-27 01:30:00.000000",
updated_at="2026-08-27 09:30:00",
)
assert current == 1
def test_success_path_persists_steps_and_execution_checkpoint(execution_store):
"""成功路径应保留每步证据，并提交可供唯一 durable writer 结算的检查点。"""
_seed_pending(execution_store)
@@ -161,7 +360,7 @@ def test_success_path_persists_steps_and_execution_checkpoint(execution_store):
result=TransferStepResult(payload={"dest_exists": True}),
)
checkpoint = TransferExecutionCheckpoint.create(
payload={"dest": "/media/task-1.mkv"},
payload={"outcome": "succeeded", "dest": "/media/task-1.mkv"},
operation_ids=(succeeded.operation_id,),
)
snapshot = command.checkpoint(
@@ -178,6 +377,244 @@ def test_success_path_persists_steps_and_execution_checkpoint(execution_store):
assert step is not None and step.state == "succeeded"
def test_provider_predecessor_remains_owned_after_host_plan_promotion(
execution_store,
):
"""provider 回退升级宿主计划后，序号零证据仍应被严格重建并纳入 checkpoint。"""
task_id = "task-1"
planning_input = _planning_input(task_id)
provider = TransferProviderReference(
plugin_id="provider-a",
plugin_name="Provider A",
)
invocation = TransferProviderInvocationSnapshot(
fileitem=planning_input.source_fileitem,
meta={"title": "Movie"},
meta_kind="MetaVideo",
mediainfo={"title": "Movie"},
mediainfo_kind="MediaInfo",
)
provider_checkpoint = TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
legacy_transfer_providers=(provider,),
provider_invocation=invocation,
)
_seed_pending(execution_store)
with execution_store() as session:
pending = session.scalar(select(TransferPending))
assert pending is not None
pending.state = "provider_pending"
pending.checkpoint_payload = provider_checkpoint.to_payload()
session.commit()
_, command = _repository(execution_store)
provider_intent = TransferStepIntent.create(
task_id=task_id,
checkpoint_fingerprint=build_transfer_checkpoint_fingerprint(
provider_checkpoint.to_payload()
),
ordinal=0,
phase="provider",
kind="legacy_transfer_provider_sequence",
payload={
"providers": [provider.to_payload()],
"invocation": invocation.to_payload(),
},
)
prepared = command.prepare(
task_id=task_id,
lease_token="lease-1",
intent=provider_intent,
)
started = command.begin(
task_id=task_id,
lease_token="lease-1",
operation_id=prepared.operation_id,
)
provider_succeeded = command.complete(
task_id=task_id,
lease_token="lease-1",
step=started,
result=TransferStepResult(payload={"handled": False}),
)
promoted_checkpoint = replace(
_plan_checkpoint(task_id),
pre_execution_cleanup_completed=True,
)
with execution_store() as session:
pending = session.scalar(select(TransferPending))
assert pending is not None
pending.state = "planned"
pending.checkpoint_payload = promoted_checkpoint.to_payload()
session.commit()
promoted_fingerprint = build_transfer_checkpoint_fingerprint(
promoted_checkpoint.to_payload()
)
host_intent = TransferStepIntent.create(
task_id=task_id,
checkpoint_fingerprint=promoted_fingerprint,
ordinal=1,
phase="transfer",
kind="materialize_target",
payload=_intent().payload,
)
host_prepared = command.prepare(
task_id=task_id,
lease_token="lease-1",
intent=host_intent,
)
host_started = command.begin(
task_id=task_id,
lease_token="lease-1",
operation_id=host_prepared.operation_id,
)
host_succeeded = command.complete(
task_id=task_id,
lease_token="lease-1",
step=host_started,
result=TransferStepResult(payload={"dest_exists": True}),
)
execution_checkpoint = TransferExecutionCheckpoint.create(
payload={"outcome": "succeeded", "dest": "/media/task-1.mkv"},
operation_ids=(
provider_succeeded.operation_id,
host_succeeded.operation_id,
),
)
snapshot = command.checkpoint(
task_id=task_id,
lease_token="lease-1",
checkpoint=execution_checkpoint,
)
assert snapshot.state is TransferExecutionState.SETTLING
assert tuple(step.ordinal for step in snapshot.steps) == (0, 1)
def test_checkpoint_rejects_operation_ids_out_of_ordinal_order(execution_store):
"""执行检查点必须按严格 ordinal 保存操作身份，集合相同也不能乱序。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
completed = []
for ordinal in range(2):
prepared = command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=_intent(ordinal=ordinal),
)
started = command.begin(
task_id="task-1",
lease_token="lease-1",
operation_id=prepared.operation_id,
)
completed.append(command.complete(
task_id="task-1",
lease_token="lease-1",
step=started,
result=TransferStepResult(payload={"ordinal": ordinal}),
))
checkpoint = TransferExecutionCheckpoint.create(
payload={"outcome": "succeeded", "dest": "/media/task-1.mkv"},
operation_ids=tuple(step.operation_id for step in reversed(completed)),
)
with pytest.raises(TransferExecutionConflictError, match="步骤顺序"):
command.checkpoint(
task_id="task-1",
lease_token="lease-1",
checkpoint=checkpoint,
)
def test_checkpoint_rejects_corrupted_persisted_operation_id(execution_store):
"""checkpoint 前必须重新计算每个持久步骤的 operation ID。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
prepared = command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=_intent(),
)
started = command.begin(
task_id="task-1",
lease_token="lease-1",
operation_id=prepared.operation_id,
)
command.complete(
task_id="task-1",
lease_token="lease-1",
step=started,
result=TransferStepResult(payload={"dest_exists": True}),
)
with execution_store() as session:
step = session.scalar(select(TransferExecutionStep))
assert step is not None
step.operation_id = "e" * 64
session.commit()
checkpoint = TransferExecutionCheckpoint.create(
payload={"outcome": "succeeded", "dest": "/media/task-1.mkv"},
operation_ids=("e" * 64,),
)
with pytest.raises(TransferExecutionConflictError, match="冻结意图"):
command.checkpoint(
task_id="task-1",
lease_token="lease-1",
checkpoint=checkpoint,
)
def test_checkpoint_rejects_noncontiguous_persisted_ordinals(execution_store):
"""checkpoint 前必须拒绝缺口或从非零开始的持久步骤序列。"""
_seed_pending(execution_store)
_, command = _repository(execution_store)
prepared = command.prepare(
task_id="task-1",
lease_token="lease-1",
intent=_intent(),
)
started = command.begin(
task_id="task-1",
lease_token="lease-1",
operation_id=prepared.operation_id,
)
command.complete(
task_id="task-1",
lease_token="lease-1",
step=started,
result=TransferStepResult(payload={"dest_exists": True}),
)
with execution_store() as session:
step = session.scalar(select(TransferExecutionStep))
assert step is not None
step.ordinal = 2
step.operation_id = build_transfer_operation_id(
task_id=step.task_id,
checkpoint_fingerprint=step.checkpoint_fingerprint,
ordinal=step.ordinal,
phase=step.phase,
kind=step.kind,
intent_payload=step.intent_payload,
)
corrupted_operation_id = step.operation_id
session.commit()
checkpoint = TransferExecutionCheckpoint.create(
payload={"outcome": "succeeded", "dest": "/media/task-1.mkv"},
operation_ids=(corrupted_operation_id,),
)
with pytest.raises(TransferExecutionConflictError, match="全局序号不连续"):
command.checkpoint(
task_id="task-1",
lease_token="lease-1",
checkpoint=checkpoint,
)
def test_retry_wait_resumes_same_failed_operation_with_new_attempt(execution_store):
"""到期重试必须复用 operation ID、保留失败证据并轮换 attempt token。"""
_seed_pending(execution_store)
@@ -264,7 +701,7 @@ def test_zero_side_effect_checkpoint_is_vacuously_complete(execution_store):
_seed_pending(execution_store)
_, command = _repository(execution_store)
checkpoint = TransferExecutionCheckpoint.create(
payload={"preview": True, "accepted": False},
payload={"outcome": "failed", "preview": True, "accepted": False},
operation_ids=(),
skip_reason="preview",
)
tests/test_transfer_execution_runner.py
+202 -16
View File
@@ -15,6 +15,12 @@ from app.application.transfer.execution import (
TransferOperationObservationState,
TransferStepIntent,
TransferStepResult,
build_transfer_checkpoint_fingerprint,
)
from app.application.transfer.workflow import (
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
)
from app.chain import transfer as transfer_chain_module
from app.db.adapters.transfer.execution import (
@@ -28,6 +34,33 @@ from app.modules.filemanager.transhandler import TransHandler
from app.schemas.workflow import FileItem
def _runner_plan_checkpoint() -> TransferPlanCheckpoint:
"""构造 runner fixture 使用的完整冻结计划。"""
planning_input = TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": "/source.mkv",
"type": "file",
},
target_storage="local",
target_path="/target.mkv",
requested_transfer_type="copy",
)
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/",
final_target_path="/target.mkv",
resolved_transfer_type="copy",
items=(TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path="/target.mkv",
),),
)
@pytest.fixture
def execution_repository():
"""构造带有效 pending 租约的独立执行仓储。"""
@@ -41,6 +74,8 @@ def execution_repository():
],
)
factory = sessionmaker(bind=engine, expire_on_commit=False)
plan_checkpoint = _runner_plan_checkpoint()
planning_input = plan_checkpoint.planning_input
with factory() as session:
session.add(TransferPending(
task_id="task-runner",
@@ -50,10 +85,10 @@ def execution_repository():
state="planned",
updated_at="2026-08-27 09:00:00",
input_version=1,
planning_input={"schema_version": 1},
input_fingerprint="input",
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
checkpoint_version=1,
checkpoint_payload={"schema_version": 1},
checkpoint_payload=plan_checkpoint.to_payload(),
planned_at="2026-08-27 09:00:00",
lease_owner="worker",
lease_token="lease",
@@ -82,11 +117,30 @@ def _runner(repository):
return transfer_chain_module._DurableTransferStepRunner(
task_id="task-runner",
lease_token="lease",
checkpoint_fingerprint="plan",
checkpoint_fingerprint=_runner_plan_fingerprint(),
repository=repository,
)
def _runner_plan_fingerprint() -> str:
"""返回 runner fixture 中完整冻结计划的 canonical 指纹。"""
return build_transfer_checkpoint_fingerprint(
_runner_plan_checkpoint().to_payload()
)
def _runner_step_payload() -> dict:
"""返回由 runner 冻结计划唯一叶操作导出的目标落地参数。"""
checkpoint = _runner_plan_checkpoint()
item = checkpoint.items[0]
return {
"source": item.source_fileitem,
"target_storage": item.target_storage,
"target_path": item.target_path,
"transfer_type": checkpoint.resolved_transfer_type,
}
def test_runner_replay_returns_persisted_result_without_repeating_side_effect(
execution_repository,
):
@@ -94,8 +148,8 @@ def test_runner_replay_returns_persisted_result_without_repeating_side_effect(
calls = []
first = _runner(execution_repository).run(
phase="transfer",
kind="copy",
payload={"source": "/source.mkv", "target": "/target.mkv"},
kind="materialize_target",
payload=_runner_step_payload(),
execute=lambda: calls.append("executed") or TransferStepResult(
payload={"item": {"path": "/target.mkv"}}
),
@@ -103,8 +157,8 @@ def test_runner_replay_returns_persisted_result_without_repeating_side_effect(
)
second = _runner(execution_repository).run(
phase="transfer",
kind="copy",
payload={"source": "/source.mkv", "target": "/target.mkv"},
kind="materialize_target",
payload=_runner_step_payload(),
execute=lambda: pytest.fail("已成功步骤不得重复执行"),
observe=lambda: pytest.fail("已成功步骤不得执行恢复探测"),
)
@@ -122,11 +176,11 @@ def test_runner_routes_unknown_orphaned_attempt_to_manual_review(
lease_token="lease",
intent=TransferStepIntent.create(
task_id="task-runner",
checkpoint_fingerprint="plan",
ordinal=0,
phase="provider",
kind="opaque",
payload={"provider": "legacy"},
checkpoint_fingerprint=_runner_plan_fingerprint(),
ordinal=0,
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
),
)
command.begin(
@@ -140,9 +194,9 @@ def test_runner_routes_unknown_orphaned_attempt_to_manual_review(
match="禁止自动重放",
):
_runner(execution_repository).run(
phase="provider",
kind="opaque",
payload={"provider": "legacy"},
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
execute=lambda: pytest.fail("未知遗留步骤不得重放"),
observe=lambda: TransferOperationObservation(
state=TransferOperationObservationState.UNKNOWN,
@@ -154,6 +208,138 @@ def test_runner_routes_unknown_orphaned_attempt_to_manual_review(
assert snapshot.state is TransferExecutionState.MANUAL_REVIEW
def test_runner_observes_applied_after_execute_error_and_completes_step(
execution_repository,
) -> None:
"""execute 抛错后若外部事实已生效，必须以观察证据完成而非重放。"""
evidence = TransferStepResult(payload={"target": "/target.mkv"})
def execute() -> TransferStepResult:
"""模拟副作用成功后调用方在返回前崩溃。"""
raise OSError("connection reset after apply")
result = _runner(execution_repository).run(
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
execute=execute,
observe=lambda: TransferOperationObservation(
state=TransferOperationObservationState.APPLIED,
evidence=evidence,
),
)
snapshot = execution_repository.get_snapshot(task_id="task-runner")
assert result == evidence
assert snapshot is not None
assert snapshot.state is TransferExecutionState.RUNNING
assert snapshot.steps[0].result == evidence
assert snapshot.steps[0].state.value == "succeeded"
def test_runner_defers_after_execute_error_observed_not_applied(
execution_repository,
) -> None:
"""execute 抛错且确认未生效时只能进入持久退避，不得误判成功。"""
evidence = TransferStepResult(payload={"target_exists": False})
def execute() -> TransferStepResult:
"""模拟外部操作在应用前失败。"""
raise OSError("write rejected")
with pytest.raises(transfer_chain_module._TransferRetryDeferred):
_runner(execution_repository).run(
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
execute=execute,
observe=lambda: TransferOperationObservation(
state=TransferOperationObservationState.NOT_APPLIED,
evidence=evidence,
),
)
snapshot = execution_repository.get_snapshot(task_id="task-runner")
assert snapshot is not None
assert snapshot.state is TransferExecutionState.RETRY_WAIT
assert snapshot.steps[0].result == evidence
assert snapshot.steps[0].state.value == "failed"
@pytest.mark.parametrize(
"observation_state",
[
TransferOperationObservationState.UNKNOWN,
TransferOperationObservationState.CONFLICT,
],
)
def test_runner_freezes_uncertain_execute_error_for_manual_review(
execution_repository,
observation_state,
) -> None:
"""execute 异常后的未知或冲突结果必须冻结，禁止自动重放。"""
evidence = TransferStepResult(payload={"receipt": "ambiguous"})
def execute() -> TransferStepResult:
"""模拟外部结果未知的执行异常。"""
raise TimeoutError("provider timeout")
with pytest.raises(
transfer_chain_module._TransferManualReviewRequired,
match=observation_state.value,
):
_runner(execution_repository).run(
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
execute=execute,
observe=lambda: TransferOperationObservation(
state=observation_state,
evidence=evidence,
),
)
snapshot = execution_repository.get_snapshot(task_id="task-runner")
assert snapshot is not None
assert snapshot.state is TransferExecutionState.MANUAL_REVIEW
assert snapshot.steps[0].result == evidence
assert snapshot.steps[0].state.value == "manual_review"
def test_runner_freezes_when_observer_errors_after_execute_error(
execution_repository,
) -> None:
"""execute 与 observer 同时失败时必须保留双重证据并进入人工复核。"""
def execute() -> TransferStepResult:
"""模拟调用结果未知的执行超时。"""
raise TimeoutError("execute timeout")
def observe() -> TransferOperationObservation:
"""模拟外部状态查询端点同时不可用。"""
raise ConnectionError("observer unavailable")
with pytest.raises(
transfer_chain_module._TransferManualReviewRequired,
match="unknown",
):
_runner(execution_repository).run(
phase="transfer",
kind="materialize_target",
payload=_runner_step_payload(),
execute=execute,
observe=observe,
)
snapshot = execution_repository.get_snapshot(task_id="task-runner")
assert snapshot is not None
assert snapshot.state is TransferExecutionState.MANUAL_REVIEW
assert snapshot.steps[0].result is not None
assert snapshot.steps[0].result.payload == {
"execute_error": "execute timeout",
"observe_error": "observer unavailable",
}
class _ImmediateStepRunner:
"""记录 TransHandler 拆分顺序并立即执行步骤的测试 runner。"""
tests/test_transfer_failed_retry_scheduler.py
-126
View File
@@ -1,126 +0,0 @@
"""失败整理 AI 重试调度器的生命周期测试。"""
import asyncio
from unittest.mock import Mock, patch
import pytest
from app.application.transfer.workflow import FailedRetryScheduler
def test_retry_scheduler_close_cancels_buffered_timer_and_rejects_new_work():
"""关闭应取消尚未触发的 timer、清空缓冲并拒绝新增记录。"""
async def exercise() -> None:
"""在独立事件循环内验证 timer 与关闭状态。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 60
await scheduler.schedule_retry(11, group_key="media:test")
timer = scheduler._retry_transfer_timers["media:test"]
await scheduler.close()
await scheduler.close()
assert timer.cancelled()
assert scheduler._retry_transfer_buffer == {}
assert scheduler._retry_transfer_timers == {}
with pytest.raises(RuntimeError, match="正在关闭"):
await scheduler.schedule_retry(12, group_key="media:test")
asyncio.run(exercise())
def test_retry_scheduler_close_cancels_and_waits_for_active_flush_task():
"""关闭返回前应等待已经启动的 flush 任务完成取消收尾。"""
async def exercise() -> None:
"""启动一个不会自行结束的 flush，并通过关闭流程取消它。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
started = asyncio.Event()
stopped = asyncio.Event()
async def blocking_flush(_group_key: str, _generation: int) -> None:
"""等待取消信号，并在 finally 中证明收尾已经完成。"""
started.set()
try:
await asyncio.Event().wait()
finally:
stopped.set()
scheduler._flush_retry_transfer = blocking_flush
await scheduler.schedule_retry(11, group_key="media:test")
await asyncio.wait_for(started.wait(), timeout=1)
task = next(iter(scheduler._retry_transfer_tasks))
await scheduler.close()
assert stopped.is_set()
assert task.cancelled()
assert task.get_name() == "transfer.failed_retry.flush"
assert scheduler._retry_transfer_tasks == set()
asyncio.run(exercise())
def test_retry_scheduler_observes_unexpected_background_task_error():
"""flush 协程越过自身防线的异常仍应由任务 owner 统一观察。"""
async def exercise() -> None:
"""让受管 flush 任务直接失败，并等待完成回调处理异常。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 0
async def failing_flush(_group_key: str, _generation: int) -> None:
"""模拟 flush 外层出现未处理异常。"""
raise RuntimeError("flush failed")
scheduler._flush_retry_transfer = failing_flush
with patch("app.application.transfer.workflow.logger.error", Mock()) as log_error:
await scheduler.schedule_retry(11, group_key="media:test")
for _ in range(5):
await asyncio.sleep(0)
if scheduler._retry_transfer_tasks:
break
assert scheduler._retry_transfer_tasks
for _ in range(5):
await asyncio.sleep(0)
if not scheduler._retry_transfer_tasks:
break
assert scheduler._retry_transfer_tasks == set()
log_error.assert_called_once()
assert "flush failed" in log_error.call_args.args[0]
await scheduler.close()
asyncio.run(exercise())
def test_retry_scheduler_old_flush_cannot_consume_renewed_generation():
"""旧 timer 已建 task 后的新失败应续期，不能被旧 flush 提前取走。"""
async def exercise() -> None:
"""稳定复现 timer callback 与同组新 schedule 交错的窗口。"""
scheduler = FailedRetryScheduler()
scheduler.RETRY_TRANSFER_DEBOUNCE_SECONDS = 3600
await scheduler.schedule_retry(11, group_key="media:test")
old_timer = scheduler._retry_transfer_timers["media:test"]
old_generation = scheduler._retry_transfer_generations["media:test"]
# 模拟旧 timer callback 已进入事件循环，但 flush task 尚未取得分组锁。
scheduler._start_retry_transfer_task("media:test", old_generation)
await scheduler.schedule_retry(12, group_key="media:test")
renewed_timer = scheduler._retry_transfer_timers["media:test"]
await asyncio.sleep(0)
await asyncio.sleep(0)
assert old_timer.cancelled()
assert renewed_timer.cancelled() is False
assert scheduler._retry_transfer_buffer["media:test"] == [11, 12]
assert scheduler._retry_transfer_timers["media:test"] is renewed_timer
assert scheduler._retry_transfer_tasks == set()
await scheduler.close()
assert renewed_timer.cancelled()
asyncio.run(exercise())
tests/test_transfer_job_manager.py
+94 -40
View File
@@ -9,6 +9,10 @@ from app.application.history import (
failed_retry_count,
record_transfer_failure,
)
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferSettlementResult,
)
from app.application.transfer.workflow import (
TransferAdmission,
TransferPlanningInput,
@@ -223,13 +227,80 @@ def make_transfer_chain() -> TransferChain:
admissions.admit.side_effect = admit
admissions.claim_task.side_effect = claim_task
admissions.checkpoint_plan.side_effect = checkpoint_plan
admissions.discard_claimed.return_value = 1
admissions.abandon_unstarted.return_value = 1
admissions.release_claim.return_value = True
chain._transfer_admissions = admissions
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
class ImmediateStepRunner:
"""为 JobManager 测试提交纯内存步骤与聚合执行检查点。"""
def run(self, *, phase, kind, payload, execute, observe):
"""立即执行确定性测试步骤，不使用恢复探测。"""
del phase, kind, payload, observe
return execute()
def checkpoint(self, transferinfo):
"""把测试整理结果冻结为可供 task-aware writer 使用的检查点。"""
return TransferExecutionCheckpoint.create(
payload={
"outcome": "succeeded" if transferinfo.success else "failed",
"transferinfo": transferinfo.model_dump(mode="json"),
},
operation_ids=("job-test-operation",),
)
step_runner = ImmediateStepRunner()
chain._TransferChain__build_durable_step_runner = MagicMock(
return_value=step_runner
)
def transfer_result(**kwargs):
"""执行测试历史暂存与发布，并返回已删除 pending 的原子回执。"""
staging = SimpleNamespace(
get_success_by_src=lambda *_args, **_kwargs: SimpleNamespace(
id=99,
status=True,
)
)
history = kwargs["stage_history"](staging)
if kwargs["publish"] is not None:
kwargs["publish"](kwargs["event_payload"])
return TransferSettlementResult(
history_id=getattr(history, "id", 1) if history is not None else 1,
settlement_revision=1,
pending_deleted=True,
)
chain.durable_event_writer = MagicMock()
chain.durable_event_writer.transfer_result.side_effect = transfer_result
return chain
def bind_terminal_checkpoint(
task: TransferTask,
transferinfo: TransferInfo,
) -> None:
"""为直接回调测试绑定 task identity、lease 与聚合执行检查点。"""
task_id = f"terminal-{abs(hash(task.fileitem.path))}"
task.bind_admission_task_id(task_id)
task.bind_execution_lease(
owner_id="job-test-owner",
lease_token=f"lease-{task_id}",
)
task.bind_execution_checkpoint(TransferExecutionCheckpoint.create(
payload={
"outcome": (
"overwrite_skipped"
if transferinfo.overwrite_skipped
else "succeeded" if transferinfo.success else "failed"
),
"transferinfo": transferinfo.model_dump(mode="json"),
},
operation_ids=("job-test-operation",),
))
def make_fileitem(path: str, size: int = 1024) -> FileItem:
file_path = path
name = file_path.rsplit("/", 1)[-1]
@@ -545,6 +616,7 @@ class TransferJobManagerTest(unittest.TestCase):
need_scrape=True,
need_notify=False,
)
bind_terminal_checkpoint(task, transferinfo)
with patch(
"app.chain.transfer.get_chain_transfer_history_port", return_value=SimpleNamespace()
@@ -969,6 +1041,7 @@ class TransferJobManagerTest(unittest.TestCase):
transfer_type="copy",
need_notify=False,
)
bind_terminal_checkpoint(task, failed_transferinfo)
failed_history_oper = SimpleNamespace()
with patch(
"app.chain.transfer.get_chain_transfer_history_port",
@@ -1010,6 +1083,7 @@ class TransferJobManagerTest(unittest.TestCase):
need_scrape=False,
need_notify=False,
)
bind_terminal_checkpoint(task, success_transferinfo)
with patch(
"app.chain.transfer.get_chain_transfer_history_port", return_value=SimpleNamespace()
), patch(
@@ -1023,7 +1097,8 @@ class TransferJobManagerTest(unittest.TestCase):
finally:
_reset_failed_retries(src_path, storage)
def test_unrecognized_task_marks_downloader_hash_completed(self):
def test_unrecognized_task_waits_for_durable_settlement_before_completion(self):
"""拒绝检查点建立后、writer 结算前不得提前完成下载种子或移除作业。"""
chain = make_transfer_chain()
chain.post_message = lambda *_args, **_kwargs: None
completed = []
@@ -1058,15 +1133,14 @@ class TransferJobManagerTest(unittest.TestCase):
self.assertFalse(state)
self.assertEqual("未识别到媒体信息", errmsg)
self.assertEqual([("abc123", "qbittorrent")], completed)
self.assertEqual([], chain.jobview.list_jobs())
self.assertEqual([], completed)
self.assertIsNotNone(task.plan_checkpoint)
self.assertIsNotNone(task.execution_checkpoint)
self.assertEqual(1, len(chain.jobview.list_jobs()))
chain.durable_event_writer.transfer_result.assert_not_called()
def test_unrecognized_task_survives_missing_failure_history(self):
"""
写整理历史失败（``add_transfer_fail`` 返回 None）时，未识别分支仍须走完
通知、作业清理与种子完成标记：历史落库是通知的附属信息，不是前置条件。
通知正文只省去 ``/redo`` 指引，不得因读取 ``his.id`` 抛 NoneType。
"""
def test_unrecognized_task_does_not_read_history_before_writer(self):
"""拒绝步骤完成但 writer 未调用时，不得读取失败历史或发送通知。"""
chain = make_transfer_chain()
notifications = []
chain.post_message = lambda message, **_kwargs: notifications.append(message)
@@ -1100,21 +1174,13 @@ class TransferJobManagerTest(unittest.TestCase):
self.assertFalse(state)
self.assertEqual("未识别到媒体信息", errmsg)
# 种子完成标记与作业清理都排在通知之后，通知崩掉会把它们一并跳过
self.assertEqual([("abc123", "qbittorrent")], completed)
self.assertEqual([], chain.jobview.list_jobs())
# 通知照发，但不含无法使用的 /redo 指引
self.assertEqual(1, len(notifications))
notification = notifications[0]
self.assertIn("未识别到媒体信息", notification.text)
self.assertNotIn("/redo", notification.text)
self.assertIsNone(notification.buttons)
self.assertEqual([], completed)
self.assertEqual([], notifications)
self.assertEqual(1, len(chain.jobview.list_jobs()))
chain.durable_event_writer.transfer_result.assert_not_called()
def test_unrecognized_task_keeps_redo_hint_when_history_written(self):
"""
整理历史正常落库时，未识别通知须保留两条 ``/redo`` 指引与操作按钮，
防止上一条用例被「一律删掉 /redo」这种偷懒实现蒙混过关。
"""
def test_unrecognized_task_does_not_publish_redo_before_writer(self):
"""即使历史函数可用，未经过 task-aware writer 也不得发布 redo。"""
chain = make_transfer_chain()
notifications = []
chain.post_message = lambda message, **_kwargs: notifications.append(message)
@@ -1141,22 +1207,8 @@ class TransferJobManagerTest(unittest.TestCase):
media_chain_cls.return_value.recognize_by_meta.return_value = None
chain._TransferChain__handle_transfer(task)
self.assertEqual(1, len(notifications))
notification = notifications[0]
self.assertIn("/redo 77\n", notification.text)
self.assertIn("/redo 77 [media_source]|[media_id]|[类型]", notification.text)
self.assertEqual(
[
[
{"text": "重试", "callback_data": "transfer_retry_77"},
{
"text": "智能助手接管",
"callback_data": "transfer_ai_retry_77",
},
]
],
notification.buttons,
)
self.assertEqual([], notifications)
chain.durable_event_writer.transfer_result.assert_not_called()
def test_do_transfer_syncs_same_stem_extra_files_by_default(self):
chain = make_transfer_chain()
@@ -1594,6 +1646,7 @@ class TransferJobManagerTest(unittest.TestCase):
) as storage_chain_cls:
storage_chain_cls.return_value.is_bluray_folder.return_value = False
for task, transferinfo in zip(tasks, transferinfos):
bind_terminal_checkpoint(task, transferinfo)
chain._TransferChain__default_callback(task, transferinfo)
chain._finish_scrape_batch_task(task)
@@ -1649,6 +1702,7 @@ class TransferJobManagerTest(unittest.TestCase):
need_scrape=True,
need_notify=False,
)
bind_terminal_checkpoint(task, transferinfo)
with patch(
"app.chain.transfer.get_chain_transfer_history_port", return_value=SimpleNamespace()
tests/test_transfer_lease_migration.py
+27
View File
@@ -185,6 +185,33 @@ def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None:
engine.dispose()
def test_replayed_upgrade_repairs_named_lease_index(monkeypatch) -> None:
"""同名但列和唯一性错误的租约索引必须精确重建。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_planning_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
connection.execute(sa.text(
"DROP INDEX ix_transferpending_recovery_lease"
))
connection.execute(sa.text(
"CREATE UNIQUE INDEX ix_transferpending_recovery_lease "
"ON transferpending (task_id)"
))
migration.upgrade()
index = next(
item for item in sa.inspect(connection).get_indexes("transferpending")
if item["name"] == "ix_transferpending_recovery_lease"
)
assert index["column_names"] == [
"state", "lease_expires_at", "created_at", "id",
]
assert index["unique"] == 0
def test_partial_transfer_lease_upgrade_preserves_existing_owner(
monkeypatch,
) -> None:
tests/test_transfer_lease_persistence.py
+5 -3
View File
@@ -20,6 +20,7 @@ from app.application.transfer.workflow import (
TransferProviderReference,
)
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.models.transferexecutionstep import TransferExecutionStep
from app.db.models.transferhistory import TransferHistory
from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
@@ -93,6 +94,7 @@ def repository_factory(tmp_path):
)
TransferHistory.__table__.create(engine)
TransferPending.__table__.create(engine)
TransferExecutionStep.__table__.create(engine)
factory = sessionmaker(bind=engine)
yield lambda: TransactionalTransferAdmissionRepository(factory)
engine.dispose()
@@ -207,7 +209,7 @@ def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
lease_token=first.lease_token,
error="expired worker",
) is False
assert repository.discard_claimed(
assert repository.abandon_unstarted(
task_id=admitted.task_id,
lease_token=first.lease_token,
) == 0
@@ -231,7 +233,7 @@ def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
lease_token=first.lease_token,
error="stale worker",
) is False
assert repository.discard_claimed(
assert repository.abandon_unstarted(
task_id=admitted.task_id,
lease_token=first.lease_token,
) == 0
@@ -257,7 +259,7 @@ def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
)
assert third is not None
assert third.attempt_count == 3
assert repository.discard_claimed(
assert repository.abandon_unstarted(
task_id=admitted.task_id,
lease_token=third.lease_token,
) == 1
tests/test_transfer_legacy_terminal_compat.py
+31 -3
View File
@@ -76,9 +76,9 @@ def _compat_chain(result_factory):
task.bind_execution_checkpoint(TransferExecutionCheckpoint.create(
payload={
"outcome": (
"succeeded"
if result.success
else "failed"
"overwrite_skipped"
if result.overwrite_skipped
else "succeeded" if result.success else "failed"
),
"transferinfo": result.model_dump(mode="json"),
},
@@ -192,6 +192,34 @@ def test_legacy_settlement_response_loss_replays_receipt_by_same_task_id() -> No
assert second["settlement"] == first["settlement"]
def test_legacy_settlement_double_failure_releases_claim_without_deleting_evidence(
) -> None:
"""两次 writer 均失败时必须释放 lease，并保留 pending 与步骤供恢复。"""
chain, executed = _compat_chain(lambda task: _result(task, success=True))
chain._transfer_admissions = Mock()
chain._transfer_admissions.release_claim.return_value = True
chain._TransferChain__ensure_recovery_scheduler = Mock()
chain.durable_event_writer.transfer_result.side_effect = RuntimeError(
"writer unavailable"
)
returned = _invoke(chain, _fileitem())
assert returned.success is False
assert "writer unavailable" in (returned.message or "")
assert executed == ["source-v1"]
assert chain.durable_event_writer.transfer_result.call_count == 2
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="task-source-v1",
lease_token="lease-source-v1",
error=(
"旧整理兼容命令 durable 终态结算失败：writer unavailable"
),
)
chain._transfer_admissions.abandon_unstarted.assert_not_called()
assert chain._owned_leases == {}
def test_legacy_overwrite_skip_binds_existing_success_in_atomic_writer() -> None:
"""覆盖跳过复用既有成功历史并以 succeeded 终态结算。"""
chain, executed = _compat_chain(
tests/test_transfer_manual_review_discovery.py
+43 -8
View File
@@ -15,6 +15,11 @@ from app.application.transfer.execution import (
TransferStepIntent,
TransferStepResult,
)
from app.application.transfer.workflow import (
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
)
from app.db.adapters.transfer.execution import (
TransactionalTransferExecutionRepository,
)
@@ -58,19 +63,44 @@ def _put_in_manual_review(factory, *, task_id: str) -> tuple[
str,
]:
"""建立一个外部结果 UNKNOWN 且已释放租约的人工复核任务。"""
source_path = f"/downloads/{task_id}.mkv"
target_path = f"/library/{task_id}.mkv"
planning_input = TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": source_path,
"type": "file",
},
target_storage="local",
target_path=target_path,
requested_transfer_type="copy",
)
checkpoint = TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/library",
final_target_path=target_path,
resolved_transfer_type="copy",
items=(TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path=target_path,
),),
)
with factory() as session:
session.add(TransferPending(
task_id=task_id,
storage="local",
src_path=f"/downloads/{task_id}.mkv",
src_path=source_path,
created_at="2026-08-27 09:00:00",
state="planned",
updated_at="2026-08-27 09:00:00",
input_version=1,
planning_input={"schema_version": 1, "source": task_id},
input_fingerprint=f"input-{task_id}",
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
checkpoint_version=1,
checkpoint_payload={"schema_version": 1, "task_id": task_id},
checkpoint_payload=checkpoint.to_payload(),
planned_at="2026-08-27 09:00:00",
lease_owner="worker-secret",
lease_token=f"lease-{task_id}",
@@ -90,13 +120,15 @@ def _put_in_manual_review(factory, *, task_id: str) -> tuple[
)
intent = TransferStepIntent.create(
task_id=task_id,
checkpoint_fingerprint=f"checkpoint-{task_id}",
checkpoint_fingerprint=checkpoint.fingerprint,
ordinal=0,
phase="transfer",
kind="materialize_target",
payload={
"source": f"/downloads/{task_id}.mkv",
"target": f"/library/{task_id}.mkv",
"source": planning_input.source_fileitem,
"target_storage": "local",
"target_path": target_path,
"transfer_type": "copy",
},
)
prepared = command.prepare(
@@ -189,7 +221,10 @@ def test_unknown_manual_review_is_discoverable_and_resumes_via_api(
}
assert discovered.step.operation_id == operation_id
assert discovered.step.kind == "materialize_target"
assert discovered.step.intent["target"] == f"/library/task-{decision}.mkv"
assert (
discovered.step.intent["target_path"]
== f"/library/task-{decision}.mkv"
)
assert discovered.step.evidence == {
"observation": "unknown",
"target_exists": True,
tests/test_transfer_overwrite_declined.py
+39 -8
View File
@@ -15,11 +15,12 @@ from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferSettlementResult,
)
from app.chain.transfer import TransferChain
from app.chain.transfer import TransferChain, _DurableTransferStepRunner
from app.schemas.transfer import TransferInfo
from app.schemas.types import EventType
from tests.test_transfer_job_manager import (
FakeMedia,
bind_terminal_checkpoint,
make_fileitem,
make_task,
make_transfer_chain,
@@ -130,16 +131,19 @@ def test_overwrite_declined_uses_successful_durable_settlement():
task = make_task(1)
task.bind_admission_task_id("task-overwrite-declined")
task.bind_execution_lease(owner_id="worker", lease_token="lease")
task.bind_execution_checkpoint(TransferExecutionCheckpoint.create(
payload={"outcome": "overwrite_skipped"},
operation_ids=(),
skip_reason="overwrite_declined",
))
transferinfo = TransferInfo(
success=False,
overwrite_skipped=True,
message="目标已存在，按覆盖策略跳过覆盖",
)
task.bind_execution_checkpoint(TransferExecutionCheckpoint.create(
payload={
"outcome": "overwrite_skipped",
"transferinfo": transferinfo.model_dump(mode="json"),
},
operation_ids=(),
skip_reason="overwrite_declined",
))
settlement = TransferChain._TransferChain__build_transfer_result_settlement(
task,
@@ -152,6 +156,27 @@ def test_overwrite_declined_uses_successful_durable_settlement():
assert settlement.error is None
def test_durable_step_runner_records_overwrite_skip_as_explicit_outcome():
"""生产 runner 必须冻结覆盖跳过事实，不能提前把它记成普通失败。"""
runner = object.__new__(_DurableTransferStepRunner)
runner._task_id = "task-overwrite-skipped"
runner._lease_token = "lease"
runner._operation_ids = []
runner._command = MagicMock()
runner._command.checkpoint.side_effect = lambda **kwargs: SimpleNamespace(
checkpoint=kwargs["checkpoint"]
)
transferinfo = TransferInfo(
success=False,
overwrite_skipped=True,
message="目标已存在，按覆盖策略跳过覆盖",
)
checkpoint = runner.checkpoint(transferinfo)
assert checkpoint.payload["outcome"] == "overwrite_skipped"
def test_overwrite_skip_without_success_history_uses_failed_settlement():
"""未核实既有成功历史时，覆盖跳过标志不能伪造成功终态。"""
task = make_task(1)
@@ -216,6 +241,7 @@ def test_default_callback_skips_history_and_notification_when_overwrite_declined
overwrite_skipped=True,
need_notify=False,
)
bind_terminal_checkpoint(task, transferinfo)
with patch(
"app.chain.transfer.get_chain_transfer_history_port",
@@ -262,6 +288,7 @@ def test_default_callback_keeps_original_failure_semantics_without_success_histo
overwrite_skipped=True,
need_notify=False,
)
bind_terminal_checkpoint(task, transferinfo)
with patch(
"app.chain.transfer.get_chain_transfer_history_port",
@@ -312,7 +339,6 @@ def test_durable_callback_settles_overwrite_skip_without_history_as_failed():
overwrite_skipped=True,
need_notify=False,
)
def durable_transfer_result(**kwargs):
"""执行失败历史暂存并返回 task-aware 结算回执。"""
history = kwargs["stage_history"](SimpleNamespace())
@@ -363,6 +389,7 @@ def test_default_callback_delegates_primary_failure_to_durable_writer():
transfer_type="copy",
need_notify=False,
)
bind_terminal_checkpoint(task, transferinfo)
def durable_transfer_result(**kwargs):
"""执行 writer 收到的历史暂存与提交后发布回调。"""
@@ -371,7 +398,11 @@ def test_default_callback_delegates_primary_failure_to_durable_writer():
payload["transfer_history_id"] = history.id
payload["idempotency_key"] = f"transfer.failed:{history.id}:v1"
kwargs["publish"](payload)
return history
return TransferSettlementResult(
history_id=history.id,
settlement_revision=1,
pending_deleted=True,
)
chain.durable_event_writer.transfer_result.side_effect = durable_transfer_result
with patch(
tests/test_transfer_pending_initial_migration.py
+97
View File
@@ -0,0 +1,97 @@
"""整理待处理表 3.0.4 初始迁移的中断恢复测试。"""
import importlib
import pytest
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
MIGRATION = "database.versions.e3d9f4b7c806_3_0_4"
def _bind_migration(monkeypatch, connection):
"""把 3.0.4 迁移绑定到隔离 SQLite 连接。"""
migration = importlib.import_module(MIGRATION)
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
return migration
def test_upgrade_repairs_missing_and_malformed_identity_index(monkeypatch) -> None:
"""建表后中断或同名错误索引都必须收敛为精确唯一索引。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
connection.execute(sa.text(
"CREATE TABLE transferpending ("
"id INTEGER PRIMARY KEY, storage VARCHAR NOT NULL, "
"src_path VARCHAR NOT NULL, created_at VARCHAR)"
))
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
connection.execute(sa.text(
"DROP INDEX ux_transferpending_storage_path"
))
connection.execute(sa.text(
"CREATE INDEX ux_transferpending_storage_path "
"ON transferpending (src_path)"
))
migration.upgrade()
index = sa.inspect(connection).get_indexes("transferpending")[0]
assert index["name"] == "ux_transferpending_storage_path"
assert index["column_names"] == ["storage", "src_path"]
assert index["unique"] == 1
def test_upgrade_recreates_empty_partial_table(monkeypatch) -> None:
"""中断留下的空残表可以无损重建为完整 3.0.4 结构。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
connection.execute(sa.text(
"CREATE TABLE transferpending (id INTEGER PRIMARY KEY)"
))
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
assert {
column["name"]
for column in sa.inspect(connection).get_columns("transferpending")
} == {"id", "storage", "src_path", "created_at"}
def test_upgrade_rejects_nonempty_partial_table(monkeypatch) -> None:
"""含数据残表无法可靠推断源身份时必须显式拒绝迁移。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
connection.execute(sa.text(
"CREATE TABLE transferpending (id INTEGER PRIMARY KEY)"
))
connection.execute(sa.text(
"INSERT INTO transferpending (id) VALUES (1)"
))
migration = _bind_migration(monkeypatch, connection)
with pytest.raises(RuntimeError, match="含数据的不完整 transferpending"):
migration.upgrade()
def test_downgrade_tolerates_interrupted_missing_index(monkeypatch) -> None:
"""索引创建前中断时降级仍应安全删除残留表。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
connection.execute(sa.text(
"CREATE TABLE transferpending ("
"id INTEGER PRIMARY KEY, storage VARCHAR NOT NULL, "
"src_path VARCHAR NOT NULL, created_at VARCHAR)"
))
migration = _bind_migration(monkeypatch, connection)
migration.downgrade()
assert "transferpending" not in sa.inspect(connection).get_table_names()
tests/test_transfer_pending_replay.py
+146 -13
View File
@@ -11,9 +11,20 @@ import threading
import time
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
from app.application.transfer.workflow import TransferAdmission, TransferPlanningInput, TransferTask
import pytest
from app.application.transfer.execution import (
TransferExecutionSnapshot,
TransferExecutionState,
)
from app.application.transfer.workflow import (
TransferAdmission,
TransferPlanningInput,
TransferTask,
)
from app.chain.transfer import TransferChain
from app.schemas.file import FileItem
@@ -26,6 +37,10 @@ def _build_chain(admissions) -> TransferChain:
"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = admissions
chain._transfer_executions = MagicMock()
chain._transfer_executions.get_snapshot.side_effect = (
lambda *, task_id: _execution_snapshot(task_id=task_id)
)
chain._worker_owner_id = "test-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
@@ -40,8 +55,34 @@ def _build_chain(admissions) -> TransferChain:
return chain
def _execution_snapshot(
*,
task_id: str = "task-1",
state: TransferExecutionState = TransferExecutionState.NOT_STARTED,
steps: tuple[object, ...] = (),
) -> TransferExecutionSnapshot:
"""构造回放判定所需的最小执行状态投影。"""
return TransferExecutionSnapshot(
task_id=task_id,
state=state,
checkpoint=None,
retry_generation=0,
retry_count=0,
retry_due_at=None,
settlement_revision=0,
terminal_history_id=None,
last_error=None,
steps=steps,
)
def _admission(path: str, task_id: str = "task-1") -> TransferAdmission:
"""构造一条可脱离数据库会话使用的准入快照。"""
planning_input = TransferPlanningInput(
source_fileitem=_task(path).fileitem.model_dump(mode="json"),
meta=None,
mediainfo=None,
)
return TransferAdmission(
task_id=task_id,
storage="local",
@@ -49,6 +90,7 @@ def _admission(path: str, task_id: str = "task-1") -> TransferAdmission:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=planning_input,
lease_owner="test-owner",
lease_token=f"lease-{task_id}",
lease_expires_at="2026-08-27 10:02:00.000000",
@@ -97,12 +139,12 @@ def test_admit_transfer_records_storage_and_path():
assert result.task_id == "task-1"
def test_discard_pending_on_terminal_state():
"""
整理到达终态后必须注销登记，否则每次重启都会重复回放。
"""
def test_terminal_without_settlement_releases_claim_and_keeps_pending():
"""缺少原子终态回执时必须释放租约并保留 pending 供恢复。"""
admissions = MagicMock()
admissions.release_claim.return_value = True
chain = _build_chain(admissions)
chain.jobview = MagicMock()
task = _task("/mnt/cd2/downloads/Movie.2024.mkv")
task.bind_admission_task_id("task-1")
task.bind_execution_lease(owner_id="test-owner", lease_token="lease-task-1")
@@ -110,14 +152,18 @@ def test_discard_pending_on_terminal_state():
chain._owned_leases = {
"task-1": ("lease-task-1", time.monotonic() + 120)
}
admissions.discard_claimed.return_value = 1
assert chain._TransferChain__finish_job_execution(
task,
terminal=True,
terminal_settlement=None,
) is False
assert chain._TransferChain__discard_pending(task) is True
admissions.discard_claimed.assert_called_once_with(
admissions.release_claim.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
error="整理终态未完成 durable 原子结算",
)
admissions.abandon_unstarted.assert_not_called()
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
@@ -156,17 +202,104 @@ def test_replay_discards_vanished_files(tmp_path):
admissions = MagicMock()
missing = tmp_path / "gone.mkv"
admissions.claim_recoverable.return_value = [_admission(str(missing))]
admissions.discard_claimed.return_value = 1
admissions.abandon_unstarted.return_value = 1
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain._execute_transfer.assert_not_called()
admissions.discard_claimed.assert_called_once_with(
admissions.abandon_unstarted.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
)
admissions.release_claim.assert_not_called()
assert chain._owned_leases == {}
def test_replay_releases_claim_when_vanished_source_abandon_is_rejected(
tmp_path,
) -> None:
"""注销 CAS 被执行证据拒绝时必须释放 claim，不能留下无人续期的毒任务。"""
admissions = MagicMock()
missing = tmp_path / "state-changed.mkv"
admission = _admission(str(missing))
admissions.claim_recoverable.return_value = [admission]
admissions.abandon_unstarted.return_value = 0
admissions.release_claim.return_value = True
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
admissions.abandon_unstarted.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
)
admissions.release_claim.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
error="源已消失但任务状态已变化，保留登记供恢复",
)
assert chain._owned_leases == {}
@pytest.mark.parametrize(
("execution_state", "steps"),
[
(TransferExecutionState.NOT_STARTED, ()),
(TransferExecutionState.RUNNING, ()),
(TransferExecutionState.RETRY_WAIT, ()),
(TransferExecutionState.NOT_STARTED, (SimpleNamespace(),)),
],
)
def test_replay_with_execution_evidence_uses_frozen_source_when_source_vanished(
tmp_path,
monkeypatch,
execution_state,
steps,
) -> None:
"""已有执行状态或步骤证据时不得用源消失推断任务可删除。"""
missing = tmp_path / "already-moved.mkv"
planning_input = TransferPlanningInput(
source_fileitem=_task(str(missing)).fileitem.model_dump(mode="json"),
meta=None,
mediainfo=None,
requested_transfer_type="move",
)
admission = replace(
_admission(str(missing)),
state="planned",
planning_input=planning_input,
checkpoint=MagicMock(),
)
admissions = MagicMock()
admissions.claim_recoverable.return_value = [admission]
chain = _build_chain(admissions)
chain._transfer_executions.get_snapshot.return_value = _execution_snapshot(
state=execution_state,
steps=steps,
)
chain._transfer_executions.get_snapshot.side_effect = None
queue_planned = MagicMock(return_value=True)
monkeypatch.setattr(
chain,
"_TransferChain__queue_planned_replay",
queue_planned,
)
def reject_stat(*_args, **_kwargs):
"""冻结恢复触碰源文件即判定测试失败。"""
pytest.fail("已有执行证据的恢复不得探测已经消失的源文件")
monkeypatch.setattr(Path, "stat", reject_stat)
chain._TransferChain__replay_pending()
queued_fileitem = queue_planned.call_args.args[0]
assert queued_fileitem.path == str(missing)
admissions.abandon_unstarted.assert_not_called()
admissions.release_claim.assert_not_called()
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
@@ -195,7 +328,7 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
chain._TransferChain__replay_pending()
chain._execute_transfer.assert_not_called()
admissions.discard_claimed.assert_not_called()
admissions.abandon_unstarted.assert_not_called()
admissions.release_claim.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
@@ -316,7 +449,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
chain._TransferChain__replay_pending(stop_event)
assert transferred == [first.as_posix()]
admissions.discard_claimed.assert_not_called()
admissions.abandon_unstarted.assert_not_called()
assert admissions.release_claim.call_count == 2
tests/test_transfer_planning_checkpoint.py
+283 -28
View File
@@ -15,7 +15,11 @@ from sqlalchemy.orm import sessionmaker
from app.application.transfer import workflow as transfer_application
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferExecutionSnapshot,
TransferExecutionState,
TransferExecutionStep,
TransferSettlementResult,
TransferStepState,
)
from app.application.transfer.workflow import TransferTask
from app.chain.transfer import TransferChain
@@ -27,7 +31,6 @@ from app.domain.meta.metabase import MetaBase
from app.modules.filemanager.module import FileManagerModule
from app.modules.filemanager.transhandler import TransHandler
from app.runtime.extensions.module.dispatcher import (
FrozenModuleProviderMissingError,
ModuleInvocationDispatcher,
)
from app.schemas.exception import StorageQueryError
@@ -204,10 +207,136 @@ def _planned_admission(task: TransferTask, checkpoint):
)
class _ExecutionRepositoryStub:
"""为规划编排测试提供严格但内存化的 execution repository。"""
def __init__(self) -> None:
"""初始化空步骤集合与未启动执行态。"""
self.steps = {}
self.state = TransferExecutionState.NOT_STARTED
self.checkpoint = None
def get_snapshot(self, *, task_id):
"""返回当前任务的类型化执行投影。"""
return TransferExecutionSnapshot(
task_id=task_id,
state=self.state,
checkpoint=self.checkpoint,
retry_generation=0,
retry_count=0,
retry_due_at=None,
settlement_revision=0,
terminal_history_id=None,
last_error=None,
steps=tuple(self.steps.values()),
)
def prepare_step(self, *, task_id, lease_token, intent):
"""幂等保存准备态步骤。"""
del lease_token
existing = self.steps.get(intent.operation_id)
if existing is not None:
return existing
step = TransferExecutionStep(
task_id=task_id,
operation_id=intent.operation_id,
checkpoint_fingerprint=intent.checkpoint_fingerprint,
ordinal=intent.ordinal,
phase=intent.phase,
kind=intent.kind,
state=TransferStepState.PREPARED,
attempt_token=None,
attempt_count=0,
intent=intent,
result=None,
last_error=None,
prepared_at="2026-08-27 10:00:00",
started_at=None,
completed_at=None,
updated_at="2026-08-27 10:00:00",
)
self.steps[intent.operation_id] = step
return step
def start_step(
self,
*,
task_id,
lease_token,
operation_id,
attempt_token,
):
"""把准备态步骤推进到已开始。"""
del task_id, lease_token
step = replace(
self.steps[operation_id],
state=TransferStepState.STARTED,
attempt_token=attempt_token,
attempt_count=1,
started_at="2026-08-27 10:00:01",
)
self.steps[operation_id] = step
self.state = TransferExecutionState.RUNNING
return step
def complete_step(
self,
*,
task_id,
lease_token,
operation_id,
attempt_token,
result,
):
"""以当前 attempt 提交成功证据。"""
del task_id, lease_token
assert self.steps[operation_id].attempt_token == attempt_token
step = replace(
self.steps[operation_id],
state=TransferStepState.SUCCEEDED,
result=result,
completed_at="2026-08-27 10:00:02",
)
self.steps[operation_id] = step
return step
def checkpoint_execution(self, *, task_id, lease_token, checkpoint):
"""保存可重放终态的聚合检查点。"""
del lease_token
self.state = TransferExecutionState.SETTLING
self.checkpoint = checkpoint
return self.get_snapshot(task_id=task_id)
def mark_manual_review(
self,
*,
task_id,
lease_token,
operation_id,
attempt_token,
error,
evidence,
):
"""把执行结果不确定的步骤隔离到人工复核态。"""
del lease_token
step = self.steps[operation_id]
assert step.attempt_token == attempt_token
self.steps[operation_id] = replace(
step,
state=TransferStepState.MANUAL_REVIEW,
result=evidence,
last_error=error,
)
self.state = TransferExecutionState.MANUAL_REVIEW
return self.get_snapshot(task_id=task_id)
def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
"""构造只保留规划编排依赖的 TransferChain 骨架。"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = repository or Mock()
chain._transfer_executions = _ExecutionRepositoryStub()
chain.durable_event_writer = Mock()
chain._worker_owner_id = "planning-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
@@ -225,6 +354,7 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(),
lease_owner=kwargs["owner_id"],
lease_token=f"lease-{kwargs['task_id']}",
lease_expires_at="2026-08-27 10:02:00.000000",
@@ -245,6 +375,14 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
transfer_type="copy",
)
)
def run_module(method, *args, **kwargs):
"""让规划测试沿正式模块入口调用其可观察的宿主执行替身。"""
assert method == "execute_transfer_plan"
checkpoint_arg = kwargs.pop("checkpoint")
return chain.execute_transfer_plan(checkpoint_arg, *args, **kwargs)
chain.run_module = Mock(side_effect=run_module)
return chain
@@ -252,6 +390,21 @@ def _replay_chain(repository) -> TransferChain:
"""构造绑定固定恢复 owner 且不启动真实 heartbeat 线程的测试链。"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = repository
chain._transfer_executions = Mock()
chain._transfer_executions.get_snapshot.side_effect = (
lambda *, task_id: TransferExecutionSnapshot(
task_id=task_id,
state=TransferExecutionState.NOT_STARTED,
checkpoint=None,
retry_generation=0,
retry_count=0,
retry_due_at=None,
settlement_revision=0,
terminal_history_id=None,
last_error=None,
steps=(),
)
)
chain._worker_owner_id = "replay-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
@@ -275,6 +428,43 @@ def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher:
)
def test_non_preview_missing_durable_writer_stops_before_planning_or_execution():
"""缺少原子 writer 时，持久任务取得租约后也不得开始任何外部流程。"""
task = _task()
task.bind_admission_task_id("task-missing-writer")
_bind_planning_input(task, _planning_input())
chain = _chain()
chain.durable_event_writer = None
with pytest.raises(RuntimeError, match="缺少 durable 原子写入端口"):
chain._plan_checkpoint_and_execute(task)
chain._module_dispatcher.freeze_plugin_providers.assert_not_called()
chain.plan_transfer.assert_not_called()
chain.execute_transfer_plan.assert_not_called()
def test_non_preview_missing_execution_repository_stops_before_side_effects():
"""缺少 execution repository 时不得调用 provider 或文件执行器。"""
task = _task()
task.bind_admission_task_id("task-missing-execution-repository")
_bind_planning_input(task, _planning_input())
_bind_checkpoint(task, _checkpoint())
chain = _chain()
chain._transfer_executions = None
chain._TransferChain__restore_planned_task = Mock()
with pytest.raises(RuntimeError, match="缺少 execution repository"):
chain._plan_checkpoint_and_execute(
task,
source_oper=object(),
target_oper=object(),
)
chain._module_dispatcher.execute_frozen_plugin_providers.assert_not_called()
chain.execute_transfer_plan.assert_not_called()
def test_legacy_provider_runs_only_after_checkpoint_commit_and_short_circuits_host():
"""旧插件 provider 必须随计划冻结，并在 CAS 提交后才能接管执行。"""
calls = []
@@ -314,7 +504,7 @@ def test_legacy_provider_runs_only_after_checkpoint_commit_and_short_circuits_ho
returned = chain._plan_checkpoint_and_execute(task)
assert returned is plugin_result
assert returned == plugin_result
assert calls == ["checkpoint", "plugin"]
chain.plan_transfer.assert_not_called()
chain.execute_transfer_plan.assert_not_called()
@@ -548,8 +738,8 @@ def test_missing_frozen_provider_keeps_pending_and_skips_cleanup() -> None:
chain._transfer_storage_chain = Mock(return_value=storage_chain)
with pytest.raises(
FrozenModuleProviderMissingError,
match=r"ProviderTwo/插件二\.transfer",
RuntimeError,
match=r"禁止自动重放.*ProviderTwo/插件二\.transfer",
):
chain._plan_checkpoint_and_execute(task)
@@ -824,7 +1014,7 @@ def test_provider_pending_crash_replay_executes_snapshot_without_host_planning()
returned = recovered_chain._plan_checkpoint_and_execute(recovered_task)
assert returned is recovered_result
assert returned == recovered_result
recovered_chain._module_dispatcher.freeze_plugin_providers.assert_not_called()
recovered_chain.plan_transfer.assert_not_called()
recovered_chain._transfer_admissions.checkpoint_plan.assert_not_called()
@@ -905,7 +1095,7 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
assert returned is result
assert calls == ["admit", "checkpoint", "execute", "settle"]
repository.discard_claimed.assert_not_called()
repository.abandon_unstarted.assert_not_called()
writer_call = chain.durable_event_writer.transfer_result.call_args.kwargs
assert writer_call["topic"] is None
assert writer_call["publish"] is None
@@ -1561,31 +1751,22 @@ def test_filemanager_resolves_drifted_target_from_checkpoint(monkeypatch):
def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch):
"""准入后、checkpoint 前的业务失败必须写 last_error 并保持 accepted。"""
"""未识别拒绝必须先建立 plan/execution checkpoint，且不提前写终态副作用。"""
task = _task()
task.meta = MetaBase("Unrecognized.Movie.2026.mkv")
task.bind_admission_task_id("task-before-checkpoint")
task.bind_execution_lease(
owner_id="recognition-owner",
lease_token="lease-task-before-checkpoint",
)
chain = object.__new__(TransferChain)
chain._transfer_admissions = Mock()
chain = _chain()
chain._worker_owner_id = "recognition-owner"
chain._owned_leases = {
"task-before-checkpoint": (
"lease-task-before-checkpoint",
float("inf"),
)
}
chain._worker_state_lock = threading.RLock()
chain.jobview = Mock()
chain.queue_failed_transfer_notification = Mock()
chain.runtime_config = SimpleNamespace(
ai_agent_enable=False,
ai_agent_retry_transfer=False,
ai_agent_enable=True,
ai_agent_retry_transfer=True,
)
chain._TransferChain__mark_torrent_completed_if_done = Mock()
chain._transfer_admissions.checkpoint_plan.side_effect = (
lambda **kwargs: _planned_admission(task, kwargs["checkpoint"])
)
media_chain = Mock()
media_chain.recognize_by_meta.return_value = None
monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain)
@@ -1593,17 +1774,91 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
"app.chain.transfer.get_chain_transfer_history_port",
lambda: SimpleNamespace(),
)
monkeypatch.setattr("app.chain.transfer.record_transfer_failure", Mock())
monkeypatch.setattr("app.chain.transfer.add_transfer_fail", lambda **_kwargs: None)
record_transfer_failure = Mock()
add_transfer_fail = Mock()
monkeypatch.setattr(
"app.chain.transfer.record_transfer_failure",
record_transfer_failure,
)
monkeypatch.setattr("app.chain.transfer.add_transfer_fail", add_transfer_fail)
result = chain._TransferChain__handle_transfer(task)
assert result == (False, "未识别到媒体信息")
chain._transfer_admissions.record_planning_failure.assert_called_once_with(
task_id="task-before-checkpoint",
lease_token="lease-task-before-checkpoint",
error="未识别到媒体信息",
assert task.plan_checkpoint is not None
assert task.plan_checkpoint.rejection_error == "未识别到媒体信息"
assert task.plan_checkpoint.items == ()
assert task.execution_checkpoint is not None
assert task.execution_checkpoint.payload["outcome"] == "failed"
assert [step.kind for step in chain._transfer_executions.steps.values()] == [
"reject"
]
chain._transfer_admissions.record_planning_failure.assert_not_called()
record_transfer_failure.assert_not_called()
add_transfer_fail.assert_not_called()
chain.queue_failed_transfer_notification.assert_not_called()
chain._TransferChain__mark_torrent_completed_if_done.assert_not_called()
def test_recognition_rejection_without_writer_has_zero_terminal_side_effects(
monkeypatch,
) -> None:
"""缺 writer 时未识别拒绝不得提交计划、历史、通知或 AI 重试。"""
task = _task()
task.meta = MetaBase("Unrecognized.Movie.2026.mkv")
task.bind_admission_task_id("task-rejection-missing-writer")
chain = _chain()
chain.durable_event_writer = None
chain.jobview = Mock()
chain.queue_failed_transfer_notification = Mock()
chain._TransferChain__mark_torrent_completed_if_done = Mock()
chain.runtime_config = SimpleNamespace(
ai_agent_enable=True,
ai_agent_retry_transfer=True,
)
media_chain = Mock()
media_chain.recognize_by_meta.return_value = None
monkeypatch.setattr("app.chain.transfer.MediaChain", lambda: media_chain)
monkeypatch.setattr(
"app.chain.transfer.get_chain_transfer_history_port",
lambda: SimpleNamespace(),
)
record_transfer_failure = Mock()
add_transfer_fail = Mock()
monkeypatch.setattr(
"app.chain.transfer.record_transfer_failure",
record_transfer_failure,
)
monkeypatch.setattr("app.chain.transfer.add_transfer_fail", add_transfer_fail)
with pytest.raises(RuntimeError, match="缺少 durable 原子写入端口"):
chain._TransferChain__handle_transfer(task)
assert task.plan_checkpoint is None
assert task.execution_checkpoint is None
chain._transfer_admissions.checkpoint_plan.assert_not_called()
record_transfer_failure.assert_not_called()
add_transfer_fail.assert_not_called()
chain.queue_failed_transfer_notification.assert_not_called()
chain._TransferChain__mark_torrent_completed_if_done.assert_not_called()
def test_planning_rejection_checkpoint_round_trips_and_rejects_file_steps():
"""拒绝原因必须稳定序列化，且不能与真实文件步骤同时存在。"""
checkpoint = replace(
_checkpoint(),
items=(),
rejection_error="未识别到媒体信息",
)
restored = transfer_application.TransferPlanCheckpoint.from_payload(
checkpoint.to_payload()
)
assert restored == checkpoint
assert restored.rejection_error == "未识别到媒体信息"
with pytest.raises(ValueError, match="不得包含文件步骤"):
replace(checkpoint, items=_checkpoint().items)
def test_preview_plans_without_persistence_or_file_side_effects():
tests/test_transfer_planning_migration.py
+57 -4
View File
@@ -132,10 +132,11 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
if isinstance(planning_payload, str):
planning_payload = json.loads(planning_payload)
planning_input = TransferPlanningInput.from_payload(planning_payload)
assert planning_input == TransferPlanningInput.legacy(
storage="local",
src_path="/downloads/Movie.mkv",
)
assert planning_input.source_fileitem == {
"storage": "local",
"path": "/downloads/Movie.mkv",
}
assert planning_input.options == {"legacy_replan": True}
assert upgraded["input_version"] == 1
assert upgraded["input_fingerprint"] == planning_input.fingerprint
assert upgraded["checkpoint_payload"] is None
@@ -282,6 +283,58 @@ def test_partial_upgrade_preserves_existing_planning_json(monkeypatch) -> None:
)).scalar_one() == "future-state"
def test_partial_upgrade_recomputes_inconsistent_planning_identity(monkeypatch) -> None:
"""部分升级留下的版本和指纹必须按最终 JSON 重算，不能保留伪身份。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_admission_table(connection)
connection.execute(sa.text(
"ALTER TABLE transferpending ADD COLUMN input_version INTEGER"
))
connection.execute(sa.text(
"ALTER TABLE transferpending ADD COLUMN input_fingerprint VARCHAR(64)"
))
connection.execute(sa.text(
"UPDATE transferpending SET input_version = 99, "
"input_fingerprint = 'bogus' WHERE id = 1"
))
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
upgraded = _planning_row(connection)
payload = upgraded["planning_input"]
if isinstance(payload, str):
payload = json.loads(payload)
planning_input = TransferPlanningInput.from_payload(payload)
assert upgraded["input_version"] == planning_input.schema_version == 1
assert upgraded["input_fingerprint"] == planning_input.fingerprint
assert upgraded["input_fingerprint"] != "bogus"
def test_replayed_upgrade_repairs_complete_but_mismatched_identity(monkeypatch) -> None:
"""重复执行升级也必须修复完整三元组中与 payload 不一致的旧值。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_admission_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
connection.execute(sa.text(
"UPDATE transferpending SET input_version = 7, "
"input_fingerprint = 'stale' WHERE id = 1"
))
migration.upgrade()
upgraded = _planning_row(connection)
payload = upgraded["planning_input"]
if isinstance(payload, str):
payload = json.loads(payload)
planning_input = TransferPlanningInput.from_payload(payload)
assert upgraded["input_version"] == 1
assert upgraded["input_fingerprint"] == planning_input.fingerprint
def test_transfer_planning_migration_runs_on_postgresql(monkeypatch) -> None:
"""配置隔离 PostgreSQL 时真实验证规划字段的完整可逆迁移。"""
prefix = "MOVIEPILOT_TEST_POSTGRESQL_"
tests/test_transfer_planning_persistence.py
+21 -12
View File
@@ -4,6 +4,7 @@ from dataclasses import replace
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from app.application.transfer.workflow import (
@@ -545,8 +546,10 @@ def test_checkpoint_rejects_missing_task(repository) -> None:
)
def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
"""兼容直接构造 ORM 行时也必须生成匹配路径的版本化输入与指纹。"""
def test_canonical_admission_requires_explicit_versioned_planning_input(
tmp_path,
) -> None:
"""直接 ORM 写入不得伪造默认输入，canonical 仓储必须显式保存完整快照。"""
engine = create_engine(f"sqlite:///{tmp_path / 'orm-defaults.db'}")
factory = sessionmaker(bind=engine)
TransferPending.__table__.create(engine)
@@ -559,21 +562,27 @@ def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
updated_at="2026-08-27 10:00:00",
)
session.add(pending)
session.commit()
task_id = pending.task_id
with pytest.raises(IntegrityError):
session.commit()
session.rollback()
repository = TransactionalTransferAdmissionRepository(factory)
admitted = repository.claim_task(
task_id=task_id,
owner_id="legacy-projection-worker",
lease_seconds=3600,
planning_input = replace(
_planning_input(),
source_fileitem={
"storage": "local",
"path": "/downloads/legacy.mkv",
"type": "file",
},
)
admitted = repository.admit(
storage="local",
src_path="/downloads/legacy.mkv",
planning_input=planning_input,
)
assert admitted is not None
assert admitted.planning_input == TransferPlanningInput.legacy(
storage="local",
src_path="/downloads/legacy.mkv",
)
assert admitted.planning_input == planning_input
assert admitted.input_fingerprint == admitted.planning_input.fingerprint
engine.dispose()
tests/test_transfer_queue_service.py
+23 -1
View File
@@ -5,7 +5,11 @@ import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application.transfer.workflow import TransferAdmission, TransferQueueService
from app.application.transfer.workflow import (
TransferAdmission,
TransferPlanningInput,
TransferQueueService,
)
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.models.transferhistory import TransferHistory
from app.db.models.transferpending import TransferPending
@@ -13,6 +17,20 @@ from app.schemas.file import FileItem
from tests.test_transfer_job_manager import make_task, make_transfer_chain
def _planning_input(path: str = "/tmp/demo.mkv") -> TransferPlanningInput:
"""构造队列准入测试要求的显式版本化输入。"""
return TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": path,
"type": "file",
"name": path.rsplit("/", 1)[-1],
},
meta=None,
mediainfo=None,
)
def _service(**overrides):
"""构造可观测整理队列服务及其默认依赖。"""
dependencies = {
@@ -24,6 +42,7 @@ def _service(**overrides):
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(),
)),
"enqueue": Mock(),
"before_enqueue": Mock(),
@@ -48,6 +67,7 @@ def test_transfer_queue_service_put_preserves_registration_order():
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(),
),
before_enqueue=lambda _task: calls.append("batch"),
enqueue=lambda _item: calls.append("queue"),
@@ -123,10 +143,12 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
factory = sessionmaker(bind=engine)
repository = TransactionalTransferAdmissionRepository(factory)
task = make_task(1)
task.bind_planning_input(_planning_input(task.fileitem.path))
service, _ = _service(
admit_task=lambda item: repository.admit(
storage=item.fileitem.storage,
src_path=item.fileitem.path,
planning_input=item.planning_input,
),
enqueue=Mock(side_effect=RuntimeError("queue closed")),
enqueue_failed=lambda item, error: repository.record_enqueue_failure(
tests/test_transfer_reconciliation_migration.py
+796
View File
@@ -0,0 +1,796 @@
"""整理状态 3.0.17 数据收口与完整迁移链测试。"""
import hashlib
import importlib
import json
import os
import uuid
from datetime import datetime, timezone
import pytest
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy.orm import sessionmaker
from app.application.transfer.execution import (
TransferExecutionCommand,
TransferExecutionState,
TransferManualReviewDecision,
TransferManualReviewQuery,
TransferStepResult,
)
from app.application.transfer.workflow import (
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.adapters.transfer.admission import (
TransactionalTransferAdmissionRepository,
)
from app.db.adapters.transfer.execution import (
TransactionalTransferExecutionRepository,
)
try:
import psycopg2 as postgres_driver
from psycopg2 import sql
POSTGRESQL_DIALECT = "postgresql+psycopg2"
except ModuleNotFoundError:
import psycopg as postgres_driver
from psycopg import sql
POSTGRESQL_DIALECT = "postgresql+psycopg"
INITIAL_MIGRATION = "database.versions.e3d9f4b7c806_3_0_4"
ADMISSION_MIGRATION = "database.versions.b1e7d3f5a9c2_3_0_13"
PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
EXECUTION_MIGRATION = "database.versions.e5c7a9b1d3f6_3_0_16"
RECONCILIATION_MIGRATION = "database.versions.f6d8b0c2e4a7_3_0_17"
def _bind_migration(monkeypatch, connection, module_name: str):
"""把指定整理迁移绑定到隔离数据库连接。"""
migration = importlib.import_module(module_name)
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
return migration
def _canonical_fingerprint(payload: dict[str, object]) -> str:
"""按运行时规范计算测试检查点指纹。"""
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _create_3_0_15_tables(connection) -> sa.Table:
"""创建执行迁移前的 pending 与最小 history 表。"""
metadata = sa.MetaData()
pending = sa.Table(
"transferpending",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("task_id", sa.String(64), nullable=False),
sa.Column("storage", sa.String(), nullable=False),
sa.Column("src_path", sa.String(), nullable=False),
sa.Column("created_at", sa.String()),
sa.Column("state", sa.String(32), nullable=False),
sa.Column("updated_at", sa.String(40), nullable=False),
sa.Column("last_error", sa.Text()),
sa.Column("input_version", sa.Integer(), nullable=False),
sa.Column("planning_input", sa.JSON(), nullable=False),
sa.Column("input_fingerprint", sa.String(64), nullable=False),
sa.Column("checkpoint_version", sa.Integer()),
sa.Column("checkpoint_payload", sa.JSON()),
sa.Column("planned_at", sa.String(40)),
sa.Column("lease_owner", sa.String(128)),
sa.Column("lease_token", sa.String(64)),
sa.Column("lease_expires_at", sa.String(40)),
sa.Column("heartbeat_at", sa.String(40)),
sa.Column("attempt_count", sa.Integer(), nullable=False),
sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"),
)
sa.Table(
"transferhistory",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("src", sa.String()),
sa.Column("src_storage", sa.String(), nullable=False),
sa.Column("status", sa.Boolean()),
)
metadata.create_all(connection)
return pending
def _seed_execution_rows(connection, pending: sa.Table) -> None:
"""写入待收口的非法状态以及应保持不变的合法状态。"""
base = {
"storage": "local",
"created_at": "2026-08-27 10:00:00",
"updated_at": "2026-08-27 10:00:00",
"last_error": None,
"state": "accepted",
"input_version": 1,
"planning_input": {
"schema_version": 1,
"source_fileitem": {"storage": "local", "path": "/source"},
},
"input_fingerprint": "input",
"checkpoint_version": None,
"checkpoint_payload": None,
"planned_at": None,
"lease_owner": None,
"lease_token": None,
"lease_expires_at": None,
"heartbeat_at": None,
"attempt_count": 0,
}
task_ids = (
"unknown",
"settling-missing",
"failed-missing",
"retry-missing-due",
"partial-checkpoint",
"completed",
"manual-lease",
"settling-valid",
"failed-valid",
"accepted-checkpoint",
"rejection-checkpoint",
"invalid-outcome",
"invalid-overwrite",
)
connection.execute(pending.insert(), [
{
**base,
"id": index,
"task_id": task_id,
"src_path": f"/{task_id}",
}
for index, task_id in enumerate(task_ids, start=1)
])
def _execution_checkpoint() -> tuple[dict[str, object], str]:
"""构造合法且可由运行时恢复的零副作用执行检查点。"""
payload = {
"schema_version": 1,
"payload": {"outcome": "succeeded", "preview": True},
"operation_ids": [],
"skip_reason": "preview",
}
return payload, _canonical_fingerprint(payload)
def test_upgrade_reconciles_invalid_execution_combinations(monkeypatch) -> None:
"""非法执行组合必须留证转人工态，合法结算与失败终态不得被破坏。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
pending = _create_3_0_15_tables(connection)
_seed_execution_rows(connection, pending)
execution = _bind_migration(monkeypatch, connection, EXECUTION_MIGRATION)
execution.upgrade()
payload, fingerprint = _execution_checkpoint()
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'future' "
"WHERE task_id = 'unknown'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'settling' "
"WHERE task_id = 'settling-missing'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'failed' "
"WHERE task_id = 'failed-missing'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'retry_wait', "
"retry_due_at = NULL WHERE task_id = 'retry-missing-due'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'running', "
"execution_version = 1 WHERE task_id = 'partial-checkpoint'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'completed' "
"WHERE task_id = 'completed'"
))
connection.execute(sa.text(
"UPDATE transferpending SET execution_state = 'manual_review', "
"lease_owner = 'old-worker', lease_token = 'old-lease', "
"lease_expires_at = '2099-01-01 00:00:00.000000', "
"heartbeat_at = '2026-08-27 10:00:00.000000' "
"WHERE task_id = 'manual-lease'"
))
current = sa.table(
"transferpending",
sa.column("task_id", sa.String(64)),
sa.column("execution_state", sa.String(32)),
sa.column("state", sa.String(32)),
sa.column("input_version", sa.Integer()),
sa.column("planning_input", sa.JSON()),
sa.column("input_fingerprint", sa.String(64)),
sa.column("checkpoint_version", sa.Integer()),
sa.column("checkpoint_payload", sa.JSON()),
sa.column("planned_at", sa.String(40)),
sa.column("execution_version", sa.Integer()),
sa.column("execution_payload", sa.JSON()),
sa.column("execution_fingerprint", sa.String(64)),
sa.column("settlement_revision", sa.Integer()),
sa.column("terminal_history_id", sa.Integer()),
sa.column("lease_owner", sa.String(128)),
sa.column("lease_token", sa.String(64)),
sa.column("lease_expires_at", sa.String(40)),
)
host_input = _planning_input("/settling-valid")
host_checkpoint = _host_checkpoint(host_input)
failed_input = _planning_input("/failed-valid")
failed_checkpoint = _host_checkpoint(failed_input)
accepted_input = _planning_input("/accepted-checkpoint")
accepted_checkpoint = _host_checkpoint(accepted_input)
rejection_input = _planning_input("/rejection-checkpoint")
rejection_checkpoint = TransferPlanCheckpoint(
planning_input=rejection_input,
target_storage="local",
root_target_path="/library",
final_target_path="/library/Movies",
resolved_transfer_type="copy",
items=(),
rejection_error="未识别到媒体信息",
)
invalid_outcome_input = _planning_input("/invalid-outcome")
invalid_outcome_checkpoint = _host_checkpoint(invalid_outcome_input)
invalid_overwrite_input = _planning_input("/invalid-overwrite")
invalid_overwrite_checkpoint = _host_checkpoint(invalid_overwrite_input)
for task_id, planning_input, checkpoint, state in (
(
"settling-valid",
host_input,
host_checkpoint,
"planned",
),
(
"failed-valid",
failed_input,
failed_checkpoint,
"planned",
),
(
"accepted-checkpoint",
accepted_input,
accepted_checkpoint,
"accepted",
),
(
"rejection-checkpoint",
rejection_input,
rejection_checkpoint,
"accepted",
),
(
"invalid-outcome",
invalid_outcome_input,
invalid_outcome_checkpoint,
"planned",
),
(
"invalid-overwrite",
invalid_overwrite_input,
invalid_overwrite_checkpoint,
"planned",
),
):
connection.execute(
current.update()
.where(current.c.task_id == task_id)
.values(
state=state,
input_version=1,
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
checkpoint_version=1,
checkpoint_payload=checkpoint.to_payload(),
planned_at="2026-08-27 10:30:00",
)
)
connection.execute(
current.update()
.where(current.c.task_id == "settling-valid")
.values(
execution_state="settling",
execution_version=1,
execution_payload=payload,
execution_fingerprint=fingerprint,
lease_owner="active-worker",
lease_token="active-lease",
lease_expires_at="2099-01-01 00:00:00.000000",
)
)
invalid_outcome_payload = {
**payload,
"payload": {"outcome": "future", "preview": True},
}
invalid_overwrite_payload = {
**payload,
"payload": {"outcome": "overwrite_skipped", "preview": True},
}
for task_id, invalid_payload in (
("invalid-outcome", invalid_outcome_payload),
("invalid-overwrite", invalid_overwrite_payload),
):
connection.execute(
current.update()
.where(current.c.task_id == task_id)
.values(
execution_state="settling",
execution_version=1,
execution_payload=invalid_payload,
execution_fingerprint=_canonical_fingerprint(invalid_payload),
)
)
connection.execute(sa.text(
"INSERT INTO transferhistory (id, src, src_storage, status) "
"VALUES (42, '/failed-valid', 'local', 0)"
))
connection.execute(
current.update()
.where(current.c.task_id == "failed-valid")
.values(
execution_state="failed",
execution_version=1,
execution_payload=payload,
execution_fingerprint=fingerprint,
settlement_revision=1,
terminal_history_id=42,
)
)
connection.execute(sa.text(
"INSERT INTO transfersettlementreceipt ("
"task_id, history_id, settlement_revision, outcome, "
"execution_fingerprint, lease_token, history_status, src, src_storage, "
"pending_deleted, error, created_at, updated_at"
") VALUES ("
"'failed-valid', 42, 1, 'failed', :fingerprint, 'failed-lease', 0, "
"'/failed-valid', 'local', 0, 'failed', "
"'2026-08-27 11:00:00', '2026-08-27 11:00:00'"
")"
), {"fingerprint": fingerprint})
reconciliation = _bind_migration(
monkeypatch,
connection,
RECONCILIATION_MIGRATION,
)
reconciliation.upgrade()
reconciliation.upgrade()
rows = {
row["task_id"]: dict(row)
for row in connection.execute(sa.text(
"SELECT task_id, execution_state, execution_version, "
"execution_payload, execution_fingerprint, lease_owner, lease_token "
"FROM transferpending"
)).mappings()
}
invalid = {
"unknown",
"settling-missing",
"failed-missing",
"retry-missing-due",
"partial-checkpoint",
"completed",
"invalid-outcome",
"invalid-overwrite",
}
assert {rows[task_id]["execution_state"] for task_id in invalid} == {
"manual_review"
}
assert all(
rows[task_id]["execution_version"] is None
and rows[task_id]["execution_payload"] is None
and rows[task_id]["execution_fingerprint"] is None
for task_id in invalid
)
assert rows["manual-lease"]["execution_state"] == "manual_review"
assert rows["manual-lease"]["lease_owner"] is None
assert rows["manual-lease"]["lease_token"] is None
assert rows["settling-valid"]["execution_state"] == "settling"
assert rows["settling-valid"]["lease_token"] == "active-lease"
assert rows["failed-valid"]["execution_state"] == "failed"
planning_states = dict(connection.execute(sa.text(
"SELECT task_id, state FROM transferpending "
"WHERE task_id IN ('accepted-checkpoint', 'rejection-checkpoint')"
)).all())
assert planning_states == {
"accepted-checkpoint": "planned",
"rejection-checkpoint": "planned",
}
assert connection.execute(sa.text(
"SELECT COUNT(*) FROM transferexecutionstep "
"WHERE kind = 'legacy_execution_review' "
"AND task_id IN ('unknown', 'settling-missing', 'failed-missing', "
"'retry-missing-due', 'partial-checkpoint', 'completed', "
"'invalid-outcome', 'invalid-overwrite')"
)).scalar_one() == len(invalid)
factory = sessionmaker(bind=engine, expire_on_commit=False)
reviews = TransferManualReviewQuery(
TransactionalTransferExecutionRepository(factory)
).list(page=1, page_size=20)
assert reviews.total == len(invalid) + 1
assert {item.task_id for item in reviews.items} == invalid | {"manual-lease"}
engine.dispose()
def test_reconciliation_migration_runs_on_postgresql(monkeypatch) -> None:
"""配置隔离 PostgreSQL 时真实验证人工态归一、租约清理与可逆 DDL。"""
prefix = "MOVIEPILOT_TEST_POSTGRESQL_"
host = os.getenv(f"{prefix}HOST")
database = os.getenv(f"{prefix}DATABASE")
username = os.getenv(f"{prefix}USERNAME")
if not host or not database or not username:
pytest.skip("未配置隔离 PostgreSQL migration 测试库")
port = os.getenv(f"{prefix}PORT", "5432")
password = os.getenv(f"{prefix}PASSWORD", "")
schema = f"transfer_reconciliation_{uuid.uuid4().hex}"
with postgres_driver.connect(
host=host,
port=port,
dbname=database,
user=username,
password=password,
) as connection:
connection.autocommit = True
with connection.cursor() as cursor:
cursor.execute(sql.SQL("CREATE SCHEMA {}").format(sql.Identifier(schema)))
engine = None
try:
engine = sa.create_engine(
sa.URL.create(
POSTGRESQL_DIALECT,
username=username,
password=password,
host=host,
port=int(port),
database=database,
),
connect_args={"options": f"-csearch_path={schema}"},
)
with engine.begin() as connection:
pending = _create_3_0_15_tables(connection)
_seed_execution_rows(connection, pending)
execution = _bind_migration(
monkeypatch,
connection,
EXECUTION_MIGRATION,
)
reconciliation = _bind_migration(
monkeypatch,
connection,
RECONCILIATION_MIGRATION,
)
execution.upgrade()
connection.execute(sa.text(
"UPDATE transferpending SET state = 'manual_review', "
"execution_state = 'manual_review', lease_owner = 'old-worker', "
"lease_token = 'old-token', "
"lease_expires_at = '2099-01-01 00:00:00.000000' "
"WHERE task_id = 'manual-lease'"
))
reconciliation.upgrade()
row = connection.execute(sa.text(
"SELECT state, execution_state, lease_owner, lease_token "
"FROM transferpending WHERE task_id = 'manual-lease'"
)).one()
assert row == ("accepted", "manual_review", None, None)
reconciliation.downgrade()
execution.downgrade()
assert "execution_state" not in {
column["name"]
for column in sa.inspect(connection).get_columns("transferpending")
}
finally:
if engine is not None:
engine.dispose()
with postgres_driver.connect(
host=host,
port=port,
dbname=database,
user=username,
password=password,
) as connection:
connection.autocommit = True
with connection.cursor() as cursor:
cursor.execute(
sql.SQL("DROP SCHEMA IF EXISTS {} CASCADE").format(
sql.Identifier(schema)
)
)
def _planning_input(path: str) -> TransferPlanningInput:
"""构造完整且可持久恢复的规划输入。"""
return TransferPlanningInput(
source_fileitem={"storage": "local", "path": path, "type": "file"},
meta={"name": "Movie", "year": 2026},
mediainfo={"title": "Movie", "tmdb_id": 42},
target_directory={"storage": "local", "path": "/library"},
target_storage="local",
target_path="/library/Movies",
requested_transfer_type="copy",
)
def _host_checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
"""构造完整宿主计划检查点。"""
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/library",
final_target_path="/library/Movies/Movie.mkv",
resolved_transfer_type="copy",
items=(TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path="/library/Movies/Movie.mkv",
),),
)
def _provider_checkpoint(
planning_input: TransferPlanningInput,
) -> TransferPlanCheckpoint:
"""构造完整 provider 待执行检查点。"""
invocation = TransferProviderInvocationSnapshot(
fileitem=planning_input.source_fileitem,
meta=planning_input.meta,
meta_kind="MetaVideo",
mediainfo=planning_input.mediainfo,
mediainfo_kind="MediaInfo",
)
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
legacy_transfer_providers=(TransferProviderReference(
plugin_id="provider-a",
plugin_name="Provider A",
),),
provider_invocation=invocation,
)
def _create_history_table(connection) -> None:
"""创建迁移链所需的最小整理历史表。"""
connection.execute(sa.text(
"CREATE TABLE transferhistory ("
"id INTEGER PRIMARY KEY, src VARCHAR, "
"src_storage VARCHAR NOT NULL, status BOOLEAN)"
))
def _run_upgrade_chain(monkeypatch, connection) -> list[object]:
"""从 3.0.13 顺序升级到 3.0.17 并返回迁移模块。"""
migrations = [
_bind_migration(monkeypatch, connection, module_name)
for module_name in (
ADMISSION_MIGRATION,
PLANNING_MIGRATION,
LEASE_MIGRATION,
EXECUTION_MIGRATION,
RECONCILIATION_MIGRATION,
)
]
for migration in migrations:
migration.upgrade()
return migrations
def test_full_legacy_chain_projects_after_downgrade_and_reupgrade(
monkeypatch,
) -> None:
"""旧登记经完整升级、人工判定、降级再升级后仍可被真实仓储投影。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
initial = _bind_migration(monkeypatch, connection, INITIAL_MIGRATION)
initial.upgrade()
_create_history_table(connection)
connection.execute(sa.text(
"INSERT INTO transferpending (id, storage, src_path, created_at) VALUES "
"(1, 'local', '/accepted.mkv', '2026-08-27 09:00:00'), "
"(2, 'local', '/planned.mkv', '2026-08-27 09:00:01'), "
"(3, 'local', '/provider.mkv', '2026-08-27 09:00:02')"
))
admission = _bind_migration(
monkeypatch,
connection,
ADMISSION_MIGRATION,
)
planning = _bind_migration(
monkeypatch,
connection,
PLANNING_MIGRATION,
)
lease = _bind_migration(monkeypatch, connection, LEASE_MIGRATION)
execution = _bind_migration(
monkeypatch,
connection,
EXECUTION_MIGRATION,
)
reconciliation = _bind_migration(
monkeypatch,
connection,
RECONCILIATION_MIGRATION,
)
admission.upgrade()
planning.upgrade()
lease.upgrade()
rows = connection.execute(sa.text(
"SELECT id, task_id, src_path, planning_input FROM transferpending"
)).mappings().all()
by_path = {row["src_path"]: row for row in rows}
for path, checkpoint in (
(
"/planned.mkv",
_host_checkpoint(_planning_input("/planned.mkv")),
),
(
"/provider.mkv",
_provider_checkpoint(_planning_input("/provider.mkv")),
),
):
planning_input = checkpoint.planning_input
connection.execute(sa.text(
"UPDATE transferpending SET state = 'manual_review', "
"input_version = 1, planning_input = :planning_input, "
"input_fingerprint = :input_fingerprint, checkpoint_version = 1, "
"checkpoint_payload = :checkpoint_payload, "
"planned_at = '2026-08-27 09:30:00', "
"lease_owner = 'old-worker', lease_token = 'old-token', "
"lease_expires_at = '2099-01-01 00:00:00.000000' "
"WHERE id = :id"
), {
"id": by_path[path]["id"],
"planning_input": json.dumps(planning_input.to_payload()),
"input_fingerprint": planning_input.fingerprint,
"checkpoint_payload": json.dumps(checkpoint.to_payload()),
})
connection.execute(sa.text(
"UPDATE transferpending SET state = 'manual_review', "
"lease_owner = 'old-worker', lease_token = 'old-token', "
"lease_expires_at = '2099-01-01 00:00:00.000000' WHERE id = 1"
))
execution.upgrade()
reconciliation.upgrade()
normalized = dict(connection.execute(sa.text(
"SELECT src_path, state FROM transferpending ORDER BY id"
)).all())
assert normalized == {
"/accepted.mkv": "accepted",
"/planned.mkv": "planned",
"/provider.mkv": "provider_pending",
}
assert connection.execute(sa.text(
"SELECT COUNT(*) FROM transferpending WHERE lease_token IS NOT NULL"
)).scalar_one() == 0
factory = sessionmaker(bind=engine, expire_on_commit=False)
execution_repository = TransactionalTransferExecutionRepository(
factory,
local_clock=lambda: datetime(2026, 8, 27, 10, 0, 0),
lease_clock=lambda: datetime(2026, 8, 27, 2, 0, 0, tzinfo=timezone.utc),
)
reviews = TransferManualReviewQuery(execution_repository).list(
page=1,
page_size=10,
)
assert reviews.total == 3
command = TransferExecutionCommand(execution_repository)
for review in reviews.items:
resolved = command.resolve_manual_review(
task_id=review.task_id,
operation_id=review.step.operation_id,
decision=TransferManualReviewDecision.NOT_APPLIED,
actor="migration-test",
reason="确认旧执行未发生",
result=TransferStepResult(payload={"confirmed": False}),
)
assert resolved.state is TransferExecutionState.RETRY_WAIT
admission_repository = TransactionalTransferAdmissionRepository(factory)
monkeypatch.setattr(
admission_repository,
"_now",
lambda: "2026-08-27 10:00:01",
)
monkeypatch.setattr(
admission_repository,
"_lease_now",
lambda: datetime(2026, 8, 27, 2, 0, 1, tzinfo=timezone.utc),
)
claimed_states = set()
for task_id in (row["task_id"] for row in rows):
claimed = admission_repository.claim_task(
task_id=task_id,
owner_id="migration-worker",
lease_seconds=60,
)
assert claimed is not None
claimed_states.add(claimed.state)
assert claimed_states == {"accepted", "planned", "provider_pending"}
with engine.begin() as connection:
for migration in (
reconciliation,
execution,
lease,
planning,
admission,
):
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
migration.downgrade()
assert {
column["name"]
for column in sa.inspect(connection).get_columns("transferpending")
} == {"id", "storage", "src_path", "created_at"}
migrations = _run_upgrade_chain(monkeypatch, connection)
assert connection.execute(sa.text(
"SELECT COUNT(*) FROM transferpending"
)).scalar_one() == 3
assert connection.execute(sa.text(
"SELECT COUNT(*) FROM transferpending "
"WHERE state = 'accepted' AND execution_state = 'not_started' "
"AND lease_token IS NULL"
)).scalar_one() == 3
assert all(migration is not None for migration in migrations)
reupgraded_repository = TransactionalTransferAdmissionRepository(factory)
monkeypatch.setattr(
reupgraded_repository,
"_now",
lambda: "2026-08-27 10:01:01",
)
monkeypatch.setattr(
reupgraded_repository,
"_lease_now",
lambda: datetime(2026, 8, 27, 2, 1, 1, tzinfo=timezone.utc),
)
reupgraded = [
reupgraded_repository.claim_task(
task_id=row["task_id"],
owner_id="reupgraded-worker",
lease_seconds=60,
)
for row in rows
]
assert all(item is not None for item in reupgraded)
assert {item.state for item in reupgraded if item is not None} == {"accepted"}
engine.dispose()
tests/test_transfer_settling_recovery.py
+229 -1
View File
@@ -9,21 +9,33 @@ import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application.chain.events import TransferResultSettlement
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferExecutionCommand,
TransferExecutionSnapshot,
TransferExecutionState,
TransferOperationObservation,
TransferOperationObservationState,
TransferStepIntent,
TransferStepResult,
)
from app.application.transfer.workflow import (
TransferAdmission,
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
TransferTask,
)
from app.chain import transfer as transfer_chain_module
from app.chain.transfer import TransferChain
from app.db.adapters.chain import TransactionalChainDurableEventWriter
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
from app.db.adapters.transfer.execution import TransactionalTransferExecutionRepository
from app.db.models.transferexecutionstep import TransferExecutionStep
from app.db.models.transferhistory import TransferHistory
from app.db.models.transferpending import TransferPending
from app.db.models.transfersettlementreceipt import TransferSettlementReceipt
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
@@ -173,6 +185,8 @@ def admission_store(tmp_path):
engine = create_engine(f"sqlite:///{tmp_path / 'settling-recovery.db'}")
TransferHistory.__table__.create(engine)
TransferPending.__table__.create(engine)
TransferExecutionStep.__table__.create(engine)
TransferSettlementReceipt.__table__.create(engine)
factory = sessionmaker(bind=engine, expire_on_commit=False)
try:
yield TransactionalTransferAdmissionRepository(factory), factory
@@ -202,6 +216,7 @@ def _build_chain(admissions) -> TransferChain:
"""构造只允许执行 settling 终态恢复的 TransferChain 骨架。"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = admissions
chain._transfer_executions = MagicMock()
chain._worker_owner_id = "recovery-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
@@ -251,6 +266,18 @@ def _recovered_task(
admission.lease_token,
time.monotonic() + 120,
)
chain._transfer_executions.get_snapshot.return_value = TransferExecutionSnapshot(
task_id=admission.task_id,
state=TransferExecutionState.SETTLING,
checkpoint=execution_checkpoint,
retry_generation=0,
retry_count=0,
retry_due_at=None,
settlement_revision=0,
terminal_history_id=None,
last_error=None,
steps=(),
)
return task
@@ -381,7 +408,7 @@ def test_replay_settling_uses_frozen_source_without_filesystem_probe(
queued_task = chain.put_to_queue.call_args.args[0]
assert queued_task.fileitem.path == path
assert queued_task.execution_checkpoint == execution_checkpoint
admissions.discard_claimed.assert_not_called()
admissions.abandon_unstarted.assert_not_called()
admissions.release_claim.assert_not_called()
chain._plan_checkpoint_and_execute.assert_not_called()
@@ -454,3 +481,204 @@ def test_writer_failure_releases_and_reclaims_same_settling_checkpoint(
assert second_admission.lease_token != first_admission.lease_token
chain._TransferChain__select_storage_oper.assert_not_called()
chain._plan_checkpoint_and_execute.assert_not_called()
def test_bound_checkpoint_is_not_restored_outside_settling() -> None:
"""旧终态检查点留在 retry_wait 时不得跳过步骤恢复直接再次结算。"""
path = "/downloads/retry-state.mkv"
planning_input = _planning_input(path)
plan_checkpoint = _plan_checkpoint(planning_input)
execution_checkpoint = _execution_checkpoint(path, success=False)
admission = TransferAdmission(
task_id="retry-task",
storage="local",
src_path=path,
state="planned",
created_at="2026-08-27 09:00:00",
updated_at="2026-08-27 09:00:00",
planning_input=planning_input,
checkpoint=plan_checkpoint,
lease_owner="recovery-owner",
lease_token="retry-token",
)
chain = _build_chain(MagicMock())
task = _recovered_task(chain, admission, execution_checkpoint)
chain._transfer_executions.get_snapshot.return_value = TransferExecutionSnapshot(
task_id=admission.task_id,
state=TransferExecutionState.RETRY_WAIT,
checkpoint=execution_checkpoint,
retry_generation=1,
retry_count=1,
retry_due_at="2026-08-27 09:00:00.000000",
settlement_revision=1,
terminal_history_id=41,
last_error="copy failed",
steps=(),
)
restored = chain._TransferChain__restore_settling_transfer_result(task)
assert restored is None
def test_failed_settlement_retry_replays_step_and_commits_new_receipt(
admission_store,
) -> None:
"""失败结算请求重试后应恢复 FAILED 步骤，并以新检查点完成下一版结算。"""
admissions, factory = admission_store
path = "/downloads/retry-success.mkv"
planning_input = _planning_input(path)
plan_checkpoint = TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/library",
final_target_path="/library/retry-success.mkv",
resolved_transfer_type="copy",
items=(TransferPlanItem(
sequence=0,
source_fileitem=planning_input.source_fileitem,
target_storage="local",
target_path="/library/retry-success.mkv",
),),
need_notify=False,
)
admitted = admissions.admit(
storage="local",
src_path=path,
planning_input=planning_input,
)
first_claim = admissions.claim_task(
task_id=admitted.task_id,
owner_id="first-owner",
lease_seconds=120,
)
assert first_claim is not None
assert first_claim.lease_token is not None
admissions.checkpoint_plan(
task_id=admitted.task_id,
lease_token=first_claim.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=plan_checkpoint,
)
executions = TransactionalTransferExecutionRepository(factory)
command = TransferExecutionCommand(
executions,
attempt_token_factory=iter(("attempt-1", "attempt-2")).__next__,
)
plan_fingerprint = (
TransferChain._TransferChain__transfer_plan_fingerprint(plan_checkpoint)
)
intent = TransferStepIntent.create(
task_id=admitted.task_id,
checkpoint_fingerprint=plan_fingerprint,
ordinal=0,
phase="transfer",
kind="copy",
payload={"source": path, "target": "/library/retry-success.mkv"},
)
prepared = command.prepare(
task_id=admitted.task_id,
lease_token=first_claim.lease_token,
intent=intent,
)
started = command.begin(
task_id=admitted.task_id,
lease_token=first_claim.lease_token,
operation_id=prepared.operation_id,
)
exhausted = command.exhaust(
task_id=admitted.task_id,
lease_token=first_claim.lease_token,
step=started,
error="copy failed",
evidence=TransferStepResult(payload={"target_exists": False}),
)
assert exhausted.checkpoint is not None
writer = TransactionalChainDurableEventWriter(factory)
first_settlement = writer.transfer_result(
topic=None,
stage_history=lambda repository: repository.add_force(
src=path,
src_storage="local",
status=False,
errmsg="copy failed",
),
event_payload={},
publish=None,
settlement=TransferResultSettlement(
task_id=admitted.task_id,
lease_token=first_claim.lease_token,
execution_fingerprint=exhausted.checkpoint.fingerprint,
outcome="failed",
error="copy failed",
),
)
assert first_settlement is not None
retry = command.request_retry(
task_id=admitted.task_id,
reason="manual retry",
requested_by="test",
)
assert retry.accepted is True
chain = _build_chain(admissions)
chain._transfer_executions = executions
chain.put_to_queue = MagicMock(return_value=True)
chain._TransferChain__replay_pending()
replayed_task = chain.put_to_queue.call_args.args[0]
assert replayed_task.execution_checkpoint is None
assert replayed_task.lease_token is not None
runner = transfer_chain_module._DurableTransferStepRunner(
task_id=admitted.task_id,
lease_token=replayed_task.lease_token,
checkpoint_fingerprint=plan_fingerprint,
repository=executions,
)
resumed = []
step_result = runner.run(
phase="transfer",
kind="copy",
payload={"source": path, "target": "/library/retry-success.mkv"},
execute=lambda: resumed.append("executed") or TransferStepResult(
payload={"target_exists": True}
),
observe=lambda: TransferOperationObservation(
state=TransferOperationObservationState.NOT_APPLIED,
evidence=TransferStepResult(payload={"target_exists": False}),
),
)
assert step_result.payload == {"target_exists": True}
assert resumed == ["executed"]
new_checkpoint = runner.checkpoint(_transfer_result(path, success=True))
assert new_checkpoint.fingerprint != exhausted.checkpoint.fingerprint
second_settlement = writer.transfer_result(
topic=None,
stage_history=lambda repository: repository.add_force(
src=path,
src_storage="local",
status=True,
errmsg=None,
),
event_payload={},
publish=None,
settlement=TransferResultSettlement(
task_id=admitted.task_id,
lease_token=replayed_task.lease_token,
execution_fingerprint=new_checkpoint.fingerprint,
outcome="succeeded",
),
)
assert second_settlement is not None
with factory() as session:
receipts = session.scalars(
select(TransferSettlementReceipt).order_by(
TransferSettlementReceipt.settlement_revision
)
).all()
pending = session.scalar(select(TransferPending))
steps = session.scalars(select(TransferExecutionStep)).all()
assert [receipt.outcome for receipt in receipts] == ["failed", "succeeded"]
assert pending is None
assert steps == []
tests/test_transfer_tmdb_category.py
+54 -1
View File
@@ -1,5 +1,8 @@
import threading
from types import SimpleNamespace
from unittest.mock import Mock
from app.application.transfer.execution import TransferExecutionCheckpoint
from app.application.transfer.workflow import TransferTask
from app.chain.transfer import TransferChain
from app.domain.context import MediaInfo
@@ -76,6 +79,43 @@ def test_transfer_stops_when_automatic_category_has_no_tmdb_result(monkeypatch)
"""启用自动类别目录时，缺少 TMDB 分类必须在文件操作前明确失败。"""
chain = object.__new__(TransferChain)
chain.jobview = SimpleNamespace(try_remove_job=lambda _task: None)
chain._transfer_admissions = Mock()
chain._worker_owner_id = "category-owner"
chain._owned_leases = {
"task-before-category": ("lease-before-category", float("inf"))
}
chain._worker_state_lock = threading.RLock()
chain.durable_event_writer = Mock()
chain.runtime_config = SimpleNamespace(
scrape_follow_tmdb=True,
ai_agent_enable=True,
ai_agent_retry_transfer=True,
)
chain.queue_failed_transfer_notification = Mock()
chain._TransferChain__mark_torrent_completed_if_done = Mock()
record_transfer_failure = Mock()
add_transfer_fail = Mock()
monkeypatch.setattr(
"app.chain.transfer.record_transfer_failure",
record_transfer_failure,
)
monkeypatch.setattr("app.chain.transfer.add_transfer_fail", add_transfer_fail)
chain._transfer_admissions.checkpoint_plan.side_effect = (
lambda **kwargs: SimpleNamespace(checkpoint=kwargs["checkpoint"])
)
step_runner = Mock()
step_runner.checkpoint.side_effect = lambda transferinfo: (
TransferExecutionCheckpoint.create(
payload={
"outcome": "failed",
"transferinfo": transferinfo.model_dump(mode="json"),
},
operation_ids=("planning-reject",),
)
)
chain._TransferChain__build_durable_step_runner = Mock(
return_value=step_runner
)
monkeypatch.setattr(
"app.chain.transfer.get_chain_transfer_history_port",
lambda: SimpleNamespace(),
@@ -114,7 +154,12 @@ def test_transfer_stops_when_automatic_category_has_no_tmdb_result(monkeypatch)
library_category_folder=True,
),
library_category_folder=True,
preview=True,
preview=False,
)
task.bind_admission_task_id("task-before-category")
task.bind_execution_lease(
owner_id="category-owner",
lease_token="lease-before-category",
)
state, message = chain._TransferChain__handle_transfer(task)
@@ -123,3 +168,11 @@ def test_transfer_stops_when_automatic_category_has_no_tmdb_result(monkeypatch)
assert message == "未识别到 TMDB 辅助信息，无法按媒体类别整理"
assert task.mediainfo.media_source == MediaSource.AniList
assert task.mediainfo.media_id == "1234"
assert task.plan_checkpoint is not None
assert task.plan_checkpoint.rejection_error == message
assert task.execution_checkpoint is not None
chain._transfer_admissions.record_planning_failure.assert_not_called()
record_transfer_failure.assert_not_called()
add_transfer_fail.assert_not_called()
chain.queue_failed_transfer_notification.assert_not_called()
chain._TransferChain__mark_torrent_completed_if_done.assert_not_called()
tests/test_transfer_worker_lifecycle.py
+52 -67
View File
@@ -4,13 +4,17 @@ import asyncio
import queue
import threading
import time
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from unittest.mock import AsyncMock, MagicMock
import pytest
from app.application.transfer.workflow import TransferAdmission, TransferQueue, TransferTask
from app.application.transfer.workflow import (
TransferAdmission,
TransferPlanningInput,
TransferQueue,
TransferTask,
)
from app.chain.transfer import TransferChain
from app.foundation.singleton import Singleton
from app.runtime.config import global_vars
@@ -19,6 +23,15 @@ from app.schemas.transfer import TransferInfo
from app.startup.initializers import transfer as transfer_initializer
def _planning_input(fileitem: FileItem) -> TransferPlanningInput:
"""构造 worker 准入与 claim 投影使用的真实规划输入。"""
return TransferPlanningInput(
source_fileitem=fileitem.model_dump(mode="json"),
meta=None,
mediainfo=None,
)
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
chain = object.__new__(TransferChain)
@@ -51,6 +64,7 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=kwargs["planning_input"],
)
admissions.claim_task.side_effect = lambda **kwargs: TransferAdmission(
task_id=kwargs["task_id"],
@@ -59,13 +73,18 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(FileItem(
storage="local",
path="/downloads/test.mkv",
type="file",
)),
lease_owner=kwargs["owner_id"],
lease_token=f"lease-{kwargs['task_id']}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
admissions.discard_claimed.return_value = 1
admissions.abandon_unstarted.return_value = 1
admissions.release_claim.return_value = True
chain._transfer_admissions = admissions
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
@@ -82,6 +101,7 @@ def _claimed_admission(task: TransferTask, task_id: str) -> TransferAdmission:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(task.fileitem),
lease_owner="worker-owner",
lease_token=f"lease-{task_id}",
lease_expires_at="2026-08-27 10:02:00.000000",
@@ -224,33 +244,29 @@ def test_close_workers_lock_wait_uses_the_same_timeout_budget() -> None:
assert chain.close_workers(timeout_seconds=1) is True
def test_close_keeps_timer_dependencies_when_workers_do_not_converge() -> None:
"""活跃整理线程超时后，通知和重试 owner 必须继续供线程使用。"""
def test_close_keeps_failure_notification_when_workers_do_not_converge() -> None:
"""活跃整理线程超时后，失败通知 owner 必须继续供线程使用。"""
chain = _build_chain()
chain.close_workers = MagicMock(return_value=False)
chain.failure_notification_aggregator = MagicMock()
chain.retry_scheduler = MagicMock(close=AsyncMock())
completed = asyncio.run(chain.close(timeout_seconds=0.01))
assert completed is False
chain.close_workers.assert_called_once_with(0.01)
chain.failure_notification_aggregator.close.assert_not_called()
chain.retry_scheduler.close.assert_not_awaited()
def test_close_releases_timer_dependencies_after_workers_converge() -> None:
"""worker 和回放退出后，整理链应继续刷新通知并关闭 AI 重试。"""
def test_close_releases_failure_notification_after_workers_converge() -> None:
"""worker 和回放退出后，整理链应刷新并关闭失败通知 owner。"""
chain = _build_chain()
chain.close_workers = MagicMock(return_value=True)
chain.failure_notification_aggregator = MagicMock()
chain.retry_scheduler = MagicMock(close=AsyncMock())
completed = asyncio.run(chain.close(timeout_seconds=0.01))
assert completed is True
chain.failure_notification_aggregator.close.assert_called_once_with()
chain.retry_scheduler.close.assert_awaited_once_with()
def test_stop_transfer_runtime_does_not_construct_chain(monkeypatch) -> None:
@@ -329,49 +345,6 @@ def test_constructor_failure_publishes_started_worker_to_cleanup(monkeypatch) ->
assert workers[0].is_alive() is False
def test_failed_retry_schedule_future_error_is_observed() -> None:
"""跨线程调度协程的延迟异常必须被取回并写入日志。"""
future: Future[None] = Future()
future.set_exception(RuntimeError("scheduler closed"))
with patch("app.chain.transfer.logger.error") as log_error:
TransferChain._observe_failed_retry_schedule(future)
log_error.assert_called_once()
assert "scheduler closed" in log_error.call_args.args[0]
def test_failed_retry_schedule_registers_future_observer(monkeypatch) -> None:
"""整理线程提交 AI 重试后应让 Future 持续连接到异常观察回调。"""
chain = _build_chain()
async def schedule_retry(_history_id: int, *, group_key: str) -> None:
"""提供不会实际执行的调度协程，供跨线程提交边界检查。"""
chain.retry_scheduler = MagicMock(schedule_retry=schedule_retry)
future = MagicMock(spec=Future)
event_loop = MagicMock()
event_loop.is_running.return_value = True
event_loop.is_closed.return_value = False
monkeypatch.setattr(global_vars, "CURRENT_EVENT_LOOP", event_loop)
def submit(coroutine, loop):
"""关闭测试协程并返回可检查的并发 Future。"""
assert loop is event_loop
coroutine.close()
return future
with patch(
"app.chain.transfer.asyncio.run_coroutine_threadsafe",
side_effect=submit,
):
chain._schedule_failed_transfer_retry(42, "media:test")
future.add_done_callback.assert_called_once()
callback = future.add_done_callback.call_args.args[0]
assert callback is TransferChain._observe_failed_retry_schedule
def test_worker_requeues_item_taken_during_shutdown(monkeypatch) -> None:
"""停止信号与 queue.get 竞态时，未开始处理的任务必须原样放回队列。"""
chain = _build_chain()
@@ -465,8 +438,8 @@ def test_worker_settles_progress_when_only_stop_sentinel_remains(monkeypatch) ->
assert list(chain._queue.queue) == [chain._QUEUE_STOP_SENTINEL]
def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch) -> None:
"""准入生成的稳定身份必须随队列任务到 worker 终态并准确注销。"""
def test_durable_task_identity_flows_to_unsettled_terminal_claim_release(monkeypatch) -> None:
"""终态无原子回执时稳定身份必须用于释放 claim，pending 保持可恢复。"""
chain = _build_chain()
chain.runtime_config.transfer_task_timeout = 0
task = TransferTask(fileitem=FileItem(
@@ -486,12 +459,13 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=_planning_input(task.fileitem),
)
admissions.claim_task.return_value = _claimed_admission(
task,
"durable-task-id",
)
admissions.discard_claimed.side_effect = (
admissions.release_claim.side_effect = (
lambda **_kwargs: discarded.set() or 1
)
chain._transfer_admissions = admissions
@@ -534,10 +508,12 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
owner_id="worker-owner",
lease_seconds=120,
)
admissions.discard_claimed.assert_called_once_with(
admissions.release_claim.assert_called_once_with(
task_id="durable-task-id",
lease_token="lease-durable-task-id",
error="整理终态未完成 durable 原子结算",
)
admissions.abandon_unstarted.assert_not_called()
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
@@ -646,7 +622,7 @@ def test_recovered_worker_reuses_claimed_token_without_second_claim(
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 1
chain._transfer_admissions.release_claim.return_value = True
stop_event = threading.Event()
def complete_recovery(*, task, callback):
@@ -670,10 +646,12 @@ def test_recovered_worker_reuses_claimed_token_without_second_claim(
assert worker.is_alive() is False
chain._transfer_admissions.claim_task.assert_not_called()
chain._transfer_admissions.discard_claimed.assert_called_once_with(
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="recovered-task",
lease_token="lease-recovered-task",
error="整理终态未完成 durable 原子结算",
)
chain._transfer_admissions.abandon_unstarted.assert_not_called()
def test_heartbeat_refreshes_current_token_and_forgets_lost_lease() -> None:
@@ -761,7 +739,7 @@ def test_worker_reports_failed_settlement_without_skipping_queue_bookkeeping(
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 0
chain._transfer_admissions.release_claim.return_value = False
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
stop_event = threading.Event()
@@ -877,8 +855,10 @@ def test_worker_fenced_releases_lost_lease_and_completes_queue_bookkeeping(
assert chain._recovery_wakeup_event.is_set() is False
def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) -> None:
"""终态 CAS 被拒绝时不得写成功历史、事件或通知。"""
def test_callback_without_terminal_settlement_releases_claim_and_counts_failure(
monkeypatch,
) -> None:
"""回调未给出原子结算回执时必须保留 pending、释放 claim 并计失败。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
@@ -896,7 +876,7 @@ def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) ->
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 0
chain._transfer_admissions.release_claim.return_value = False
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
success_callback = MagicMock(return_value=(True, ""))
chain._TransferChain__default_callback = success_callback
@@ -924,8 +904,13 @@ def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) ->
worker.join(timeout=1)
assert worker.is_alive() is False
success_callback.assert_not_called()
chain.jobview.fail_unfinished_task.assert_called_once_with(task)
success_callback.assert_called_once()
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="admitted-task",
lease_token="lease-admitted-task",
error="整理终态未完成 durable 原子结算",
)
chain._transfer_admissions.abandon_unstarted.assert_not_called()
assert chain._fail_num == 1
assert chain._queue.unfinished_tasks == 0