From e82ce8447c92547ee45f826191f16de23724fa7e Mon Sep 17 00:00:00 2001 From: jxxghp Date: Thu, 27 Aug 2026 20:26:27 +0800 Subject: [PATCH] refactor: complete durable transfer execution settlement --- app/agent/prompt/System Tasks.yaml | 21 +- .../tools/impl/delete_transfer_history.py | 46 +- app/api/endpoints/history.py | 199 +++- app/api/endpoints/transfer.py | 155 ++- app/application/chain/data.py | 9 + app/application/chain/durable_events.py | 49 +- app/application/history.py | 10 +- app/application/outbox.py | 24 +- app/application/transfer.py | 23 + app/application/transfer_execution.py | 884 ++++++++++++++ app/chain/_transfer.py | 127 +- app/chain/transfer.py | 813 +++++++++++-- app/db/adapters/chain.py | 360 +++++- app/db/adapters/transfer.py | 6 +- app/db/adapters/transfer_execution.py | 1030 +++++++++++++++++ app/db/models/__init__.py | 8 + app/db/models/transferexecutionstep.py | 470 ++++++++ app/db/models/transferhistory.py | 104 +- app/db/models/transferpending.py | 409 +++++++ app/db/models/transfersettlementreceipt.py | 147 +++ app/db/oper/__init__.py | 3 + app/db/oper/transferexecutionstep.py | 243 ++++ app/db/oper/transferhistory.py | 101 +- app/db/oper/transferpending.py | 207 ++++ app/db/oper/transfersettlementreceipt.py | 71 ++ app/locales/en-US.json | 2 + app/locales/zh-CN.json | 2 + app/locales/zh-TW.json | 2 + app/modules/filemanager/module.py | 11 + app/modules/filemanager/transhandler.py | 694 ++++++++++- app/schemas/exports.py | 14 +- app/schemas/history.py | 2 + app/schemas/transfer.py | 74 +- app/sdk/_legacy/transferpending.py | 52 +- app/startup/initializers/modules.py | 6 + database/versions/e5c7a9b1d3f6_3_0_16.py | 856 ++++++++++++++ docs/architecture-optimization-checklist.md | 40 +- docs/architecture-overview.md | 4 +- docs/architecture-refactor-roadmap.md | 4 +- docs/mcp-api.md | 3 + docs/rules/05-architecture.md | 18 + tests/conftest.py | 6 + .../architecture/dependency-baseline.json | 61 +- .../fixtures/architecture/ruff-baseline.json | 15 - .../runtime-contract-baseline.json | 7 +- tests/test_architecture_dependencies.py | 2 + tests/test_chain_durable_events.py | 876 +++++++++++++- tests/test_db_transferhistory_queries.py | 60 +- tests/test_db_transferpending_queries.py | 2 + tests/test_history_ai_retry_gate.py | 437 +++++++ tests/test_history_batch_ai_redo_prompt.py | 19 + tests/test_history_mutation_command.py | 17 + tests/test_legacy_db_behavior_compat.py | 45 + tests/test_transfer_durable_retry_owner.py | 213 ++++ tests/test_transfer_execution_migration.py | 769 ++++++++++++ tests/test_transfer_execution_persistence.py | 454 ++++++++ tests/test_transfer_execution_runner.py | 217 ++++ tests/test_transfer_lease_migration.py | 24 +- tests/test_transfer_lease_persistence.py | 2 + tests/test_transfer_legacy_terminal_compat.py | 264 +++++ tests/test_transfer_manual_review_api.py | 201 ++++ .../test_transfer_manual_review_discovery.py | 250 ++++ tests/test_transfer_overwrite_declined.py | 117 ++ tests/test_transfer_pending_legacy_compat.py | 286 ++++- tests/test_transfer_planning_checkpoint.py | 60 +- tests/test_transfer_planning_migration.py | 30 +- tests/test_transfer_planning_persistence.py | 28 + tests/test_transfer_queue_service.py | 2 + tests/test_transfer_settling_recovery.py | 456 ++++++++ 69 files changed, 11948 insertions(+), 275 deletions(-) create mode 100644 app/application/transfer_execution.py create mode 100644 app/db/adapters/transfer_execution.py create mode 100644 app/db/models/transferexecutionstep.py create mode 100644 app/db/models/transfersettlementreceipt.py create mode 100644 app/db/oper/transferexecutionstep.py create mode 100644 app/db/oper/transfersettlementreceipt.py create mode 100644 database/versions/e5c7a9b1d3f6_3_0_16.py create mode 100644 tests/test_history_ai_retry_gate.py create mode 100644 tests/test_transfer_durable_retry_owner.py create mode 100644 tests/test_transfer_execution_migration.py create mode 100644 tests/test_transfer_execution_persistence.py create mode 100644 tests/test_transfer_execution_runner.py create mode 100644 tests/test_transfer_legacy_terminal_compat.py create mode 100644 tests/test_transfer_manual_review_api.py create mode 100644 tests/test_transfer_manual_review_discovery.py create mode 100644 tests/test_transfer_settling_recovery.py diff --git a/app/agent/prompt/System Tasks.yaml b/app/agent/prompt/System Tasks.yaml index ceca65122..883f6b12d 100644 --- a/app/agent/prompt/System Tasks.yaml +++ b/app/agent/prompt/System Tasks.yaml @@ -39,10 +39,11 @@ task_types: - "Use `query_transfer_history` with status='failed' to find the record with id={history_id} and understand the failure details such as source path, error message, and media info." - "Analyze the error message to determine the best retry strategy." - "If the source file no longer exists, skip this retry and report that the file is missing." - - "Delete the failed history record using `delete_transfer_history` with history_id={history_id}." - - "Re-identify the media using `recognize_media` with the source file path. For audio files, set media_type='music' and preserve artist/title/album context." - - "If recognition fails, try `search_media` with keywords from the filename. For music, distinguish recording, album, and browse-only artist results." - - "Re-transfer using `transfer_file` with the source path and exact identity fields. Reuse media_source + media_id for every media type, plus media_type + music_type for music." + - "Call `delete_transfer_history` with history_id={history_id}. Durable records are not deleted: the tool submits them to the persistent retry scheduler." + - "If `delete_transfer_history` reports that a durable retry was accepted or rejected, stop and report the exact scheduler result. Do not call `transfer_file`; do not delete the target, history, or retry evidence." + - "Only for a legacy history that was actually deleted, re-identify the media using `recognize_media` with the source file path. For audio files, set media_type='music' and preserve artist/title/album context." + - "If legacy recognition fails, try `search_media` with keywords from the filename. For music, distinguish recording, album, and browse-only artist results." + - "Re-transfer only the deleted legacy history using `transfer_file` with the source path and exact identity fields. Reuse media_source + media_id for every media type, plus media_type + music_type for music." - "Report the final result." batch_transfer_failed_retry: header: "[System Task - Batch Transfer Failed Retry]" @@ -56,7 +57,7 @@ task_types: - "Use `query_transfer_history` with status='failed' to find all records with these IDs and understand the failure details." - "Group records by exact media identity and source directory before retrying. Do not assume all selected files belong to one media." - "If the error is about media recognition, identify each group once using `recognize_media` or `search_media`, then reuse that result inside the group. Album tracks should normally be retried from the shared album directory with the album identity." - - "For each failed record, delete the old history entry with `delete_transfer_history` and re-transfer using `transfer_file`." + - "For each failed record, call `delete_transfer_history`. Durable records are submitted to the persistent retry scheduler, and that result is final for this task: report its exact accepted or rejected state and do not call `transfer_file` or delete its evidence. Only an actually deleted legacy history may be re-transferred with `transfer_file`." - "Report how many retries succeeded and how many still failed." task_rules: - "Within one verified group, do NOT call `recognize_media` or `search_media` repeatedly for each file. A music recording is one track; a music album is one multi-track directory; an artist is never a transfer target." @@ -90,8 +91,9 @@ task_types: - "If the current recognition is wrong or the record should be reorganized, determine the correct media identity first." - "Prefer `recognize_media` with the source path. If recognition is not reliable, use `search_media` with keywords from filename, title, or year." - "Only continue when you have high confidence in the target media." - - "Before re-organizing, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file." - - "Then use `transfer_file` to organize the source path directly." + - "Before re-organizing, call `delete_transfer_history`. Durable records are not deleted: the tool submits them to the persistent retry scheduler." + - "If `delete_transfer_history` reports that a durable retry was accepted or rejected, stop for that record and report the exact scheduler result. Do not call `transfer_file`; do not delete the target, history, or retry evidence. Only a legacy history deletion may be followed by `transfer_file`." + - "For a legacy history that was actually deleted, use `transfer_file` to organize the source path directly." - "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. For an album, retry the album directory once with the album identity when the records share that directory." - "If this record is already correct and no re-organize is needed, do not perform destructive actions; simply report that no change is necessary." task_rules: @@ -116,8 +118,9 @@ task_types: - "For each group, decide whether the current recognition is trustworthy." - "If multiple records clearly belong to the same movie, series, or music album, identify the media once with `recognize_media` or `search_media`, then reuse that result for the related records. A recording remains a single-track target, and an artist is browse-only." - "If a source file no longer exists or cannot be safely processed, skip that record and note the reason." - - "Before re-organizing a record, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file." - - "Then use `transfer_file` to organize the source path directly." + - "Before re-organizing each record, call `delete_transfer_history`. Durable records are not deleted: the tool submits each task to the persistent retry scheduler." + - "For each durable result, report the exact accepted or rejected scheduler state and stop processing that record. Do not call `transfer_file`; do not delete its target, history, or retry evidence." + - "Only after a legacy history was actually deleted, use `transfer_file` to organize that source path directly." - "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. Prefer one directory transfer for a verified complete album instead of treating each track as an unrelated media item." - "If a record is already correct and no re-organize is needed, do not perform destructive actions; simply mark it as skipped." - "Report only the aggregate outcome, including how many records succeeded, skipped, and failed." diff --git a/app/agent/tools/impl/delete_transfer_history.py b/app/agent/tools/impl/delete_transfer_history.py index 50e58810e..3e43128ac 100644 --- a/app/agent/tools/impl/delete_transfer_history.py +++ b/app/agent/tools/impl/delete_transfer_history.py @@ -7,8 +7,13 @@ from pydantic import BaseModel, Field from app.agent.tools.base import MoviePilotTool from app.agent.tools.tags import ToolTag -from app.chain.storage import StorageChain from app.application.agentdata import get_agent_transfer_history_port +from app.application.chain.data import get_chain_transfer_execution_port +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferRetryRequestResult, +) +from app.chain.storage import StorageChain from app.runtime.log import logger from app.schemas.workflow import FileItem @@ -29,6 +34,22 @@ def _delete_history_destination_file(fileitem: FileItem) -> tuple[bool, bool]: return True, bool(storage_chain.delete_media_file(fileitem)) +def _request_transfer_retry( + *, + history_id: int, + task_id: str, + user_id: str, +) -> TransferRetryRequestResult: + """在线程池中登记 durable 重试，避免 Agent 事件循环执行同步数据库 I/O。""" + return TransferExecutionCommand( + get_chain_transfer_execution_port() + ).request_retry( + task_id=task_id, + reason=f"Agent 请求重试整理历史 #{history_id}", + requested_by=f"agent:{user_id or 'unknown'}", + ) + + class DeleteTransferHistoryTool(MoviePilotTool): name: str = "delete_transfer_history" tags: list[str] = [ @@ -37,9 +58,10 @@ class DeleteTransferHistoryTool(MoviePilotTool): ToolTag.Admin, ] description: str = ( - "Delete a specific transfer history record by its ID. For non-successful-move records with an old " - "destination file, the tool removes that media-library file before deleting the history record. This is " - "useful before retrying or re-organizing because the system skips files that already have transfer history." + "Request a safe retry for durable transfer history, or delete a legacy transfer history record by its ID. " + "Durable records keep their files and history and are retried only by the persistent scheduler. For legacy " + "non-successful-move records, the tool removes the old destination before deleting the history. If a durable " + "retry is accepted or rejected, stop and report that result; do not call transfer_file for the same record." ) args_schema: Type[BaseModel] = DeleteTransferHistoryInput require_admin: bool = True @@ -58,6 +80,22 @@ class DeleteTransferHistoryTool(MoviePilotTool): if not history: return f"错误：整理历史记录不存在，ID={history_id}" + task_id = getattr(history, "transfer_task_id", None) + if task_id: + retry = await self.run_blocking( + "db", + _request_transfer_retry, + history_id=history_id, + task_id=task_id, + user_id=self._user_id, + ) + outcome = "已登记" if retry.accepted else "未登记" + return ( + f"durable 整理任务{outcome}重试：ID={history_id}，" + f"task_id={task_id}，state={retry.state.value}，{retry.message}。" + "已保留目标文件、历史记录和失败计数；不要调用 transfer_file。" + ) + title = history.title or "未知" src = history.src or "未知" status = "成功" if history.status else "失败" diff --git a/app/api/endpoints/history.py b/app/api/endpoints/history.py index 49f742c76..973feeaf4 100644 --- a/app/api/endpoints/history.py +++ b/app/api/endpoints/history.py @@ -1,3 +1,4 @@ +import asyncio import time from collections.abc import Coroutine from typing import Any, Callable, List, Optional @@ -27,12 +28,17 @@ from app.api.dependencies.history import ( ) from app.api.response import ResponseAPIRouter from app.application.agent import get_running_agent_manager +from app.application.chain.data import get_chain_transfer_execution_port from app.application.configuration import ApiRuntimeConfig from app.application.history import ( DownloadHistoryMutationCommand, HistoryQueryService, TransferHistoryMutationCommand, ) +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferRetryRequestResult, +) from app.runtime.config import global_vars from app.runtime.log import logger from app.runtime.progress import AsyncProgressHelper @@ -58,6 +64,120 @@ def normalize_history_ids(history_ids: list[int]) -> list[int]: return normalized_ids +def _request_durable_transfer_retry( + *, + history_id: int, + task_id: str, + requested_by: str, +) -> TransferRetryRequestResult: + """把 durable 历史重试交给唯一持久调度器，不在请求线程执行整理。""" + return TransferExecutionCommand( + get_chain_transfer_execution_port() + ).request_retry( + task_id=task_id, + reason=f"AI REST 请求重试整理历史 #{history_id}", + requested_by=requested_by, + ) + + +def _format_retry_rejections( + rejections: list[tuple[int, TransferRetryRequestResult]], +) -> str: + """把批量 durable 重试拒绝原因格式化为可审计的接口提示。""" + return "；".join( + f"#{history_id} [{result.state.value}]: {result.message}" + for history_id, result in rejections + ) + + +def _partition_durable_histories( + histories: list[_SchemaTransferHistory], +) -> tuple[list[_SchemaTransferHistory], list[_SchemaTransferHistory]]: + """按是否绑定持久任务回执分离 durable 与旧整理历史。""" + return ( + [history for history in histories if history.transfer_task_id], + [history for history in histories if not history.transfer_task_id], + ) + + +async def _request_batch_durable_retries( + histories: list[_SchemaTransferHistory], +) -> tuple[int, list[tuple[int, TransferRetryRequestResult]]]: + """逐任务登记 durable 重试并保留每条拒绝的稳定状态。""" + accepted_count = 0 + rejections: list[tuple[int, TransferRetryRequestResult]] = [] + for history in histories: + retry = await asyncio.to_thread( + _request_durable_transfer_retry, + history_id=history.id, + task_id=history.transfer_task_id or "", + requested_by="history_ai_redo_batch", + ) + if retry.accepted: + accepted_count += 1 + else: + rejections.append((history.id, retry)) + return accepted_count, rejections + + +def _durable_retry_messages( + *, + accepted_count: int, + rejections: list[tuple[int, TransferRetryRequestResult]], +) -> list[str]: + """构造 durable 批量登记结果消息，供纯 durable 和混合请求复用。""" + messages: list[str] = [] + if accepted_count: + messages.append(f"已登记 {accepted_count} 个持久整理任务重试") + if rejections: + messages.append("以下任务未登记重试：" + _format_retry_rejections(rejections)) + return messages + + +async def _complete_durable_retry_batch( + *, + histories: list[_SchemaTransferHistory], + messages: list[str], + rejections: list[tuple[int, TransferRetryRequestResult]], +) -> Any: + """完成纯 durable 批量响应；存在拒绝时不伪造成功进度。""" + message = "；".join(messages) + if rejections: + return _SchemaResponse(success=False, message=message) + progress_key = f"transfer_retry_batch_{int(time.time() * 1000)}" + history_ids = [history.id for history in histories] + await _complete_durable_retry_progress( + progress_key=progress_key, + text=message, + history_ids=history_ids, + ) + return _SchemaResponse( + success=True, + message=message, + data={"progress_key": progress_key, "history_ids": history_ids}, + ) + + +async def _complete_durable_retry_progress( + *, + progress_key: str, + text: str, + history_ids: list[int], +) -> None: + """写入可被现有 SSE 客户端立即消费的 durable 重试完成进度。""" + progress = AsyncProgressHelper(progress_key) + await progress.start() + await progress.end( + text=text, + data={ + "history_ids": history_ids, + "success": True, + "completed": True, + "message": text, + }, + ) + + def _build_progress_output_callback( progress: AsyncProgressHelper, data: dict[str, Any], @@ -285,13 +405,34 @@ async def ai_redo_transfer_history( 手动触发单条历史记录的 AI 重新整理，并返回进度键。 """ runtime_config = resolve_api_runtime_config(runtime_config) - if not runtime_config.ai_agent_enable: - return _SchemaResponse(success=False, message="MoviePilot智能助手未启用") - history = await query.get_transfer(history_id) if not history: return _SchemaResponse(success=False, message="整理记录不存在") + if history.transfer_task_id: + retry = await asyncio.to_thread( + _request_durable_transfer_retry, + history_id=history.id, + task_id=history.transfer_task_id, + requested_by="history_ai_redo", + ) + if not retry.accepted: + return _SchemaResponse(success=False, message=retry.message) + progress_key = f"transfer_retry_{history_id}_{int(time.time() * 1000)}" + await _complete_durable_retry_progress( + progress_key=progress_key, + text=retry.message, + history_ids=[history.id], + ) + return _SchemaResponse( + success=True, + message=retry.message, + data={"progress_key": progress_key}, + ) + + if not runtime_config.ai_agent_enable: + return _SchemaResponse(success=False, message="MoviePilot智能助手未启用") + prompt = build_manual_redo_prompt(history) progress_key = f"ai_redo_transfer_{history_id}_{int(time.time() * 1000)}" _start_ai_redo_task( @@ -320,9 +461,6 @@ async def batch_ai_redo_transfer_history( 手动触发多条历史记录的 AI 批量重新整理，并返回进度键。 """ runtime_config = resolve_api_runtime_config(runtime_config) - if not runtime_config.ai_agent_enable: - return _SchemaResponse(success=False, message="MoviePilot智能助手未启用") - history_ids = normalize_history_ids(payload.history_ids) if not history_ids: return _SchemaResponse(success=False, message="未提供有效的整理记录") @@ -336,18 +474,59 @@ async def batch_ai_redo_transfer_history( + ", ".join(str(history_id) for history_id in missing_ids), ) - prompt = build_batch_manual_redo_prompt(histories) + durable_histories, legacy_histories = _partition_durable_histories(histories) + accepted_count, rejections = await _request_batch_durable_retries( + durable_histories + ) + response_message_parts = _durable_retry_messages( + accepted_count=accepted_count, + rejections=rejections, + ) + + if not legacy_histories: + return await _complete_durable_retry_batch( + histories=durable_histories, + messages=response_message_parts, + rejections=rejections, + ) + + if rejections: + response_message_parts.append( + f"{len(legacy_histories)} 条旧历史未提交：批量请求包含被拒绝的持久任务" + ) + return _SchemaResponse( + success=False, + message="；".join(response_message_parts), + ) + + if not runtime_config.ai_agent_enable: + response_message_parts.append( + f"{len(legacy_histories)} 条旧历史未处理：MoviePilot智能助手未启用" + ) + return _SchemaResponse( + success=False, + message="；".join(response_message_parts), + ) + + prompt = build_batch_manual_redo_prompt(legacy_histories) progress_key = f"ai_redo_transfer_batch_{int(time.time() * 1000)}" _start_batch_ai_redo_task( - history_ids=history_ids, + history_ids=[history.id for history in legacy_histories], prompt=prompt, progress_key=progress_key, task_registry=task_registry, ) + response_message_parts.append( + f"已提交 {len(legacy_histories)} 条旧历史给智能助手处理" + ) return _SchemaResponse( - success=True, - data={"progress_key": progress_key, "history_ids": history_ids}, + success=not rejections, + message="；".join(response_message_parts), + data={ + "progress_key": progress_key, + "history_ids": [history.id for history in histories], + }, ) diff --git a/app/api/endpoints/transfer.py b/app/api/endpoints/transfer.py index 1611e8b57..1caf9314c 100644 --- a/app/api/endpoints/transfer.py +++ b/app/api/endpoints/transfer.py @@ -1,15 +1,25 @@ from pathlib import Path -from typing import Annotated, Any, List, Optional +from typing import Annotated, Any, List, Literal, Optional, cast -from fastapi import Depends +from fastapi import Depends, HTTPException, Query, status from app.adapters.web.security.access import verify_apitoken, verify_token from app.api.dependencies.auth import get_current_active_manage_user from app.api.dependencies.history import get_transfer_history_lookup_service from app.api.response import ResponseAPIRouter +from app.application.chain.data import get_chain_transfer_execution_port from app.application.configuration import get_api_runtime_config_snapshot from app.application.directory import DirectoryHelper from app.application.history import TransferHistoryLookupService +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferExecutionConflictError, + TransferExecutionState, + TransferManualReviewDecision, + TransferManualReviewQuery, + TransferManualReviewTaskView, + TransferStepResult, +) from app.chain.media import MediaChain from app.chain.transfer import TransferChain from app.runtime.log import logger @@ -25,6 +35,10 @@ from app.schemas.transfer import ManualTransferHistoryInfo as _SchemaManualTrans from app.schemas.transfer import ManualTransferResultData as _SchemaManualTransferResultData from app.schemas.transfer import ManualTransferTargetPath as _SchemaManualTransferTargetPath from app.schemas.transfer import TransferJob as _SchemaTransferJob +from app.schemas.transfer import TransferManualReviewData as _SchemaTransferManualReviewData +from app.schemas.transfer import TransferManualReviewPageData as _SchemaTransferManualReviewPageData +from app.schemas.transfer import TransferManualReviewRequest as _SchemaTransferManualReviewRequest +from app.schemas.transfer import TransferManualReviewTaskData as _SchemaTransferManualReviewTaskData from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaType from app.schemas.workflow import FileItem from app.schemas.workflow import FileItem as _SchemaFileItem @@ -32,6 +46,143 @@ from app.schemas.workflow import FileItem as _SchemaFileItem router = ResponseAPIRouter() +def _manual_review_actor(current_user: object) -> str: + """按名称、用户名和用户 ID 的稳定顺序提取人工复核操作者。""" + for attribute in ("name", "username", "id"): + value = getattr(current_user, attribute, None) + if value is not None and str(value).strip(): + return str(value).strip() + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="当前管理用户缺少可审计身份", + ) + + +def _manual_review_task_data( + task: TransferManualReviewTaskView, +) -> _SchemaTransferManualReviewTaskData: + """把 Application 人工复核投影映射为严格公开响应。""" + return cast( + _SchemaTransferManualReviewTaskData, + _SchemaTransferManualReviewTaskData.model_validate({ + "task_id": task.task_id, + "source": { + "storage": task.source.storage, + "path": task.source.path, + }, + "state": task.state.value, + "step": { + "operation_id": task.step.operation_id, + "kind": task.step.kind, + "intent": task.step.intent, + "evidence": task.step.evidence, + "error": task.step.error, + }, + "review_revision": task.review_revision, + }), + ) + + +@router.get( # type: ignore[misc] + "/tasks/manual-reviews", + summary="分页查询 durable 整理人工复核任务", + response_model=_SchemaResponse[_SchemaTransferManualReviewPageData], +) +def list_transfer_manual_reviews( + state_filter: Literal["manual_review", "retry_wait"] = Query( + default="manual_review", + alias="state", + ), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=30, ge=1, le=100), + current_user: object = Depends(get_current_active_manage_user), +) -> Any: + """分页返回待复核或已判定等待 durable 恢复的任务。""" + del current_user + result = TransferManualReviewQuery( + get_chain_transfer_execution_port() + ).list( + state=TransferExecutionState(state_filter), + page=page, + page_size=page_size, + ) + return _SchemaResponse( + success=True, + data=_SchemaTransferManualReviewPageData( + items=[_manual_review_task_data(item) for item in result.items], + total=result.total, + page=result.page, + page_size=result.page_size, + ), + ) + + +@router.get( # type: ignore[misc] + "/tasks/{task_id}/manual-review", + summary="查询 durable 整理人工复核详情", + response_model=_SchemaResponse[_SchemaTransferManualReviewTaskData], +) +def get_transfer_manual_review( + task_id: str, + current_user: object = Depends(get_current_active_manage_user), +) -> Any: + """按任务标识返回严格裁剪的人工复核详情。""" + del current_user + task = TransferManualReviewQuery( + get_chain_transfer_execution_port() + ).get(task_id=task_id) + if task is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="人工复核任务不存在", + ) + return _SchemaResponse(success=True, data=_manual_review_task_data(task)) + + +@router.post( # type: ignore[misc] + "/tasks/{task_id}/manual-review", + summary="人工判定整理步骤的外部执行结果", + response_model=_SchemaResponse[_SchemaTransferManualReviewData], +) +def resolve_transfer_manual_review( + task_id: str, + review: _SchemaTransferManualReviewRequest, + current_user: object = Depends(get_current_active_manage_user), +) -> Any: + """提交无租约人工判定，并返回不含 attempt 与 lease 的公开状态。""" + result = ( + TransferStepResult(payload=dict(review.result_payload)) + if review.result_payload is not None + else None + ) + try: + resolved = TransferExecutionCommand( + get_chain_transfer_execution_port() + ).resolve_manual_review( + task_id=task_id, + operation_id=review.operation_id, + decision=TransferManualReviewDecision(review.decision), + actor=_manual_review_actor(current_user), + reason=review.reason, + result=result, + ) + except TransferExecutionConflictError as error: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(error), + ) from error + return _SchemaResponse( + success=True, + data=_SchemaTransferManualReviewData( + task_id=resolved.task_id, + operation_id=resolved.operation_id, + decision=resolved.decision.value, + state=resolved.state.value, + review_revision=resolved.review_revision, + ), + ) + + @router.get( "/name", summary="查询整理后的名称", diff --git a/app/application/chain/data.py b/app/application/chain/data.py index 645ab0c1f..f5fd5addf 100644 --- a/app/application/chain/data.py +++ b/app/application/chain/data.py @@ -11,10 +11,12 @@ from dataclasses import dataclass from typing import Any, Optional from app.application.transfer import TransferAdmissionRepository +from app.application.transfer_execution import TransferExecutionRepository OperFactory = Callable[[], Any] TransferAdmissionRepositoryFactory = Callable[[], TransferAdmissionRepository] +TransferExecutionRepositoryFactory = Callable[[], TransferExecutionRepository] @dataclass(frozen=True, slots=True) @@ -27,6 +29,7 @@ class ChainDataPorts: download_history: OperFactory transfer_history: OperFactory transfer_pending: TransferAdmissionRepositoryFactory + transfer_execution: TransferExecutionRepositoryFactory media_server: OperFactory download_failure: OperFactory user: OperFactory @@ -110,6 +113,7 @@ def configure_chain_data_ports(**factories: OperFactory) -> None: "download_history", "transfer_history", "transfer_pending", + "transfer_execution", "media_server", "download_failure", "user", @@ -158,6 +162,11 @@ def get_chain_transfer_pending_port() -> TransferAdmissionRepository: return get_chain_data_ports().transfer_pending() +def get_chain_transfer_execution_port() -> TransferExecutionRepository: + """创建类型化的整理步骤执行与终态结算仓储。""" + return get_chain_data_ports().transfer_execution() + + def get_chain_media_server_port() -> Any: """创建媒体服务器数据端口实例。""" return get_chain_data_ports().media_server() diff --git a/app/application/chain/durable_events.py b/app/application/chain/durable_events.py index d12343ee2..848f7337e 100644 --- a/app/application/chain/durable_events.py +++ b/app/application/chain/durable_events.py @@ -11,6 +11,7 @@ from typing import Any, Protocol, cast from uuid import uuid4 from app.application.history import TransferHistoryRecord, TransferHistoryWriter +from app.application.transfer_execution import TransferSettlementResult from app.domain.context import Context, MediaInfo, MusicInfo, TorrentInfo from app.domain.meta.metabase import MetaBase from app.domain.meta.metamusic import MetaMusic @@ -37,12 +38,33 @@ class ChainDurableEventWriter(Protocol): def transfer_result( self, *, - topic: str, + topic: str | None, stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None], event_payload: dict[str, Any], - publish: Callable[[dict[str, Any]], None], - ) -> TransferHistoryRecord | None: - """提交整理历史与结果 intent，并在提交后广播兼容事件。""" + publish: Callable[[dict[str, Any]], None] | None, + settlement: "TransferResultSettlement | None" = None, + ) -> TransferHistoryRecord | TransferSettlementResult | None: + """提交历史、可选任务终态和结果 intent，再按 topic 广播事件。""" + + +@dataclass(frozen=True, slots=True) +class TransferResultSettlement: + """描述一次受 lease fencing 保护的整理任务终态结算。""" + + task_id: str + lease_token: str + execution_fingerprint: str + outcome: str + error: str | None = None + + def __post_init__(self) -> None: + """拒绝缺少稳定身份、非法结果或不可诊断的失败终态。""" + if not self.task_id or not self.lease_token or not self.execution_fingerprint: + raise ValueError("整理终态结算缺少任务、租约或执行检查点身份") + if self.outcome not in {"succeeded", "failed"}: + raise ValueError(f"不支持的整理终态：{self.outcome}") + if self.outcome == "failed" and not self.error: + raise ValueError("整理失败终态必须包含可诊断原因") @dataclass(frozen=True, slots=True) @@ -61,8 +83,23 @@ def download_added_event_key(history_id: int) -> str: return f"download.added:{history_id}:{uuid4().hex}:v1" -def transfer_result_event_key(topic: str, history_id: int) -> str: - """由结果 topic、整理历史 ID 与本次事实标识构造幂等键。""" +def transfer_result_event_key( + topic: str, + history_id: int, + *, + settlement: TransferResultSettlement | None = None, + settlement_revision: int | None = None, +) -> str: + """为旧结果事实生成 occurrence key，为任务结算生成确定性幂等键。""" + if settlement is not None: + if settlement_revision is None or settlement_revision <= 0: + raise ValueError("整理终态事件键缺少有效结算修订号") + return ( + f"transfer.result:{settlement.task_id}:{settlement_revision}:" + f"{settlement.outcome}:v1" + ) + if settlement_revision is not None: + raise ValueError("旧整理结果事件键不能单独指定结算修订号") return f"{topic}:{history_id}:{uuid4().hex}:v1" diff --git a/app/application/history.py b/app/application/history.py index af2fa9618..ace1f7763 100644 --- a/app/application/history.py +++ b/app/application/history.py @@ -40,6 +40,7 @@ class TransferHistoryRecord(Protocol): src: Optional[str] src_storage: Optional[str] src_fileitem: Optional[dict] + transfer_task_id: Optional[str] class TransferHistoryWriter(Protocol): @@ -414,6 +415,11 @@ class TransferHistoryMutationCommand: history = self._repository.get(history_id) if not history: return HistoryMutationResult(False, "记录不存在") + if getattr(history, "transfer_task_id", None): + return HistoryMutationResult( + False, + "持久整理失败记录不可删除，请使用重试或人工复核入口", + ) if delete_destination and history.dest_fileitem: destination = self._file_item_factory(history.dest_fileitem) @@ -440,10 +446,10 @@ class TransferHistoryMutationCommand: return HistoryMutationResult(True) def truncate(self) -> HistoryMutationResult: - """在单一事务中清空全部整理历史。""" + """在单一事务中清空旧历史，并保留当前失败任务记录。""" self._repository.stage_truncate() self._commit() - return HistoryMutationResult(True) + return HistoryMutationResult(True, "已清空旧整理记录，失败任务记录已保留") def _commit(self) -> None: """提交历史事务，失败时回滚且不发布事件或清缓存。""" diff --git a/app/application/outbox.py b/app/application/outbox.py index 793810d6f..5feaee870 100644 --- a/app/application/outbox.py +++ b/app/application/outbox.py @@ -167,16 +167,18 @@ class DurableEventCommand: def execute( self, *, - intent: OutboxIntent | Callable[[T], OutboxIntent], + intent: OutboxIntent | Callable[[T], OutboxIntent] | None, stage_business: Callable[[], T], - publish: Callable[[], None], + publish: Callable[[], None] | None, after_commit: Callable[[], None] | None = None, ) -> T: - """先原子提交业务与 intent，再保持原顺序执行提交后动作和即时广播。""" + """原子提交业务与可选 intent，再执行可选提交后动作和广播。""" + resolved_intent: OutboxIntent | None = None try: result = stage_business() - resolved_intent = intent(result) if callable(intent) else intent - self._outbox.stage(resolved_intent, datetime.now(timezone.utc)) + if intent is not None: + resolved_intent = intent(result) if callable(intent) else intent + self._outbox.stage(resolved_intent, datetime.now(timezone.utc)) self._unit_of_work.commit() except Exception: self._unit_of_work.rollback() @@ -184,11 +186,13 @@ class DurableEventCommand: if after_commit: after_commit() - publish() - self._outbox.complete_by_event_key( - resolved_intent.event_key, - datetime.now(timezone.utc), - ) + if publish: + publish() + if resolved_intent is not None and publish is not None: + self._outbox.complete_by_event_key( + resolved_intent.event_key, + datetime.now(timezone.utc), + ) return result diff --git a/app/application/transfer.py b/app/application/transfer.py index 5d01871ca..8e165a174 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -37,6 +37,7 @@ from pydantic import BaseModel, ConfigDict, PrivateAttr from app.adapters.system.host import SystemUtils from app.application.agent import get_prompt_manager, get_running_agent_manager +from app.application.transfer_execution import TransferExecutionCheckpoint from app.domain.context import MediaInfo, MusicInfo from app.domain.media import normalize_music_type from app.domain.meta.metabase import MetaBase @@ -668,6 +669,8 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): _admission_task_id: Optional[str] = PrivateAttr(default=None) _planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None) _plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None) + _execution_checkpoint: Optional[TransferExecutionCheckpoint] = PrivateAttr(default=None) + _terminal_settled: bool = PrivateAttr(default=False) _planning_context_restored: bool = PrivateAttr(default=False) _lease_owner: Optional[str] = PrivateAttr(default=None) _lease_token: Optional[str] = PrivateAttr(default=None) @@ -699,6 +702,26 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): """绑定持久执行检查点，不改变插件可见序列化字段。""" self._plan_checkpoint = checkpoint + @property + def execution_checkpoint(self) -> Optional[TransferExecutionCheckpoint]: + """返回仅供宿主终态结算使用的内部执行检查点。""" + return self._execution_checkpoint + + def bind_execution_checkpoint( + self, checkpoint: TransferExecutionCheckpoint + ) -> None: + """绑定执行结果检查点，不改变插件可见的旧任务序列化字段。""" + self._execution_checkpoint = checkpoint + + @property + def terminal_settled(self) -> bool: + """返回历史、事件与 pending 是否已由同一 UoW 提交。""" + return self._terminal_settled + + def mark_terminal_settled(self) -> None: + """仅在 task-aware writer 成功返回后标记终态已经提交。""" + self._terminal_settled = True + @property def planning_context_restored(self) -> bool: """返回当前领域上下文是否来自持久快照。""" diff --git a/app/application/transfer_execution.py b/app/application/transfer_execution.py new file mode 100644 index 000000000..335dc5fcf --- /dev/null +++ b/app/application/transfer_execution.py @@ -0,0 +1,884 @@ +"""整理步骤执行检查点、人工判定与终态结算的应用契约。""" + +from __future__ import annotations + +import hashlib +import json +from copy import deepcopy +from dataclasses import dataclass +from enum import StrEnum +from typing import Any, Callable, Mapping, Optional, Protocol +from uuid import uuid4 + +TRANSFER_EXECUTION_VERSION = 1 +TRANSFER_STEP_INTENT_VERSION = 1 +TRANSFER_STEP_RESULT_VERSION = 1 + + +class TransferExecutionState(StrEnum): + """描述 planning phase 正交的持久执行状态。""" + + NOT_STARTED = "not_started" + RUNNING = "running" + RETRY_WAIT = "retry_wait" + SETTLING = "settling" + FAILED = "failed" + MANUAL_REVIEW = "manual_review" + + +class TransferStepState(StrEnum): + """描述单个稳定外部操作的持久执行状态。""" + + PREPARED = "prepared" + STARTED = "started" + SUCCEEDED = "succeeded" + FAILED = "failed" + MANUAL_REVIEW = "manual_review" + + +class TransferTerminalState(StrEnum): + """描述允许写入整理历史的确定终态。""" + + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class TransferOperationObservationState(StrEnum): + """描述重启后对遗留 STARTED 外部操作的严格探测结论。""" + + APPLIED = "applied" + NOT_APPLIED = "not_applied" + UNKNOWN = "unknown" + CONFLICT = "conflict" + + +class TransferManualReviewDecision(StrEnum): + """描述人工对外部结果不确定步骤作出的显式判定。""" + + NOT_APPLIED = "not_applied" + APPLIED = "applied" + FAILED = "failed" + + +class TransferExecutionError(RuntimeError): + """表示整理执行持久化状态无法按请求推进。""" + + +class TransferExecutionConflictError(TransferExecutionError): + """表示稳定操作身份、尝试身份或检查点证据发生冲突。""" + + +class TransferExecutionLeaseLostError(TransferExecutionError): + """表示持久化写入时任务租约已失效或已被其他 worker 接管。""" + + +def _canonical_json(payload: Mapping[str, Any]) -> str: + """生成稳定操作身份使用的规范 JSON。""" + return json.dumps( + payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + + +def build_transfer_operation_id( + *, + task_id: str, + checkpoint_fingerprint: str, + ordinal: int, + phase: str, + kind: str, + intent_payload: Mapping[str, Any], +) -> str: + """ + 由冻结计划和操作语义生成跨 lease、attempt 与重启稳定的身份。 + + :param task_id: 稳定任务标识 + :param checkpoint_fingerprint: 冻结计划指纹 + :param ordinal: 全局执行序号 + :param phase: 执行阶段 + :param kind: 操作类型 + :param intent_payload: 冻结操作参数 + :return: SHA-256 操作标识 + """ + if not task_id or not checkpoint_fingerprint or ordinal < 0 or not phase or not kind: + raise ValueError("整理操作身份缺少稳定任务、计划或步骤信息") + canonical = _canonical_json({ + "schema_version": TRANSFER_STEP_INTENT_VERSION, + "task_id": task_id, + "checkpoint_fingerprint": checkpoint_fingerprint, + "ordinal": ordinal, + "phase": phase, + "kind": kind, + "intent_payload": dict(intent_payload), + }) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def build_transfer_checkpoint_fingerprint( + checkpoint_payload: Mapping[str, Any], +) -> str: + """由版本化执行检查点 payload 生成稳定 SHA-256 指纹。""" + return hashlib.sha256( + _canonical_json(checkpoint_payload).encode("utf-8") + ).hexdigest() + + +@dataclass(frozen=True, slots=True) +class TransferStepIntent: + """保存一次可持久化外部操作的稳定意图。""" + + operation_id: str + checkpoint_fingerprint: str + ordinal: int + phase: str + kind: str + payload: dict[str, Any] + version: int = TRANSFER_STEP_INTENT_VERSION + + def __post_init__(self) -> None: + """拒绝版本未知、身份不完整或不可 JSON 序列化的操作意图。""" + object.__setattr__(self, "payload", deepcopy(self.payload)) + if self.version != TRANSFER_STEP_INTENT_VERSION: + raise ValueError(f"不支持的整理步骤意图版本：{self.version}") + if ( + not self.operation_id + or not self.checkpoint_fingerprint + or self.ordinal < 0 + or not self.phase + or not self.kind + ): + raise ValueError("整理步骤意图缺少稳定身份或顺序") + _canonical_json(self.payload) + + @classmethod + def create( + cls, + *, + task_id: str, + checkpoint_fingerprint: str, + ordinal: int, + phase: str, + kind: str, + payload: Mapping[str, Any], + ) -> "TransferStepIntent": + """构造并绑定稳定 operation ID 的步骤意图。""" + frozen_payload = dict(payload) + return cls( + operation_id=build_transfer_operation_id( + task_id=task_id, + checkpoint_fingerprint=checkpoint_fingerprint, + ordinal=ordinal, + phase=phase, + kind=kind, + intent_payload=frozen_payload, + ), + checkpoint_fingerprint=checkpoint_fingerprint, + ordinal=ordinal, + phase=phase, + kind=kind, + payload=frozen_payload, + ) + + +@dataclass(frozen=True, slots=True) +class TransferStepResult: + """保存严格执行或探测后得到的版本化步骤证据。""" + + payload: dict[str, Any] + version: int = TRANSFER_STEP_RESULT_VERSION + + def __post_init__(self) -> None: + """拒绝未知版本或不可 JSON 序列化的步骤结果。""" + object.__setattr__(self, "payload", deepcopy(self.payload)) + if self.version != TRANSFER_STEP_RESULT_VERSION: + raise ValueError(f"不支持的整理步骤结果版本：{self.version}") + _canonical_json(self.payload) + + +@dataclass(frozen=True, slots=True) +class TransferOperationObservation: + """保存外部操作探测结论及其版本化事实证据。""" + + state: TransferOperationObservationState + evidence: TransferStepResult + + +@dataclass(frozen=True, slots=True) +class TransferExecutionStep: + """提供脱离 ORM Session 的单步骤持久状态投影。""" + + task_id: str + operation_id: str + checkpoint_fingerprint: str + ordinal: int + phase: str + kind: str + state: TransferStepState + attempt_token: Optional[str] + attempt_count: int + intent: TransferStepIntent + result: Optional[TransferStepResult] + last_error: Optional[str] + prepared_at: str + started_at: Optional[str] + completed_at: Optional[str] + updated_at: str + + +@dataclass(frozen=True, slots=True) +class TransferExecutionCheckpoint: + """保存所有必要步骤完成后供终态结算重放的聚合结果。""" + + fingerprint: str + payload: dict[str, Any] + operation_ids: tuple[str, ...] + skip_reason: Optional[str] = None + version: int = TRANSFER_EXECUTION_VERSION + + def __post_init__(self) -> None: + """拒绝重复步骤、空指纹和不可序列化的执行检查点。""" + object.__setattr__(self, "payload", deepcopy(self.payload)) + if self.version != TRANSFER_EXECUTION_VERSION or not self.fingerprint: + raise ValueError("整理执行检查点版本或指纹无效") + if len(set(self.operation_ids)) != len(self.operation_ids): + raise ValueError("整理执行检查点不能引用重复操作") + if not all(isinstance(item, str) and item for item in self.operation_ids): + raise ValueError("整理执行检查点包含无效操作标识") + if not self.operation_ids and not self.skip_reason: + raise ValueError("零副作用整理执行检查点必须记录 skip_reason") + _canonical_json(self.payload) + if build_transfer_checkpoint_fingerprint(self.to_payload()) != self.fingerprint: + raise ValueError("整理执行检查点内容与指纹不一致") + + @classmethod + def create( + cls, + *, + payload: Mapping[str, Any], + operation_ids: tuple[str, ...], + skip_reason: Optional[str] = None, + ) -> "TransferExecutionCheckpoint": + """构造并绑定完整序列化内容指纹的执行检查点。""" + frozen_payload = dict(payload) + serialized = { + "schema_version": TRANSFER_EXECUTION_VERSION, + "payload": frozen_payload, + "operation_ids": list(operation_ids), + "skip_reason": skip_reason, + } + return cls( + fingerprint=build_transfer_checkpoint_fingerprint(serialized), + payload=frozen_payload, + operation_ids=operation_ids, + skip_reason=skip_reason, + ) + + def to_payload(self) -> dict[str, Any]: + """编码可跨重启恢复的完整版本化执行检查点。""" + return { + "schema_version": self.version, + "payload": dict(self.payload), + "operation_ids": list(self.operation_ids), + "skip_reason": self.skip_reason, + } + + @classmethod + def from_payload( + cls, + payload: Mapping[str, Any], + *, + fingerprint: str, + ) -> "TransferExecutionCheckpoint": + """解析并校验数据库中的执行检查点版本与稳定指纹。""" + serialized = dict(payload) + if build_transfer_checkpoint_fingerprint(serialized) != fingerprint: + raise TransferExecutionConflictError("整理执行检查点 JSON 与指纹不一致") + version = serialized.get("schema_version") + if version != TRANSFER_EXECUTION_VERSION: + raise TransferExecutionConflictError("整理执行检查点版本不受支持") + result_payload = serialized.get("payload") + operation_ids = serialized.get("operation_ids") + if not isinstance(result_payload, dict) or not isinstance(operation_ids, list): + raise TransferExecutionConflictError("整理执行检查点结构无效") + if not all(isinstance(item, str) and item for item in operation_ids): + raise TransferExecutionConflictError("整理执行检查点包含无效操作标识") + skip_reason = serialized.get("skip_reason") + if skip_reason is not None and not isinstance(skip_reason, str): + raise TransferExecutionConflictError("整理执行跳过原因类型无效") + return cls( + fingerprint=fingerprint, + payload=result_payload, + operation_ids=tuple(operation_ids), + skip_reason=skip_reason, + version=version, + ) + + +@dataclass(frozen=True, slots=True) +class TransferExecutionSnapshot: + """提供任务执行 checkpoint、重试和人工状态的稳定投影。""" + + task_id: str + state: TransferExecutionState + checkpoint: Optional[TransferExecutionCheckpoint] + retry_generation: int + retry_count: int + retry_due_at: Optional[str] + settlement_revision: int + terminal_history_id: Optional[int] + last_error: Optional[str] + steps: tuple[TransferExecutionStep, ...] + + +@dataclass(frozen=True, slots=True) +class TransferSettlementIntent: + """描述已确定结果对应的历史写入与 pending 终态。""" + + terminal_state: TransferTerminalState + history_payload: dict[str, Any] + + def __post_init__(self) -> None: + """确保历史 payload 可持久化且包含稳定源身份。""" + object.__setattr__(self, "history_payload", deepcopy(self.history_payload)) + if not self.history_payload.get("src"): + raise ValueError("整理终态历史缺少源路径") + _canonical_json(self.history_payload) + + +@dataclass(frozen=True, slots=True) +class TransferSettlementResult: + """描述终态历史是否新提交以及 pending 是否已删除。""" + + history_id: int + settlement_revision: int + pending_deleted: bool + already_settled: bool = False + + +@dataclass(frozen=True, slots=True) +class TransferRetryRequestResult: + """描述用户重试请求是否被持久调度器接受。""" + + accepted: bool + state: TransferExecutionState + retry_generation: int + message: str + + +@dataclass(frozen=True, slots=True) +class TransferManualReviewResult: + """描述一次已持久审计的人工判定及后续调度状态。""" + + task_id: str + operation_id: str + decision: TransferManualReviewDecision + state: TransferExecutionState + review_revision: int + step: TransferExecutionStep + + +@dataclass(frozen=True, slots=True) +class TransferManualReviewSource: + """提供人工复核任务的最小源文件身份。""" + + storage: str + path: str + + +@dataclass(frozen=True, slots=True) +class TransferManualReviewStepView: + """提供人工复核所需且不含租约或尝试令牌的步骤证据。""" + + operation_id: str + kind: str + intent: dict[str, Any] + evidence: Optional[dict[str, Any]] + error: Optional[str] + + +@dataclass(frozen=True, slots=True) +class TransferManualReviewTaskView: + """提供管理员发现与复核 durable 整理任务的公开投影。""" + + task_id: str + source: TransferManualReviewSource + state: TransferExecutionState + step: TransferManualReviewStepView + review_revision: int + + +@dataclass(frozen=True, slots=True) +class TransferManualReviewPage: + """提供稳定分页的人工复核任务结果。""" + + items: tuple[TransferManualReviewTaskView, ...] + total: int + page: int + page_size: int + + +class TransferExecutionRepository(Protocol): + """定义步骤状态、执行 checkpoint 与终态历史的持久化端口。""" + + def get_snapshot(self, *, task_id: str) -> Optional[TransferExecutionSnapshot]: + """读取任务执行快照。""" + + def list_manual_reviews( + self, + *, + state: TransferExecutionState, + page: int, + page_size: int, + ) -> TransferManualReviewPage: + """按严格公开状态分页读取人工复核任务。""" + + def get_manual_review( + self, + *, + task_id: str, + ) -> Optional[TransferManualReviewTaskView]: + """按任务标识读取人工复核公开详情。""" + + def prepare_step( + self, + *, + task_id: str, + lease_token: str, + intent: TransferStepIntent, + ) -> TransferExecutionStep: + """在外部副作用前持久化稳定步骤意图。""" + + def start_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + ) -> TransferExecutionStep: + """以当前 lease 和新 attempt token 标记步骤开始。""" + + def restart_after_not_applied( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + previous_attempt_token: str, + attempt_token: str, + evidence: TransferStepResult, + ) -> TransferExecutionStep: + """严格探测为未发生后，以新 attempt token 安全重启遗留步骤。""" + + def resume_failed_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + ) -> TransferExecutionStep: + """任务到期并重新 claim 后，以新 attempt token 重试 FAILED 步骤。""" + + def complete_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + result: TransferStepResult, + ) -> TransferExecutionStep: + """以 lease 和 attempt 双 CAS 提交成功证据。""" + + def defer_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + retry_due_at: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """持久化已知失败、到期时间并原子释放当前 lease。""" + + def exhaust_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """重试预算耗尽时保留 lease，并建立可 durable 失败结算的检查点。""" + + def request_retry( + self, + *, + task_id: str, + reason: str, + requested_by: str, + ) -> TransferRetryRequestResult: + """仅把 FAILED 终态转入到期可 claim 的 retry_wait。""" + + def resolve_manual_review( + self, + *, + task_id: str, + operation_id: str, + decision: TransferManualReviewDecision, + actor: str, + reason: str, + result: Optional[TransferStepResult] = None, + ) -> TransferManualReviewResult: + """无 lease 地原子提交人工判定审计并交回唯一调度器。""" + + def mark_manual_review( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + error: str, + evidence: Optional[TransferStepResult] = None, + attempt_token: Optional[str] = None, + ) -> TransferExecutionSnapshot: + """持久化不可判定结果并从自动调度中隔离任务。""" + + def checkpoint_execution( + self, + *, + task_id: str, + lease_token: str, + checkpoint: TransferExecutionCheckpoint, + ) -> TransferExecutionSnapshot: + """确认所有引用步骤成功后提交聚合执行检查点。""" + +class TransferStepRunner(Protocol): + """定义文件执行方可注入的单步骤持久执行边界。""" + + def run( + self, + *, + phase: str, + kind: str, + payload: Mapping[str, Any], + execute: Callable[[], TransferStepResult], + observe: Callable[[], TransferOperationObservation], + ) -> TransferStepResult: + """持久编排一次外部副作用，并在遗留尝试时先严格探测。""" + + +class TransferExecutionCommand: + """以类型化命令收口步骤状态机，外部 I/O 由调用方在事务外执行。""" + + def __init__( + self, + repository: TransferExecutionRepository, + *, + attempt_token_factory: Callable[[], str] = lambda: uuid4().hex, + ) -> None: + """保存持久化端口和可替换的 attempt token 工厂。""" + self._repository = repository + self._attempt_token_factory = attempt_token_factory + + def prepare( + self, + *, + task_id: str, + lease_token: str, + intent: TransferStepIntent, + ) -> TransferExecutionStep: + """在外部调用前提交步骤意图。""" + return self._repository.prepare_step( + task_id=task_id, + lease_token=lease_token, + intent=intent, + ) + + def begin( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + ) -> TransferExecutionStep: + """生成单次 attempt token 并持久化开始状态。""" + attempt_token = self._attempt_token_factory() + if not attempt_token: + raise ValueError("整理步骤 attempt token 不能为空") + return self._repository.start_step( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + ) + + def restart_after_not_applied( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + evidence: TransferStepResult, + ) -> TransferExecutionStep: + """在严格 NOT_APPLIED 证据成立时轮换 attempt token 并继续执行。""" + if step.state is not TransferStepState.STARTED or not step.attempt_token: + raise ValueError("只有遗留 STARTED 步骤可按未发生证据安全重启") + attempt_token = self._attempt_token_factory() + if not attempt_token or attempt_token == step.attempt_token: + raise ValueError("安全重启必须生成不同的 attempt token") + return self._repository.restart_after_not_applied( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + previous_attempt_token=step.attempt_token, + attempt_token=attempt_token, + evidence=evidence, + ) + + def resume_failed( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + ) -> TransferExecutionStep: + """使用重试调度取得的新 lease 为 FAILED 步骤创建下一次尝试。""" + if step.state is not TransferStepState.FAILED or step.attempt_token is not None: + raise ValueError("只有已释放 attempt 的 FAILED 步骤可以恢复重试") + attempt_token = self._attempt_token_factory() + if not attempt_token: + raise ValueError("恢复重试的 attempt token 不能为空") + return self._repository.resume_failed_step( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + attempt_token=attempt_token, + ) + + def complete( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + result: TransferStepResult, + ) -> TransferExecutionStep: + """使用已开始步骤携带的 attempt token 提交成功证据。""" + if not step.attempt_token: + raise ValueError("尚未开始的整理步骤不能提交成功") + return self._repository.complete_step( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + attempt_token=step.attempt_token, + result=result, + ) + + def defer( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + error: str, + retry_due_at: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """记录已知失败并把唯一重试权交回持久调度器。""" + if not step.attempt_token: + raise ValueError("尚未开始的整理步骤不能进入重试等待") + return self._repository.defer_step( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + attempt_token=step.attempt_token, + error=error, + retry_due_at=retry_due_at, + evidence=evidence, + ) + + def exhaust( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + error: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """提交达到预算的确定失败，并把任务交给唯一 durable 终态 writer。""" + if step.state is not TransferStepState.STARTED or not step.attempt_token: + raise ValueError("只有 STARTED 步骤可提交预算耗尽失败") + return self._repository.exhaust_step( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + attempt_token=step.attempt_token, + error=error, + evidence=evidence, + ) + + def request_retry( + self, + *, + task_id: str, + reason: str, + requested_by: str, + ) -> TransferRetryRequestResult: + """登记用户重试意图，不直接 claim 或执行任务。""" + if not task_id or not reason or not requested_by: + raise ValueError("用户重试请求缺少任务、原因或请求身份") + return self._repository.request_retry( + task_id=task_id, + reason=reason, + requested_by=requested_by, + ) + + def resolve_manual_review( + self, + *, + task_id: str, + operation_id: str, + decision: TransferManualReviewDecision, + actor: str, + reason: str, + result: Optional[TransferStepResult] = None, + ) -> TransferManualReviewResult: + """提交人工判定；FAILED 在无 lease durable 结算落地前明确拒绝。""" + if not all((task_id, operation_id, actor, reason)): + raise ValueError("人工判定缺少任务、步骤、操作者或原因") + if decision is TransferManualReviewDecision.APPLIED and result is None: + raise ValueError("人工判定已发生时必须提供结果证据") + if decision is TransferManualReviewDecision.FAILED: + raise TransferExecutionConflictError( + "人工失败终态尚不能绕过 lease durable 结算" + ) + return self._repository.resolve_manual_review( + task_id=task_id, + operation_id=operation_id, + decision=decision, + actor=actor, + reason=reason, + result=result, + ) + + def manual_review( + self, + *, + task_id: str, + lease_token: str, + step: TransferExecutionStep, + error: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """隔离无法严格判断外部结果的步骤，禁止自动再次执行。""" + return self._repository.mark_manual_review( + task_id=task_id, + lease_token=lease_token, + operation_id=step.operation_id, + attempt_token=step.attempt_token, + error=error, + evidence=evidence, + ) + + def checkpoint( + self, + *, + task_id: str, + lease_token: str, + checkpoint: TransferExecutionCheckpoint, + ) -> TransferExecutionSnapshot: + """提交可独立重放终态结算的聚合执行结果。""" + return self._repository.checkpoint_execution( + task_id=task_id, + lease_token=lease_token, + checkpoint=checkpoint, + ) + + +class TransferManualReviewQuery: + """收口管理员可发现的人工复核只读用例。""" + + _visible_states = frozenset({ + TransferExecutionState.MANUAL_REVIEW, + TransferExecutionState.RETRY_WAIT, + }) + + def __init__(self, repository: TransferExecutionRepository) -> None: + """保存整理执行查询端口。""" + self._repository = repository + + @classmethod + def _validate_state(cls, state: TransferExecutionState) -> None: + """拒绝把内部执行状态扩展到人工复核查询面。""" + if state not in cls._visible_states: + raise ValueError(f"人工复核查询不支持状态：{state.value}") + + def list( + self, + *, + state: TransferExecutionState = TransferExecutionState.MANUAL_REVIEW, + page: int = 1, + page_size: int = 30, + ) -> TransferManualReviewPage: + """分页返回待复核或刚完成复核并等待恢复的任务。""" + self._validate_state(state) + if page < 1 or not 1 <= page_size <= 100: + raise ValueError("人工复核分页参数超出允许范围") + return self._repository.list_manual_reviews( + state=state, + page=page, + page_size=page_size, + ) + + def get(self, *, task_id: str) -> Optional[TransferManualReviewTaskView]: + """读取一条人工复核详情，内部状态任务按不存在处理。""" + if not task_id: + raise ValueError("人工复核详情缺少任务标识") + result = self._repository.get_manual_review(task_id=task_id) + if result is not None: + self._validate_state(result.state) + return result + +__all__ = [ + "TRANSFER_EXECUTION_VERSION", + "TRANSFER_STEP_INTENT_VERSION", + "TRANSFER_STEP_RESULT_VERSION", + "TransferExecutionCheckpoint", + "TransferExecutionCommand", + "TransferExecutionConflictError", + "TransferExecutionError", + "TransferExecutionLeaseLostError", + "TransferOperationObservation", + "TransferOperationObservationState", + "TransferRetryRequestResult", + "TransferExecutionRepository", + "TransferExecutionSnapshot", + "TransferExecutionState", + "TransferExecutionStep", + "TransferManualReviewDecision", + "TransferManualReviewPage", + "TransferManualReviewQuery", + "TransferManualReviewResult", + "TransferManualReviewSource", + "TransferManualReviewStepView", + "TransferManualReviewTaskView", + "TransferSettlementIntent", + "TransferSettlementResult", + "TransferStepIntent", + "TransferStepResult", + "TransferStepRunner", + "TransferStepState", + "TransferTerminalState", + "build_transfer_checkpoint_fingerprint", + "build_transfer_operation_id", +] diff --git a/app/chain/_transfer.py b/app/chain/_transfer.py index 7b2afb5e9..c6a8eb695 100644 --- a/app/chain/_transfer.py +++ b/app/chain/_transfer.py @@ -9,12 +9,13 @@ TransferChain 中。mixin 方法运行时经 MRO 解析，共享 TransferChain """ from copy import deepcopy from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from app.adapters.system.host import SystemUtils from app.application.agent import build_manual_redo_prompt, get_running_agent_manager from app.application.chain.data import ( get_chain_download_history_port, + get_chain_transfer_execution_port, get_chain_transfer_history_port, ) from app.application.configuration import ( @@ -24,6 +25,7 @@ from app.application.configuration import ( from app.application.formatting import EpisodeFormatRuleHelper from app.application.history import clear_transfer_failures, resolve_history from app.application.transfer import TransferTask, job_lock +from app.application.transfer_execution import TransferExecutionCommand from app.chain._contracts import TransferMixinHost from app.chain.media import MediaChain from app.chain.storage import StorageChain @@ -61,6 +63,34 @@ DownloadFiles = Any DownloadHistory = Any TransferHistory = Any + +def _request_durable_transfer_retry( + history: TransferHistory, + *, + requested_by: str, +) -> Optional[Tuple[bool, str]]: + """将 durable 历史重试交还持久调度器，旧历史返回 ``None``。""" + task_id = getattr(history, "transfer_task_id", None) + if not task_id: + return None + try: + result = TransferExecutionCommand( + get_chain_transfer_execution_port() + ).request_retry( + task_id=task_id, + reason=f"用户请求重试整理历史 #{history.id}", + requested_by=requested_by, + ) + except (RuntimeError, ValueError) as error: + logger.warning( + "登记 durable 整理重试失败：history_id=%s task_id=%s error=%s", + history.id, + task_id, + error, + ) + return False, str(error) + return result.accepted, result.message + # 字幕文件常见的语言/默认/强制标记，整理同名字幕时只允许剥离这些字幕专属尾缀。 SUBTITLE_STEM_TAGS = { "cc", @@ -1233,12 +1263,40 @@ class ManualHistoryMixin: histories[history.id] = history return list(histories.values()) - @staticmethod + def _request_durable_transfer_retry( + self, + history: TransferHistory, + *, + requested_by: str, + ) -> Optional[Tuple[bool, str]]: + """将 durable 历史重试交还持久调度器，旧历史返回 ``None``。 + + durable 任务的历史、目标文件和步骤证据共同描述一次可恢复执行。任何历史 + 入口都只能登记重试意图，不能删除这些证据后重新准入一条并行任务。 + + :param history: 整理历史 + :param requested_by: 发起重试的稳定入口身份 + :return: durable 请求结果；旧历史返回 ``None`` 继续兼容流程 + """ + return _request_durable_transfer_retry( + history, + requested_by=requested_by, + ) + def _delete_manual_transfer_history( + self, history: TransferHistory, transfer_history_oper: Any, ) -> Tuple[bool, str]: """删除手动重整历史；非成功移动记录同时清理可能存在的旧目标。""" + durable_retry = self._request_durable_transfer_retry( + history, + requested_by="manual_reorganize", + ) + if durable_retry is not None: + _, message = durable_retry + # 返回 False 阻止旧 caller 把“已登记”误当成“历史已删除”并继续执行。 + return False, message if ( history.dest_fileitem and not ManualHistoryMixin._is_successful_move_history(history) @@ -1365,7 +1423,7 @@ class FailedRetryMixin: source=source, userid=userid, username=username, - title=f"整理记录 #{history_id} 已重新整理", + title=errmsg or f"整理记录 #{history_id} 已重新整理", link=self.runtime_config.history_url, save_history=False, ) @@ -1397,6 +1455,47 @@ class FailedRetryMixin: 由智能助手接管一条失败的整理记录。 """ + history = get_chain_transfer_history_port().get(history_id) + if not history: + host = cast(TransferMixinHost, self) + host.post_message( + Message( + channel=channel, + source=source, + userid=userid, + username=username, + title="重新整理失败", + text=f"整理记录 #{history_id} 不存在", + link=host.runtime_config.history_url, + save_history=False, + ) + ) + return + + durable_retry = _request_durable_transfer_retry( + history, + requested_by="ai_retry_button", + ) + if durable_retry is not None: + accepted, message = durable_retry + self.post_message( + Message( + channel=channel, + source=source, + userid=userid, + username=username, + title=( + message + if accepted + else "重新整理失败" + ), + text=None if accepted else message, + link=self.runtime_config.history_url, + save_history=False, + ) + ) + return + if not self.runtime_config.ai_agent_enable: self.post_message( Message( @@ -1410,22 +1509,6 @@ class FailedRetryMixin: ) return - history = get_chain_transfer_history_port().get(history_id) - if not history: - self.post_message( - Message( - channel=channel, - source=source, - userid=userid, - username=username, - title="重新整理失败", - text=f"整理记录 #{history_id} 不存在", - link=self.runtime_config.history_url, - save_history=False, - ) - ) - return - redo_prompt = build_manual_redo_prompt(history) async def _run_ai_takeover(): @@ -1528,6 +1611,12 @@ class FailedRetryMixin: if not history: logger.error(f"整理记录不存在，ID：{logid}") return False, "整理记录不存在" + durable_retry = _request_durable_transfer_retry( + history, + requested_by="history_redo", + ) + if durable_retry is not None: + return durable_retry # 按源目录路径重新整理 src_path = Path(history.src) if not src_path.exists(): diff --git a/app/chain/transfer.py b/app/chain/transfer.py index 0bad6bead..636a9a56e 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -1,4 +1,6 @@ import asyncio +import hashlib +import json import queue import re import threading @@ -10,16 +12,19 @@ from concurrent.futures import CancelledError as FutureCancelledError from concurrent.futures import Future from copy import deepcopy from dataclasses import replace +from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast +from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union, cast from pydantic_core import to_jsonable_python from app.application.chain.data import ( get_chain_download_history_port, + get_chain_transfer_execution_port, get_chain_transfer_history_port, get_chain_transfer_pending_port, ) +from app.application.chain.durable_events import TransferResultSettlement from app.application.configuration import get_configured_system_config from app.application.directory import DirectoryHelper from app.application.formatting import FormatParser @@ -30,6 +35,7 @@ from app.application.history import ( describe_history_gate, evaluate_history_gate, is_skip_action, + max_failed_retries, record_transfer_failure, ) from app.application.outbox import ( @@ -58,6 +64,20 @@ from app.application.transfer import ( build_transfer_failure_group_key, job_lock, ) +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferExecutionCommand, + TransferExecutionConflictError, + TransferExecutionRepository, + TransferExecutionSnapshot, + TransferExecutionState, + TransferOperationObservation, + TransferOperationObservationState, + TransferSettlementResult, + TransferStepIntent, + TransferStepResult, + TransferStepState, +) from app.chain import ChainBase from app.chain._transfer import ( EpisodeFormatMixin, @@ -113,6 +133,213 @@ downloader_lock = threading.Lock() task_lock = threading.Lock() +class _TransferRetryDeferred(RuntimeError): + """表示已把确定失败交回唯一持久重试调度器。""" + + +class _TransferManualReviewRequired(RuntimeError): + """表示外部结果无法严格判定，任务已隔离等待人工复核。""" + + +class _TransferRetryExhausted(RuntimeError): + """表示步骤重试预算耗尽，任务可进入原子失败结算。""" + + def __init__(self, message: str, snapshot: TransferExecutionSnapshot) -> None: + """保存失败原因和数据库建立的结算检查点。""" + super().__init__(message) + self.snapshot = snapshot + + +class _DurableTransferStepRunner: + """以稳定顺序、lease 和 attempt fencing 执行整理外部步骤。""" + + _RETRY_DELAY_SECONDS = 30 + + def __init__( + self, + *, + task_id: str, + lease_token: str, + checkpoint_fingerprint: str, + repository: TransferExecutionRepository, + ) -> None: + """绑定单个任务的持久执行命令与全局步骤序号。""" + self._task_id = task_id + self._lease_token = lease_token + self._checkpoint_fingerprint = checkpoint_fingerprint + self._repository = repository + self._command = TransferExecutionCommand(self._repository) + snapshot = self._repository.get_snapshot(task_id=task_id) + existing_steps = list(snapshot.steps) if snapshot else [] + matching_ordinals = [ + step.ordinal + for step in existing_steps + if step.checkpoint_fingerprint == checkpoint_fingerprint + ] + self._ordinal = ( + min(matching_ordinals) + if matching_ordinals + else max((step.ordinal for step in existing_steps), default=-1) + 1 + ) + self._operation_ids = [ + step.operation_id + for step in existing_steps + if step.ordinal < self._ordinal + ] + + @staticmethod + def _retry_due_at() -> str: + """生成固定退避后的 UTC 调度时间。""" + return ( + datetime.now(timezone.utc) + + timedelta(seconds=_DurableTransferStepRunner._RETRY_DELAY_SECONDS) + ).strftime("%Y-%m-%d %H:%M:%S.%f") + + def _execute_attempt( + self, + *, + step: Any, + execute: Callable[[], TransferStepResult], + ) -> TransferStepResult: + """在事务外执行副作用，并把失败交给持久重试或终态结算。""" + try: + result = execute() + except Exception as error: + snapshot = self._repository.get_snapshot(task_id=self._task_id) + if snapshot is None: + raise RuntimeError("整理执行快照在步骤失败后丢失") from error + if snapshot.retry_count >= max_failed_retries(): + exhausted = self._command.exhaust( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + error=str(error), + ) + raise _TransferRetryExhausted(str(error), exhausted) from error + self._command.defer( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + error=str(error), + retry_due_at=self._retry_due_at(), + ) + raise _TransferRetryDeferred(str(error)) from error + completed = self._command.complete( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + result=result, + ) + if completed.result is None: + raise RuntimeError("整理步骤完成后缺少持久结果") + return completed.result + + def run( + self, + *, + phase: str, + kind: str, + payload: Mapping[str, Any], + execute: Callable[[], TransferStepResult], + observe: Callable[[], TransferOperationObservation], + ) -> TransferStepResult: + """幂等消费一个步骤，遗留 STARTED 必须先取得严格观察结论。""" + intent = TransferStepIntent.create( + task_id=self._task_id, + checkpoint_fingerprint=self._checkpoint_fingerprint, + ordinal=self._ordinal, + phase=phase, + kind=kind, + payload=payload, + ) + self._ordinal += 1 + self._operation_ids.append(intent.operation_id) + step = self._command.prepare( + task_id=self._task_id, + lease_token=self._lease_token, + intent=intent, + ) + if step.state is TransferStepState.SUCCEEDED: + if step.result is None: + raise RuntimeError("已成功整理步骤缺少结果证据") + return step.result + if step.state is TransferStepState.MANUAL_REVIEW: + raise _TransferManualReviewRequired( + step.last_error or "整理步骤正在等待人工复核" + ) + if step.state is TransferStepState.PREPARED: + step = self._command.begin( + task_id=self._task_id, + lease_token=self._lease_token, + operation_id=step.operation_id, + ) + elif step.state is TransferStepState.FAILED: + step = self._command.resume_failed( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + ) + elif step.state is TransferStepState.STARTED: + observation = observe() + if observation.state is TransferOperationObservationState.APPLIED: + completed = self._command.complete( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + result=observation.evidence, + ) + if completed.result is None: + raise RuntimeError("探测完成的整理步骤缺少结果证据") + return completed.result + if observation.state is TransferOperationObservationState.NOT_APPLIED: + step = self._command.restart_after_not_applied( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + evidence=observation.evidence, + ) + else: + reason = ( + f"遗留步骤 {step.operation_id} 外部结果为 " + f"{observation.state.value}，禁止自动重放" + ) + self._command.manual_review( + task_id=self._task_id, + lease_token=self._lease_token, + step=step, + error=reason, + evidence=observation.evidence, + ) + raise _TransferManualReviewRequired(reason) + else: + raise RuntimeError(f"不支持的整理步骤状态：{step.state}") + return self._execute_attempt(step=step, execute=execute) + + def checkpoint(self, transferinfo: TransferInfo) -> TransferExecutionCheckpoint: + """提交足以独立重放终态结算的聚合执行结果。""" + successful_outcome = bool(transferinfo.success) + checkpoint = TransferExecutionCheckpoint.create( + payload={ + "outcome": "succeeded" if successful_outcome else "failed", + "transferinfo": transferinfo.model_dump(mode="json"), + }, + operation_ids=tuple(self._operation_ids), + skip_reason=( + (transferinfo.message or "zero-side-effect transfer") + if not self._operation_ids + else None + ), + ) + snapshot = self._command.checkpoint( + task_id=self._task_id, + lease_token=self._lease_token, + checkpoint=checkpoint, + ) + if snapshot.checkpoint is None: + raise RuntimeError("整理执行检查点提交后无法回读") + return snapshot.checkpoint + + class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, HistoryMatchMixin, FileKeyMixin, ManualHistoryMixin, FailedRetryMixin, ChainBase, ConfigReloadMixin, metaclass=Singleton): """ @@ -152,6 +379,18 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo _RECOVERY_POLL_INTERVAL_SECONDS = 15.0 _RECOVERY_CLAIM_LIMIT = 100 + @staticmethod + def __transfer_plan_fingerprint(checkpoint: TransferPlanCheckpoint) -> str: + """由完整冻结计划生成执行步骤使用的稳定 SHA-256 身份。""" + canonical = json.dumps( + checkpoint.to_payload(), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + @staticmethod def _transfer_result_payload( task: TransferTask, @@ -195,6 +434,115 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo return AUDIO_TRANSFER_FAILED_TOPIC, EventType.AudioTransferFailed return None + @staticmethod + def __build_transfer_result_settlement( + task: TransferTask, + transferinfo: TransferInfo, + *, + overwrite_declined: bool = False, + ) -> Optional[TransferResultSettlement]: + """由执行结果与已核实的覆盖裁决构造受 lease fencing 保护的终态命令。""" + checkpoint = task.execution_checkpoint + if checkpoint is None: + return None + if not task.admission_task_id or not task.lease_token: + raise TransferLeaseLostError("整理终态缺少持久任务身份或租约") + successful_outcome = bool(transferinfo.success or overwrite_declined) + return TransferResultSettlement( + task_id=task.admission_task_id, + lease_token=task.lease_token, + execution_fingerprint=checkpoint.fingerprint, + outcome="succeeded" if successful_outcome else "failed", + error=( + None + if successful_outcome + else (transferinfo.message or "整理失败") + ), + ) + + def __settle_legacy_transfer_result( + self, + task: TransferTask, + transferinfo: TransferInfo, + ) -> None: + """无公开事件地原子提交旧同步调用的历史回执与任务终态。""" + transferhis = get_chain_transfer_history_port() + overwrite_declined = self._is_overwrite_declined( + task, + transferinfo, + transferhis, + ) + settlement = self.__build_transfer_result_settlement( + task, + transferinfo, + overwrite_declined=overwrite_declined, + ) + if settlement is None: + raise RuntimeError("旧整理兼容命令缺少可验证的执行检查点") + writer = getattr(self, "durable_event_writer", None) + if writer is None: + raise RuntimeError("旧整理兼容命令缺少 durable 原子写入端口") + + def stage_history(staging: Any) -> Any: + """按兼容结果暂存成功、失败或覆盖跳过的唯一历史投影。""" + if overwrite_declined: + return staging.get_success_by_src( + task.fileitem.path, + task.fileitem.storage, + ) + if transferinfo.success: + return add_transfer_success( + fileitem=task.fileitem, + mode=transferinfo.transfer_type or "", + downloader=task.downloader, + download_hash=task.download_hash, + meta=cast(MetaBase, task.meta), + mediainfo=cast(Union[MediaInfo, MusicInfo], task.mediainfo), + transferinfo=transferinfo, + transfer_history_oper=staging, + ) + return add_transfer_fail( + fileitem=task.fileitem, + mode=transferinfo.transfer_type or "", + downloader=task.downloader, + download_hash=task.download_hash, + meta=cast(MetaBase, task.meta), + mediainfo=task.mediainfo, + transferinfo=transferinfo, + transfer_history_oper=staging, + ) + def write_result() -> Any: + """以相同 task_id 和执行指纹提交或回读同一终态。""" + return writer.transfer_result( + topic=None, + stage_history=stage_history, + event_payload=self._transfer_result_payload(task, transferinfo), + publish=None, + settlement=settlement, + ) + + try: + result = write_result() + except Exception as first_error: + logger.warning( + "旧整理兼容命令首次结算响应不确定，按同一 task_id 回读 durable 回执：%s", + first_error, + ) + result = write_result() + if not isinstance(result, TransferSettlementResult): + raise RuntimeError("旧整理兼容命令没有返回 durable 结算结果") + task.mark_terminal_settled() + assert task.admission_task_id is not None + assert task.lease_token is not None + self.__forget_owned_lease(task.admission_task_id, task.lease_token) + + @staticmethod + def __transfer_history_id(history: Any) -> Optional[int]: + """统一读取旧历史投影和 task-aware 结算结果的历史标识。""" + if isinstance(history, TransferSettlementResult): + return history.history_id + return getattr(history, "id", None) if history is not None else None + def _publish_transfer_result( self, event_type: EventType, @@ -243,6 +591,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.failure_notification_aggregator = TransferFailureNotificationAggregator() # durable admission 仓储先于内存入队保存任务，进程退出后仍可恢复。 self._transfer_admissions = get_chain_transfer_pending_port() + self._transfer_executions = get_chain_transfer_execution_port() # 转移成功的文件清单 self._success_target_files: Dict[Tuple, List[str]] = {} # 批次级刮削缓冲，避免同一批多文件入库重复触发目录刮削 @@ -568,6 +917,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo transferhis = get_chain_transfer_history_port() target_dir_path = self.__get_transfer_target_dir_path(transferinfo) job_id = self.jobview.get_job_id(task) + overwrite_declined = False + if not transferinfo.success: + overwrite_declined = self._is_overwrite_declined( + task, transferinfo, transferhis + ) + settlement = self.__build_transfer_result_settlement( + task, + transferinfo, + overwrite_declined=overwrite_declined, + ) # 转移失败 if not transferinfo.success: @@ -575,14 +934,28 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # 媒体库里原有版本仍然在位，写失败记录会用 add_force 顶掉原成功记录，此后该路径 # 永远处于失败态，每个新事件都会重试并重推失败通知。此时保留原记录、不写历史、 # 不发事件与通知、不触发重试，仅把任务置为未入库 - overwrite_declined = self._is_overwrite_declined( - task, transferinfo, transferhis - ) history = None if overwrite_declined: logger.info( f"{task.fileitem.name} 未入库并保留原整理记录：{transferinfo.message}" ) + if settlement is not None: + writer = getattr(self, "durable_event_writer", None) + if writer is None: + raise RuntimeError("覆盖跳过的 durable 终态缺少原子写入端口") + history = writer.transfer_result( + topic=None, + stage_history=lambda staging: staging.get_success_by_src( + task.fileitem.path, + task.fileitem.storage, + ), + event_payload=self._transfer_result_payload(task, transferinfo), + publish=None, + settlement=settlement, + ) + if not isinstance(history, TransferSettlementResult): + raise RuntimeError("覆盖跳过的 durable 终态没有返回结算结果") + task.mark_terminal_settled() else: logger.warn(f"{task.fileitem.name} 入库失败：{transferinfo.message}") @@ -597,11 +970,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo durable_event = self._durable_transfer_event(task, success=False) durable_transfer_failed = bool( - getattr(self, "durable_event_writer", None) and durable_event + getattr(self, "durable_event_writer", None) + and (durable_event or settlement) ) if durable_transfer_failed: - assert durable_event is not None - topic, event_type = durable_event + topic = durable_event[0] if durable_event else None + event_type = durable_event[1] if durable_event else None event_payload = self._transfer_result_payload(task, transferinfo) history = self.durable_event_writer.transfer_result( topic=topic, @@ -616,10 +990,19 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo transfer_history_oper=writer, ), event_payload=event_payload, - publish=lambda payload: self._publish_transfer_result( - event_type, payload + publish=( + lambda payload: self._publish_transfer_result( + event_type, payload + ) + if event_type is not None + else None ), + settlement=settlement, ) + if settlement is not None: + if not isinstance(history, TransferSettlementResult): + raise RuntimeError("整理失败 durable 终态没有返回结算结果") + task.mark_terminal_settled() else: history = add_transfer_fail( fileitem=task.fileitem, @@ -639,14 +1022,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._transfer_result_payload( task, transferinfo, - history.id if history else None, + self.__transfer_history_id(history), ), ) self.queue_failed_transfer_notification( task=task, transferinfo=transferinfo, - history_id=history.id if history else None, + history_id=self.__transfer_history_id(history), ) # 设置任务失败 @@ -655,13 +1038,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo # AI智能体自动重试整理 if ( history + and settlement is None and self.runtime_config.ai_agent_enable and self.runtime_config.ai_agent_retry_transfer ): # 使用 download_hash 或源文件父目录作为分组键， # 同一批次（如同一个种子）的失败记录会被合并为一次agent调用 self._schedule_failed_transfer_retry( - history.id, + cast(int, self.__transfer_history_id(history)), build_transfer_failure_group_key(task), ) @@ -681,11 +1065,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo durable_event = self._durable_transfer_event(task, success=True) durable_transfer_complete = bool( - getattr(self, "durable_event_writer", None) and durable_event + getattr(self, "durable_event_writer", None) + and (durable_event or settlement) ) if durable_transfer_complete: - assert durable_event is not None - topic, event_type = durable_event + topic = durable_event[0] if durable_event else None + event_type = durable_event[1] if durable_event else None event_payload = self._transfer_result_payload(task, transferinfo) history = self.durable_event_writer.transfer_result( topic=topic, @@ -700,10 +1085,19 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo transfer_history_oper=writer, ), event_payload=event_payload, - publish=lambda payload: self._publish_transfer_result( - event_type, payload + publish=( + lambda payload: self._publish_transfer_result( + event_type, payload + ) + if event_type is not None + else None ), + settlement=settlement, ) + if settlement is not None: + if not isinstance(history, TransferSettlementResult): + raise RuntimeError("整理成功 durable 终态没有返回结算结果") + task.mark_terminal_settled() else: history = add_transfer_success( fileitem=task.fileitem, @@ -723,7 +1117,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._transfer_result_payload( task, transferinfo, - history.id if history else None, + self.__transfer_history_id(history), ), ) @@ -1364,11 +1758,23 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo storage = admission.storage src_path = admission.src_path try: - fileitem, should_discard = self.__build_replay_fileitem( - storage, - src_path, - admission.planning_input, - ) + settling_checkpoint = self.__settling_replay_checkpoint(admission) + if settling_checkpoint is not None: + planning_input = admission.planning_input + if planning_input is None: + raise TransferPlanningStateError( + "settling 回放缺少整理规划输入" + ) + fileitem = FileItem.model_validate( + planning_input.source_fileitem + ) + should_discard = False + else: + fileitem, should_discard = self.__build_replay_fileitem( + storage, + src_path, + admission.planning_input, + ) # stat 等同步 I/O 返回后重新检查，关闭期间不得注销尚未完成的登记。 if stop_event.is_set(): self.__release_admission_claim( @@ -1408,7 +1814,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) continue if admission.checkpoint: - if self.__queue_planned_replay(fileitem, admission): + if self.__queue_planned_replay( + fileitem, + admission, + execution_checkpoint=settling_checkpoint, + ): replayed += 1 else: self.__release_admission_claim( @@ -1459,6 +1869,27 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo else: logger.info(f"✓ 待整理文件回放完成，{replayed} 个文件已重新送入整理链") + def __settling_replay_checkpoint( + self, + admission: TransferAdmission, + ) -> Optional[TransferExecutionCheckpoint]: + """读取已 claim 的 settling 检查点，避免终态恢复再次探测源文件。""" + repository: Optional[TransferExecutionRepository] = getattr( + self, + "_transfer_executions", + None, + ) + if repository is None: + return None + snapshot = repository.get_snapshot(task_id=admission.task_id) + if snapshot is None or snapshot.state is not TransferExecutionState.SETTLING: + return None + if snapshot.checkpoint is None: + raise TransferExecutionConflictError( + "settling 整理任务缺少可重放终态检查点" + ) + return snapshot.checkpoint + @staticmethod def __build_replay_kwargs( planning_input: Optional[TransferPlanningInput], @@ -1577,6 +2008,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self, fileitem: FileItem, admission: TransferAdmission, + *, + execution_checkpoint: Optional[TransferExecutionCheckpoint] = None, ) -> bool: """把已提交检查点直接恢复为单个任务，避免重新扫描、识别和规划。""" checkpoint = admission.checkpoint @@ -1623,6 +2056,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__bind_claimed_admission(task, admission) task.bind_planning_input(checkpoint.planning_input) task.bind_plan_checkpoint(checkpoint) + if execution_checkpoint is not None: + task.bind_execution_checkpoint(execution_checkpoint) self.__restore_planned_task(task) return self.put_to_queue(task) @@ -1919,6 +2354,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo """直接执行已提交检查点，不再触发识别、分类、选目录或重命名。""" self.__restore_planned_task(task) self.jobview.running_task(task) + settling_result = self.__restore_settling_transfer_result(task) + if settling_result is not None: + if callback: + return callback(task, settling_result) + return settling_result.success, settling_result.message or "" checkpoint = task.plan_checkpoint assert checkpoint is not None target_storage = ( @@ -1937,6 +2377,124 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo return callback(task, transferinfo) return transferinfo.success, transferinfo.message or "" + def __restore_settling_transfer_result( + self, + task: TransferTask, + ) -> Optional[TransferInfo]: + """从 execution checkpoint 恢复终态结果，禁止再次执行外部步骤。""" + checkpoint = task.execution_checkpoint + if checkpoint is None: + repository: Optional[TransferExecutionRepository] = getattr( + self, + "_transfer_executions", + None, + ) + task_id = task.admission_task_id + if repository is None or not task_id: + return None + snapshot = repository.get_snapshot(task_id=task_id) + if snapshot is None or snapshot.state is not TransferExecutionState.SETTLING: + return None + checkpoint = snapshot.checkpoint + if checkpoint is None: + raise TransferExecutionConflictError( + "settling 整理任务缺少可重放终态检查点" + ) + self.__assert_owned_lease(task) + payload = checkpoint.payload + outcome = payload.get("outcome") + if outcome not in {"succeeded", "failed"}: + raise TransferExecutionConflictError("整理执行检查点缺少确定终态") + transfer_payload = payload.get("transferinfo") + if isinstance(transfer_payload, dict): + transferinfo = cast( + TransferInfo, + TransferInfo.model_validate(transfer_payload), + ) + elif outcome == "failed": + plan_checkpoint = task.plan_checkpoint + transferinfo = TransferInfo( + success=False, + message=str(payload.get("error") or "整理失败"), + fileitem=task.fileitem, + fail_list=[task.fileitem.path], + transfer_type=( + plan_checkpoint.resolved_transfer_type + if plan_checkpoint is not None + else None + ), + need_notify=( + plan_checkpoint.need_notify + if plan_checkpoint is not None + else False + ), + ) + else: + raise TransferExecutionConflictError( + "成功整理执行检查点缺少可重放 TransferInfo" + ) + successful_outcome = bool(transferinfo.success) + if successful_outcome != (outcome == "succeeded"): + raise TransferExecutionConflictError( + "整理执行检查点终态与 TransferInfo 不一致" + ) + task.bind_execution_checkpoint(checkpoint) + return transferinfo + + def __build_durable_step_runner( + self, + task: TransferTask, + checkpoint: TransferPlanCheckpoint, + ) -> Optional[_DurableTransferStepRunner]: + """为非预览持久任务构造绑定计划指纹和当前租约的步骤 runner。""" + if task.preview: + return None + repository = getattr(self, "_transfer_executions", None) + if repository is None: + return None + if not task.admission_task_id or not task.lease_token: + raise TransferLeaseLostError("整理执行缺少持久任务身份或租约") + return _DurableTransferStepRunner( + task_id=task.admission_task_id, + lease_token=task.lease_token, + checkpoint_fingerprint=self.__transfer_plan_fingerprint(checkpoint), + repository=repository, + ) + + def __execute_host_transfer_plan( + self, + task: TransferTask, + checkpoint: TransferPlanCheckpoint, + *, + source_oper: Any, + target_oper: Any, + step_runner: Optional[_DurableTransferStepRunner], + ) -> Optional[TransferInfo]: + """通过模块边界执行宿主计划，并只向持久任务注入内部步骤 runner。""" + if step_runner is None: + return self.execute_transfer_plan( + checkpoint, + meta=task.meta, + mediainfo=task.mediainfo, + source_oper=source_oper, + target_oper=target_oper, + cleanup_media_file=self.__cleanup_transfer_destination, + ) + return cast( + Optional[TransferInfo], + self.run_module( + "execute_transfer_plan", + checkpoint=checkpoint, + meta=task.meta, + mediainfo=task.mediainfo, + source_oper=source_oper, + target_oper=target_oper, + cleanup_media_file=self.__cleanup_transfer_destination, + observe_cleanup_media_file=self.__observe_cleanup_destination, + step_runner=step_runner, + ), + ) + def _plan_checkpoint_and_execute( self, task: TransferTask, @@ -2001,13 +2559,29 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__restore_planned_task(task) self.__assert_owned_lease(task) - legacy_result = self.__execute_legacy_transfer_providers( - task, - checkpoint=checkpoint, - source_oper=source_oper, - target_oper=target_oper, - ) + step_runner = self.__build_durable_step_runner(task, checkpoint) + try: + legacy_result = self.__execute_legacy_transfer_providers( + task, + checkpoint=checkpoint, + source_oper=source_oper, + target_oper=target_oper, + step_runner=step_runner, + ) + except _TransferRetryExhausted as error: + assert error.snapshot.checkpoint is not None + task.bind_execution_checkpoint(error.snapshot.checkpoint) + return TransferInfo( + success=False, + message=str(error), + fileitem=task.fileitem, + fail_list=[task.fileitem.path], + transfer_type=checkpoint.resolved_transfer_type, + need_notify=checkpoint.need_notify, + ) if legacy_result is not None: + if step_runner is not None: + task.bind_execution_checkpoint(step_runner.checkpoint(legacy_result)) return legacy_result if checkpoint.is_provider_pending: @@ -2027,21 +2601,35 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) task.bind_plan_checkpoint(checkpoint) self.__restore_planned_task(task) + step_runner = self.__build_durable_step_runner(task, checkpoint) except Exception as error: self.__record_checkpoint_failure(task, error) raise self.__assert_owned_lease(task) - result = self.execute_transfer_plan( - checkpoint, - meta=task.meta, - mediainfo=task.mediainfo, - source_oper=source_oper, - target_oper=target_oper, - cleanup_media_file=self.__cleanup_transfer_destination, - ) + try: + result = self.__execute_host_transfer_plan( + task, + checkpoint, + source_oper=source_oper, + target_oper=target_oper, + step_runner=step_runner, + ) + except _TransferRetryExhausted as error: + assert error.snapshot.checkpoint is not None + task.bind_execution_checkpoint(error.snapshot.checkpoint) + return TransferInfo( + success=False, + message=str(error), + fileitem=task.fileitem, + fail_list=[task.fileitem.path], + transfer_type=checkpoint.resolved_transfer_type, + need_notify=checkpoint.need_notify, + ) if result is None: raise RuntimeError("文件整理模块未返回检查点执行结果") + if step_runner is not None: + task.bind_execution_checkpoint(step_runner.checkpoint(result)) return result def __plan_host_transfer( @@ -2121,6 +2709,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo checkpoint: TransferPlanCheckpoint, source_oper: Any, target_oper: Any, + step_runner: Optional[_DurableTransferStepRunner] = None, ) -> Optional[TransferInfo]: """在 checkpoint 提交后按冻结身份执行旧插件 transfer provider。""" providers = checkpoint.legacy_transfer_providers @@ -2153,30 +2742,67 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo TmdbEpisode.model_validate(item) for item in invocation.episodes_info ] - result = self._module_dispatcher.execute_frozen_plugin_providers( - "transfer", - providers, - before_invoke=lambda: self.__prepare_legacy_provider_execution( - checkpoint=checkpoint - ), - fileitem=FileItem.model_validate(invocation.fileitem), - meta=provider_meta, - mediainfo=provider_mediainfo, - target_directory=target_directory, - target_storage=invocation.target_storage, - target_path=(Path(invocation.target_path) if invocation.target_path else None), - transfer_type=invocation.transfer_type, - scrape=invocation.scrape, - library_type_folder=invocation.library_type_folder, - library_category_folder=invocation.library_category_folder, - episodes_info=provider_episodes, - source_oper=source_oper, - target_oper=target_oper, - preview=invocation.preview, - ) - if result is None or isinstance(result, TransferInfo): + def invoke_provider_sequence() -> Optional[TransferInfo]: + """按冻结顺序调用旧 provider，并校验其兼容返回类型。""" + result = self._module_dispatcher.execute_frozen_plugin_providers( + "transfer", + providers, + before_invoke=lambda: self.__prepare_legacy_provider_execution( + checkpoint=checkpoint + ), + fileitem=FileItem.model_validate(invocation.fileitem), + meta=provider_meta, + mediainfo=provider_mediainfo, + target_directory=target_directory, + target_storage=invocation.target_storage, + target_path=(Path(invocation.target_path) if invocation.target_path else None), + transfer_type=invocation.transfer_type, + scrape=invocation.scrape, + library_type_folder=invocation.library_type_folder, + library_category_folder=invocation.library_category_folder, + episodes_info=provider_episodes, + source_oper=source_oper, + target_oper=target_oper, + preview=invocation.preview, + ) + if result is not None and not isinstance(result, TransferInfo): + raise TypeError("旧插件 transfer provider 返回了不支持的结果类型") return result - raise TypeError("旧插件 transfer provider 返回了不支持的结果类型") + + def execute_provider_sequence() -> TransferStepResult: + """执行冻结旧 provider 序列，并冻结是否接管及兼容结果。""" + result = invoke_provider_sequence() + return TransferStepResult(payload={ + "handled": result is not None, + "transferinfo": ( + result.model_dump(mode="json") if result is not None else None + ), + }) + + if step_runner is None: + return invoke_provider_sequence() + provider_result = step_runner.run( + phase="provider", + kind="legacy_transfer_provider_sequence", + payload={ + "providers": [provider.to_payload() for provider in providers], + "invocation": invocation.to_payload(), + }, + execute=execute_provider_sequence, + observe=lambda: TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={ + "reason": "legacy provider has no stable operation receipt", + }), + ), + ) + if not provider_result.payload.get("handled"): + return None + payload = provider_result.payload.get("transferinfo") + if not isinstance(payload, dict): + raise TransferPlanningStateError("旧 provider 成功证据缺少整理结果") + validated_result: TransferInfo = TransferInfo.model_validate(payload) + return validated_result def __prepare_legacy_provider_execution( self, @@ -2213,6 +2839,15 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo return True return bool(storage_chain.delete_media_file(current_item)) + def __observe_cleanup_destination(self, fileitem: FileItem) -> bool: + """只读确认统一清理目标已经不存在，查询错误由步骤 runner 隔离。""" + if not fileitem.path: + return False + return self._transfer_storage_chain().get_file_item_strict( + storage=fileitem.storage, + path=Path(fileitem.path), + ) is None + def execute_legacy_transfer_command( self, fileitem: FileItem, @@ -2263,8 +2898,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo fail_list=[fileitem.path], transfer_type=transfer_type, ) - if not self.__discard_pending(task): - message = "旧整理兼容命令已执行，但 durable 终态结算失去租约" + if preview: + return result + try: + self.__settle_legacy_transfer_result(task, result) + except Exception as error: + message = f"旧整理兼容命令 durable 终态结算失败：{error}" logger.error(message) return TransferInfo( success=False, @@ -2414,6 +3053,15 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo marker(task) if terminal: if terminal_settlement is not None: + if ( + terminal_settlement + and task.admission_task_id + and task.lease_token + ): + self.__forget_owned_lease( + task.admission_task_id, + task.lease_token, + ) return terminal_settlement return self.__discard_pending(task) self.__release_task_claim(task) @@ -2570,9 +3218,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo callback_task: TransferTask, transferinfo: TransferInfo, ) -> Tuple[bool, str]: - """成功结果先提交 durable 终态，再执行历史、事件和通知回调。""" + """由默认回调原子提交历史、事件与 durable 终态。""" nonlocal terminal_settlement - if transferinfo.success and not callback_task.preview: + if ( + transferinfo.success + and not callback_task.preview + and callback_task.execution_checkpoint is None + ): terminal_settlement = self.__discard_pending(callback_task) if not terminal_settlement: self.__fail_transfer_task(callback_task) @@ -2582,6 +3234,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo callback_task, transferinfo, ) + if callback_task.execution_checkpoint is not None: + terminal_settlement = callback_task.terminal_settled return callback_result return transferinfo.success, transferinfo.message or "" @@ -3956,13 +4610,27 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo transferd = self._get_manual_transfer_history( fileitem=file_item, transfer_history_oper=transfer_history_oper, - include_move_dest=manual and reorganize, + include_move_dest=bool(manual and reorganize), ) if transferd: should_reorganize = manual and ( reorganize or not transferd.status ) if should_reorganize: + durable_retry = self._request_durable_transfer_retry( + transferd, + requested_by="manual_reorganize", + ) + if durable_retry is not None: + accepted, message = durable_retry + if accepted: + logger.info(message) + else: + all_success = False + logger.error(message) + err_msgs.append(message) + # durable 历史只登记唯一调度重试，不在当前调用重新准入。 + continue state, message = self._delete_manual_transfer_history( history=transferd, transfer_history_oper=transfer_history_oper, @@ -4232,9 +4900,13 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo callback_task: TransferTask, transferinfo: TransferInfo, ) -> Tuple[bool, str]: - """同步成功结果先结算 durable 行，再执行兼容回调副作用。""" + """同步路径也由默认回调原子提交历史、事件与 durable 终态。""" nonlocal terminal_settlement - if transferinfo.success and not callback_task.preview: + if ( + transferinfo.success + and not callback_task.preview + and callback_task.execution_checkpoint is None + ): terminal_settlement = self.__discard_pending(callback_task) if not terminal_settlement: self.__fail_transfer_task(callback_task) @@ -4244,7 +4916,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo if preview else self.__default_callback ) - return callback(callback_task, transferinfo) + callback_result = callback(callback_task, transferinfo) + if callback_task.execution_checkpoint is not None: + terminal_settlement = callback_task.terminal_settled + return callback_result try: self.__claim_task_for_execution(transfer_task) diff --git a/app/db/adapters/chain.py b/app/db/adapters/chain.py index d3175f17c..ee422b13d 100644 --- a/app/db/adapters/chain.py +++ b/app/db/adapters/chain.py @@ -3,13 +3,17 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timezone from typing import Any +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from app.application.chain.durable_events import ( ChainDurableEventWriter, TransferHistoryRef, + TransferResultSettlement, download_added_event_key, snapshot_download_added, snapshot_transfer_result, @@ -17,22 +21,40 @@ from app.application.chain.durable_events import ( ) from app.application.history import TransferHistoryRecord, TransferHistoryWriter from app.application.outbox import ( - DurableEventCommand, DOWNLOAD_ADDED_TOPIC, + DurableEventCommand, OutboxIntent, ) +from app.application.transfer_execution import ( + TransferExecutionConflictError, + TransferExecutionLeaseLostError, + TransferExecutionState, + TransferSettlementResult, +) from app.db.adapters.outbox import SqlAlchemyOutboxRepository +from app.db.models.transfersettlementreceipt import TransferSettlementReceipt from app.db.oper.downloadhistory import DownloadHistoryOper +from app.db.oper.transferexecutionstep import TransferExecutionStepOper from app.db.oper.transferhistory import TransferHistoryOper +from app.db.oper.transferpending import TransferPendingOper +from app.db.oper.transfersettlementreceipt import TransferSettlementReceiptOper from app.db.uow import SqlAlchemyUnitOfWork class _StagingTransferHistoryWriter: """让既有历史字段映射复用无提交的 replace 适配器。""" - def __init__(self, repository: TransferHistoryOper) -> None: - """保存绑定调用方 Session 的整理历史仓储。""" + def __init__( + self, + repository: TransferHistoryOper, + *, + settlement: TransferResultSettlement | None = None, + settlement_revision: int | None = None, + ) -> None: + """保存仓储，并让历史继续表达同源最新业务投影。""" self._repository = repository + self._settlement = settlement + self._settlement_revision = settlement_revision def get_by_src( self, @@ -47,14 +69,40 @@ class _StagingTransferHistoryWriter: src: str, storage: str | None = None, ) -> TransferHistoryRecord | None: - """转发按源路径读取成功记录。""" + """读取成功记录；任务结算时绑定当前任务投影。""" + if self._settlement is not None: + if self._settlement_revision is None: + raise RuntimeError("整理任务结算缺少事务内修订号") + return self._repository.stage_bind_settlement( + task_id=self._settlement.task_id, + settlement_revision=self._settlement_revision, + src=src, + storage=storage, + ) return self._repository.get_success_by_src(src, storage) def add_force(self, **payload: Any) -> TransferHistoryRecord: - """保持应用层旧端口名，但只暂存替换而不自行提交。""" + """保持旧端口名，并按是否存在任务身份选择暂存策略。""" + if self._settlement is not None: + if self._settlement_revision is None: + raise RuntimeError("整理任务结算缺少事务内修订号") + return self._repository.stage_upsert_by_transfer_task_id( + task_id=self._settlement.task_id, + settlement_revision=self._settlement_revision, + retain_task_mapping=self._settlement.outcome == "failed", + payload=payload, + ) return self._repository.stage_replace_by_src(**payload) +@dataclass(frozen=True, slots=True) +class _StagedTransferResult: + """保存构造 outbox 所需历史投影及可选任务结算结果。""" + + history: TransferHistoryRef + settlement: TransferSettlementResult | None = None + + class TransactionalChainDurableEventWriter(ChainDurableEventWriter): """为每次 Chain 结果事件创建独占同步 Session 和 UoW。""" @@ -109,41 +157,111 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter): def transfer_result( self, *, - topic: str, + topic: str | None, stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None], event_payload: dict[str, Any], - publish: Callable[[dict[str, Any]], None], - ) -> TransferHistoryRecord | None: - """原子写整理历史与结果 intent，并返回脱离 Session 的最小投影。""" + publish: Callable[[dict[str, Any]], None] | None, + settlement: TransferResultSettlement | None = None, + ) -> TransferHistoryRecord | TransferSettlementResult | None: + """原子写历史、可选任务终态与 intent，再返回稳定投影。""" + if topic is None and settlement is None: + raise ValueError("无事件 topic 的整理写入必须绑定 durable 任务结算") session = self._session_factory() try: - staging = _StagingTransferHistoryWriter(TransferHistoryOper(session)) + history_repository = TransferHistoryOper(session) + pending_repository = TransferPendingOper(session) + receipt_repository = TransferSettlementReceiptOper(session) + already_settled = self._read_settlement_result( + pending_repository=pending_repository, + receipt_repository=receipt_repository, + settlement=settlement, + ) + if already_settled is not None: + return already_settled + command = DurableEventCommand( unit_of_work=SqlAlchemyUnitOfWork(session), outbox=SqlAlchemyOutboxRepository(session), ) - def stage_business() -> TransferHistoryRef | None: - """复用历史字段映射，并在 flush 后冻结安全投影。""" + def stage_business() -> _StagedTransferResult: + """同一事务暂存历史及受 fencing 保护的 pending 终态。""" + expected_revision = self._settlement_revision( + pending_repository=pending_repository, + settlement=settlement, + ) + next_revision = ( + expected_revision + 1 + if expected_revision is not None + else None + ) + staging = _StagingTransferHistoryWriter( + history_repository, + settlement=settlement, + settlement_revision=next_revision, + ) history = stage_history(staging) if history is None: - return None - return TransferHistoryRef( + raise RuntimeError("整理历史暂存失败，无法登记 durable 结果事件") + projected = TransferHistoryRef( id=history.id, status=bool(history.status), src=history.src, src_storage=history.src_storage, src_fileitem=history.src_fileitem, ) + if settlement is None: + return _StagedTransferResult(history=projected) + assert expected_revision is not None + assert next_revision is not None + self._validate_history_outcome(projected, settlement) + pending_deleted = self._stage_pending_terminal( + session=session, + repository=pending_repository, + settlement=settlement, + expected_revision=expected_revision, + history_id=projected.id, + ) + settled_at = datetime.now(timezone.utc).isoformat() + receipt_repository.stage_append( + task_id=settlement.task_id, + history_id=projected.id, + settlement_revision=next_revision, + outcome=settlement.outcome, + execution_fingerprint=settlement.execution_fingerprint, + lease_token=settlement.lease_token, + history_status=projected.status, + src=projected.src, + src_storage=projected.src_storage, + pending_deleted=pending_deleted, + error=settlement.error, + settled_at=settled_at, + ) + return _StagedTransferResult( + history=projected, + settlement=TransferSettlementResult( + history_id=projected.id, + settlement_revision=next_revision, + pending_deleted=pending_deleted, + ), + ) def build_intent( - history: TransferHistoryRef | None, + result: _StagedTransferResult, ) -> OutboxIntent: """历史 ID 确定后构造事件键与可恢复快照。""" - if history is None: - raise RuntimeError("整理历史暂存失败，无法登记 durable 结果事件") - event_key = transfer_result_event_key(topic, history.id) - event_payload["transfer_history_id"] = history.id + assert topic is not None + event_key = transfer_result_event_key( + topic, + result.history.id, + settlement=settlement, + settlement_revision=( + result.settlement.settlement_revision + if result.settlement is not None + else None + ), + ) + event_payload["transfer_history_id"] = result.history.id event_payload["idempotency_key"] = event_key return OutboxIntent( event_key=event_key, @@ -151,10 +269,204 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter): payload=snapshot_transfer_result(event_payload), ) - return command.execute( - intent=build_intent, - stage_business=stage_business, - publish=lambda: publish(event_payload), - ) + try: + result = command.execute( + intent=build_intent if topic is not None else None, + stage_business=stage_business, + publish=( + (lambda: publish(event_payload)) + if ( + settlement is None + and topic is not None + and publish is not None + ) + else None + ), + ) + except ( + IntegrityError, + TransferExecutionConflictError, + TransferExecutionLeaseLostError, + ValueError, + ): + if settlement is None: + raise + session.rollback() + replay = self._read_settlement_result( + pending_repository=pending_repository, + receipt_repository=receipt_repository, + settlement=settlement, + ) + if replay is None: + raise + return replay + return result.settlement or result.history finally: session.close() + + @staticmethod + def _read_settlement_result( + *, + pending_repository: TransferPendingOper, + receipt_repository: TransferSettlementReceiptOper, + settlement: TransferResultSettlement | None, + ) -> TransferSettlementResult | None: + """识别已提交终态并返回幂等结果，未结算时返回空。""" + if settlement is None: + return None + pending = pending_repository.get_by_task_id(task_id=settlement.task_id) + latest = receipt_repository.get_latest_by_task_id( + task_id=settlement.task_id + ) + receipt = receipt_repository.get_by_identity( + task_id=settlement.task_id, + execution_fingerprint=settlement.execution_fingerprint, + lease_token=settlement.lease_token, + outcome=settlement.outcome, + ) + if ( + pending is not None + and pending.execution_state == TransferExecutionState.FAILED.value + ): + if ( + latest is None + or pending.terminal_history_id != latest.history_id + or pending.settlement_revision != latest.settlement_revision + or pending.execution_fingerprint != latest.execution_fingerprint + or latest.outcome != "failed" + or latest.pending_deleted + ): + raise TransferExecutionConflictError("失败终态与最新结算回执不一致") + if receipt is not None: + TransactionalChainDurableEventWriter._validate_receipt( + receipt=receipt, + settlement=settlement, + ) + return TransferSettlementResult( + history_id=receipt.history_id, + settlement_revision=receipt.settlement_revision, + pending_deleted=receipt.pending_deleted, + already_settled=True, + ) + if pending is None: + if latest is None: + raise TransferExecutionConflictError( + "pending 已不存在且没有可验证的终态回执" + ) + raise TransferExecutionConflictError("整理终态与 durable 回执不一致") + if pending.execution_state != TransferExecutionState.FAILED.value: + return None + raise TransferExecutionConflictError("失败终态缺少匹配的结算回执") + + @staticmethod + def _validate_receipt( + *, + receipt: TransferSettlementReceipt, + settlement: TransferResultSettlement, + ) -> None: + """校验重放请求与独立回执中的终态身份完全一致。""" + expected_status = settlement.outcome == "succeeded" + if ( + receipt.task_id != settlement.task_id + or receipt.outcome != settlement.outcome + or receipt.execution_fingerprint != settlement.execution_fingerprint + or receipt.lease_token != settlement.lease_token + or receipt.history_status is not expected_status + or receipt.error != settlement.error + ): + raise TransferExecutionConflictError("整理终态与 durable 回执不一致") + + @staticmethod + def _settlement_revision( + *, + pending_repository: TransferPendingOper, + settlement: TransferResultSettlement | None, + ) -> int | None: + """从当前 pending 读取 CAS 基准修订号并校验执行身份。""" + if settlement is None: + return None + pending = pending_repository.get_by_task_id(task_id=settlement.task_id) + if pending is None: + raise TransferExecutionLeaseLostError("整理任务 pending 已不存在") + now_utc = TransactionalChainDurableEventWriter._format_utc( + datetime.now(timezone.utc) + ) + if ( + pending.lease_token != settlement.lease_token + or pending.lease_expires_at is None + or pending.lease_expires_at <= now_utc + ): + raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管") + if ( + pending.execution_state != TransferExecutionState.SETTLING.value + or pending.execution_fingerprint + != settlement.execution_fingerprint + ): + raise TransferExecutionConflictError("整理终态与执行检查点不匹配") + return int(pending.settlement_revision) + + @staticmethod + def _stage_pending_terminal( + *, + session: Session, + repository: TransferPendingOper, + settlement: TransferResultSettlement, + expected_revision: int, + history_id: int, + ) -> bool: + """以同一修订和 lease CAS 收口 pending，成功时同时清理步骤。""" + now = datetime.now(timezone.utc) + now_utc = TransactionalChainDurableEventWriter._format_utc(now) + if settlement.outcome == "succeeded": + TransferExecutionStepOper(session).stage_delete_task( + task_id=settlement.task_id + ) + updated = repository.stage_delete_terminal_success( + task_id=settlement.task_id, + lease_token=settlement.lease_token, + execution_fingerprint=settlement.execution_fingerprint, + expected_revision=expected_revision, + now_utc=now_utc, + ) + pending_deleted = True + else: + updated = repository.stage_terminal_failure( + task_id=settlement.task_id, + lease_token=settlement.lease_token, + execution_fingerprint=settlement.execution_fingerprint, + expected_revision=expected_revision, + history_id=history_id, + error=settlement.error, + now_utc=now_utc, + updated_at=now.astimezone().strftime("%Y-%m-%d %H:%M:%S"), + ) + pending_deleted = False + if updated != 1: + session.expire_all() + current = repository.get_by_task_id(task_id=settlement.task_id) + if ( + current is None + or current.lease_token != settlement.lease_token + or current.lease_expires_at is None + or current.lease_expires_at <= now_utc + ): + raise TransferExecutionLeaseLostError( + "整理任务租约已失效或被其他 worker 接管" + ) + raise TransferExecutionConflictError("整理终态结算版本发生冲突") + return pending_deleted + + @staticmethod + def _validate_history_outcome( + history: TransferHistoryRef, + settlement: TransferResultSettlement, + ) -> None: + """拒绝结算终态与历史状态不一致的调用。""" + expected_status = settlement.outcome == "succeeded" + if history.status is not expected_status: + raise TransferExecutionConflictError("整理终态与历史状态不一致") + + @staticmethod + def _format_utc(value: datetime) -> str: + """编码与 pending lease 列一致的固定宽度 UTC 时间。""" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") diff --git a/app/db/adapters/transfer.py b/app/db/adapters/transfer.py index 51d48ff2c..5cac4dd46 100644 --- a/app/db/adapters/transfer.py +++ b/app/db/adapters/transfer.py @@ -145,6 +145,8 @@ class TransactionalTransferAdmissionRepository: planning_input: Optional[TransferPlanningInput] = None, ) -> TransferAdmission: """按输入指纹幂等持久化准入事实，并返回跨重启稳定身份。""" + if not storage or not src_path: + raise ValueError("整理任务的存储与源路径不能为空") effective_input = planning_input or TransferPlanningInput.legacy( storage=storage, src_path=src_path, @@ -170,7 +172,9 @@ class TransactionalTransferAdmissionRepository: input_fingerprint=effective_input.fingerprint, ) if pending is None: - raise ValueError("整理任务的存储与源路径不能为空") + raise TransferAdmissionConflictError( + f"整理源文件已有持久终态回执: {storage}:{src_path}" + ) session.flush() self._assert_input_match(pending, effective_input) admission = self._project(pending) diff --git a/app/db/adapters/transfer_execution.py b/app/db/adapters/transfer_execution.py new file mode 100644 index 000000000..f8a34793c --- /dev/null +++ b/app/db/adapters/transfer_execution.py @@ -0,0 +1,1030 @@ +"""整理步骤执行与终态结算端口的 SQLAlchemy 短事务适配器。""" + +from __future__ import annotations + +from collections.abc import Callable +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import and_, func, or_, select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferExecutionConflictError, + TransferExecutionLeaseLostError, + TransferExecutionSnapshot, + TransferExecutionState, + TransferExecutionStep, + TransferManualReviewDecision, + TransferManualReviewPage, + TransferManualReviewResult, + TransferManualReviewSource, + TransferManualReviewStepView, + TransferManualReviewTaskView, + TransferRetryRequestResult, + TransferStepIntent, + TransferStepResult, + TransferStepState, +) +from app.db.models.transferexecutionstep import ( + TransferExecutionStep as TransferExecutionStepModel, +) +from app.db.models.transferpending import TransferPending +from app.db.oper.transferexecutionstep import TransferExecutionStepOper +from app.db.oper.transferpending import TransferPendingOper +from app.db.uow import SqlAlchemyUnitOfWork + +_LEGACY_REVIEW_STEP_KIND = "legacy_execution_review" + + +def _local_now() -> datetime: + """返回业务审计字段使用的宿主本地时间。""" + return datetime.now() + + +def _utc_now() -> datetime: + """返回租约 fencing 使用的 UTC 时间。""" + return datetime.now(timezone.utc) + + +class TransactionalTransferExecutionRepository: + """以操作级短 Session 实现整理执行持久化端口。""" + + def __init__( + self, + session_factory: Callable[[], Session], + *, + local_clock: Callable[[], datetime] = _local_now, + lease_clock: Callable[[], datetime] = _utc_now, + ) -> None: + """保存 Session 工厂及可测试时钟，不持有跨外部 I/O 事务。""" + self._session_factory = session_factory + self._local_clock = local_clock + self._lease_clock = lease_clock + + @staticmethod + def _format_local(value: datetime) -> str: + """编码与既有业务审计列一致的本地时间。""" + return value.strftime("%Y-%m-%d %H:%M:%S") + + @staticmethod + def _format_utc(value: datetime) -> str: + """编码可按字典序比较的固定宽度 UTC 租约时间。""" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + + @staticmethod + def _execution_steps( + steps: list[TransferExecutionStepModel], + ) -> list[TransferExecutionStepModel]: + """排除只用于迁移人工审计、不得参与真实执行的 synthetic 步骤。""" + return [step for step in steps if step.kind != _LEGACY_REVIEW_STEP_KIND] + + @staticmethod + def _project_step(step: TransferExecutionStepModel) -> TransferExecutionStep: + """在 Session 有效期内冻结 ORM 步骤为 Application DTO。""" + if (step.result_version is None) != (step.result_payload is None): + raise TransferExecutionConflictError("整理步骤结果版本与 payload 不完整") + result = ( + TransferStepResult( + version=step.result_version, + payload=dict(step.result_payload), + ) + if step.result_version is not None and step.result_payload is not None + else None + ) + intent = TransferStepIntent( + operation_id=step.operation_id, + checkpoint_fingerprint=step.checkpoint_fingerprint, + ordinal=step.ordinal, + phase=step.phase, + kind=step.kind, + payload=dict(step.intent_payload), + version=step.intent_version, + ) + return TransferExecutionStep( + task_id=step.task_id, + operation_id=step.operation_id, + checkpoint_fingerprint=step.checkpoint_fingerprint, + ordinal=step.ordinal, + phase=step.phase, + kind=step.kind, + state=TransferStepState(step.state), + attempt_token=step.attempt_token, + attempt_count=step.attempt_count, + intent=intent, + result=result, + last_error=step.last_error, + prepared_at=step.prepared_at, + started_at=step.started_at, + completed_at=step.completed_at, + updated_at=step.updated_at, + ) + + @classmethod + def _project_snapshot( + cls, + pending: TransferPending, + steps: list[TransferExecutionStepModel], + ) -> TransferExecutionSnapshot: + """在 Session 有效期内冻结 pending 与步骤聚合状态。""" + has_checkpoint_column = pending.execution_payload is not None + has_checkpoint_identity = ( + pending.execution_version is not None + and pending.execution_fingerprint is not None + ) + if has_checkpoint_column != has_checkpoint_identity: + raise TransferExecutionConflictError("整理执行检查点列不完整") + checkpoint = None + if pending.execution_payload is not None: + checkpoint = TransferExecutionCheckpoint.from_payload( + pending.execution_payload, + fingerprint=pending.execution_fingerprint or "", + ) + if checkpoint.version != pending.execution_version: + raise TransferExecutionConflictError("整理执行检查点列版本不一致") + return TransferExecutionSnapshot( + task_id=pending.task_id, + state=TransferExecutionState(pending.execution_state), + checkpoint=checkpoint, + retry_generation=pending.retry_generation, + retry_count=pending.retry_count, + retry_due_at=pending.retry_due_at, + settlement_revision=pending.settlement_revision, + terminal_history_id=pending.terminal_history_id, + last_error=pending.last_error, + steps=tuple( + cls._project_step(step) + for step in cls._execution_steps(steps) + ), + ) + + @classmethod + def _project_manual_review( + cls, + pending: TransferPending, + steps: list[TransferExecutionStepModel], + ) -> TransferManualReviewTaskView: + """冻结人工复核详情，并只选择当前或最近一次复核步骤。""" + state = TransferExecutionState(pending.execution_state) + if state is TransferExecutionState.MANUAL_REVIEW: + candidates = [ + step + for step in steps + if step.state == TransferStepState.MANUAL_REVIEW.value + ] + elif ( + state is TransferExecutionState.RETRY_WAIT + and pending.manual_review_revision > 0 + ): + candidates = [ + step + for step in steps + if step.state in { + TransferStepState.FAILED.value, + TransferStepState.SUCCEEDED.value, + } + and step.updated_at == pending.reviewed_at + ] + else: + raise TransferExecutionConflictError("整理任务不属于公开人工复核状态") + if not candidates: + raise TransferExecutionConflictError("人工复核任务缺少对应步骤证据") + step = max(candidates, key=lambda item: (item.ordinal, item.id)) + evidence = dict(step.result_payload) if step.result_payload is not None else None + return TransferManualReviewTaskView( + task_id=pending.task_id, + source=TransferManualReviewSource( + storage=pending.storage, + path=pending.src_path, + ), + state=state, + step=TransferManualReviewStepView( + operation_id=step.operation_id, + kind=step.kind, + intent=dict(step.intent_payload), + evidence=evidence, + error=step.last_error, + ), + review_revision=pending.manual_review_revision, + ) + + @staticmethod + def _intent_matches( + step: TransferExecutionStepModel, + *, + task_id: str, + intent: TransferStepIntent, + ) -> bool: + """判断同 operation ID 的既有步骤是否为完全相同的冻结意图。""" + return all(( + step.task_id == task_id, + step.checkpoint_fingerprint == intent.checkpoint_fingerprint, + step.ordinal == intent.ordinal, + step.phase == intent.phase, + step.kind == intent.kind, + step.intent_version == intent.version, + step.intent_payload == intent.payload, + )) + + @staticmethod + def _raise_fenced_failure( + pending: Optional[TransferPending], + *, + lease_token: str, + now_utc: str, + detail: str, + ) -> None: + """区分租约丢失与同租约内的状态或 attempt 冲突。""" + if ( + pending is None + or pending.lease_token != lease_token + or pending.lease_expires_at is None + or pending.lease_expires_at <= now_utc + ): + raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管") + raise TransferExecutionConflictError(detail) + + def _times(self) -> tuple[str, str]: + """生成一次事务内共享的 UTC fencing 与本地审计时间。""" + return ( + self._format_utc(self._lease_clock()), + self._format_local(self._local_clock()), + ) + + @staticmethod + def _rollback(transaction: SqlAlchemyUnitOfWork) -> None: + """统一回滚失败的操作级事务。""" + transaction.rollback() + + def get_snapshot(self, *, task_id: str) -> Optional[TransferExecutionSnapshot]: + """使用一次只读短 Session 获取任务执行快照。""" + with self._session_factory() as session: + pending = TransferPendingOper(session).get_by_task_id(task_id=task_id) + if pending is None: + return None + steps = TransferExecutionStepOper(session).list_by_task_id(task_id=task_id) + return self._project_snapshot(pending, steps) + + def list_manual_reviews( + self, + *, + state: TransferExecutionState, + page: int, + page_size: int, + ) -> TransferManualReviewPage: + """在数据库内过滤和分页人工复核任务，再冻结步骤证据。""" + if state not in { + TransferExecutionState.MANUAL_REVIEW, + TransferExecutionState.RETRY_WAIT, + }: + raise ValueError(f"人工复核查询不支持状态：{state.value}") + state_predicate = TransferPending.execution_state == state.value + if state is TransferExecutionState.RETRY_WAIT: + state_predicate = and_( + state_predicate, + TransferPending.manual_review_revision > 0, + ) + with self._session_factory() as session: + total = int(session.scalar( + select(func.count()).select_from(TransferPending).where(state_predicate) + ) or 0) + pendings = list(session.scalars( + select(TransferPending) + .where(state_predicate) + .order_by(TransferPending.updated_at.desc(), TransferPending.id.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + ).all()) + step_oper = TransferExecutionStepOper(session) + items = tuple( + self._project_manual_review( + pending, + step_oper.list_by_task_id(task_id=pending.task_id), + ) + for pending in pendings + ) + return TransferManualReviewPage( + items=items, + total=total, + page=page, + page_size=page_size, + ) + + def get_manual_review( + self, + *, + task_id: str, + ) -> Optional[TransferManualReviewTaskView]: + """读取待人工复核或已判定等待恢复的单个任务。""" + with self._session_factory() as session: + pending = session.scalar( + select(TransferPending).where( + TransferPending.task_id == task_id, + or_( + TransferPending.execution_state + == TransferExecutionState.MANUAL_REVIEW.value, + and_( + TransferPending.execution_state + == TransferExecutionState.RETRY_WAIT.value, + TransferPending.manual_review_revision > 0, + ), + ), + ) + ) + if pending is None: + return None + return self._project_manual_review( + pending, + TransferExecutionStepOper(session).list_by_task_id(task_id=task_id), + ) + + def prepare_step( + self, + *, + task_id: str, + lease_token: str, + intent: TransferStepIntent, + ) -> TransferExecutionStep: + """在外部副作用前以有效 lease 幂等提交稳定步骤意图。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + updated = pending_oper.stage_execution_running( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务当前状态不能准备外部步骤", + ) + oper = TransferExecutionStepOper(session) + step = oper.get_by_operation_id(operation_id=intent.operation_id) + if step is None: + step = oper.stage_prepare( + task_id=task_id, + operation_id=intent.operation_id, + checkpoint_fingerprint=intent.checkpoint_fingerprint, + ordinal=intent.ordinal, + phase=intent.phase, + kind=intent.kind, + intent_version=intent.version, + intent_payload=dict(intent.payload), + now_time=updated_at, + ) + session.flush() + elif not self._intent_matches(step, task_id=task_id, intent=intent): + raise TransferExecutionConflictError( + "稳定 operation ID 已绑定不同步骤意图" + ) + projected = self._project_step(step) + transaction.commit() + return projected + except IntegrityError as error: + self._rollback(transaction) + raise TransferExecutionConflictError( + "整理步骤 operation ID 或全局序号发生并发冲突" + ) from error + except Exception: + self._rollback(transaction) + raise + + def start_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + ) -> TransferExecutionStep: + """以当前 lease 把 PREPARED 步骤推进为 STARTED。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferExecutionStepOper(session) + updated = oper.stage_start_attempt( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + TransferPendingOper(session).get_by_task_id(task_id=task_id), + lease_token=lease_token, + now_utc=now_utc, + detail="整理步骤不是可开始的 PREPARED 状态", + ) + session.expire_all() + step = oper.get_by_operation_id(operation_id=operation_id) + if step is None: + raise TransferExecutionConflictError("整理步骤开始后无法回读") + projected = self._project_step(step) + transaction.commit() + return projected + except Exception: + self._rollback(transaction) + raise + + def restart_after_not_applied( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + previous_attempt_token: str, + attempt_token: str, + evidence: TransferStepResult, + ) -> TransferExecutionStep: + """以严格 NOT_APPLIED 证据和旧 attempt CAS 重启遗留步骤。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferExecutionStepOper(session) + updated = oper.stage_restart_after_not_applied( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + previous_attempt_token=previous_attempt_token, + attempt_token=attempt_token, + result_version=evidence.version, + result_payload=dict(evidence.payload), + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + TransferPendingOper(session).get_by_task_id(task_id=task_id), + lease_token=lease_token, + now_utc=now_utc, + detail="遗留步骤状态或旧 attempt token 已变化", + ) + session.expire_all() + step = oper.get_by_operation_id(operation_id=operation_id) + if step is None: + raise TransferExecutionConflictError("整理步骤重启后无法回读") + projected = self._project_step(step) + transaction.commit() + return projected + except Exception: + self._rollback(transaction) + raise + + def resume_failed_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + ) -> TransferExecutionStep: + """在 retry_wait 到期并重新 claim 后恢复同一 FAILED 操作。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + pending_updated = pending_oper.stage_execution_running( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + updated_at=updated_at, + ) + if pending_updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="重试任务未到期或当前状态不能恢复", + ) + oper = TransferExecutionStepOper(session) + updated = oper.stage_resume_failed_attempt( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理步骤不是可恢复的 FAILED 状态", + ) + session.expire_all() + step = oper.get_by_operation_id(operation_id=operation_id) + if step is None: + raise TransferExecutionConflictError("恢复重试后无法回读整理步骤") + projected = self._project_step(step) + transaction.commit() + return projected + except Exception: + self._rollback(transaction) + raise + + def complete_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + result: TransferStepResult, + ) -> TransferExecutionStep: + """以 lease 与 attempt 双 CAS 提交外部操作成功证据。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferExecutionStepOper(session) + updated = oper.stage_complete_attempt( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + result_version=result.version, + result_payload=dict(result.payload), + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + TransferPendingOper(session).get_by_task_id(task_id=task_id), + lease_token=lease_token, + now_utc=now_utc, + detail="整理步骤 attempt token 已变化或状态不是 STARTED", + ) + session.expire_all() + step = oper.get_by_operation_id(operation_id=operation_id) + if step is None: + raise TransferExecutionConflictError("整理步骤完成后无法回读") + projected = self._project_step(step) + transaction.commit() + return projected + except Exception: + self._rollback(transaction) + raise + + def defer_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + retry_due_at: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """原子提交已知失败证据、重试世代和到期时间并释放 lease。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + step_oper = TransferExecutionStepOper(session) + updated = step_oper.stage_fail_attempt( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + error=error, + result_version=evidence.version if evidence else None, + result_payload=dict(evidence.payload) if evidence else None, + now_utc=now_utc, + updated_at=updated_at, + ) + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理步骤已知失败证据与当前 attempt 冲突", + ) + pending_updated = pending_oper.stage_defer_execution( + task_id=task_id, + lease_token=lease_token, + error=error, + retry_due_at=retry_due_at, + now_utc=now_utc, + updated_at=updated_at, + ) + if pending_updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务不能进入重试等待", + ) + session.flush() + session.expire_all() + pending = pending_oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("重试任务无法回读") + snapshot = self._project_snapshot( + pending, + step_oper.list_by_task_id(task_id=task_id), + ) + transaction.commit() + return snapshot + except Exception: + self._rollback(transaction) + raise + + def exhaust_step( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + evidence: Optional[TransferStepResult] = None, + ) -> TransferExecutionSnapshot: + """原子提交预算耗尽步骤失败，并保留 lease 建立失败结算检查点。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + step_oper = TransferExecutionStepOper(session) + updated = step_oper.stage_fail_attempt( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + error=error, + result_version=evidence.version if evidence else None, + result_payload=dict(evidence.payload) if evidence else None, + now_utc=now_utc, + updated_at=updated_at, + ) + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="预算耗尽失败证据与当前 attempt 冲突", + ) + session.flush() + steps = self._execution_steps( + step_oper.list_by_task_id(task_id=task_id) + ) + checkpoint = TransferExecutionCheckpoint.create( + payload={ + "outcome": "failed", + "failed_operation_id": operation_id, + "error": error, + "evidence": ( + { + "schema_version": evidence.version, + "payload": dict(evidence.payload), + } + if evidence + else None + ), + }, + operation_ids=tuple(step.operation_id for step in steps), + ) + pending_updated = pending_oper.stage_checkpoint_exhausted_failure( + task_id=task_id, + lease_token=lease_token, + execution_version=checkpoint.version, + execution_payload=checkpoint.to_payload(), + execution_fingerprint=checkpoint.fingerprint, + error=error, + now_utc=now_utc, + updated_at=updated_at, + ) + if pending_updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务不能进入失败结算状态", + ) + session.flush() + session.expire_all() + pending = pending_oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("失败结算任务无法回读") + snapshot = self._project_snapshot(pending, steps) + transaction.commit() + return snapshot + except Exception: + self._rollback(transaction) + raise + + def request_retry( + self, + *, + task_id: str, + reason: str, + requested_by: str, + ) -> TransferRetryRequestResult: + """以短事务登记用户重试意图，不 claim、执行或删除既有证据。""" + retry_due_at, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferPendingOper(session) + pending = oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("未找到可重试的整理任务") + state = TransferExecutionState(pending.execution_state) + if state is TransferExecutionState.RETRY_WAIT: + return TransferRetryRequestResult( + accepted=True, + state=state, + retry_generation=pending.retry_generation, + message="整理任务已在等待重试", + ) + if state is not TransferExecutionState.FAILED: + message = ( + "人工复核任务必须先完成专门判定" + if state is TransferExecutionState.MANUAL_REVIEW + else "整理任务当前状态不接受用户重试" + ) + return TransferRetryRequestResult( + accepted=False, + state=state, + retry_generation=pending.retry_generation, + message=message, + ) + updated = oper.stage_request_execution_retry( + task_id=task_id, + reason=reason, + requested_by=requested_by, + retry_due_at=retry_due_at, + updated_at=updated_at, + ) + if updated != 1: + session.expire_all() + pending = oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("重试请求竞争后任务已不存在") + state = TransferExecutionState(pending.execution_state) + if state is TransferExecutionState.RETRY_WAIT: + return TransferRetryRequestResult( + accepted=True, + state=state, + retry_generation=pending.retry_generation, + message="整理任务已由并发请求登记重试", + ) + return TransferRetryRequestResult( + accepted=False, + state=state, + retry_generation=pending.retry_generation, + message="整理任务状态已变化，未登记重试", + ) + session.flush() + session.expire_all() + pending = oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("登记重试后任务无法回读") + result = TransferRetryRequestResult( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + retry_generation=pending.retry_generation, + message="整理任务已登记重试", + ) + transaction.commit() + return result + except Exception: + self._rollback(transaction) + raise + + def resolve_manual_review( + self, + *, + task_id: str, + operation_id: str, + decision: TransferManualReviewDecision, + actor: str, + reason: str, + result: Optional[TransferStepResult] = None, + ) -> TransferManualReviewResult: + """在无 lease 短事务中原子提交步骤与 pending 的人工判定。""" + if decision is TransferManualReviewDecision.FAILED: + raise TransferExecutionConflictError( + "人工失败终态尚不能绕过 lease durable 结算" + ) + if decision is TransferManualReviewDecision.APPLIED and result is None: + raise ValueError("人工判定已发生时必须提供结果证据") + retry_due_at, updated_at = self._times() + target_state = ( + TransferStepState.SUCCEEDED.value + if decision is TransferManualReviewDecision.APPLIED + else TransferStepState.FAILED.value + ) + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + step_oper = TransferExecutionStepOper(session) + step = step_oper.get_by_operation_id(operation_id=operation_id) + if ( + decision is TransferManualReviewDecision.APPLIED + and step is not None + and step.task_id == task_id + and step.kind == _LEGACY_REVIEW_STEP_KIND + ): + raise TransferExecutionConflictError( + "升级遗留步骤没有足够证据证明外部操作已发生；" + "请先人工回滚，或确认未发生后选择 NOT_APPLIED" + ) + step_updated = step_oper.stage_resolve_manual_review( + task_id=task_id, + operation_id=operation_id, + target_state=target_state, + reason=reason, + result_version=result.version if result else None, + result_payload=dict(result.payload) if result else None, + updated_at=updated_at, + ) + if step_updated != 1: + raise TransferExecutionConflictError( + "步骤已不处于可判定的无租约人工复核态" + ) + pending_oper = TransferPendingOper(session) + pending_updated = pending_oper.stage_resolve_manual_review( + task_id=task_id, + decision=decision.value, + actor=actor, + reason=reason, + retry_due_at=retry_due_at, + updated_at=updated_at, + ) + if pending_updated != 1: + raise TransferExecutionConflictError( + "任务已不处于可判定的无租约人工复核态" + ) + session.flush() + session.expire_all() + pending = pending_oper.get_by_task_id(task_id=task_id) + step = step_oper.get_by_operation_id(operation_id=operation_id) + if pending is None or step is None: + raise TransferExecutionConflictError("人工判定提交后无法回读") + resolved = TransferManualReviewResult( + task_id=task_id, + operation_id=operation_id, + decision=decision, + state=TransferExecutionState(pending.execution_state), + review_revision=pending.manual_review_revision, + step=self._project_step(step), + ) + transaction.commit() + return resolved + except Exception: + self._rollback(transaction) + raise + + def mark_manual_review( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + error: str, + evidence: Optional[TransferStepResult] = None, + attempt_token: Optional[str] = None, + ) -> TransferExecutionSnapshot: + """原子隔离外部结果未知的步骤与任务，并释放自动调度 lease。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + step_oper = TransferExecutionStepOper(session) + updated = step_oper.stage_manual_review( + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + error=error, + result_version=evidence.version if evidence else None, + result_payload=dict(evidence.payload) if evidence else None, + now_utc=now_utc, + updated_at=updated_at, + ) + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="人工复核证据与当前步骤状态冲突", + ) + pending_updated = pending_oper.stage_mark_execution_manual_review( + task_id=task_id, + lease_token=lease_token, + error=error, + now_utc=now_utc, + updated_at=updated_at, + ) + if pending_updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务不能进入人工复核", + ) + session.flush() + session.expire_all() + pending = pending_oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("人工复核任务无法回读") + snapshot = self._project_snapshot( + pending, + step_oper.list_by_task_id(task_id=task_id), + ) + transaction.commit() + return snapshot + except Exception: + self._rollback(transaction) + raise + + def checkpoint_execution( + self, + *, + task_id: str, + lease_token: str, + checkpoint: TransferExecutionCheckpoint, + ) -> TransferExecutionSnapshot: + """仅在完整步骤集合均成功时提交可重放终态执行检查点。""" + now_utc, updated_at = self._times() + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + step_oper = TransferExecutionStepOper(session) + steps = self._execution_steps( + step_oper.list_by_task_id(task_id=task_id) + ) + step_ids = {step.operation_id for step in steps} + if step_ids != set(checkpoint.operation_ids): + raise TransferExecutionConflictError( + "执行检查点引用的步骤集合与持久步骤不一致" + ) + if any(step.state != TransferStepState.SUCCEEDED.value for step in steps): + raise TransferExecutionConflictError( + "存在未成功步骤，不能提交执行检查点" + ) + pending_oper = TransferPendingOper(session) + pending = pending_oper.get_by_task_id(task_id=task_id) + running = pending_oper.stage_execution_running( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + updated_at=updated_at, + ) + if running != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务当前状态不能建立执行检查点", + ) + updated = pending_oper.stage_checkpoint_execution( + task_id=task_id, + lease_token=lease_token, + execution_version=checkpoint.version, + execution_payload=checkpoint.to_payload(), + execution_fingerprint=checkpoint.fingerprint, + now_utc=now_utc, + updated_at=updated_at, + ) + if updated != 1: + self._raise_fenced_failure( + pending, + lease_token=lease_token, + now_utc=now_utc, + detail="整理任务当前状态不能提交执行检查点", + ) + session.flush() + session.expire_all() + pending = pending_oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferExecutionConflictError("执行检查点任务无法回读") + snapshot = self._project_snapshot(pending, steps) + transaction.commit() + return snapshot + except Exception: + self._rollback(transaction) + raise diff --git a/app/db/models/__init__.py b/app/db/models/__init__.py index 38aa37e98..b44e8d134 100644 --- a/app/db/models/__init__.py +++ b/app/db/models/__init__.py @@ -37,6 +37,14 @@ _MODEL_EXPORTS = { ), "SystemConfig": ("app.db.models.systemconfig", "SystemConfig"), "TransferHistory": ("app.db.models.transferhistory", "TransferHistory"), + "TransferExecutionStep": ( + "app.db.models.transferexecutionstep", + "TransferExecutionStep", + ), + "TransferSettlementReceipt": ( + "app.db.models.transfersettlementreceipt", + "TransferSettlementReceipt", + ), "TransferPending": ("app.db.models.transferpending", "TransferPending"), "User": ("app.db.models.user", "User"), "UserConfig": ("app.db.models.userconfig", "UserConfig"), diff --git a/app/db/models/transferexecutionstep.py b/app/db/models/transferexecutionstep.py new file mode 100644 index 000000000..da4034d56 --- /dev/null +++ b/app/db/models/transferexecutionstep.py @@ -0,0 +1,470 @@ +"""整理任务外部操作步骤的持久化模型。""" + +from __future__ import annotations + +from typing import Any, Optional, cast + +from sqlalchemy import ( + JSON, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, + and_, + delete, + exists, + or_, + select, + update, +) +from sqlalchemy.orm import Mapped, Session, mapped_column +from sqlalchemy.sql.selectable import Exists + +from app.db.base import Base, execute_dml, get_id_column +from app.db.models.transferpending import TransferPending + + +class TransferExecutionStep(Base): + """保存一次稳定外部操作的意图、尝试身份与结果证据。""" + + id = get_id_column() + task_id: Mapped[str] = mapped_column( + String(64), + ForeignKey("transferpending.task_id", ondelete="CASCADE"), + nullable=False, + ) + operation_id: Mapped[str] = mapped_column(String(64), nullable=False) + checkpoint_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + ordinal: Mapped[int] = mapped_column(Integer, nullable=False) + phase: Mapped[str] = mapped_column(String(32), nullable=False) + kind: Mapped[str] = mapped_column(String(32), nullable=False) + state: Mapped[str] = mapped_column(String(32), nullable=False, default="prepared") + attempt_token: Mapped[Optional[str]] = mapped_column(String(64)) + attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + intent_version: Mapped[int] = mapped_column(Integer, nullable=False) + intent_payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) + result_version: Mapped[Optional[int]] = mapped_column(Integer) + result_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON) + last_error: Mapped[Optional[str]] = mapped_column(Text) + prepared_at: Mapped[str] = mapped_column(String(40), nullable=False) + started_at: Mapped[Optional[str]] = mapped_column(String(40)) + completed_at: Mapped[Optional[str]] = mapped_column(String(40)) + updated_at: Mapped[str] = mapped_column(String(40), nullable=False) + + __table_args__ = ( + UniqueConstraint("operation_id", name="uq_transferexecutionstep_operation_id"), + UniqueConstraint( + "task_id", + "ordinal", + name="uq_transferexecutionstep_task_ordinal", + ), + Index( + "ix_transferexecutionstep_task_state_ordinal", + "task_id", + "state", + "ordinal", + ), + ) + + @classmethod + def get_by_operation_id( + cls, + db: Session, + *, + operation_id: str, + ) -> Optional["TransferExecutionStep"]: + """按稳定操作标识读取步骤。""" + if not operation_id: + return None + return cast( + Optional["TransferExecutionStep"], + db.execute( + select(cls).where(cls.operation_id == operation_id) + ).scalars().first(), + ) + + @classmethod + def list_by_task_id( + cls, + db: Session, + *, + task_id: str, + ) -> list["TransferExecutionStep"]: + """按全局序号读取任务的全部外部操作步骤。""" + if not task_id: + return [] + return list( + db.execute( + select(cls) + .where(cls.task_id == task_id) + .order_by(cls.ordinal.asc()) + ).scalars().all() + ) + + @classmethod + def stage_prepare( + cls, + db: Session, + *, + task_id: str, + operation_id: str, + checkpoint_fingerprint: str, + ordinal: int, + phase: str, + kind: str, + intent_version: int, + intent_payload: dict[str, Any], + now_time: str, + ) -> "TransferExecutionStep": + """在调用方事务中暂存尚未执行的稳定步骤意图。""" + step = cls( + task_id=task_id, + operation_id=operation_id, + checkpoint_fingerprint=checkpoint_fingerprint, + ordinal=ordinal, + phase=phase, + kind=kind, + state="prepared", + attempt_count=0, + intent_version=intent_version, + intent_payload=intent_payload, + prepared_at=now_time, + updated_at=now_time, + ) + db.add(step) + return step + + @classmethod + def start_attempt( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效任务租约 CAS 开始一次新的步骤尝试。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "prepared", + cls.attempt_token.is_(None), + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + state="started", + attempt_token=attempt_token, + attempt_count=cls.attempt_count + 1, + started_at=updated_at, + completed_at=None, + last_error=None, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def restart_after_not_applied( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + previous_attempt_token: str, + attempt_token: str, + result_version: int, + result_payload: dict[str, Any], + now_utc: str, + updated_at: str, + ) -> int: + """以 NOT_APPLIED 证据和旧 attempt token CAS 重启遗留步骤。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "started", + cls.attempt_token == previous_attempt_token, + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + attempt_token=attempt_token, + attempt_count=cls.attempt_count + 1, + result_version=result_version, + result_payload=result_payload, + started_at=updated_at, + completed_at=None, + last_error=None, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def resume_failed_attempt( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以重试调度的新 lease CAS 恢复 FAILED 步骤并保留失败证据。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "failed", + cls.attempt_token.is_(None), + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + state="started", + attempt_token=attempt_token, + attempt_count=cls.attempt_count + 1, + started_at=updated_at, + completed_at=None, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def complete_attempt( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + result_version: int, + result_payload: dict[str, Any], + now_utc: str, + updated_at: str, + ) -> int: + """以租约与 attempt 双 CAS 提交步骤成功证据。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "started", + cls.attempt_token == attempt_token, + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + state="succeeded", + attempt_token=None, + result_version=result_version, + result_payload=result_payload, + last_error=None, + completed_at=updated_at, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def fail_attempt( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + now_utc: str, + updated_at: str, + ) -> int: + """以租约与 attempt 双 CAS 提交已知失败证据。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "started", + cls.attempt_token == attempt_token, + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + state="failed", + attempt_token=None, + result_version=result_version, + result_payload=result_payload, + last_error=error, + completed_at=updated_at, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def mark_manual_review( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: Optional[str], + error: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + now_utc: str, + updated_at: str, + ) -> int: + """以当前尝试身份隔离外部结果不可判定的步骤。""" + attempt_match = ( + cls.attempt_token == attempt_token + if attempt_token is not None + else cls.attempt_token.is_(None) + ) + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state.in_(("prepared", "started", "failed")), + attempt_match, + cls._active_lease_exists( + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + ), + ) + .values( + state="manual_review", + attempt_token=None, + result_version=result_version, + result_payload=result_payload, + last_error=error, + completed_at=updated_at, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def resolve_manual_review( + cls, + db: Session, + *, + task_id: str, + operation_id: str, + target_state: str, + reason: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + updated_at: str, + ) -> int: + """仅在 pending 同为无租约人工态时 CAS 提交步骤判定。""" + if target_state not in {"failed", "succeeded"}: + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.operation_id == operation_id, + cls.state == "manual_review", + cls.attempt_token.is_(None), + cls._manual_review_pending_exists(task_id=task_id), + ) + .values( + state=target_state, + result_version=result_version, + result_payload=result_payload, + last_error=(reason if target_state == "failed" else None), + completed_at=updated_at, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def delete_by_task_id(cls, db: Session, *, task_id: str) -> int: + """在终态成功结算事务中删除任务的步骤证据。""" + if not task_id: + return 0 + return execute_dml( + db, + delete(cls).where(cls.task_id == task_id), + execution_options={"synchronize_session": False}, + ) + + @staticmethod + def _active_lease_exists( + *, + task_id: str, + lease_token: str, + now_utc: str, + ) -> Exists: + """构造关联 pending 行仍持有当前有效租约的 SQL 谓词。""" + return exists( + select(TransferPending.id).where( + and_( + TransferPending.task_id == task_id, + TransferPending.lease_token == lease_token, + TransferPending.lease_expires_at.is_not(None), + TransferPending.lease_expires_at > now_utc, + or_( + TransferPending.execution_state == "running", + TransferPending.execution_state == "not_started", + ), + ) + ) + ) + + @staticmethod + def _manual_review_pending_exists(*, task_id: str) -> Exists: + """构造关联 pending 行处于无租约人工复核态的 SQL 谓词。""" + return exists( + select(TransferPending.id).where( + TransferPending.task_id == task_id, + TransferPending.execution_state == "manual_review", + TransferPending.lease_token.is_(None), + TransferPending.lease_owner.is_(None), + ) + ) diff --git a/app/db/models/transferhistory.py b/app/db/models/transferhistory.py index bbea0f5ee..dc40003a6 100644 --- a/app/db/models/transferhistory.py +++ b/app/db/models/transferhistory.py @@ -1,9 +1,9 @@ import re import time from pathlib import Path -from typing import Any, List, Optional +from typing import Any, List, Optional, cast -from sqlalchemy import Boolean, Index, Integer, JSON, String, delete, func, or_, select, update +from sqlalchemy import JSON, Boolean, Index, Integer, String, delete, func, or_, select, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Mapped, Session, mapped_column @@ -24,6 +24,10 @@ class TransferHistory(Base): 整理记录 """ id = get_id_column() + # 失败 pending 当前映射使用的稳定整理任务标识 + transfer_task_id: Mapped[Optional[str]] = mapped_column(String(64)) + # 失败 pending 当前映射对应的结算版本 + transfer_settlement_revision: Mapped[Optional[int]] = mapped_column(Integer) # 源路径 src: Mapped[Optional[str]] = mapped_column(String, index=True) # 源存储 @@ -90,6 +94,11 @@ class TransferHistory(Base): Index('ix_transferhistory_date_id', 'date', 'id'), Index('ix_transferhistory_media_identity', 'media_source', 'media_id'), Index('ux_transferhistory_src_storage', 'src', 'src_storage', unique=True), + Index( + 'ux_transferhistory_transfer_task_id', + 'transfer_task_id', + unique=True, + ), ) @classmethod @@ -208,6 +217,82 @@ class TransferHistory(Base): statement = statement.where(cls.src_storage == storage) return db.execute(statement.order_by(cls.id.desc())).scalars().first() + @classmethod + def get_by_transfer_task_id( + cls, + db: Session, + *, + task_id: str, + ) -> Optional["TransferHistory"]: + """按稳定整理任务标识读取终态结算历史。""" + if not task_id: + return None + return cast( + Optional["TransferHistory"], + db.execute( + select(cls).where(cls.transfer_task_id == task_id) + ).scalars().first(), + ) + + @classmethod + def upsert_by_transfer_task_id( + cls, + db: Session, + *, + task_id: str, + settlement_revision: int, + retain_task_mapping: bool, + payload: dict[str, Any], + ) -> "TransferHistory": + """按任务幂等写历史，并维持同源存储仅保留一条的既有约束。""" + if not task_id or settlement_revision <= 0: + raise ValueError("整理历史结算缺少稳定任务或正向版本") + column_names = {column.name for column in cls.__table__.columns} + values = { + key: value + for key, value in payload.items() + if key in column_names and key not in { + "id", + "transfer_task_id", + "transfer_settlement_revision", + } + } + src = values.get("src") + if not src: + raise ValueError("整理历史结算缺少源路径") + src_storage = values.get("src_storage") or "local" + values["src_storage"] = src_storage + history = cls.get_by_transfer_task_id(db, task_id=task_id) + if history is None: + history = db.execute( + select(cls).where( + cls.src == src, + cls.src_storage == src_storage, + ) + ).scalars().first() + if history is None: + history = cls( + transfer_task_id=(task_id if retain_task_mapping else None), + transfer_settlement_revision=( + settlement_revision if retain_task_mapping else None + ), + **values, + ) + db.add(history) + else: + if (history.transfer_task_id == task_id + and history.transfer_settlement_revision is not None + and settlement_revision <= history.transfer_settlement_revision): + raise ValueError("整理历史结算版本必须单调递增") + history.transfer_task_id = task_id if retain_task_mapping else None + history.transfer_settlement_revision = ( + settlement_revision if retain_task_mapping else None + ) + for key, value in values.items(): + setattr(history, key, value) + db.flush() + return history + @classmethod def get_success_by_src( cls, db: Session, src: str, @@ -557,10 +642,20 @@ class TransferHistory(Base): src_storage = kwargs.get("src_storage") or "local" kwargs["src_storage"] = src_storage if src: + durable = db.execute( + select(cls.id).where( + cls.src == src, + cls.src_storage == src_storage, + cls.transfer_task_id.is_not(None), + ) + ).scalar_one_or_none() + if durable is not None: + raise ValueError("持久整理回执不能由旧历史写入口覆盖") db.execute( delete(cls).where( cls.src == src, cls.src_storage == src_storage, + cls.transfer_task_id.is_(None), ), execution_options={"synchronize_session": False}, ) @@ -590,7 +685,10 @@ class TransferHistory(Base): """ ids = db.execute( select(cls.id) - .where(cls.date < before_time) + .where( + cls.date < before_time, + cls.transfer_task_id.is_(None), + ) .order_by(cls.id.asc()) .limit(limit) ).scalars().all() diff --git a/app/db/models/transferpending.py b/app/db/models/transferpending.py index 3964a4385..fc0650b67 100644 --- a/app/db/models/transferpending.py +++ b/app/db/models/transferpending.py @@ -134,6 +134,42 @@ class TransferPending(Base): heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40)) # 真正取得新 token 的累计次数 attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 与规划状态正交的执行状态 + execution_state: Mapped[str] = mapped_column( + String(32), nullable=False, default="not_started" + ) + # 聚合执行检查点格式版本 + execution_version: Mapped[Optional[int]] = mapped_column(Integer) + # 可独立重放终态结算的聚合执行结果 + execution_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON) + # 聚合执行结果规范 JSON 的 SHA-256 指纹 + execution_fingerprint: Mapped[Optional[str]] = mapped_column(String(64)) + # 每次进入 retry_wait 都递增的调度世代 + retry_generation: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 已持久提交的步骤重试次数 + retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 下一次允许 claim 的 UTC 时间 + retry_due_at: Mapped[Optional[str]] = mapped_column(String(40)) + # 最近一次终态失败重试请求身份 + retry_requested_by: Mapped[Optional[str]] = mapped_column(String(128)) + # 最近一次终态失败重试请求原因 + retry_reason: Mapped[Optional[str]] = mapped_column(Text) + # 已完成终态结算的单调版本 + settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + # 失败终态保留的整理历史标识 + terminal_history_id: Mapped[Optional[int]] = mapped_column(Integer) + # 人工判定的单调审计版本 + manual_review_revision: Mapped[int] = mapped_column( + Integer, nullable=False, default=0 + ) + # 最近一次人工判定时间 + reviewed_at: Mapped[Optional[str]] = mapped_column(String(40)) + # 最近一次人工判定操作者 + reviewed_by: Mapped[Optional[str]] = mapped_column(String(128)) + # 最近一次人工判定原因 + review_reason: Mapped[Optional[str]] = mapped_column(Text) + # 最近一次人工判定结论 + review_decision: Mapped[Optional[str]] = mapped_column(String(32)) __table_args__ = ( # 同一个文件重复入队只保留一条，回放时不会重复送入整理链 @@ -153,6 +189,14 @@ class TransferPending(Base): "created_at", "id", ), + Index( + "ix_transferpending_execution_due", + "execution_state", + "retry_due_at", + "state", + "created_at", + "id", + ), UniqueConstraint("task_id", name="uq_transferpending_task_id"), ) @@ -264,6 +308,19 @@ class TransferPending(Base): cursor_created_at = func.coalesce(cls.created_at, "") statement = select(cls.task_id, cursor_created_at, cls.id).where( cls.state.in_(states), + cls.execution_state.in_(( + "not_started", + "running", + "retry_wait", + "settling", + )), + or_( + cls.execution_state != "retry_wait", + and_( + cls.retry_due_at.is_not(None), + cls.retry_due_at <= now_time, + ), + ), or_( cls.lease_token.is_(None), cls.lease_expires_at.is_(None), @@ -331,6 +388,19 @@ class TransferPending(Base): .where( cls.task_id == task_id, cls.state.in_(states), + cls.execution_state.in_(( + "not_started", + "running", + "retry_wait", + "settling", + )), + or_( + cls.execution_state != "retry_wait", + and_( + cls.retry_due_at.is_not(None), + cls.retry_due_at <= now_time, + ), + ), or_( cls.lease_token.is_(None), cls.lease_expires_at.is_(None), @@ -348,6 +418,345 @@ class TransferPending(Base): execution_options={"synchronize_session": False}, ) + @classmethod + def stage_execution_running( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约把可执行任务推进或保持为 running。""" + if not all((task_id, lease_token, now_utc, updated_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + cls.execution_state.in_(("not_started", "running", "retry_wait")), + ) + .values( + execution_state="running", + retry_due_at=None, + last_error=None, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def defer_execution( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + error: str, + retry_due_at: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约进入 retry_wait，并原子释放当前租约。""" + if not all((task_id, lease_token, error, retry_due_at, now_utc, updated_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state == "planned", + cls.checkpoint_version.is_not(None), + cls.checkpoint_payload.is_not(None), + cls.execution_state == "running", + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ) + .values( + execution_state="retry_wait", + retry_generation=cls.retry_generation + 1, + retry_count=cls.retry_count + 1, + retry_due_at=retry_due_at, + lease_owner=None, + lease_token=None, + lease_expires_at=None, + heartbeat_at=None, + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def mark_execution_manual_review( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + error: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约隔离执行结果未知的任务，并释放自动调度租约。""" + if not all((task_id, lease_token, error, now_utc, updated_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.execution_state.in_(("not_started", "running", "retry_wait")), + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ) + .values( + execution_state="manual_review", + lease_owner=None, + lease_token=None, + lease_expires_at=None, + heartbeat_at=None, + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def checkpoint_execution( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + execution_version: int, + execution_payload: dict[str, Any], + execution_fingerprint: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约保存可重放执行检查点并进入 settling。""" + if not all(( + task_id, + lease_token, + execution_version, + execution_payload, + execution_fingerprint, + now_utc, + updated_at, + )): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state == "planned", + cls.checkpoint_version.is_not(None), + cls.checkpoint_payload.is_not(None), + cls.execution_state == "running", + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ) + .values( + execution_state="settling", + execution_version=execution_version, + execution_payload=execution_payload, + execution_fingerprint=execution_fingerprint, + retry_due_at=None, + last_error=None, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def checkpoint_exhausted_failure( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + execution_version: int, + execution_payload: dict[str, Any], + execution_fingerprint: str, + error: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效 lease 保存预算耗尽失败检查点并保持租约进入 settling。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state == "planned", + cls.checkpoint_version.is_not(None), + cls.checkpoint_payload.is_not(None), + cls.execution_state == "running", + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ) + .values( + execution_state="settling", + execution_version=execution_version, + execution_payload=execution_payload, + execution_fingerprint=execution_fingerprint, + retry_due_at=None, + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def request_execution_retry( + cls, + db: Session, + *, + task_id: str, + reason: str, + requested_by: str, + retry_due_at: str, + updated_at: str, + ) -> int: + """仅将无租约 FAILED 任务 CAS 为立即到期的 retry_wait。""" + if not all((task_id, reason, requested_by, retry_due_at, updated_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.execution_state == "failed", + cls.lease_token.is_(None), + ) + .values( + execution_state="retry_wait", + retry_generation=cls.retry_generation + 1, + retry_due_at=retry_due_at, + retry_requested_by=requested_by, + retry_reason=reason, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def resolve_manual_review( + cls, + db: Session, + *, + task_id: str, + decision: str, + actor: str, + reason: str, + retry_due_at: str, + updated_at: str, + ) -> int: + """无 lease 地 CAS 提交人工判定审计并交回 retry_wait 调度。""" + if decision not in {"not_applied", "applied"}: + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.execution_state == "manual_review", + cls.lease_token.is_(None), + cls.lease_owner.is_(None), + ) + .values( + execution_state="retry_wait", + retry_generation=cls.retry_generation + 1, + retry_due_at=retry_due_at, + manual_review_revision=cls.manual_review_revision + 1, + reviewed_at=updated_at, + reviewed_by=actor, + review_reason=reason, + review_decision=decision, + last_error=(reason if decision == "not_applied" else None), + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def stage_terminal_failure( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + execution_fingerprint: str, + expected_revision: int, + history_id: int, + error: Optional[str], + now_utc: str, + updated_at: str, + ) -> int: + """以执行指纹和结算版本 CAS 保留失败终态及其历史。""" + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.execution_state == "settling", + cls.execution_fingerprint == execution_fingerprint, + cls.settlement_revision == expected_revision, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ) + .values( + execution_state="failed", + settlement_revision=cls.settlement_revision + 1, + terminal_history_id=history_id, + lease_owner=None, + lease_token=None, + lease_expires_at=None, + heartbeat_at=None, + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def delete_terminal_success( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + execution_fingerprint: str, + expected_revision: int, + now_utc: str, + ) -> int: + """以执行指纹和结算版本 CAS 删除已成功结算的 pending。""" + return execute_dml( + db, + delete(cls).where( + cls.task_id == task_id, + cls.execution_state == "settling", + cls.execution_fingerprint == execution_fingerprint, + cls.settlement_revision == expected_revision, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_utc, + ), + execution_options={"synchronize_session": False}, + ) + @classmethod def record_projection_failure( cls, diff --git a/app/db/models/transfersettlementreceipt.py b/app/db/models/transfersettlementreceipt.py new file mode 100644 index 000000000..451595096 --- /dev/null +++ b/app/db/models/transfersettlementreceipt.py @@ -0,0 +1,147 @@ +"""整理任务终态结算回执模型。""" + +from __future__ import annotations + +from typing import Optional, cast + +from sqlalchemy import Boolean, Index, Integer, String, Text, UniqueConstraint, desc, select +from sqlalchemy.orm import Mapped, Session, mapped_column + +from app.db.base import Base, get_id_column + + +class TransferSettlementReceipt(Base): + """按任务保存独立于最新历史投影的 durable 终态证据。""" + + id = get_id_column() + task_id: Mapped[str] = mapped_column(String(64), nullable=False) + history_id: Mapped[int] = mapped_column(Integer, nullable=False) + settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False) + outcome: Mapped[str] = mapped_column(String(16), nullable=False) + execution_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + lease_token: Mapped[str] = mapped_column(String(64), nullable=False) + history_status: Mapped[bool] = mapped_column(Boolean, nullable=False) + src: Mapped[Optional[str]] = mapped_column(String) + src_storage: Mapped[Optional[str]] = mapped_column(String) + pending_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False) + error: Mapped[Optional[str]] = mapped_column(Text) + created_at: Mapped[str] = mapped_column(String(40), nullable=False) + updated_at: Mapped[str] = mapped_column(String(40), nullable=False) + + __table_args__ = ( + UniqueConstraint( + "task_id", + "settlement_revision", + name="uq_transfersettlementreceipt_task_revision", + ), + Index( + "ix_transfersettlementreceipt_task_revision", + "task_id", + "settlement_revision", + ), + Index( + "ix_transfersettlementreceipt_history_id", + "history_id", + ), + ) + + @classmethod + def get_latest_by_task_id( + cls, + db: Session, + *, + task_id: str, + ) -> Optional["TransferSettlementReceipt"]: + """按稳定任务标识读取最新已提交结算回执。""" + if not task_id: + return None + return cast( + Optional["TransferSettlementReceipt"], + db.execute( + select(cls) + .where(cls.task_id == task_id) + .order_by(desc(cls.settlement_revision)) + ).scalars().first(), + ) + + @classmethod + def get_by_identity( + cls, + db: Session, + *, + task_id: str, + execution_fingerprint: str, + lease_token: str, + outcome: str, + ) -> Optional["TransferSettlementReceipt"]: + """按原始执行身份读取不可变结算回执。""" + if not all((task_id, execution_fingerprint, lease_token, outcome)): + return None + return cast( + Optional["TransferSettlementReceipt"], + db.execute( + select(cls).where( + cls.task_id == task_id, + cls.execution_fingerprint == execution_fingerprint, + cls.lease_token == lease_token, + cls.outcome == outcome, + ).order_by(desc(cls.settlement_revision)) + ).scalars().first(), + ) + + @classmethod + def stage_append( + cls, + db: Session, + *, + task_id: str, + history_id: int, + settlement_revision: int, + outcome: str, + execution_fingerprint: str, + lease_token: str, + history_status: bool, + src: Optional[str], + src_storage: Optional[str], + pending_deleted: bool, + error: Optional[str], + settled_at: str, + ) -> "TransferSettlementReceipt": + """按连续修订追加任务回执，旧修订证据永不覆盖。""" + if not all((task_id, history_id, settlement_revision, outcome, + execution_fingerprint, lease_token, settled_at)): + raise ValueError("整理结算回执缺少稳定身份或结果证据") + if outcome not in {"succeeded", "failed"}: + raise ValueError(f"不支持的整理结算结果：{outcome}") + if cls.get_by_identity( + db, + task_id=task_id, + execution_fingerprint=execution_fingerprint, + lease_token=lease_token, + outcome=outcome, + ) is not None: + raise ValueError("同一整理执行身份不能追加多个结算修订") + latest = cls.get_latest_by_task_id(db, task_id=task_id) + if latest is None: + if settlement_revision != 1: + raise ValueError("整理结算回执必须从修订 1 开始") + elif settlement_revision != latest.settlement_revision + 1: + raise ValueError("整理结算回执修订必须连续递增") + receipt = cls( + task_id=task_id, + history_id=history_id, + settlement_revision=settlement_revision, + outcome=outcome, + execution_fingerprint=execution_fingerprint, + lease_token=lease_token, + history_status=history_status, + src=src, + src_storage=src_storage, + pending_deleted=pending_deleted, + error=error, + created_at=settled_at, + updated_at=settled_at, + ) + db.add(receipt) + db.flush() + return receipt diff --git a/app/db/oper/__init__.py b/app/db/oper/__init__.py index 4cd94c720..07794a1cf 100644 --- a/app/db/oper/__init__.py +++ b/app/db/oper/__init__.py @@ -33,6 +33,7 @@ if TYPE_CHECKING: from app.db.oper.systemconfig import SystemConfigOper from app.db.oper.transferhistory import TransferHistoryOper from app.db.oper.transferpending import TransferPendingOper + from app.db.oper.transfersettlementreceipt import TransferSettlementReceiptOper from app.db.oper.user import UserOper from app.db.oper.userconfig import UserConfigOper from app.db.oper.workflow import WorkflowOper @@ -52,6 +53,7 @@ _OPER_MODULES = { "SystemConfigOper": "systemconfig", "TransferHistoryOper": "transferhistory", "TransferPendingOper": "transferpending", + "TransferSettlementReceiptOper": "transfersettlementreceipt", "UserConfigOper": "userconfig", "UserOper": "user", "WorkflowOper": "workflow", @@ -94,6 +96,7 @@ __all__ = [ "SystemConfigOper", "TransferHistoryOper", "TransferPendingOper", + "TransferSettlementReceiptOper", "UserConfigOper", "UserOper", "WorkflowOper", diff --git a/app/db/oper/transferexecutionstep.py b/app/db/oper/transferexecutionstep.py new file mode 100644 index 000000000..5024f6ed6 --- /dev/null +++ b/app/db/oper/transferexecutionstep.py @@ -0,0 +1,243 @@ +"""整理外部操作步骤的显式 Session 数据访问对象。""" + +from __future__ import annotations + +from typing import Any, Optional + +from sqlalchemy.orm import Session + +from app.db.base import DbOper +from app.db.models.transferexecutionstep import TransferExecutionStep + + +class TransferExecutionStepOper(DbOper): + """在调用方事务中查询或暂存整理外部操作步骤。""" + + def _session(self) -> Session: + """返回调用方同步 Session，拒绝隐式事务破坏原子状态推进。""" + if not isinstance(self._db, Session): + raise RuntimeError("整理执行步骤写入需要调用方提供同步 Session") + return self._db + + def get_by_operation_id( + self, + *, + operation_id: str, + ) -> Optional[TransferExecutionStep]: + """按稳定操作标识查询步骤。""" + return TransferExecutionStep.get_by_operation_id( + self._session(), + operation_id=operation_id, + ) + + def list_by_task_id(self, *, task_id: str) -> list[TransferExecutionStep]: + """按全局序号查询任务的全部步骤。""" + return TransferExecutionStep.list_by_task_id( + self._session(), + task_id=task_id, + ) + + def stage_prepare( + self, + *, + task_id: str, + operation_id: str, + checkpoint_fingerprint: str, + ordinal: int, + phase: str, + kind: str, + intent_version: int, + intent_payload: dict[str, Any], + now_time: str, + ) -> TransferExecutionStep: + """暂存尚未执行的稳定步骤意图。""" + return TransferExecutionStep.stage_prepare( + self._session(), + task_id=task_id, + operation_id=operation_id, + checkpoint_fingerprint=checkpoint_fingerprint, + ordinal=ordinal, + phase=phase, + kind=kind, + intent_version=intent_version, + intent_payload=intent_payload, + now_time=now_time, + ) + + def stage_start_attempt( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效任务租约暂存新步骤尝试。""" + return TransferExecutionStep.start_attempt( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_restart_after_not_applied( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + previous_attempt_token: str, + attempt_token: str, + result_version: int, + result_payload: dict[str, Any], + now_utc: str, + updated_at: str, + ) -> int: + """以严格未发生证据暂存遗留 STARTED 步骤的安全重启。""" + return TransferExecutionStep.restart_after_not_applied( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + previous_attempt_token=previous_attempt_token, + attempt_token=attempt_token, + result_version=result_version, + result_payload=result_payload, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_resume_failed_attempt( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以重试调度的新 lease 暂存 FAILED 步骤恢复。""" + return TransferExecutionStep.resume_failed_attempt( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_complete_attempt( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + result_version: int, + result_payload: dict[str, Any], + now_utc: str, + updated_at: str, + ) -> int: + """以 lease 与 attempt 双 CAS 暂存成功证据。""" + return TransferExecutionStep.complete_attempt( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + result_version=result_version, + result_payload=result_payload, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_fail_attempt( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: str, + error: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + now_utc: str, + updated_at: str, + ) -> int: + """以 lease 与 attempt 双 CAS 暂存已知失败证据。""" + return TransferExecutionStep.fail_attempt( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + error=error, + result_version=result_version, + result_payload=result_payload, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_manual_review( + self, + *, + task_id: str, + lease_token: str, + operation_id: str, + attempt_token: Optional[str], + error: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + now_utc: str, + updated_at: str, + ) -> int: + """以当前尝试身份暂存人工复核证据。""" + return TransferExecutionStep.mark_manual_review( + self._session(), + task_id=task_id, + lease_token=lease_token, + operation_id=operation_id, + attempt_token=attempt_token, + error=error, + result_version=result_version, + result_payload=result_payload, + now_utc=now_utc, + updated_at=updated_at, + ) + + def stage_resolve_manual_review( + self, + *, + task_id: str, + operation_id: str, + target_state: str, + reason: str, + result_version: Optional[int], + result_payload: Optional[dict[str, Any]], + updated_at: str, + ) -> int: + """在 pending 同为无租约人工态时暂存步骤判定。""" + return TransferExecutionStep.resolve_manual_review( + self._session(), + task_id=task_id, + operation_id=operation_id, + target_state=target_state, + reason=reason, + result_version=result_version, + result_payload=result_payload, + updated_at=updated_at, + ) + + def stage_delete_task(self, *, task_id: str) -> int: + """暂存任务全部步骤删除。""" + return TransferExecutionStep.delete_by_task_id( + self._session(), + task_id=task_id, + ) diff --git a/app/db/oper/transferhistory.py b/app/db/oper/transferhistory.py index 500402c5f..8b3b44fc2 100644 --- a/app/db/oper/transferhistory.py +++ b/app/db/oper/transferhistory.py @@ -2,6 +2,7 @@ import time from typing import Any, List, Optional from sqlalchemy import delete as sqlalchemy_delete +from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import Session from app.db.base import DbOper @@ -116,6 +117,19 @@ class TransferHistoryOper(DbOper): lambda session: TransferHistory.get_by_src(session, src, storage) ) + def get_by_transfer_task_id( + self, + *, + task_id: str, + ) -> Optional[TransferHistory]: + """按稳定整理任务标识读取终态历史。""" + return self._execute_sync_query( + lambda session: TransferHistory.get_by_transfer_task_id( + session, + task_id=task_id, + ) + ) + def get_success_by_src( self, src: str, storage: Optional[str] = None ) -> Optional[TransferHistory]: @@ -264,33 +278,60 @@ class TransferHistoryOper(DbOper): def delete(self, historyid): """ - 删除转移记录 + 删除旧转移记录，失败任务历史由状态机独占。 """ - self._stage_delete(TransferHistory, historyid) + self._execute_sync_write( + lambda session: session.execute( + sqlalchemy_delete(TransferHistory).where( + TransferHistory.id == historyid, + TransferHistory.transfer_task_id.is_(None), + ) + ) + ) def stage_delete(self, historyid: int) -> None: """暂存整理记录删除，事务由调用方统一提交。""" self._db.execute( sqlalchemy_delete(TransferHistory).where( - TransferHistory.id == historyid + TransferHistory.id == historyid, + TransferHistory.transfer_task_id.is_(None), ) ) def stage_truncate(self) -> None: - """暂存全部整理记录删除，由请求级事务统一提交。""" - self._db.execute(sqlalchemy_delete(TransferHistory)) + """暂存旧整理记录删除，只保留当前失败任务历史。""" + self._db.execute( + sqlalchemy_delete(TransferHistory).where( + TransferHistory.transfer_task_id.is_(None) + ) + ) async def async_delete(self, historyid): """ - 异步删除转移记录。 + 异步删除旧转移记录，失败任务历史由状态机独占。 """ - await self._stage_async_delete(TransferHistory, historyid) + async def stage(session: AsyncSession) -> None: + """在异步事务内只删除没有任务回执的历史。""" + await session.execute( + sqlalchemy_delete(TransferHistory).where( + TransferHistory.id == historyid, + TransferHistory.transfer_task_id.is_(None), + ) + ) + + await self._execute_async_write(stage) def truncate(self): """ - 清空转移记录 + 清空旧转移记录，只保留当前失败任务历史。 """ - self._stage_truncate(TransferHistory) + self._execute_sync_write( + lambda session: session.execute( + sqlalchemy_delete(TransferHistory).where( + TransferHistory.transfer_task_id.is_(None) + ) + ) + ) def add_force(self, **kwargs) -> Optional[TransferHistory]: """ @@ -327,6 +368,7 @@ class TransferHistoryOper(DbOper): sqlalchemy_delete(TransferHistory).where( TransferHistory.src == kwargs.get("src"), TransferHistory.src_storage == kwargs["src_storage"], + TransferHistory.transfer_task_id.is_(None), ) ) self._db.flush() @@ -335,6 +377,47 @@ class TransferHistoryOper(DbOper): self._db.flush() return history + def stage_upsert_by_transfer_task_id( + self, + *, + task_id: str, + settlement_revision: int, + retain_task_mapping: bool, + payload: dict[str, Any], + ) -> TransferHistory: + """在调用方事务内按任务标识幂等暂存终态历史。""" + if not isinstance(self._db, Session): + raise RuntimeError("整理历史任务结算需要调用方提供同步 Session") + payload = dict(payload) + payload["src_storage"] = payload.get("src_storage") or "local" + payload["date"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return TransferHistory.upsert_by_transfer_task_id( + self._db, + task_id=task_id, + settlement_revision=settlement_revision, + retain_task_mapping=retain_task_mapping, + payload=payload, + ) + + def stage_bind_settlement( + self, + *, + task_id: str, + settlement_revision: int, + src: str, + storage: Optional[str] = None, + ) -> Optional[TransferHistory]: + """复用已有成功历史且清除失败任务映射，不改写业务字段。""" + if not isinstance(self._db, Session): + raise RuntimeError("整理历史任务回执绑定需要调用方提供同步 Session") + history = TransferHistory.get_success_by_src(self._db, src, storage) + if history is None: + return None + history.transfer_task_id = None + history.transfer_settlement_revision = None + self._db.flush() + return history + def update_download_hash(self, historyid, download_hash): """ 补充转移记录download_hash diff --git a/app/db/oper/transferpending.py b/app/db/oper/transferpending.py index 4bc34fbae..c919146be 100644 --- a/app/db/oper/transferpending.py +++ b/app/db/oper/transferpending.py @@ -328,3 +328,210 @@ class TransferPendingOper(DbOper): now_time=now_time, ) ) + + def stage_execution_running( + self, + *, + task_id: str, + lease_token: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约暂存任务执行状态为 running。""" + return self._execute_sync_write( + lambda session: TransferPending.stage_execution_running( + session, + task_id=task_id, + lease_token=lease_token, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_defer_execution( + self, + *, + task_id: str, + lease_token: str, + error: str, + retry_due_at: str, + now_utc: str, + updated_at: str, + ) -> int: + """暂存重试世代和到期时间，并原子释放当前租约。""" + return self._execute_sync_write( + lambda session: TransferPending.defer_execution( + session, + task_id=task_id, + lease_token=lease_token, + error=error, + retry_due_at=retry_due_at, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_mark_execution_manual_review( + self, + *, + task_id: str, + lease_token: str, + error: str, + now_utc: str, + updated_at: str, + ) -> int: + """暂存人工复核隔离状态，并原子释放当前租约。""" + return self._execute_sync_write( + lambda session: TransferPending.mark_execution_manual_review( + session, + task_id=task_id, + lease_token=lease_token, + error=error, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_checkpoint_execution( + self, + *, + task_id: str, + lease_token: str, + execution_version: int, + execution_payload: dict[str, Any], + execution_fingerprint: str, + now_utc: str, + updated_at: str, + ) -> int: + """以有效租约暂存聚合执行检查点并进入 settling。""" + return self._execute_sync_write( + lambda session: TransferPending.checkpoint_execution( + session, + task_id=task_id, + lease_token=lease_token, + execution_version=execution_version, + execution_payload=execution_payload, + execution_fingerprint=execution_fingerprint, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_checkpoint_exhausted_failure( + self, + *, + task_id: str, + lease_token: str, + execution_version: int, + execution_payload: dict[str, Any], + execution_fingerprint: str, + error: str, + now_utc: str, + updated_at: str, + ) -> int: + """暂存预算耗尽失败检查点，并保留有效 lease 进入 settling。""" + return self._execute_sync_write( + lambda session: TransferPending.checkpoint_exhausted_failure( + session, + task_id=task_id, + lease_token=lease_token, + execution_version=execution_version, + execution_payload=execution_payload, + execution_fingerprint=execution_fingerprint, + error=error, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_request_execution_retry( + self, + *, + task_id: str, + reason: str, + requested_by: str, + retry_due_at: str, + updated_at: str, + ) -> int: + """仅将 FAILED 任务暂存为立即到期的 retry_wait。""" + return self._execute_sync_write( + lambda session: TransferPending.request_execution_retry( + session, + task_id=task_id, + reason=reason, + requested_by=requested_by, + retry_due_at=retry_due_at, + updated_at=updated_at, + ) + ) + + def stage_resolve_manual_review( + self, + *, + task_id: str, + decision: str, + actor: str, + reason: str, + retry_due_at: str, + updated_at: str, + ) -> int: + """无 lease 地暂存人工判定审计并交回 retry_wait 调度。""" + return self._execute_sync_write( + lambda session: TransferPending.resolve_manual_review( + session, + task_id=task_id, + decision=decision, + actor=actor, + reason=reason, + retry_due_at=retry_due_at, + updated_at=updated_at, + ) + ) + + def stage_terminal_failure( + self, + *, + task_id: str, + lease_token: str, + execution_fingerprint: str, + expected_revision: int, + history_id: int, + error: Optional[str], + now_utc: str, + updated_at: str, + ) -> int: + """以执行指纹和结算版本暂存失败终态。""" + return self._execute_sync_write( + lambda session: TransferPending.stage_terminal_failure( + session, + task_id=task_id, + lease_token=lease_token, + execution_fingerprint=execution_fingerprint, + expected_revision=expected_revision, + history_id=history_id, + error=error, + now_utc=now_utc, + updated_at=updated_at, + ) + ) + + def stage_delete_terminal_success( + self, + *, + task_id: str, + lease_token: str, + execution_fingerprint: str, + expected_revision: int, + now_utc: str, + ) -> int: + """以执行指纹和结算版本暂存成功 pending 删除。""" + return self._execute_sync_write( + lambda session: TransferPending.delete_terminal_success( + session, + task_id=task_id, + lease_token=lease_token, + execution_fingerprint=execution_fingerprint, + expected_revision=expected_revision, + now_utc=now_utc, + ) + ) diff --git a/app/db/oper/transfersettlementreceipt.py b/app/db/oper/transfersettlementreceipt.py new file mode 100644 index 000000000..239334f40 --- /dev/null +++ b/app/db/oper/transfersettlementreceipt.py @@ -0,0 +1,71 @@ +"""整理任务终态结算回执的事务内数据访问。""" + +from typing import Optional + +from app.db.base import DbOper +from app.db.models.transfersettlementreceipt import TransferSettlementReceipt + + +class TransferSettlementReceiptOper(DbOper): + """在调用方 Session 内读取和推进 durable 结算回执。""" + + def get_latest_by_task_id( + self, + *, + task_id: str, + ) -> Optional[TransferSettlementReceipt]: + """按稳定任务标识读取最新回执。""" + return TransferSettlementReceipt.get_latest_by_task_id( + self._db, + task_id=task_id, + ) + + def get_by_identity( + self, + *, + task_id: str, + execution_fingerprint: str, + lease_token: str, + outcome: str, + ) -> Optional[TransferSettlementReceipt]: + """按原始执行身份读取不可变回执。""" + return TransferSettlementReceipt.get_by_identity( + self._db, + task_id=task_id, + execution_fingerprint=execution_fingerprint, + lease_token=lease_token, + outcome=outcome, + ) + + def stage_append( + self, + *, + task_id: str, + history_id: int, + settlement_revision: int, + outcome: str, + execution_fingerprint: str, + lease_token: str, + history_status: bool, + src: Optional[str], + src_storage: Optional[str], + pending_deleted: bool, + error: Optional[str], + settled_at: str, + ) -> TransferSettlementReceipt: + """在调用方事务内按连续修订追加任务结算回执。""" + return TransferSettlementReceipt.stage_append( + self._db, + task_id=task_id, + history_id=history_id, + settlement_revision=settlement_revision, + outcome=outcome, + execution_fingerprint=execution_fingerprint, + lease_token=lease_token, + history_status=history_status, + src=src, + src_storage=src_storage, + pending_deleted=pending_deleted, + error=error, + settled_at=settled_at, + ) diff --git a/app/locales/en-US.json b/app/locales/en-US.json index a5d193456..360813013 100644 --- a/app/locales/en-US.json +++ b/app/locales/en-US.json @@ -116,6 +116,8 @@ "会话不存在或无权访问": "The conversation does not exist or you do not have access", "会话保存失败": "Failed to save conversation", "后台服务不存在": "Background service does not exist", + "当前管理用户缺少可审计身份": "The current administrator does not have an auditable identity", + "人工复核任务不存在": "The manual review task does not exist", "任务添加失败": "Failed to add task", "无法识别媒体信息": "Unable to recognize media information", "未识别到媒体信息": "Unable to recognize media information", diff --git a/app/locales/zh-CN.json b/app/locales/zh-CN.json index 840b16c51..d3fedebda 100644 --- a/app/locales/zh-CN.json +++ b/app/locales/zh-CN.json @@ -100,6 +100,8 @@ } }, "messages": { + "当前管理用户缺少可审计身份": "当前管理用户缺少可审计身份", + "人工复核任务不存在": "人工复核任务不存在", "媒体来源和媒体 ID 必须同时提供": "媒体来源和媒体 ID 必须同时提供", "media_source 和 media_id 必须同时提供": "media_source 和 media_id 必须同时提供", "模块不支持测试": "模块不支持测试", diff --git a/app/locales/zh-TW.json b/app/locales/zh-TW.json index 0c216f977..081da1e6b 100644 --- a/app/locales/zh-TW.json +++ b/app/locales/zh-TW.json @@ -112,6 +112,8 @@ "会话不存在或无权访问": "會話不存在或無權存取", "会话保存失败": "會話儲存失敗", "后台服务不存在": "背景服務不存在", + "当前管理用户缺少可审计身份": "目前管理使用者缺少可稽核身分", + "人工复核任务不存在": "人工複核任務不存在", "任务添加失败": "任務新增失敗", "无法识别媒体信息": "無法識別媒體資訊", "未识别到媒体信息": "未識別到媒體資訊", diff --git a/app/modules/filemanager/module.py b/app/modules/filemanager/module.py index dfe77a3af..2d3585563 100644 --- a/app/modules/filemanager/module.py +++ b/app/modules/filemanager/module.py @@ -5,6 +5,7 @@ from app.adapters.system.host import SystemUtils from app.application.directory import DirectoryHelper from app.application.messaging.message import MessageHelper from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput +from app.application.transfer_execution import TransferStepRunner from app.domain.context import MediaInfo, MusicInfo from app.domain.meta.metabase import MetaBase from app.domain.meta.metamusic import MetaMusic @@ -542,10 +543,13 @@ class FileManagerModule(_ModuleBase): source_oper: Optional[StorageBase] = None, target_oper: Optional[StorageBase] = None, cleanup_media_file: Optional[Callable[[FileItem], bool]] = None, + observe_cleanup_media_file: Optional[Callable[[FileItem], bool]] = None, + step_runner: Optional[TransferStepRunner] = None, ) -> TransferInfo: """解析存储适配器并通过统一删除能力执行已冻结计划。""" source_fileitem = FileItem(**checkpoint.planning_input.source_fileitem) cleanup_before_transfer = None + observe_cleanup_before_transfer = None cleanup_payload = checkpoint.planning_input.options.get( "cleanup_dest_fileitem" ) @@ -565,6 +569,11 @@ class FileManagerModule(_ModuleBase): f"{cleanup_fileitem.path} 删除失败，整理计划保留待重试" ) + if observe_cleanup_media_file: + def observe_cleanup_before_transfer() -> bool: + """只读确认旧目标是否已经由统一能力清理。""" + return observe_cleanup_media_file(cleanup_fileitem) + source_storage = source_fileitem.storage or "local" if not source_oper: source_oper = self.__get_storage_oper(source_storage) @@ -595,6 +604,8 @@ class FileManagerModule(_ModuleBase): source_oper=source_oper, target_oper=target_oper, cleanup_before_transfer=cleanup_before_transfer, + observe_cleanup_before_transfer=observe_cleanup_before_transfer, + step_runner=step_runner, ) @staticmethod diff --git a/app/modules/filemanager/transhandler.py b/app/modules/filemanager/transhandler.py index 5120a4dd3..61a41c5d9 100644 --- a/app/modules/filemanager/transhandler.py +++ b/app/modules/filemanager/transhandler.py @@ -1,3 +1,4 @@ +import filecmp import re from copy import deepcopy from pathlib import Path @@ -14,6 +15,12 @@ from app.application.transfer import ( TransferPlanItem, TransferPlanningInput, ) +from app.application.transfer_execution import ( + TransferOperationObservation, + TransferOperationObservationState, + TransferStepResult, + TransferStepRunner, +) from app.domain.context import MediaInfo, MusicInfo from app.domain.meta.metabase import MetaBase from app.domain.meta.metamusic import MetaMusic @@ -625,6 +632,456 @@ class TransHandler: ) return False, False, None + @staticmethod + def __serialize_step_item(fileitem: Optional[FileItem]) -> Optional[dict[str, Any]]: + """把步骤结果中的文件投影冻结为 JSON 对象。""" + return fileitem.model_dump(mode="json") if fileitem else None + + @staticmethod + def __restore_step_item(result: TransferStepResult) -> Optional[FileItem]: + """从已持久成功证据恢复目标文件，不再次访问外部存储。""" + payload = result.payload.get("item") + return FileItem.model_validate(payload) if isinstance(payload, dict) else None + + @staticmethod + def __observe_item_presence( + storage_oper: StorageBase, + path: Path, + *, + applied_when_present: bool, + ) -> TransferOperationObservation: + """以严格查询判断目标存在性，查询异常一律视为未知。""" + try: + item = storage_oper.get_item_strict(path) + except Exception as error: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={ + "path": path.as_posix(), + "query_error": str(error), + }), + ) + exists = item is not None + applied = exists if applied_when_present else not exists + return TransferOperationObservation( + state=( + TransferOperationObservationState.APPLIED + if applied + else TransferOperationObservationState.NOT_APPLIED + ), + evidence=TransferStepResult(payload={ + "path": path.as_posix(), + "exists": exists, + "item": TransHandler.__serialize_step_item(item), + }), + ) + + @staticmethod + def __run_persisted_step( + step_runner: Optional[TransferStepRunner], + *, + phase: str, + kind: str, + payload: dict[str, Any], + execute: Callable[[], TransferStepResult], + observe: Callable[[], TransferOperationObservation], + ) -> TransferStepResult: + """在持久任务中委托步骤账本，旧同步调用则直接执行。""" + if step_runner is None: + return execute() + return step_runner.run( + phase=phase, + kind=kind, + payload=payload, + execute=execute, + observe=observe, + ) + + @staticmethod + def __observe_transfer_operation( + *, + fileitem: FileItem, + target_storage: str, + source_oper: StorageBase, + target_oper: StorageBase, + target_file: Path, + transfer_type: str, + ) -> TransferOperationObservation: + """对遗留传输尝试作保守判定，证据不足时禁止自动重放。""" + try: + source_item = source_oper.get_item_strict(Path(cast(str, fileitem.path))) + target_item = target_oper.get_item_strict(target_file) + except Exception as error: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={"query_error": str(error)}), + ) + + source_exists = source_item is not None + target_exists = target_item is not None + evidence = TransferStepResult(payload={ + "source_exists": source_exists, + "target_exists": target_exists, + "item": TransHandler.__serialize_step_item(target_item), + }) + if transfer_type == "move": + if not source_exists and target_exists: + return TransferOperationObservation( + state=TransferOperationObservationState.APPLIED, + evidence=evidence, + ) + if source_exists and not target_exists: + return TransferOperationObservation( + state=TransferOperationObservationState.NOT_APPLIED, + evidence=evidence, + ) + return TransferOperationObservation( + state=TransferOperationObservationState.CONFLICT, + evidence=evidence, + ) + + if not target_exists: + return TransferOperationObservation( + state=TransferOperationObservationState.NOT_APPLIED, + evidence=evidence, + ) + if fileitem.storage != "local" or target_storage != "local": + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=evidence, + ) + source_path = Path(cast(str, fileitem.path)) + try: + if transfer_type == "copy": + applied = source_path.is_file() and filecmp.cmp( + source_path, target_file, shallow=False + ) + elif transfer_type == "link": + applied = source_path.is_file() and target_file.samefile(source_path) + elif transfer_type == "softlink": + applied = target_file.is_symlink() and target_file.resolve() == source_path.resolve() + else: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=evidence, + ) + except OSError as error: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={ + **evidence.payload, + "verification_error": str(error), + }), + ) + return TransferOperationObservation( + state=( + TransferOperationObservationState.APPLIED + if applied + else TransferOperationObservationState.CONFLICT + ), + evidence=evidence, + ) + + @classmethod + def __execute_transfer_with_steps( + cls, + *, + step_runner: Optional[TransferStepRunner], + fileitem: FileItem, + target_storage: str, + source_oper: StorageBase, + target_oper: StorageBase, + target_file: Path, + transfer_type: str, + ) -> tuple[Optional[FileItem], str]: + """执行稳定传输步骤，并把跨存储 move 拆为落地与源删除。""" + cross_storage_move = ( + transfer_type == "move" and fileitem.storage != target_storage + ) + materialize_type = "copy" if cross_storage_move else transfer_type + intent_payload = { + "source": fileitem.model_dump(mode="json"), + "target_storage": target_storage, + "target_path": target_file.as_posix(), + "transfer_type": materialize_type, + } + + def execute_materialize() -> TransferStepResult: + """执行一次目标落地并冻结其返回对象。""" + new_item, error = cls.__transfer_command( + fileitem=fileitem, + target_storage=target_storage, + source_oper=source_oper, + target_oper=target_oper, + target_file=target_file, + transfer_type=materialize_type, + ) + if not new_item: + raise RuntimeError(error or f"{fileitem.path} 整理失败") + return TransferStepResult(payload={ + "item": cls.__serialize_step_item(new_item), + "message": error, + }) + + materialized = cls.__run_persisted_step( + step_runner, + phase="transfer", + kind="materialize_target", + payload=intent_payload, + execute=execute_materialize, + observe=lambda: cls.__observe_transfer_operation( + fileitem=fileitem, + target_storage=target_storage, + source_oper=source_oper, + target_oper=target_oper, + target_file=target_file, + transfer_type=materialize_type, + ), + ) + new_item = cls.__restore_step_item(materialized) + if not new_item: + return None, "整理步骤成功证据缺少目标文件" + if not cross_storage_move: + return new_item, str(materialized.payload.get("message") or "") + + def execute_source_delete() -> TransferStepResult: + """在目标已落地后单独删除跨存储 move 的源文件。""" + if not source_oper.delete(fileitem): + raise RuntimeError(f"{fileitem.path} 源文件删除失败") + return TransferStepResult(payload={ + "source_path": fileitem.path, + "deleted": True, + }) + + cls.__run_persisted_step( + step_runner, + phase="transfer", + kind="delete_move_source", + payload={ + "source": fileitem.model_dump(mode="json"), + "target_storage": target_storage, + "target_path": target_file.as_posix(), + }, + execute=execute_source_delete, + observe=lambda: cls.__observe_item_presence( + source_oper, + Path(cast(str, fileitem.path)), + applied_when_present=False, + ), + ) + return new_item, str(materialized.payload.get("message") or "") + + @classmethod + def __ensure_directory_with_step( + cls, + *, + step_runner: Optional[TransferStepRunner], + target_oper: StorageBase, + target_storage: str, + path: Path, + ) -> Optional[FileItem]: + """持久记录可能创建目录的 get_folder 操作并恢复其结果。""" + def execute() -> TransferStepResult: + """获取或创建目标目录并冻结目录对象。""" + directory = target_oper.get_folder(path) + if not directory: + raise RuntimeError(f"目标目录 {path} 获取失败") + return TransferStepResult(payload={ + "item": cls.__serialize_step_item(directory), + }) + + result = cls.__run_persisted_step( + step_runner, + phase="prepare", + kind="ensure_target_directory", + payload={"storage": target_storage, "path": path.as_posix()}, + execute=execute, + observe=lambda: cls.__observe_item_presence( + target_oper, + path, + applied_when_present=True, + ), + ) + return cls.__restore_step_item(result) + + @classmethod + def __cleanup_with_step( + cls, + *, + step_runner: Optional[TransferStepRunner], + cleanup: Optional[Callable[[], None]], + observe_cleanup: Optional[Callable[[], bool]], + source_path: str, + ) -> None: + """把兼容清理能力纳入步骤账本，未知遗留结果必须人工复核。""" + if cleanup is None: + return + + def execute() -> TransferStepResult: + """执行统一旧目标清理能力。""" + cleanup() + return TransferStepResult(payload={"cleaned": True}) + + def observe() -> TransferOperationObservation: + """通过只读兼容能力确认旧目标是否已经消失。""" + if observe_cleanup is None: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={ + "reason": "cleanup observer unavailable", + }), + ) + try: + cleaned = observe_cleanup() + except Exception as error: + return TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={"query_error": str(error)}), + ) + return TransferOperationObservation( + state=( + TransferOperationObservationState.APPLIED + if cleaned + else TransferOperationObservationState.NOT_APPLIED + ), + evidence=TransferStepResult(payload={"cleaned": cleaned}), + ) + + cls.__run_persisted_step( + step_runner, + phase="prepare", + kind="cleanup_previous_destination", + payload={"source_path": source_path}, + execute=execute, + observe=observe, + ) + + @classmethod + def __delete_target_with_step( + cls, + *, + step_runner: Optional[TransferStepRunner], + target_oper: StorageBase, + target_storage: str, + target_file: Path, + ) -> None: + """幂等删除覆盖目标，并持久记录删除意图和严格存在性证据。""" + def execute() -> TransferStepResult: + """只删除当前冻结目标，目标已不存在视为成功。""" + current = target_oper.get_item_strict(target_file) + if current is not None and not target_oper.delete(current): + raise RuntimeError(f"【{target_storage}】{target_file} 删除失败") + return TransferStepResult(payload={"deleted": True}) + + cls.__run_persisted_step( + step_runner, + phase="prepare", + kind="delete_overwrite_target", + payload={"storage": target_storage, "path": target_file.as_posix()}, + execute=execute, + observe=lambda: cls.__observe_item_presence( + target_oper, + target_file, + applied_when_present=False, + ), + ) + + def __resolve_overwrite_with_step( + self, + *, + step_runner: Optional[TransferStepRunner], + fileitem: FileItem, + meta: MetaBase, + mediainfo: MediaInfo | MusicInfo, + target_oper: StorageBase, + target_storage: str, + target_file: Path, + transfer_type: str, + overwrite_mode: Optional[str], + need_notify: bool, + ) -> tuple[bool, bool, Optional[TransferInfo]]: + """冻结覆盖策略判定，避免目标变化后重启得到不同步骤序列。""" + def execute() -> TransferStepResult: + """执行一次覆盖策略判定并冻结完整裁决。""" + over_flag, delete_versions, failure = self.__resolve_overwrite( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_oper=target_oper, + target_storage=target_storage, + target_file=target_file, + transfer_type=transfer_type, + overwrite_mode=overwrite_mode, + need_notify=need_notify, + ) + return TransferStepResult(payload={ + "over_flag": over_flag, + "delete_versions": delete_versions, + "failure": failure.model_dump(mode="json") if failure else None, + }) + + result = self.__run_persisted_step( + step_runner, + phase="decision", + kind="resolve_overwrite", + payload={ + "source": fileitem.model_dump(mode="json"), + "target_storage": target_storage, + "target_path": target_file.as_posix(), + "transfer_type": transfer_type, + "overwrite_mode": overwrite_mode, + "need_notify": need_notify, + }, + execute=execute, + observe=lambda: TransferOperationObservation( + state=TransferOperationObservationState.NOT_APPLIED, + evidence=TransferStepResult(payload={ + "reason": "read-only overwrite decision may be repeated", + }), + ), + ) + failure_payload = result.payload.get("failure") + return ( + bool(result.payload.get("over_flag")), + bool(result.payload.get("delete_versions")), + ( + TransferInfo.model_validate(failure_payload) + if isinstance(failure_payload, dict) + else None + ), + ) + + def __intercept_with_step( + self, + *, + step_runner: Optional[TransferStepRunner], + payload: dict[str, Any], + invoke: Callable[[], tuple[bool, str]], + ) -> tuple[bool, str]: + """冻结插件拦截裁决；遗留未回执调用因插件不透明而转人工复核。""" + def execute() -> TransferStepResult: + """执行插件拦截并冻结允许标记与原因。""" + allowed, reason = invoke() + return TransferStepResult(payload={ + "allowed": allowed, + "reason": reason, + }) + + result = self.__run_persisted_step( + step_runner, + phase="decision", + kind="plugin_transfer_intercept", + payload=payload, + execute=execute, + observe=lambda: TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={ + "reason": "plugin intercept has no stable invocation receipt", + }), + ), + ) + return bool(result.payload.get("allowed")), str(result.payload.get("reason") or "") + def execute_transfer_plan( self, checkpoint: TransferPlanCheckpoint, @@ -634,6 +1091,8 @@ class TransHandler: source_oper: StorageBase, target_oper: StorageBase, cleanup_before_transfer: Optional[Callable[[], None]] = None, + observe_cleanup_before_transfer: Optional[Callable[[], bool]] = None, + step_runner: Optional[TransferStepRunner] = None, ) -> TransferInfo: """只消费冻结目标和有序操作，并在执行期处理覆盖与插件拦截。""" fileitem = FileItem(**checkpoint.planning_input.source_fileitem) @@ -684,7 +1143,7 @@ class TransHandler: ) if fileitem.type == "dir": - stream_path = Path(fileitem.path) / "BDMV" / "STREAM" + stream_path = Path(cast(str, fileitem.path)) / "BDMV" / "STREAM" stream_sizes = [ FileItem(**item.source_fileitem).size or 0 for item in checkpoint.items @@ -693,13 +1152,22 @@ class TransHandler: ] if stream_sizes: fileitem.size = sum(stream_sizes) - allowed, reason = self.__intercept_transfer( - fileitem=fileitem, - meta=meta, - mediainfo=mediainfo, - target_storage=target_storage, - target_path=target_path, - transfer_type=transfer_type, + allowed, reason = self.__intercept_with_step( + step_runner=step_runner, + payload={ + "source": fileitem.model_dump(mode="json"), + "target_storage": target_storage, + "target_path": target_path.as_posix(), + "transfer_type": transfer_type, + }, + invoke=lambda: self.__intercept_transfer( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_path, + transfer_type=transfer_type, + ), ) if not allowed: return TransferInfo( @@ -709,9 +1177,18 @@ class TransHandler: transfer_type=transfer_type, need_notify=checkpoint.need_notify, ) - if cleanup_before_transfer: - cleanup_before_transfer() - target_diritem = target_oper.get_folder(target_path) + self.__cleanup_with_step( + step_runner=step_runner, + cleanup=cleanup_before_transfer, + observe_cleanup=observe_cleanup_before_transfer, + source_path=cast(str, fileitem.path), + ) + target_diritem = self.__ensure_directory_with_step( + step_runner=step_runner, + target_oper=target_oper, + target_storage=target_storage, + path=target_path, + ) if not target_diritem: return TransferInfo( success=False, @@ -733,7 +1210,8 @@ class TransHandler: need_notify=checkpoint.need_notify, ) source_item = FileItem(**planned_item.source_fileitem) - new_item, error = self.__transfer_command( + new_item, error = self.__execute_transfer_with_steps( + step_runner=step_runner, fileitem=source_item, target_storage=planned_item.target_storage, source_oper=source_oper, @@ -790,7 +1268,8 @@ class TransHandler: need_notify=checkpoint.need_notify, ) target_file = Path(planned_item.target_path) - over_flag, delete_versions, overwrite_failure = self.__resolve_overwrite( + over_flag, delete_versions, overwrite_failure = self.__resolve_overwrite_with_step( + step_runner=step_runner, fileitem=fileitem, meta=meta, mediainfo=mediainfo, @@ -803,14 +1282,24 @@ class TransHandler: ) if overwrite_failure: return overwrite_failure - allowed, reason = self.__intercept_transfer( - fileitem=fileitem, - meta=meta, - mediainfo=mediainfo, - target_storage=target_storage, - target_path=target_file, - transfer_type=transfer_type, - over_flag=over_flag, + allowed, reason = self.__intercept_with_step( + step_runner=step_runner, + payload={ + "source": fileitem.model_dump(mode="json"), + "target_storage": target_storage, + "target_path": target_file.as_posix(), + "transfer_type": transfer_type, + "over_flag": over_flag, + }, + invoke=lambda: self.__intercept_transfer( + fileitem=fileitem, + meta=meta, + mediainfo=mediainfo, + target_storage=target_storage, + target_path=target_file, + transfer_type=transfer_type, + over_flag=over_flag, + ), ) if not allowed: return TransferInfo( @@ -822,9 +1311,18 @@ class TransHandler: need_notify=checkpoint.need_notify, ) - if cleanup_before_transfer: - cleanup_before_transfer() - target_diritem = target_oper.get_folder(target_file.parent) + self.__cleanup_with_step( + step_runner=step_runner, + cleanup=cleanup_before_transfer, + observe_cleanup=observe_cleanup_before_transfer, + source_path=cast(str, fileitem.path), + ) + target_diritem = self.__ensure_directory_with_step( + step_runner=step_runner, + target_oper=target_oper, + target_storage=target_storage, + path=target_file.parent, + ) if not target_diritem: return TransferInfo( success=False, @@ -835,17 +1333,51 @@ class TransHandler: need_notify=checkpoint.need_notify, ) if delete_versions: - self.__delete_version_files(target_oper, target_file) - new_item, error = self.__transfer_file( - fileitem=fileitem, - target_storage=target_storage, - target_file=target_file, - transfer_type=transfer_type, - over_flag=over_flag, - source_oper=source_oper, - target_oper=target_oper, - result=result, - ) + if step_runner is None: + self.__delete_version_files(target_oper, target_file) + else: + self.__delete_version_files_with_steps( + step_runner=step_runner, + storage_oper=target_oper, + target_storage=target_storage, + path=target_file, + ) + if step_runner is None: + new_item, error = self.__transfer_file( + fileitem=fileitem, + target_storage=target_storage, + target_file=target_file, + transfer_type=transfer_type, + over_flag=over_flag, + source_oper=source_oper, + target_oper=target_oper, + result=result, + ) + else: + if over_flag: + self.__delete_target_with_step( + step_runner=step_runner, + target_oper=target_oper, + target_storage=target_storage, + target_file=target_file, + ) + new_item, error = self.__execute_transfer_with_steps( + step_runner=step_runner, + fileitem=fileitem, + target_storage=target_storage, + source_oper=source_oper, + target_oper=target_oper, + target_file=target_file, + transfer_type=transfer_type, + ) + if new_item: + self.__update_result( + result=result, + file_list=[fileitem.path], + file_list_new=[new_item.path], + file_count=1, + total_size=fileitem.size, + ) if not new_item: error = error or f"{fileitem.path} 整理后未获取到目标文件信息" return TransferInfo( @@ -1286,6 +1818,98 @@ class TransHandler: naming_context.pop("media_id", None) return naming_context + @staticmethod + def __find_version_files( + storage_oper: StorageBase, + path: Path, + ) -> list[FileItem]: + """稳定列出与冻结目标相同季集和 Part 的其它视频版本。""" + meta = MetaInfoPath(path) + parent_item = storage_oper.get_item_strict(path.parent) + if not parent_item: + return [] + media_files = storage_oper.list(parent_item) or [] + result: list[FileItem] = [] + for media_file in media_files: + media_path = Path(media_file.path) + if media_path == path or media_file.type != "file": + continue + if f".{cast(str, media_file.extension).lower()}" not in get_runtime_setting('RMT_MEDIAEXT'): + continue + filemeta = MetaInfoPath(media_path) + if filemeta.season != meta.season or filemeta.episode != meta.episode: + continue + if meta.part and filemeta.part and filemeta.part != meta.part: + continue + result.append(media_file) + return sorted(result, key=lambda item: (item.path, item.fileid or "")) + + @classmethod + def __delete_version_files_with_steps( + cls, + *, + step_runner: TransferStepRunner, + storage_oper: StorageBase, + target_storage: str, + path: Path, + ) -> None: + """先冻结版本清单，再把每一个删除作为独立稳定步骤执行。""" + def discover() -> TransferStepResult: + """读取并冻结当前版本删除候选，读失败不产生副作用。""" + candidates = cls.__find_version_files(storage_oper, path) + return TransferStepResult(payload={ + "items": [item.model_dump(mode="json") for item in candidates], + }) + + discovery = cls.__run_persisted_step( + step_runner, + phase="prepare", + kind="discover_version_targets", + payload={"storage": target_storage, "path": path.as_posix()}, + execute=discover, + observe=lambda: TransferOperationObservation( + state=TransferOperationObservationState.NOT_APPLIED, + evidence=TransferStepResult(payload={ + "reason": "read-only discovery may be repeated", + }), + ), + ) + raw_items = discovery.payload.get("items") + if not isinstance(raw_items, list): + raise RuntimeError("版本删除候选检查点格式无效") + for raw_item in raw_items: + candidate = FileItem.model_validate(raw_item) + + def delete_candidate(item: FileItem = candidate) -> TransferStepResult: + """删除一个冻结版本候选，已不存在时保持幂等成功。""" + current = storage_oper.get_item_strict(Path(cast(str, item.path))) + if current is not None and not storage_oper.delete(current): + raise RuntimeError(f"版本文件 {item.path} 删除失败") + return TransferStepResult(payload={ + "path": item.path, + "deleted": True, + }) + + def observe_candidate(item: FileItem = candidate) -> TransferOperationObservation: + """查询一个冻结版本候选是否已经删除。""" + return cls.__observe_item_presence( + storage_oper, + Path(cast(str, item.path)), + applied_when_present=False, + ) + + cls.__run_persisted_step( + step_runner, + phase="prepare", + kind="delete_version_target", + payload={ + "storage": target_storage, + "item": candidate.model_dump(mode="json"), + }, + execute=delete_candidate, + observe=observe_candidate, + ) + @staticmethod def __delete_version_files(storage_oper: StorageBase, path: Path) -> bool: """ diff --git a/app/schemas/exports.py b/app/schemas/exports.py index 8d7c154aa..90d80157d 100644 --- a/app/schemas/exports.py +++ b/app/schemas/exports.py @@ -417,6 +417,12 @@ SCHEMA_EXPORTS = { 'TransferInterceptEventData': ('app.schemas.event', 'TransferInterceptEventData'), 'TransferJob': ('app.schemas.transfer', 'TransferJob'), 'TransferJobTask': ('app.schemas.transfer', 'TransferJobTask'), + 'TransferManualReviewData': ('app.schemas.transfer', 'TransferManualReviewData'), + 'TransferManualReviewPageData': ('app.schemas.transfer', 'TransferManualReviewPageData'), + 'TransferManualReviewRequest': ('app.schemas.transfer', 'TransferManualReviewRequest'), + 'TransferManualReviewSourceData': ('app.schemas.transfer', 'TransferManualReviewSourceData'), + 'TransferManualReviewStepData': ('app.schemas.transfer', 'TransferManualReviewStepData'), + 'TransferManualReviewTaskData': ('app.schemas.transfer', 'TransferManualReviewTaskData'), 'TransferOverwriteCheckEventData': ('app.schemas.event', 'TransferOverwriteCheckEventData'), 'TransferRenameBuildEventData': ('app.schemas.event', 'TransferRenameBuildEventData'), 'TransferRenameEventData': ('app.schemas.event', 'TransferRenameEventData'), @@ -459,7 +465,7 @@ SCHEMA_EXPORTS = { 'field_serializer': ('app.schemas.agent', 'field_serializer'), 'field_validator': ('app.schemas.system', 'field_validator'), 'json': ('app.schemas.subscribe', 'json'), - 'model_validator': ('app.schemas.subscribe', 'model_validator'), + 'model_validator': ('app.schemas.transfer', 'model_validator'), 're': ('app.schemas.file', 're'), } @@ -474,9 +480,9 @@ SCHEMA_CONFLICTS = { 'Field': ['app.schemas.agent', 'app.schemas.cache', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.download', 'app.schemas.event', 'app.schemas.file', 'app.schemas.history', 'app.schemas.llm', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.notification', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.storage', 'app.schemas.openai', 'app.schemas.servarr', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.subscribe', 'app.schemas.system', 'app.schemas.tmdb', 'app.schemas.token', 'app.schemas.transfer', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'], 'FileItem': ['app.schemas.event', 'app.schemas.file', 'app.schemas.transfer', 'app.schemas.workflow'], 'FilterRuleGroup': ['app.schemas.rule', 'app.schemas.system'], - 'JsonData': ['app.schemas.agent', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.openai', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'], + 'JsonData': ['app.schemas.agent', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.openai', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.transfer', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'], 'List': ['app.schemas.agent', 'app.schemas.context', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.monitoring', 'app.schemas.plugin', 'app.schemas.openai', 'app.schemas.subscribe', 'app.schemas.transfer', 'app.schemas.workflow'], - 'Literal': ['app.schemas.agent', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.servcookie', 'app.schemas.system', 'app.schemas.mcp'], + 'Literal': ['app.schemas.agent', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.servcookie', 'app.schemas.system', 'app.schemas.transfer', 'app.schemas.mcp'], 'LocaleHelper': ['app.schemas.dashboard', 'app.schemas.response'], 'MediaInfo': ['app.schemas.context', 'app.schemas.system', 'app.schemas.transfer', 'app.schemas.workflow'], 'MediaSource': ['app.schemas.cache', 'app.schemas.context', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.music', 'app.schemas.subscribe', 'app.schemas.transfer'], @@ -502,5 +508,5 @@ SCHEMA_CONFLICTS = { 'dataclass': ['app.schemas.notification', 'app.schemas.system'], 'datetime': ['app.schemas.agent', 'app.schemas.monitoring'], 'field_validator': ['app.schemas.event', 'app.schemas.message', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.response', 'app.schemas.subscribe', 'app.schemas.system'], - 'model_validator': ['app.schemas.dashboard', 'app.schemas.event', 'app.schemas.mediaserver', 'app.schemas.subscribe'], + 'model_validator': ['app.schemas.dashboard', 'app.schemas.event', 'app.schemas.mediaserver', 'app.schemas.subscribe', 'app.schemas.transfer'], } diff --git a/app/schemas/history.py b/app/schemas/history.py index ba9cdc229..1f8c44ae2 100644 --- a/app/schemas/history.py +++ b/app/schemas/history.py @@ -69,6 +69,8 @@ class TransferHistory(OptionalMediaIdentityMixin, BaseModel): # ID id: int + # durable 整理任务标识仅供宿主入口选择重试协议，不属于公开历史响应 + transfer_task_id: Optional[str] = Field(default=None, exclude=True) # 源存储类型 src_storage: Optional[str] = None # 目标存储类型 diff --git a/app/schemas/transfer.py b/app/schemas/transfer.py index b33b05739..5065dca3c 100644 --- a/app/schemas/transfer.py +++ b/app/schemas/transfer.py @@ -1,8 +1,9 @@ from pathlib import Path -from typing import List, Optional, Union +from typing import Literal, List, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator +from app.schemas.common import JsonData from app.schemas.media import OptionalMediaIdentityMixin from app.schemas.types import MediaSource, MusicTargetEntityType @@ -157,6 +158,75 @@ class TransferInfo(BaseModel): return dicts +class TransferManualReviewRequest(BaseModel): # type: ignore[misc] + """人工判定外部结果不确定整理步骤的请求。""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + operation_id: str = Field(min_length=1, description="待判定的稳定操作标识") + decision: Literal["not_applied", "applied"] = Field( + description="人工判定；不公开 failed，失败终态只能由 durable 结算写入", + ) + reason: str = Field(min_length=1, max_length=2000, description="人工判定理由") + result_payload: Optional[dict[str, JsonData]] = Field( + default=None, + description="判定为 applied 时必填的外部结果证据", + ) + + @model_validator(mode="after") # type: ignore[misc] + def validate_result_payload(self) -> "TransferManualReviewRequest": + """要求已发生判定携带可持久化的结果证据。""" + if self.decision == "applied" and self.result_payload is None: + raise ValueError("判定为 applied 时必须提供 result_payload") + return self + + +class TransferManualReviewData(BaseModel): # type: ignore[misc] + """人工复核提交后的公开状态投影。""" + + task_id: str + operation_id: str + decision: Literal["not_applied", "applied"] + state: str + review_revision: int + + +class TransferManualReviewSourceData(BaseModel): # type: ignore[misc] + """人工复核任务的源文件身份。""" + + storage: str + path: str + + +class TransferManualReviewStepData(BaseModel): # type: ignore[misc] + """人工复核步骤的公开意图与事实证据。""" + + operation_id: str + kind: str + intent: dict[str, JsonData] + evidence: Optional[dict[str, JsonData]] = None + error: Optional[str] = None + + +class TransferManualReviewTaskData(BaseModel): # type: ignore[misc] + """可由管理员发现和判定的 durable 整理任务。""" + + task_id: str + source: TransferManualReviewSourceData + state: Literal["manual_review", "retry_wait"] + step: TransferManualReviewStepData + review_revision: int + + +class TransferManualReviewPageData(BaseModel): # type: ignore[misc] + """人工复核任务分页结果。""" + + items: list[TransferManualReviewTaskData] = Field(default_factory=list) + total: int = 0 + page: int = 1 + page_size: int = 30 + + class EpisodeFormat(BaseModel): """ 剧集自定义识别格式 diff --git a/app/sdk/_legacy/transferpending.py b/app/sdk/_legacy/transferpending.py index 791e0a22e..98393fce3 100644 --- a/app/sdk/_legacy/transferpending.py +++ b/app/sdk/_legacy/transferpending.py @@ -1,21 +1,55 @@ """兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。""" from datetime import datetime -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple from uuid import uuid4 -from sqlalchemy import delete, select +from sqlalchemy import column, delete, exists, select, table from app.db.base import DbOper, execute_dml from app.db.models.transferpending import TransferPending as _TransferPending +_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id")) +_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id")) + + +def _safe_legacy_delete_predicates() -> tuple[Any, ...]: + """只允许旧接口删除从未 claim、执行或结算的新鲜登记。""" + return ( + _TransferPending.lease_owner.is_(None), + _TransferPending.lease_token.is_(None), + _TransferPending.lease_expires_at.is_(None), + _TransferPending.heartbeat_at.is_(None), + _TransferPending.attempt_count == 0, + _TransferPending.execution_state == "not_started", + _TransferPending.execution_version.is_(None), + _TransferPending.execution_payload.is_(None), + _TransferPending.execution_fingerprint.is_(None), + _TransferPending.retry_generation == 0, + _TransferPending.retry_count == 0, + _TransferPending.retry_due_at.is_(None), + _TransferPending.settlement_revision == 0, + _TransferPending.terminal_history_id.is_(None), + _TransferPending.last_error.is_(None), + ~exists( + select(_TRANSFER_EXECUTION_STEP.c.task_id).where( + _TRANSFER_EXECUTION_STEP.c.task_id == _TransferPending.task_id + ) + ), + ~exists( + select(_TRANSFER_HISTORY.c.transfer_task_id).where( + _TRANSFER_HISTORY.c.transfer_task_id == _TransferPending.task_id + ) + ), + ) + class TransferPendingOper(DbOper): """ 保留旧待整理登记 ABI，并对执行租约实施兼容写 fencing。 本类只由精确旧导入映射加载。宿主整理链仍使用 Application Port 和显式 - Session 的 canonical Oper；旧删除入口只能处理从未取得租约的记录。 + Session 的 canonical Oper；旧删除入口只能处理无任何执行证据的新鲜登记。 """ def register(self, storage: str, src_path: str) -> Optional[_TransferPending]: @@ -123,10 +157,10 @@ class TransferPendingOper(DbOper): def discard(self, storage: str, src_path: str) -> int: """ - 按旧签名删除未 claim 的指定登记。 + 按旧签名删除从未 claim 且没有执行证据的指定登记。 - 任何带 token 的记录都由当前租约拥有者通过 fenced canonical API 收口； - 即使租约已经过期，旧插件也不得越权代替恢复调度器删除。 + 任何租约、重试、步骤或终态证据都归 canonical 状态机所有；即使租约 + 已经过期，旧插件也不得越权代替恢复调度器删除。 :param storage: 存储 :param src_path: 源文件路径 :return: 删除的记录数 @@ -139,7 +173,7 @@ class TransferPendingOper(DbOper): delete(_TransferPending).where( _TransferPending.storage == storage, _TransferPending.src_path == src_path, - _TransferPending.lease_token.is_(None), + *_safe_legacy_delete_predicates(), ), execution_options={"synchronize_session": False}, ) @@ -167,7 +201,7 @@ class TransferPendingOper(DbOper): def clear(self) -> int: """ - 清空全部未 claim 登记，保留任何带租约 token 的任务。 + 清空从未 claim 且没有执行证据的登记，保留状态机拥有的任务。 :return: 删除的记录数 """ @@ -175,7 +209,7 @@ class TransferPendingOper(DbOper): lambda session: execute_dml( session, delete(_TransferPending).where( - _TransferPending.lease_token.is_(None), + *_safe_legacy_delete_predicates(), ), execution_options={"synchronize_session": False}, ) diff --git a/app/startup/initializers/modules.py b/app/startup/initializers/modules.py index d51fa49a1..7c1084c54 100644 --- a/app/startup/initializers/modules.py +++ b/app/startup/initializers/modules.py @@ -111,6 +111,9 @@ from app.db.adapters.site import TransactionalSiteRepository from app.db.adapters.subscription import TransactionalSubscribeWriter from app.db.adapters.transaction import TransactionalWriteRunner from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, +) from app.db.adapters.workflow import TransactionalWorkflowExecutionService from app.db.oper.agentchat import AgentChatOper from app.db.oper.agenttask import AgentTaskOper @@ -865,6 +868,9 @@ async def init_modules() -> HostRuntime: transfer_pending=lambda: TransactionalTransferAdmissionRepository( SessionFactory ), + transfer_execution=lambda: TransactionalTransferExecutionRepository( + SessionFactory + ), media_server=lambda: MediaServerOper(), download_failure=lambda: TransactionalDownloadFailureRepository( SessionFactory diff --git a/database/versions/e5c7a9b1d3f6_3_0_16.py b/database/versions/e5c7a9b1d3f6_3_0_16.py new file mode 100644 index 000000000..c2f5c8667 --- /dev/null +++ b/database/versions/e5c7a9b1d3f6_3_0_16.py @@ -0,0 +1,856 @@ +"""3.0.16 增加整理步骤执行证据与幂等终态结算字段。 + +Revision ID: e5c7a9b1d3f6 +Revises: d3a9e5f7b2c4 +Create Date: 2026-08-27 +""" + +from collections.abc import Callable +import hashlib +import json + +import sqlalchemy as sa +from alembic import op + +revision = "e5c7a9b1d3f6" +down_revision = "d3a9e5f7b2c4" +branch_labels = None +depends_on = None + +_PENDING_TABLE = "transferpending" +_HISTORY_TABLE = "transferhistory" +_STEP_TABLE = "transferexecutionstep" +_RECEIPT_TABLE = "transfersettlementreceipt" +_PENDING_INDEX = "ix_transferpending_execution_due" +_HISTORY_INDEX = "ux_transferhistory_transfer_task_id" +_STEP_OPERATION_UNIQUE = "uq_transferexecutionstep_operation_id" +_STEP_ORDINAL_UNIQUE = "uq_transferexecutionstep_task_ordinal" +_STEP_INDEX = "ix_transferexecutionstep_task_state_ordinal" +_RECEIPT_TASK_REVISION_UNIQUE = "uq_transfersettlementreceipt_task_revision" +_RECEIPT_HISTORY_INDEX = "ix_transfersettlementreceipt_history_id" +_RECEIPT_TASK_REVISION_INDEX = "ix_transfersettlementreceipt_task_revision" +_MANUAL_REVIEW_DIAGNOSTIC = "升级检测到既有执行迹象，需人工确认后再处理" +_LEGACY_REVIEW_STEP_KIND = "legacy_execution_review" +_LEGACY_REVIEW_STEP_ORDINAL = 2_147_483_647 +_LEGACY_REVIEW_FALLBACK_TIME = "1970-01-01 00:00:00" + +_PENDING_COLUMNS = { + "execution_state", + "execution_version", + "execution_payload", + "execution_fingerprint", + "retry_generation", + "retry_count", + "retry_due_at", + "retry_requested_by", + "retry_reason", + "settlement_revision", + "terminal_history_id", + "manual_review_revision", + "reviewed_at", + "reviewed_by", + "review_reason", + "review_decision", +} +_HISTORY_COLUMNS = { + "transfer_task_id", + "transfer_settlement_revision", +} +_STEP_COLUMNS = { + "id", + "task_id", + "operation_id", + "checkpoint_fingerprint", + "ordinal", + "phase", + "kind", + "state", + "attempt_token", + "attempt_count", + "intent_version", + "intent_payload", + "result_version", + "result_payload", + "last_error", + "prepared_at", + "started_at", + "completed_at", + "updated_at", +} +_STEP_NULLABLE_COLUMNS = { + "attempt_token", + "result_version", + "result_payload", + "last_error", + "started_at", + "completed_at", +} +_STEP_COLUMN_TYPES = { + "id": ("integer", None), + "task_id": ("string", 64), + "operation_id": ("string", 64), + "checkpoint_fingerprint": ("string", 64), + "ordinal": ("integer", None), + "phase": ("string", 32), + "kind": ("string", 32), + "state": ("string", 32), + "attempt_token": ("string", 64), + "attempt_count": ("integer", None), + "intent_version": ("integer", None), + "intent_payload": ("json", None), + "result_version": ("integer", None), + "result_payload": ("json", None), + "last_error": ("text", None), + "prepared_at": ("string", 40), + "started_at": ("string", 40), + "completed_at": ("string", 40), + "updated_at": ("string", 40), +} +_RECEIPT_COLUMNS = { + "id", + "task_id", + "history_id", + "settlement_revision", + "outcome", + "execution_fingerprint", + "lease_token", + "history_status", + "src", + "src_storage", + "pending_deleted", + "error", + "created_at", + "updated_at", +} +_RECEIPT_NULLABLE_COLUMNS = { + "src", + "src_storage", + "error", +} +_RECEIPT_COLUMN_TYPES = { + "id": ("integer", None), + "task_id": ("string", 64), + "history_id": ("integer", None), + "settlement_revision": ("integer", None), + "outcome": ("string", 16), + "execution_fingerprint": ("string", 64), + "lease_token": ("string", 64), + "history_status": ("boolean", None), + "src": ("string", None), + "src_storage": ("string", None), + "pending_deleted": ("boolean", None), + "error": ("text", None), + "created_at": ("string", 40), + "updated_at": ("string", 40), +} + + +def _table_names() -> set[str]: + """返回当前数据库的表名集合。""" + return set(sa.inspect(op.get_bind()).get_table_names()) + + +def _column_names(table_name: str) -> set[str]: + """返回指定表的字段集合。""" + if table_name not in _table_names(): + return set() + return { + column["name"] + for column in sa.inspect(op.get_bind()).get_columns(table_name) + } + + +def _index_names(table_name: str) -> set[str]: + """返回指定表的显式索引名称集合。""" + if table_name not in _table_names(): + return set() + return { + index["name"] + for index in sa.inspect(op.get_bind()).get_indexes(table_name) + if index.get("name") + } + + +def _table_row_count(table_name: str) -> int: + """不依赖业务字段读取迁移表行数，供残缺结构修复决策使用。""" + return op.get_bind().execute( + sa.select(sa.func.count()).select_from(sa.table(table_name)) + ).scalar_one() + + +def _repair_unique_constraints( + *, + table_name: str, + expected: dict[str, tuple[str, ...]], + create_table: Callable[[], None], +) -> None: + """把迁移自有表的唯一约束收敛到命名的 ORM 精确集合。""" + constraints = sa.inspect(op.get_bind()).get_unique_constraints(table_name) + unnamed = [item for item in constraints if not item.get("name")] + if unnamed: + if _table_row_count(table_name) > 0: + raise RuntimeError( + f"检测到含数据迁移表 {table_name} 存在未命名唯一约束，" + "无法安全收敛到当前模型" + ) + op.drop_table(table_name) + create_table() + return + actual = { + item["name"]: tuple(item.get("column_names") or ()) + for item in constraints + } + to_drop = { + name + for name, columns in actual.items() + if name not in expected or columns != expected[name] + } + to_create = { + name + for name, columns in expected.items() + if actual.get(name) != columns + } + if not to_drop and not to_create: + return + with op.batch_alter_table(table_name) as batch_op: + for constraint_name in sorted(to_drop): + batch_op.drop_constraint(constraint_name, type_="unique") + for constraint_name in sorted(to_create): + batch_op.create_unique_constraint( + constraint_name, + list(expected[constraint_name]), + ) + + +def _repair_indexes( + *, + table_name: str, + expected: dict[str, tuple[str, ...]], +) -> None: + """把迁移自有表的显式索引收敛到 ORM 精确集合。""" + actual = { + item["name"]: tuple(item.get("column_names") or ()) + for item in sa.inspect(op.get_bind()).get_indexes(table_name) + if item.get("name") and not item.get("duplicates_constraint") + } + for index_name, columns in actual.items(): + if index_name not in expected or columns != expected[index_name]: + op.drop_index(index_name, table_name=table_name) + remaining = _index_names(table_name) + for index_name, columns in expected.items(): + if index_name not in remaining: + op.create_index( + index_name, + table_name, + list(columns), + unique=False, + ) + + +def _column_type_signature(column_type: sa.types.TypeEngine) -> tuple[str, object]: + """把方言反射类型归一为迁移可稳定比较的类型与长度。""" + if isinstance(column_type, sa.JSON): + return "json", None + if isinstance(column_type, sa.Boolean): + return "boolean", None + if isinstance(column_type, sa.Integer): + return "integer", None + if isinstance(column_type, sa.Text): + return "text", None + if isinstance(column_type, sa.String): + return "string", column_type.length + return column_type.__class__.__name__.lower(), None + + +def _validate_or_recreate_empty_table( + *, + table_name: str, + expected_columns: set[str], + nullable_columns: set[str], + expected_types: dict[str, tuple[str, object]], + expected_foreign_keys: set[tuple[str, str, str, str]], + create_table: Callable[[], None], +) -> None: + """校验中断升级留下的表结构；仅空表允许无损重建。""" + inspected_columns = sa.inspect(op.get_bind()).get_columns(table_name) + actual_columns = {column["name"] for column in inspected_columns} + actual_nullable = { + column["name"] + for column in inspected_columns + if column.get("nullable", True) + } + missing = expected_columns - actual_columns + unexpected = actual_columns - expected_columns + wrong_nullable = { + column_name + for column_name in expected_columns & actual_columns + if (column_name in actual_nullable) != (column_name in nullable_columns) + } + wrong_types = { + column["name"] + for column in inspected_columns + if ( + column["name"] in expected_types + and _column_type_signature(column["type"]) + != expected_types[column["name"]] + ) + } + primary_key = tuple( + sa.inspect(op.get_bind()).get_pk_constraint(table_name) + .get("constrained_columns") or () + ) + foreign_keys = { + ( + foreign_key["constrained_columns"][0], + foreign_key["referred_table"], + foreign_key["referred_columns"][0], + str(foreign_key.get("options", {}).get("ondelete") or "").upper(), + ) + for foreign_key in sa.inspect(op.get_bind()).get_foreign_keys(table_name) + if ( + len(foreign_key.get("constrained_columns") or ()) == 1 + and len(foreign_key.get("referred_columns") or ()) == 1 + ) + } + wrong_primary_key = primary_key != ("id",) + wrong_foreign_keys = foreign_keys != expected_foreign_keys + if not any(( + missing, + unexpected, + wrong_nullable, + wrong_types, + wrong_primary_key, + wrong_foreign_keys, + )): + return + row_count = _table_row_count(table_name) + if row_count == 0: + op.drop_table(table_name) + create_table() + return + details = ", ".join(filter(None, ( + f"缺少字段 {sorted(missing)}" if missing else "", + f"未知字段 {sorted(unexpected)}" if unexpected else "", + f"空值约束不一致 {sorted(wrong_nullable)}" if wrong_nullable else "", + f"字段类型不一致 {sorted(wrong_types)}" if wrong_types else "", + "主键不一致" if wrong_primary_key else "", + "外键不一致" if wrong_foreign_keys else "", + ))) + raise RuntimeError( + f"检测到含数据的不完整迁移表 {table_name}（{details}），" + "无法自动修复，请先恢复该版本的完整表结构后重试" + ) + + +def _add_pending_columns() -> None: + """补齐 pending 执行 checkpoint、重试与结算字段。""" + columns = _column_names(_PENDING_TABLE) + additions = ( + ("execution_state", sa.Column("execution_state", sa.String(32))), + ("execution_version", sa.Column("execution_version", sa.Integer())), + ("execution_payload", sa.Column("execution_payload", sa.JSON())), + ( + "execution_fingerprint", + sa.Column("execution_fingerprint", sa.String(64)), + ), + ("retry_generation", sa.Column("retry_generation", sa.Integer())), + ("retry_count", sa.Column("retry_count", sa.Integer())), + ("retry_due_at", sa.Column("retry_due_at", sa.String(40))), + ("retry_requested_by", sa.Column("retry_requested_by", sa.String(128))), + ("retry_reason", sa.Column("retry_reason", sa.Text())), + ("settlement_revision", sa.Column("settlement_revision", sa.Integer())), + ("terminal_history_id", sa.Column("terminal_history_id", sa.Integer())), + ("manual_review_revision", sa.Column("manual_review_revision", sa.Integer())), + ("reviewed_at", sa.Column("reviewed_at", sa.String(40))), + ("reviewed_by", sa.Column("reviewed_by", sa.String(128))), + ("review_reason", sa.Column("review_reason", sa.Text())), + ("review_decision", sa.Column("review_decision", sa.String(32))), + ) + for column_name, column in additions: + if column_name not in columns: + op.add_column(_PENDING_TABLE, column) + + +def _backfill_pending() -> None: + """保守隔离旧执行迹象，并补齐新增非空字段。""" + columns = _column_names(_PENDING_TABLE) + if not _PENDING_COLUMNS.issubset(columns): + return + pending = sa.table( + _PENDING_TABLE, + sa.column("state", sa.String(32)), + sa.column("checkpoint_version", sa.Integer()), + sa.column("checkpoint_payload", sa.JSON()), + sa.column("lease_token", sa.String(64)), + sa.column("attempt_count", sa.Integer()), + sa.column("last_error", sa.Text()), + sa.column("execution_state", sa.String(32)), + sa.column("retry_generation", sa.Integer()), + sa.column("retry_count", sa.Integer()), + sa.column("settlement_revision", sa.Integer()), + sa.column("manual_review_revision", sa.Integer()), + ) + uncertain = sa.or_( + pending.c.state.in_(("provider_pending", "planned", "manual_review")), + pending.c.checkpoint_version.is_not(None), + pending.c.lease_token.is_not(None), + pending.c.attempt_count > 0, + pending.c.last_error.contains(_MANUAL_REVIEW_DIAGNOSTIC), + ) + bind = op.get_bind() + bind.execute( + pending.update() + .where(pending.c.execution_state.is_(None), uncertain) + .values( + execution_state="manual_review", + last_error=sa.func.coalesce( + pending.c.last_error, + _MANUAL_REVIEW_DIAGNOSTIC, + ), + ) + ) + bind.execute( + pending.update() + .where(pending.c.execution_state.is_(None)) + .values(execution_state="not_started") + ) + for column_name in ( + "retry_generation", + "retry_count", + "settlement_revision", + "manual_review_revision", + ): + column = getattr(pending.c, column_name) + bind.execute( + pending.update().where(column.is_(None)).values({column_name: 0}) + ) + with op.batch_alter_table(_PENDING_TABLE) as batch_op: + batch_op.alter_column( + "execution_state", + existing_type=sa.String(32), + nullable=False, + ) + for column_name in ( + "retry_generation", + "retry_count", + "settlement_revision", + "manual_review_revision", + ): + batch_op.alter_column( + column_name, + existing_type=sa.Integer(), + nullable=False, + ) + + +def _add_history_columns() -> None: + """补齐成功删除 pending 后仍可幂等回读的历史身份字段。""" + columns = _column_names(_HISTORY_TABLE) + if "transfer_task_id" not in columns: + op.add_column( + _HISTORY_TABLE, + sa.Column("transfer_task_id", sa.String(64), nullable=True), + ) + if "transfer_settlement_revision" not in columns: + op.add_column( + _HISTORY_TABLE, + sa.Column("transfer_settlement_revision", sa.Integer(), nullable=True), + ) + + +def _create_step_table() -> None: + """按 ORM 契约创建整理步骤执行证据表。""" + op.create_table( + _STEP_TABLE, + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("operation_id", sa.String(64), nullable=False), + sa.Column("checkpoint_fingerprint", sa.String(64), nullable=False), + sa.Column("ordinal", sa.Integer(), nullable=False), + sa.Column("phase", sa.String(32), nullable=False), + sa.Column("kind", sa.String(32), nullable=False), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("attempt_token", sa.String(64), nullable=True), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.Column("intent_version", sa.Integer(), nullable=False), + sa.Column("intent_payload", sa.JSON(), nullable=False), + sa.Column("result_version", sa.Integer(), nullable=True), + sa.Column("result_payload", sa.JSON(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("prepared_at", sa.String(40), nullable=False), + sa.Column("started_at", sa.String(40), nullable=True), + sa.Column("completed_at", sa.String(40), nullable=True), + sa.Column("updated_at", sa.String(40), nullable=False), + sa.ForeignKeyConstraint( + ["task_id"], + ["transferpending.task_id"], + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "operation_id", + name=_STEP_OPERATION_UNIQUE, + ), + sa.UniqueConstraint( + "task_id", + "ordinal", + name=_STEP_ORDINAL_UNIQUE, + ), + ) + + +def _create_or_repair_step_table() -> None: + """创建步骤证据表，并在任何查询前拒绝有数据的残缺结构。""" + if _STEP_TABLE not in _table_names(): + _create_step_table() + else: + _validate_or_recreate_empty_table( + table_name=_STEP_TABLE, + expected_columns=_STEP_COLUMNS, + nullable_columns=_STEP_NULLABLE_COLUMNS, + expected_types=_STEP_COLUMN_TYPES, + expected_foreign_keys={( + "task_id", + _PENDING_TABLE, + "task_id", + "CASCADE", + )}, + create_table=_create_step_table, + ) + _repair_unique_constraints( + table_name=_STEP_TABLE, + expected={ + _STEP_OPERATION_UNIQUE: ("operation_id",), + _STEP_ORDINAL_UNIQUE: ("task_id", "ordinal"), + }, + create_table=_create_step_table, + ) + + +def _create_receipt_table() -> None: + """按 append-only ORM 契约创建任务终态结算回执表。""" + op.create_table( + _RECEIPT_TABLE, + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("history_id", sa.Integer(), nullable=False), + sa.Column("settlement_revision", sa.Integer(), nullable=False), + sa.Column("outcome", sa.String(16), nullable=False), + sa.Column("execution_fingerprint", sa.String(64), nullable=False), + sa.Column("lease_token", sa.String(64), nullable=False), + sa.Column("history_status", sa.Boolean(), nullable=False), + sa.Column("src", sa.String(), nullable=True), + sa.Column("src_storage", sa.String(), nullable=True), + sa.Column("pending_deleted", sa.Boolean(), nullable=False), + sa.Column("error", sa.Text(), nullable=True), + sa.Column("created_at", sa.String(40), nullable=False), + sa.Column("updated_at", sa.String(40), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint( + "task_id", + "settlement_revision", + name=_RECEIPT_TASK_REVISION_UNIQUE, + ), + ) + + +def _create_or_repair_receipt_table() -> None: + """创建结算回执表，并只对空的残缺表执行无损重建。""" + if _RECEIPT_TABLE not in _table_names(): + _create_receipt_table() + else: + _validate_or_recreate_empty_table( + table_name=_RECEIPT_TABLE, + expected_columns=_RECEIPT_COLUMNS, + nullable_columns=_RECEIPT_NULLABLE_COLUMNS, + expected_types=_RECEIPT_COLUMN_TYPES, + expected_foreign_keys=set(), + create_table=_create_receipt_table, + ) + _repair_unique_constraints( + table_name=_RECEIPT_TABLE, + expected={ + _RECEIPT_TASK_REVISION_UNIQUE: ( + "task_id", + "settlement_revision", + ), + }, + create_table=_create_receipt_table, + ) + _repair_indexes( + table_name=_RECEIPT_TABLE, + expected={ + _RECEIPT_HISTORY_INDEX: ("history_id",), + _RECEIPT_TASK_REVISION_INDEX: ( + "task_id", + "settlement_revision", + ), + }, + ) + + +def _legacy_review_identity(*, task_id: str, suffix: str) -> str: + """生成迁移遗留复核步骤使用的确定性 SHA-256 身份。""" + return hashlib.sha256( + f"moviepilot:3.0.16:legacy-review:{suffix}:{task_id}".encode("utf-8") + ).hexdigest() + + +def _backfill_legacy_review_steps() -> None: + """为没有可判定人工步骤的遗留人工态补一条 synthetic 审计步骤。""" + if _STEP_TABLE not in _table_names(): + return + pending = sa.table( + _PENDING_TABLE, + sa.column("task_id", sa.String(64)), + sa.column("state", sa.String(32)), + sa.column("execution_state", sa.String(32)), + sa.column("attempt_count", sa.Integer()), + sa.column("last_error", sa.Text()), + sa.column("created_at", sa.String(40)), + sa.column("updated_at", sa.String(40)), + ) + steps = sa.table( + _STEP_TABLE, + sa.column("task_id", sa.String(64)), + sa.column("operation_id", sa.String(64)), + sa.column("checkpoint_fingerprint", sa.String(64)), + sa.column("ordinal", sa.Integer()), + sa.column("phase", sa.String(32)), + sa.column("kind", sa.String(32)), + sa.column("state", sa.String(32)), + sa.column("attempt_token", sa.String(64)), + sa.column("attempt_count", sa.Integer()), + sa.column("intent_version", sa.Integer()), + sa.column("intent_payload", sa.JSON()), + sa.column("result_version", sa.Integer()), + sa.column("result_payload", sa.JSON()), + sa.column("last_error", sa.Text()), + sa.column("prepared_at", sa.String(40)), + sa.column("started_at", sa.String(40)), + sa.column("completed_at", sa.String(40)), + sa.column("updated_at", sa.String(40)), + ) + bind = op.get_bind() + rows = bind.execute( + sa.select( + pending.c.task_id, + pending.c.state, + pending.c.attempt_count, + pending.c.last_error, + pending.c.created_at, + pending.c.updated_at, + ).where( + pending.c.execution_state == "manual_review", + ~sa.exists( + sa.select(steps.c.task_id).where( + steps.c.task_id == pending.c.task_id, + steps.c.state == "manual_review", + ) + ), + ) + ).mappings().all() + values = [] + for row in rows: + task_id = row["task_id"] + intent_payload = { + "schema_version": 1, + "origin": "3.0.16_migration", + "legacy_state": row["state"], + "diagnostic": row["last_error"] or _MANUAL_REVIEW_DIAGNOSTIC, + } + checkpoint_fingerprint = hashlib.sha256( + json.dumps( + intent_payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ).hexdigest() + evidence_time = ( + row["updated_at"] + or row["created_at"] + or _LEGACY_REVIEW_FALLBACK_TIME + ) + values.append({ + "task_id": task_id, + "operation_id": _legacy_review_identity( + task_id=task_id, + suffix="operation", + ), + "checkpoint_fingerprint": checkpoint_fingerprint, + "ordinal": _LEGACY_REVIEW_STEP_ORDINAL, + "phase": "legacy_upgrade", + "kind": _LEGACY_REVIEW_STEP_KIND, + "state": "manual_review", + "attempt_token": None, + "attempt_count": row["attempt_count"] or 0, + "intent_version": 1, + "intent_payload": intent_payload, + "result_version": None, + "result_payload": None, + "last_error": row["last_error"] or _MANUAL_REVIEW_DIAGNOSTIC, + "prepared_at": evidence_time, + "started_at": None, + "completed_at": None, + "updated_at": evidence_time, + }) + if values: + bind.execute(steps.insert(), values) + + +def upgrade() -> None: + """增加执行证据、幂等结算身份和恢复调度字段，支持中断重跑。""" + tables = _table_names() + if _PENDING_TABLE not in tables: + return + _add_pending_columns() + _backfill_pending() + if _PENDING_INDEX not in _index_names(_PENDING_TABLE): + op.create_index( + _PENDING_INDEX, + _PENDING_TABLE, + [ + "execution_state", + "retry_due_at", + "state", + "created_at", + "id", + ], + unique=False, + ) + if _HISTORY_TABLE in tables: + _add_history_columns() + if _HISTORY_INDEX not in _index_names(_HISTORY_TABLE): + op.create_index( + _HISTORY_INDEX, + _HISTORY_TABLE, + ["transfer_task_id"], + unique=True, + ) + _create_or_repair_receipt_table() + _create_or_repair_step_table() + _backfill_legacy_review_steps() + _repair_indexes( + table_name=_STEP_TABLE, + expected={ + _STEP_INDEX: ("task_id", "state", "ordinal"), + }, + ) + + +def _mark_downgrade_uncertain() -> None: + """在丢弃执行证据前恢复旧版可消费状态并保留再升级诊断。""" + columns = _column_names(_PENDING_TABLE) + if "execution_state" not in columns: + return + pending = sa.table( + _PENDING_TABLE, + sa.column("task_id", sa.String(64)), + sa.column("state", sa.String(32)), + sa.column("last_error", sa.Text()), + sa.column("lease_owner", sa.String(128)), + sa.column("lease_token", sa.String(64)), + sa.column("lease_expires_at", sa.String(40)), + sa.column("heartbeat_at", sa.String(40)), + sa.column("execution_state", sa.String(32)), + sa.column("execution_version", sa.Integer()), + sa.column("execution_fingerprint", sa.String(64)), + sa.column("retry_count", sa.Integer()), + sa.column("settlement_revision", sa.Integer()), + sa.column("terminal_history_id", sa.Integer()), + ) + uncertain = sa.or_( + pending.c.execution_state != "not_started", + pending.c.execution_version.is_not(None), + pending.c.execution_fingerprint.is_not(None), + pending.c.retry_count > 0, + pending.c.settlement_revision > 0, + pending.c.terminal_history_id.is_not(None), + ) + if _STEP_TABLE in _table_names(): + steps = sa.table( + _STEP_TABLE, + sa.column("task_id", sa.String(64)), + ) + uncertain = sa.or_( + uncertain, + sa.exists(sa.select(steps.c.task_id).where( + steps.c.task_id == pending.c.task_id + )), + ) + op.get_bind().execute( + pending.update() + .where(uncertain) + .values( + state="accepted", + lease_owner=None, + lease_token=None, + lease_expires_at=None, + heartbeat_at=None, + last_error=sa.case( + ( + sa.or_( + pending.c.last_error.is_(None), + pending.c.last_error == "", + ), + _MANUAL_REVIEW_DIAGNOSTIC, + ), + ( + pending.c.last_error.contains(_MANUAL_REVIEW_DIAGNOSTIC), + pending.c.last_error, + ), + else_=( + pending.c.last_error + + "\n" + + _MANUAL_REVIEW_DIAGNOSTIC + ), + ), + ) + ) + + +def downgrade() -> None: + """保守折叠执行状态后移除 3.0.16 字段、索引与步骤表。""" + if _PENDING_TABLE not in _table_names(): + return + _mark_downgrade_uncertain() + if _STEP_TABLE in _table_names(): + op.drop_table(_STEP_TABLE) + if _RECEIPT_TABLE in _table_names(): + op.drop_table(_RECEIPT_TABLE) + if _HISTORY_TABLE in _table_names(): + history_columns = _column_names(_HISTORY_TABLE) + if _HISTORY_INDEX in _index_names(_HISTORY_TABLE): + op.drop_index(_HISTORY_INDEX, table_name=_HISTORY_TABLE) + with op.batch_alter_table(_HISTORY_TABLE) as batch_op: + for column_name in ( + "transfer_settlement_revision", + "transfer_task_id", + ): + if column_name in history_columns: + batch_op.drop_column(column_name) + pending_columns = _column_names(_PENDING_TABLE) + if _PENDING_INDEX in _index_names(_PENDING_TABLE): + op.drop_index(_PENDING_INDEX, table_name=_PENDING_TABLE) + with op.batch_alter_table(_PENDING_TABLE) as batch_op: + for column_name in ( + "terminal_history_id", + "review_decision", + "review_reason", + "reviewed_by", + "reviewed_at", + "manual_review_revision", + "settlement_revision", + "retry_reason", + "retry_requested_by", + "retry_due_at", + "retry_count", + "retry_generation", + "execution_fingerprint", + "execution_payload", + "execution_version", + "execution_state", + ): + if column_name in pending_columns: + batch_op.drop_column(column_name) diff --git a/docs/architecture-optimization-checklist.md b/docs/architecture-optimization-checklist.md index d62b9a95e..f6eec7acd 100644 --- a/docs/architecture-optimization-checklist.md +++ b/docs/architecture-optimization-checklist.md @@ -69,16 +69,16 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` | 指标 | 当前值 | 解释 | |---|---:|---| -| 宿主 Python 模块 / 内部依赖边 | 837 / 6,834 | `dependency-baseline.json` 当前快照 | +| 宿主 Python 模块 / 内部依赖边 | 843 / 6,883 | `dependency-baseline.json` 当前快照 | | 非平凡 SCC | 2 | 新增 Chain 包根环；另一个是隔离的 29 模块 TMDB 移植包环 | | 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 | | Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 | -| Module Contract | 217 specs / 215 methods / 265 calls | 动态方法名为 0；内部 planning 合同不进入插件调度，旧 transfer 只保留 provider ABI | +| Module Contract | 217 specs / 215 methods / 266 calls | 动态方法名为 0；内部 planning 合同不进入插件调度，旧 transfer 只保留 provider ABI | | Event Contract | 53 | 均已有 payload model，但当前全部是 diagnostic enforcement | | Python 源码量 | 约 271,400 行 | 60 个文件超过 1,000 行，14 个超过 2,000 行 | | 长方法 | 281 个超过 80 行 | 67 个超过 150 行，23 个超过 250 行；大量是私有方法 | | 全量 mypy 历史债务 | 11,983 / 601 文件 | strict frontier 当前只覆盖 41 个文件，且 ratchet 已新增 2 个错误 | -| Ruff 历史诊断 | 934 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` | +| Ruff 历史诊断 | 929 | 低水位门禁通过，但规则集只覆盖 `E4/E7/E9/F/I` | | 覆盖率低水位 | Application 78.24%，Domain 79.29% | Chain、Runtime、Agent、Adapter、Startup 未进入包级覆盖率门禁 | ### 3.3 热点文件 @@ -194,8 +194,8 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` 返回空后再以第二次 CAS 提交 `planned`；planned 重放只消费冻结上下文和目标。 - `S1-L1.3 Lease 与恢复调度`：`VERIFIED`。已交付 token fencing 的 claim/lease/heartbeat/attempt、过期接管、固定退避的唯一恢复入口和有界关闭 owner。 -- `S1-L1.4 幂等执行与终态结算`：`PLANNED`。交付文件/历史幂等、唯一 retry owner 和 - `manual_review` 语义。 +- `S1-L1.4 幂等执行与终态结算`：`VERIFIED`。已交付稳定 operation ledger、严格结果探测、 + 唯一 retry owner、`manual_review` 人工判定和 history/pending/outbox 同 UoW 终态结算。 - `S1-L1.5 E3 全链收口`：`PLANNED`。完成崩溃矩阵、兼容验收与旧路径删除。此叶交付前， ARCH-102 父项保持“执行中”，不得以局部绿色宣称 E3 完成。 @@ -216,10 +216,18 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` fencing。启动和同进程恢复共享唯一 scheduler，确定性失败按固定轮询退避，关闭时 worker、replay、 lease release 和 heartbeat 都由有界生命周期 owner 持有。损坏投影以无有效租约 CAS 留痕，同错不 重复刷写，且不会阻塞后续健康任务。 -- `TransferPending` 仍缺少逐步骤执行结果和 `manual_review`，因此还不能判定“文件已移动、历史未提交” - 等外部结果未知的后续中间态。 -- 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、 - lease/heartbeat 和人工恢复要求不一致。 +- `S1-L1.4` 已增加 `TransferExecutionStep` 独立账本：每一步在副作用前冻结 intent 和稳定 + operation ID，以 lease + attempt 双 CAS 提交结果；重启遇到遗留 `STARTED` 时必须先严格探测， + 只有 `NOT_APPLIED` 能轮换 attempt 自动重试，`UNKNOWN/CONFLICT` 进入 `manual_review`。 +- 文件 cleanup、目录创建、版本发现/删除、覆盖目标删除、目标物化和跨存储 move 的源删除均已拆为 + 可重放步骤；本地复制使用完整内容比较，远端结果证据不足时不会伪造 exactly-once。 +- 成功、失败及覆盖拒绝均通过 task-aware writer 在一个 UoW 内提交 history、pending、step cleanup + 与可选 outbox；revision 和确定性 occurrence key 使“文件已移动、历史未提交”在恢复后只补历史， + 不重复文件副作用。历史/API/Agent 重试只登记 durable retry intent，由唯一 scheduler 重新 claim。 +- 管理员人工判定 API 只公开 `not_applied` 与带结果证据的 `applied`，并持久记录操作者、理由、结论 + 和 revision；无租约人工路径不能直接伪造失败终态。 +- 以上实现已满足 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 稳定身份、步骤状态、 + lease/heartbeat 和人工恢复的阶段性要求；完整崩溃矩阵与兼容收口仍由 `S1-L1.5` 验收。 **目标与步骤** @@ -228,9 +236,9 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` - [x] 初始登记保存稳定源身份、版本化请求和状态；目标与有序操作在纯规划完成后以 planning checkpoint 原子更新，任何文件副作用不得早于该提交。 - [x] 增加 claim/lease/heartbeat/attempt 与过期接管，同一任务同时只能有一个 worker owner。 -- [ ] 设计幂等文件操作和历史提交；只有所有必要步骤达到持久终态后才能删除记录。 -- [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner，定义旧记录迁移和兼容规则。 -- [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界，不直接套用 E2 +- [x] 设计幂等文件操作和历史提交；只有所有必要步骤达到持久终态后才能删除记录。 +- [x] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner，定义旧记录迁移和兼容规则。 +- [x] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界，不直接套用 E2 Outbox 的 dead-letter 语义；禁止按年龄通用清理 pending。 - [x] 当前 admission/planning 数据模型变更均配套 Alembic migration，并验证升级、降级和中断重跑。 @@ -238,10 +246,10 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` - [x] 登记后、内存入队前崩溃，重启可继续。 - [x] 持久登记成功但内存入队失败，重启可继续。 -- [ ] 文件移动后、历史提交前崩溃，在支持稳定身份/幂等操作的存储上不重复移动且可补齐历史。 -- [ ] worker 未知异常和 lease 超时后保留可诊断状态。 -- [ ] 重复回放、重复消息和人工重试都保持幂等。 -- [ ] 外部存储返回结果未知时进入 `manual_review`，不得伪装成 exactly-once 成功。 +- [x] 文件移动后、历史提交前崩溃，在支持稳定身份/幂等操作的存储上不重复移动且可补齐历史。 +- [x] worker 未知异常和 lease 超时后保留可诊断状态。 +- [x] 重复回放、重复消息和人工重试都保持幂等。 +- [x] 外部存储返回结果未知时进入 `manual_review`，不得伪装成 exactly-once 成功。 ```bash .venv/bin/python -m pytest \ diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index e48ea6340..fe7d65b59 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -704,8 +704,8 @@ flowchart LR | 指标 | 当前值 | |---|---:| -| Python 模块 | 837 | -| 内部导入边 | 6,834 | +| Python 模块 | 843 | +| 内部导入边 | 6,883 | | 非平凡 SCC | 2（`ARCH-107` 临时 Chain 包根环；精确 containment 的 TMDB 移植包环） | | Direct egress | 66（12 条待迁移债务，54 条精确 containment） | | Module Contract V2 spec | 217（其中 215 个进入 `run_module` 观察面） | diff --git a/docs/architecture-refactor-roadmap.md b/docs/architecture-refactor-roadmap.md index 816d9105e..03b312009 100644 --- a/docs/architecture-refactor-roadmap.md +++ b/docs/architecture-refactor-roadmap.md @@ -95,7 +95,7 @@ G-ARCH 只有在以下条件全部满足后才可完成： | S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地；先持久 commit 再入队，入队失败保留可恢复记录；宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission | | S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化；无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划，有 provider 时先提交 `provider_pending`，全部返回空后再以第二次 CAS 提交 `planned`；重放只执行冻结目标，所有文件副作用晚于对应 checkpoint commit | | S1-L1.3 Lease 与恢复调度 | `VERIFIED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地；启动回放和同进程恢复共用唯一调度入口，同一任务同时只有一个 worker owner | -| S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放；唯一 retry owner 生效，未知外部结果进入 `manual_review`，仅完整终态删除 pending | +| S1-L1.4 幂等执行与终态结算 | `VERIFIED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放；唯一 retry owner 生效，未知外部结果进入 `manual_review`，仅完整终态删除 pending | | S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整；旧 fail-open、重复状态与兼容层外旧入口删除，ARCH-102 债务归零 | | S1-L2 Workflow typed query | `PLANNED` | S0 | Workflow Application Port 不返回 `Any`/ORM，Session 内投影 DTO，正式调用方全部切换 | | S1-L3 Chain/Agent typed data ports | `PLANNED` | S1-L2 | `ChainDataPorts`/`AgentDataPorts` 的 raw Oper/`Any` factory 全部清零，兼容调用进入 Legacy 层 | @@ -144,7 +144,7 @@ G-ARCH 只有在以下条件全部满足后才可完成： | S4-L2 Event strict contract | `PLANNED` | S0-L2.6,S1-L6 | 宿主事件输入/输出按风险 strict，诊断例外只属于第三方插件兼容 | | S4-L3 Complexity v2 | `PLANNED` | S3 | 私有方法、class/file、圈复杂度进入门禁；所有超限通过职责拆分归零 | | S4-L4 全量 mypy 清零 | `PLANNED` | S3,S4-L1,S4-L2 | `mypy-baseline.json` 归零并删除债务接受路径，全宿主 strict 类型通过 | -| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 934 条诊断归零，规则集扩展经过独立审查且新增诊断为零 | +| S4-L5 Ruff 治理债务清零 | `PLANNED` | S3 | 当前受控 929 条诊断归零，规则集扩展经过独立审查且新增诊断为零 | | S4-L6 Coverage/并发/质量证据 | `PLANNED` | S3,S4-L1,S4-L2 | 高风险包纳入 coverage；raw concurrency 分类清零；Module Quality 有真实 evidence test | ### S5：Plugin、Agent、Domain、Startup 与最终收口 diff --git a/docs/mcp-api.md b/docs/mcp-api.md index 783775bbb..36734c732 100644 --- a/docs/mcp-api.md +++ b/docs/mcp-api.md @@ -158,6 +158,9 @@ FastAPI 的 HTTP 异常和参数校验异常统一使用 `message`，不再返 | POST | `/api/v1/transfer/manual/target-path` | 按源文件与目录配置匹配手动整理目标路径；请求体为 `ManualTransferItem`，该接口不执行媒体识别 | | POST | `/api/v1/transfer/manual/history` | 查询文件、批量文件或目录命中的成功整理历史摘要，用于进入手动整理界面时显示重新整理状态 | | POST | `/api/v1/transfer/manual` | 手动整理；请求体可用 `media_source` + `media_id` 指定本次识别与刮削数据源；音乐请求未传 `music_type` 时，目录按 `album`、文件按 `recording` 解释；命中失败历史时自动清理旧目标和记录后重试，`reorganize=true` 时清理命中的成功历史和非移动模式旧目标后重新整理 | +| GET | `/api/v1/transfer/tasks/manual-reviews` | 管理员分页查询 durable 人工复核任务；`state` 仅允许 `manual_review`（默认）或已经人工判定、等待调度恢复的 `retry_wait`，支持 `page` 与 `page_size`。响应只公开任务、源文件、状态、步骤意图/证据/错误和复核修订号，不返回 lease 或 attempt 身份 | +| GET | `/api/v1/transfer/tasks/{task_id}/manual-review` | 管理员查询单个 durable 人工复核任务详情；仅可读取 `manual_review` 或已经人工判定的 `retry_wait` 任务，其余状态按不存在处理 | +| POST | `/api/v1/transfer/tasks/{task_id}/manual-review` | 管理员判定处于 `manual_review` 的 durable 整理步骤；请求包含 `operation_id`、`decision=not_applied|applied`、`reason`，`applied` 还必须提供 `result_payload`。`failed` 不属于公开决策，失败终态只能由持租约的 durable 结算写入；响应仅返回任务、操作、决策、后续状态和复核修订号 | #### 站点 diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index ab2f25521..a188bff09 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -68,6 +68,7 @@ to make the directory tree look symmetrical. | `app/application/chain/` | Injectable Chain runtime context and compatibility provider | | `app/application/agentdata.py` | Named Agent data ports; canonical Agent consumers use `get_agent_*_port()` and do not alias legacy proxies to Oper classes | | `app/application/outbox.py` | Durable intent and Outbox repository/dispatcher contracts for post-commit side effects | +| `app/application/transfer_execution.py` | Durable transfer execution contracts: stable operation identity, step/checkpoint state, retry/manual-review commands and terminal-settlement DTOs; contains no SQLAlchemy or external I/O | | `app/application/plugin/` | Plugin market catalog, installation command, installed-plugin identity contract, runtime port, folder operations and dynamic-route use cases; filenames remain single words (`catalog.py`, `identity.py`, `install.py`, `runtime.py`, `folders.py`, `routes.py`) | | `app/application/server/` | MoviePilot Server reporting and sharing use cases; local data readers and transport callbacks are injected by startup | | `app/application/site/` | Configured site catalog, authentication level and index-resource capability; the generated extension and its data bundle stay together here | @@ -148,6 +149,23 @@ must not be reintroduced as Oper aliases in canonical Agent modules. Monitor history checks use `get_transfer_history_port()` from `app/application/history.py`; the constructible `TransferHistoryPort` facade is retained only for compatibility and is not a canonical Oper substitute. + +Durable transfer execution follows one explicit boundary. The Chain freezes each +external file operation into the Application-owned contract in +`app/application/transfer_execution.py`; `app/db/adapters/transfer_execution.py` +uses short transactions to persist the task ledger and fences every state change +with the current lease and attempt token. `app/db/oper/transferexecutionstep.py` +remains table-oriented and never owns retry or recovery policy. External file I/O +runs outside those transactions. A legacy or remote operation whose result cannot +be proven as applied or not applied enters `manual_review` and must not be replayed +automatically. Terminal history, pending state, execution-step cleanup and the +optional outbox intent are committed only by the task-aware implementation in +`app/db/adapters/chain.py`; canonical callers must not add a second settlement or +direct pending-deletion path. Task-aware settlement never performs synchronous +event publication inside the worker callback; the committed outbox owns delivery. +History mutation and maintenance paths may delete or replace only legacy rows with +no `transfer_task_id`, because durable receipts are recovery evidence rather than +ordinary user-maintained history. Canonical Chain, API, Scheduler and Agent consumers read notification and media server configuration through the named helpers in `app/application/notification.py` and `app/application/mediaserver.py`. `ServiceConfigHelper` remains the parser at diff --git a/tests/conftest.py b/tests/conftest.py index 02dc5cecd..770df6085 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -231,6 +231,9 @@ def configure_plugin_system_services(): from app.db.oper.subscribehistory import SubscribeHistoryOper from app.db.oper.transferhistory import TransferHistoryOper from app.db.adapters.transfer import TransactionalTransferAdmissionRepository + from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, + ) from app.db.oper.user import UserOper from app.db.oper.workflow import WorkflowOper, configure_workflow_legacy_writer from app.db.oper.message import MessageOper @@ -306,6 +309,9 @@ def configure_plugin_system_services(): transfer_pending=lambda: TransactionalTransferAdmissionRepository( SessionFactory ), + transfer_execution=lambda: TransactionalTransferExecutionRepository( + SessionFactory + ), media_server=lambda: MediaServerOper(), download_failure=lambda: TransactionalDownloadFailureRepository( SessionFactory diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index 1c13d2e24..6dc04d645 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -1441,8 +1441,8 @@ "runtime_only": true } }, - "edge_count": 6834, - "edge_sha256": "07c0b6f24ef3ce3e7ea0fa7c1b617388e11e2453ce2b0b35bddb38ed2d7f5835", + "edge_count": 6883, + "edge_sha256": "f44ae63f222eda2fc1b8ce805ab5138550a560bef6c266e3f2fae6338a243cd2", "edges": [ "app -> app.runtime", "app -> app.runtime.compat", @@ -2217,6 +2217,9 @@ "app.agent.tools.impl.delete_transfer_history -> app.agent.tools.tags", "app.agent.tools.impl.delete_transfer_history -> app.application", "app.agent.tools.impl.delete_transfer_history -> app.application.agentdata", + "app.agent.tools.impl.delete_transfer_history -> app.application.chain", + "app.agent.tools.impl.delete_transfer_history -> app.application.chain.data", + "app.agent.tools.impl.delete_transfer_history -> app.application.transfer_execution", "app.agent.tools.impl.delete_transfer_history -> app.chain", "app.agent.tools.impl.delete_transfer_history -> app.chain.storage", "app.agent.tools.impl.delete_transfer_history -> app.runtime", @@ -3306,8 +3309,11 @@ "app.api.endpoints.history -> app.api.response", "app.api.endpoints.history -> app.application", "app.api.endpoints.history -> app.application.agent", + "app.api.endpoints.history -> app.application.chain", + "app.api.endpoints.history -> app.application.chain.data", "app.api.endpoints.history -> app.application.configuration", "app.api.endpoints.history -> app.application.history", + "app.api.endpoints.history -> app.application.transfer_execution", "app.api.endpoints.history -> app.runtime", "app.api.endpoints.history -> app.runtime.config", "app.api.endpoints.history -> app.runtime.log", @@ -3814,9 +3820,12 @@ "app.api.endpoints.transfer -> app.api.dependencies.history", "app.api.endpoints.transfer -> app.api.response", "app.api.endpoints.transfer -> app.application", + "app.api.endpoints.transfer -> app.application.chain", + "app.api.endpoints.transfer -> app.application.chain.data", "app.api.endpoints.transfer -> app.application.configuration", "app.api.endpoints.transfer -> app.application.directory", "app.api.endpoints.transfer -> app.application.history", + "app.api.endpoints.transfer -> app.application.transfer_execution", "app.api.endpoints.transfer -> app.chain", "app.api.endpoints.transfer -> app.chain.media", "app.api.endpoints.transfer -> app.chain.transfer", @@ -3975,8 +3984,10 @@ "app.application.chain.context -> app.runtime.stop", "app.application.chain.data -> app.application", "app.application.chain.data -> app.application.transfer", + "app.application.chain.data -> app.application.transfer_execution", "app.application.chain.durable_events -> app.application", "app.application.chain.durable_events -> app.application.history", + "app.application.chain.durable_events -> app.application.transfer_execution", "app.application.chain.durable_events -> app.domain", "app.application.chain.durable_events -> app.domain.context", "app.application.chain.durable_events -> app.domain.meta", @@ -4436,6 +4447,7 @@ "app.application.transfer -> app.adapters.system.host", "app.application.transfer -> app.application", "app.application.transfer -> app.application.agent", + "app.application.transfer -> app.application.transfer_execution", "app.application.transfer -> app.domain", "app.application.transfer -> app.domain.context", "app.application.transfer -> app.domain.media", @@ -4557,6 +4569,7 @@ "app.chain._transfer -> app.application.formatting", "app.chain._transfer -> app.application.history", "app.chain._transfer -> app.application.transfer", + "app.chain._transfer -> app.application.transfer_execution", "app.chain._transfer -> app.chain", "app.chain._transfer -> app.chain._contracts", "app.chain._transfer -> app.chain.media", @@ -4982,12 +4995,14 @@ "app.chain.transfer -> app.application", "app.chain.transfer -> app.application.chain", "app.chain.transfer -> app.application.chain.data", + "app.chain.transfer -> app.application.chain.durable_events", "app.chain.transfer -> app.application.configuration", "app.chain.transfer -> app.application.directory", "app.chain.transfer -> app.application.formatting", "app.chain.transfer -> app.application.history", "app.chain.transfer -> app.application.outbox", "app.chain.transfer -> app.application.transfer", + "app.chain.transfer -> app.application.transfer_execution", "app.chain.transfer -> app.chain", "app.chain.transfer -> app.chain._transfer", "app.chain.transfer -> app.chain.media", @@ -5094,12 +5109,18 @@ "app.db.adapters.chain -> app.application.chain.durable_events", "app.db.adapters.chain -> app.application.history", "app.db.adapters.chain -> app.application.outbox", + "app.db.adapters.chain -> app.application.transfer_execution", "app.db.adapters.chain -> app.db", "app.db.adapters.chain -> app.db.adapters", "app.db.adapters.chain -> app.db.adapters.outbox", + "app.db.adapters.chain -> app.db.models", + "app.db.adapters.chain -> app.db.models.transfersettlementreceipt", "app.db.adapters.chain -> app.db.oper", "app.db.adapters.chain -> app.db.oper.downloadhistory", + "app.db.adapters.chain -> app.db.oper.transferexecutionstep", "app.db.adapters.chain -> app.db.oper.transferhistory", + "app.db.adapters.chain -> app.db.oper.transferpending", + "app.db.adapters.chain -> app.db.oper.transfersettlementreceipt", "app.db.adapters.chain -> app.db.uow", "app.db.adapters.download -> app.db", "app.db.adapters.download -> app.db.oper", @@ -5153,6 +5174,16 @@ "app.db.adapters.transfer -> app.db.oper", "app.db.adapters.transfer -> app.db.oper.transferpending", "app.db.adapters.transfer -> app.db.uow", + "app.db.adapters.transfer_execution -> app.application", + "app.db.adapters.transfer_execution -> app.application.transfer_execution", + "app.db.adapters.transfer_execution -> app.db", + "app.db.adapters.transfer_execution -> app.db.models", + "app.db.adapters.transfer_execution -> app.db.models.transferexecutionstep", + "app.db.adapters.transfer_execution -> app.db.models.transferpending", + "app.db.adapters.transfer_execution -> app.db.oper", + "app.db.adapters.transfer_execution -> app.db.oper.transferexecutionstep", + "app.db.adapters.transfer_execution -> app.db.oper.transferpending", + "app.db.adapters.transfer_execution -> app.db.uow", "app.db.adapters.workflow -> app.application", "app.db.adapters.workflow -> app.application.workflow", "app.db.adapters.workflow -> app.db", @@ -5258,6 +5289,10 @@ "app.db.models.subscribehistory -> app.schemas.types", "app.db.models.systemconfig -> app.db", "app.db.models.systemconfig -> app.db.base", + "app.db.models.transferexecutionstep -> app.db", + "app.db.models.transferexecutionstep -> app.db.base", + "app.db.models.transferexecutionstep -> app.db.models", + "app.db.models.transferexecutionstep -> app.db.models.transferpending", "app.db.models.transferhistory -> app.db", "app.db.models.transferhistory -> app.db.base", "app.db.models.transferhistory -> app.db.models", @@ -5266,6 +5301,8 @@ "app.db.models.transferhistory -> app.schemas.types", "app.db.models.transferpending -> app.db", "app.db.models.transferpending -> app.db.base", + "app.db.models.transfersettlementreceipt -> app.db", + "app.db.models.transfersettlementreceipt -> app.db.base", "app.db.models.user -> app.db", "app.db.models.user -> app.db.base", "app.db.models.userconfig -> app.db", @@ -5345,6 +5382,10 @@ "app.db.oper.systemconfig -> app.foundation.singleton", "app.db.oper.systemconfig -> app.schemas", "app.db.oper.systemconfig -> app.schemas.types", + "app.db.oper.transferexecutionstep -> app.db", + "app.db.oper.transferexecutionstep -> app.db.base", + "app.db.oper.transferexecutionstep -> app.db.models", + "app.db.oper.transferexecutionstep -> app.db.models.transferexecutionstep", "app.db.oper.transferhistory -> app.db", "app.db.oper.transferhistory -> app.db.base", "app.db.oper.transferhistory -> app.db.models", @@ -5355,6 +5396,10 @@ "app.db.oper.transferpending -> app.db.base", "app.db.oper.transferpending -> app.db.models", "app.db.oper.transferpending -> app.db.models.transferpending", + "app.db.oper.transfersettlementreceipt -> app.db", + "app.db.oper.transfersettlementreceipt -> app.db.base", + "app.db.oper.transfersettlementreceipt -> app.db.models", + "app.db.oper.transfersettlementreceipt -> app.db.models.transfersettlementreceipt", "app.db.oper.user -> app.db", "app.db.oper.user -> app.db.base", "app.db.oper.user -> app.db.models", @@ -5808,6 +5853,7 @@ "app.modules.filemanager.module -> app.application.messaging", "app.modules.filemanager.module -> app.application.messaging.message", "app.modules.filemanager.module -> app.application.transfer", + "app.modules.filemanager.module -> app.application.transfer_execution", "app.modules.filemanager.module -> app.domain", "app.modules.filemanager.module -> app.domain.context", "app.modules.filemanager.module -> app.domain.meta", @@ -5960,6 +6006,7 @@ "app.modules.filemanager.transhandler -> app.application.messaging", "app.modules.filemanager.transhandler -> app.application.messaging.message", "app.modules.filemanager.transhandler -> app.application.transfer", + "app.modules.filemanager.transhandler -> app.application.transfer_execution", "app.modules.filemanager.transhandler -> app.domain", "app.modules.filemanager.transhandler -> app.domain.context", "app.modules.filemanager.transhandler -> app.domain.meta", @@ -7587,6 +7634,7 @@ "app.schemas.token -> app.schemas", "app.schemas.token -> app.schemas.user", "app.schemas.transfer -> app.schemas", + "app.schemas.transfer -> app.schemas.common", "app.schemas.transfer -> app.schemas.context", "app.schemas.transfer -> app.schemas.file", "app.schemas.transfer -> app.schemas.media", @@ -7923,6 +7971,7 @@ "app.startup.initializers.modules -> app.db.adapters.subscription", "app.startup.initializers.modules -> app.db.adapters.transaction", "app.startup.initializers.modules -> app.db.adapters.transfer", + "app.startup.initializers.modules -> app.db.adapters.transfer_execution", "app.startup.initializers.modules -> app.db.adapters.workflow", "app.startup.initializers.modules -> app.db.oper", "app.startup.initializers.modules -> app.db.oper.agentchat", @@ -8279,7 +8328,7 @@ "app.workflow.actions.transfer_file -> app.workflow", "app.workflow.actions.transfer_file -> app.workflow.actions" ], - "module_count": 837, + "module_count": 843, "modules": [ "app", "app.adapters", @@ -8627,6 +8676,7 @@ "app.application.torrent", "app.application.torrent_cache", "app.application.transfer", + "app.application.transfer_execution", "app.application.workflow", "app.chain", "app.chain._contracts", @@ -8678,6 +8728,7 @@ "app.db.adapters.subscription", "app.db.adapters.transaction", "app.db.adapters.transfer", + "app.db.adapters.transfer_execution", "app.db.adapters.workflow", "app.db.base", "app.db.decorators", @@ -8707,8 +8758,10 @@ "app.db.models.subscribe", "app.db.models.subscribehistory", "app.db.models.systemconfig", + "app.db.models.transferexecutionstep", "app.db.models.transferhistory", "app.db.models.transferpending", + "app.db.models.transfersettlementreceipt", "app.db.models.user", "app.db.models.userconfig", "app.db.models.workflow", @@ -8726,8 +8779,10 @@ "app.db.oper.subscribe", "app.db.oper.subscribehistory", "app.db.oper.systemconfig", + "app.db.oper.transferexecutionstep", "app.db.oper.transferhistory", "app.db.oper.transferpending", + "app.db.oper.transfersettlementreceipt", "app.db.oper.user", "app.db.oper.userconfig", "app.db.oper.workflow", diff --git a/tests/fixtures/architecture/ruff-baseline.json b/tests/fixtures/architecture/ruff-baseline.json index 9c5bb903a..25ad1a815 100644 --- a/tests/fixtures/architecture/ruff-baseline.json +++ b/tests/fixtures/architecture/ruff-baseline.json @@ -149,9 +149,6 @@ "app/agent/tools/impl/delete_rule_group.py": { "I001": 1 }, - "app/agent/tools/impl/delete_transfer_history.py": { - "I001": 1 - }, "app/agent/tools/impl/execute_command.py": { "I001": 1 }, @@ -411,9 +408,6 @@ "app/db/__init__.py": { "I001": 1 }, - "app/db/adapters/chain.py": { - "I001": 1 - }, "app/db/adapters/outbox.py": { "I001": 1 }, @@ -477,9 +471,6 @@ "app/db/models/systemconfig.py": { "I001": 1 }, - "app/db/models/transferhistory.py": { - "I001": 1 - }, "app/db/models/user.py": { "I001": 1 }, @@ -1265,9 +1256,6 @@ "tests/test_capability_runtime.py": { "I001": 1 }, - "tests/test_chain_durable_events.py": { - "I001": 1 - }, "tests/test_chain_layering.py": { "I001": 1 }, @@ -1334,9 +1322,6 @@ "tests/test_db_session_lifecycle.py": { "I001": 1 }, - "tests/test_db_transferhistory_queries.py": { - "F401": 1 - }, "tests/test_delete_transfer_history_tool.py": { "I001": 1 }, diff --git a/tests/fixtures/architecture/runtime-contract-baseline.json b/tests/fixtures/architecture/runtime-contract-baseline.json index 956c04fa7..5a69ca4a6 100644 --- a/tests/fixtures/architecture/runtime-contract-baseline.json +++ b/tests/fixtures/architecture/runtime-contract-baseline.json @@ -7974,7 +7974,7 @@ } }, "run_module": { - "call_count": 265, + "call_count": 266, "dynamic_call_count": 0, "dynamic_calls": [], "method_count": 215, @@ -8801,6 +8801,11 @@ "caller": "app.chain", "count": 1, "mode": "sync" + }, + { + "caller": "app.chain.transfer", + "count": 1, + "mode": "sync" } ], "filter_torrents": [ diff --git a/tests/test_architecture_dependencies.py b/tests/test_architecture_dependencies.py index eff18dd46..d9396c3a7 100644 --- a/tests/test_architecture_dependencies.py +++ b/tests/test_architecture_dependencies.py @@ -482,6 +482,8 @@ def test_transfer_pending_oper_import_is_confined_to_database_boundary(): """宿主仅允许事务适配器和兼容导出直接导入整理待处理 Oper。""" allowed_paths = { "app/db/adapters/transfer.py", + "app/db/adapters/chain.py", + "app/db/adapters/transfer_execution.py", "app/db/oper/__init__.py", } violations: list[str] = [] diff --git a/tests/test_chain_durable_events.py b/tests/test_chain_durable_events.py index d1f03f52e..a516f3f43 100644 --- a/tests/test_chain_durable_events.py +++ b/tests/test_chain_durable_events.py @@ -1,13 +1,18 @@ """下载与整理 durable 事件的原子写入和对象恢复测试。""" import json +from concurrent.futures import ThreadPoolExecutor +from threading import Barrier +from unittest.mock import Mock from uuid import UUID import pytest from sqlalchemy import create_engine, delete, select +from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import sessionmaker from app.application.chain.durable_events import ( + TransferResultSettlement, download_added_event_key, restore_download_added, restore_transfer_result, @@ -15,16 +20,26 @@ from app.application.chain.durable_events import ( snapshot_transfer_result, transfer_result_event_key, ) +from app.application.history import TransferHistoryMutationCommand +from app.application.transfer_execution import ( + TransferExecutionLeaseLostError, + TransferSettlementResult, +) +from app.db.adapters.chain import TransactionalChainDurableEventWriter from app.db.base import Base from app.db.models.downloadhistory import DownloadFiles, DownloadHistory from app.db.models.outbox import OutboxMessage +from app.db.models.transferexecutionstep import TransferExecutionStep from app.db.models.transferhistory import TransferHistory +from app.db.models.transferpending import TransferPending +from app.db.models.transfersettlementreceipt import TransferSettlementReceipt +from app.db.oper.transferhistory import TransferHistoryOper +from app.db.uow import SqlAlchemyUnitOfWork from app.domain.context import Context, MediaInfo, TorrentInfo from app.domain.metainfo import MetaInfo from app.schemas.file import FileItem from app.schemas.transfer import TransferInfo from app.schemas.types import MediaSource, MediaType -from app.db.adapters.chain import TransactionalChainDurableEventWriter def _session_factory(): @@ -66,6 +81,79 @@ def _objects(): return meta, media, context, fileitem, transferinfo +def _add_settling_pending( + factory, + *, + task_id: str = "task-1", + lease_token: str = "lease-1", + execution_fingerprint: str = "execution-1", + settlement_revision: int = 0, + src_path: str | None = None, +) -> None: + """写入具备有效长租约和执行检查点的待结算任务。""" + with factory() as session: + session.add(TransferPending( + task_id=task_id, + storage="local", + src_path=src_path or f"/downloads/{task_id}.mkv", + created_at="2026-08-27 09:00:00", + state="planned", + updated_at="2026-08-27 09:00:00", + input_version=1, + planning_input={"schema_version": 1, "source": task_id}, + input_fingerprint=f"input-{task_id}", + checkpoint_version=1, + checkpoint_payload={"schema_version": 1, "task_id": task_id}, + planned_at="2026-08-27 09:00:00", + lease_owner="worker-1", + lease_token=lease_token, + lease_expires_at="2099-01-01 00:00:00.000000", + heartbeat_at="2026-08-27 01:00:00.000000", + attempt_count=1, + execution_state="settling", + execution_version=1, + execution_payload={"schema_version": 1}, + execution_fingerprint=execution_fingerprint, + retry_generation=0, + retry_count=0, + settlement_revision=settlement_revision, + )) + session.commit() + + +def _settlement( + *, + outcome: str, + task_id: str = "task-1", + lease_token: str = "lease-1", + execution_fingerprint: str = "execution-1", +) -> TransferResultSettlement: + """构造测试使用的稳定终态结算身份。""" + return TransferResultSettlement( + task_id=task_id, + lease_token=lease_token, + execution_fingerprint=execution_fingerprint, + outcome=outcome, + error="目标文件校验失败" if outcome == "failed" else None, + ) + + +def _stage_result_history( + repository, + *, + task_id: str, + succeeded: bool, + src_path: str | None = None, +): + """通过兼容历史端口暂存一条最小任务结算记录。""" + return repository.add_force( + src=src_path or f"/downloads/{task_id}.mkv", + src_storage="local", + status=succeeded, + errmsg=None if succeeded else "目标文件校验失败", + ) + + def _assert_event_key( event_key: str, topic: str, @@ -213,6 +301,71 @@ def test_event_keys_distinguish_reused_history_ids(): _assert_event_key(event_key, "transfer.completed", 7) +def test_task_settlement_event_key_is_deterministic_and_revision_scoped(): + """任务结算按稳定任务、修订号和结果生成可重放的唯一事件键。""" + succeeded = TransferResultSettlement( + task_id="task-1", + lease_token="lease-1", + execution_fingerprint="execution-1", + outcome="succeeded", + ) + failed = TransferResultSettlement( + task_id="task-1", + lease_token="lease-1", + execution_fingerprint="execution-1", + outcome="failed", + error="目标文件校验失败", + ) + + assert transfer_result_event_key( + "transfer.completed", 7, settlement=succeeded, settlement_revision=2 + ) == "transfer.result:task-1:2:succeeded:v1" + assert transfer_result_event_key( + "transfer.completed", 99, settlement=succeeded, settlement_revision=2 + ) == "transfer.result:task-1:2:succeeded:v1" + assert transfer_result_event_key( + "transfer.failed", 7, settlement=failed, settlement_revision=3 + ) == "transfer.result:task-1:3:failed:v1" + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"task_id": ""}, "缺少任务"), + ({"outcome": "unknown"}, "不支持的整理终态"), + ({"outcome": "failed", "error": None}, "必须包含可诊断原因"), + ], +) +def test_task_settlement_rejects_incomplete_identity(kwargs, message): + """任务结算在进入数据库适配器前拒绝不完整的 fencing 身份。""" + values = { + "task_id": "task-1", + "lease_token": "lease-1", + "execution_fingerprint": "execution-1", + "outcome": "succeeded", + "error": None, + } + values.update(kwargs) + with pytest.raises(ValueError, match=message): + TransferResultSettlement(**values) + + +def test_task_settlement_event_key_requires_transaction_revision(): + """任务结算事件键只能使用持久层已取得的正向修订号。""" + settlement = TransferResultSettlement( + task_id="task-1", + lease_token="lease-1", + execution_fingerprint="execution-1", + outcome="succeeded", + ) + with pytest.raises(ValueError, match="缺少有效结算修订号"): + transfer_result_event_key( + "transfer.completed", + 7, + settlement=settlement, + ) + + def test_transfer_succeeds_when_history_id_is_reused_with_retained_outbox(): """整理历史删除而 outbox 保留时，复用主键不得阻断新整理记录。""" factory = _session_factory() @@ -302,3 +455,724 @@ def test_transfer_event_failure_leaves_committed_intent_pending(): assert history.status is True assert outbox.status == "pending" _assert_event_key(outbox.event_key, "transfer.completed", history.id) + + +def test_task_success_settlement_atomically_deletes_pending_and_steps(): + """成功终态原子提交历史、pending、步骤和待异步投递的 intent。""" + factory = _session_factory() + _add_settling_pending(factory) + with factory() as session: + session.add(TransferExecutionStep( + task_id="task-1", + operation_id="operation-1", + checkpoint_fingerprint="plan-1", + ordinal=0, + phase="transfer", + kind="copy", + state="succeeded", + attempt_count=1, + intent_version=1, + intent_payload={"src": "/downloads/task-1.mkv"}, + result_version=1, + result_payload={"dest": "/library/task-1.mkv"}, + prepared_at="2026-08-27 09:00:00", + completed_at="2026-08-27 09:01:00", + updated_at="2026-08-27 09:01:00", + )) + session.commit() + writer = TransactionalChainDurableEventWriter(factory) + published = [] + + def publish(payload): + """验证即时发布只能观察到已提交的完整终态。""" + with factory() as session: + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert session.execute(select(TransferHistory)).scalar_one().status is True + assert session.execute(select(OutboxMessage)).scalar_one().status == "pending" + published.append(dict(payload)) + + result = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=True, + ), + event_payload={}, + publish=publish, + settlement=_settlement(outcome="succeeded"), + ) + + assert result == TransferSettlementResult( + history_id=1, + settlement_revision=1, + pending_deleted=True, + ) + with factory() as session: + history = session.execute(select(TransferHistory)).scalar_one() + receipt = session.execute(select(TransferSettlementReceipt)).scalar_one() + outbox = session.execute(select(OutboxMessage)).scalar_one() + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert session.execute(select(TransferExecutionStep)).scalar_one_or_none() is None + assert history.transfer_task_id is None + assert history.transfer_settlement_revision is None + assert receipt.task_id == "task-1" + assert receipt.history_id == history.id + assert receipt.outcome == "succeeded" + assert receipt.execution_fingerprint == "execution-1" + assert receipt.lease_token == "lease-1" + assert receipt.history_status is True + assert receipt.src == "/downloads/task-1.mkv" + assert receipt.src_storage == "local" + assert receipt.pending_deleted is True + assert outbox.event_key == "transfer.result:task-1:1:succeeded:v1" + assert outbox.status == "pending" + assert outbox.payload["idempotency_key"] == outbox.event_key + assert "task_id" not in outbox.payload + assert published == [] + + +def test_task_success_replay_reads_history_without_new_event(): + """成功删除 pending 后重复结算只回读历史，不重复登记或发布事件。""" + factory = _session_factory() + _add_settling_pending(factory) + writer = TransactionalChainDurableEventWriter(factory) + settlement = _settlement(outcome="succeeded") + calls = [] + + first = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=True, + ), + event_payload={}, + publish=lambda _payload: calls.append("first"), + settlement=settlement, + ) + replay = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda _repository: pytest.fail("幂等回读不得重写历史"), + event_payload={}, + publish=lambda _payload: pytest.fail("幂等回读不得重复发布"), + settlement=settlement, + ) + + assert isinstance(first, TransferSettlementResult) + assert replay == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + with factory() as session: + assert len(session.execute(select(TransferHistory)).scalars().all()) == 1 + assert len(session.execute(select(OutboxMessage)).scalars().all()) == 1 + assert calls == [] + + +def test_multiple_same_source_tasks_keep_independent_replay_receipts(): + """同源多代任务可依次完成，旧任务仍由独立回执幂等回读。""" + factory = _session_factory() + writer = TransactionalChainDurableEventWriter(factory) + shared_src = "/downloads/shared-generation.mkv" + _add_settling_pending( + factory, + task_id="old-task", + src_path=shared_src, + ) + old_settlement = _settlement(outcome="succeeded", task_id="old-task") + old_result = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="old-task", + succeeded=True, + src_path=shared_src, + ), + event_payload={}, + publish=None, + settlement=old_settlement, + ) + _add_settling_pending( + factory, + task_id="new-task", + lease_token="lease-2", + execution_fingerprint="execution-2", + src_path=shared_src, + ) + + new_result = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="new-task", + succeeded=True, + src_path=shared_src, + ), + event_payload={}, + publish=None, + settlement=_settlement( + outcome="succeeded", + task_id="new-task", + lease_token="lease-2", + execution_fingerprint="execution-2", + ), + ) + old_replay = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda _repository: pytest.fail("旧任务不得改写最新投影"), + event_payload={}, + publish=None, + settlement=old_settlement, + ) + + with factory() as session: + history = session.execute(select(TransferHistory)).scalar_one() + receipts = session.execute( + select(TransferSettlementReceipt).order_by(TransferSettlementReceipt.id) + ).scalars().all() + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert isinstance(old_result, TransferSettlementResult) + assert isinstance(new_result, TransferSettlementResult) + assert old_replay == TransferSettlementResult( + history_id=old_result.history_id, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + assert history.transfer_task_id is None + assert history.transfer_settlement_revision is None + assert [receipt.task_id for receipt in receipts] == ["old-task", "new-task"] + assert [receipt.history_id for receipt in receipts] == [history.id, history.id] + + +def test_task_settlement_without_public_topic_commits_no_outbox(): + """无公共事件的文件仍原子结算历史和 pending，且不登记或发布事件。""" + factory = _session_factory() + _add_settling_pending(factory, task_id="lyrics-task") + writer = TransactionalChainDurableEventWriter(factory) + + result = writer.transfer_result( + topic=None, + stage_history=lambda repository: _stage_result_history( + repository, + task_id="lyrics-task", + succeeded=True, + ), + event_payload={"unexpected": "must-not-publish"}, + publish=lambda _payload: pytest.fail("无公共 topic 不得发布"), + settlement=_settlement( + outcome="succeeded", + task_id="lyrics-task", + ), + ) + + assert result == TransferSettlementResult( + history_id=1, + settlement_revision=1, + pending_deleted=True, + ) + with factory() as session: + history = session.execute(select(TransferHistory)).scalar_one() + assert history.transfer_task_id is None + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None + + +def test_task_settlement_binds_receipt_without_overwriting_success_history(): + """不覆盖裁决只绑定任务回执，保留旧成功历史的全部业务字段。""" + factory = _session_factory() + _add_settling_pending(factory, task_id="declined-task") + with factory() as session: + session.add(TransferHistory( + src="/downloads/declined-task.mkv", + src_storage="local", + dest="/library/original.mkv", + title="Original", + status=True, + date="2026-08-26 20:00:00", + )) + session.commit() + writer = TransactionalChainDurableEventWriter(factory) + settlement = _settlement( + outcome="succeeded", + task_id="declined-task", + ) + + first = writer.transfer_result( + topic=None, + stage_history=lambda repository: repository.get_success_by_src( + "/downloads/declined-task.mkv", + "local", + ), + event_payload={}, + publish=None, + settlement=settlement, + ) + replay = writer.transfer_result( + topic=None, + stage_history=lambda _repository: pytest.fail("回执重放不得重新查写历史"), + event_payload={}, + publish=None, + settlement=settlement, + ) + + assert isinstance(first, TransferSettlementResult) + assert replay == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + with factory() as session: + history = session.execute(select(TransferHistory)).scalar_one() + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None + assert history.dest == "/library/original.mkv" + assert history.title == "Original" + assert history.status is True + assert history.date == "2026-08-26 20:00:00" + assert history.transfer_task_id is None + assert history.transfer_settlement_revision is None + + +@pytest.mark.parametrize("cleanup", ["delete", "truncate"]) +def test_receipt_replay_survives_real_history_command_cleanup(cleanup): + """真实历史删除或清空命令执行后，独立回执仍可重放成功终态。""" + factory = _session_factory() + _add_settling_pending(factory, task_id="cleanup-task") + writer = TransactionalChainDurableEventWriter(factory) + settlement = _settlement(outcome="succeeded", task_id="cleanup-task") + first = writer.transfer_result( + topic=None, + stage_history=lambda repository: _stage_result_history( + repository, + task_id="cleanup-task", + succeeded=True, + ), + event_payload={}, + publish=None, + settlement=settlement, + ) + with factory() as session: + command = TransferHistoryMutationCommand( + repository=TransferHistoryOper(session), + download_repository=Mock(), + unit_of_work=SqlAlchemyUnitOfWork(session), + file_item_factory=Mock(), + delete_media_file=Mock(return_value=True), + publish_download_file_deleted=Mock(), + clear_failures=Mock(), + ) + cleanup_result = ( + command.delete(first.history_id) + if cleanup == "delete" + else command.truncate() + ) + assert cleanup_result.success is True + + replay = writer.transfer_result( + topic=None, + stage_history=lambda _repository: pytest.fail("清理后重放不得重建历史"), + event_payload={}, + publish=None, + settlement=settlement, + ) + + with factory() as session: + assert session.execute(select(TransferHistory)).scalar_one_or_none() is None + receipts = session.execute( + select(TransferSettlementReceipt) + .order_by(TransferSettlementReceipt.settlement_revision) + ).scalars().all() + assert isinstance(first, TransferSettlementResult) + assert replay == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + assert len(receipts) == 1 + assert receipts[0].task_id == "cleanup-task" + assert receipts[0].history_id == first.history_id + + +def test_success_receipt_allows_expiry_and_legacy_same_source_replace(): + """成功回执不锁死业务历史，过期清理和旧兼容替换仍按原契约工作。""" + factory = _session_factory() + shared_src = "/downloads/cleanup-compatible.mkv" + _add_settling_pending( + factory, + task_id="compatible-task", + src_path=shared_src, + ) + writer = TransactionalChainDurableEventWriter(factory) + settlement = _settlement(outcome="succeeded", task_id="compatible-task") + first = writer.transfer_result( + topic=None, + stage_history=lambda repository: _stage_result_history( + repository, + task_id="compatible-task", + succeeded=True, + src_path=shared_src, + ), + event_payload={}, + publish=None, + settlement=settlement, + ) + + with factory() as session: + assert TransferHistory.delete_before( + session, + before_time="9999-12-31 23:59:59", + limit=100, + ) == 1 + session.commit() + legacy = TransferHistoryOper(session).stage_replace_by_src( + src=shared_src, + src_storage="local", + status=True, + ) + session.commit() + assert legacy.transfer_task_id is None + + replay = writer.transfer_result( + topic=None, + stage_history=lambda _repository: pytest.fail("旧兼容替换后不得重写历史"), + event_payload={}, + publish=None, + settlement=settlement, + ) + assert isinstance(first, TransferSettlementResult) + assert replay == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + + +def test_task_failure_settlement_is_replayable_and_retry_advances_revision(): + """失败保留终态证据，重复调用幂等，显式重试后才递增修订号。""" + factory = _session_factory() + _add_settling_pending(factory) + writer = TransactionalChainDurableEventWriter(factory) + calls = [] + first_settlement = _settlement(outcome="failed") + + first = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=False, + ), + event_payload={}, + publish=lambda _payload: calls.append("first"), + settlement=first_settlement, + ) + replay = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda _repository: pytest.fail("失败回读不得重写历史"), + event_payload={}, + publish=lambda _payload: pytest.fail("失败回读不得重复发布"), + settlement=first_settlement, + ) + assert isinstance(first, TransferSettlementResult) + assert replay.already_settled is True + assert replay.settlement_revision == 1 + + with factory() as session: + pending = session.execute(select(TransferPending)).scalar_one() + assert pending.execution_state == "failed" + assert pending.lease_token is None + assert pending.settlement_revision == 1 + assert pending.terminal_history_id == first.history_id + pending.execution_state = "settling" + pending.execution_fingerprint = "execution-2" + pending.lease_owner = "worker-2" + pending.lease_token = "lease-2" + pending.lease_expires_at = "2099-01-01 00:00:00.000000" + session.commit() + + retried = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=False, + ), + event_payload={}, + publish=lambda _payload: calls.append("retry"), + settlement=_settlement( + outcome="failed", + lease_token="lease-2", + execution_fingerprint="execution-2", + ), + ) + + assert retried == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=2, + pending_deleted=False, + ) + stale_replay = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda _repository: pytest.fail( + "旧修订延迟重放不得覆盖最新历史" + ), + event_payload={}, + publish=None, + settlement=first_settlement, + ) + assert stale_replay == TransferSettlementResult( + history_id=first.history_id, + settlement_revision=1, + pending_deleted=False, + already_settled=True, + ) + with factory() as session: + histories = session.execute(select(TransferHistory)).scalars().all() + outboxes = session.execute( + select(OutboxMessage).order_by(OutboxMessage.id) + ).scalars().all() + pending = session.execute(select(TransferPending)).scalar_one() + receipts = session.execute( + select(TransferSettlementReceipt) + .order_by(TransferSettlementReceipt.settlement_revision) + ).scalars().all() + assert len(histories) == 1 + assert histories[0].transfer_settlement_revision == 2 + assert pending.settlement_revision == 2 + assert [receipt.settlement_revision for receipt in receipts] == [1, 2] + assert all(receipt.task_id == "task-1" for receipt in receipts) + assert all(receipt.history_id == first.history_id for receipt in receipts) + assert receipts[0].execution_fingerprint == "execution-1" + assert receipts[0].lease_token == "lease-1" + assert receipts[1].outcome == "failed" + assert receipts[1].execution_fingerprint == "execution-2" + assert receipts[1].lease_token == "lease-2" + assert receipts[1].pending_deleted is False + assert receipts[1].error == "目标文件校验失败" + assert [item.event_key for item in outboxes] == [ + "transfer.result:task-1:1:failed:v1", + "transfer.result:task-1:2:failed:v1", + ] + assert calls == [] + + +def test_failed_revision_replays_after_later_success_deleted_pending(): + """后续重试成功删除 pending 后，旧失败修订仍按原执行身份幂等回读。""" + factory = _session_factory() + _add_settling_pending(factory) + writer = TransactionalChainDurableEventWriter(factory) + failed_settlement = _settlement(outcome="failed") + failed = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=False, + ), + event_payload={}, + publish=None, + settlement=failed_settlement, + ) + with factory() as session: + pending = session.execute(select(TransferPending)).scalar_one() + pending.execution_state = "settling" + pending.execution_fingerprint = "execution-2" + pending.lease_owner = "worker-2" + pending.lease_token = "lease-2" + pending.lease_expires_at = "2099-01-01 00:00:00.000000" + session.commit() + succeeded_settlement = _settlement( + outcome="succeeded", + lease_token="lease-2", + execution_fingerprint="execution-2", + ) + succeeded = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=True, + ), + event_payload={}, + publish=None, + settlement=succeeded_settlement, + ) + + stale_replay = writer.transfer_result( + topic="transfer.failed", + stage_history=lambda _repository: pytest.fail("旧失败回执不得重写历史"), + event_payload={}, + publish=None, + settlement=failed_settlement, + ) + success_replay = writer.transfer_result( + topic="transfer.completed", + stage_history=lambda _repository: pytest.fail("成功回执不得重写历史"), + event_payload={}, + publish=None, + settlement=succeeded_settlement, + ) + + assert isinstance(failed, TransferSettlementResult) + assert isinstance(succeeded, TransferSettlementResult) + assert stale_replay == TransferSettlementResult( + history_id=failed.history_id, + settlement_revision=1, + pending_deleted=False, + already_settled=True, + ) + assert success_replay == TransferSettlementResult( + history_id=succeeded.history_id, + settlement_revision=2, + pending_deleted=True, + already_settled=True, + ) + with factory() as session: + history = session.execute(select(TransferHistory)).scalar_one() + receipts = session.execute( + select(TransferSettlementReceipt) + .order_by(TransferSettlementReceipt.settlement_revision) + ).scalars().all() + assert session.execute(select(TransferPending)).scalar_one_or_none() is None + assert history.transfer_task_id is None + assert history.transfer_settlement_revision is None + assert [receipt.outcome for receipt in receipts] == ["failed", "succeeded"] + + +def test_task_settlement_outbox_conflict_rolls_back_history_and_pending(): + """intent 唯一键冲突时回滚此前已暂存的历史与 pending 终态。""" + factory = _session_factory() + _add_settling_pending(factory) + event_key = "transfer.result:task-1:1:succeeded:v1" + with factory() as session: + session.add(OutboxMessage( + event_key=event_key, + topic="transfer.completed", + payload_version=1, + payload={}, + status="completed", + attempt=0, + next_retry_at="2026-08-27T01:00:00+00:00", + created_at="2026-08-27T01:00:00+00:00", + )) + session.commit() + writer = TransactionalChainDurableEventWriter(factory) + + with pytest.raises(IntegrityError): + writer.transfer_result( + topic="transfer.completed", + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=True, + ), + event_payload={}, + publish=lambda _payload: pytest.fail("事务失败不得发布"), + settlement=_settlement(outcome="succeeded"), + ) + + with factory() as session: + pending = session.execute(select(TransferPending)).scalar_one() + histories = session.execute(select(TransferHistory)).scalars().all() + receipts = session.execute(select(TransferSettlementReceipt)).scalars().all() + outboxes = session.execute(select(OutboxMessage)).scalars().all() + assert pending.execution_state == "settling" + assert pending.settlement_revision == 0 + assert pending.lease_token == "lease-1" + assert histories == [] + assert receipts == [] + assert len(outboxes) == 1 + assert outboxes[0].event_key == event_key + + +def test_task_settlement_rejects_stale_lease_without_business_writes(): + """陈旧 lease 在历史回调前即被 fencing，不能留下历史或 intent。""" + factory = _session_factory() + _add_settling_pending(factory) + writer = TransactionalChainDurableEventWriter(factory) + + with pytest.raises(TransferExecutionLeaseLostError): + writer.transfer_result( + topic="transfer.completed", + stage_history=lambda _repository: pytest.fail("陈旧 lease 不得写历史"), + event_payload={}, + publish=lambda _payload: pytest.fail("陈旧 lease 不得发布"), + settlement=_settlement( + outcome="succeeded", + lease_token="stale-lease", + ), + ) + + with factory() as session: + pending = session.execute(select(TransferPending)).scalar_one() + assert session.execute(select(TransferHistory)).scalar_one_or_none() is None + assert session.execute( + select(TransferSettlementReceipt) + ).scalar_one_or_none() is None + assert session.execute(select(OutboxMessage)).scalar_one_or_none() is None + assert pending.execution_state == "settling" + assert pending.lease_token == "lease-1" + + +def test_concurrent_duplicate_settlement_returns_one_commit_and_one_replay( + tmp_path, + monkeypatch, +): + """并发重复结算只有一个事务写入，竞争输家回读同一不可变回执。""" + engine = create_engine( + f"sqlite+pysqlite:///{tmp_path / 'settlement-race.db'}", + connect_args={"check_same_thread": False, "timeout": 10}, + ) + Base.metadata.create_all(engine) + factory = sessionmaker(bind=engine) + _add_settling_pending(factory) + writer = TransactionalChainDurableEventWriter(factory) + settlement = _settlement(outcome="succeeded") + barrier = Barrier(2) + original_read = TransactionalChainDurableEventWriter._read_settlement_result + + def synchronized_read(**kwargs): + """让两个调用都先观察到未结算，再同时进入事务竞争。""" + result = original_read(**kwargs) + if result is None: + barrier.wait(timeout=10) + return result + + monkeypatch.setattr( + TransactionalChainDurableEventWriter, + "_read_settlement_result", + staticmethod(synchronized_read), + ) + + def settle_once(): + """用相同 fencing 身份提交同一成功终态。""" + return writer.transfer_result( + topic=None, + stage_history=lambda repository: _stage_result_history( + repository, + task_id="task-1", + succeeded=True, + ), + event_payload={}, + publish=None, + settlement=settlement, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(lambda _index: settle_once(), range(2))) + + assert sorted(result.already_settled for result in results) == [False, True] + assert {result.history_id for result in results} == {1} + with factory() as session: + assert len(session.execute(select(TransferHistory)).scalars().all()) == 1 + assert len( + session.execute(select(TransferSettlementReceipt)).scalars().all() + ) == 1 + assert session.execute(select(TransferPending)).scalar_one_or_none() is None diff --git a/tests/test_db_transferhistory_queries.py b/tests/test_db_transferhistory_queries.py index 5bd9bdb94..3ec6a867f 100644 --- a/tests/test_db_transferhistory_queries.py +++ b/tests/test_db_transferhistory_queries.py @@ -5,7 +5,6 @@ 来的）、统计。查重误判会重复整理或永久漏件——挂载故障那一类问题最终就落在这张表上； 溯源查错会让「重新整理」把不相干的文件搬走。 """ -import asyncio import time as _time import pytest @@ -515,6 +514,65 @@ def test_delete_before_is_batched_and_keeps_recent(db): assert TransferHistory.get_by_src(db.session, "/data/recent.mkv") is not None +def test_delete_before_preserves_current_failed_task_history(db): + """过期维护不得删除当前失败 pending 映射使用的历史。""" + durable = _hist( + "durable-old", + src="/data/durable-old.mkv", + date="2026-01-01 10:00:00", + status=False, + ) + durable.transfer_task_id = "task-durable-old" + durable.transfer_settlement_revision = 1 + db.add(durable) + + assert TransferHistory.delete_before( + db.session, + before_time="2026-08-01", + limit=100, + ) == 0 + assert TransferHistory.get_by_transfer_task_id( + db.session, + task_id="task-durable-old", + ) is not None + + +def test_upsert_durable_projection_advances_to_new_same_source_task(db): + """同源历史只表达最新任务投影，旧任务重放身份由独立回执持有。""" + durable = _hist( + "old-task", + src="/data/reused.mkv", + date="2026-08-01 10:00:00", + ) + durable.transfer_task_id = "old-task" + durable.transfer_settlement_revision = 1 + db.add(durable) + + projected = TransferHistory.upsert_by_transfer_task_id( + db.session, + task_id="new-task", + settlement_revision=1, + retain_task_mapping=False, + payload={ + "src": "/data/reused.mkv", + "src_storage": "local", + "status": True, + }, + ) + + assert TransferHistory.get_by_transfer_task_id( + db.session, + task_id="old-task", + ) is None + assert TransferHistory.get_by_transfer_task_id( + db.session, + task_id="new-task", + ) is None + assert projected is durable + assert projected.transfer_task_id is None + assert projected.transfer_settlement_revision is None + + def test_delete_before_keeps_the_row_exactly_at_the_boundary(db): """ 保留时间点上的整理历史属于「保留期内」，不能被清理（``date < before_time``）。 diff --git a/tests/test_db_transferpending_queries.py b/tests/test_db_transferpending_queries.py index 39a7367a6..942896378 100644 --- a/tests/test_db_transferpending_queries.py +++ b/tests/test_db_transferpending_queries.py @@ -13,6 +13,7 @@ from sqlalchemy.orm import sessionmaker from app.db import base as db_base from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending from app.db.oper.transferpending import TransferPendingOper @@ -143,6 +144,7 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch): def test_transactional_repository_commits_frozen_projections(tmp_path): """适配器应独立提交 UoW，并在会话关闭前冻结应用 DTO。""" engine = create_engine(f"sqlite:///{tmp_path / 'transfer.db'}") + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) factory = sessionmaker(bind=engine) repository = TransactionalTransferAdmissionRepository(factory) diff --git a/tests/test_history_ai_retry_gate.py b/tests/test_history_ai_retry_gate.py new file mode 100644 index 000000000..fdcac2bf3 --- /dev/null +++ b/tests/test_history_ai_retry_gate.py @@ -0,0 +1,437 @@ +"""验证 AI 历史入口不会绕过 durable 整理任务的唯一重试权。""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +from app.agent.tools.impl.delete_transfer_history import DeleteTransferHistoryTool +from app.api.endpoints import history as history_endpoint +from app.application.configuration import ApiRuntimeConfig +from app.application.transfer_execution import ( + TransferExecutionState, + TransferRetryRequestResult, +) +from app.runtime.progress import AsyncProgressHelper +from app.schemas.history import BatchTransferHistoryRedoRequest, TransferHistory + + +class _HistoryQuery: + """按测试输入返回脱离数据库的整理历史 DTO。""" + + def __init__(self, histories: list[TransferHistory]) -> None: + """保存按 ID 可查的测试历史。""" + self._histories = {history.id: history for history in histories} + + async def get_transfer(self, history_id: int) -> TransferHistory | None: + """返回单条测试历史。""" + return self._histories.get(history_id) + + async def get_transfers( + self, + history_ids: list[int], + ) -> tuple[list[TransferHistory], list[int]]: + """按输入顺序返回存在和缺失的测试历史。""" + records = [self._histories[item] for item in history_ids if item in self._histories] + missing = [item for item in history_ids if item not in self._histories] + return records, missing + + +class _RetryCommand: + """记录 execution 重试请求并返回逐任务测试结果。""" + + calls: list[tuple[object, dict]] = [] + results: dict[str, TransferRetryRequestResult] = {} + + def __init__(self, repository: object) -> None: + """保存调用方取得的 execution 仓储。""" + self._repository = repository + + def request_retry(self, **kwargs) -> TransferRetryRequestResult: + """记录调用并返回任务对应结果。""" + self.calls.append((self._repository, kwargs)) + return self.results[kwargs["task_id"]] + + +async def _record_async(target: list[dict], payload: dict) -> None: + """记录被 await 的异步边界调用。""" + target.append(payload) + + +def _runtime(*, ai_enabled: bool = True) -> ApiRuntimeConfig: + """构造历史端点需要的最小稳定配置快照。""" + return ApiRuntimeConfig( + advanced_mode=False, + access_token_expire_minutes=30, + btrfs_fsid_dedup=False, + ai_agent_enable=ai_enabled, + ) + + +def _retry_result( + *, + accepted: bool, + state: TransferExecutionState, + message: str, +) -> TransferRetryRequestResult: + """构造 execution 重试登记结果。""" + return TransferRetryRequestResult( + accepted=accepted, + state=state, + retry_generation=3, + message=message, + ) + + +def _install_retry_command(monkeypatch, results: dict[str, TransferRetryRequestResult]) -> object: + """安装不会接触真实数据库的 execution 端口和命令替身。""" + repository = object() + _RetryCommand.calls = [] + _RetryCommand.results = results + monkeypatch.setattr( + history_endpoint, + "get_chain_transfer_execution_port", + lambda: repository, + ) + monkeypatch.setattr(history_endpoint, "TransferExecutionCommand", _RetryCommand) + return repository + + +def test_transfer_history_task_id_is_internal_projection_only() -> None: + """durable 任务标识可供宿主读取，但不得扩展公开历史响应。""" + history = TransferHistory(id=7, transfer_task_id="task-7") + + assert history.transfer_task_id == "task-7" + assert "transfer_task_id" not in history.model_dump() + + +def test_durable_retry_progress_is_immediately_completed_for_existing_sse() -> None: + """durable 重试完成进度应让既有 SSE 客户端首次读取即可收口。""" + + async def scenario() -> dict: + """写入并回读同一个测试进度键。""" + progress_key = "test_history_durable_retry_completed" + await history_endpoint._complete_durable_retry_progress( + progress_key=progress_key, + text="整理任务已登记重试", + history_ids=[8], + ) + detail = await AsyncProgressHelper(progress_key).get() + assert detail is not None + return detail + + detail = asyncio.run(scenario()) + + assert detail["enable"] is False + assert detail["value"] == 100 + assert detail["data"]["history_ids"] == [8] + assert detail["data"]["success"] is True + assert detail["data"]["completed"] is True + assert detail["data"]["message"] == "整理任务已登记重试" + + +def test_single_ai_redo_requests_durable_retry_without_agent(monkeypatch) -> None: + """单条 durable AI 重做只登记调度重试，即使 Agent 功能未启用。""" + repository = _install_retry_command( + monkeypatch, + { + "task-11": _retry_result( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + message="整理任务已登记重试", + ) + }, + ) + monkeypatch.setattr( + history_endpoint, + "_start_ai_redo_task", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("durable 重试不得启动 Agent") + ), + ) + completed_progress: list[dict] = [] + monkeypatch.setattr( + history_endpoint, + "_complete_durable_retry_progress", + lambda **kwargs: _record_async(completed_progress, kwargs), + ) + + response = asyncio.run( + history_endpoint.ai_redo_transfer_history( + 11, + query=_HistoryQuery([TransferHistory(id=11, transfer_task_id="task-11")]), + runtime_config=_runtime(ai_enabled=False), + task_registry=object(), + _=object(), + ) + ) + + assert response.success is True + assert response.data is not None + assert response.data["progress_key"].startswith("transfer_retry_11_") + assert response.message == "整理任务已登记重试" + assert completed_progress[0]["history_ids"] == [11] + assert _RetryCommand.calls == [ + ( + repository, + { + "task_id": "task-11", + "reason": "AI REST 请求重试整理历史 #11", + "requested_by": "history_ai_redo", + }, + ) + ] + + +def test_single_ai_redo_reports_manual_review_rejection(monkeypatch) -> None: + """人工复核状态必须原样拒绝，且不得回退到破坏性 Agent 流程。""" + _install_retry_command( + monkeypatch, + { + "task-12": _retry_result( + accepted=False, + state=TransferExecutionState.MANUAL_REVIEW, + message="人工复核任务必须先完成专门判定", + ) + }, + ) + monkeypatch.setattr( + history_endpoint, + "build_manual_redo_prompt", + lambda _history: (_ for _ in ()).throw( + AssertionError("拒绝后不得生成 Agent 提示词") + ), + ) + + response = asyncio.run( + history_endpoint.ai_redo_transfer_history( + 12, + query=_HistoryQuery([TransferHistory(id=12, transfer_task_id="task-12")]), + runtime_config=_runtime(), + task_registry=object(), + _=object(), + ) + ) + + assert response.success is False + assert response.message == "人工复核任务必须先完成专门判定" + + +def test_batch_ai_redo_returns_completed_progress_for_durable_tasks(monkeypatch) -> None: + """全 durable 批量接受后保留前端既有 progress_key 协议。""" + _install_retry_command( + monkeypatch, + { + "task-18": _retry_result( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + message="整理任务已登记重试", + ), + "task-19": _retry_result( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + message="整理任务已在等待重试", + ), + }, + ) + completed_progress: list[dict] = [] + monkeypatch.setattr( + history_endpoint, + "_complete_durable_retry_progress", + lambda **kwargs: _record_async(completed_progress, kwargs), + ) + monkeypatch.setattr( + history_endpoint, + "_start_batch_ai_redo_task", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("全 durable 批量不得启动 Agent") + ), + ) + + response = asyncio.run( + history_endpoint.batch_ai_redo_transfer_history( + BatchTransferHistoryRedoRequest(history_ids=[18, 19]), + query=_HistoryQuery([ + TransferHistory(id=18, transfer_task_id="task-18"), + TransferHistory(id=19, transfer_task_id="task-19"), + ]), + runtime_config=_runtime(ai_enabled=False), + task_registry=object(), + _=object(), + ) + ) + + assert response.success is True + assert response.data is not None + assert response.data["progress_key"].startswith("transfer_retry_batch_") + assert response.data["history_ids"] == [18, 19] + assert completed_progress[0]["history_ids"] == [18, 19] + + +def test_batch_ai_redo_reports_each_rejection_without_starting_legacy_agent( + monkeypatch, +) -> None: + """混合批量逐 task 返回拒绝，避免同时产生无人监听的旧 Agent 任务。""" + _install_retry_command( + monkeypatch, + { + "task-21": _retry_result( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + message="整理任务已登记重试", + ), + "task-22": _retry_result( + accepted=False, + state=TransferExecutionState.RUNNING, + message="整理任务当前状态不接受用户重试", + ), + }, + ) + started: list[dict] = [] + prompted: list[list[int]] = [] + monkeypatch.setattr( + history_endpoint, + "build_batch_manual_redo_prompt", + lambda histories: prompted.append([history.id for history in histories]) or "legacy prompt", + ) + monkeypatch.setattr( + history_endpoint, + "_start_batch_ai_redo_task", + lambda **kwargs: started.append(kwargs), + ) + histories = [ + TransferHistory(id=21, transfer_task_id="task-21"), + TransferHistory(id=22, transfer_task_id="task-22"), + TransferHistory(id=23), + ] + + response = asyncio.run( + history_endpoint.batch_ai_redo_transfer_history( + BatchTransferHistoryRedoRequest(history_ids=[21, 22, 23]), + query=_HistoryQuery(histories), + runtime_config=_runtime(), + task_registry=object(), + _=object(), + ) + ) + + assert response.success is False + assert "已登记 1 个持久整理任务重试" in response.message + assert "#22 [running]: 整理任务当前状态不接受用户重试" in response.message + assert response.data is None + assert "1 条旧历史未提交" in response.message + assert prompted == [] + assert [call[1]["task_id"] for call in _RetryCommand.calls] == [ + "task-21", + "task-22", + ] + assert started == [] + + +def test_batch_ai_redo_sends_only_legacy_records_after_durable_acceptance( + monkeypatch, +) -> None: + """混合批量全接受时 durable 只登记重试，旧历史才进入 Agent。""" + _install_retry_command( + monkeypatch, + { + "task-24": _retry_result( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + message="整理任务已登记重试", + ) + }, + ) + prompted: list[list[int]] = [] + started: list[dict] = [] + monkeypatch.setattr( + history_endpoint, + "build_batch_manual_redo_prompt", + lambda histories: prompted.append([history.id for history in histories]) or "legacy prompt", + ) + monkeypatch.setattr( + history_endpoint, + "_start_batch_ai_redo_task", + lambda **kwargs: started.append(kwargs), + ) + + response = asyncio.run( + history_endpoint.batch_ai_redo_transfer_history( + BatchTransferHistoryRedoRequest(history_ids=[24, 25]), + query=_HistoryQuery([ + TransferHistory(id=24, transfer_task_id="task-24"), + TransferHistory(id=25), + ]), + runtime_config=_runtime(), + task_registry=object(), + _=object(), + ) + ) + + assert response.success is True + assert response.data is not None + assert response.data["history_ids"] == [24, 25] + assert prompted == [[25]] + assert started[0]["history_ids"] == [25] + + +def test_agent_delete_tool_requests_retry_before_any_destructive_action(monkeypatch) -> None: + """Agent 工具命中 durable 历史时保留目标、历史和失败证据。""" + history = SimpleNamespace( + id=31, + transfer_task_id="task-31", + dest_fileitem={"path": "/library/demo.mkv"}, + ) + delete_calls: list[int] = [] + + class _HistoryPort: + """提供 durable 历史并观察是否发生删除。""" + + async def async_get(self, history_id: int) -> object: + """返回 durable 历史。""" + assert history_id == 31 + return history + + async def async_delete(self, history_id: int) -> None: + """记录不应发生的历史删除。""" + delete_calls.append(history_id) + + retry_calls: list[dict] = [] + monkeypatch.setattr( + "app.agent.tools.impl.delete_transfer_history.get_agent_transfer_history_port", + _HistoryPort, + ) + monkeypatch.setattr( + "app.agent.tools.impl.delete_transfer_history._request_transfer_retry", + lambda **kwargs: retry_calls.append(kwargs) + or _retry_result( + accepted=False, + state=TransferExecutionState.MANUAL_REVIEW, + message="人工复核任务必须先完成专门判定", + ), + ) + monkeypatch.setattr( + "app.agent.tools.impl.delete_transfer_history.StorageChain", + lambda: (_ for _ in ()).throw( + AssertionError("durable 目标文件不得删除") + ), + ) + + result = asyncio.run( + DeleteTransferHistoryTool( + session_id="redo-session", + user_id="10001", + ).run(history_id=31) + ) + + assert "未登记重试" in result + assert "state=manual_review" in result + assert "不要调用 transfer_file" in result + assert retry_calls == [ + { + "history_id": 31, + "task_id": "task-31", + "user_id": "10001", + } + ] + assert delete_calls == [] diff --git a/tests/test_history_batch_ai_redo_prompt.py b/tests/test_history_batch_ai_redo_prompt.py index 7a5e4f80a..ccb15d409 100644 --- a/tests/test_history_batch_ai_redo_prompt.py +++ b/tests/test_history_batch_ai_redo_prompt.py @@ -45,3 +45,22 @@ def test_batch_manual_redo_job_definition_contains_plain_text_rules(): assert any("plain text only" in rule for rule in task_rules) assert any("Markdown formatting" in rule for rule in task_rules) + + +def test_manual_redo_tasks_stop_after_durable_retry_result(): + """系统任务必须禁止 durable 历史在登记重试后继续直接整理。""" + definition = prompt_manager.load_system_tasks_definition() + + for task_name in ( + "transfer_failed_retry", + "batch_transfer_failed_retry", + "manual_transfer_redo", + "batch_manual_transfer_redo", + ): + task = definition.task_types[task_name] + instructions = [*task.steps, *task.task_rules] + assert any("persistent retry scheduler" in item for item in instructions) + assert any( + "do not call `transfer_file`" in item.lower() + for item in instructions + ) diff --git a/tests/test_history_mutation_command.py b/tests/test_history_mutation_command.py index 59e66fc64..7fc0b511e 100644 --- a/tests/test_history_mutation_command.py +++ b/tests/test_history_mutation_command.py @@ -37,6 +37,7 @@ def _history(): download_hash="abc", src_fileitem={"path": "/downloads/demo.mkv"}, dest_fileitem={"path": "/media/demo.mkv"}, + transfer_task_id=None, ) @@ -119,5 +120,21 @@ def test_transfer_truncate_uses_single_transaction(): result = command.truncate() assert result.success is True + assert result.message == "已清空旧整理记录，失败任务记录已保留" dependencies["repository"].stage_truncate.assert_called_once_with() dependencies["unit_of_work"].commit.assert_called_once_with() + + +def test_transfer_delete_rejects_durable_receipt_before_file_side_effects(): + """durable 回执不能被历史 API 连同源或目标文件一起删除。""" + history = _history() + history.transfer_task_id = "task-durable" + command, dependencies = _transfer_command(history=history) + + result = command.delete(7, delete_source=True, delete_destination=True) + + assert result.success is False + assert result.message == "持久整理失败记录不可删除，请使用重试或人工复核入口" + dependencies["delete_media_file"].assert_not_called() + dependencies["repository"].stage_delete.assert_not_called() + dependencies["unit_of_work"].commit.assert_not_called() diff --git a/tests/test_legacy_db_behavior_compat.py b/tests/test_legacy_db_behavior_compat.py index d451f3348..938c930e7 100644 --- a/tests/test_legacy_db_behavior_compat.py +++ b/tests/test_legacy_db_behavior_compat.py @@ -3,6 +3,7 @@ import importlib import pytest from app.application.transfer import TransferTask as CanonicalTransferTask +from app.db.models.transferhistory import TransferHistory from app.schemas.file import FileItem @@ -136,6 +137,50 @@ def test_legacy_transfer_history_writes_delegate_to_application_service( assert captured == {**arguments, "transfer_history_oper": oper} +def test_legacy_transfer_history_mutations_preserve_durable_receipts(db): + """旧插件 delete/truncate/add_force 不能删除或覆盖 durable 终态回执。""" + legacy = importlib.import_module("app.db.transferhistory_oper") + durable = TransferHistory( + src="/downloads/durable.mkv", + src_storage="local", + status=True, + transfer_task_id="task-durable", + transfer_settlement_revision=1, + ) + legacy_row = TransferHistory( + src="/downloads/legacy.mkv", + src_storage="local", + status=True, + ) + db.add(durable, legacy_row) + oper = legacy.TransferHistoryOper(db.session) + + oper.delete(durable.id) + oper.truncate() + + assert TransferHistory.get_by_transfer_task_id( + db.session, + task_id="task-durable", + ) is not None + assert TransferHistory.get_by_src( + db.session, + "/downloads/legacy.mkv", + "local", + ) is None + with pytest.raises(ValueError, match="持久整理回执"): + oper.add_force( + src="/downloads/durable.mkv", + src_storage="local", + status=False, + ) + receipt = TransferHistory.get_by_transfer_task_id( + db.session, + task_id="task-durable", + ) + assert receipt is not None + assert receipt.status is True + + class LegacyPydanticValue: """模拟旧插件放进 TransferTask 的 Pydantic 风格对象。""" diff --git a/tests/test_transfer_durable_retry_owner.py b/tests/test_transfer_durable_retry_owner.py new file mode 100644 index 000000000..e5a33000d --- /dev/null +++ b/tests/test_transfer_durable_retry_owner.py @@ -0,0 +1,213 @@ +"""验证历史重试入口只把 durable 任务交给持久恢复调度器。""" + +from types import SimpleNamespace + +from app.application.transfer_execution import ( + TransferExecutionState, + TransferRetryRequestResult, +) +from app.chain.transfer import TransferChain +from app.schemas.types import NotificationChannel + + +class _RetryCommand: + """记录历史入口提交的类型化重试请求。""" + + calls: list[tuple[object, dict]] = [] + result = TransferRetryRequestResult( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + retry_generation=2, + message="整理任务已登记重试", + ) + + def __init__(self, repository: object) -> None: + """保存测试仓储实例。""" + self._repository = repository + + def request_retry(self, **kwargs) -> TransferRetryRequestResult: + """记录请求并返回用例指定结果。""" + self.calls.append((self._repository, kwargs)) + return self.result + + +def _install_retry_port(monkeypatch) -> object: + """安装不会接触数据库的 execution 端口与命令替身。""" + repository = object() + _RetryCommand.calls = [] + _RetryCommand.result = TransferRetryRequestResult( + accepted=True, + state=TransferExecutionState.RETRY_WAIT, + retry_generation=2, + message="整理任务已登记重试", + ) + monkeypatch.setattr( + "app.chain._transfer.get_chain_transfer_execution_port", + lambda: repository, + ) + monkeypatch.setattr( + "app.chain._transfer.TransferExecutionCommand", + _RetryCommand, + ) + return repository + + +def test_durable_history_redo_only_requests_persistent_retry(monkeypatch): + """durable 重做不得检查源文件、重新识别或重新准入执行。""" + repository = _install_retry_port(monkeypatch) + history = SimpleNamespace( + id=81, + transfer_task_id="transfer-task-81", + src="/missing/source.mkv", + ) + monkeypatch.setattr( + "app.chain._transfer.get_chain_transfer_history_port", + lambda: SimpleNamespace(get=lambda history_id: history), + ) + monkeypatch.setattr( + "app.chain._transfer.Path.exists", + lambda _path: (_ for _ in ()).throw( + AssertionError("durable 重试不应检查源路径") + ), + ) + chain = object.__new__(TransferChain) + monkeypatch.setattr( + chain, + "do_transfer", + lambda **_kwargs: (_ for _ in ()).throw( + AssertionError("durable 重试不应重新准入") + ), + ) + + state, message = chain._re_transfer(logid=81) + + assert state is True + assert message == "整理任务已登记重试" + assert _RetryCommand.calls == [ + ( + repository, + { + "task_id": "transfer-task-81", + "reason": "用户请求重试整理历史 #81", + "requested_by": "history_redo", + }, + ) + ] + + +def test_durable_manual_cleanup_keeps_target_history_and_failure_budget(monkeypatch): + """手动重整命中 durable 历史时不得先删目标、历史或失败计数。""" + _install_retry_port(monkeypatch) + history = SimpleNamespace( + id=82, + transfer_task_id="transfer-task-82", + status=False, + mode="copy", + src="/downloads/source.mkv", + src_storage="local", + dest_fileitem={ + "storage": "local", + "path": "/library/source.mkv", + "type": "file", + }, + ) + history_port = SimpleNamespace( + delete=lambda _history_id: (_ for _ in ()).throw( + AssertionError("durable 历史不得删除") + ) + ) + monkeypatch.setattr( + "app.chain._transfer.StorageChain", + lambda: (_ for _ in ()).throw( + AssertionError("durable 目标不得删除") + ), + ) + monkeypatch.setattr( + "app.chain._transfer.clear_transfer_failures", + lambda *_args: (_ for _ in ()).throw( + AssertionError("durable 失败计数不得清零") + ), + ) + chain = object.__new__(TransferChain) + + state, message = chain._delete_manual_transfer_history( + history=history, + transfer_history_oper=history_port, + ) + + assert state is False + assert message == "整理任务已登记重试" + assert _RetryCommand.calls[0][1]["requested_by"] == "manual_reorganize" + + +def test_durable_ai_button_bypasses_agent_and_requests_scheduler(monkeypatch): + """AI 按钮命中 durable 历史时也只能登记调度重试。""" + _install_retry_port(monkeypatch) + history = SimpleNamespace(id=83, transfer_task_id="transfer-task-83") + monkeypatch.setattr( + "app.chain._transfer.get_chain_transfer_history_port", + lambda: SimpleNamespace(get=lambda history_id: history), + ) + monkeypatch.setattr( + "app.chain._transfer.build_manual_redo_prompt", + lambda _history: (_ for _ in ()).throw( + AssertionError("durable 重试不得生成 Agent 破坏性提示词") + ), + ) + monkeypatch.setattr( + "app.chain._transfer.get_task_registry", + lambda: (_ for _ in ()).throw( + AssertionError("durable 重试不得提交 Agent 任务") + ), + ) + messages = [] + chain = object.__new__(TransferChain) + chain.runtime_config = SimpleNamespace( + ai_agent_enable=False, + history_url="/history", + ) + chain.post_message = messages.append + + chain._take_over_transfer_history_by_ai( + history_id=83, + channel=NotificationChannel.Telegram, + source="telegram-test", + userid="10001", + username="tester", + ) + + assert len(messages) == 1 + assert messages[0].title == "整理任务已登记重试" + assert _RetryCommand.calls[0][1]["requested_by"] == "ai_retry_button" + + +def test_durable_manual_review_rejection_does_not_fall_back_to_legacy(monkeypatch): + """人工复核任务被拒绝后不得回退到旧识别和重整流程。""" + _install_retry_port(monkeypatch) + _RetryCommand.result = TransferRetryRequestResult( + accepted=False, + state=TransferExecutionState.MANUAL_REVIEW, + retry_generation=1, + message="人工复核任务必须先完成专门判定", + ) + history = SimpleNamespace( + id=84, + transfer_task_id="transfer-task-84", + src="/downloads/source.mkv", + ) + monkeypatch.setattr( + "app.chain._transfer.get_chain_transfer_history_port", + lambda: SimpleNamespace(get=lambda history_id: history), + ) + monkeypatch.setattr( + "app.chain._transfer.Path.exists", + lambda _path: (_ for _ in ()).throw( + AssertionError("拒绝后不得回退旧流程") + ), + ) + chain = object.__new__(TransferChain) + + state, message = chain._re_transfer(logid=84) + + assert state is False + assert message == "人工复核任务必须先完成专门判定" diff --git a/tests/test_transfer_execution_migration.py b/tests/test_transfer_execution_migration.py new file mode 100644 index 000000000..a45d5466e --- /dev/null +++ b/tests/test_transfer_execution_migration.py @@ -0,0 +1,769 @@ +"""整理执行证据 3.0.16 Alembic 迁移的保守升级与可逆性测试。""" + +import importlib +from datetime import datetime, timezone +from io import StringIO +from types import SimpleNamespace + +import pytest +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations +from sqlalchemy.orm import sessionmaker + +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferExecutionConflictError, + TransferExecutionState, + TransferManualReviewDecision, + TransferManualReviewQuery, + TransferStepIntent, + TransferStepResult, +) +from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, +) +from app.db.models.transferpending import TransferPending + +MIGRATION = "database.versions.e5c7a9b1d3f6_3_0_16" +LEGACY_DIAGNOSTIC = "升级检测到既有执行迹象，需人工确认后再处理" + + +def _bind_migration(monkeypatch, connection): + """把 3.0.16 迁移绑定到隔离 SQLite 连接。""" + migration = importlib.import_module(MIGRATION) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + return migration + + +def _create_legacy_tables(connection) -> tuple[sa.Table, sa.Table]: + """创建具备 3.0.15 租约字段的最小 pending/history 表。""" + metadata = sa.MetaData() + pending = sa.Table( + "transferpending", + metadata, + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("storage", sa.String(), nullable=False), + sa.Column("src_path", sa.String(), nullable=False), + sa.Column("created_at", sa.String()), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("updated_at", sa.String(40)), + sa.Column("last_error", sa.Text()), + sa.Column("input_version", sa.Integer(), nullable=False), + sa.Column("planning_input", sa.JSON(), nullable=False), + sa.Column("input_fingerprint", sa.String(64), nullable=False), + sa.Column("checkpoint_version", sa.Integer()), + sa.Column("checkpoint_payload", sa.JSON()), + sa.Column("planned_at", sa.String(40)), + sa.Column("lease_owner", sa.String(128)), + sa.Column("lease_token", sa.String(64)), + sa.Column("lease_expires_at", sa.String(40)), + sa.Column("heartbeat_at", sa.String(40)), + sa.Column("attempt_count", sa.Integer(), nullable=False), + sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"), + ) + history = sa.Table( + "transferhistory", + metadata, + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("src", sa.String()), + sa.Column("src_storage", sa.String(), nullable=False), + ) + metadata.create_all(connection) + return pending, history + + +def _insert_legacy_rows(connection, pending: sa.Table) -> None: + """写入安全未开始和多种执行结果未知的旧任务。""" + base = { + "storage": "local", + "created_at": "2026-08-27 10:00:00", + "updated_at": "2026-08-27 10:00:00", + "last_error": None, + "input_version": 1, + "planning_input": {"schema_version": 1}, + "input_fingerprint": "0" * 64, + "checkpoint_version": None, + "checkpoint_payload": None, + "planned_at": None, + "lease_owner": None, + "lease_token": None, + "lease_expires_at": None, + "heartbeat_at": None, + "attempt_count": 0, + } + rows = [ + {**base, "id": 1, "task_id": "safe", "src_path": "/safe", "state": "accepted"}, + { + **base, + "id": 2, + "task_id": "planned", + "src_path": "/planned", + "state": "planned", + "checkpoint_version": 1, + "checkpoint_payload": {"schema_version": 1}, + }, + { + **base, + "id": 3, + "task_id": "attempted", + "src_path": "/attempted", + "state": "accepted", + "attempt_count": 1, + }, + { + **base, + "id": 4, + "task_id": "provider", + "src_path": "/provider", + "state": "provider_pending", + "checkpoint_version": 1, + "checkpoint_payload": {"provider": True}, + }, + ] + connection.execute(pending.insert(), rows) + + +def _assert_execution_tables_match_models(connection) -> None: + """断言步骤与 append-only 回执表的字段、约束和索引精确匹配 ORM。""" + inspector = sa.inspect(connection) + step_columns = { + column["name"]: column["nullable"] + for column in inspector.get_columns("transferexecutionstep") + } + assert step_columns == { + "id": False, + "task_id": False, + "operation_id": False, + "checkpoint_fingerprint": False, + "ordinal": False, + "phase": False, + "kind": False, + "state": False, + "attempt_token": True, + "attempt_count": False, + "intent_version": False, + "intent_payload": False, + "result_version": True, + "result_payload": True, + "last_error": True, + "prepared_at": False, + "started_at": True, + "completed_at": True, + "updated_at": False, + } + assert { + column["name"]: str(column["type"]) + for column in inspector.get_columns("transferexecutionstep") + } == { + "id": "INTEGER", + "task_id": "VARCHAR(64)", + "operation_id": "VARCHAR(64)", + "checkpoint_fingerprint": "VARCHAR(64)", + "ordinal": "INTEGER", + "phase": "VARCHAR(32)", + "kind": "VARCHAR(32)", + "state": "VARCHAR(32)", + "attempt_token": "VARCHAR(64)", + "attempt_count": "INTEGER", + "intent_version": "INTEGER", + "intent_payload": "JSON", + "result_version": "INTEGER", + "result_payload": "JSON", + "last_error": "TEXT", + "prepared_at": "VARCHAR(40)", + "started_at": "VARCHAR(40)", + "completed_at": "VARCHAR(40)", + "updated_at": "VARCHAR(40)", + } + assert inspector.get_pk_constraint("transferexecutionstep")[ + "constrained_columns" + ] == ["id"] + assert [ + ( + item["constrained_columns"], + item["referred_table"], + item["referred_columns"], + item["options"].get("ondelete"), + ) + for item in inspector.get_foreign_keys("transferexecutionstep") + ] == [(["task_id"], "transferpending", ["task_id"], "CASCADE")] + step_uniques = { + item["name"]: item["column_names"] + for item in inspector.get_unique_constraints("transferexecutionstep") + } + assert step_uniques == { + "uq_transferexecutionstep_operation_id": ["operation_id"], + "uq_transferexecutionstep_task_ordinal": ["task_id", "ordinal"], + } + step_indexes = { + item["name"]: item["column_names"] + for item in inspector.get_indexes("transferexecutionstep") + } + assert step_indexes == { + "ix_transferexecutionstep_task_state_ordinal": [ + "task_id", + "state", + "ordinal", + ], + } + + receipt_columns = { + column["name"]: column["nullable"] + for column in inspector.get_columns("transfersettlementreceipt") + } + assert receipt_columns == { + "id": False, + "task_id": False, + "history_id": False, + "settlement_revision": False, + "outcome": False, + "execution_fingerprint": False, + "lease_token": False, + "history_status": False, + "src": True, + "src_storage": True, + "pending_deleted": False, + "error": True, + "created_at": False, + "updated_at": False, + } + assert { + column["name"]: str(column["type"]) + for column in inspector.get_columns("transfersettlementreceipt") + } == { + "id": "INTEGER", + "task_id": "VARCHAR(64)", + "history_id": "INTEGER", + "settlement_revision": "INTEGER", + "outcome": "VARCHAR(16)", + "execution_fingerprint": "VARCHAR(64)", + "lease_token": "VARCHAR(64)", + "history_status": "BOOLEAN", + "src": "VARCHAR", + "src_storage": "VARCHAR", + "pending_deleted": "BOOLEAN", + "error": "TEXT", + "created_at": "VARCHAR(40)", + "updated_at": "VARCHAR(40)", + } + assert inspector.get_pk_constraint("transfersettlementreceipt")[ + "constrained_columns" + ] == ["id"] + assert inspector.get_foreign_keys("transfersettlementreceipt") == [] + receipt_uniques = { + item["name"]: item["column_names"] + for item in inspector.get_unique_constraints("transfersettlementreceipt") + } + assert receipt_uniques == { + "uq_transfersettlementreceipt_task_revision": [ + "task_id", + "settlement_revision", + ], + } + receipt_indexes = { + item["name"]: item["column_names"] + for item in inspector.get_indexes("transfersettlementreceipt") + } + assert receipt_indexes == { + "ix_transfersettlementreceipt_history_id": ["history_id"], + "ix_transfersettlementreceipt_task_revision": [ + "task_id", + "settlement_revision", + ], + } + + +def test_upgrade_is_conservative_and_repairs_interrupted_indexes(monkeypatch): + """旧执行迹象必须隔离，重复升级应补齐索引且不覆盖保守状态。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + connection.execute(sa.text( + "DROP INDEX ix_transferexecutionstep_task_state_ordinal" + )) + connection.execute(sa.text( + "DROP INDEX ix_transfersettlementreceipt_history_id" + )) + connection.execute(sa.text( + "DROP INDEX ix_transfersettlementreceipt_task_revision" + )) + migration.upgrade() + states = dict(connection.execute(sa.text( + "SELECT task_id, execution_state FROM transferpending ORDER BY id" + )).all()) + assert states == { + "safe": "not_started", + "planned": "manual_review", + "attempted": "manual_review", + "provider": "manual_review", + } + inspector = sa.inspect(connection) + assert { + "transferexecutionstep", + "transfersettlementreceipt", + }.issubset(inspector.get_table_names()) + execution_due_index = next( + index + for index in inspector.get_indexes("transferpending") + if index["name"] == "ix_transferpending_execution_due" + ) + assert execution_due_index["column_names"] == [ + "execution_state", + "retry_due_at", + "state", + "created_at", + "id", + ] + assert "ix_transferexecutionstep_task_state_ordinal" in { + index["name"] for index in inspector.get_indexes("transferexecutionstep") + } + assert "ux_transferhistory_transfer_task_id" in { + index["name"] for index in inspector.get_indexes("transferhistory") + } + _assert_execution_tables_match_models(connection) + engine.dispose() + + +def test_downgrade_marks_step_evidence_then_reupgrade_keeps_manual_review( + monkeypatch, +) -> None: + """降级不得丢失执行不确定性，再升级也不能把该任务自动重放。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + connection.execute(sa.text( + "INSERT INTO transferexecutionstep (" + "task_id, operation_id, checkpoint_fingerprint, ordinal, phase, kind, " + "state, attempt_token, attempt_count, intent_version, intent_payload, " + "prepared_at, updated_at" + ") VALUES (" + "'safe', 'operation', 'plan', 0, 'transfer', 'copy', " + "'prepared', NULL, 0, 1, '{}', " + "'2026-08-27 10:00:00', '2026-08-27 10:00:00'" + ")" + )) + connection.execute(sa.text( + "UPDATE transferpending SET last_error = '原始失败细节' " + "WHERE task_id = 'safe'" + )) + migration.downgrade() + assert connection.execute(sa.text( + "SELECT state FROM transferpending WHERE task_id = 'safe'" + )).scalar_one() == "accepted" + assert connection.execute(sa.text( + "SELECT last_error FROM transferpending WHERE task_id = 'safe'" + )).scalar_one() == f"原始失败细节\n{LEGACY_DIAGNOSTIC}" + inspector = sa.inspect(connection) + assert "transferexecutionstep" not in inspector.get_table_names() + assert "transfersettlementreceipt" not in inspector.get_table_names() + assert "execution_state" not in { + column["name"] for column in inspector.get_columns("transferpending") + } + migration.upgrade() + assert connection.execute(sa.text( + "SELECT execution_state FROM transferpending WHERE task_id = 'safe'" + )).scalar_one() == "manual_review" + engine.dispose() + + +def test_upgrade_without_pending_table_is_a_safe_noop(monkeypatch): + """全新数据库尚未执行前置迁移时本版本应安全等待迁移链建表。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + assert sa.inspect(connection).get_table_names() == [] + engine.dispose() + + +def test_execution_table_ddl_compiles_for_postgresql(monkeypatch) -> None: + """步骤与回执建表 DDL 必须在生产 PostgreSQL 方言下可编译。""" + output = StringIO() + context = MigrationContext.configure( + dialect_name="postgresql", + opts={"as_sql": True, "output_buffer": output}, + ) + migration = importlib.import_module(MIGRATION) + monkeypatch.setattr(migration, "op", Operations(context)) + migration._create_step_table() + migration._create_receipt_table() + ddl = output.getvalue() + assert "CREATE TABLE transferexecutionstep" in ddl + assert "FOREIGN KEY(task_id) REFERENCES transferpending (task_id) ON DELETE CASCADE" in ddl + assert "CREATE TABLE transfersettlementreceipt" in ddl + assert "CONSTRAINT uq_transfersettlementreceipt_task_revision UNIQUE" in ddl + + +def test_repair_indexes_ignores_postgresql_unique_backing_index(monkeypatch) -> None: + """PG 唯一约束后端索引应留给约束治理，普通意外索引仍须删除。""" + migration = importlib.import_module(MIGRATION) + table_name = "transfersettlementreceipt" + indexes = [ + { + "name": "uq_transfersettlementreceipt_task_revision", + "column_names": ["task_id", "settlement_revision"], + "unique": True, + "duplicates_constraint": "uq_transfersettlementreceipt_task_revision", + }, + { + "name": "ix_transfersettlementreceipt_task_revision", + "column_names": ["task_id", "settlement_revision"], + "unique": False, + }, + { + "name": "ix_transfersettlementreceipt_unexpected", + "column_names": ["outcome"], + "unique": False, + }, + ] + dropped = [] + created = [] + + def drop_index(index_name: str, *, table_name: str) -> None: + """记录删除并模拟 PostgreSQL 反射结果随 DDL 更新。""" + dropped.append((index_name, table_name)) + indexes[:] = [item for item in indexes if item["name"] != index_name] + + inspector = SimpleNamespace( + get_table_names=lambda: [table_name], + get_indexes=lambda inspected_table: list(indexes), + ) + monkeypatch.setattr(migration.sa, "inspect", lambda bind: inspector) + monkeypatch.setattr( + migration, + "op", + SimpleNamespace( + get_bind=lambda: object(), + drop_index=drop_index, + create_index=lambda *args, **kwargs: created.append((args, kwargs)), + ), + ) + + migration._repair_indexes( + table_name=table_name, + expected={ + "ix_transfersettlementreceipt_task_revision": ( + "task_id", + "settlement_revision", + ), + }, + ) + + assert dropped == [ + ("ix_transfersettlementreceipt_unexpected", table_name), + ] + assert created == [] + assert any( + item["name"] == "uq_transfersettlementreceipt_task_revision" + for item in indexes + ) + + +def test_interrupted_column_upgrade_preserves_existing_manual_state(monkeypatch): + """字段阶段中断后重跑应补齐 schema 且保留已写入的人工复核状态。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + connection.execute(sa.text( + "ALTER TABLE transferpending ADD COLUMN execution_state VARCHAR(32)" + )) + connection.execute(sa.text( + "UPDATE transferpending SET execution_state = 'manual_review' " + "WHERE task_id = 'safe'" + )) + connection.execute(sa.text( + "ALTER TABLE transferhistory ADD COLUMN transfer_task_id VARCHAR(64)" + )) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + assert connection.execute(sa.text( + "SELECT execution_state FROM transferpending WHERE task_id = 'safe'" + )).scalar_one() == "manual_review" + pending_columns = { + column["name"] + for column in sa.inspect(connection).get_columns("transferpending") + } + assert { + "execution_fingerprint", + "retry_generation", + "retry_requested_by", + "settlement_revision", + }.issubset(pending_columns) + assert "transfer_settlement_revision" in { + column["name"] + for column in sa.inspect(connection).get_columns("transferhistory") + } + engine.dispose() + + +def test_upgrade_recreates_empty_partial_execution_tables(monkeypatch) -> None: + """中断升级留下的空残表应无损重建为完整步骤与回执 schema。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + connection.execute(sa.text( + "CREATE TABLE transferexecutionstep (" + "id INTEGER PRIMARY KEY, task_id VARCHAR(64) NOT NULL)" + )) + connection.execute(sa.text( + "CREATE TABLE transfersettlementreceipt (" + "id INTEGER PRIMARY KEY, task_id VARCHAR(64) NOT NULL)" + )) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + _assert_execution_tables_match_models(connection) + engine.dispose() + + +def test_upgrade_replaces_old_single_task_receipt_unique(monkeypatch) -> None: + """中断版本的 task 单列唯一约束必须移除，保留数据后允许追加新 revision。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + connection.execute(sa.text( + "INSERT INTO transfersettlementreceipt (" + "task_id, history_id, settlement_revision, outcome, " + "execution_fingerprint, lease_token, history_status, src, src_storage, " + "pending_deleted, error, created_at, updated_at" + ") VALUES (" + "'task-a', 1, 1, 'succeeded', 'fingerprint-1', 'lease-1', 1, " + "'/src', 'local', 1, NULL, " + "'2026-08-27 10:00:00', '2026-08-27 10:00:00'" + ")" + )) + with migration.op.batch_alter_table("transfersettlementreceipt") as batch_op: + batch_op.drop_constraint( + "uq_transfersettlementreceipt_task_revision", + type_="unique", + ) + batch_op.create_unique_constraint( + "uq_transfersettlementreceipt_task_id", + ["task_id"], + ) + migration.upgrade() + connection.execute(sa.text( + "INSERT INTO transfersettlementreceipt (" + "task_id, history_id, settlement_revision, outcome, " + "execution_fingerprint, lease_token, history_status, src, src_storage, " + "pending_deleted, error, created_at, updated_at" + ") VALUES (" + "'task-a', 2, 2, 'failed', 'fingerprint-2', 'lease-2', 0, " + "'/src', 'local', 1, 'failed', " + "'2026-08-27 11:00:00', '2026-08-27 11:00:00'" + ")" + )) + assert connection.execute(sa.text( + "SELECT settlement_revision FROM transfersettlementreceipt " + "WHERE task_id = 'task-a' ORDER BY settlement_revision" + )).scalars().all() == [1, 2] + _assert_execution_tables_match_models(connection) + engine.dispose() + + +@pytest.mark.parametrize( + ("table_name", "create_sql", "insert_sql"), + ( + ( + "transferexecutionstep", + "CREATE TABLE transferexecutionstep (" + "id INTEGER PRIMARY KEY, task_id VARCHAR(64) NOT NULL)", + "INSERT INTO transferexecutionstep (id, task_id) VALUES (1, 'planned')", + ), + ( + "transfersettlementreceipt", + "CREATE TABLE transfersettlementreceipt (" + "id INTEGER PRIMARY KEY, task_id VARCHAR(64) NOT NULL)", + "INSERT INTO transfersettlementreceipt (id, task_id) " + "VALUES (1, 'settled')", + ), + ), +) +def test_upgrade_rejects_nonempty_partial_execution_table( + monkeypatch, + table_name: str, + create_sql: str, + insert_sql: str, +) -> None: + """含数据残表不能猜测修复，且必须报告明确迁移冲突而非缺列 SQL。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + connection.execute(sa.text(create_sql)) + connection.execute(sa.text(insert_sql)) + migration = _bind_migration(monkeypatch, connection) + with pytest.raises( + RuntimeError, + match=rf"含数据的不完整迁移表 {table_name}.*缺少字段", + ): + migration.upgrade() + engine.dispose() + + +def test_upgrade_adds_synthetic_review_when_nonmanual_step_already_exists( + monkeypatch, +) -> None: + """已有普通步骤不代表可人工判定，迁移仍须补 synthetic 并稳定回填时间。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + connection.execute(sa.text( + "UPDATE transferpending SET updated_at = NULL WHERE task_id = 'planned'" + )) + migration = _bind_migration(monkeypatch, connection) + migration._add_pending_columns() + migration._backfill_pending() + migration._create_step_table() + connection.execute(sa.text( + "INSERT INTO transferexecutionstep (" + "task_id, operation_id, checkpoint_fingerprint, ordinal, phase, kind, " + "state, attempt_token, attempt_count, intent_version, intent_payload, " + "prepared_at, updated_at" + ") VALUES (" + "'planned', 'existing-operation', 'existing-plan', 0, 'transfer', 'copy', " + "'prepared', NULL, 0, 1, '{}', " + "'2026-08-27 09:00:00', '2026-08-27 09:00:00'" + ")" + )) + migration.upgrade() + migration.upgrade() + rows = connection.execute(sa.text( + "SELECT kind, state, prepared_at FROM transferexecutionstep " + "WHERE task_id = 'planned' ORDER BY ordinal" + )).all() + assert rows == [ + ("copy", "prepared", "2026-08-27 09:00:00"), + ("legacy_execution_review", "manual_review", "2026-08-27 10:00:00"), + ] + engine.dispose() + + +def test_migrated_legacy_reviews_are_discoverable_resolvable_and_retryable( + monkeypatch, +) -> None: + """迁移遗留任务应可分页判定，并在判定后准备真实首步骤。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + pending, _ = _create_legacy_tables(connection) + _insert_legacy_rows(connection, pending) + migration = _bind_migration(monkeypatch, connection) + migration.upgrade() + + factory = sessionmaker(bind=engine, expire_on_commit=False) + repository = TransactionalTransferExecutionRepository( + factory, + local_clock=lambda: datetime(2026, 8, 27, 11, 0, 0), + lease_clock=lambda: datetime( + 2026, + 8, + 27, + 3, + 0, + 0, + tzinfo=timezone.utc, + ), + ) + query = TransferManualReviewQuery(repository) + command = TransferExecutionCommand(repository) + + first_page = query.list(page=1, page_size=2) + second_page = query.list(page=2, page_size=2) + assert first_page.total == 3 + assert len(first_page.items) == 2 + assert len(second_page.items) == 1 + reviews = { + item.task_id: item + for item in (*first_page.items, *second_page.items) + } + assert set(reviews) == {"planned", "attempted", "provider"} + assert all( + item.step.kind == "legacy_execution_review" + for item in reviews.values() + ) + assert query.get(task_id="planned") == reviews["planned"] + + with pytest.raises( + TransferExecutionConflictError, + match="没有足够证据证明外部操作已发生", + ): + command.resolve_manual_review( + task_id="planned", + operation_id=reviews["planned"].step.operation_id, + decision=TransferManualReviewDecision.APPLIED, + actor="admin", + reason="无法仅凭旧状态确认外部结果", + result=TransferStepResult(payload={"confirmed": True}), + ) + assert query.get(task_id="planned").state is TransferExecutionState.MANUAL_REVIEW + + resolved = [] + for task_id in ("planned", "attempted"): + resolved.append(command.resolve_manual_review( + task_id=task_id, + operation_id=reviews[task_id].step.operation_id, + decision=TransferManualReviewDecision.NOT_APPLIED, + actor="admin", + reason="已回滚或确认旧步骤未发生", + result=TransferStepResult(payload={"confirmed": False}), + )) + assert all( + item.state is TransferExecutionState.RETRY_WAIT + for item in resolved + ) + retry_page = query.list( + state=TransferExecutionState.RETRY_WAIT, + page=1, + page_size=10, + ) + assert {item.task_id for item in retry_page.items} == {"planned", "attempted"} + assert query.get(task_id="planned").step.evidence == {"confirmed": False} + assert query.get(task_id="attempted").step.evidence == {"confirmed": False} + + with factory() as session: + rows = list(session.scalars( + sa.select(TransferPending).where( + TransferPending.task_id.in_(("planned", "attempted")) + ) + ).all()) + for row in rows: + row.lease_owner = "worker" + row.lease_token = f"lease-{row.task_id}" + row.lease_expires_at = "2099-01-01 00:00:00.000000" + session.commit() + + for task_id in ("planned", "attempted"): + snapshot = repository.get_snapshot(task_id=task_id) + assert snapshot is not None + assert snapshot.steps == () + prepared = command.prepare( + task_id=task_id, + lease_token=f"lease-{task_id}", + intent=TransferStepIntent.create( + task_id=task_id, + checkpoint_fingerprint="f" * 64, + ordinal=0, + phase="transfer", + kind="copy", + payload={"source": task_id}, + ), + ) + assert prepared.ordinal == 0 + engine.dispose() diff --git a/tests/test_transfer_execution_persistence.py b/tests/test_transfer_execution_persistence.py new file mode 100644 index 000000000..1249ee570 --- /dev/null +++ b/tests/test_transfer_execution_persistence.py @@ -0,0 +1,454 @@ +"""验证整理执行证据、CAS fencing 与终态结算持久化。""" + +from datetime import datetime, timezone + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferExecutionCommand, + TransferExecutionConflictError, + TransferExecutionState, + TransferManualReviewDecision, + TransferOperationObservation, + TransferOperationObservationState, + TransferStepIntent, + TransferStepResult, + TransferStepState, + build_transfer_checkpoint_fingerprint, + build_transfer_operation_id, +) +from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, +) +from app.db.base import Base +from app.db.models.transferexecutionstep import TransferExecutionStep +from app.db.models.transferhistory import TransferHistory +from app.db.models.transferpending import TransferPending + + +@pytest.fixture +def execution_store(): + """构造只含整理执行相关表的独立内存数据库。""" + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[ + TransferPending.__table__, + TransferHistory.__table__, + TransferExecutionStep.__table__, + ], + ) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield factory + finally: + engine.dispose() + + +def _seed_pending(factory, *, task_id: str = "task-1", lease_token: str = "lease-1"): + """写入一条带有效租约与合法 planning checkpoint 的待执行任务。""" + with factory() as session: + session.add(TransferPending( + task_id=task_id, + storage="local", + src_path=f"/{task_id}.mkv", + created_at="2026-08-27 09:00:00", + state="planned", + updated_at="2026-08-27 09:00:00", + input_version=1, + planning_input={"schema_version": 1, "source": task_id}, + input_fingerprint="input-fingerprint", + checkpoint_version=1, + checkpoint_payload={"schema_version": 1, "task_id": task_id}, + planned_at="2026-08-27 09:00:00", + lease_owner="worker-1", + lease_token=lease_token, + lease_expires_at="2099-01-01 00:00:00.000000", + heartbeat_at="2026-08-27 01:00:00.000000", + attempt_count=1, + execution_state="not_started", + retry_generation=0, + retry_count=0, + settlement_revision=0, + )) + session.commit() + + +def _repository(factory, token_values: list[str] | None = None): + """构造固定时钟与可预测 attempt token 的执行命令。""" + repository = TransactionalTransferExecutionRepository( + factory, + local_clock=lambda: datetime(2026, 8, 27, 9, 30, 0), + lease_clock=lambda: datetime(2026, 8, 27, 1, 30, 0, tzinfo=timezone.utc), + ) + values = iter(token_values or ["attempt-1", "attempt-2", "attempt-3"]) + return repository, TransferExecutionCommand( + repository, + attempt_token_factory=lambda: next(values), + ) + + +def _intent(*, task_id: str = "task-1", ordinal: int = 0) -> TransferStepIntent: + """构造稳定且可重复计算身份的测试步骤意图。""" + return TransferStepIntent.create( + task_id=task_id, + checkpoint_fingerprint="plan-fingerprint", + ordinal=ordinal, + phase="transfer", + kind="copy", + payload={"src": f"/{task_id}.mkv", "dest": f"/media/{task_id}.mkv"}, + ) + + +def test_stable_operation_and_checkpoint_identities_are_canonical(): + """字段顺序不能改变 operation ID 或执行 checkpoint 指纹。""" + first = build_transfer_operation_id( + task_id="task", + checkpoint_fingerprint="plan", + ordinal=2, + phase="scrape", + kind="write", + intent_payload={"b": 2, "a": 1}, + ) + second = build_transfer_operation_id( + task_id="task", + checkpoint_fingerprint="plan", + ordinal=2, + phase="scrape", + kind="write", + intent_payload={"a": 1, "b": 2}, + ) + assert first == second + assert build_transfer_checkpoint_fingerprint({"b": 2, "a": 1}) == ( + build_transfer_checkpoint_fingerprint({"a": 1, "b": 2}) + ) + mutable_payload = {"path": "/original"} + intent = TransferStepIntent.create( + task_id="task", + checkpoint_fingerprint="plan", + ordinal=0, + phase="transfer", + kind="copy", + payload=mutable_payload, + ) + mutable_payload["path"] = "/mutated" + assert intent.payload == {"path": "/original"} + + +def test_success_path_persists_steps_and_execution_checkpoint(execution_store): + """成功路径应保留每步证据，并提交可供唯一 durable writer 结算的检查点。""" + _seed_pending(execution_store) + repository, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", + lease_token="lease-1", + intent=_intent(), + ) + assert prepared.state is TransferStepState.PREPARED + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + assert started.attempt_token == "attempt-1" + succeeded = command.complete( + task_id="task-1", + lease_token="lease-1", + step=started, + result=TransferStepResult(payload={"dest_exists": True}), + ) + checkpoint = TransferExecutionCheckpoint.create( + payload={"dest": "/media/task-1.mkv"}, + operation_ids=(succeeded.operation_id,), + ) + snapshot = command.checkpoint( + task_id="task-1", + lease_token="lease-1", + checkpoint=checkpoint, + ) + assert snapshot.state is TransferExecutionState.SETTLING + with execution_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + assert pending.execution_fingerprint == checkpoint.fingerprint + step = session.scalar(select(TransferExecutionStep)) + assert step is not None and step.state == "succeeded" + + +def test_retry_wait_resumes_same_failed_operation_with_new_attempt(execution_store): + """到期重试必须复用 operation ID、保留失败证据并轮换 attempt token。""" + _seed_pending(execution_store) + repository, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", lease_token="lease-1", intent=_intent() + ) + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + deferred = command.defer( + task_id="task-1", + lease_token="lease-1", + step=started, + error="destination unavailable", + retry_due_at="2026-08-27 01:30:01.000000", + evidence=TransferStepResult(payload={"applied": False}), + ) + assert deferred.state is TransferExecutionState.RETRY_WAIT + assert deferred.retry_generation == 1 + with execution_store() as session: + claimed = TransferPending.claim_task( + session, + task_id="task-1", + states=("planned",), + owner_id="worker-2", + lease_token="lease-2", + now_time="2026-08-27 01:30:02.000000", + lease_expires_at="2099-01-01 00:00:00.000000", + updated_at="2026-08-27 09:30:02", + ) + session.commit() + assert claimed == 1 + resumed = command.resume_failed( + task_id="task-1", + lease_token="lease-2", + step=deferred.steps[0], + ) + assert resumed.operation_id == prepared.operation_id + assert resumed.attempt_token == "attempt-2" + assert resumed.attempt_count == 2 + assert resumed.result == TransferStepResult(payload={"applied": False}) + assert resumed.last_error == "destination unavailable" + + +def test_orphan_started_requires_observation_before_attempt_rotation(execution_store): + """遗留 STARTED 只能凭 NOT_APPLIED 证据轮换 attempt，旧 attempt 随即失效。""" + _seed_pending(execution_store) + repository, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", lease_token="lease-1", intent=_intent() + ) + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + observation = TransferOperationObservation( + state=TransferOperationObservationState.NOT_APPLIED, + evidence=TransferStepResult(payload={"dest_exists": False}), + ) + restarted = command.restart_after_not_applied( + task_id="task-1", + lease_token="lease-1", + step=started, + evidence=observation.evidence, + ) + assert restarted.attempt_token == "attempt-2" + assert restarted.attempt_count == 2 + with pytest.raises(TransferExecutionConflictError): + repository.complete_step( + task_id="task-1", + lease_token="lease-1", + operation_id=started.operation_id, + attempt_token="attempt-1", + result=TransferStepResult(payload={"stale": True}), + ) + + +def test_zero_side_effect_checkpoint_is_vacuously_complete(execution_store): + """纯策略拒绝可在没有步骤行时提交带 skip_reason 的确定执行结果。""" + _seed_pending(execution_store) + _, command = _repository(execution_store) + checkpoint = TransferExecutionCheckpoint.create( + payload={"preview": True, "accepted": False}, + operation_ids=(), + skip_reason="preview", + ) + snapshot = command.checkpoint( + task_id="task-1", + lease_token="lease-1", + checkpoint=checkpoint, + ) + assert snapshot.state is TransferExecutionState.SETTLING + assert snapshot.checkpoint == checkpoint + assert snapshot.steps == () + + +def test_exhausted_step_builds_failure_checkpoint_and_keeps_lease(execution_store): + """预算耗尽应原子建立失败结算检查点，并为 durable writer 保留 lease。""" + _seed_pending(execution_store) + _, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", lease_token="lease-1", intent=_intent() + ) + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + snapshot = command.exhaust( + task_id="task-1", + lease_token="lease-1", + step=started, + error="retry budget exhausted", + evidence=TransferStepResult(payload={"applied": False}), + ) + assert snapshot.state is TransferExecutionState.SETTLING + assert snapshot.checkpoint is not None + assert snapshot.checkpoint.payload["outcome"] == "failed" + assert snapshot.checkpoint.payload["error"] == "retry budget exhausted" + assert snapshot.checkpoint.operation_ids == (started.operation_id,) + assert snapshot.steps[0].state is TransferStepState.FAILED + with execution_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + assert pending.lease_token == "lease-1" + + +def test_user_retry_is_single_generation_and_rejects_manual_review(execution_store): + """FAILED 用户重试只递增一次世代，重复请求幂等且人工复核必须拒绝。""" + _seed_pending(execution_store) + with execution_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + pending.execution_state = "failed" + pending.lease_owner = None + pending.lease_token = None + pending.lease_expires_at = None + pending.terminal_history_id = 42 + pending.retry_count = 3 + session.commit() + _, command = _repository(execution_store) + first = command.request_retry( + task_id="task-1", + reason="用户确认目标未落地", + requested_by="admin", + ) + repeated = command.request_retry( + task_id="task-1", + reason="重复点击", + requested_by="admin", + ) + assert first.accepted and repeated.accepted + assert first.retry_generation == repeated.retry_generation == 1 + with execution_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + assert pending.execution_state == "retry_wait" + assert pending.retry_count == 3 + assert pending.terminal_history_id == 42 + assert pending.retry_reason == "用户确认目标未落地" + assert pending.retry_requested_by == "admin" + pending.execution_state = "manual_review" + session.commit() + rejected = command.request_retry( + task_id="task-1", + reason="强制重试", + requested_by="admin", + ) + assert not rejected.accepted + assert rejected.state is TransferExecutionState.MANUAL_REVIEW + assert "人工" in rejected.message + + +def test_manual_not_applied_decision_is_audited_and_schedules_same_step( + execution_store, +) -> None: + """人工判定未发生应无 lease 地恢复 FAILED，并只交给唯一调度器。""" + _seed_pending(execution_store) + _, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", lease_token="lease-1", intent=_intent() + ) + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + manual = command.manual_review( + task_id="task-1", + lease_token="lease-1", + step=started, + error="external result unknown", + ) + assert manual.state is TransferExecutionState.MANUAL_REVIEW + resolved = command.resolve_manual_review( + task_id="task-1", + operation_id=started.operation_id, + decision=TransferManualReviewDecision.NOT_APPLIED, + actor="admin", + reason="目标与临时文件均不存在", + result=TransferStepResult(payload={"dest_exists": False}), + ) + assert resolved.state is TransferExecutionState.RETRY_WAIT + assert resolved.review_revision == 1 + assert resolved.step.state is TransferStepState.FAILED + with execution_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + assert pending.lease_token is None + assert pending.reviewed_by == "admin" + assert pending.review_decision == "not_applied" + assert pending.review_reason == "目标与临时文件均不存在" + with pytest.raises(TransferExecutionConflictError): + command.resolve_manual_review( + task_id="task-1", + operation_id=started.operation_id, + decision=TransferManualReviewDecision.NOT_APPLIED, + actor="admin", + reason="重复判定", + ) + + +def test_manual_applied_requires_result_and_failed_decision_is_rejected( + execution_store, +) -> None: + """人工判定已发生必须带结果证据，FAILED 不得绕过 lease durable 结算。""" + _seed_pending(execution_store) + _, command = _repository(execution_store) + prepared = command.prepare( + task_id="task-1", lease_token="lease-1", intent=_intent() + ) + started = command.begin( + task_id="task-1", + lease_token="lease-1", + operation_id=prepared.operation_id, + ) + command.manual_review( + task_id="task-1", + lease_token="lease-1", + step=started, + error="external result unknown", + ) + with pytest.raises(ValueError, match="结果证据"): + command.resolve_manual_review( + task_id="task-1", + operation_id=started.operation_id, + decision=TransferManualReviewDecision.APPLIED, + actor="admin", + reason="已确认目标存在", + ) + with pytest.raises(TransferExecutionConflictError, match="durable"): + command.resolve_manual_review( + task_id="task-1", + operation_id=started.operation_id, + decision=TransferManualReviewDecision.FAILED, + actor="admin", + reason="确认失败", + ) + resolved = command.resolve_manual_review( + task_id="task-1", + operation_id=started.operation_id, + decision=TransferManualReviewDecision.APPLIED, + actor="admin", + reason="目标摘要匹配", + result=TransferStepResult(payload={"dest_exists": True, "hash_match": True}), + ) + assert resolved.step.state is TransferStepState.SUCCEEDED + assert resolved.step.result == TransferStepResult( + payload={"dest_exists": True, "hash_match": True} + ) diff --git a/tests/test_transfer_execution_runner.py b/tests/test_transfer_execution_runner.py new file mode 100644 index 000000000..e9f8637d6 --- /dev/null +++ b/tests/test_transfer_execution_runner.py @@ -0,0 +1,217 @@ +"""验证 TransferChain 步骤 runner 与文件执行器的崩溃恢复边界。""" + +from datetime import datetime, timezone +from pathlib import Path +from unittest.mock import Mock + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferExecutionState, + TransferOperationObservation, + TransferOperationObservationState, + TransferStepIntent, + TransferStepResult, +) +from app.chain import transfer as transfer_chain_module +from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, +) +from app.db.base import Base +from app.db.models.transferexecutionstep import TransferExecutionStep +from app.db.models.transferhistory import TransferHistory +from app.db.models.transferpending import TransferPending +from app.modules.filemanager.transhandler import TransHandler +from app.schemas.workflow import FileItem + + +@pytest.fixture +def execution_repository(): + """构造带有效 pending 租约的独立执行仓储。""" + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[ + TransferPending.__table__, + TransferHistory.__table__, + TransferExecutionStep.__table__, + ], + ) + factory = sessionmaker(bind=engine, expire_on_commit=False) + with factory() as session: + session.add(TransferPending( + task_id="task-runner", + storage="local", + src_path="/source.mkv", + created_at="2026-08-27 09:00:00", + state="planned", + updated_at="2026-08-27 09:00:00", + input_version=1, + planning_input={"schema_version": 1}, + input_fingerprint="input", + checkpoint_version=1, + checkpoint_payload={"schema_version": 1}, + planned_at="2026-08-27 09:00:00", + lease_owner="worker", + lease_token="lease", + lease_expires_at="2099-01-01 00:00:00.000000", + heartbeat_at="2026-08-27 01:00:00.000000", + attempt_count=1, + execution_state="not_started", + retry_generation=0, + retry_count=0, + settlement_revision=0, + )) + session.commit() + repository = TransactionalTransferExecutionRepository( + factory, + local_clock=lambda: datetime(2026, 8, 27, 9, 30, 0), + lease_clock=lambda: datetime(2026, 8, 27, 1, 30, 0, tzinfo=timezone.utc), + ) + try: + yield repository + finally: + engine.dispose() + + +def _runner(repository): + """构造绑定固定任务、租约与计划身份的 durable runner。""" + return transfer_chain_module._DurableTransferStepRunner( + task_id="task-runner", + lease_token="lease", + checkpoint_fingerprint="plan", + repository=repository, + ) + + +def test_runner_replay_returns_persisted_result_without_repeating_side_effect( + execution_repository, +): + """成功步骤重放只能回读结果，不能再次调用外部执行函数。""" + calls = [] + first = _runner(execution_repository).run( + phase="transfer", + kind="copy", + payload={"source": "/source.mkv", "target": "/target.mkv"}, + execute=lambda: calls.append("executed") or TransferStepResult( + payload={"item": {"path": "/target.mkv"}} + ), + observe=lambda: pytest.fail("新步骤不应执行恢复探测"), + ) + second = _runner(execution_repository).run( + phase="transfer", + kind="copy", + payload={"source": "/source.mkv", "target": "/target.mkv"}, + execute=lambda: pytest.fail("已成功步骤不得重复执行"), + observe=lambda: pytest.fail("已成功步骤不得执行恢复探测"), + ) + assert first == second + assert calls == ["executed"] + + +def test_runner_routes_unknown_orphaned_attempt_to_manual_review( + execution_repository, +): + """遗留 STARTED 无法严格判断时必须隔离，不能再次执行副作用。""" + command = TransferExecutionCommand(execution_repository) + prepared = command.prepare( + task_id="task-runner", + lease_token="lease", + intent=TransferStepIntent.create( + task_id="task-runner", + checkpoint_fingerprint="plan", + ordinal=0, + phase="provider", + kind="opaque", + payload={"provider": "legacy"}, + ), + ) + command.begin( + task_id="task-runner", + lease_token="lease", + operation_id=prepared.operation_id, + ) + + with pytest.raises( + transfer_chain_module._TransferManualReviewRequired, + match="禁止自动重放", + ): + _runner(execution_repository).run( + phase="provider", + kind="opaque", + payload={"provider": "legacy"}, + execute=lambda: pytest.fail("未知遗留步骤不得重放"), + observe=lambda: TransferOperationObservation( + state=TransferOperationObservationState.UNKNOWN, + evidence=TransferStepResult(payload={"receipt": None}), + ), + ) + snapshot = execution_repository.get_snapshot(task_id="task-runner") + assert snapshot is not None + assert snapshot.state is TransferExecutionState.MANUAL_REVIEW + + +class _ImmediateStepRunner: + """记录 TransHandler 拆分顺序并立即执行步骤的测试 runner。""" + + def __init__(self) -> None: + """初始化步骤记录。""" + self.steps = [] + + def run(self, *, phase, kind, payload, execute, observe): + """记录稳定意图并直接执行，不触发恢复探测。""" + self.steps.append((phase, kind, payload)) + return execute() + + +def test_cross_storage_move_materializes_before_independent_source_delete(tmp_path): + """跨存储 move 必须先复制目标，再用独立步骤删除源。""" + source_path = tmp_path / "source.mkv" + source_path.write_bytes(b"movie") + source_item = FileItem( + storage="local", + path=source_path.as_posix(), + name=source_path.name, + type="file", + size=source_path.stat().st_size, + extension="mkv", + ) + target_item = FileItem( + storage="remote", + path="/library/source.mkv", + name="source.mkv", + type="file", + size=source_item.size, + extension="mkv", + ) + source_oper = Mock() + source_oper.delete.return_value = True + target_oper = Mock() + target_oper.get_folder.return_value = FileItem( + storage="remote", path="/library", name="library", type="dir" + ) + target_oper.upload.return_value = target_item + runner = _ImmediateStepRunner() + + result, error = TransHandler._TransHandler__execute_transfer_with_steps( + step_runner=runner, + fileitem=source_item, + target_storage="remote", + source_oper=source_oper, + target_oper=target_oper, + target_file=Path("/library/source.mkv"), + transfer_type="move", + ) + + assert error == "" + assert result == target_item + assert [kind for _phase, kind, _payload in runner.steps] == [ + "materialize_target", + "delete_move_source", + ] + assert runner.steps[0][2]["transfer_type"] == "copy" + target_oper.upload.assert_called_once() + source_oper.delete.assert_called_once_with(source_item) diff --git a/tests/test_transfer_lease_migration.py b/tests/test_transfer_lease_migration.py index ac8f2deb2..9f23c82e2 100644 --- a/tests/test_transfer_lease_migration.py +++ b/tests/test_transfer_lease_migration.py @@ -16,6 +16,24 @@ LEASE_COLUMNS = { "heartbeat_at", "attempt_count", } +TRANSFER_EXECUTION_COLUMNS = { + "execution_state", + "execution_version", + "execution_payload", + "execution_fingerprint", + "retry_generation", + "retry_count", + "retry_due_at", + "retry_requested_by", + "retry_reason", + "settlement_revision", + "terminal_history_id", + "manual_review_revision", + "reviewed_at", + "reviewed_by", + "review_reason", + "review_decision", +} def _bind_migration(monkeypatch, connection): @@ -105,7 +123,11 @@ def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None: assert { column["name"] for column in inspector.get_columns("transferpending") - } == {column.name for column in TransferPending.__table__.columns} + } == { + column.name + for column in TransferPending.__table__.columns + if column.name not in TRANSFER_EXECUTION_COLUMNS + } assert "ix_transferpending_recovery_lease" in { index["name"] for index in inspector.get_indexes("transferpending") diff --git a/tests/test_transfer_lease_persistence.py b/tests/test_transfer_lease_persistence.py index b8355923d..f45c73396 100644 --- a/tests/test_transfer_lease_persistence.py +++ b/tests/test_transfer_lease_persistence.py @@ -20,6 +20,7 @@ from app.application.transfer import ( TransferProviderReference, ) from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending from app.db.oper.transferpending import TransferPendingOper @@ -90,6 +91,7 @@ def repository_factory(tmp_path): f"sqlite:///{tmp_path / 'transfer-lease.db'}", connect_args={"check_same_thread": False, "timeout": 10}, ) + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) factory = sessionmaker(bind=engine) yield lambda: TransactionalTransferAdmissionRepository(factory) diff --git a/tests/test_transfer_legacy_terminal_compat.py b/tests/test_transfer_legacy_terminal_compat.py new file mode 100644 index 000000000..cf90f8e57 --- /dev/null +++ b/tests/test_transfer_legacy_terminal_compat.py @@ -0,0 +1,264 @@ +"""验证旧插件同步整理 ABI 复用 canonical durable 终态写入口。""" + +import threading +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferSettlementResult, +) +from app.chain.transfer import TransferChain +from app.domain.context import MediaInfo +from app.domain.meta.metabase import MetaBase +from app.schemas.file import FileItem +from app.schemas.transfer import TransferInfo +from app.schemas.types import MediaType + + +def _fileitem(*, fileid: str = "source-v1", size: int = 1024) -> FileItem: + """构造可区分同路径版本的兼容调用文件项。""" + return FileItem( + storage="local", + path="/downloads/Movie.2026.mkv", + type="file", + name="Movie.2026.mkv", + basename="Movie.2026", + extension="mkv", + size=size, + modify_time=1770000000 + size, + fileid=fileid, + ) + + +def _result(task, *, success: bool, overwrite_skipped: bool = False) -> TransferInfo: + """按当前任务构造足以写历史的兼容整理结果。""" + return TransferInfo( + success=success, + overwrite_skipped=overwrite_skipped, + message=None if success else "copy failed", + fileitem=task.fileitem, + target_item=FileItem( + storage="local", + path="/library/Movie (2026)/Movie.mkv", + type="file", + name="Movie.mkv", + basename="Movie", + extension="mkv", + ), + transfer_type="copy", + file_list=[task.fileitem.path], + ) + + +def _compat_chain(result_factory): + """构造只执行旧同步命令和 task-aware 结算的 TransferChain 骨架。""" + chain = object.__new__(TransferChain) + chain._worker_owner_id = "compat-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + chain._worker_state_lock = threading.RLock() + chain.durable_event_writer = Mock() + executed = [] + + def execute(task, **_kwargs): + """模拟外部步骤已完成并建立可独立结算的执行检查点。""" + executed.append(task.fileitem.fileid) + task_id = f"task-{task.fileitem.fileid}" + lease_token = f"lease-{task.fileitem.fileid}" + task.bind_admission_task_id(task_id) + task.bind_execution_lease( + owner_id=chain._worker_owner_id, + lease_token=lease_token, + ) + chain._owned_leases[task_id] = (lease_token, float("inf")) + result = result_factory(task) + task.bind_execution_checkpoint(TransferExecutionCheckpoint.create( + payload={ + "outcome": ( + "succeeded" + if result.success + else "failed" + ), + "transferinfo": result.model_dump(mode="json"), + }, + operation_ids=(f"operation-{task.fileitem.fileid}",), + )) + return result + + chain._plan_checkpoint_and_execute = Mock(side_effect=execute) + return chain, executed + + +def _invoke(chain: TransferChain, fileitem: FileItem) -> TransferInfo: + """以插件可见参数调用同步兼容入口。""" + return chain.execute_legacy_transfer_command( + fileitem=fileitem, + meta=MetaBase(fileitem.name), + mediainfo=MediaInfo(type=MediaType.MOVIE, title="Movie", year="2026"), + target_storage="local", + target_path="/library", + transfer_type="copy", + ) + + +def _settlement_writer(*, status: bool, history_id: int = 41): + """返回执行 stage_history 并产出 task-aware 结果的 writer side effect。""" + staged_payloads = [] + + def write(**kwargs): + """暂存历史后返回与当前结算修订一致的投影。""" + staging = Mock() + + def add_force(**payload): + """保存历史 payload 并返回 writer 所需的最小记录。""" + staged_payloads.append(payload) + return SimpleNamespace( + id=history_id, + status=bool(payload["status"]), + src=payload["src"], + src_storage=payload["src_storage"], + src_fileitem=payload["src_fileitem"], + ) + + staging.add_force.side_effect = add_force + staging.get_success_by_src.return_value = SimpleNamespace( + id=history_id, + status=True, + src="/downloads/Movie.2026.mkv", + src_storage="local", + src_fileitem=_fileitem().model_dump(mode="json"), + ) + history = kwargs["stage_history"](staging) + assert bool(history.status) is status + return TransferSettlementResult( + history_id=history.id, + settlement_revision=1, + pending_deleted=status, + ) + + return write, staged_payloads + + +def test_legacy_failed_result_uses_atomic_terminal_writer() -> None: + """失败结果也必须原子写失败历史并保留 pending，不得直接注销。""" + chain, executed = _compat_chain(lambda task: _result(task, success=False)) + write, staged_payloads = _settlement_writer(status=False) + chain.durable_event_writer.transfer_result.side_effect = write + + returned = _invoke(chain, _fileitem()) + + assert returned.success is False + assert returned.message == "copy failed" + assert executed == ["source-v1"] + assert staged_payloads[0]["status"] == 0 + call = chain.durable_event_writer.transfer_result.call_args.kwargs + assert call["topic"] is None + assert call["publish"] is None + assert call["settlement"].outcome == "failed" + assert call["settlement"].error == "copy failed" + + +def test_legacy_settlement_response_loss_replays_receipt_by_same_task_id() -> None: + """首次提交后响应丢失只能用同一 task_id 回读，不得重做外部步骤。""" + chain, executed = _compat_chain(lambda task: _result(task, success=True)) + write, staged_payloads = _settlement_writer(status=True) + writer_calls = 0 + + def response_lost_then_receipt(**kwargs): + """首次提交历史后模拟响应丢失，第二次只返回 immutable receipt。""" + nonlocal writer_calls + writer_calls += 1 + if writer_calls == 1: + write(**kwargs) + raise RuntimeError("response lost after commit") + return TransferSettlementResult( + history_id=41, + settlement_revision=1, + pending_deleted=True, + already_settled=True, + ) + + chain.durable_event_writer.transfer_result.side_effect = response_lost_then_receipt + + returned = _invoke(chain, _fileitem()) + + assert returned.success is True + assert executed == ["source-v1"] + assert writer_calls == 2 + assert len(staged_payloads) == 1 + first = chain.durable_event_writer.transfer_result.call_args_list[0].kwargs + second = chain.durable_event_writer.transfer_result.call_args_list[1].kwargs + assert second["settlement"] == first["settlement"] + + +def test_legacy_overwrite_skip_binds_existing_success_in_atomic_writer() -> None: + """覆盖跳过复用既有成功历史并以 succeeded 终态结算。""" + chain, executed = _compat_chain( + lambda task: _result(task, success=False, overwrite_skipped=True) + ) + write, staged_payloads = _settlement_writer(status=True) + chain.durable_event_writer.transfer_result.side_effect = write + + success_history = SimpleNamespace(id=41, status=True) + history_port = SimpleNamespace( + get_by_src=lambda _src, storage=None: success_history, + ) + with patch( + "app.chain.transfer.get_chain_transfer_history_port", + return_value=history_port, + ): + returned = _invoke(chain, _fileitem()) + + assert returned.success is False + assert returned.overwrite_skipped is True + assert executed == ["source-v1"] + assert staged_payloads == [] + settlement = chain.durable_event_writer.transfer_result.call_args.kwargs[ + "settlement" + ] + assert settlement.outcome == "succeeded" + assert settlement.error is None + + +def test_legacy_overwrite_skip_without_success_history_settles_failed() -> None: + """兼容入口没有既有成功历史时按失败结算并保留 pending。""" + chain, executed = _compat_chain( + lambda task: _result(task, success=False, overwrite_skipped=True) + ) + write, staged_payloads = _settlement_writer(status=False) + chain.durable_event_writer.transfer_result.side_effect = write + history_port = SimpleNamespace( + get_by_src=lambda _src, storage=None: None, + ) + + with patch( + "app.chain.transfer.get_chain_transfer_history_port", + return_value=history_port, + ): + returned = _invoke(chain, _fileitem()) + + assert returned.success is False + assert returned.overwrite_skipped is True + assert executed == ["source-v1"] + assert len(staged_payloads) == 1 + assert staged_payloads[0]["status"] == 0 + settlement = chain.durable_event_writer.transfer_result.call_args.kwargs[ + "settlement" + ] + assert settlement.outcome == "failed" + assert settlement.error == "copy failed" + + +def test_legacy_same_path_changed_version_still_executes_new_task() -> None: + """兼容入口不能仅凭同源历史吞掉 fileid/size/mtime 已变化的新版本。""" + chain, executed = _compat_chain(lambda task: _result(task, success=True)) + write, _staged_payloads = _settlement_writer(status=True) + chain.durable_event_writer.transfer_result.side_effect = write + + first = _invoke(chain, _fileitem(fileid="source-v1", size=1024)) + second = _invoke(chain, _fileitem(fileid="source-v2", size=2048)) + + assert first.success is True + assert second.success is True + assert executed == ["source-v1", "source-v2"] diff --git a/tests/test_transfer_manual_review_api.py b/tests/test_transfer_manual_review_api.py new file mode 100644 index 000000000..ed2342f98 --- /dev/null +++ b/tests/test_transfer_manual_review_api.py @@ -0,0 +1,201 @@ +"""验证 durable 整理人工复核 API 的鉴权和公开契约。""" + +from __future__ import annotations + +import inspect +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException +from pydantic import ValidationError + +from app.api.dependencies.auth import get_current_active_manage_user +from app.api.endpoints import transfer as transfer_endpoint +from app.application.transfer_execution import ( + TransferExecutionConflictError, + TransferExecutionState, + TransferManualReviewDecision, + TransferManualReviewResult, + TransferStepResult, +) +from app.schemas.transfer import TransferManualReviewRequest + + +class _ManualReviewCommand: + """记录人工复核调用并返回可控结果。""" + + calls: list[tuple[object, dict]] = [] + result: TransferManualReviewResult | None = None + error: Exception | None = None + + def __init__(self, repository: object) -> None: + """保存端点取得的仓储替身。""" + self._repository = repository + + def resolve_manual_review(self, **kwargs) -> TransferManualReviewResult: + """记录参数，并按测试配置返回或抛出结果。""" + self.calls.append((self._repository, kwargs)) + if self.error: + raise self.error + assert self.result is not None + return self.result + + +def _install_command(monkeypatch) -> object: + """安装不接触数据库的人工复核命令替身。""" + repository = object() + _ManualReviewCommand.calls = [] + _ManualReviewCommand.error = None + monkeypatch.setattr( + transfer_endpoint, + "get_chain_transfer_execution_port", + lambda: repository, + ) + monkeypatch.setattr( + transfer_endpoint, + "TransferExecutionCommand", + _ManualReviewCommand, + ) + return repository + + +@pytest.mark.parametrize( + "endpoint", + [ + transfer_endpoint.list_transfer_manual_reviews, + transfer_endpoint.get_transfer_manual_review, + transfer_endpoint.resolve_transfer_manual_review, + ], +) +def test_manual_review_endpoints_require_manage_permission(endpoint) -> None: + """人工复核发现、详情与判定必须复用全局 manage 权限依赖。""" + dependency = inspect.signature( + endpoint + ).parameters["current_user"].default.dependency + + assert dependency is get_current_active_manage_user + + +@pytest.mark.parametrize( + ("payload", "message"), + [ + ( + {"operation_id": "op-1", "decision": "failed", "reason": "确认失败"}, + "Input should be 'not_applied' or 'applied'", + ), + ( + {"operation_id": "op-1", "decision": "applied", "reason": "已完成"}, + "必须提供 result_payload", + ), + ( + {"operation_id": "op-1", "decision": "not_applied", "reason": " "}, + "String should have at least 1 character", + ), + ], +) +def test_manual_review_request_rejects_unsafe_decisions( + payload: dict, + message: str, +) -> None: + """公开 schema 不允许 FAILED，且 APPLIED 必须携带结果证据。""" + with pytest.raises(ValidationError, match=message): + TransferManualReviewRequest.model_validate(payload) + + +def test_manual_review_applied_wraps_result_and_hides_internal_state( + monkeypatch, +) -> None: + """APPLIED 应包装版本化结果，响应不得泄漏 lease 或 attempt。""" + repository = _install_command(monkeypatch) + _ManualReviewCommand.result = SimpleNamespace( + task_id="task-1", + operation_id="op-1", + decision=TransferManualReviewDecision.APPLIED, + state=TransferExecutionState.RETRY_WAIT, + review_revision=4, + ) + + response = transfer_endpoint.resolve_transfer_manual_review( + task_id="task-1", + review=TransferManualReviewRequest( + operation_id="op-1", + decision="applied", + reason="目标摘要匹配", + result_payload={"dest_exists": True, "hash_match": True}, + ), + current_user=SimpleNamespace(name=" admin ", username="other", id=7), + ) + + assert _ManualReviewCommand.calls == [ + ( + repository, + { + "task_id": "task-1", + "operation_id": "op-1", + "decision": TransferManualReviewDecision.APPLIED, + "actor": "admin", + "reason": "目标摘要匹配", + "result": TransferStepResult( + payload={"dest_exists": True, "hash_match": True} + ), + }, + ) + ] + assert response.data is not None + assert response.data.model_dump() == { + "task_id": "task-1", + "operation_id": "op-1", + "decision": "applied", + "state": "retry_wait", + "review_revision": 4, + } + assert "lease" not in response.model_dump_json() + assert "attempt" not in response.model_dump_json() + + +def test_manual_review_not_applied_uses_username_fallback(monkeypatch) -> None: + """名称为空时应稳定回退到 username，并允许安全重新调度。""" + _install_command(monkeypatch) + _ManualReviewCommand.result = SimpleNamespace( + task_id="task-2", + operation_id="op-2", + decision=TransferManualReviewDecision.NOT_APPLIED, + state=TransferExecutionState.RETRY_WAIT, + review_revision=2, + ) + + response = transfer_endpoint.resolve_transfer_manual_review( + task_id="task-2", + review=TransferManualReviewRequest( + operation_id="op-2", + decision="not_applied", + reason="确认源文件仍存在", + ), + current_user=SimpleNamespace(name="", username="reviewer", id=8), + ) + + assert _ManualReviewCommand.calls[0][1]["actor"] == "reviewer" + assert _ManualReviewCommand.calls[0][1]["result"] is None + assert response.data is not None + assert response.data.state == "retry_wait" + + +def test_manual_review_conflict_returns_http_409(monkeypatch) -> None: + """重复或过期人工判定必须返回资源冲突而非伪成功。""" + _install_command(monkeypatch) + _ManualReviewCommand.error = TransferExecutionConflictError("步骤已被判定") + + with pytest.raises(HTTPException) as error: + transfer_endpoint.resolve_transfer_manual_review( + task_id="task-1", + review=TransferManualReviewRequest( + operation_id="op-1", + decision="not_applied", + reason="重复判定", + ), + current_user=SimpleNamespace(name=None, username=None, id=11), + ) + + assert error.value.status_code == 409 + assert error.value.detail == "步骤已被判定" + assert _ManualReviewCommand.calls[0][1]["actor"] == "11" diff --git a/tests/test_transfer_manual_review_discovery.py b/tests/test_transfer_manual_review_discovery.py new file mode 100644 index 000000000..6d9011602 --- /dev/null +++ b/tests/test_transfer_manual_review_discovery.py @@ -0,0 +1,250 @@ +"""验证 durable 人工复核从未知证据到调度恢复的可发现闭环。""" + +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from app.api.endpoints import transfer as transfer_endpoint +from app.application.transfer_execution import ( + TransferExecutionCommand, + TransferExecutionState, + TransferManualReviewQuery, + TransferStepIntent, + TransferStepResult, +) +from app.db.adapters.transfer_execution import ( + TransactionalTransferExecutionRepository, +) +from app.db.base import Base +from app.db.models.transferexecutionstep import TransferExecutionStep +from app.db.models.transferhistory import TransferHistory +from app.db.models.transferpending import TransferPending +from app.schemas.transfer import TransferManualReviewRequest + + +@pytest.fixture +def review_store(): + """构造隔离的 durable 整理人工复核数据库。""" + engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[ + TransferPending.__table__, + TransferHistory.__table__, + TransferExecutionStep.__table__, + ], + ) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield factory + finally: + engine.dispose() + + +def _repository(factory) -> TransactionalTransferExecutionRepository: + """构造使用确定时钟的整理执行仓储。""" + return TransactionalTransferExecutionRepository( + factory, + local_clock=lambda: datetime(2026, 8, 27, 9, 30, 0), + lease_clock=lambda: datetime(2026, 8, 27, 1, 30, 0, tzinfo=timezone.utc), + ) + + +def _put_in_manual_review(factory, *, task_id: str) -> tuple[ + TransactionalTransferExecutionRepository, + str, +]: + """建立一个外部结果 UNKNOWN 且已释放租约的人工复核任务。""" + with factory() as session: + session.add(TransferPending( + task_id=task_id, + storage="local", + src_path=f"/downloads/{task_id}.mkv", + created_at="2026-08-27 09:00:00", + state="planned", + updated_at="2026-08-27 09:00:00", + input_version=1, + planning_input={"schema_version": 1, "source": task_id}, + input_fingerprint=f"input-{task_id}", + checkpoint_version=1, + checkpoint_payload={"schema_version": 1, "task_id": task_id}, + planned_at="2026-08-27 09:00:00", + lease_owner="worker-secret", + lease_token=f"lease-{task_id}", + lease_expires_at="2099-01-01 00:00:00.000000", + heartbeat_at="2026-08-27 01:00:00.000000", + attempt_count=1, + execution_state="not_started", + retry_generation=0, + retry_count=0, + settlement_revision=0, + )) + session.commit() + repository = _repository(factory) + command = TransferExecutionCommand( + repository, + attempt_token_factory=lambda: f"attempt-{task_id}", + ) + intent = TransferStepIntent.create( + task_id=task_id, + checkpoint_fingerprint=f"checkpoint-{task_id}", + ordinal=0, + phase="transfer", + kind="materialize_target", + payload={ + "source": f"/downloads/{task_id}.mkv", + "target": f"/library/{task_id}.mkv", + }, + ) + prepared = command.prepare( + task_id=task_id, + lease_token=f"lease-{task_id}", + intent=intent, + ) + started = command.begin( + task_id=task_id, + lease_token=f"lease-{task_id}", + operation_id=prepared.operation_id, + ) + snapshot = command.manual_review( + task_id=task_id, + lease_token=f"lease-{task_id}", + step=started, + error="external result unknown", + evidence=TransferStepResult(payload={ + "observation": "unknown", + "target_exists": True, + }), + ) + assert snapshot.state is TransferExecutionState.MANUAL_REVIEW + return repository, started.operation_id + + +def test_manual_review_list_is_database_paginated(review_store) -> None: + """待复核任务应按稳定顺序在数据库分页，不混入普通任务。""" + repository, _ = _put_in_manual_review(review_store, task_id="task-1") + _put_in_manual_review(review_store, task_id="task-2") + + first = repository.list_manual_reviews( + state=TransferExecutionState.MANUAL_REVIEW, + page=1, + page_size=1, + ) + second = repository.list_manual_reviews( + state=TransferExecutionState.MANUAL_REVIEW, + page=2, + page_size=1, + ) + + assert first.total == second.total == 2 + assert len(first.items) == len(second.items) == 1 + assert first.items[0].task_id != second.items[0].task_id + with pytest.raises(ValueError, match="不支持状态"): + TransferManualReviewQuery(repository).list( + state=TransferExecutionState.RUNNING, + ) + + +@pytest.mark.parametrize( + ("decision", "result_payload", "expected_step_state"), + [ + ("applied", {"target_exists": True, "hash_match": True}, "succeeded"), + ("not_applied", None, "failed"), + ], +) +def test_unknown_manual_review_is_discoverable_and_resumes_via_api( + monkeypatch, + review_store, + decision: str, + result_payload: dict[str, bool] | None, + expected_step_state: str, +) -> None: + """UNKNOWN 任务应可发现，人工判定后进入唯一 retry_wait 恢复路径。""" + repository, operation_id = _put_in_manual_review( + review_store, + task_id=f"task-{decision}", + ) + monkeypatch.setattr( + transfer_endpoint, + "get_chain_transfer_execution_port", + lambda: repository, + ) + + listed = transfer_endpoint.list_transfer_manual_reviews( + state_filter="manual_review", + page=1, + page_size=10, + current_user=object(), + ) + assert listed.data is not None + assert listed.data.total == 1 + discovered = listed.data.items[0] + assert discovered.task_id == f"task-{decision}" + assert discovered.source.model_dump() == { + "storage": "local", + "path": f"/downloads/task-{decision}.mkv", + } + assert discovered.step.operation_id == operation_id + assert discovered.step.kind == "materialize_target" + assert discovered.step.intent["target"] == f"/library/task-{decision}.mkv" + assert discovered.step.evidence == { + "observation": "unknown", + "target_exists": True, + } + assert discovered.step.error == "external result unknown" + assert discovered.review_revision == 0 + public_json = listed.model_dump_json() + assert "worker-secret" not in public_json + assert "lease-" not in public_json + assert "attempt-" not in public_json + + detail = transfer_endpoint.get_transfer_manual_review( + task_id=f"task-{decision}", + current_user=object(), + ) + assert detail.data == discovered + + resolved = transfer_endpoint.resolve_transfer_manual_review( + task_id=f"task-{decision}", + review=TransferManualReviewRequest( + operation_id=operation_id, + decision=decision, + reason=f"reviewed-{decision}", + result_payload=result_payload, + ), + current_user=SimpleNamespace(name="admin"), + ) + assert resolved.data is not None + assert resolved.data.state == "retry_wait" + assert resolved.data.review_revision == 1 + + waiting = transfer_endpoint.get_transfer_manual_review( + task_id=f"task-{decision}", + current_user=object(), + ) + assert waiting.data is not None + assert waiting.data.state == "retry_wait" + assert waiting.data.review_revision == 1 + retry_page = transfer_endpoint.list_transfer_manual_reviews( + state_filter="retry_wait", + page=1, + page_size=10, + current_user=object(), + ) + assert retry_page.data is not None + assert [item.task_id for item in retry_page.data.items] == [f"task-{decision}"] + + snapshot = repository.get_snapshot(task_id=f"task-{decision}") + assert snapshot is not None + assert snapshot.state is TransferExecutionState.RETRY_WAIT + assert snapshot.retry_due_at is not None + assert snapshot.steps[0].state.value == expected_step_state + with review_store() as session: + pending = session.scalar(select(TransferPending)) + assert pending is not None + assert pending.lease_owner is None + assert pending.lease_token is None + assert pending.retry_generation == 1 diff --git a/tests/test_transfer_overwrite_declined.py b/tests/test_transfer_overwrite_declined.py index 519ff0f79..c69cba4bc 100644 --- a/tests/test_transfer_overwrite_declined.py +++ b/tests/test_transfer_overwrite_declined.py @@ -11,6 +11,10 @@ from unittest.mock import MagicMock, patch import pytest +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferSettlementResult, +) from app.chain.transfer import TransferChain from app.schemas.transfer import TransferInfo from app.schemas.types import EventType @@ -121,6 +125,60 @@ def test_overwrite_declined_false_when_query_raises(): assert result is False +def test_overwrite_declined_uses_successful_durable_settlement(): + """覆盖拒绝保留旧成功历史时，durable 终态必须按成功结算。""" + task = make_task(1) + task.bind_admission_task_id("task-overwrite-declined") + task.bind_execution_lease(owner_id="worker", lease_token="lease") + task.bind_execution_checkpoint(TransferExecutionCheckpoint.create( + payload={"outcome": "overwrite_skipped"}, + operation_ids=(), + skip_reason="overwrite_declined", + )) + transferinfo = TransferInfo( + success=False, + overwrite_skipped=True, + message="目标已存在，按覆盖策略跳过覆盖", + ) + + settlement = TransferChain._TransferChain__build_transfer_result_settlement( + task, + transferinfo, + overwrite_declined=True, + ) + + assert settlement is not None + assert settlement.outcome == "succeeded" + assert settlement.error is None + + +def test_overwrite_skip_without_success_history_uses_failed_settlement(): + """未核实既有成功历史时，覆盖跳过标志不能伪造成功终态。""" + task = make_task(1) + task.bind_admission_task_id("task-overwrite-missing-history") + task.bind_execution_lease(owner_id="worker", lease_token="lease") + task.bind_execution_checkpoint(TransferExecutionCheckpoint.create( + payload={"outcome": "failed"}, + operation_ids=(), + skip_reason="overwrite_without_history", + )) + transferinfo = TransferInfo( + success=False, + overwrite_skipped=True, + message="目标已存在，按覆盖策略跳过覆盖", + ) + + settlement = TransferChain._TransferChain__build_transfer_result_settlement( + task, + transferinfo, + overwrite_declined=False, + ) + + assert settlement is not None + assert settlement.outcome == "failed" + assert settlement.error == transferinfo.message + + # --------------------------------------------------------------------------- # __default_callback 失败分支 # --------------------------------------------------------------------------- @@ -230,6 +288,65 @@ def test_default_callback_keeps_original_failure_semantics_without_success_histo assert len(transfer_failed_events) == 1 +def test_durable_callback_settles_overwrite_skip_without_history_as_failed(): + """durable 回调没有既有成功历史时必须原子提交失败历史和失败终态。""" + chain = make_transfer_chain() + chain.eventmanager = MagicMock() + chain.post_message = MagicMock() + chain.durable_event_writer = MagicMock() + task = _make_failed_task() + task.bind_admission_task_id("task-overwrite-no-history") + task.bind_execution_lease(owner_id="worker", lease_token="lease") + task.bind_execution_checkpoint(TransferExecutionCheckpoint.create( + payload={"outcome": "failed"}, + operation_ids=(), + skip_reason="overwrite_without_history", + )) + add_fail_calls = [] + transfer_history_oper = make_history_oper(history=None) + transferinfo = TransferInfo( + success=False, + fileitem=task.fileitem, + message="目标已存在，按覆盖策略跳过覆盖", + transfer_type="copy", + overwrite_skipped=True, + need_notify=False, + ) + + def durable_transfer_result(**kwargs): + """执行失败历史暂存并返回 task-aware 结算回执。""" + history = kwargs["stage_history"](SimpleNamespace()) + assert kwargs["settlement"].outcome == "failed" + return TransferSettlementResult( + history_id=history.id, + settlement_revision=1, + pending_deleted=False, + ) + + chain.durable_event_writer.transfer_result.side_effect = durable_transfer_result + with patch( + "app.chain.transfer.get_chain_transfer_history_port", + return_value=transfer_history_oper, + ), patch( + "app.chain.transfer.add_transfer_fail", + make_fail_recorder(add_fail_calls), + ), patch( + "app.runtime.config.settings.AI_AGENT_ENABLE", False + ), patch( + "app.runtime.config.settings.AI_AGENT_RETRY_TRANSFER", False + ): + state, errmsg = chain._TransferChain__default_callback(task, transferinfo) + + assert state is False + assert errmsg == transferinfo.message + assert len(add_fail_calls) == 1 + settlement = chain.durable_event_writer.transfer_result.call_args.kwargs[ + "settlement" + ] + assert settlement.outcome == "failed" + assert settlement.error == transferinfo.message + + def test_default_callback_delegates_primary_failure_to_durable_writer(): """正式上下文存在 writer 时，主要媒体失败历史和事件必须走同一事务端口。""" chain = make_transfer_chain() diff --git a/tests/test_transfer_pending_legacy_compat.py b/tests/test_transfer_pending_legacy_compat.py index 34061608b..7a5de43d7 100644 --- a/tests/test_transfer_pending_legacy_compat.py +++ b/tests/test_transfer_pending_legacy_compat.py @@ -1,14 +1,17 @@ """旧待整理 Oper 的精确插件兼容与租约 fencing 测试。""" import importlib +import inspect from collections.abc import Callable from typing import Any import pytest -from sqlalchemy import create_engine, select +from sqlalchemy import create_engine, select, update from sqlalchemy.orm import Session, sessionmaker from app.db import base as db_base +from app.db.models.transferexecutionstep import TransferExecutionStep +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending from app.db.oper.transferpending import TransferPendingOper as CanonicalTransferPendingOper from app.runtime.compat.manifest import MODULE_ALIASES @@ -19,6 +22,8 @@ def legacy_session_factory(tmp_path, monkeypatch): """为无 Session 兼容 Oper 提供独占事务，并在提交后保留返回快照。""" engine = create_engine(f"sqlite:///{tmp_path / 'legacy-transferpending.db'}") TransferPending.__table__.create(engine) + TransferExecutionStep.__table__.create(engine) + TransferHistory.__table__.create(engine) factory = sessionmaker(bind=engine) def run_transaction(operation: Callable[[Session], Any]) -> Any: @@ -49,6 +54,8 @@ def test_legacy_import_targets_private_sdk_facade() -> None: assert legacy is importlib.import_module(alias.target) assert legacy.__all__ == ["TransferPendingOper"] assert not hasattr(legacy, "TransferPending") + assert not hasattr(legacy, "TransferExecutionStep") + assert not hasattr(legacy, "TransferExecutionState") assert legacy.TransferPendingOper is not CanonicalTransferPendingOper for internal_method in ( "stage_admit", @@ -59,6 +66,55 @@ def test_legacy_import_targets_private_sdk_facade() -> None: assert not hasattr(legacy.TransferPendingOper, internal_method) +def test_legacy_oper_preserves_exact_public_method_abi() -> None: + """兼容门面只公开历史八方法，且位置参数与关键字参数边界保持不变。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper_type = legacy.TransferPendingOper + public_methods = { + name + for name, value in oper_type.__dict__.items() + if not name.startswith("_") and callable(value) + } + + assert public_methods == { + "register", + "list_by_state", + "list_by_states", + "get_by_identity", + "get_by_task_id", + "discard", + "list_all", + "clear", + } + assert str(inspect.signature(oper_type.register)) == ( + "(self, storage: str, src_path: str) -> " + "app.db.models.transferpending.TransferPending | None" + ) + assert str(inspect.signature(oper_type.list_by_state)) == ( + "(self, *, state: str, limit: int | None = 5000) -> " + "List[app.db.models.transferpending.TransferPending]" + ) + assert str(inspect.signature(oper_type.list_by_states)) == ( + "(self, *, states: tuple[str, ...], limit: int | None = 5000) -> " + "List[app.db.models.transferpending.TransferPending]" + ) + assert str(inspect.signature(oper_type.get_by_identity)) == ( + "(self, *, storage: str, src_path: str) -> " + "app.db.models.transferpending.TransferPending | None" + ) + assert str(inspect.signature(oper_type.get_by_task_id)) == ( + "(self, *, task_id: str) -> " + "app.db.models.transferpending.TransferPending | None" + ) + assert str(inspect.signature(oper_type.discard)) == ( + "(self, storage: str, src_path: str) -> int" + ) + assert str(inspect.signature(oper_type.list_all)) == ( + "(self, limit: int | None = 5000) -> List[Tuple[str, str]]" + ) + assert str(inspect.signature(oper_type.clear)) == "(self) -> int" + + def test_legacy_no_session_queries_preserve_historical_shapes( legacy_session_factory, ) -> None: @@ -96,10 +152,35 @@ def test_legacy_no_session_queries_preserve_historical_shapes( assert oper.list_by_states(states=()) == [] +def test_legacy_register_allows_new_task_for_previously_settled_source( + legacy_session_factory, +) -> None: + """旧插件登记入口允许同源文件形成新的合法任务世代。""" + with legacy_session_factory() as session: + session.add(TransferHistory( + transfer_task_id="settled-task", + transfer_settlement_revision=1, + src="/downloads/settled.mkv", + src_storage="local", + status=True, + )) + session.commit() + + legacy = importlib.import_module("app.db.transferpending_oper") + pending = legacy.TransferPendingOper().register( + "local", + "/downloads/settled.mkv", + ) + assert pending is not None + assert pending.task_id != "settled-task" + with legacy_session_factory() as session: + assert session.execute(select(TransferPending)).scalar_one().task_id == pending.task_id + + def test_legacy_mutations_never_delete_claimed_rows( legacy_session_factory, ) -> None: - """旧 discard/clear 只处理未 claim 行，有效或过期 token 均受保护。""" + """旧 discard/clear 只处理新鲜行，有效或过期 token 均受保护。""" legacy = importlib.import_module("app.db.transferpending_oper") oper = legacy.TransferPendingOper() active = oper.register("local", "/downloads/active.mkv") @@ -156,3 +237,204 @@ def test_legacy_mutations_never_delete_claimed_rows( (active.task_id, "active-token"), (expired.task_id, "expired-token"), ] + + +def test_legacy_mutations_protect_every_execution_and_terminal_state( + legacy_session_factory, +) -> None: + """无租约的运行、等待、结算、失败和人工判定任务仍归状态机所有。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper = legacy.TransferPendingOper() + protected = { + state: oper.register("local", f"/downloads/{state}.mkv") + for state in ( + "running", + "retry_wait", + "settling", + "failed", + "manual_review", + ) + } + expired = oper.register("local", "/downloads/expired-running.mkv") + safe_discard = oper.register("local", "/downloads/safe-discard.mkv") + safe_clear = oper.register("local", "/downloads/safe-clear.mkv") + assert all(protected.values()) + assert expired is not None + assert safe_discard is not None + assert safe_clear is not None + + with legacy_session_factory() as session: + for state, row in protected.items(): + assert row is not None + session.execute( + update(TransferPending) + .where(TransferPending.task_id == row.task_id) + .values(execution_state=state) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == expired.task_id) + .values( + execution_state="running", + lease_owner="expired-worker", + lease_token="expired-token", + lease_expires_at="2026-08-27 09:59:00.000000", + heartbeat_at="2026-08-27 09:58:00.000000", + attempt_count=1, + ) + ) + session.commit() + + assert oper.discard("local", safe_discard.src_path) == 1 + for state, row in protected.items(): + assert row is not None + assert oper.discard("local", row.src_path) == 0, state + assert oper.discard("local", expired.src_path) == 0 + assert oper.clear() == 1 + assert set(oper.list_all()) == { + ("local", row.src_path) + for row in (*protected.values(), expired) + if row is not None + } + + +def test_legacy_mutations_protect_execution_evidence_even_if_not_started( + legacy_session_factory, +) -> None: + """状态字段异常回退时，claim、步骤、重试和结算证据仍阻止旧接口删除。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper = legacy.TransferPendingOper() + evidence_rows = { + name: oper.register("local", f"/downloads/evidence-{name}.mkv") + for name in ( + "claim", + "checkpoint", + "retry", + "settlement", + "error", + "step", + "history", + ) + } + safe = oper.register("local", "/downloads/evidence-free.mkv") + assert all(evidence_rows.values()) + assert safe is not None + + with legacy_session_factory() as session: + rows = { + name: row + for name, row in evidence_rows.items() + if row is not None + } + session.execute( + update(TransferPending) + .where(TransferPending.task_id == rows["claim"].task_id) + .values(attempt_count=1) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == rows["checkpoint"].task_id) + .values( + execution_version=1, + execution_payload={"operation_ids": ["op-checkpoint"]}, + execution_fingerprint="f" * 64, + ) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == rows["retry"].task_id) + .values( + retry_generation=1, + retry_count=1, + retry_due_at="2026-08-27 12:00:00.000000", + ) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == rows["settlement"].task_id) + .values(settlement_revision=1, terminal_history_id=42) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == rows["error"].task_id) + .values(last_error="execution outcome unknown") + ) + session.add( + TransferExecutionStep( + task_id=rows["step"].task_id, + operation_id="op-step-evidence", + checkpoint_fingerprint="c" * 64, + ordinal=0, + phase="materialize", + kind="copy", + state="prepared", + attempt_count=0, + intent_version=1, + intent_payload={"src": rows["step"].src_path}, + prepared_at="2026-08-27 10:00:00.000000", + updated_at="2026-08-27 10:00:00.000000", + ) + ) + session.add( + TransferHistory( + transfer_task_id=rows["history"].task_id, + transfer_settlement_revision=1, + src=rows["history"].src_path, + src_storage="local", + status=True, + ) + ) + session.commit() + + for name, row in evidence_rows.items(): + assert row is not None + assert oper.discard("local", row.src_path) == 0, name + assert oper.clear() == 1 + assert set(oper.list_all()) == { + ("local", row.src_path) + for row in evidence_rows.values() + if row is not None + } + + +def test_legacy_register_returns_existing_terminal_or_manual_review_task( + legacy_session_factory, +) -> None: + """重复登记不得绕过失败或人工判定任务创建平行执行身份。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper = legacy.TransferPendingOper() + failed = oper.register("local", "/downloads/retry-failed.mkv") + manual = oper.register("local", "/downloads/retry-manual.mkv") + assert failed is not None + assert manual is not None + + with legacy_session_factory() as session: + session.execute( + update(TransferPending) + .where(TransferPending.task_id == failed.task_id) + .values( + execution_state="failed", + settlement_revision=1, + terminal_history_id=81, + ) + ) + session.execute( + update(TransferPending) + .where(TransferPending.task_id == manual.task_id) + .values( + execution_state="manual_review", + last_error="provider outcome unknown", + ) + ) + session.commit() + + repeated_failed = oper.register("local", failed.src_path) + repeated_manual = oper.register("local", manual.src_path) + assert repeated_failed is not None + assert repeated_failed.task_id == failed.task_id + assert repeated_failed.execution_state == "failed" + assert repeated_failed.terminal_history_id == 81 + assert repeated_manual is not None + assert repeated_manual.task_id == manual.task_id + assert repeated_manual.execution_state == "manual_review" + assert repeated_manual.last_error == "provider outcome unknown" diff --git a/tests/test_transfer_planning_checkpoint.py b/tests/test_transfer_planning_checkpoint.py index c72e3c9f0..83948261a 100644 --- a/tests/test_transfer_planning_checkpoint.py +++ b/tests/test_transfer_planning_checkpoint.py @@ -14,8 +14,13 @@ from sqlalchemy.orm import sessionmaker from app.application import transfer as transfer_application from app.application.transfer import TransferTask +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferSettlementResult, +) from app.chain.transfer import TransferChain from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending from app.domain.context import MediaInfo from app.domain.meta.metabase import MetaBase @@ -831,7 +836,7 @@ def test_provider_pending_crash_replay_executes_snapshot_without_host_planning() def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): - """旧同步调用必须经过 admission、checkpoint、执行和终态注销。""" + """旧同步调用必须经过 admission、checkpoint、执行和原子终态结算。""" calls = [] repository = Mock() repository.admit.side_effect = ( @@ -845,18 +850,47 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): return SimpleNamespace(checkpoint=kwargs["checkpoint"]) repository.checkpoint_plan.side_effect = checkpoint_plan - repository.discard_claimed.side_effect = ( - lambda **_kwargs: calls.append("discard") or 1 - ) result = TransferInfo( success=True, fileitem=_task().fileitem, transfer_type="copy", ) chain = _chain(repository=repository, checkpoint=_checkpoint(), result=result) - chain.execute_transfer_plan.side_effect = ( - lambda *_args, **_kwargs: calls.append("execute") or result + execution_checkpoint = TransferExecutionCheckpoint.create( + payload={ + "outcome": "succeeded", + "transferinfo": result.model_dump(mode="json"), + }, + operation_ids=("operation-legacy-command",), ) + step_runner = Mock() + step_runner.checkpoint.return_value = execution_checkpoint + chain._TransferChain__build_durable_step_runner = Mock(return_value=step_runner) + chain.run_module = Mock( + side_effect=lambda *_args, **_kwargs: calls.append("execute") or result + ) + chain.durable_event_writer = Mock() + + def settle_result(**kwargs): + """执行历史暂存并模拟 writer 返回 task-aware 结算投影。""" + staging = Mock() + staging.add_force.return_value = SimpleNamespace( + id=31, + status=True, + src=result.fileitem.path, + src_storage=result.fileitem.storage, + src_fileitem=result.fileitem.model_dump(mode="json"), + ) + history = kwargs["stage_history"](staging) + assert history.status is True + calls.append("settle") + return TransferSettlementResult( + history_id=history.id, + settlement_revision=1, + pending_deleted=True, + ) + + chain.durable_event_writer.transfer_result.side_effect = settle_result meta = MetaBase("Movie.2026.mkv") mediainfo = MediaInfo() @@ -870,11 +904,13 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): ) assert returned is result - assert calls == ["admit", "checkpoint", "execute", "discard"] - repository.discard_claimed.assert_called_once_with( - task_id="task-legacy-command", - lease_token="lease-task-legacy-command", - ) + assert calls == ["admit", "checkpoint", "execute", "settle"] + repository.discard_claimed.assert_not_called() + writer_call = chain.durable_event_writer.transfer_result.call_args.kwargs + assert writer_call["topic"] is None + assert writer_call["publish"] is None + assert writer_call["settlement"].task_id == "task-legacy-command" + assert writer_call["settlement"].outcome == "succeeded" def test_cleanup_destination_is_idempotent_and_uses_storage_safety_policy(): @@ -932,6 +968,7 @@ def test_planning_payload_round_trip_is_self_contained_and_stable(): def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_path): """checkpoint 内嵌输入与 accepted 指纹不一致时必须拒绝状态跃迁。""" engine = create_engine(f"sqlite:///{tmp_path / 'fingerprint.db'}") + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) accepted_input = _planning_input(target_path="/library/A") @@ -975,6 +1012,7 @@ def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path): """仓储必须同时恢复 accepted 输入和 planned 自包含 checkpoint。""" engine = create_engine(f"sqlite:///{tmp_path / 'recoverable.db'}") + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) planning_input = _planning_input() diff --git a/tests/test_transfer_planning_migration.py b/tests/test_transfer_planning_migration.py index 20cb877f2..f37a1e5cd 100644 --- a/tests/test_transfer_planning_migration.py +++ b/tests/test_transfer_planning_migration.py @@ -26,6 +26,24 @@ except ModuleNotFoundError: PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14" LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15" +POST_LEASE_EXECUTION_COLUMNS = { + "execution_state", + "execution_version", + "execution_payload", + "execution_fingerprint", + "retry_generation", + "retry_count", + "retry_due_at", + "retry_requested_by", + "retry_reason", + "settlement_revision", + "terminal_history_id", + "manual_review_revision", + "reviewed_at", + "reviewed_by", + "review_reason", + "review_decision", +} def _bind_migration(monkeypatch, connection, module_name=PLANNING_MIGRATION): @@ -104,7 +122,11 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: assert { column["name"] for column in inspector.get_columns("transferpending") - } == {column.name for column in TransferPending.__table__.columns} + } == { + column.name + for column in TransferPending.__table__.columns + if column.name not in POST_LEASE_EXECUTION_COLUMNS + } upgraded = _planning_row(connection) planning_payload = upgraded["planning_input"] if isinstance(planning_payload, str): @@ -168,7 +190,11 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: assert { column["name"] for column in sa.inspect(connection).get_columns("transferpending") - } == {column.name for column in TransferPending.__table__.columns} + } == { + column.name + for column in TransferPending.__table__.columns + if column.name not in POST_LEASE_EXECUTION_COLUMNS + } def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None: diff --git a/tests/test_transfer_planning_persistence.py b/tests/test_transfer_planning_persistence.py index 043d5e2e9..6df131c74 100644 --- a/tests/test_transfer_planning_persistence.py +++ b/tests/test_transfer_planning_persistence.py @@ -20,6 +20,7 @@ from app.application.transfer import ( TransferProviderReference, ) from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending @@ -27,6 +28,7 @@ from app.db.models.transferpending import TransferPending def repository(tmp_path): """创建只服务单个测试的 SQLite 整理计划仓储。""" engine = create_engine(f"sqlite:///{tmp_path / 'transfer-planning.db'}") + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) @@ -347,6 +349,30 @@ def test_admit_reuses_identical_input_and_rejects_conflict(repository) -> None: ) +def test_admit_allows_new_generation_when_history_has_previous_task(repository) -> None: + """历史保留上一代任务投影时，同源新事实仍可形成新任务世代。""" + with repository._session_factory() as session: # noqa: SLF001 + session.add(TransferHistory( + transfer_task_id="settled-task", + transfer_settlement_revision=1, + src="/downloads/Movie.2026.mkv", + src_storage="local", + status=True, + )) + session.commit() + + admission = repository.admit( + storage="local", + src_path="/downloads/Movie.2026.mkv", + planning_input=_planning_input(), + ) + + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute(select(TransferPending)).scalar_one() + assert admission.task_id == pending.task_id + assert admission.task_id != "settled-task" + + def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None: """完整计划和 planned 状态应同事务提交且允许相同检查点重试。""" planning_input = _planning_input() @@ -556,6 +582,7 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) - """列版本、JSON 和指纹任一不一致时都不得返回伪冻结 DTO。""" engine = create_engine(f"sqlite:///{tmp_path / 'input-corruption.db'}") factory = sessionmaker(bind=engine) + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) repository = TransactionalTransferAdmissionRepository(factory) planning_input = _planning_input() @@ -601,6 +628,7 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None: """planned 行的列版本与自包含 checkpoint JSON 必须严格一致。""" engine = create_engine(f"sqlite:///{tmp_path / 'checkpoint-corruption.db'}") factory = sessionmaker(bind=engine) + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) repository = TransactionalTransferAdmissionRepository(factory) planning_input = _planning_input() diff --git a/tests/test_transfer_queue_service.py b/tests/test_transfer_queue_service.py index 36bfd6f14..cdc8bb547 100644 --- a/tests/test_transfer_queue_service.py +++ b/tests/test_transfer_queue_service.py @@ -7,6 +7,7 @@ from sqlalchemy.orm import sessionmaker from app.application.transfer import TransferAdmission, TransferQueueService from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory from app.db.models.transferpending import TransferPending from app.schemas.file import FileItem from tests.test_transfer_job_manager import make_task, make_transfer_chain @@ -117,6 +118,7 @@ def test_transfer_queue_service_cleans_up_when_batch_registration_fails(): def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path): """真实仓储已提交后即使内存入队失败，任务也必须带原因留待恢复。""" engine = create_engine(f"sqlite:///{tmp_path / 'durable-admission.db'}") + TransferHistory.__table__.create(engine) TransferPending.__table__.create(engine) factory = sessionmaker(bind=engine) repository = TransactionalTransferAdmissionRepository(factory) diff --git a/tests/test_transfer_settling_recovery.py b/tests/test_transfer_settling_recovery.py new file mode 100644 index 000000000..a29669d36 --- /dev/null +++ b/tests/test_transfer_settling_recovery.py @@ -0,0 +1,456 @@ +"""验证 settling 终态在崩溃后只重放持久结算。""" + +import threading +import time +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import sessionmaker + +from app.application.transfer import ( + TransferAdmission, + TransferPlanCheckpoint, + TransferPlanningInput, + TransferTask, +) +from app.application.transfer_execution import ( + TransferExecutionCheckpoint, + TransferExecutionSnapshot, + TransferExecutionState, +) +from app.chain.transfer import TransferChain +from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferhistory import TransferHistory +from app.db.models.transferpending import TransferPending +from app.schemas.file import FileItem +from app.schemas.transfer import TransferInfo + + +def _planning_input(path: str) -> TransferPlanningInput: + """构造保留源文件身份的最小持久规划输入。""" + return TransferPlanningInput( + source_fileitem={ + "storage": "local", + "path": path, + "type": "file", + "name": Path(path).name, + "basename": Path(path).stem, + "extension": Path(path).suffix.lstrip("."), + "size": 1024, + }, + meta=None, + mediainfo=None, + requested_transfer_type="move", + ) + + +def _plan_checkpoint( + planning_input: TransferPlanningInput, +) -> TransferPlanCheckpoint: + """构造外部步骤已经结束后可直接结算的冻结计划。""" + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="local", + root_target_path="/library", + final_target_path="/library/Movie.mkv", + resolved_transfer_type="move", + items=(), + need_notify=False, + skip_reason="测试已完成外部步骤", + ) + + +def _transfer_result(path: str, *, success: bool) -> TransferInfo: + """构造可完整写入 execution checkpoint 的整理结果。""" + fileitem = FileItem( + storage="local", + path=path, + type="file", + name=Path(path).name, + basename=Path(path).stem, + extension=Path(path).suffix.lstrip("."), + size=1024, + ) + return TransferInfo( + success=success, + fileitem=fileitem, + target_item=( + FileItem( + storage="local", + path="/library/Movie.mkv", + type="file", + name="Movie.mkv", + ) + if success + else None + ), + transfer_type="move", + fail_list=[] if success else [path], + message="整理完成" if success else "整理失败", + need_notify=False, + ) + + +def _execution_checkpoint( + path: str, + *, + success: bool, + include_transferinfo: bool = True, +) -> TransferExecutionCheckpoint: + """构造成功或确定失败的聚合执行检查点。""" + payload = { + "outcome": "succeeded" if success else "failed", + "error": None if success else "整理失败", + } + if include_transferinfo: + payload["transferinfo"] = _transfer_result( + path, + success=success, + ).model_dump(mode="json") + return TransferExecutionCheckpoint.create( + payload=payload, + operation_ids=("operation-1",), + ) + + +def _add_settling_pending( + factory, + *, + path: str, + lease_state: str, +) -> tuple[TransferPlanCheckpoint, TransferExecutionCheckpoint]: + """写入带完整计划和执行检查点的 settling 任务。""" + planning_input = _planning_input(path) + plan_checkpoint = _plan_checkpoint(planning_input) + execution_checkpoint = _execution_checkpoint(path, success=True) + lease_values = { + "lease_owner": None, + "lease_token": None, + "lease_expires_at": None, + "heartbeat_at": None, + "attempt_count": 0, + } + if lease_state == "expired": + lease_values.update({ + "lease_owner": "old-owner", + "lease_token": "old-token", + "lease_expires_at": "2000-01-01 00:00:00.000000", + "heartbeat_at": "1999-12-31 23:59:00.000000", + "attempt_count": 1, + }) + with factory() as session: + session.add(TransferPending( + task_id="settling-task", + storage="local", + src_path=path, + created_at="2026-08-27 09:00:00", + state="planned", + updated_at="2026-08-27 09:00:00", + input_version=planning_input.schema_version, + planning_input=planning_input.to_payload(), + input_fingerprint=planning_input.fingerprint, + checkpoint_version=plan_checkpoint.schema_version, + checkpoint_payload=plan_checkpoint.to_payload(), + planned_at="2026-08-27 09:00:00", + execution_state="settling", + execution_version=execution_checkpoint.version, + execution_payload=execution_checkpoint.to_payload(), + execution_fingerprint=execution_checkpoint.fingerprint, + retry_generation=0, + retry_count=0, + settlement_revision=0, + **lease_values, + )) + session.commit() + return plan_checkpoint, execution_checkpoint + + +@pytest.fixture +def admission_store(tmp_path): + """创建独立 SQLite admission 仓储及其 Session 工厂。""" + engine = create_engine(f"sqlite:///{tmp_path / 'settling-recovery.db'}") + TransferHistory.__table__.create(engine) + TransferPending.__table__.create(engine) + factory = sessionmaker(bind=engine, expire_on_commit=False) + try: + yield TransactionalTransferAdmissionRepository(factory), factory + finally: + engine.dispose() + + +def _snapshot( + checkpoint: TransferExecutionCheckpoint, +) -> TransferExecutionSnapshot: + """构造 settling 状态的脱离 Session 执行投影。""" + return TransferExecutionSnapshot( + task_id="settling-task", + state=TransferExecutionState.SETTLING, + checkpoint=checkpoint, + retry_generation=0, + retry_count=0, + retry_due_at=None, + settlement_revision=0, + terminal_history_id=None, + last_error=None, + steps=(), + ) + + +def _build_chain(admissions) -> TransferChain: + """构造只允许执行 settling 终态恢复的 TransferChain 骨架。""" + chain = object.__new__(TransferChain) + chain._transfer_admissions = admissions + chain._worker_owner_id = "recovery-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + chain._worker_state_lock = threading.RLock() + chain._closing = False + chain._recovery_wakeup_event = threading.Event() + chain._replay_stop_event = threading.Event() + chain._lease_heartbeat_stop_event = threading.Event() + chain._lease_heartbeat_thread = None + chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock() + chain._TransferChain__ensure_recovery_scheduler = MagicMock() + chain._TransferChain__restore_planned_task = MagicMock() + chain._TransferChain__select_storage_oper = MagicMock( + side_effect=AssertionError("settling 恢复不得选择存储适配器") + ) + chain._plan_checkpoint_and_execute = MagicMock( + side_effect=AssertionError("settling 恢复不得重新执行计划") + ) + chain.jobview = MagicMock() + return chain + + +def _recovered_task( + chain: TransferChain, + admission: TransferAdmission, + execution_checkpoint: TransferExecutionCheckpoint, +) -> TransferTask: + """把 claim 投影绑定为只待终态 writer 处理的恢复任务。""" + assert admission.planning_input is not None + assert admission.checkpoint is not None + assert admission.lease_owner is not None + assert admission.lease_token is not None + task = TransferTask( + fileitem=FileItem.model_validate( + admission.planning_input.source_fileitem + ) + ) + task.bind_admission_task_id(admission.task_id) + task.bind_planning_input(admission.planning_input) + task.bind_plan_checkpoint(admission.checkpoint) + task.bind_execution_checkpoint(execution_checkpoint) + task.bind_execution_lease( + owner_id=admission.lease_owner, + lease_token=admission.lease_token, + ) + chain._owned_leases[admission.task_id] = ( + admission.lease_token, + time.monotonic() + 120, + ) + return task + + +@pytest.mark.parametrize("lease_state", ["missing", "expired"]) +def test_settling_task_can_be_claimed_by_only_one_owner( + admission_store, + lease_state, +) -> None: + """空租约和过期租约的 settling 任务都只能由一个 worker 取得。""" + repository, factory = admission_store + _add_settling_pending( + factory, + path="/downloads/Movie.mkv", + lease_state=lease_state, + ) + + claimed = repository.claim_recoverable( + owner_id="first-owner", + limit=1, + lease_seconds=120, + ) + competing = repository.claim_recoverable( + owner_id="second-owner", + limit=1, + lease_seconds=120, + ) + + assert len(claimed) == 1 + assert claimed[0].lease_owner == "first-owner" + assert competing == [] + with factory() as session: + pending = session.execute(select(TransferPending)).scalar_one() + assert pending.execution_state == "settling" + assert pending.lease_token == claimed[0].lease_token + + +@pytest.mark.parametrize( + ("success", "include_transferinfo"), + [(True, True), (False, False)], +) +def test_settling_result_calls_only_task_aware_terminal_writer( + success, + include_transferinfo, +) -> None: + """成功和确定失败都只从检查点恢复结果并调用携带 task 的 writer。""" + path = "/downloads/Movie.mkv" + planning_input = _planning_input(path) + plan_checkpoint = _plan_checkpoint(planning_input) + execution_checkpoint = _execution_checkpoint( + path, + success=success, + include_transferinfo=include_transferinfo, + ) + admission = TransferAdmission( + task_id="settling-task", + storage="local", + src_path=path, + state="planned", + created_at="2026-08-27 09:00:00", + updated_at="2026-08-27 09:00:00", + planning_input=planning_input, + checkpoint=plan_checkpoint, + lease_owner="recovery-owner", + lease_token="recovery-token", + ) + chain = _build_chain(MagicMock()) + task = _recovered_task(chain, admission, execution_checkpoint) + writer_calls = [] + + def terminal_writer( + callback_task: TransferTask, + transferinfo: TransferInfo, + ) -> tuple[bool, str]: + """记录 task-aware 终态 writer 收到的恢复事实。""" + writer_calls.append((callback_task, transferinfo)) + return transferinfo.success, transferinfo.message or "" + + result = chain._TransferChain__handle_planned_transfer( + task, + terminal_writer, + ) + + assert result[0] is success + assert len(writer_calls) == 1 + assert writer_calls[0][0] is task + assert writer_calls[0][0].execution_checkpoint == execution_checkpoint + assert writer_calls[0][1].success is success + chain._TransferChain__select_storage_oper.assert_not_called() + chain._plan_checkpoint_and_execute.assert_not_called() + + +def test_replay_settling_uses_frozen_source_without_filesystem_probe( + monkeypatch, +) -> None: + """move 后源文件已消失时，settling 回放仍应直接入队结算。""" + path = "/already-moved/Movie.mkv" + planning_input = _planning_input(path) + plan_checkpoint = _plan_checkpoint(planning_input) + execution_checkpoint = _execution_checkpoint(path, success=True) + admission = TransferAdmission( + task_id="settling-task", + storage="local", + src_path=path, + state="planned", + created_at="2026-08-27 09:00:00", + updated_at="2026-08-27 09:00:00", + planning_input=planning_input, + checkpoint=plan_checkpoint, + lease_owner="recovery-owner", + lease_token="recovery-token", + ) + admissions = MagicMock() + admissions.claim_recoverable.return_value = [admission] + executions = MagicMock() + executions.get_snapshot.return_value = _snapshot(execution_checkpoint) + chain = _build_chain(admissions) + chain._transfer_executions = executions + chain.put_to_queue = MagicMock(return_value=True) + + def reject_stat(*_args, **_kwargs): + """任何源文件探测都表示 settling 恢复走回了旧执行路径。""" + pytest.fail("settling 恢复不得探测已经移动的源文件") + + monkeypatch.setattr(Path, "stat", reject_stat) + + chain._TransferChain__replay_pending() + + queued_task = chain.put_to_queue.call_args.args[0] + assert queued_task.fileitem.path == path + assert queued_task.execution_checkpoint == execution_checkpoint + admissions.discard_claimed.assert_not_called() + admissions.release_claim.assert_not_called() + chain._plan_checkpoint_and_execute.assert_not_called() + + +def test_writer_failure_releases_and_reclaims_same_settling_checkpoint( + admission_store, +) -> None: + """writer 临时失败后释放租约，再次恢复不得重新执行外部步骤。""" + repository, factory = admission_store + _, execution_checkpoint = _add_settling_pending( + factory, + path="/downloads/Movie.mkv", + lease_state="missing", + ) + first_admission = repository.claim_recoverable( + owner_id="recovery-owner", + limit=1, + lease_seconds=120, + )[0] + chain = _build_chain(repository) + first_task = _recovered_task( + chain, + first_admission, + execution_checkpoint, + ) + + def unavailable_writer(*_args, **_kwargs): + """模拟历史与 pending 原子 writer 的暂时性数据库失败。""" + raise RuntimeError("writer temporarily unavailable") + + with pytest.raises(RuntimeError, match="temporarily unavailable"): + chain._TransferChain__handle_planned_transfer( + first_task, + unavailable_writer, + ) + assert chain._TransferChain__release_task_claim( + first_task, + error="writer temporarily unavailable", + ) + + chain._worker_owner_id = "second-recovery-owner" + second_admission = repository.claim_recoverable( + owner_id="second-recovery-owner", + limit=1, + lease_seconds=120, + )[0] + second_task = _recovered_task( + chain, + second_admission, + execution_checkpoint, + ) + settled = [] + + def available_writer( + callback_task: TransferTask, + transferinfo: TransferInfo, + ) -> tuple[bool, str]: + """模拟下一轮恢复时恢复正常的 task-aware writer。""" + settled.append((callback_task, transferinfo)) + return transferinfo.success, transferinfo.message or "" + + result = chain._TransferChain__handle_planned_transfer( + second_task, + available_writer, + ) + + assert result == (True, "整理完成") + assert len(settled) == 1 + assert settled[0][0].execution_checkpoint == execution_checkpoint + assert second_admission.lease_token != first_admission.lease_token + chain._TransferChain__select_storage_oper.assert_not_called() + chain._plan_checkpoint_and_execute.assert_not_called()