mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-04 23:17:20 +08:00
refactor: complete durable transfer execution settlement
This commit is contained in:
@@ -39,10 +39,11 @@ task_types:
|
||||
- "Use `query_transfer_history` with status='failed' to find the record with id={history_id} and understand the failure details such as source path, error message, and media info."
|
||||
- "Analyze the error message to determine the best retry strategy."
|
||||
- "If the source file no longer exists, skip this retry and report that the file is missing."
|
||||
- "Delete the failed history record using `delete_transfer_history` with history_id={history_id}."
|
||||
- "Re-identify the media using `recognize_media` with the source file path. For audio files, set media_type='music' and preserve artist/title/album context."
|
||||
- "If recognition fails, try `search_media` with keywords from the filename. For music, distinguish recording, album, and browse-only artist results."
|
||||
- "Re-transfer using `transfer_file` with the source path and exact identity fields. Reuse media_source + media_id for every media type, plus media_type + music_type for music."
|
||||
- "Call `delete_transfer_history` with history_id={history_id}. Durable records are not deleted: the tool submits them to the persistent retry scheduler."
|
||||
- "If `delete_transfer_history` reports that a durable retry was accepted or rejected, stop and report the exact scheduler result. Do not call `transfer_file`; do not delete the target, history, or retry evidence."
|
||||
- "Only for a legacy history that was actually deleted, re-identify the media using `recognize_media` with the source file path. For audio files, set media_type='music' and preserve artist/title/album context."
|
||||
- "If legacy recognition fails, try `search_media` with keywords from the filename. For music, distinguish recording, album, and browse-only artist results."
|
||||
- "Re-transfer only the deleted legacy history using `transfer_file` with the source path and exact identity fields. Reuse media_source + media_id for every media type, plus media_type + music_type for music."
|
||||
- "Report the final result."
|
||||
batch_transfer_failed_retry:
|
||||
header: "[System Task - Batch Transfer Failed Retry]"
|
||||
@@ -56,7 +57,7 @@ task_types:
|
||||
- "Use `query_transfer_history` with status='failed' to find all records with these IDs and understand the failure details."
|
||||
- "Group records by exact media identity and source directory before retrying. Do not assume all selected files belong to one media."
|
||||
- "If the error is about media recognition, identify each group once using `recognize_media` or `search_media`, then reuse that result inside the group. Album tracks should normally be retried from the shared album directory with the album identity."
|
||||
- "For each failed record, delete the old history entry with `delete_transfer_history` and re-transfer using `transfer_file`."
|
||||
- "For each failed record, call `delete_transfer_history`. Durable records are submitted to the persistent retry scheduler, and that result is final for this task: report its exact accepted or rejected state and do not call `transfer_file` or delete its evidence. Only an actually deleted legacy history may be re-transferred with `transfer_file`."
|
||||
- "Report how many retries succeeded and how many still failed."
|
||||
task_rules:
|
||||
- "Within one verified group, do NOT call `recognize_media` or `search_media` repeatedly for each file. A music recording is one track; a music album is one multi-track directory; an artist is never a transfer target."
|
||||
@@ -90,8 +91,9 @@ task_types:
|
||||
- "If the current recognition is wrong or the record should be reorganized, determine the correct media identity first."
|
||||
- "Prefer `recognize_media` with the source path. If recognition is not reliable, use `search_media` with keywords from filename, title, or year."
|
||||
- "Only continue when you have high confidence in the target media."
|
||||
- "Before re-organizing, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file."
|
||||
- "Then use `transfer_file` to organize the source path directly."
|
||||
- "Before re-organizing, call `delete_transfer_history`. Durable records are not deleted: the tool submits them to the persistent retry scheduler."
|
||||
- "If `delete_transfer_history` reports that a durable retry was accepted or rejected, stop for that record and report the exact scheduler result. Do not call `transfer_file`; do not delete the target, history, or retry evidence. Only a legacy history deletion may be followed by `transfer_file`."
|
||||
- "For a legacy history that was actually deleted, use `transfer_file` to organize the source path directly."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. For an album, retry the album directory once with the album identity when the records share that directory."
|
||||
- "If this record is already correct and no re-organize is needed, do not perform destructive actions; simply report that no change is necessary."
|
||||
task_rules:
|
||||
@@ -116,8 +118,9 @@ task_types:
|
||||
- "For each group, decide whether the current recognition is trustworthy."
|
||||
- "If multiple records clearly belong to the same movie, series, or music album, identify the media once with `recognize_media` or `search_media`, then reuse that result for the related records. A recording remains a single-track target, and an artist is browse-only."
|
||||
- "If a source file no longer exists or cannot be safely processed, skip that record and note the reason."
|
||||
- "Before re-organizing a record, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file."
|
||||
- "Then use `transfer_file` to organize the source path directly."
|
||||
- "Before re-organizing each record, call `delete_transfer_history`. Durable records are not deleted: the tool submits each task to the persistent retry scheduler."
|
||||
- "For each durable result, report the exact accepted or rejected scheduler state and stop processing that record. Do not call `transfer_file`; do not delete its target, history, or retry evidence."
|
||||
- "Only after a legacy history was actually deleted, use `transfer_file` to organize that source path directly."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. Prefer one directory transfer for a verified complete album instead of treating each track as an unrelated media item."
|
||||
- "If a record is already correct and no re-organize is needed, do not perform destructive actions; simply mark it as skipped."
|
||||
- "Report only the aggregate outcome, including how many records succeeded, skipped, and failed."
|
||||
|
||||
@@ -7,8 +7,13 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.tools.base import MoviePilotTool
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.chain.storage import StorageChain
|
||||
from app.application.agentdata import get_agent_transfer_history_port
|
||||
from app.application.chain.data import get_chain_transfer_execution_port
|
||||
from app.application.transfer_execution import (
|
||||
TransferExecutionCommand,
|
||||
TransferRetryRequestResult,
|
||||
)
|
||||
from app.chain.storage import StorageChain
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.workflow import FileItem
|
||||
|
||||
@@ -29,6 +34,22 @@ def _delete_history_destination_file(fileitem: FileItem) -> tuple[bool, bool]:
|
||||
return True, bool(storage_chain.delete_media_file(fileitem))
|
||||
|
||||
|
||||
def _request_transfer_retry(
|
||||
*,
|
||||
history_id: int,
|
||||
task_id: str,
|
||||
user_id: str,
|
||||
) -> TransferRetryRequestResult:
|
||||
"""在线程池中登记 durable 重试,避免 Agent 事件循环执行同步数据库 I/O。"""
|
||||
return TransferExecutionCommand(
|
||||
get_chain_transfer_execution_port()
|
||||
).request_retry(
|
||||
task_id=task_id,
|
||||
reason=f"Agent 请求重试整理历史 #{history_id}",
|
||||
requested_by=f"agent:{user_id or 'unknown'}",
|
||||
)
|
||||
|
||||
|
||||
class DeleteTransferHistoryTool(MoviePilotTool):
|
||||
name: str = "delete_transfer_history"
|
||||
tags: list[str] = [
|
||||
@@ -37,9 +58,10 @@ class DeleteTransferHistoryTool(MoviePilotTool):
|
||||
ToolTag.Admin,
|
||||
]
|
||||
description: str = (
|
||||
"Delete a specific transfer history record by its ID. For non-successful-move records with an old "
|
||||
"destination file, the tool removes that media-library file before deleting the history record. This is "
|
||||
"useful before retrying or re-organizing because the system skips files that already have transfer history."
|
||||
"Request a safe retry for durable transfer history, or delete a legacy transfer history record by its ID. "
|
||||
"Durable records keep their files and history and are retried only by the persistent scheduler. For legacy "
|
||||
"non-successful-move records, the tool removes the old destination before deleting the history. If a durable "
|
||||
"retry is accepted or rejected, stop and report that result; do not call transfer_file for the same record."
|
||||
)
|
||||
args_schema: Type[BaseModel] = DeleteTransferHistoryInput
|
||||
require_admin: bool = True
|
||||
@@ -58,6 +80,22 @@ class DeleteTransferHistoryTool(MoviePilotTool):
|
||||
if not history:
|
||||
return f"错误:整理历史记录不存在,ID={history_id}"
|
||||
|
||||
task_id = getattr(history, "transfer_task_id", None)
|
||||
if task_id:
|
||||
retry = await self.run_blocking(
|
||||
"db",
|
||||
_request_transfer_retry,
|
||||
history_id=history_id,
|
||||
task_id=task_id,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
outcome = "已登记" if retry.accepted else "未登记"
|
||||
return (
|
||||
f"durable 整理任务{outcome}重试:ID={history_id},"
|
||||
f"task_id={task_id},state={retry.state.value},{retry.message}。"
|
||||
"已保留目标文件、历史记录和失败计数;不要调用 transfer_file。"
|
||||
)
|
||||
|
||||
title = history.title or "未知"
|
||||
src = history.src or "未知"
|
||||
status = "成功" if history.status else "失败"
|
||||
|
||||
+189
-10
@@ -1,3 +1,4 @@
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Coroutine
|
||||
from typing import Any, Callable, List, Optional
|
||||
@@ -27,12 +28,17 @@ from app.api.dependencies.history import (
|
||||
)
|
||||
from app.api.response import ResponseAPIRouter
|
||||
from app.application.agent import get_running_agent_manager
|
||||
from app.application.chain.data import get_chain_transfer_execution_port
|
||||
from app.application.configuration import ApiRuntimeConfig
|
||||
from app.application.history import (
|
||||
DownloadHistoryMutationCommand,
|
||||
HistoryQueryService,
|
||||
TransferHistoryMutationCommand,
|
||||
)
|
||||
from app.application.transfer_execution import (
|
||||
TransferExecutionCommand,
|
||||
TransferRetryRequestResult,
|
||||
)
|
||||
from app.runtime.config import global_vars
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.progress import AsyncProgressHelper
|
||||
@@ -58,6 +64,120 @@ def normalize_history_ids(history_ids: list[int]) -> list[int]:
|
||||
return normalized_ids
|
||||
|
||||
|
||||
def _request_durable_transfer_retry(
|
||||
*,
|
||||
history_id: int,
|
||||
task_id: str,
|
||||
requested_by: str,
|
||||
) -> TransferRetryRequestResult:
|
||||
"""把 durable 历史重试交给唯一持久调度器,不在请求线程执行整理。"""
|
||||
return TransferExecutionCommand(
|
||||
get_chain_transfer_execution_port()
|
||||
).request_retry(
|
||||
task_id=task_id,
|
||||
reason=f"AI REST 请求重试整理历史 #{history_id}",
|
||||
requested_by=requested_by,
|
||||
)
|
||||
|
||||
|
||||
def _format_retry_rejections(
|
||||
rejections: list[tuple[int, TransferRetryRequestResult]],
|
||||
) -> str:
|
||||
"""把批量 durable 重试拒绝原因格式化为可审计的接口提示。"""
|
||||
return ";".join(
|
||||
f"#{history_id} [{result.state.value}]: {result.message}"
|
||||
for history_id, result in rejections
|
||||
)
|
||||
|
||||
|
||||
def _partition_durable_histories(
|
||||
histories: list[_SchemaTransferHistory],
|
||||
) -> tuple[list[_SchemaTransferHistory], list[_SchemaTransferHistory]]:
|
||||
"""按是否绑定持久任务回执分离 durable 与旧整理历史。"""
|
||||
return (
|
||||
[history for history in histories if history.transfer_task_id],
|
||||
[history for history in histories if not history.transfer_task_id],
|
||||
)
|
||||
|
||||
|
||||
async def _request_batch_durable_retries(
|
||||
histories: list[_SchemaTransferHistory],
|
||||
) -> tuple[int, list[tuple[int, TransferRetryRequestResult]]]:
|
||||
"""逐任务登记 durable 重试并保留每条拒绝的稳定状态。"""
|
||||
accepted_count = 0
|
||||
rejections: list[tuple[int, TransferRetryRequestResult]] = []
|
||||
for history in histories:
|
||||
retry = await asyncio.to_thread(
|
||||
_request_durable_transfer_retry,
|
||||
history_id=history.id,
|
||||
task_id=history.transfer_task_id or "",
|
||||
requested_by="history_ai_redo_batch",
|
||||
)
|
||||
if retry.accepted:
|
||||
accepted_count += 1
|
||||
else:
|
||||
rejections.append((history.id, retry))
|
||||
return accepted_count, rejections
|
||||
|
||||
|
||||
def _durable_retry_messages(
|
||||
*,
|
||||
accepted_count: int,
|
||||
rejections: list[tuple[int, TransferRetryRequestResult]],
|
||||
) -> list[str]:
|
||||
"""构造 durable 批量登记结果消息,供纯 durable 和混合请求复用。"""
|
||||
messages: list[str] = []
|
||||
if accepted_count:
|
||||
messages.append(f"已登记 {accepted_count} 个持久整理任务重试")
|
||||
if rejections:
|
||||
messages.append("以下任务未登记重试:" + _format_retry_rejections(rejections))
|
||||
return messages
|
||||
|
||||
|
||||
async def _complete_durable_retry_batch(
|
||||
*,
|
||||
histories: list[_SchemaTransferHistory],
|
||||
messages: list[str],
|
||||
rejections: list[tuple[int, TransferRetryRequestResult]],
|
||||
) -> Any:
|
||||
"""完成纯 durable 批量响应;存在拒绝时不伪造成功进度。"""
|
||||
message = ";".join(messages)
|
||||
if rejections:
|
||||
return _SchemaResponse(success=False, message=message)
|
||||
progress_key = f"transfer_retry_batch_{int(time.time() * 1000)}"
|
||||
history_ids = [history.id for history in histories]
|
||||
await _complete_durable_retry_progress(
|
||||
progress_key=progress_key,
|
||||
text=message,
|
||||
history_ids=history_ids,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
message=message,
|
||||
data={"progress_key": progress_key, "history_ids": history_ids},
|
||||
)
|
||||
|
||||
|
||||
async def _complete_durable_retry_progress(
|
||||
*,
|
||||
progress_key: str,
|
||||
text: str,
|
||||
history_ids: list[int],
|
||||
) -> None:
|
||||
"""写入可被现有 SSE 客户端立即消费的 durable 重试完成进度。"""
|
||||
progress = AsyncProgressHelper(progress_key)
|
||||
await progress.start()
|
||||
await progress.end(
|
||||
text=text,
|
||||
data={
|
||||
"history_ids": history_ids,
|
||||
"success": True,
|
||||
"completed": True,
|
||||
"message": text,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _build_progress_output_callback(
|
||||
progress: AsyncProgressHelper,
|
||||
data: dict[str, Any],
|
||||
@@ -285,13 +405,34 @@ async def ai_redo_transfer_history(
|
||||
手动触发单条历史记录的 AI 重新整理,并返回进度键。
|
||||
"""
|
||||
runtime_config = resolve_api_runtime_config(runtime_config)
|
||||
if not runtime_config.ai_agent_enable:
|
||||
return _SchemaResponse(success=False, message="MoviePilot智能助手未启用")
|
||||
|
||||
history = await query.get_transfer(history_id)
|
||||
if not history:
|
||||
return _SchemaResponse(success=False, message="整理记录不存在")
|
||||
|
||||
if history.transfer_task_id:
|
||||
retry = await asyncio.to_thread(
|
||||
_request_durable_transfer_retry,
|
||||
history_id=history.id,
|
||||
task_id=history.transfer_task_id,
|
||||
requested_by="history_ai_redo",
|
||||
)
|
||||
if not retry.accepted:
|
||||
return _SchemaResponse(success=False, message=retry.message)
|
||||
progress_key = f"transfer_retry_{history_id}_{int(time.time() * 1000)}"
|
||||
await _complete_durable_retry_progress(
|
||||
progress_key=progress_key,
|
||||
text=retry.message,
|
||||
history_ids=[history.id],
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
message=retry.message,
|
||||
data={"progress_key": progress_key},
|
||||
)
|
||||
|
||||
if not runtime_config.ai_agent_enable:
|
||||
return _SchemaResponse(success=False, message="MoviePilot智能助手未启用")
|
||||
|
||||
prompt = build_manual_redo_prompt(history)
|
||||
progress_key = f"ai_redo_transfer_{history_id}_{int(time.time() * 1000)}"
|
||||
_start_ai_redo_task(
|
||||
@@ -320,9 +461,6 @@ async def batch_ai_redo_transfer_history(
|
||||
手动触发多条历史记录的 AI 批量重新整理,并返回进度键。
|
||||
"""
|
||||
runtime_config = resolve_api_runtime_config(runtime_config)
|
||||
if not runtime_config.ai_agent_enable:
|
||||
return _SchemaResponse(success=False, message="MoviePilot智能助手未启用")
|
||||
|
||||
history_ids = normalize_history_ids(payload.history_ids)
|
||||
if not history_ids:
|
||||
return _SchemaResponse(success=False, message="未提供有效的整理记录")
|
||||
@@ -336,18 +474,59 @@ async def batch_ai_redo_transfer_history(
|
||||
+ ", ".join(str(history_id) for history_id in missing_ids),
|
||||
)
|
||||
|
||||
prompt = build_batch_manual_redo_prompt(histories)
|
||||
durable_histories, legacy_histories = _partition_durable_histories(histories)
|
||||
accepted_count, rejections = await _request_batch_durable_retries(
|
||||
durable_histories
|
||||
)
|
||||
response_message_parts = _durable_retry_messages(
|
||||
accepted_count=accepted_count,
|
||||
rejections=rejections,
|
||||
)
|
||||
|
||||
if not legacy_histories:
|
||||
return await _complete_durable_retry_batch(
|
||||
histories=durable_histories,
|
||||
messages=response_message_parts,
|
||||
rejections=rejections,
|
||||
)
|
||||
|
||||
if rejections:
|
||||
response_message_parts.append(
|
||||
f"{len(legacy_histories)} 条旧历史未提交:批量请求包含被拒绝的持久任务"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=";".join(response_message_parts),
|
||||
)
|
||||
|
||||
if not runtime_config.ai_agent_enable:
|
||||
response_message_parts.append(
|
||||
f"{len(legacy_histories)} 条旧历史未处理:MoviePilot智能助手未启用"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=False,
|
||||
message=";".join(response_message_parts),
|
||||
)
|
||||
|
||||
prompt = build_batch_manual_redo_prompt(legacy_histories)
|
||||
progress_key = f"ai_redo_transfer_batch_{int(time.time() * 1000)}"
|
||||
_start_batch_ai_redo_task(
|
||||
history_ids=history_ids,
|
||||
history_ids=[history.id for history in legacy_histories],
|
||||
prompt=prompt,
|
||||
progress_key=progress_key,
|
||||
task_registry=task_registry,
|
||||
)
|
||||
|
||||
response_message_parts.append(
|
||||
f"已提交 {len(legacy_histories)} 条旧历史给智能助手处理"
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
data={"progress_key": progress_key, "history_ids": history_ids},
|
||||
success=not rejections,
|
||||
message=";".join(response_message_parts),
|
||||
data={
|
||||
"progress_key": progress_key,
|
||||
"history_ids": [history.id for history in histories],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,15 +1,25 @@
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, List, Optional
|
||||
from typing import Annotated, Any, List, Literal, Optional, cast
|
||||
|
||||
from fastapi import Depends
|
||||
from fastapi import Depends, HTTPException, Query, status
|
||||
|
||||
from app.adapters.web.security.access import verify_apitoken, verify_token
|
||||
from app.api.dependencies.auth import get_current_active_manage_user
|
||||
from app.api.dependencies.history import get_transfer_history_lookup_service
|
||||
from app.api.response import ResponseAPIRouter
|
||||
from app.application.chain.data import get_chain_transfer_execution_port
|
||||
from app.application.configuration import get_api_runtime_config_snapshot
|
||||
from app.application.directory import DirectoryHelper
|
||||
from app.application.history import TransferHistoryLookupService
|
||||
from app.application.transfer_execution import (
|
||||
TransferExecutionCommand,
|
||||
TransferExecutionConflictError,
|
||||
TransferExecutionState,
|
||||
TransferManualReviewDecision,
|
||||
TransferManualReviewQuery,
|
||||
TransferManualReviewTaskView,
|
||||
TransferStepResult,
|
||||
)
|
||||
from app.chain.media import MediaChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.runtime.log import logger
|
||||
@@ -25,6 +35,10 @@ from app.schemas.transfer import ManualTransferHistoryInfo as _SchemaManualTrans
|
||||
from app.schemas.transfer import ManualTransferResultData as _SchemaManualTransferResultData
|
||||
from app.schemas.transfer import ManualTransferTargetPath as _SchemaManualTransferTargetPath
|
||||
from app.schemas.transfer import TransferJob as _SchemaTransferJob
|
||||
from app.schemas.transfer import TransferManualReviewData as _SchemaTransferManualReviewData
|
||||
from app.schemas.transfer import TransferManualReviewPageData as _SchemaTransferManualReviewPageData
|
||||
from app.schemas.transfer import TransferManualReviewRequest as _SchemaTransferManualReviewRequest
|
||||
from app.schemas.transfer import TransferManualReviewTaskData as _SchemaTransferManualReviewTaskData
|
||||
from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaType
|
||||
from app.schemas.workflow import FileItem
|
||||
from app.schemas.workflow import FileItem as _SchemaFileItem
|
||||
@@ -32,6 +46,143 @@ from app.schemas.workflow import FileItem as _SchemaFileItem
|
||||
router = ResponseAPIRouter()
|
||||
|
||||
|
||||
def _manual_review_actor(current_user: object) -> str:
|
||||
"""按名称、用户名和用户 ID 的稳定顺序提取人工复核操作者。"""
|
||||
for attribute in ("name", "username", "id"):
|
||||
value = getattr(current_user, attribute, None)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="当前管理用户缺少可审计身份",
|
||||
)
|
||||
|
||||
|
||||
def _manual_review_task_data(
|
||||
task: TransferManualReviewTaskView,
|
||||
) -> _SchemaTransferManualReviewTaskData:
|
||||
"""把 Application 人工复核投影映射为严格公开响应。"""
|
||||
return cast(
|
||||
_SchemaTransferManualReviewTaskData,
|
||||
_SchemaTransferManualReviewTaskData.model_validate({
|
||||
"task_id": task.task_id,
|
||||
"source": {
|
||||
"storage": task.source.storage,
|
||||
"path": task.source.path,
|
||||
},
|
||||
"state": task.state.value,
|
||||
"step": {
|
||||
"operation_id": task.step.operation_id,
|
||||
"kind": task.step.kind,
|
||||
"intent": task.step.intent,
|
||||
"evidence": task.step.evidence,
|
||||
"error": task.step.error,
|
||||
},
|
||||
"review_revision": task.review_revision,
|
||||
}),
|
||||
)
|
||||
|
||||
|
||||
@router.get( # type: ignore[misc]
|
||||
"/tasks/manual-reviews",
|
||||
summary="分页查询 durable 整理人工复核任务",
|
||||
response_model=_SchemaResponse[_SchemaTransferManualReviewPageData],
|
||||
)
|
||||
def list_transfer_manual_reviews(
|
||||
state_filter: Literal["manual_review", "retry_wait"] = Query(
|
||||
default="manual_review",
|
||||
alias="state",
|
||||
),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=30, ge=1, le=100),
|
||||
current_user: object = Depends(get_current_active_manage_user),
|
||||
) -> Any:
|
||||
"""分页返回待复核或已判定等待 durable 恢复的任务。"""
|
||||
del current_user
|
||||
result = TransferManualReviewQuery(
|
||||
get_chain_transfer_execution_port()
|
||||
).list(
|
||||
state=TransferExecutionState(state_filter),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
data=_SchemaTransferManualReviewPageData(
|
||||
items=[_manual_review_task_data(item) for item in result.items],
|
||||
total=result.total,
|
||||
page=result.page,
|
||||
page_size=result.page_size,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get( # type: ignore[misc]
|
||||
"/tasks/{task_id}/manual-review",
|
||||
summary="查询 durable 整理人工复核详情",
|
||||
response_model=_SchemaResponse[_SchemaTransferManualReviewTaskData],
|
||||
)
|
||||
def get_transfer_manual_review(
|
||||
task_id: str,
|
||||
current_user: object = Depends(get_current_active_manage_user),
|
||||
) -> Any:
|
||||
"""按任务标识返回严格裁剪的人工复核详情。"""
|
||||
del current_user
|
||||
task = TransferManualReviewQuery(
|
||||
get_chain_transfer_execution_port()
|
||||
).get(task_id=task_id)
|
||||
if task is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail="人工复核任务不存在",
|
||||
)
|
||||
return _SchemaResponse(success=True, data=_manual_review_task_data(task))
|
||||
|
||||
|
||||
@router.post( # type: ignore[misc]
|
||||
"/tasks/{task_id}/manual-review",
|
||||
summary="人工判定整理步骤的外部执行结果",
|
||||
response_model=_SchemaResponse[_SchemaTransferManualReviewData],
|
||||
)
|
||||
def resolve_transfer_manual_review(
|
||||
task_id: str,
|
||||
review: _SchemaTransferManualReviewRequest,
|
||||
current_user: object = Depends(get_current_active_manage_user),
|
||||
) -> Any:
|
||||
"""提交无租约人工判定,并返回不含 attempt 与 lease 的公开状态。"""
|
||||
result = (
|
||||
TransferStepResult(payload=dict(review.result_payload))
|
||||
if review.result_payload is not None
|
||||
else None
|
||||
)
|
||||
try:
|
||||
resolved = TransferExecutionCommand(
|
||||
get_chain_transfer_execution_port()
|
||||
).resolve_manual_review(
|
||||
task_id=task_id,
|
||||
operation_id=review.operation_id,
|
||||
decision=TransferManualReviewDecision(review.decision),
|
||||
actor=_manual_review_actor(current_user),
|
||||
reason=review.reason,
|
||||
result=result,
|
||||
)
|
||||
except TransferExecutionConflictError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail=str(error),
|
||||
) from error
|
||||
return _SchemaResponse(
|
||||
success=True,
|
||||
data=_SchemaTransferManualReviewData(
|
||||
task_id=resolved.task_id,
|
||||
operation_id=resolved.operation_id,
|
||||
decision=resolved.decision.value,
|
||||
state=resolved.state.value,
|
||||
review_revision=resolved.review_revision,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/name",
|
||||
summary="查询整理后的名称",
|
||||
|
||||
@@ -11,10 +11,12 @@ from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.application.transfer import TransferAdmissionRepository
|
||||
from app.application.transfer_execution import TransferExecutionRepository
|
||||
|
||||
|
||||
OperFactory = Callable[[], Any]
|
||||
TransferAdmissionRepositoryFactory = Callable[[], TransferAdmissionRepository]
|
||||
TransferExecutionRepositoryFactory = Callable[[], TransferExecutionRepository]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -27,6 +29,7 @@ class ChainDataPorts:
|
||||
download_history: OperFactory
|
||||
transfer_history: OperFactory
|
||||
transfer_pending: TransferAdmissionRepositoryFactory
|
||||
transfer_execution: TransferExecutionRepositoryFactory
|
||||
media_server: OperFactory
|
||||
download_failure: OperFactory
|
||||
user: OperFactory
|
||||
@@ -110,6 +113,7 @@ def configure_chain_data_ports(**factories: OperFactory) -> None:
|
||||
"download_history",
|
||||
"transfer_history",
|
||||
"transfer_pending",
|
||||
"transfer_execution",
|
||||
"media_server",
|
||||
"download_failure",
|
||||
"user",
|
||||
@@ -158,6 +162,11 @@ def get_chain_transfer_pending_port() -> TransferAdmissionRepository:
|
||||
return get_chain_data_ports().transfer_pending()
|
||||
|
||||
|
||||
def get_chain_transfer_execution_port() -> TransferExecutionRepository:
|
||||
"""创建类型化的整理步骤执行与终态结算仓储。"""
|
||||
return get_chain_data_ports().transfer_execution()
|
||||
|
||||
|
||||
def get_chain_media_server_port() -> Any:
|
||||
"""创建媒体服务器数据端口实例。"""
|
||||
return get_chain_data_ports().media_server()
|
||||
|
||||
@@ -11,6 +11,7 @@ from typing import Any, Protocol, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from app.application.history import TransferHistoryRecord, TransferHistoryWriter
|
||||
from app.application.transfer_execution import TransferSettlementResult
|
||||
from app.domain.context import Context, MediaInfo, MusicInfo, TorrentInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
@@ -37,12 +38,33 @@ class ChainDurableEventWriter(Protocol):
|
||||
def transfer_result(
|
||||
self,
|
||||
*,
|
||||
topic: str,
|
||||
topic: str | None,
|
||||
stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None],
|
||||
event_payload: dict[str, Any],
|
||||
publish: Callable[[dict[str, Any]], None],
|
||||
) -> TransferHistoryRecord | None:
|
||||
"""提交整理历史与结果 intent,并在提交后广播兼容事件。"""
|
||||
publish: Callable[[dict[str, Any]], None] | None,
|
||||
settlement: "TransferResultSettlement | None" = None,
|
||||
) -> TransferHistoryRecord | TransferSettlementResult | None:
|
||||
"""提交历史、可选任务终态和结果 intent,再按 topic 广播事件。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferResultSettlement:
|
||||
"""描述一次受 lease fencing 保护的整理任务终态结算。"""
|
||||
|
||||
task_id: str
|
||||
lease_token: str
|
||||
execution_fingerprint: str
|
||||
outcome: str
|
||||
error: str | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""拒绝缺少稳定身份、非法结果或不可诊断的失败终态。"""
|
||||
if not self.task_id or not self.lease_token or not self.execution_fingerprint:
|
||||
raise ValueError("整理终态结算缺少任务、租约或执行检查点身份")
|
||||
if self.outcome not in {"succeeded", "failed"}:
|
||||
raise ValueError(f"不支持的整理终态:{self.outcome}")
|
||||
if self.outcome == "failed" and not self.error:
|
||||
raise ValueError("整理失败终态必须包含可诊断原因")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
@@ -61,8 +83,23 @@ def download_added_event_key(history_id: int) -> str:
|
||||
return f"download.added:{history_id}:{uuid4().hex}:v1"
|
||||
|
||||
|
||||
def transfer_result_event_key(topic: str, history_id: int) -> str:
|
||||
"""由结果 topic、整理历史 ID 与本次事实标识构造幂等键。"""
|
||||
def transfer_result_event_key(
|
||||
topic: str,
|
||||
history_id: int,
|
||||
*,
|
||||
settlement: TransferResultSettlement | None = None,
|
||||
settlement_revision: int | None = None,
|
||||
) -> str:
|
||||
"""为旧结果事实生成 occurrence key,为任务结算生成确定性幂等键。"""
|
||||
if settlement is not None:
|
||||
if settlement_revision is None or settlement_revision <= 0:
|
||||
raise ValueError("整理终态事件键缺少有效结算修订号")
|
||||
return (
|
||||
f"transfer.result:{settlement.task_id}:{settlement_revision}:"
|
||||
f"{settlement.outcome}:v1"
|
||||
)
|
||||
if settlement_revision is not None:
|
||||
raise ValueError("旧整理结果事件键不能单独指定结算修订号")
|
||||
return f"{topic}:{history_id}:{uuid4().hex}:v1"
|
||||
|
||||
|
||||
|
||||
@@ -40,6 +40,7 @@ class TransferHistoryRecord(Protocol):
|
||||
src: Optional[str]
|
||||
src_storage: Optional[str]
|
||||
src_fileitem: Optional[dict]
|
||||
transfer_task_id: Optional[str]
|
||||
|
||||
|
||||
class TransferHistoryWriter(Protocol):
|
||||
@@ -414,6 +415,11 @@ class TransferHistoryMutationCommand:
|
||||
history = self._repository.get(history_id)
|
||||
if not history:
|
||||
return HistoryMutationResult(False, "记录不存在")
|
||||
if getattr(history, "transfer_task_id", None):
|
||||
return HistoryMutationResult(
|
||||
False,
|
||||
"持久整理失败记录不可删除,请使用重试或人工复核入口",
|
||||
)
|
||||
|
||||
if delete_destination and history.dest_fileitem:
|
||||
destination = self._file_item_factory(history.dest_fileitem)
|
||||
@@ -440,10 +446,10 @@ class TransferHistoryMutationCommand:
|
||||
return HistoryMutationResult(True)
|
||||
|
||||
def truncate(self) -> HistoryMutationResult:
|
||||
"""在单一事务中清空全部整理历史。"""
|
||||
"""在单一事务中清空旧历史,并保留当前失败任务记录。"""
|
||||
self._repository.stage_truncate()
|
||||
self._commit()
|
||||
return HistoryMutationResult(True)
|
||||
return HistoryMutationResult(True, "已清空旧整理记录,失败任务记录已保留")
|
||||
|
||||
def _commit(self) -> None:
|
||||
"""提交历史事务,失败时回滚且不发布事件或清缓存。"""
|
||||
|
||||
+14
-10
@@ -167,16 +167,18 @@ class DurableEventCommand:
|
||||
def execute(
|
||||
self,
|
||||
*,
|
||||
intent: OutboxIntent | Callable[[T], OutboxIntent],
|
||||
intent: OutboxIntent | Callable[[T], OutboxIntent] | None,
|
||||
stage_business: Callable[[], T],
|
||||
publish: Callable[[], None],
|
||||
publish: Callable[[], None] | None,
|
||||
after_commit: Callable[[], None] | None = None,
|
||||
) -> T:
|
||||
"""先原子提交业务与 intent,再保持原顺序执行提交后动作和即时广播。"""
|
||||
"""原子提交业务与可选 intent,再执行可选提交后动作和广播。"""
|
||||
resolved_intent: OutboxIntent | None = None
|
||||
try:
|
||||
result = stage_business()
|
||||
resolved_intent = intent(result) if callable(intent) else intent
|
||||
self._outbox.stage(resolved_intent, datetime.now(timezone.utc))
|
||||
if intent is not None:
|
||||
resolved_intent = intent(result) if callable(intent) else intent
|
||||
self._outbox.stage(resolved_intent, datetime.now(timezone.utc))
|
||||
self._unit_of_work.commit()
|
||||
except Exception:
|
||||
self._unit_of_work.rollback()
|
||||
@@ -184,11 +186,13 @@ class DurableEventCommand:
|
||||
|
||||
if after_commit:
|
||||
after_commit()
|
||||
publish()
|
||||
self._outbox.complete_by_event_key(
|
||||
resolved_intent.event_key,
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
if publish:
|
||||
publish()
|
||||
if resolved_intent is not None and publish is not None:
|
||||
self._outbox.complete_by_event_key(
|
||||
resolved_intent.event_key,
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ from pydantic import BaseModel, ConfigDict, PrivateAttr
|
||||
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.application.agent import get_prompt_manager, get_running_agent_manager
|
||||
from app.application.transfer_execution import TransferExecutionCheckpoint
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.media import normalize_music_type
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
@@ -668,6 +669,8 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
_admission_task_id: Optional[str] = PrivateAttr(default=None)
|
||||
_planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None)
|
||||
_plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None)
|
||||
_execution_checkpoint: Optional[TransferExecutionCheckpoint] = PrivateAttr(default=None)
|
||||
_terminal_settled: bool = PrivateAttr(default=False)
|
||||
_planning_context_restored: bool = PrivateAttr(default=False)
|
||||
_lease_owner: Optional[str] = PrivateAttr(default=None)
|
||||
_lease_token: Optional[str] = PrivateAttr(default=None)
|
||||
@@ -699,6 +702,26 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
"""绑定持久执行检查点,不改变插件可见序列化字段。"""
|
||||
self._plan_checkpoint = checkpoint
|
||||
|
||||
@property
|
||||
def execution_checkpoint(self) -> Optional[TransferExecutionCheckpoint]:
|
||||
"""返回仅供宿主终态结算使用的内部执行检查点。"""
|
||||
return self._execution_checkpoint
|
||||
|
||||
def bind_execution_checkpoint(
|
||||
self, checkpoint: TransferExecutionCheckpoint
|
||||
) -> None:
|
||||
"""绑定执行结果检查点,不改变插件可见的旧任务序列化字段。"""
|
||||
self._execution_checkpoint = checkpoint
|
||||
|
||||
@property
|
||||
def terminal_settled(self) -> bool:
|
||||
"""返回历史、事件与 pending 是否已由同一 UoW 提交。"""
|
||||
return self._terminal_settled
|
||||
|
||||
def mark_terminal_settled(self) -> None:
|
||||
"""仅在 task-aware writer 成功返回后标记终态已经提交。"""
|
||||
self._terminal_settled = True
|
||||
|
||||
@property
|
||||
def planning_context_restored(self) -> bool:
|
||||
"""返回当前领域上下文是否来自持久快照。"""
|
||||
|
||||
@@ -0,0 +1,884 @@
|
||||
"""整理步骤执行检查点、人工判定与终态结算的应用契约。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import Any, Callable, Mapping, Optional, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
TRANSFER_EXECUTION_VERSION = 1
|
||||
TRANSFER_STEP_INTENT_VERSION = 1
|
||||
TRANSFER_STEP_RESULT_VERSION = 1
|
||||
|
||||
|
||||
class TransferExecutionState(StrEnum):
|
||||
"""描述 planning phase 正交的持久执行状态。"""
|
||||
|
||||
NOT_STARTED = "not_started"
|
||||
RUNNING = "running"
|
||||
RETRY_WAIT = "retry_wait"
|
||||
SETTLING = "settling"
|
||||
FAILED = "failed"
|
||||
MANUAL_REVIEW = "manual_review"
|
||||
|
||||
|
||||
class TransferStepState(StrEnum):
|
||||
"""描述单个稳定外部操作的持久执行状态。"""
|
||||
|
||||
PREPARED = "prepared"
|
||||
STARTED = "started"
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
MANUAL_REVIEW = "manual_review"
|
||||
|
||||
|
||||
class TransferTerminalState(StrEnum):
|
||||
"""描述允许写入整理历史的确定终态。"""
|
||||
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class TransferOperationObservationState(StrEnum):
|
||||
"""描述重启后对遗留 STARTED 外部操作的严格探测结论。"""
|
||||
|
||||
APPLIED = "applied"
|
||||
NOT_APPLIED = "not_applied"
|
||||
UNKNOWN = "unknown"
|
||||
CONFLICT = "conflict"
|
||||
|
||||
|
||||
class TransferManualReviewDecision(StrEnum):
|
||||
"""描述人工对外部结果不确定步骤作出的显式判定。"""
|
||||
|
||||
NOT_APPLIED = "not_applied"
|
||||
APPLIED = "applied"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class TransferExecutionError(RuntimeError):
|
||||
"""表示整理执行持久化状态无法按请求推进。"""
|
||||
|
||||
|
||||
class TransferExecutionConflictError(TransferExecutionError):
|
||||
"""表示稳定操作身份、尝试身份或检查点证据发生冲突。"""
|
||||
|
||||
|
||||
class TransferExecutionLeaseLostError(TransferExecutionError):
|
||||
"""表示持久化写入时任务租约已失效或已被其他 worker 接管。"""
|
||||
|
||||
|
||||
def _canonical_json(payload: Mapping[str, Any]) -> str:
|
||||
"""生成稳定操作身份使用的规范 JSON。"""
|
||||
return json.dumps(
|
||||
payload,
|
||||
ensure_ascii=True,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
|
||||
|
||||
def build_transfer_operation_id(
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint_fingerprint: str,
|
||||
ordinal: int,
|
||||
phase: str,
|
||||
kind: str,
|
||||
intent_payload: Mapping[str, Any],
|
||||
) -> str:
|
||||
"""
|
||||
由冻结计划和操作语义生成跨 lease、attempt 与重启稳定的身份。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param checkpoint_fingerprint: 冻结计划指纹
|
||||
:param ordinal: 全局执行序号
|
||||
:param phase: 执行阶段
|
||||
:param kind: 操作类型
|
||||
:param intent_payload: 冻结操作参数
|
||||
:return: SHA-256 操作标识
|
||||
"""
|
||||
if not task_id or not checkpoint_fingerprint or ordinal < 0 or not phase or not kind:
|
||||
raise ValueError("整理操作身份缺少稳定任务、计划或步骤信息")
|
||||
canonical = _canonical_json({
|
||||
"schema_version": TRANSFER_STEP_INTENT_VERSION,
|
||||
"task_id": task_id,
|
||||
"checkpoint_fingerprint": checkpoint_fingerprint,
|
||||
"ordinal": ordinal,
|
||||
"phase": phase,
|
||||
"kind": kind,
|
||||
"intent_payload": dict(intent_payload),
|
||||
})
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def build_transfer_checkpoint_fingerprint(
|
||||
checkpoint_payload: Mapping[str, Any],
|
||||
) -> str:
|
||||
"""由版本化执行检查点 payload 生成稳定 SHA-256 指纹。"""
|
||||
return hashlib.sha256(
|
||||
_canonical_json(checkpoint_payload).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferStepIntent:
|
||||
"""保存一次可持久化外部操作的稳定意图。"""
|
||||
|
||||
operation_id: str
|
||||
checkpoint_fingerprint: str
|
||||
ordinal: int
|
||||
phase: str
|
||||
kind: str
|
||||
payload: dict[str, Any]
|
||||
version: int = TRANSFER_STEP_INTENT_VERSION
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""拒绝版本未知、身份不完整或不可 JSON 序列化的操作意图。"""
|
||||
object.__setattr__(self, "payload", deepcopy(self.payload))
|
||||
if self.version != TRANSFER_STEP_INTENT_VERSION:
|
||||
raise ValueError(f"不支持的整理步骤意图版本:{self.version}")
|
||||
if (
|
||||
not self.operation_id
|
||||
or not self.checkpoint_fingerprint
|
||||
or self.ordinal < 0
|
||||
or not self.phase
|
||||
or not self.kind
|
||||
):
|
||||
raise ValueError("整理步骤意图缺少稳定身份或顺序")
|
||||
_canonical_json(self.payload)
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint_fingerprint: str,
|
||||
ordinal: int,
|
||||
phase: str,
|
||||
kind: str,
|
||||
payload: Mapping[str, Any],
|
||||
) -> "TransferStepIntent":
|
||||
"""构造并绑定稳定 operation ID 的步骤意图。"""
|
||||
frozen_payload = dict(payload)
|
||||
return cls(
|
||||
operation_id=build_transfer_operation_id(
|
||||
task_id=task_id,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
ordinal=ordinal,
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
intent_payload=frozen_payload,
|
||||
),
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
ordinal=ordinal,
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
payload=frozen_payload,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferStepResult:
|
||||
"""保存严格执行或探测后得到的版本化步骤证据。"""
|
||||
|
||||
payload: dict[str, Any]
|
||||
version: int = TRANSFER_STEP_RESULT_VERSION
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""拒绝未知版本或不可 JSON 序列化的步骤结果。"""
|
||||
object.__setattr__(self, "payload", deepcopy(self.payload))
|
||||
if self.version != TRANSFER_STEP_RESULT_VERSION:
|
||||
raise ValueError(f"不支持的整理步骤结果版本:{self.version}")
|
||||
_canonical_json(self.payload)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferOperationObservation:
|
||||
"""保存外部操作探测结论及其版本化事实证据。"""
|
||||
|
||||
state: TransferOperationObservationState
|
||||
evidence: TransferStepResult
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferExecutionStep:
|
||||
"""提供脱离 ORM Session 的单步骤持久状态投影。"""
|
||||
|
||||
task_id: str
|
||||
operation_id: str
|
||||
checkpoint_fingerprint: str
|
||||
ordinal: int
|
||||
phase: str
|
||||
kind: str
|
||||
state: TransferStepState
|
||||
attempt_token: Optional[str]
|
||||
attempt_count: int
|
||||
intent: TransferStepIntent
|
||||
result: Optional[TransferStepResult]
|
||||
last_error: Optional[str]
|
||||
prepared_at: str
|
||||
started_at: Optional[str]
|
||||
completed_at: Optional[str]
|
||||
updated_at: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferExecutionCheckpoint:
|
||||
"""保存所有必要步骤完成后供终态结算重放的聚合结果。"""
|
||||
|
||||
fingerprint: str
|
||||
payload: dict[str, Any]
|
||||
operation_ids: tuple[str, ...]
|
||||
skip_reason: Optional[str] = None
|
||||
version: int = TRANSFER_EXECUTION_VERSION
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""拒绝重复步骤、空指纹和不可序列化的执行检查点。"""
|
||||
object.__setattr__(self, "payload", deepcopy(self.payload))
|
||||
if self.version != TRANSFER_EXECUTION_VERSION or not self.fingerprint:
|
||||
raise ValueError("整理执行检查点版本或指纹无效")
|
||||
if len(set(self.operation_ids)) != len(self.operation_ids):
|
||||
raise ValueError("整理执行检查点不能引用重复操作")
|
||||
if not all(isinstance(item, str) and item for item in self.operation_ids):
|
||||
raise ValueError("整理执行检查点包含无效操作标识")
|
||||
if not self.operation_ids and not self.skip_reason:
|
||||
raise ValueError("零副作用整理执行检查点必须记录 skip_reason")
|
||||
_canonical_json(self.payload)
|
||||
if build_transfer_checkpoint_fingerprint(self.to_payload()) != self.fingerprint:
|
||||
raise ValueError("整理执行检查点内容与指纹不一致")
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
*,
|
||||
payload: Mapping[str, Any],
|
||||
operation_ids: tuple[str, ...],
|
||||
skip_reason: Optional[str] = None,
|
||||
) -> "TransferExecutionCheckpoint":
|
||||
"""构造并绑定完整序列化内容指纹的执行检查点。"""
|
||||
frozen_payload = dict(payload)
|
||||
serialized = {
|
||||
"schema_version": TRANSFER_EXECUTION_VERSION,
|
||||
"payload": frozen_payload,
|
||||
"operation_ids": list(operation_ids),
|
||||
"skip_reason": skip_reason,
|
||||
}
|
||||
return cls(
|
||||
fingerprint=build_transfer_checkpoint_fingerprint(serialized),
|
||||
payload=frozen_payload,
|
||||
operation_ids=operation_ids,
|
||||
skip_reason=skip_reason,
|
||||
)
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
"""编码可跨重启恢复的完整版本化执行检查点。"""
|
||||
return {
|
||||
"schema_version": self.version,
|
||||
"payload": dict(self.payload),
|
||||
"operation_ids": list(self.operation_ids),
|
||||
"skip_reason": self.skip_reason,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_payload(
|
||||
cls,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
fingerprint: str,
|
||||
) -> "TransferExecutionCheckpoint":
|
||||
"""解析并校验数据库中的执行检查点版本与稳定指纹。"""
|
||||
serialized = dict(payload)
|
||||
if build_transfer_checkpoint_fingerprint(serialized) != fingerprint:
|
||||
raise TransferExecutionConflictError("整理执行检查点 JSON 与指纹不一致")
|
||||
version = serialized.get("schema_version")
|
||||
if version != TRANSFER_EXECUTION_VERSION:
|
||||
raise TransferExecutionConflictError("整理执行检查点版本不受支持")
|
||||
result_payload = serialized.get("payload")
|
||||
operation_ids = serialized.get("operation_ids")
|
||||
if not isinstance(result_payload, dict) or not isinstance(operation_ids, list):
|
||||
raise TransferExecutionConflictError("整理执行检查点结构无效")
|
||||
if not all(isinstance(item, str) and item for item in operation_ids):
|
||||
raise TransferExecutionConflictError("整理执行检查点包含无效操作标识")
|
||||
skip_reason = serialized.get("skip_reason")
|
||||
if skip_reason is not None and not isinstance(skip_reason, str):
|
||||
raise TransferExecutionConflictError("整理执行跳过原因类型无效")
|
||||
return cls(
|
||||
fingerprint=fingerprint,
|
||||
payload=result_payload,
|
||||
operation_ids=tuple(operation_ids),
|
||||
skip_reason=skip_reason,
|
||||
version=version,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferExecutionSnapshot:
|
||||
"""提供任务执行 checkpoint、重试和人工状态的稳定投影。"""
|
||||
|
||||
task_id: str
|
||||
state: TransferExecutionState
|
||||
checkpoint: Optional[TransferExecutionCheckpoint]
|
||||
retry_generation: int
|
||||
retry_count: int
|
||||
retry_due_at: Optional[str]
|
||||
settlement_revision: int
|
||||
terminal_history_id: Optional[int]
|
||||
last_error: Optional[str]
|
||||
steps: tuple[TransferExecutionStep, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferSettlementIntent:
|
||||
"""描述已确定结果对应的历史写入与 pending 终态。"""
|
||||
|
||||
terminal_state: TransferTerminalState
|
||||
history_payload: dict[str, Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""确保历史 payload 可持久化且包含稳定源身份。"""
|
||||
object.__setattr__(self, "history_payload", deepcopy(self.history_payload))
|
||||
if not self.history_payload.get("src"):
|
||||
raise ValueError("整理终态历史缺少源路径")
|
||||
_canonical_json(self.history_payload)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferSettlementResult:
|
||||
"""描述终态历史是否新提交以及 pending 是否已删除。"""
|
||||
|
||||
history_id: int
|
||||
settlement_revision: int
|
||||
pending_deleted: bool
|
||||
already_settled: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferRetryRequestResult:
|
||||
"""描述用户重试请求是否被持久调度器接受。"""
|
||||
|
||||
accepted: bool
|
||||
state: TransferExecutionState
|
||||
retry_generation: int
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferManualReviewResult:
|
||||
"""描述一次已持久审计的人工判定及后续调度状态。"""
|
||||
|
||||
task_id: str
|
||||
operation_id: str
|
||||
decision: TransferManualReviewDecision
|
||||
state: TransferExecutionState
|
||||
review_revision: int
|
||||
step: TransferExecutionStep
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferManualReviewSource:
|
||||
"""提供人工复核任务的最小源文件身份。"""
|
||||
|
||||
storage: str
|
||||
path: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferManualReviewStepView:
|
||||
"""提供人工复核所需且不含租约或尝试令牌的步骤证据。"""
|
||||
|
||||
operation_id: str
|
||||
kind: str
|
||||
intent: dict[str, Any]
|
||||
evidence: Optional[dict[str, Any]]
|
||||
error: Optional[str]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferManualReviewTaskView:
|
||||
"""提供管理员发现与复核 durable 整理任务的公开投影。"""
|
||||
|
||||
task_id: str
|
||||
source: TransferManualReviewSource
|
||||
state: TransferExecutionState
|
||||
step: TransferManualReviewStepView
|
||||
review_revision: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TransferManualReviewPage:
|
||||
"""提供稳定分页的人工复核任务结果。"""
|
||||
|
||||
items: tuple[TransferManualReviewTaskView, ...]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
|
||||
|
||||
class TransferExecutionRepository(Protocol):
|
||||
"""定义步骤状态、执行 checkpoint 与终态历史的持久化端口。"""
|
||||
|
||||
def get_snapshot(self, *, task_id: str) -> Optional[TransferExecutionSnapshot]:
|
||||
"""读取任务执行快照。"""
|
||||
|
||||
def list_manual_reviews(
|
||||
self,
|
||||
*,
|
||||
state: TransferExecutionState,
|
||||
page: int,
|
||||
page_size: int,
|
||||
) -> TransferManualReviewPage:
|
||||
"""按严格公开状态分页读取人工复核任务。"""
|
||||
|
||||
def get_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional[TransferManualReviewTaskView]:
|
||||
"""按任务标识读取人工复核公开详情。"""
|
||||
|
||||
def prepare_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
intent: TransferStepIntent,
|
||||
) -> TransferExecutionStep:
|
||||
"""在外部副作用前持久化稳定步骤意图。"""
|
||||
|
||||
def start_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
) -> TransferExecutionStep:
|
||||
"""以当前 lease 和新 attempt token 标记步骤开始。"""
|
||||
|
||||
def restart_after_not_applied(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
previous_attempt_token: str,
|
||||
attempt_token: str,
|
||||
evidence: TransferStepResult,
|
||||
) -> TransferExecutionStep:
|
||||
"""严格探测为未发生后,以新 attempt token 安全重启遗留步骤。"""
|
||||
|
||||
def resume_failed_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
) -> TransferExecutionStep:
|
||||
"""任务到期并重新 claim 后,以新 attempt token 重试 FAILED 步骤。"""
|
||||
|
||||
def complete_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
result: TransferStepResult,
|
||||
) -> TransferExecutionStep:
|
||||
"""以 lease 和 attempt 双 CAS 提交成功证据。"""
|
||||
|
||||
def defer_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
error: str,
|
||||
retry_due_at: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""持久化已知失败、到期时间并原子释放当前 lease。"""
|
||||
|
||||
def exhaust_step(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
error: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""重试预算耗尽时保留 lease,并建立可 durable 失败结算的检查点。"""
|
||||
|
||||
def request_retry(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
requested_by: str,
|
||||
) -> TransferRetryRequestResult:
|
||||
"""仅把 FAILED 终态转入到期可 claim 的 retry_wait。"""
|
||||
|
||||
def resolve_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
decision: TransferManualReviewDecision,
|
||||
actor: str,
|
||||
reason: str,
|
||||
result: Optional[TransferStepResult] = None,
|
||||
) -> TransferManualReviewResult:
|
||||
"""无 lease 地原子提交人工判定审计并交回唯一调度器。"""
|
||||
|
||||
def mark_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
error: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
attempt_token: Optional[str] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""持久化不可判定结果并从自动调度中隔离任务。"""
|
||||
|
||||
def checkpoint_execution(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
checkpoint: TransferExecutionCheckpoint,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""确认所有引用步骤成功后提交聚合执行检查点。"""
|
||||
|
||||
class TransferStepRunner(Protocol):
|
||||
"""定义文件执行方可注入的单步骤持久执行边界。"""
|
||||
|
||||
def run(
|
||||
self,
|
||||
*,
|
||||
phase: str,
|
||||
kind: str,
|
||||
payload: Mapping[str, Any],
|
||||
execute: Callable[[], TransferStepResult],
|
||||
observe: Callable[[], TransferOperationObservation],
|
||||
) -> TransferStepResult:
|
||||
"""持久编排一次外部副作用,并在遗留尝试时先严格探测。"""
|
||||
|
||||
|
||||
class TransferExecutionCommand:
|
||||
"""以类型化命令收口步骤状态机,外部 I/O 由调用方在事务外执行。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repository: TransferExecutionRepository,
|
||||
*,
|
||||
attempt_token_factory: Callable[[], str] = lambda: uuid4().hex,
|
||||
) -> None:
|
||||
"""保存持久化端口和可替换的 attempt token 工厂。"""
|
||||
self._repository = repository
|
||||
self._attempt_token_factory = attempt_token_factory
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
intent: TransferStepIntent,
|
||||
) -> TransferExecutionStep:
|
||||
"""在外部调用前提交步骤意图。"""
|
||||
return self._repository.prepare_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
intent=intent,
|
||||
)
|
||||
|
||||
def begin(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
) -> TransferExecutionStep:
|
||||
"""生成单次 attempt token 并持久化开始状态。"""
|
||||
attempt_token = self._attempt_token_factory()
|
||||
if not attempt_token:
|
||||
raise ValueError("整理步骤 attempt token 不能为空")
|
||||
return self._repository.start_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
)
|
||||
|
||||
def restart_after_not_applied(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
evidence: TransferStepResult,
|
||||
) -> TransferExecutionStep:
|
||||
"""在严格 NOT_APPLIED 证据成立时轮换 attempt token 并继续执行。"""
|
||||
if step.state is not TransferStepState.STARTED or not step.attempt_token:
|
||||
raise ValueError("只有遗留 STARTED 步骤可按未发生证据安全重启")
|
||||
attempt_token = self._attempt_token_factory()
|
||||
if not attempt_token or attempt_token == step.attempt_token:
|
||||
raise ValueError("安全重启必须生成不同的 attempt token")
|
||||
return self._repository.restart_after_not_applied(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
previous_attempt_token=step.attempt_token,
|
||||
attempt_token=attempt_token,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def resume_failed(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
) -> TransferExecutionStep:
|
||||
"""使用重试调度取得的新 lease 为 FAILED 步骤创建下一次尝试。"""
|
||||
if step.state is not TransferStepState.FAILED or step.attempt_token is not None:
|
||||
raise ValueError("只有已释放 attempt 的 FAILED 步骤可以恢复重试")
|
||||
attempt_token = self._attempt_token_factory()
|
||||
if not attempt_token:
|
||||
raise ValueError("恢复重试的 attempt token 不能为空")
|
||||
return self._repository.resume_failed_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
attempt_token=attempt_token,
|
||||
)
|
||||
|
||||
def complete(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
result: TransferStepResult,
|
||||
) -> TransferExecutionStep:
|
||||
"""使用已开始步骤携带的 attempt token 提交成功证据。"""
|
||||
if not step.attempt_token:
|
||||
raise ValueError("尚未开始的整理步骤不能提交成功")
|
||||
return self._repository.complete_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
attempt_token=step.attempt_token,
|
||||
result=result,
|
||||
)
|
||||
|
||||
def defer(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
error: str,
|
||||
retry_due_at: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""记录已知失败并把唯一重试权交回持久调度器。"""
|
||||
if not step.attempt_token:
|
||||
raise ValueError("尚未开始的整理步骤不能进入重试等待")
|
||||
return self._repository.defer_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
attempt_token=step.attempt_token,
|
||||
error=error,
|
||||
retry_due_at=retry_due_at,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def exhaust(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
error: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""提交达到预算的确定失败,并把任务交给唯一 durable 终态 writer。"""
|
||||
if step.state is not TransferStepState.STARTED or not step.attempt_token:
|
||||
raise ValueError("只有 STARTED 步骤可提交预算耗尽失败")
|
||||
return self._repository.exhaust_step(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
attempt_token=step.attempt_token,
|
||||
error=error,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def request_retry(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
requested_by: str,
|
||||
) -> TransferRetryRequestResult:
|
||||
"""登记用户重试意图,不直接 claim 或执行任务。"""
|
||||
if not task_id or not reason or not requested_by:
|
||||
raise ValueError("用户重试请求缺少任务、原因或请求身份")
|
||||
return self._repository.request_retry(
|
||||
task_id=task_id,
|
||||
reason=reason,
|
||||
requested_by=requested_by,
|
||||
)
|
||||
|
||||
def resolve_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
decision: TransferManualReviewDecision,
|
||||
actor: str,
|
||||
reason: str,
|
||||
result: Optional[TransferStepResult] = None,
|
||||
) -> TransferManualReviewResult:
|
||||
"""提交人工判定;FAILED 在无 lease durable 结算落地前明确拒绝。"""
|
||||
if not all((task_id, operation_id, actor, reason)):
|
||||
raise ValueError("人工判定缺少任务、步骤、操作者或原因")
|
||||
if decision is TransferManualReviewDecision.APPLIED and result is None:
|
||||
raise ValueError("人工判定已发生时必须提供结果证据")
|
||||
if decision is TransferManualReviewDecision.FAILED:
|
||||
raise TransferExecutionConflictError(
|
||||
"人工失败终态尚不能绕过 lease durable 结算"
|
||||
)
|
||||
return self._repository.resolve_manual_review(
|
||||
task_id=task_id,
|
||||
operation_id=operation_id,
|
||||
decision=decision,
|
||||
actor=actor,
|
||||
reason=reason,
|
||||
result=result,
|
||||
)
|
||||
|
||||
def manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
step: TransferExecutionStep,
|
||||
error: str,
|
||||
evidence: Optional[TransferStepResult] = None,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""隔离无法严格判断外部结果的步骤,禁止自动再次执行。"""
|
||||
return self._repository.mark_manual_review(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=step.operation_id,
|
||||
attempt_token=step.attempt_token,
|
||||
error=error,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
def checkpoint(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
checkpoint: TransferExecutionCheckpoint,
|
||||
) -> TransferExecutionSnapshot:
|
||||
"""提交可独立重放终态结算的聚合执行结果。"""
|
||||
return self._repository.checkpoint_execution(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
|
||||
|
||||
class TransferManualReviewQuery:
|
||||
"""收口管理员可发现的人工复核只读用例。"""
|
||||
|
||||
_visible_states = frozenset({
|
||||
TransferExecutionState.MANUAL_REVIEW,
|
||||
TransferExecutionState.RETRY_WAIT,
|
||||
})
|
||||
|
||||
def __init__(self, repository: TransferExecutionRepository) -> None:
|
||||
"""保存整理执行查询端口。"""
|
||||
self._repository = repository
|
||||
|
||||
@classmethod
|
||||
def _validate_state(cls, state: TransferExecutionState) -> None:
|
||||
"""拒绝把内部执行状态扩展到人工复核查询面。"""
|
||||
if state not in cls._visible_states:
|
||||
raise ValueError(f"人工复核查询不支持状态:{state.value}")
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
state: TransferExecutionState = TransferExecutionState.MANUAL_REVIEW,
|
||||
page: int = 1,
|
||||
page_size: int = 30,
|
||||
) -> TransferManualReviewPage:
|
||||
"""分页返回待复核或刚完成复核并等待恢复的任务。"""
|
||||
self._validate_state(state)
|
||||
if page < 1 or not 1 <= page_size <= 100:
|
||||
raise ValueError("人工复核分页参数超出允许范围")
|
||||
return self._repository.list_manual_reviews(
|
||||
state=state,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
|
||||
def get(self, *, task_id: str) -> Optional[TransferManualReviewTaskView]:
|
||||
"""读取一条人工复核详情,内部状态任务按不存在处理。"""
|
||||
if not task_id:
|
||||
raise ValueError("人工复核详情缺少任务标识")
|
||||
result = self._repository.get_manual_review(task_id=task_id)
|
||||
if result is not None:
|
||||
self._validate_state(result.state)
|
||||
return result
|
||||
|
||||
__all__ = [
|
||||
"TRANSFER_EXECUTION_VERSION",
|
||||
"TRANSFER_STEP_INTENT_VERSION",
|
||||
"TRANSFER_STEP_RESULT_VERSION",
|
||||
"TransferExecutionCheckpoint",
|
||||
"TransferExecutionCommand",
|
||||
"TransferExecutionConflictError",
|
||||
"TransferExecutionError",
|
||||
"TransferExecutionLeaseLostError",
|
||||
"TransferOperationObservation",
|
||||
"TransferOperationObservationState",
|
||||
"TransferRetryRequestResult",
|
||||
"TransferExecutionRepository",
|
||||
"TransferExecutionSnapshot",
|
||||
"TransferExecutionState",
|
||||
"TransferExecutionStep",
|
||||
"TransferManualReviewDecision",
|
||||
"TransferManualReviewPage",
|
||||
"TransferManualReviewQuery",
|
||||
"TransferManualReviewResult",
|
||||
"TransferManualReviewSource",
|
||||
"TransferManualReviewStepView",
|
||||
"TransferManualReviewTaskView",
|
||||
"TransferSettlementIntent",
|
||||
"TransferSettlementResult",
|
||||
"TransferStepIntent",
|
||||
"TransferStepResult",
|
||||
"TransferStepRunner",
|
||||
"TransferStepState",
|
||||
"TransferTerminalState",
|
||||
"build_transfer_checkpoint_fingerprint",
|
||||
"build_transfer_operation_id",
|
||||
]
|
||||
+108
-19
@@ -9,12 +9,13 @@ TransferChain 中。mixin 方法运行时经 MRO 解析,共享 TransferChain
|
||||
"""
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union
|
||||
from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
|
||||
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.application.agent import build_manual_redo_prompt, get_running_agent_manager
|
||||
from app.application.chain.data import (
|
||||
get_chain_download_history_port,
|
||||
get_chain_transfer_execution_port,
|
||||
get_chain_transfer_history_port,
|
||||
)
|
||||
from app.application.configuration import (
|
||||
@@ -24,6 +25,7 @@ from app.application.configuration import (
|
||||
from app.application.formatting import EpisodeFormatRuleHelper
|
||||
from app.application.history import clear_transfer_failures, resolve_history
|
||||
from app.application.transfer import TransferTask, job_lock
|
||||
from app.application.transfer_execution import TransferExecutionCommand
|
||||
from app.chain._contracts import TransferMixinHost
|
||||
from app.chain.media import MediaChain
|
||||
from app.chain.storage import StorageChain
|
||||
@@ -61,6 +63,34 @@ DownloadFiles = Any
|
||||
DownloadHistory = Any
|
||||
TransferHistory = Any
|
||||
|
||||
|
||||
def _request_durable_transfer_retry(
|
||||
history: TransferHistory,
|
||||
*,
|
||||
requested_by: str,
|
||||
) -> Optional[Tuple[bool, str]]:
|
||||
"""将 durable 历史重试交还持久调度器,旧历史返回 ``None``。"""
|
||||
task_id = getattr(history, "transfer_task_id", None)
|
||||
if not task_id:
|
||||
return None
|
||||
try:
|
||||
result = TransferExecutionCommand(
|
||||
get_chain_transfer_execution_port()
|
||||
).request_retry(
|
||||
task_id=task_id,
|
||||
reason=f"用户请求重试整理历史 #{history.id}",
|
||||
requested_by=requested_by,
|
||||
)
|
||||
except (RuntimeError, ValueError) as error:
|
||||
logger.warning(
|
||||
"登记 durable 整理重试失败:history_id=%s task_id=%s error=%s",
|
||||
history.id,
|
||||
task_id,
|
||||
error,
|
||||
)
|
||||
return False, str(error)
|
||||
return result.accepted, result.message
|
||||
|
||||
# 字幕文件常见的语言/默认/强制标记,整理同名字幕时只允许剥离这些字幕专属尾缀。
|
||||
SUBTITLE_STEM_TAGS = {
|
||||
"cc",
|
||||
@@ -1233,12 +1263,40 @@ class ManualHistoryMixin:
|
||||
histories[history.id] = history
|
||||
return list(histories.values())
|
||||
|
||||
@staticmethod
|
||||
def _request_durable_transfer_retry(
|
||||
self,
|
||||
history: TransferHistory,
|
||||
*,
|
||||
requested_by: str,
|
||||
) -> Optional[Tuple[bool, str]]:
|
||||
"""将 durable 历史重试交还持久调度器,旧历史返回 ``None``。
|
||||
|
||||
durable 任务的历史、目标文件和步骤证据共同描述一次可恢复执行。任何历史
|
||||
入口都只能登记重试意图,不能删除这些证据后重新准入一条并行任务。
|
||||
|
||||
:param history: 整理历史
|
||||
:param requested_by: 发起重试的稳定入口身份
|
||||
:return: durable 请求结果;旧历史返回 ``None`` 继续兼容流程
|
||||
"""
|
||||
return _request_durable_transfer_retry(
|
||||
history,
|
||||
requested_by=requested_by,
|
||||
)
|
||||
|
||||
def _delete_manual_transfer_history(
|
||||
self,
|
||||
history: TransferHistory,
|
||||
transfer_history_oper: Any,
|
||||
) -> Tuple[bool, str]:
|
||||
"""删除手动重整历史;非成功移动记录同时清理可能存在的旧目标。"""
|
||||
durable_retry = self._request_durable_transfer_retry(
|
||||
history,
|
||||
requested_by="manual_reorganize",
|
||||
)
|
||||
if durable_retry is not None:
|
||||
_, message = durable_retry
|
||||
# 返回 False 阻止旧 caller 把“已登记”误当成“历史已删除”并继续执行。
|
||||
return False, message
|
||||
if (
|
||||
history.dest_fileitem
|
||||
and not ManualHistoryMixin._is_successful_move_history(history)
|
||||
@@ -1365,7 +1423,7 @@ class FailedRetryMixin:
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title=f"整理记录 #{history_id} 已重新整理",
|
||||
title=errmsg or f"整理记录 #{history_id} 已重新整理",
|
||||
link=self.runtime_config.history_url,
|
||||
save_history=False,
|
||||
)
|
||||
@@ -1397,6 +1455,47 @@ class FailedRetryMixin:
|
||||
由智能助手接管一条失败的整理记录。
|
||||
"""
|
||||
|
||||
history = get_chain_transfer_history_port().get(history_id)
|
||||
if not history:
|
||||
host = cast(TransferMixinHost, self)
|
||||
host.post_message(
|
||||
Message(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="重新整理失败",
|
||||
text=f"整理记录 #{history_id} 不存在",
|
||||
link=host.runtime_config.history_url,
|
||||
save_history=False,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
durable_retry = _request_durable_transfer_retry(
|
||||
history,
|
||||
requested_by="ai_retry_button",
|
||||
)
|
||||
if durable_retry is not None:
|
||||
accepted, message = durable_retry
|
||||
self.post_message(
|
||||
Message(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title=(
|
||||
message
|
||||
if accepted
|
||||
else "重新整理失败"
|
||||
),
|
||||
text=None if accepted else message,
|
||||
link=self.runtime_config.history_url,
|
||||
save_history=False,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
if not self.runtime_config.ai_agent_enable:
|
||||
self.post_message(
|
||||
Message(
|
||||
@@ -1410,22 +1509,6 @@ class FailedRetryMixin:
|
||||
)
|
||||
return
|
||||
|
||||
history = get_chain_transfer_history_port().get(history_id)
|
||||
if not history:
|
||||
self.post_message(
|
||||
Message(
|
||||
channel=channel,
|
||||
source=source,
|
||||
userid=userid,
|
||||
username=username,
|
||||
title="重新整理失败",
|
||||
text=f"整理记录 #{history_id} 不存在",
|
||||
link=self.runtime_config.history_url,
|
||||
save_history=False,
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
redo_prompt = build_manual_redo_prompt(history)
|
||||
|
||||
async def _run_ai_takeover():
|
||||
@@ -1528,6 +1611,12 @@ class FailedRetryMixin:
|
||||
if not history:
|
||||
logger.error(f"整理记录不存在,ID:{logid}")
|
||||
return False, "整理记录不存在"
|
||||
durable_retry = _request_durable_transfer_retry(
|
||||
history,
|
||||
requested_by="history_redo",
|
||||
)
|
||||
if durable_retry is not None:
|
||||
return durable_retry
|
||||
# 按源目录路径重新整理
|
||||
src_path = Path(history.src)
|
||||
if not src_path.exists():
|
||||
|
||||
+744
-69
File diff suppressed because it is too large
Load Diff
+336
-24
@@ -3,13 +3,17 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.application.chain.durable_events import (
|
||||
ChainDurableEventWriter,
|
||||
TransferHistoryRef,
|
||||
TransferResultSettlement,
|
||||
download_added_event_key,
|
||||
snapshot_download_added,
|
||||
snapshot_transfer_result,
|
||||
@@ -17,22 +21,40 @@ from app.application.chain.durable_events import (
|
||||
)
|
||||
from app.application.history import TransferHistoryRecord, TransferHistoryWriter
|
||||
from app.application.outbox import (
|
||||
DurableEventCommand,
|
||||
DOWNLOAD_ADDED_TOPIC,
|
||||
DurableEventCommand,
|
||||
OutboxIntent,
|
||||
)
|
||||
from app.application.transfer_execution import (
|
||||
TransferExecutionConflictError,
|
||||
TransferExecutionLeaseLostError,
|
||||
TransferExecutionState,
|
||||
TransferSettlementResult,
|
||||
)
|
||||
from app.db.adapters.outbox import SqlAlchemyOutboxRepository
|
||||
from app.db.models.transfersettlementreceipt import TransferSettlementReceipt
|
||||
from app.db.oper.downloadhistory import DownloadHistoryOper
|
||||
from app.db.oper.transferexecutionstep import TransferExecutionStepOper
|
||||
from app.db.oper.transferhistory import TransferHistoryOper
|
||||
from app.db.oper.transferpending import TransferPendingOper
|
||||
from app.db.oper.transfersettlementreceipt import TransferSettlementReceiptOper
|
||||
from app.db.uow import SqlAlchemyUnitOfWork
|
||||
|
||||
|
||||
class _StagingTransferHistoryWriter:
|
||||
"""让既有历史字段映射复用无提交的 replace 适配器。"""
|
||||
|
||||
def __init__(self, repository: TransferHistoryOper) -> None:
|
||||
"""保存绑定调用方 Session 的整理历史仓储。"""
|
||||
def __init__(
|
||||
self,
|
||||
repository: TransferHistoryOper,
|
||||
*,
|
||||
settlement: TransferResultSettlement | None = None,
|
||||
settlement_revision: int | None = None,
|
||||
) -> None:
|
||||
"""保存仓储,并让历史继续表达同源最新业务投影。"""
|
||||
self._repository = repository
|
||||
self._settlement = settlement
|
||||
self._settlement_revision = settlement_revision
|
||||
|
||||
def get_by_src(
|
||||
self,
|
||||
@@ -47,14 +69,40 @@ class _StagingTransferHistoryWriter:
|
||||
src: str,
|
||||
storage: str | None = None,
|
||||
) -> TransferHistoryRecord | None:
|
||||
"""转发按源路径读取成功记录。"""
|
||||
"""读取成功记录;任务结算时绑定当前任务投影。"""
|
||||
if self._settlement is not None:
|
||||
if self._settlement_revision is None:
|
||||
raise RuntimeError("整理任务结算缺少事务内修订号")
|
||||
return self._repository.stage_bind_settlement(
|
||||
task_id=self._settlement.task_id,
|
||||
settlement_revision=self._settlement_revision,
|
||||
src=src,
|
||||
storage=storage,
|
||||
)
|
||||
return self._repository.get_success_by_src(src, storage)
|
||||
|
||||
def add_force(self, **payload: Any) -> TransferHistoryRecord:
|
||||
"""保持应用层旧端口名,但只暂存替换而不自行提交。"""
|
||||
"""保持旧端口名,并按是否存在任务身份选择暂存策略。"""
|
||||
if self._settlement is not None:
|
||||
if self._settlement_revision is None:
|
||||
raise RuntimeError("整理任务结算缺少事务内修订号")
|
||||
return self._repository.stage_upsert_by_transfer_task_id(
|
||||
task_id=self._settlement.task_id,
|
||||
settlement_revision=self._settlement_revision,
|
||||
retain_task_mapping=self._settlement.outcome == "failed",
|
||||
payload=payload,
|
||||
)
|
||||
return self._repository.stage_replace_by_src(**payload)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _StagedTransferResult:
|
||||
"""保存构造 outbox 所需历史投影及可选任务结算结果。"""
|
||||
|
||||
history: TransferHistoryRef
|
||||
settlement: TransferSettlementResult | None = None
|
||||
|
||||
|
||||
class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
|
||||
"""为每次 Chain 结果事件创建独占同步 Session 和 UoW。"""
|
||||
|
||||
@@ -109,41 +157,111 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
|
||||
def transfer_result(
|
||||
self,
|
||||
*,
|
||||
topic: str,
|
||||
topic: str | None,
|
||||
stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None],
|
||||
event_payload: dict[str, Any],
|
||||
publish: Callable[[dict[str, Any]], None],
|
||||
) -> TransferHistoryRecord | None:
|
||||
"""原子写整理历史与结果 intent,并返回脱离 Session 的最小投影。"""
|
||||
publish: Callable[[dict[str, Any]], None] | None,
|
||||
settlement: TransferResultSettlement | None = None,
|
||||
) -> TransferHistoryRecord | TransferSettlementResult | None:
|
||||
"""原子写历史、可选任务终态与 intent,再返回稳定投影。"""
|
||||
if topic is None and settlement is None:
|
||||
raise ValueError("无事件 topic 的整理写入必须绑定 durable 任务结算")
|
||||
session = self._session_factory()
|
||||
try:
|
||||
staging = _StagingTransferHistoryWriter(TransferHistoryOper(session))
|
||||
history_repository = TransferHistoryOper(session)
|
||||
pending_repository = TransferPendingOper(session)
|
||||
receipt_repository = TransferSettlementReceiptOper(session)
|
||||
already_settled = self._read_settlement_result(
|
||||
pending_repository=pending_repository,
|
||||
receipt_repository=receipt_repository,
|
||||
settlement=settlement,
|
||||
)
|
||||
if already_settled is not None:
|
||||
return already_settled
|
||||
|
||||
command = DurableEventCommand(
|
||||
unit_of_work=SqlAlchemyUnitOfWork(session),
|
||||
outbox=SqlAlchemyOutboxRepository(session),
|
||||
)
|
||||
|
||||
def stage_business() -> TransferHistoryRef | None:
|
||||
"""复用历史字段映射,并在 flush 后冻结安全投影。"""
|
||||
def stage_business() -> _StagedTransferResult:
|
||||
"""同一事务暂存历史及受 fencing 保护的 pending 终态。"""
|
||||
expected_revision = self._settlement_revision(
|
||||
pending_repository=pending_repository,
|
||||
settlement=settlement,
|
||||
)
|
||||
next_revision = (
|
||||
expected_revision + 1
|
||||
if expected_revision is not None
|
||||
else None
|
||||
)
|
||||
staging = _StagingTransferHistoryWriter(
|
||||
history_repository,
|
||||
settlement=settlement,
|
||||
settlement_revision=next_revision,
|
||||
)
|
||||
history = stage_history(staging)
|
||||
if history is None:
|
||||
return None
|
||||
return TransferHistoryRef(
|
||||
raise RuntimeError("整理历史暂存失败,无法登记 durable 结果事件")
|
||||
projected = TransferHistoryRef(
|
||||
id=history.id,
|
||||
status=bool(history.status),
|
||||
src=history.src,
|
||||
src_storage=history.src_storage,
|
||||
src_fileitem=history.src_fileitem,
|
||||
)
|
||||
if settlement is None:
|
||||
return _StagedTransferResult(history=projected)
|
||||
assert expected_revision is not None
|
||||
assert next_revision is not None
|
||||
self._validate_history_outcome(projected, settlement)
|
||||
pending_deleted = self._stage_pending_terminal(
|
||||
session=session,
|
||||
repository=pending_repository,
|
||||
settlement=settlement,
|
||||
expected_revision=expected_revision,
|
||||
history_id=projected.id,
|
||||
)
|
||||
settled_at = datetime.now(timezone.utc).isoformat()
|
||||
receipt_repository.stage_append(
|
||||
task_id=settlement.task_id,
|
||||
history_id=projected.id,
|
||||
settlement_revision=next_revision,
|
||||
outcome=settlement.outcome,
|
||||
execution_fingerprint=settlement.execution_fingerprint,
|
||||
lease_token=settlement.lease_token,
|
||||
history_status=projected.status,
|
||||
src=projected.src,
|
||||
src_storage=projected.src_storage,
|
||||
pending_deleted=pending_deleted,
|
||||
error=settlement.error,
|
||||
settled_at=settled_at,
|
||||
)
|
||||
return _StagedTransferResult(
|
||||
history=projected,
|
||||
settlement=TransferSettlementResult(
|
||||
history_id=projected.id,
|
||||
settlement_revision=next_revision,
|
||||
pending_deleted=pending_deleted,
|
||||
),
|
||||
)
|
||||
|
||||
def build_intent(
|
||||
history: TransferHistoryRef | None,
|
||||
result: _StagedTransferResult,
|
||||
) -> OutboxIntent:
|
||||
"""历史 ID 确定后构造事件键与可恢复快照。"""
|
||||
if history is None:
|
||||
raise RuntimeError("整理历史暂存失败,无法登记 durable 结果事件")
|
||||
event_key = transfer_result_event_key(topic, history.id)
|
||||
event_payload["transfer_history_id"] = history.id
|
||||
assert topic is not None
|
||||
event_key = transfer_result_event_key(
|
||||
topic,
|
||||
result.history.id,
|
||||
settlement=settlement,
|
||||
settlement_revision=(
|
||||
result.settlement.settlement_revision
|
||||
if result.settlement is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
event_payload["transfer_history_id"] = result.history.id
|
||||
event_payload["idempotency_key"] = event_key
|
||||
return OutboxIntent(
|
||||
event_key=event_key,
|
||||
@@ -151,10 +269,204 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
|
||||
payload=snapshot_transfer_result(event_payload),
|
||||
)
|
||||
|
||||
return command.execute(
|
||||
intent=build_intent,
|
||||
stage_business=stage_business,
|
||||
publish=lambda: publish(event_payload),
|
||||
)
|
||||
try:
|
||||
result = command.execute(
|
||||
intent=build_intent if topic is not None else None,
|
||||
stage_business=stage_business,
|
||||
publish=(
|
||||
(lambda: publish(event_payload))
|
||||
if (
|
||||
settlement is None
|
||||
and topic is not None
|
||||
and publish is not None
|
||||
)
|
||||
else None
|
||||
),
|
||||
)
|
||||
except (
|
||||
IntegrityError,
|
||||
TransferExecutionConflictError,
|
||||
TransferExecutionLeaseLostError,
|
||||
ValueError,
|
||||
):
|
||||
if settlement is None:
|
||||
raise
|
||||
session.rollback()
|
||||
replay = self._read_settlement_result(
|
||||
pending_repository=pending_repository,
|
||||
receipt_repository=receipt_repository,
|
||||
settlement=settlement,
|
||||
)
|
||||
if replay is None:
|
||||
raise
|
||||
return replay
|
||||
return result.settlement or result.history
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
@staticmethod
|
||||
def _read_settlement_result(
|
||||
*,
|
||||
pending_repository: TransferPendingOper,
|
||||
receipt_repository: TransferSettlementReceiptOper,
|
||||
settlement: TransferResultSettlement | None,
|
||||
) -> TransferSettlementResult | None:
|
||||
"""识别已提交终态并返回幂等结果,未结算时返回空。"""
|
||||
if settlement is None:
|
||||
return None
|
||||
pending = pending_repository.get_by_task_id(task_id=settlement.task_id)
|
||||
latest = receipt_repository.get_latest_by_task_id(
|
||||
task_id=settlement.task_id
|
||||
)
|
||||
receipt = receipt_repository.get_by_identity(
|
||||
task_id=settlement.task_id,
|
||||
execution_fingerprint=settlement.execution_fingerprint,
|
||||
lease_token=settlement.lease_token,
|
||||
outcome=settlement.outcome,
|
||||
)
|
||||
if (
|
||||
pending is not None
|
||||
and pending.execution_state == TransferExecutionState.FAILED.value
|
||||
):
|
||||
if (
|
||||
latest is None
|
||||
or pending.terminal_history_id != latest.history_id
|
||||
or pending.settlement_revision != latest.settlement_revision
|
||||
or pending.execution_fingerprint != latest.execution_fingerprint
|
||||
or latest.outcome != "failed"
|
||||
or latest.pending_deleted
|
||||
):
|
||||
raise TransferExecutionConflictError("失败终态与最新结算回执不一致")
|
||||
if receipt is not None:
|
||||
TransactionalChainDurableEventWriter._validate_receipt(
|
||||
receipt=receipt,
|
||||
settlement=settlement,
|
||||
)
|
||||
return TransferSettlementResult(
|
||||
history_id=receipt.history_id,
|
||||
settlement_revision=receipt.settlement_revision,
|
||||
pending_deleted=receipt.pending_deleted,
|
||||
already_settled=True,
|
||||
)
|
||||
if pending is None:
|
||||
if latest is None:
|
||||
raise TransferExecutionConflictError(
|
||||
"pending 已不存在且没有可验证的终态回执"
|
||||
)
|
||||
raise TransferExecutionConflictError("整理终态与 durable 回执不一致")
|
||||
if pending.execution_state != TransferExecutionState.FAILED.value:
|
||||
return None
|
||||
raise TransferExecutionConflictError("失败终态缺少匹配的结算回执")
|
||||
|
||||
@staticmethod
|
||||
def _validate_receipt(
|
||||
*,
|
||||
receipt: TransferSettlementReceipt,
|
||||
settlement: TransferResultSettlement,
|
||||
) -> None:
|
||||
"""校验重放请求与独立回执中的终态身份完全一致。"""
|
||||
expected_status = settlement.outcome == "succeeded"
|
||||
if (
|
||||
receipt.task_id != settlement.task_id
|
||||
or receipt.outcome != settlement.outcome
|
||||
or receipt.execution_fingerprint != settlement.execution_fingerprint
|
||||
or receipt.lease_token != settlement.lease_token
|
||||
or receipt.history_status is not expected_status
|
||||
or receipt.error != settlement.error
|
||||
):
|
||||
raise TransferExecutionConflictError("整理终态与 durable 回执不一致")
|
||||
|
||||
@staticmethod
|
||||
def _settlement_revision(
|
||||
*,
|
||||
pending_repository: TransferPendingOper,
|
||||
settlement: TransferResultSettlement | None,
|
||||
) -> int | None:
|
||||
"""从当前 pending 读取 CAS 基准修订号并校验执行身份。"""
|
||||
if settlement is None:
|
||||
return None
|
||||
pending = pending_repository.get_by_task_id(task_id=settlement.task_id)
|
||||
if pending is None:
|
||||
raise TransferExecutionLeaseLostError("整理任务 pending 已不存在")
|
||||
now_utc = TransactionalChainDurableEventWriter._format_utc(
|
||||
datetime.now(timezone.utc)
|
||||
)
|
||||
if (
|
||||
pending.lease_token != settlement.lease_token
|
||||
or pending.lease_expires_at is None
|
||||
or pending.lease_expires_at <= now_utc
|
||||
):
|
||||
raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管")
|
||||
if (
|
||||
pending.execution_state != TransferExecutionState.SETTLING.value
|
||||
or pending.execution_fingerprint
|
||||
!= settlement.execution_fingerprint
|
||||
):
|
||||
raise TransferExecutionConflictError("整理终态与执行检查点不匹配")
|
||||
return int(pending.settlement_revision)
|
||||
|
||||
@staticmethod
|
||||
def _stage_pending_terminal(
|
||||
*,
|
||||
session: Session,
|
||||
repository: TransferPendingOper,
|
||||
settlement: TransferResultSettlement,
|
||||
expected_revision: int,
|
||||
history_id: int,
|
||||
) -> bool:
|
||||
"""以同一修订和 lease CAS 收口 pending,成功时同时清理步骤。"""
|
||||
now = datetime.now(timezone.utc)
|
||||
now_utc = TransactionalChainDurableEventWriter._format_utc(now)
|
||||
if settlement.outcome == "succeeded":
|
||||
TransferExecutionStepOper(session).stage_delete_task(
|
||||
task_id=settlement.task_id
|
||||
)
|
||||
updated = repository.stage_delete_terminal_success(
|
||||
task_id=settlement.task_id,
|
||||
lease_token=settlement.lease_token,
|
||||
execution_fingerprint=settlement.execution_fingerprint,
|
||||
expected_revision=expected_revision,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
pending_deleted = True
|
||||
else:
|
||||
updated = repository.stage_terminal_failure(
|
||||
task_id=settlement.task_id,
|
||||
lease_token=settlement.lease_token,
|
||||
execution_fingerprint=settlement.execution_fingerprint,
|
||||
expected_revision=expected_revision,
|
||||
history_id=history_id,
|
||||
error=settlement.error,
|
||||
now_utc=now_utc,
|
||||
updated_at=now.astimezone().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
)
|
||||
pending_deleted = False
|
||||
if updated != 1:
|
||||
session.expire_all()
|
||||
current = repository.get_by_task_id(task_id=settlement.task_id)
|
||||
if (
|
||||
current is None
|
||||
or current.lease_token != settlement.lease_token
|
||||
or current.lease_expires_at is None
|
||||
or current.lease_expires_at <= now_utc
|
||||
):
|
||||
raise TransferExecutionLeaseLostError(
|
||||
"整理任务租约已失效或被其他 worker 接管"
|
||||
)
|
||||
raise TransferExecutionConflictError("整理终态结算版本发生冲突")
|
||||
return pending_deleted
|
||||
|
||||
@staticmethod
|
||||
def _validate_history_outcome(
|
||||
history: TransferHistoryRef,
|
||||
settlement: TransferResultSettlement,
|
||||
) -> None:
|
||||
"""拒绝结算终态与历史状态不一致的调用。"""
|
||||
expected_status = settlement.outcome == "succeeded"
|
||||
if history.status is not expected_status:
|
||||
raise TransferExecutionConflictError("整理终态与历史状态不一致")
|
||||
|
||||
@staticmethod
|
||||
def _format_utc(value: datetime) -> str:
|
||||
"""编码与 pending lease 列一致的固定宽度 UTC 时间。"""
|
||||
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
|
||||
@@ -145,6 +145,8 @@ class TransactionalTransferAdmissionRepository:
|
||||
planning_input: Optional[TransferPlanningInput] = None,
|
||||
) -> TransferAdmission:
|
||||
"""按输入指纹幂等持久化准入事实,并返回跨重启稳定身份。"""
|
||||
if not storage or not src_path:
|
||||
raise ValueError("整理任务的存储与源路径不能为空")
|
||||
effective_input = planning_input or TransferPlanningInput.legacy(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
@@ -170,7 +172,9 @@ class TransactionalTransferAdmissionRepository:
|
||||
input_fingerprint=effective_input.fingerprint,
|
||||
)
|
||||
if pending is None:
|
||||
raise ValueError("整理任务的存储与源路径不能为空")
|
||||
raise TransferAdmissionConflictError(
|
||||
f"整理源文件已有持久终态回执: {storage}:{src_path}"
|
||||
)
|
||||
session.flush()
|
||||
self._assert_input_match(pending, effective_input)
|
||||
admission = self._project(pending)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -37,6 +37,14 @@ _MODEL_EXPORTS = {
|
||||
),
|
||||
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
|
||||
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
|
||||
"TransferExecutionStep": (
|
||||
"app.db.models.transferexecutionstep",
|
||||
"TransferExecutionStep",
|
||||
),
|
||||
"TransferSettlementReceipt": (
|
||||
"app.db.models.transfersettlementreceipt",
|
||||
"TransferSettlementReceipt",
|
||||
),
|
||||
"TransferPending": ("app.db.models.transferpending", "TransferPending"),
|
||||
"User": ("app.db.models.user", "User"),
|
||||
"UserConfig": ("app.db.models.userconfig", "UserConfig"),
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""整理任务外部操作步骤的持久化模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
delete,
|
||||
exists,
|
||||
or_,
|
||||
select,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
from sqlalchemy.sql.selectable import Exists
|
||||
|
||||
from app.db.base import Base, execute_dml, get_id_column
|
||||
from app.db.models.transferpending import TransferPending
|
||||
|
||||
|
||||
class TransferExecutionStep(Base):
|
||||
"""保存一次稳定外部操作的意图、尝试身份与结果证据。"""
|
||||
|
||||
id = get_id_column()
|
||||
task_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("transferpending.task_id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
operation_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
checkpoint_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="prepared")
|
||||
attempt_token: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
intent_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
intent_payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
result_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
result_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
|
||||
last_error: Mapped[Optional[str]] = mapped_column(Text)
|
||||
prepared_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
started_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
completed_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("operation_id", name="uq_transferexecutionstep_operation_id"),
|
||||
UniqueConstraint(
|
||||
"task_id",
|
||||
"ordinal",
|
||||
name="uq_transferexecutionstep_task_ordinal",
|
||||
),
|
||||
Index(
|
||||
"ix_transferexecutionstep_task_state_ordinal",
|
||||
"task_id",
|
||||
"state",
|
||||
"ordinal",
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_by_operation_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> Optional["TransferExecutionStep"]:
|
||||
"""按稳定操作标识读取步骤。"""
|
||||
if not operation_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferExecutionStep"],
|
||||
db.execute(
|
||||
select(cls).where(cls.operation_id == operation_id)
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_by_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> list["TransferExecutionStep"]:
|
||||
"""按全局序号读取任务的全部外部操作步骤。"""
|
||||
if not task_id:
|
||||
return []
|
||||
return list(
|
||||
db.execute(
|
||||
select(cls)
|
||||
.where(cls.task_id == task_id)
|
||||
.order_by(cls.ordinal.asc())
|
||||
).scalars().all()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_prepare(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
checkpoint_fingerprint: str,
|
||||
ordinal: int,
|
||||
phase: str,
|
||||
kind: str,
|
||||
intent_version: int,
|
||||
intent_payload: dict[str, Any],
|
||||
now_time: str,
|
||||
) -> "TransferExecutionStep":
|
||||
"""在调用方事务中暂存尚未执行的稳定步骤意图。"""
|
||||
step = cls(
|
||||
task_id=task_id,
|
||||
operation_id=operation_id,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
ordinal=ordinal,
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
state="prepared",
|
||||
attempt_count=0,
|
||||
intent_version=intent_version,
|
||||
intent_payload=intent_payload,
|
||||
prepared_at=now_time,
|
||||
updated_at=now_time,
|
||||
)
|
||||
db.add(step)
|
||||
return step
|
||||
|
||||
@classmethod
|
||||
def start_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效任务租约 CAS 开始一次新的步骤尝试。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "prepared",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="started",
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def restart_after_not_applied(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
previous_attempt_token: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以 NOT_APPLIED 证据和旧 attempt token CAS 重启遗留步骤。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == previous_attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resume_failed_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以重试调度的新 lease CAS 恢复 FAILED 步骤并保留失败证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "failed",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="started",
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def complete_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以租约与 attempt 双 CAS 提交步骤成功证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="succeeded",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=None,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def fail_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以租约与 attempt 双 CAS 提交已知失败证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="failed",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=error,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def mark_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: Optional[str],
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以当前尝试身份隔离外部结果不可判定的步骤。"""
|
||||
attempt_match = (
|
||||
cls.attempt_token == attempt_token
|
||||
if attempt_token is not None
|
||||
else cls.attempt_token.is_(None)
|
||||
)
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state.in_(("prepared", "started", "failed")),
|
||||
attempt_match,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="manual_review",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=error,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resolve_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
target_state: str,
|
||||
reason: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""仅在 pending 同为无租约人工态时 CAS 提交步骤判定。"""
|
||||
if target_state not in {"failed", "succeeded"}:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "manual_review",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._manual_review_pending_exists(task_id=task_id),
|
||||
)
|
||||
.values(
|
||||
state=target_state,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=(reason if target_state == "failed" else None),
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def delete_by_task_id(cls, db: Session, *, task_id: str) -> int:
|
||||
"""在终态成功结算事务中删除任务的步骤证据。"""
|
||||
if not task_id:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
delete(cls).where(cls.task_id == task_id),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _active_lease_exists(
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
) -> Exists:
|
||||
"""构造关联 pending 行仍持有当前有效租约的 SQL 谓词。"""
|
||||
return exists(
|
||||
select(TransferPending.id).where(
|
||||
and_(
|
||||
TransferPending.task_id == task_id,
|
||||
TransferPending.lease_token == lease_token,
|
||||
TransferPending.lease_expires_at.is_not(None),
|
||||
TransferPending.lease_expires_at > now_utc,
|
||||
or_(
|
||||
TransferPending.execution_state == "running",
|
||||
TransferPending.execution_state == "not_started",
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _manual_review_pending_exists(*, task_id: str) -> Exists:
|
||||
"""构造关联 pending 行处于无租约人工复核态的 SQL 谓词。"""
|
||||
return exists(
|
||||
select(TransferPending.id).where(
|
||||
TransferPending.task_id == task_id,
|
||||
TransferPending.execution_state == "manual_review",
|
||||
TransferPending.lease_token.is_(None),
|
||||
TransferPending.lease_owner.is_(None),
|
||||
)
|
||||
)
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
from typing import Any, List, Optional, cast
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, JSON, String, delete, func, or_, select, update
|
||||
from sqlalchemy import JSON, Boolean, Index, Integer, String, delete, func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
@@ -24,6 +24,10 @@ class TransferHistory(Base):
|
||||
整理记录
|
||||
"""
|
||||
id = get_id_column()
|
||||
# 失败 pending 当前映射使用的稳定整理任务标识
|
||||
transfer_task_id: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
# 失败 pending 当前映射对应的结算版本
|
||||
transfer_settlement_revision: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 源路径
|
||||
src: Mapped[Optional[str]] = mapped_column(String, index=True)
|
||||
# 源存储
|
||||
@@ -90,6 +94,11 @@ class TransferHistory(Base):
|
||||
Index('ix_transferhistory_date_id', 'date', 'id'),
|
||||
Index('ix_transferhistory_media_identity', 'media_source', 'media_id'),
|
||||
Index('ux_transferhistory_src_storage', 'src', 'src_storage', unique=True),
|
||||
Index(
|
||||
'ux_transferhistory_transfer_task_id',
|
||||
'transfer_task_id',
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -208,6 +217,82 @@ class TransferHistory(Base):
|
||||
statement = statement.where(cls.src_storage == storage)
|
||||
return db.execute(statement.order_by(cls.id.desc())).scalars().first()
|
||||
|
||||
@classmethod
|
||||
def get_by_transfer_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional["TransferHistory"]:
|
||||
"""按稳定整理任务标识读取终态结算历史。"""
|
||||
if not task_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferHistory"],
|
||||
db.execute(
|
||||
select(cls).where(cls.transfer_task_id == task_id)
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def upsert_by_transfer_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
settlement_revision: int,
|
||||
retain_task_mapping: bool,
|
||||
payload: dict[str, Any],
|
||||
) -> "TransferHistory":
|
||||
"""按任务幂等写历史,并维持同源存储仅保留一条的既有约束。"""
|
||||
if not task_id or settlement_revision <= 0:
|
||||
raise ValueError("整理历史结算缺少稳定任务或正向版本")
|
||||
column_names = {column.name for column in cls.__table__.columns}
|
||||
values = {
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key in column_names and key not in {
|
||||
"id",
|
||||
"transfer_task_id",
|
||||
"transfer_settlement_revision",
|
||||
}
|
||||
}
|
||||
src = values.get("src")
|
||||
if not src:
|
||||
raise ValueError("整理历史结算缺少源路径")
|
||||
src_storage = values.get("src_storage") or "local"
|
||||
values["src_storage"] = src_storage
|
||||
history = cls.get_by_transfer_task_id(db, task_id=task_id)
|
||||
if history is None:
|
||||
history = db.execute(
|
||||
select(cls).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
)
|
||||
).scalars().first()
|
||||
if history is None:
|
||||
history = cls(
|
||||
transfer_task_id=(task_id if retain_task_mapping else None),
|
||||
transfer_settlement_revision=(
|
||||
settlement_revision if retain_task_mapping else None
|
||||
),
|
||||
**values,
|
||||
)
|
||||
db.add(history)
|
||||
else:
|
||||
if (history.transfer_task_id == task_id
|
||||
and history.transfer_settlement_revision is not None
|
||||
and settlement_revision <= history.transfer_settlement_revision):
|
||||
raise ValueError("整理历史结算版本必须单调递增")
|
||||
history.transfer_task_id = task_id if retain_task_mapping else None
|
||||
history.transfer_settlement_revision = (
|
||||
settlement_revision if retain_task_mapping else None
|
||||
)
|
||||
for key, value in values.items():
|
||||
setattr(history, key, value)
|
||||
db.flush()
|
||||
return history
|
||||
|
||||
@classmethod
|
||||
def get_success_by_src(
|
||||
cls, db: Session, src: str,
|
||||
@@ -557,10 +642,20 @@ class TransferHistory(Base):
|
||||
src_storage = kwargs.get("src_storage") or "local"
|
||||
kwargs["src_storage"] = src_storage
|
||||
if src:
|
||||
durable = db.execute(
|
||||
select(cls.id).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
cls.transfer_task_id.is_not(None),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if durable is not None:
|
||||
raise ValueError("持久整理回执不能由旧历史写入口覆盖")
|
||||
db.execute(
|
||||
delete(cls).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
cls.transfer_task_id.is_(None),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
@@ -590,7 +685,10 @@ class TransferHistory(Base):
|
||||
"""
|
||||
ids = db.execute(
|
||||
select(cls.id)
|
||||
.where(cls.date < before_time)
|
||||
.where(
|
||||
cls.date < before_time,
|
||||
cls.transfer_task_id.is_(None),
|
||||
)
|
||||
.order_by(cls.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all()
|
||||
|
||||
@@ -134,6 +134,42 @@ class TransferPending(Base):
|
||||
heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 真正取得新 token 的累计次数
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 与规划状态正交的执行状态
|
||||
execution_state: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="not_started"
|
||||
)
|
||||
# 聚合执行检查点格式版本
|
||||
execution_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 可独立重放终态结算的聚合执行结果
|
||||
execution_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
|
||||
# 聚合执行结果规范 JSON 的 SHA-256 指纹
|
||||
execution_fingerprint: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
# 每次进入 retry_wait 都递增的调度世代
|
||||
retry_generation: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 已持久提交的步骤重试次数
|
||||
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 下一次允许 claim 的 UTC 时间
|
||||
retry_due_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 最近一次终态失败重试请求身份
|
||||
retry_requested_by: Mapped[Optional[str]] = mapped_column(String(128))
|
||||
# 最近一次终态失败重试请求原因
|
||||
retry_reason: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 已完成终态结算的单调版本
|
||||
settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 失败终态保留的整理历史标识
|
||||
terminal_history_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 人工判定的单调审计版本
|
||||
manual_review_revision: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0
|
||||
)
|
||||
# 最近一次人工判定时间
|
||||
reviewed_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 最近一次人工判定操作者
|
||||
reviewed_by: Mapped[Optional[str]] = mapped_column(String(128))
|
||||
# 最近一次人工判定原因
|
||||
review_reason: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 最近一次人工判定结论
|
||||
review_decision: Mapped[Optional[str]] = mapped_column(String(32))
|
||||
|
||||
__table_args__ = (
|
||||
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
|
||||
@@ -153,6 +189,14 @@ class TransferPending(Base):
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
Index(
|
||||
"ix_transferpending_execution_due",
|
||||
"execution_state",
|
||||
"retry_due_at",
|
||||
"state",
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
UniqueConstraint("task_id", name="uq_transferpending_task_id"),
|
||||
)
|
||||
|
||||
@@ -264,6 +308,19 @@ class TransferPending(Base):
|
||||
cursor_created_at = func.coalesce(cls.created_at, "")
|
||||
statement = select(cls.task_id, cursor_created_at, cls.id).where(
|
||||
cls.state.in_(states),
|
||||
cls.execution_state.in_((
|
||||
"not_started",
|
||||
"running",
|
||||
"retry_wait",
|
||||
"settling",
|
||||
)),
|
||||
or_(
|
||||
cls.execution_state != "retry_wait",
|
||||
and_(
|
||||
cls.retry_due_at.is_not(None),
|
||||
cls.retry_due_at <= now_time,
|
||||
),
|
||||
),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
@@ -331,6 +388,19 @@ class TransferPending(Base):
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(states),
|
||||
cls.execution_state.in_((
|
||||
"not_started",
|
||||
"running",
|
||||
"retry_wait",
|
||||
"settling",
|
||||
)),
|
||||
or_(
|
||||
cls.execution_state != "retry_wait",
|
||||
and_(
|
||||
cls.retry_due_at.is_not(None),
|
||||
cls.retry_due_at <= now_time,
|
||||
),
|
||||
),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
@@ -348,6 +418,345 @@ class TransferPending(Base):
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_execution_running(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约把可执行任务推进或保持为 running。"""
|
||||
if not all((task_id, lease_token, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
)
|
||||
.values(
|
||||
execution_state="running",
|
||||
retry_due_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def defer_execution(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
retry_due_at: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约进入 retry_wait,并原子释放当前租约。"""
|
||||
if not all((task_id, lease_token, error, retry_due_at, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_count=cls.retry_count + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def mark_execution_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约隔离执行结果未知的任务,并释放自动调度租约。"""
|
||||
if not all((task_id, lease_token, error, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="manual_review",
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def checkpoint_execution(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约保存可重放执行检查点并进入 settling。"""
|
||||
if not all((
|
||||
task_id,
|
||||
lease_token,
|
||||
execution_version,
|
||||
execution_payload,
|
||||
execution_fingerprint,
|
||||
now_utc,
|
||||
updated_at,
|
||||
)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="settling",
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
retry_due_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def checkpoint_exhausted_failure(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效 lease 保存预算耗尽失败检查点并保持租约进入 settling。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="settling",
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
retry_due_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def request_execution_retry(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
requested_by: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""仅将无租约 FAILED 任务 CAS 为立即到期的 retry_wait。"""
|
||||
if not all((task_id, reason, requested_by, retry_due_at, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "failed",
|
||||
cls.lease_token.is_(None),
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
retry_requested_by=requested_by,
|
||||
retry_reason=reason,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resolve_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
decision: str,
|
||||
actor: str,
|
||||
reason: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""无 lease 地 CAS 提交人工判定审计并交回 retry_wait 调度。"""
|
||||
if decision not in {"not_applied", "applied"}:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "manual_review",
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_owner.is_(None),
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
manual_review_revision=cls.manual_review_revision + 1,
|
||||
reviewed_at=updated_at,
|
||||
reviewed_by=actor,
|
||||
review_reason=reason,
|
||||
review_decision=decision,
|
||||
last_error=(reason if decision == "not_applied" else None),
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_terminal_failure(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
history_id: int,
|
||||
error: Optional[str],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本 CAS 保留失败终态及其历史。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "settling",
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.settlement_revision == expected_revision,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="failed",
|
||||
settlement_revision=cls.settlement_revision + 1,
|
||||
terminal_history_id=history_id,
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def delete_terminal_success(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
now_utc: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本 CAS 删除已成功结算的 pending。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
delete(cls).where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "settling",
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.settlement_revision == expected_revision,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def record_projection_failure(
|
||||
cls,
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""整理任务终态结算回执模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, cast
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, String, Text, UniqueConstraint, desc, select
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
from app.db.base import Base, get_id_column
|
||||
|
||||
|
||||
class TransferSettlementReceipt(Base):
|
||||
"""按任务保存独立于最新历史投影的 durable 终态证据。"""
|
||||
|
||||
id = get_id_column()
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
history_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
outcome: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
execution_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
lease_token: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
history_status: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
src: Mapped[Optional[str]] = mapped_column(String)
|
||||
src_storage: Mapped[Optional[str]] = mapped_column(String)
|
||||
pending_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
error: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"task_id",
|
||||
"settlement_revision",
|
||||
name="uq_transfersettlementreceipt_task_revision",
|
||||
),
|
||||
Index(
|
||||
"ix_transfersettlementreceipt_task_revision",
|
||||
"task_id",
|
||||
"settlement_revision",
|
||||
),
|
||||
Index(
|
||||
"ix_transfersettlementreceipt_history_id",
|
||||
"history_id",
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_latest_by_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional["TransferSettlementReceipt"]:
|
||||
"""按稳定任务标识读取最新已提交结算回执。"""
|
||||
if not task_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferSettlementReceipt"],
|
||||
db.execute(
|
||||
select(cls)
|
||||
.where(cls.task_id == task_id)
|
||||
.order_by(desc(cls.settlement_revision))
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_by_identity(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
outcome: str,
|
||||
) -> Optional["TransferSettlementReceipt"]:
|
||||
"""按原始执行身份读取不可变结算回执。"""
|
||||
if not all((task_id, execution_fingerprint, lease_token, outcome)):
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferSettlementReceipt"],
|
||||
db.execute(
|
||||
select(cls).where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.lease_token == lease_token,
|
||||
cls.outcome == outcome,
|
||||
).order_by(desc(cls.settlement_revision))
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_append(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
history_id: int,
|
||||
settlement_revision: int,
|
||||
outcome: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
history_status: bool,
|
||||
src: Optional[str],
|
||||
src_storage: Optional[str],
|
||||
pending_deleted: bool,
|
||||
error: Optional[str],
|
||||
settled_at: str,
|
||||
) -> "TransferSettlementReceipt":
|
||||
"""按连续修订追加任务回执,旧修订证据永不覆盖。"""
|
||||
if not all((task_id, history_id, settlement_revision, outcome,
|
||||
execution_fingerprint, lease_token, settled_at)):
|
||||
raise ValueError("整理结算回执缺少稳定身份或结果证据")
|
||||
if outcome not in {"succeeded", "failed"}:
|
||||
raise ValueError(f"不支持的整理结算结果:{outcome}")
|
||||
if cls.get_by_identity(
|
||||
db,
|
||||
task_id=task_id,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
outcome=outcome,
|
||||
) is not None:
|
||||
raise ValueError("同一整理执行身份不能追加多个结算修订")
|
||||
latest = cls.get_latest_by_task_id(db, task_id=task_id)
|
||||
if latest is None:
|
||||
if settlement_revision != 1:
|
||||
raise ValueError("整理结算回执必须从修订 1 开始")
|
||||
elif settlement_revision != latest.settlement_revision + 1:
|
||||
raise ValueError("整理结算回执修订必须连续递增")
|
||||
receipt = cls(
|
||||
task_id=task_id,
|
||||
history_id=history_id,
|
||||
settlement_revision=settlement_revision,
|
||||
outcome=outcome,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
history_status=history_status,
|
||||
src=src,
|
||||
src_storage=src_storage,
|
||||
pending_deleted=pending_deleted,
|
||||
error=error,
|
||||
created_at=settled_at,
|
||||
updated_at=settled_at,
|
||||
)
|
||||
db.add(receipt)
|
||||
db.flush()
|
||||
return receipt
|
||||
@@ -33,6 +33,7 @@ if TYPE_CHECKING:
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.db.oper.transferhistory import TransferHistoryOper
|
||||
from app.db.oper.transferpending import TransferPendingOper
|
||||
from app.db.oper.transfersettlementreceipt import TransferSettlementReceiptOper
|
||||
from app.db.oper.user import UserOper
|
||||
from app.db.oper.userconfig import UserConfigOper
|
||||
from app.db.oper.workflow import WorkflowOper
|
||||
@@ -52,6 +53,7 @@ _OPER_MODULES = {
|
||||
"SystemConfigOper": "systemconfig",
|
||||
"TransferHistoryOper": "transferhistory",
|
||||
"TransferPendingOper": "transferpending",
|
||||
"TransferSettlementReceiptOper": "transfersettlementreceipt",
|
||||
"UserConfigOper": "userconfig",
|
||||
"UserOper": "user",
|
||||
"WorkflowOper": "workflow",
|
||||
@@ -94,6 +96,7 @@ __all__ = [
|
||||
"SystemConfigOper",
|
||||
"TransferHistoryOper",
|
||||
"TransferPendingOper",
|
||||
"TransferSettlementReceiptOper",
|
||||
"UserConfigOper",
|
||||
"UserOper",
|
||||
"WorkflowOper",
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""整理外部操作步骤的显式 Session 数据访问对象。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.base import DbOper
|
||||
from app.db.models.transferexecutionstep import TransferExecutionStep
|
||||
|
||||
|
||||
class TransferExecutionStepOper(DbOper):
|
||||
"""在调用方事务中查询或暂存整理外部操作步骤。"""
|
||||
|
||||
def _session(self) -> Session:
|
||||
"""返回调用方同步 Session,拒绝隐式事务破坏原子状态推进。"""
|
||||
if not isinstance(self._db, Session):
|
||||
raise RuntimeError("整理执行步骤写入需要调用方提供同步 Session")
|
||||
return self._db
|
||||
|
||||
def get_by_operation_id(
|
||||
self,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> Optional[TransferExecutionStep]:
|
||||
"""按稳定操作标识查询步骤。"""
|
||||
return TransferExecutionStep.get_by_operation_id(
|
||||
self._session(),
|
||||
operation_id=operation_id,
|
||||
)
|
||||
|
||||
def list_by_task_id(self, *, task_id: str) -> list[TransferExecutionStep]:
|
||||
"""按全局序号查询任务的全部步骤。"""
|
||||
return TransferExecutionStep.list_by_task_id(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
)
|
||||
|
||||
def stage_prepare(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
checkpoint_fingerprint: str,
|
||||
ordinal: int,
|
||||
phase: str,
|
||||
kind: str,
|
||||
intent_version: int,
|
||||
intent_payload: dict[str, Any],
|
||||
now_time: str,
|
||||
) -> TransferExecutionStep:
|
||||
"""暂存尚未执行的稳定步骤意图。"""
|
||||
return TransferExecutionStep.stage_prepare(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
operation_id=operation_id,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
ordinal=ordinal,
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
intent_version=intent_version,
|
||||
intent_payload=intent_payload,
|
||||
now_time=now_time,
|
||||
)
|
||||
|
||||
def stage_start_attempt(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效任务租约暂存新步骤尝试。"""
|
||||
return TransferExecutionStep.start_attempt(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_restart_after_not_applied(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
previous_attempt_token: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以严格未发生证据暂存遗留 STARTED 步骤的安全重启。"""
|
||||
return TransferExecutionStep.restart_after_not_applied(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
previous_attempt_token=previous_attempt_token,
|
||||
attempt_token=attempt_token,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_resume_failed_attempt(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以重试调度的新 lease 暂存 FAILED 步骤恢复。"""
|
||||
return TransferExecutionStep.resume_failed_attempt(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_complete_attempt(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以 lease 与 attempt 双 CAS 暂存成功证据。"""
|
||||
return TransferExecutionStep.complete_attempt(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_fail_attempt(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以 lease 与 attempt 双 CAS 暂存已知失败证据。"""
|
||||
return TransferExecutionStep.fail_attempt(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
error=error,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: Optional[str],
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以当前尝试身份暂存人工复核证据。"""
|
||||
return TransferExecutionStep.mark_manual_review(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
operation_id=operation_id,
|
||||
attempt_token=attempt_token,
|
||||
error=error,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_resolve_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
target_state: str,
|
||||
reason: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""在 pending 同为无租约人工态时暂存步骤判定。"""
|
||||
return TransferExecutionStep.resolve_manual_review(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
operation_id=operation_id,
|
||||
target_state=target_state,
|
||||
reason=reason,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
def stage_delete_task(self, *, task_id: str) -> int:
|
||||
"""暂存任务全部步骤删除。"""
|
||||
return TransferExecutionStep.delete_by_task_id(
|
||||
self._session(),
|
||||
task_id=task_id,
|
||||
)
|
||||
@@ -2,6 +2,7 @@ import time
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from sqlalchemy import delete as sqlalchemy_delete
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.db.base import DbOper
|
||||
@@ -116,6 +117,19 @@ class TransferHistoryOper(DbOper):
|
||||
lambda session: TransferHistory.get_by_src(session, src, storage)
|
||||
)
|
||||
|
||||
def get_by_transfer_task_id(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional[TransferHistory]:
|
||||
"""按稳定整理任务标识读取终态历史。"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: TransferHistory.get_by_transfer_task_id(
|
||||
session,
|
||||
task_id=task_id,
|
||||
)
|
||||
)
|
||||
|
||||
def get_success_by_src(
|
||||
self, src: str, storage: Optional[str] = None
|
||||
) -> Optional[TransferHistory]:
|
||||
@@ -264,33 +278,60 @@ class TransferHistoryOper(DbOper):
|
||||
|
||||
def delete(self, historyid):
|
||||
"""
|
||||
删除转移记录
|
||||
删除旧转移记录,失败任务历史由状态机独占。
|
||||
"""
|
||||
self._stage_delete(TransferHistory, historyid)
|
||||
self._execute_sync_write(
|
||||
lambda session: session.execute(
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.id == historyid,
|
||||
TransferHistory.transfer_task_id.is_(None),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def stage_delete(self, historyid: int) -> None:
|
||||
"""暂存整理记录删除,事务由调用方统一提交。"""
|
||||
self._db.execute(
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.id == historyid
|
||||
TransferHistory.id == historyid,
|
||||
TransferHistory.transfer_task_id.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
def stage_truncate(self) -> None:
|
||||
"""暂存全部整理记录删除,由请求级事务统一提交。"""
|
||||
self._db.execute(sqlalchemy_delete(TransferHistory))
|
||||
"""暂存旧整理记录删除,只保留当前失败任务历史。"""
|
||||
self._db.execute(
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.transfer_task_id.is_(None)
|
||||
)
|
||||
)
|
||||
|
||||
async def async_delete(self, historyid):
|
||||
"""
|
||||
异步删除转移记录。
|
||||
异步删除旧转移记录,失败任务历史由状态机独占。
|
||||
"""
|
||||
await self._stage_async_delete(TransferHistory, historyid)
|
||||
async def stage(session: AsyncSession) -> None:
|
||||
"""在异步事务内只删除没有任务回执的历史。"""
|
||||
await session.execute(
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.id == historyid,
|
||||
TransferHistory.transfer_task_id.is_(None),
|
||||
)
|
||||
)
|
||||
|
||||
await self._execute_async_write(stage)
|
||||
|
||||
def truncate(self):
|
||||
"""
|
||||
清空转移记录
|
||||
清空旧转移记录,只保留当前失败任务历史。
|
||||
"""
|
||||
self._stage_truncate(TransferHistory)
|
||||
self._execute_sync_write(
|
||||
lambda session: session.execute(
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.transfer_task_id.is_(None)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def add_force(self, **kwargs) -> Optional[TransferHistory]:
|
||||
"""
|
||||
@@ -327,6 +368,7 @@ class TransferHistoryOper(DbOper):
|
||||
sqlalchemy_delete(TransferHistory).where(
|
||||
TransferHistory.src == kwargs.get("src"),
|
||||
TransferHistory.src_storage == kwargs["src_storage"],
|
||||
TransferHistory.transfer_task_id.is_(None),
|
||||
)
|
||||
)
|
||||
self._db.flush()
|
||||
@@ -335,6 +377,47 @@ class TransferHistoryOper(DbOper):
|
||||
self._db.flush()
|
||||
return history
|
||||
|
||||
def stage_upsert_by_transfer_task_id(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
settlement_revision: int,
|
||||
retain_task_mapping: bool,
|
||||
payload: dict[str, Any],
|
||||
) -> TransferHistory:
|
||||
"""在调用方事务内按任务标识幂等暂存终态历史。"""
|
||||
if not isinstance(self._db, Session):
|
||||
raise RuntimeError("整理历史任务结算需要调用方提供同步 Session")
|
||||
payload = dict(payload)
|
||||
payload["src_storage"] = payload.get("src_storage") or "local"
|
||||
payload["date"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||||
return TransferHistory.upsert_by_transfer_task_id(
|
||||
self._db,
|
||||
task_id=task_id,
|
||||
settlement_revision=settlement_revision,
|
||||
retain_task_mapping=retain_task_mapping,
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
def stage_bind_settlement(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
settlement_revision: int,
|
||||
src: str,
|
||||
storage: Optional[str] = None,
|
||||
) -> Optional[TransferHistory]:
|
||||
"""复用已有成功历史且清除失败任务映射,不改写业务字段。"""
|
||||
if not isinstance(self._db, Session):
|
||||
raise RuntimeError("整理历史任务回执绑定需要调用方提供同步 Session")
|
||||
history = TransferHistory.get_success_by_src(self._db, src, storage)
|
||||
if history is None:
|
||||
return None
|
||||
history.transfer_task_id = None
|
||||
history.transfer_settlement_revision = None
|
||||
self._db.flush()
|
||||
return history
|
||||
|
||||
def update_download_hash(self, historyid, download_hash):
|
||||
"""
|
||||
补充转移记录download_hash
|
||||
|
||||
@@ -328,3 +328,210 @@ class TransferPendingOper(DbOper):
|
||||
now_time=now_time,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_execution_running(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约暂存任务执行状态为 running。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.stage_execution_running(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_defer_execution(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
retry_due_at: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""暂存重试世代和到期时间,并原子释放当前租约。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.defer_execution(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
retry_due_at=retry_due_at,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_mark_execution_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""暂存人工复核隔离状态,并原子释放当前租约。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.mark_execution_manual_review(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_checkpoint_execution(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约暂存聚合执行检查点并进入 settling。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.checkpoint_execution(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_checkpoint_exhausted_failure(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""暂存预算耗尽失败检查点,并保留有效 lease 进入 settling。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.checkpoint_exhausted_failure(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
error=error,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_request_execution_retry(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
requested_by: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""仅将 FAILED 任务暂存为立即到期的 retry_wait。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.request_execution_retry(
|
||||
session,
|
||||
task_id=task_id,
|
||||
reason=reason,
|
||||
requested_by=requested_by,
|
||||
retry_due_at=retry_due_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_resolve_manual_review(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
decision: str,
|
||||
actor: str,
|
||||
reason: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""无 lease 地暂存人工判定审计并交回 retry_wait 调度。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.resolve_manual_review(
|
||||
session,
|
||||
task_id=task_id,
|
||||
decision=decision,
|
||||
actor=actor,
|
||||
reason=reason,
|
||||
retry_due_at=retry_due_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_terminal_failure(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
history_id: int,
|
||||
error: Optional[str],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本暂存失败终态。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.stage_terminal_failure(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
expected_revision=expected_revision,
|
||||
history_id=history_id,
|
||||
error=error,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_delete_terminal_success(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
now_utc: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本暂存成功 pending 删除。"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.delete_terminal_success(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
expected_revision=expected_revision,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""整理任务终态结算回执的事务内数据访问。"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from app.db.base import DbOper
|
||||
from app.db.models.transfersettlementreceipt import TransferSettlementReceipt
|
||||
|
||||
|
||||
class TransferSettlementReceiptOper(DbOper):
|
||||
"""在调用方 Session 内读取和推进 durable 结算回执。"""
|
||||
|
||||
def get_latest_by_task_id(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional[TransferSettlementReceipt]:
|
||||
"""按稳定任务标识读取最新回执。"""
|
||||
return TransferSettlementReceipt.get_latest_by_task_id(
|
||||
self._db,
|
||||
task_id=task_id,
|
||||
)
|
||||
|
||||
def get_by_identity(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
outcome: str,
|
||||
) -> Optional[TransferSettlementReceipt]:
|
||||
"""按原始执行身份读取不可变回执。"""
|
||||
return TransferSettlementReceipt.get_by_identity(
|
||||
self._db,
|
||||
task_id=task_id,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
outcome=outcome,
|
||||
)
|
||||
|
||||
def stage_append(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
history_id: int,
|
||||
settlement_revision: int,
|
||||
outcome: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
history_status: bool,
|
||||
src: Optional[str],
|
||||
src_storage: Optional[str],
|
||||
pending_deleted: bool,
|
||||
error: Optional[str],
|
||||
settled_at: str,
|
||||
) -> TransferSettlementReceipt:
|
||||
"""在调用方事务内按连续修订追加任务结算回执。"""
|
||||
return TransferSettlementReceipt.stage_append(
|
||||
self._db,
|
||||
task_id=task_id,
|
||||
history_id=history_id,
|
||||
settlement_revision=settlement_revision,
|
||||
outcome=outcome,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
history_status=history_status,
|
||||
src=src,
|
||||
src_storage=src_storage,
|
||||
pending_deleted=pending_deleted,
|
||||
error=error,
|
||||
settled_at=settled_at,
|
||||
)
|
||||
@@ -116,6 +116,8 @@
|
||||
"会话不存在或无权访问": "The conversation does not exist or you do not have access",
|
||||
"会话保存失败": "Failed to save conversation",
|
||||
"后台服务不存在": "Background service does not exist",
|
||||
"当前管理用户缺少可审计身份": "The current administrator does not have an auditable identity",
|
||||
"人工复核任务不存在": "The manual review task does not exist",
|
||||
"任务添加失败": "Failed to add task",
|
||||
"无法识别媒体信息": "Unable to recognize media information",
|
||||
"未识别到媒体信息": "Unable to recognize media information",
|
||||
|
||||
@@ -100,6 +100,8 @@
|
||||
}
|
||||
},
|
||||
"messages": {
|
||||
"当前管理用户缺少可审计身份": "当前管理用户缺少可审计身份",
|
||||
"人工复核任务不存在": "人工复核任务不存在",
|
||||
"媒体来源和媒体 ID 必须同时提供": "媒体来源和媒体 ID 必须同时提供",
|
||||
"media_source 和 media_id 必须同时提供": "media_source 和 media_id 必须同时提供",
|
||||
"模块不支持测试": "模块不支持测试",
|
||||
|
||||
@@ -112,6 +112,8 @@
|
||||
"会话不存在或无权访问": "會話不存在或無權存取",
|
||||
"会话保存失败": "會話儲存失敗",
|
||||
"后台服务不存在": "背景服務不存在",
|
||||
"当前管理用户缺少可审计身份": "目前管理使用者缺少可稽核身分",
|
||||
"人工复核任务不存在": "人工複核任務不存在",
|
||||
"任务添加失败": "任務新增失敗",
|
||||
"无法识别媒体信息": "無法識別媒體資訊",
|
||||
"未识别到媒体信息": "未識別到媒體資訊",
|
||||
|
||||
@@ -5,6 +5,7 @@ from app.adapters.system.host import SystemUtils
|
||||
from app.application.directory import DirectoryHelper
|
||||
from app.application.messaging.message import MessageHelper
|
||||
from app.application.transfer import TransferPlanCheckpoint, TransferPlanningInput
|
||||
from app.application.transfer_execution import TransferStepRunner
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
@@ -542,10 +543,13 @@ class FileManagerModule(_ModuleBase):
|
||||
source_oper: Optional[StorageBase] = None,
|
||||
target_oper: Optional[StorageBase] = None,
|
||||
cleanup_media_file: Optional[Callable[[FileItem], bool]] = None,
|
||||
observe_cleanup_media_file: Optional[Callable[[FileItem], bool]] = None,
|
||||
step_runner: Optional[TransferStepRunner] = None,
|
||||
) -> TransferInfo:
|
||||
"""解析存储适配器并通过统一删除能力执行已冻结计划。"""
|
||||
source_fileitem = FileItem(**checkpoint.planning_input.source_fileitem)
|
||||
cleanup_before_transfer = None
|
||||
observe_cleanup_before_transfer = None
|
||||
cleanup_payload = checkpoint.planning_input.options.get(
|
||||
"cleanup_dest_fileitem"
|
||||
)
|
||||
@@ -565,6 +569,11 @@ class FileManagerModule(_ModuleBase):
|
||||
f"{cleanup_fileitem.path} 删除失败,整理计划保留待重试"
|
||||
)
|
||||
|
||||
if observe_cleanup_media_file:
|
||||
def observe_cleanup_before_transfer() -> bool:
|
||||
"""只读确认旧目标是否已经由统一能力清理。"""
|
||||
return observe_cleanup_media_file(cleanup_fileitem)
|
||||
|
||||
source_storage = source_fileitem.storage or "local"
|
||||
if not source_oper:
|
||||
source_oper = self.__get_storage_oper(source_storage)
|
||||
@@ -595,6 +604,8 @@ class FileManagerModule(_ModuleBase):
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
cleanup_before_transfer=cleanup_before_transfer,
|
||||
observe_cleanup_before_transfer=observe_cleanup_before_transfer,
|
||||
step_runner=step_runner,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import filecmp
|
||||
import re
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
@@ -14,6 +15,12 @@ from app.application.transfer import (
|
||||
TransferPlanItem,
|
||||
TransferPlanningInput,
|
||||
)
|
||||
from app.application.transfer_execution import (
|
||||
TransferOperationObservation,
|
||||
TransferOperationObservationState,
|
||||
TransferStepResult,
|
||||
TransferStepRunner,
|
||||
)
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
@@ -625,6 +632,456 @@ class TransHandler:
|
||||
)
|
||||
return False, False, None
|
||||
|
||||
@staticmethod
|
||||
def __serialize_step_item(fileitem: Optional[FileItem]) -> Optional[dict[str, Any]]:
|
||||
"""把步骤结果中的文件投影冻结为 JSON 对象。"""
|
||||
return fileitem.model_dump(mode="json") if fileitem else None
|
||||
|
||||
@staticmethod
|
||||
def __restore_step_item(result: TransferStepResult) -> Optional[FileItem]:
|
||||
"""从已持久成功证据恢复目标文件,不再次访问外部存储。"""
|
||||
payload = result.payload.get("item")
|
||||
return FileItem.model_validate(payload) if isinstance(payload, dict) else None
|
||||
|
||||
@staticmethod
|
||||
def __observe_item_presence(
|
||||
storage_oper: StorageBase,
|
||||
path: Path,
|
||||
*,
|
||||
applied_when_present: bool,
|
||||
) -> TransferOperationObservation:
|
||||
"""以严格查询判断目标存在性,查询异常一律视为未知。"""
|
||||
try:
|
||||
item = storage_oper.get_item_strict(path)
|
||||
except Exception as error:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={
|
||||
"path": path.as_posix(),
|
||||
"query_error": str(error),
|
||||
}),
|
||||
)
|
||||
exists = item is not None
|
||||
applied = exists if applied_when_present else not exists
|
||||
return TransferOperationObservation(
|
||||
state=(
|
||||
TransferOperationObservationState.APPLIED
|
||||
if applied
|
||||
else TransferOperationObservationState.NOT_APPLIED
|
||||
),
|
||||
evidence=TransferStepResult(payload={
|
||||
"path": path.as_posix(),
|
||||
"exists": exists,
|
||||
"item": TransHandler.__serialize_step_item(item),
|
||||
}),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __run_persisted_step(
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
*,
|
||||
phase: str,
|
||||
kind: str,
|
||||
payload: dict[str, Any],
|
||||
execute: Callable[[], TransferStepResult],
|
||||
observe: Callable[[], TransferOperationObservation],
|
||||
) -> TransferStepResult:
|
||||
"""在持久任务中委托步骤账本,旧同步调用则直接执行。"""
|
||||
if step_runner is None:
|
||||
return execute()
|
||||
return step_runner.run(
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
payload=payload,
|
||||
execute=execute,
|
||||
observe=observe,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __observe_transfer_operation(
|
||||
*,
|
||||
fileitem: FileItem,
|
||||
target_storage: str,
|
||||
source_oper: StorageBase,
|
||||
target_oper: StorageBase,
|
||||
target_file: Path,
|
||||
transfer_type: str,
|
||||
) -> TransferOperationObservation:
|
||||
"""对遗留传输尝试作保守判定,证据不足时禁止自动重放。"""
|
||||
try:
|
||||
source_item = source_oper.get_item_strict(Path(cast(str, fileitem.path)))
|
||||
target_item = target_oper.get_item_strict(target_file)
|
||||
except Exception as error:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={"query_error": str(error)}),
|
||||
)
|
||||
|
||||
source_exists = source_item is not None
|
||||
target_exists = target_item is not None
|
||||
evidence = TransferStepResult(payload={
|
||||
"source_exists": source_exists,
|
||||
"target_exists": target_exists,
|
||||
"item": TransHandler.__serialize_step_item(target_item),
|
||||
})
|
||||
if transfer_type == "move":
|
||||
if not source_exists and target_exists:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.APPLIED,
|
||||
evidence=evidence,
|
||||
)
|
||||
if source_exists and not target_exists:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.NOT_APPLIED,
|
||||
evidence=evidence,
|
||||
)
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.CONFLICT,
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
if not target_exists:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.NOT_APPLIED,
|
||||
evidence=evidence,
|
||||
)
|
||||
if fileitem.storage != "local" or target_storage != "local":
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=evidence,
|
||||
)
|
||||
source_path = Path(cast(str, fileitem.path))
|
||||
try:
|
||||
if transfer_type == "copy":
|
||||
applied = source_path.is_file() and filecmp.cmp(
|
||||
source_path, target_file, shallow=False
|
||||
)
|
||||
elif transfer_type == "link":
|
||||
applied = source_path.is_file() and target_file.samefile(source_path)
|
||||
elif transfer_type == "softlink":
|
||||
applied = target_file.is_symlink() and target_file.resolve() == source_path.resolve()
|
||||
else:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=evidence,
|
||||
)
|
||||
except OSError as error:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={
|
||||
**evidence.payload,
|
||||
"verification_error": str(error),
|
||||
}),
|
||||
)
|
||||
return TransferOperationObservation(
|
||||
state=(
|
||||
TransferOperationObservationState.APPLIED
|
||||
if applied
|
||||
else TransferOperationObservationState.CONFLICT
|
||||
),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __execute_transfer_with_steps(
|
||||
cls,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
fileitem: FileItem,
|
||||
target_storage: str,
|
||||
source_oper: StorageBase,
|
||||
target_oper: StorageBase,
|
||||
target_file: Path,
|
||||
transfer_type: str,
|
||||
) -> tuple[Optional[FileItem], str]:
|
||||
"""执行稳定传输步骤,并把跨存储 move 拆为落地与源删除。"""
|
||||
cross_storage_move = (
|
||||
transfer_type == "move" and fileitem.storage != target_storage
|
||||
)
|
||||
materialize_type = "copy" if cross_storage_move else transfer_type
|
||||
intent_payload = {
|
||||
"source": fileitem.model_dump(mode="json"),
|
||||
"target_storage": target_storage,
|
||||
"target_path": target_file.as_posix(),
|
||||
"transfer_type": materialize_type,
|
||||
}
|
||||
|
||||
def execute_materialize() -> TransferStepResult:
|
||||
"""执行一次目标落地并冻结其返回对象。"""
|
||||
new_item, error = cls.__transfer_command(
|
||||
fileitem=fileitem,
|
||||
target_storage=target_storage,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
target_file=target_file,
|
||||
transfer_type=materialize_type,
|
||||
)
|
||||
if not new_item:
|
||||
raise RuntimeError(error or f"{fileitem.path} 整理失败")
|
||||
return TransferStepResult(payload={
|
||||
"item": cls.__serialize_step_item(new_item),
|
||||
"message": error,
|
||||
})
|
||||
|
||||
materialized = cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="transfer",
|
||||
kind="materialize_target",
|
||||
payload=intent_payload,
|
||||
execute=execute_materialize,
|
||||
observe=lambda: cls.__observe_transfer_operation(
|
||||
fileitem=fileitem,
|
||||
target_storage=target_storage,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
target_file=target_file,
|
||||
transfer_type=materialize_type,
|
||||
),
|
||||
)
|
||||
new_item = cls.__restore_step_item(materialized)
|
||||
if not new_item:
|
||||
return None, "整理步骤成功证据缺少目标文件"
|
||||
if not cross_storage_move:
|
||||
return new_item, str(materialized.payload.get("message") or "")
|
||||
|
||||
def execute_source_delete() -> TransferStepResult:
|
||||
"""在目标已落地后单独删除跨存储 move 的源文件。"""
|
||||
if not source_oper.delete(fileitem):
|
||||
raise RuntimeError(f"{fileitem.path} 源文件删除失败")
|
||||
return TransferStepResult(payload={
|
||||
"source_path": fileitem.path,
|
||||
"deleted": True,
|
||||
})
|
||||
|
||||
cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="transfer",
|
||||
kind="delete_move_source",
|
||||
payload={
|
||||
"source": fileitem.model_dump(mode="json"),
|
||||
"target_storage": target_storage,
|
||||
"target_path": target_file.as_posix(),
|
||||
},
|
||||
execute=execute_source_delete,
|
||||
observe=lambda: cls.__observe_item_presence(
|
||||
source_oper,
|
||||
Path(cast(str, fileitem.path)),
|
||||
applied_when_present=False,
|
||||
),
|
||||
)
|
||||
return new_item, str(materialized.payload.get("message") or "")
|
||||
|
||||
@classmethod
|
||||
def __ensure_directory_with_step(
|
||||
cls,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
target_oper: StorageBase,
|
||||
target_storage: str,
|
||||
path: Path,
|
||||
) -> Optional[FileItem]:
|
||||
"""持久记录可能创建目录的 get_folder 操作并恢复其结果。"""
|
||||
def execute() -> TransferStepResult:
|
||||
"""获取或创建目标目录并冻结目录对象。"""
|
||||
directory = target_oper.get_folder(path)
|
||||
if not directory:
|
||||
raise RuntimeError(f"目标目录 {path} 获取失败")
|
||||
return TransferStepResult(payload={
|
||||
"item": cls.__serialize_step_item(directory),
|
||||
})
|
||||
|
||||
result = cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="prepare",
|
||||
kind="ensure_target_directory",
|
||||
payload={"storage": target_storage, "path": path.as_posix()},
|
||||
execute=execute,
|
||||
observe=lambda: cls.__observe_item_presence(
|
||||
target_oper,
|
||||
path,
|
||||
applied_when_present=True,
|
||||
),
|
||||
)
|
||||
return cls.__restore_step_item(result)
|
||||
|
||||
@classmethod
|
||||
def __cleanup_with_step(
|
||||
cls,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
cleanup: Optional[Callable[[], None]],
|
||||
observe_cleanup: Optional[Callable[[], bool]],
|
||||
source_path: str,
|
||||
) -> None:
|
||||
"""把兼容清理能力纳入步骤账本,未知遗留结果必须人工复核。"""
|
||||
if cleanup is None:
|
||||
return
|
||||
|
||||
def execute() -> TransferStepResult:
|
||||
"""执行统一旧目标清理能力。"""
|
||||
cleanup()
|
||||
return TransferStepResult(payload={"cleaned": True})
|
||||
|
||||
def observe() -> TransferOperationObservation:
|
||||
"""通过只读兼容能力确认旧目标是否已经消失。"""
|
||||
if observe_cleanup is None:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={
|
||||
"reason": "cleanup observer unavailable",
|
||||
}),
|
||||
)
|
||||
try:
|
||||
cleaned = observe_cleanup()
|
||||
except Exception as error:
|
||||
return TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={"query_error": str(error)}),
|
||||
)
|
||||
return TransferOperationObservation(
|
||||
state=(
|
||||
TransferOperationObservationState.APPLIED
|
||||
if cleaned
|
||||
else TransferOperationObservationState.NOT_APPLIED
|
||||
),
|
||||
evidence=TransferStepResult(payload={"cleaned": cleaned}),
|
||||
)
|
||||
|
||||
cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="prepare",
|
||||
kind="cleanup_previous_destination",
|
||||
payload={"source_path": source_path},
|
||||
execute=execute,
|
||||
observe=observe,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __delete_target_with_step(
|
||||
cls,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
target_oper: StorageBase,
|
||||
target_storage: str,
|
||||
target_file: Path,
|
||||
) -> None:
|
||||
"""幂等删除覆盖目标,并持久记录删除意图和严格存在性证据。"""
|
||||
def execute() -> TransferStepResult:
|
||||
"""只删除当前冻结目标,目标已不存在视为成功。"""
|
||||
current = target_oper.get_item_strict(target_file)
|
||||
if current is not None and not target_oper.delete(current):
|
||||
raise RuntimeError(f"【{target_storage}】{target_file} 删除失败")
|
||||
return TransferStepResult(payload={"deleted": True})
|
||||
|
||||
cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="prepare",
|
||||
kind="delete_overwrite_target",
|
||||
payload={"storage": target_storage, "path": target_file.as_posix()},
|
||||
execute=execute,
|
||||
observe=lambda: cls.__observe_item_presence(
|
||||
target_oper,
|
||||
target_file,
|
||||
applied_when_present=False,
|
||||
),
|
||||
)
|
||||
|
||||
def __resolve_overwrite_with_step(
|
||||
self,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
fileitem: FileItem,
|
||||
meta: MetaBase,
|
||||
mediainfo: MediaInfo | MusicInfo,
|
||||
target_oper: StorageBase,
|
||||
target_storage: str,
|
||||
target_file: Path,
|
||||
transfer_type: str,
|
||||
overwrite_mode: Optional[str],
|
||||
need_notify: bool,
|
||||
) -> tuple[bool, bool, Optional[TransferInfo]]:
|
||||
"""冻结覆盖策略判定,避免目标变化后重启得到不同步骤序列。"""
|
||||
def execute() -> TransferStepResult:
|
||||
"""执行一次覆盖策略判定并冻结完整裁决。"""
|
||||
over_flag, delete_versions, failure = self.__resolve_overwrite(
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
target_file=target_file,
|
||||
transfer_type=transfer_type,
|
||||
overwrite_mode=overwrite_mode,
|
||||
need_notify=need_notify,
|
||||
)
|
||||
return TransferStepResult(payload={
|
||||
"over_flag": over_flag,
|
||||
"delete_versions": delete_versions,
|
||||
"failure": failure.model_dump(mode="json") if failure else None,
|
||||
})
|
||||
|
||||
result = self.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="decision",
|
||||
kind="resolve_overwrite",
|
||||
payload={
|
||||
"source": fileitem.model_dump(mode="json"),
|
||||
"target_storage": target_storage,
|
||||
"target_path": target_file.as_posix(),
|
||||
"transfer_type": transfer_type,
|
||||
"overwrite_mode": overwrite_mode,
|
||||
"need_notify": need_notify,
|
||||
},
|
||||
execute=execute,
|
||||
observe=lambda: TransferOperationObservation(
|
||||
state=TransferOperationObservationState.NOT_APPLIED,
|
||||
evidence=TransferStepResult(payload={
|
||||
"reason": "read-only overwrite decision may be repeated",
|
||||
}),
|
||||
),
|
||||
)
|
||||
failure_payload = result.payload.get("failure")
|
||||
return (
|
||||
bool(result.payload.get("over_flag")),
|
||||
bool(result.payload.get("delete_versions")),
|
||||
(
|
||||
TransferInfo.model_validate(failure_payload)
|
||||
if isinstance(failure_payload, dict)
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
def __intercept_with_step(
|
||||
self,
|
||||
*,
|
||||
step_runner: Optional[TransferStepRunner],
|
||||
payload: dict[str, Any],
|
||||
invoke: Callable[[], tuple[bool, str]],
|
||||
) -> tuple[bool, str]:
|
||||
"""冻结插件拦截裁决;遗留未回执调用因插件不透明而转人工复核。"""
|
||||
def execute() -> TransferStepResult:
|
||||
"""执行插件拦截并冻结允许标记与原因。"""
|
||||
allowed, reason = invoke()
|
||||
return TransferStepResult(payload={
|
||||
"allowed": allowed,
|
||||
"reason": reason,
|
||||
})
|
||||
|
||||
result = self.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="decision",
|
||||
kind="plugin_transfer_intercept",
|
||||
payload=payload,
|
||||
execute=execute,
|
||||
observe=lambda: TransferOperationObservation(
|
||||
state=TransferOperationObservationState.UNKNOWN,
|
||||
evidence=TransferStepResult(payload={
|
||||
"reason": "plugin intercept has no stable invocation receipt",
|
||||
}),
|
||||
),
|
||||
)
|
||||
return bool(result.payload.get("allowed")), str(result.payload.get("reason") or "")
|
||||
|
||||
def execute_transfer_plan(
|
||||
self,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
@@ -634,6 +1091,8 @@ class TransHandler:
|
||||
source_oper: StorageBase,
|
||||
target_oper: StorageBase,
|
||||
cleanup_before_transfer: Optional[Callable[[], None]] = None,
|
||||
observe_cleanup_before_transfer: Optional[Callable[[], bool]] = None,
|
||||
step_runner: Optional[TransferStepRunner] = None,
|
||||
) -> TransferInfo:
|
||||
"""只消费冻结目标和有序操作,并在执行期处理覆盖与插件拦截。"""
|
||||
fileitem = FileItem(**checkpoint.planning_input.source_fileitem)
|
||||
@@ -684,7 +1143,7 @@ class TransHandler:
|
||||
)
|
||||
|
||||
if fileitem.type == "dir":
|
||||
stream_path = Path(fileitem.path) / "BDMV" / "STREAM"
|
||||
stream_path = Path(cast(str, fileitem.path)) / "BDMV" / "STREAM"
|
||||
stream_sizes = [
|
||||
FileItem(**item.source_fileitem).size or 0
|
||||
for item in checkpoint.items
|
||||
@@ -693,13 +1152,22 @@ class TransHandler:
|
||||
]
|
||||
if stream_sizes:
|
||||
fileitem.size = sum(stream_sizes)
|
||||
allowed, reason = self.__intercept_transfer(
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_path=target_path,
|
||||
transfer_type=transfer_type,
|
||||
allowed, reason = self.__intercept_with_step(
|
||||
step_runner=step_runner,
|
||||
payload={
|
||||
"source": fileitem.model_dump(mode="json"),
|
||||
"target_storage": target_storage,
|
||||
"target_path": target_path.as_posix(),
|
||||
"transfer_type": transfer_type,
|
||||
},
|
||||
invoke=lambda: self.__intercept_transfer(
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_path=target_path,
|
||||
transfer_type=transfer_type,
|
||||
),
|
||||
)
|
||||
if not allowed:
|
||||
return TransferInfo(
|
||||
@@ -709,9 +1177,18 @@ class TransHandler:
|
||||
transfer_type=transfer_type,
|
||||
need_notify=checkpoint.need_notify,
|
||||
)
|
||||
if cleanup_before_transfer:
|
||||
cleanup_before_transfer()
|
||||
target_diritem = target_oper.get_folder(target_path)
|
||||
self.__cleanup_with_step(
|
||||
step_runner=step_runner,
|
||||
cleanup=cleanup_before_transfer,
|
||||
observe_cleanup=observe_cleanup_before_transfer,
|
||||
source_path=cast(str, fileitem.path),
|
||||
)
|
||||
target_diritem = self.__ensure_directory_with_step(
|
||||
step_runner=step_runner,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
path=target_path,
|
||||
)
|
||||
if not target_diritem:
|
||||
return TransferInfo(
|
||||
success=False,
|
||||
@@ -733,7 +1210,8 @@ class TransHandler:
|
||||
need_notify=checkpoint.need_notify,
|
||||
)
|
||||
source_item = FileItem(**planned_item.source_fileitem)
|
||||
new_item, error = self.__transfer_command(
|
||||
new_item, error = self.__execute_transfer_with_steps(
|
||||
step_runner=step_runner,
|
||||
fileitem=source_item,
|
||||
target_storage=planned_item.target_storage,
|
||||
source_oper=source_oper,
|
||||
@@ -790,7 +1268,8 @@ class TransHandler:
|
||||
need_notify=checkpoint.need_notify,
|
||||
)
|
||||
target_file = Path(planned_item.target_path)
|
||||
over_flag, delete_versions, overwrite_failure = self.__resolve_overwrite(
|
||||
over_flag, delete_versions, overwrite_failure = self.__resolve_overwrite_with_step(
|
||||
step_runner=step_runner,
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
@@ -803,14 +1282,24 @@ class TransHandler:
|
||||
)
|
||||
if overwrite_failure:
|
||||
return overwrite_failure
|
||||
allowed, reason = self.__intercept_transfer(
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_path=target_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=over_flag,
|
||||
allowed, reason = self.__intercept_with_step(
|
||||
step_runner=step_runner,
|
||||
payload={
|
||||
"source": fileitem.model_dump(mode="json"),
|
||||
"target_storage": target_storage,
|
||||
"target_path": target_file.as_posix(),
|
||||
"transfer_type": transfer_type,
|
||||
"over_flag": over_flag,
|
||||
},
|
||||
invoke=lambda: self.__intercept_transfer(
|
||||
fileitem=fileitem,
|
||||
meta=meta,
|
||||
mediainfo=mediainfo,
|
||||
target_storage=target_storage,
|
||||
target_path=target_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=over_flag,
|
||||
),
|
||||
)
|
||||
if not allowed:
|
||||
return TransferInfo(
|
||||
@@ -822,9 +1311,18 @@ class TransHandler:
|
||||
need_notify=checkpoint.need_notify,
|
||||
)
|
||||
|
||||
if cleanup_before_transfer:
|
||||
cleanup_before_transfer()
|
||||
target_diritem = target_oper.get_folder(target_file.parent)
|
||||
self.__cleanup_with_step(
|
||||
step_runner=step_runner,
|
||||
cleanup=cleanup_before_transfer,
|
||||
observe_cleanup=observe_cleanup_before_transfer,
|
||||
source_path=cast(str, fileitem.path),
|
||||
)
|
||||
target_diritem = self.__ensure_directory_with_step(
|
||||
step_runner=step_runner,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
path=target_file.parent,
|
||||
)
|
||||
if not target_diritem:
|
||||
return TransferInfo(
|
||||
success=False,
|
||||
@@ -835,17 +1333,51 @@ class TransHandler:
|
||||
need_notify=checkpoint.need_notify,
|
||||
)
|
||||
if delete_versions:
|
||||
self.__delete_version_files(target_oper, target_file)
|
||||
new_item, error = self.__transfer_file(
|
||||
fileitem=fileitem,
|
||||
target_storage=target_storage,
|
||||
target_file=target_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=over_flag,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
result=result,
|
||||
)
|
||||
if step_runner is None:
|
||||
self.__delete_version_files(target_oper, target_file)
|
||||
else:
|
||||
self.__delete_version_files_with_steps(
|
||||
step_runner=step_runner,
|
||||
storage_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
path=target_file,
|
||||
)
|
||||
if step_runner is None:
|
||||
new_item, error = self.__transfer_file(
|
||||
fileitem=fileitem,
|
||||
target_storage=target_storage,
|
||||
target_file=target_file,
|
||||
transfer_type=transfer_type,
|
||||
over_flag=over_flag,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
result=result,
|
||||
)
|
||||
else:
|
||||
if over_flag:
|
||||
self.__delete_target_with_step(
|
||||
step_runner=step_runner,
|
||||
target_oper=target_oper,
|
||||
target_storage=target_storage,
|
||||
target_file=target_file,
|
||||
)
|
||||
new_item, error = self.__execute_transfer_with_steps(
|
||||
step_runner=step_runner,
|
||||
fileitem=fileitem,
|
||||
target_storage=target_storage,
|
||||
source_oper=source_oper,
|
||||
target_oper=target_oper,
|
||||
target_file=target_file,
|
||||
transfer_type=transfer_type,
|
||||
)
|
||||
if new_item:
|
||||
self.__update_result(
|
||||
result=result,
|
||||
file_list=[fileitem.path],
|
||||
file_list_new=[new_item.path],
|
||||
file_count=1,
|
||||
total_size=fileitem.size,
|
||||
)
|
||||
if not new_item:
|
||||
error = error or f"{fileitem.path} 整理后未获取到目标文件信息"
|
||||
return TransferInfo(
|
||||
@@ -1286,6 +1818,98 @@ class TransHandler:
|
||||
naming_context.pop("media_id", None)
|
||||
return naming_context
|
||||
|
||||
@staticmethod
|
||||
def __find_version_files(
|
||||
storage_oper: StorageBase,
|
||||
path: Path,
|
||||
) -> list[FileItem]:
|
||||
"""稳定列出与冻结目标相同季集和 Part 的其它视频版本。"""
|
||||
meta = MetaInfoPath(path)
|
||||
parent_item = storage_oper.get_item_strict(path.parent)
|
||||
if not parent_item:
|
||||
return []
|
||||
media_files = storage_oper.list(parent_item) or []
|
||||
result: list[FileItem] = []
|
||||
for media_file in media_files:
|
||||
media_path = Path(media_file.path)
|
||||
if media_path == path or media_file.type != "file":
|
||||
continue
|
||||
if f".{cast(str, media_file.extension).lower()}" not in get_runtime_setting('RMT_MEDIAEXT'):
|
||||
continue
|
||||
filemeta = MetaInfoPath(media_path)
|
||||
if filemeta.season != meta.season or filemeta.episode != meta.episode:
|
||||
continue
|
||||
if meta.part and filemeta.part and filemeta.part != meta.part:
|
||||
continue
|
||||
result.append(media_file)
|
||||
return sorted(result, key=lambda item: (item.path, item.fileid or ""))
|
||||
|
||||
@classmethod
|
||||
def __delete_version_files_with_steps(
|
||||
cls,
|
||||
*,
|
||||
step_runner: TransferStepRunner,
|
||||
storage_oper: StorageBase,
|
||||
target_storage: str,
|
||||
path: Path,
|
||||
) -> None:
|
||||
"""先冻结版本清单,再把每一个删除作为独立稳定步骤执行。"""
|
||||
def discover() -> TransferStepResult:
|
||||
"""读取并冻结当前版本删除候选,读失败不产生副作用。"""
|
||||
candidates = cls.__find_version_files(storage_oper, path)
|
||||
return TransferStepResult(payload={
|
||||
"items": [item.model_dump(mode="json") for item in candidates],
|
||||
})
|
||||
|
||||
discovery = cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="prepare",
|
||||
kind="discover_version_targets",
|
||||
payload={"storage": target_storage, "path": path.as_posix()},
|
||||
execute=discover,
|
||||
observe=lambda: TransferOperationObservation(
|
||||
state=TransferOperationObservationState.NOT_APPLIED,
|
||||
evidence=TransferStepResult(payload={
|
||||
"reason": "read-only discovery may be repeated",
|
||||
}),
|
||||
),
|
||||
)
|
||||
raw_items = discovery.payload.get("items")
|
||||
if not isinstance(raw_items, list):
|
||||
raise RuntimeError("版本删除候选检查点格式无效")
|
||||
for raw_item in raw_items:
|
||||
candidate = FileItem.model_validate(raw_item)
|
||||
|
||||
def delete_candidate(item: FileItem = candidate) -> TransferStepResult:
|
||||
"""删除一个冻结版本候选,已不存在时保持幂等成功。"""
|
||||
current = storage_oper.get_item_strict(Path(cast(str, item.path)))
|
||||
if current is not None and not storage_oper.delete(current):
|
||||
raise RuntimeError(f"版本文件 {item.path} 删除失败")
|
||||
return TransferStepResult(payload={
|
||||
"path": item.path,
|
||||
"deleted": True,
|
||||
})
|
||||
|
||||
def observe_candidate(item: FileItem = candidate) -> TransferOperationObservation:
|
||||
"""查询一个冻结版本候选是否已经删除。"""
|
||||
return cls.__observe_item_presence(
|
||||
storage_oper,
|
||||
Path(cast(str, item.path)),
|
||||
applied_when_present=False,
|
||||
)
|
||||
|
||||
cls.__run_persisted_step(
|
||||
step_runner,
|
||||
phase="prepare",
|
||||
kind="delete_version_target",
|
||||
payload={
|
||||
"storage": target_storage,
|
||||
"item": candidate.model_dump(mode="json"),
|
||||
},
|
||||
execute=delete_candidate,
|
||||
observe=observe_candidate,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def __delete_version_files(storage_oper: StorageBase, path: Path) -> bool:
|
||||
"""
|
||||
|
||||
+10
-4
@@ -417,6 +417,12 @@ SCHEMA_EXPORTS = {
|
||||
'TransferInterceptEventData': ('app.schemas.event', 'TransferInterceptEventData'),
|
||||
'TransferJob': ('app.schemas.transfer', 'TransferJob'),
|
||||
'TransferJobTask': ('app.schemas.transfer', 'TransferJobTask'),
|
||||
'TransferManualReviewData': ('app.schemas.transfer', 'TransferManualReviewData'),
|
||||
'TransferManualReviewPageData': ('app.schemas.transfer', 'TransferManualReviewPageData'),
|
||||
'TransferManualReviewRequest': ('app.schemas.transfer', 'TransferManualReviewRequest'),
|
||||
'TransferManualReviewSourceData': ('app.schemas.transfer', 'TransferManualReviewSourceData'),
|
||||
'TransferManualReviewStepData': ('app.schemas.transfer', 'TransferManualReviewStepData'),
|
||||
'TransferManualReviewTaskData': ('app.schemas.transfer', 'TransferManualReviewTaskData'),
|
||||
'TransferOverwriteCheckEventData': ('app.schemas.event', 'TransferOverwriteCheckEventData'),
|
||||
'TransferRenameBuildEventData': ('app.schemas.event', 'TransferRenameBuildEventData'),
|
||||
'TransferRenameEventData': ('app.schemas.event', 'TransferRenameEventData'),
|
||||
@@ -459,7 +465,7 @@ SCHEMA_EXPORTS = {
|
||||
'field_serializer': ('app.schemas.agent', 'field_serializer'),
|
||||
'field_validator': ('app.schemas.system', 'field_validator'),
|
||||
'json': ('app.schemas.subscribe', 'json'),
|
||||
'model_validator': ('app.schemas.subscribe', 'model_validator'),
|
||||
'model_validator': ('app.schemas.transfer', 'model_validator'),
|
||||
're': ('app.schemas.file', 're'),
|
||||
}
|
||||
|
||||
@@ -474,9 +480,9 @@ SCHEMA_CONFLICTS = {
|
||||
'Field': ['app.schemas.agent', 'app.schemas.cache', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.download', 'app.schemas.event', 'app.schemas.file', 'app.schemas.history', 'app.schemas.llm', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.notification', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.storage', 'app.schemas.openai', 'app.schemas.servarr', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.subscribe', 'app.schemas.system', 'app.schemas.tmdb', 'app.schemas.token', 'app.schemas.transfer', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'],
|
||||
'FileItem': ['app.schemas.event', 'app.schemas.file', 'app.schemas.transfer', 'app.schemas.workflow'],
|
||||
'FilterRuleGroup': ['app.schemas.rule', 'app.schemas.system'],
|
||||
'JsonData': ['app.schemas.agent', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.openai', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'],
|
||||
'JsonData': ['app.schemas.agent', 'app.schemas.common', 'app.schemas.context', 'app.schemas.dashboard', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.mfa', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.openai', 'app.schemas.servcookie', 'app.schemas.site', 'app.schemas.transfer', 'app.schemas.user', 'app.schemas.workflow', 'app.schemas.mcp'],
|
||||
'List': ['app.schemas.agent', 'app.schemas.context', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.monitoring', 'app.schemas.plugin', 'app.schemas.openai', 'app.schemas.subscribe', 'app.schemas.transfer', 'app.schemas.workflow'],
|
||||
'Literal': ['app.schemas.agent', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.servcookie', 'app.schemas.system', 'app.schemas.mcp'],
|
||||
'Literal': ['app.schemas.agent', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.servcookie', 'app.schemas.system', 'app.schemas.transfer', 'app.schemas.mcp'],
|
||||
'LocaleHelper': ['app.schemas.dashboard', 'app.schemas.response'],
|
||||
'MediaInfo': ['app.schemas.context', 'app.schemas.system', 'app.schemas.transfer', 'app.schemas.workflow'],
|
||||
'MediaSource': ['app.schemas.cache', 'app.schemas.context', 'app.schemas.event', 'app.schemas.history', 'app.schemas.mediaserver', 'app.schemas.music', 'app.schemas.subscribe', 'app.schemas.transfer'],
|
||||
@@ -502,5 +508,5 @@ SCHEMA_CONFLICTS = {
|
||||
'dataclass': ['app.schemas.notification', 'app.schemas.system'],
|
||||
'datetime': ['app.schemas.agent', 'app.schemas.monitoring'],
|
||||
'field_validator': ['app.schemas.event', 'app.schemas.message', 'app.schemas.music', 'app.schemas.plugin', 'app.schemas.response', 'app.schemas.subscribe', 'app.schemas.system'],
|
||||
'model_validator': ['app.schemas.dashboard', 'app.schemas.event', 'app.schemas.mediaserver', 'app.schemas.subscribe'],
|
||||
'model_validator': ['app.schemas.dashboard', 'app.schemas.event', 'app.schemas.mediaserver', 'app.schemas.subscribe', 'app.schemas.transfer'],
|
||||
}
|
||||
|
||||
@@ -69,6 +69,8 @@ class TransferHistory(OptionalMediaIdentityMixin, BaseModel):
|
||||
|
||||
# ID
|
||||
id: int
|
||||
# durable 整理任务标识仅供宿主入口选择重试协议,不属于公开历史响应
|
||||
transfer_task_id: Optional[str] = Field(default=None, exclude=True)
|
||||
# 源存储类型
|
||||
src_storage: Optional[str] = None
|
||||
# 目标存储类型
|
||||
|
||||
+72
-2
@@ -1,8 +1,9 @@
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Union
|
||||
from typing import Literal, List, Optional, Union
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
||||
|
||||
from app.schemas.common import JsonData
|
||||
from app.schemas.media import OptionalMediaIdentityMixin
|
||||
from app.schemas.types import MediaSource, MusicTargetEntityType
|
||||
|
||||
@@ -157,6 +158,75 @@ class TransferInfo(BaseModel):
|
||||
return dicts
|
||||
|
||||
|
||||
class TransferManualReviewRequest(BaseModel): # type: ignore[misc]
|
||||
"""人工判定外部结果不确定整理步骤的请求。"""
|
||||
|
||||
model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
|
||||
|
||||
operation_id: str = Field(min_length=1, description="待判定的稳定操作标识")
|
||||
decision: Literal["not_applied", "applied"] = Field(
|
||||
description="人工判定;不公开 failed,失败终态只能由 durable 结算写入",
|
||||
)
|
||||
reason: str = Field(min_length=1, max_length=2000, description="人工判定理由")
|
||||
result_payload: Optional[dict[str, JsonData]] = Field(
|
||||
default=None,
|
||||
description="判定为 applied 时必填的外部结果证据",
|
||||
)
|
||||
|
||||
@model_validator(mode="after") # type: ignore[misc]
|
||||
def validate_result_payload(self) -> "TransferManualReviewRequest":
|
||||
"""要求已发生判定携带可持久化的结果证据。"""
|
||||
if self.decision == "applied" and self.result_payload is None:
|
||||
raise ValueError("判定为 applied 时必须提供 result_payload")
|
||||
return self
|
||||
|
||||
|
||||
class TransferManualReviewData(BaseModel): # type: ignore[misc]
|
||||
"""人工复核提交后的公开状态投影。"""
|
||||
|
||||
task_id: str
|
||||
operation_id: str
|
||||
decision: Literal["not_applied", "applied"]
|
||||
state: str
|
||||
review_revision: int
|
||||
|
||||
|
||||
class TransferManualReviewSourceData(BaseModel): # type: ignore[misc]
|
||||
"""人工复核任务的源文件身份。"""
|
||||
|
||||
storage: str
|
||||
path: str
|
||||
|
||||
|
||||
class TransferManualReviewStepData(BaseModel): # type: ignore[misc]
|
||||
"""人工复核步骤的公开意图与事实证据。"""
|
||||
|
||||
operation_id: str
|
||||
kind: str
|
||||
intent: dict[str, JsonData]
|
||||
evidence: Optional[dict[str, JsonData]] = None
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class TransferManualReviewTaskData(BaseModel): # type: ignore[misc]
|
||||
"""可由管理员发现和判定的 durable 整理任务。"""
|
||||
|
||||
task_id: str
|
||||
source: TransferManualReviewSourceData
|
||||
state: Literal["manual_review", "retry_wait"]
|
||||
step: TransferManualReviewStepData
|
||||
review_revision: int
|
||||
|
||||
|
||||
class TransferManualReviewPageData(BaseModel): # type: ignore[misc]
|
||||
"""人工复核任务分页结果。"""
|
||||
|
||||
items: list[TransferManualReviewTaskData] = Field(default_factory=list)
|
||||
total: int = 0
|
||||
page: int = 1
|
||||
page_size: int = 30
|
||||
|
||||
|
||||
class EpisodeFormat(BaseModel):
|
||||
"""
|
||||
剧集自定义识别格式
|
||||
|
||||
@@ -1,21 +1,55 @@
|
||||
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy import column, delete, exists, select, table
|
||||
|
||||
from app.db.base import DbOper, execute_dml
|
||||
from app.db.models.transferpending import TransferPending as _TransferPending
|
||||
|
||||
_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
|
||||
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
|
||||
|
||||
|
||||
def _safe_legacy_delete_predicates() -> tuple[Any, ...]:
|
||||
"""只允许旧接口删除从未 claim、执行或结算的新鲜登记。"""
|
||||
return (
|
||||
_TransferPending.lease_owner.is_(None),
|
||||
_TransferPending.lease_token.is_(None),
|
||||
_TransferPending.lease_expires_at.is_(None),
|
||||
_TransferPending.heartbeat_at.is_(None),
|
||||
_TransferPending.attempt_count == 0,
|
||||
_TransferPending.execution_state == "not_started",
|
||||
_TransferPending.execution_version.is_(None),
|
||||
_TransferPending.execution_payload.is_(None),
|
||||
_TransferPending.execution_fingerprint.is_(None),
|
||||
_TransferPending.retry_generation == 0,
|
||||
_TransferPending.retry_count == 0,
|
||||
_TransferPending.retry_due_at.is_(None),
|
||||
_TransferPending.settlement_revision == 0,
|
||||
_TransferPending.terminal_history_id.is_(None),
|
||||
_TransferPending.last_error.is_(None),
|
||||
~exists(
|
||||
select(_TRANSFER_EXECUTION_STEP.c.task_id).where(
|
||||
_TRANSFER_EXECUTION_STEP.c.task_id == _TransferPending.task_id
|
||||
)
|
||||
),
|
||||
~exists(
|
||||
select(_TRANSFER_HISTORY.c.transfer_task_id).where(
|
||||
_TRANSFER_HISTORY.c.transfer_task_id == _TransferPending.task_id
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TransferPendingOper(DbOper):
|
||||
"""
|
||||
保留旧待整理登记 ABI,并对执行租约实施兼容写 fencing。
|
||||
|
||||
本类只由精确旧导入映射加载。宿主整理链仍使用 Application Port 和显式
|
||||
Session 的 canonical Oper;旧删除入口只能处理从未取得租约的记录。
|
||||
Session 的 canonical Oper;旧删除入口只能处理无任何执行证据的新鲜登记。
|
||||
"""
|
||||
|
||||
def register(self, storage: str, src_path: str) -> Optional[_TransferPending]:
|
||||
@@ -123,10 +157,10 @@ class TransferPendingOper(DbOper):
|
||||
|
||||
def discard(self, storage: str, src_path: str) -> int:
|
||||
"""
|
||||
按旧签名删除未 claim 的指定登记。
|
||||
按旧签名删除从未 claim 且没有执行证据的指定登记。
|
||||
|
||||
任何带 token 的记录都由当前租约拥有者通过 fenced canonical API 收口;
|
||||
即使租约已经过期,旧插件也不得越权代替恢复调度器删除。
|
||||
任何租约、重试、步骤或终态证据都归 canonical 状态机所有;即使租约
|
||||
已经过期,旧插件也不得越权代替恢复调度器删除。
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 删除的记录数
|
||||
@@ -139,7 +173,7 @@ class TransferPendingOper(DbOper):
|
||||
delete(_TransferPending).where(
|
||||
_TransferPending.storage == storage,
|
||||
_TransferPending.src_path == src_path,
|
||||
_TransferPending.lease_token.is_(None),
|
||||
*_safe_legacy_delete_predicates(),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
@@ -167,7 +201,7 @@ class TransferPendingOper(DbOper):
|
||||
|
||||
def clear(self) -> int:
|
||||
"""
|
||||
清空全部未 claim 登记,保留任何带租约 token 的任务。
|
||||
清空从未 claim 且没有执行证据的登记,保留状态机拥有的任务。
|
||||
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
@@ -175,7 +209,7 @@ class TransferPendingOper(DbOper):
|
||||
lambda session: execute_dml(
|
||||
session,
|
||||
delete(_TransferPending).where(
|
||||
_TransferPending.lease_token.is_(None),
|
||||
*_safe_legacy_delete_predicates(),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@@ -111,6 +111,9 @@ from app.db.adapters.site import TransactionalSiteRepository
|
||||
from app.db.adapters.subscription import TransactionalSubscribeWriter
|
||||
from app.db.adapters.transaction import TransactionalWriteRunner
|
||||
from app.db.adapters.transfer import TransactionalTransferAdmissionRepository
|
||||
from app.db.adapters.transfer_execution import (
|
||||
TransactionalTransferExecutionRepository,
|
||||
)
|
||||
from app.db.adapters.workflow import TransactionalWorkflowExecutionService
|
||||
from app.db.oper.agentchat import AgentChatOper
|
||||
from app.db.oper.agenttask import AgentTaskOper
|
||||
@@ -865,6 +868,9 @@ async def init_modules() -> HostRuntime:
|
||||
transfer_pending=lambda: TransactionalTransferAdmissionRepository(
|
||||
SessionFactory
|
||||
),
|
||||
transfer_execution=lambda: TransactionalTransferExecutionRepository(
|
||||
SessionFactory
|
||||
),
|
||||
media_server=lambda: MediaServerOper(),
|
||||
download_failure=lambda: TransactionalDownloadFailureRepository(
|
||||
SessionFactory
|
||||
|
||||
Reference in New Issue
Block a user