mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-01 05:27:02 +08:00
Merge upstream/v3 into codex/feat/plugin-data-query-sdk-v3
This commit is contained in:
@@ -43,6 +43,14 @@ class TransferTerminalState(StrEnum):
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
class TransferExecutionOutcome(StrEnum):
|
||||
"""描述执行检查点可裁决的业务结果。"""
|
||||
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
OVERWRITE_SKIPPED = "overwrite_skipped"
|
||||
|
||||
|
||||
class TransferOperationObservationState(StrEnum):
|
||||
"""描述重启后对遗留 STARTED 外部操作的严格探测结论。"""
|
||||
|
||||
@@ -249,6 +257,29 @@ class TransferExecutionCheckpoint:
|
||||
raise ValueError("整理执行检查点包含无效操作标识")
|
||||
if not self.operation_ids and not self.skip_reason:
|
||||
raise ValueError("零副作用整理执行检查点必须记录 skip_reason")
|
||||
raw_outcome = self.payload.get("outcome")
|
||||
try:
|
||||
outcome = TransferExecutionOutcome(
|
||||
raw_outcome if isinstance(raw_outcome, str) else ""
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise ValueError("整理执行检查点 outcome 无效") from error
|
||||
transferinfo = self.payload.get("transferinfo")
|
||||
if transferinfo is not None:
|
||||
if not isinstance(transferinfo, dict):
|
||||
raise ValueError("整理执行检查点 TransferInfo 必须是 JSON 对象")
|
||||
expected_success = outcome is TransferExecutionOutcome.SUCCEEDED
|
||||
expected_overwrite_skip = (
|
||||
outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED
|
||||
)
|
||||
if (
|
||||
bool(transferinfo.get("success")) != expected_success
|
||||
or bool(transferinfo.get("overwrite_skipped"))
|
||||
!= expected_overwrite_skip
|
||||
):
|
||||
raise ValueError("整理执行 checkpoint outcome 与 TransferInfo 不一致")
|
||||
elif outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED:
|
||||
raise ValueError("覆盖跳过执行检查点缺少冻结 TransferInfo")
|
||||
_canonical_json(self.payload)
|
||||
if build_transfer_checkpoint_fingerprint(self.to_payload()) != self.fingerprint:
|
||||
raise ValueError("整理执行检查点内容与指纹不一致")
|
||||
@@ -285,6 +316,35 @@ class TransferExecutionCheckpoint:
|
||||
"skip_reason": self.skip_reason,
|
||||
}
|
||||
|
||||
def validate_settlement_outcome(
|
||||
self,
|
||||
settlement_outcome: str,
|
||||
) -> TransferTerminalState:
|
||||
"""解析执行事实,并校验同事务历史裁决给出的结算方向。"""
|
||||
raw_outcome = self.payload.get("outcome")
|
||||
try:
|
||||
outcome = TransferExecutionOutcome(
|
||||
raw_outcome if isinstance(raw_outcome, str) else ""
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise TransferExecutionConflictError(
|
||||
"整理执行检查点 outcome 无效"
|
||||
) from error
|
||||
try:
|
||||
terminal_state = TransferTerminalState(settlement_outcome)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise TransferExecutionConflictError(
|
||||
"整理结算 outcome 无效"
|
||||
) from error
|
||||
if (
|
||||
outcome != TransferExecutionOutcome.OVERWRITE_SKIPPED
|
||||
and outcome.value != terminal_state.value
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理结算 outcome 与执行检查点不一致"
|
||||
)
|
||||
return terminal_state
|
||||
|
||||
@classmethod
|
||||
def from_payload(
|
||||
cls,
|
||||
@@ -293,6 +353,8 @@ class TransferExecutionCheckpoint:
|
||||
fingerprint: str,
|
||||
) -> "TransferExecutionCheckpoint":
|
||||
"""解析并校验数据库中的执行检查点版本与稳定指纹。"""
|
||||
if not isinstance(payload, Mapping):
|
||||
raise TransferExecutionConflictError("整理执行检查点 JSON 结构无效")
|
||||
serialized = dict(payload)
|
||||
if build_transfer_checkpoint_fingerprint(serialized) != fingerprint:
|
||||
raise TransferExecutionConflictError("整理执行检查点 JSON 与指纹不一致")
|
||||
@@ -308,13 +370,18 @@ class TransferExecutionCheckpoint:
|
||||
skip_reason = serialized.get("skip_reason")
|
||||
if skip_reason is not None and not isinstance(skip_reason, str):
|
||||
raise TransferExecutionConflictError("整理执行跳过原因类型无效")
|
||||
return cls(
|
||||
fingerprint=fingerprint,
|
||||
payload=result_payload,
|
||||
operation_ids=tuple(operation_ids),
|
||||
skip_reason=skip_reason,
|
||||
version=version,
|
||||
)
|
||||
try:
|
||||
return cls(
|
||||
fingerprint=fingerprint,
|
||||
payload=result_payload,
|
||||
operation_ids=tuple(operation_ids),
|
||||
skip_reason=skip_reason,
|
||||
version=version,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise TransferExecutionConflictError(
|
||||
"整理执行检查点内容无效"
|
||||
) from error
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
||||
@@ -38,7 +38,6 @@ from typing import (
|
||||
from pydantic import BaseModel, ConfigDict, PrivateAttr
|
||||
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.application.agent import get_prompt_manager, get_running_agent_manager
|
||||
from app.application.transfer.execution import TransferExecutionCheckpoint
|
||||
from app.domain.context import MediaInfo, MusicInfo
|
||||
from app.domain.media import normalize_music_type
|
||||
@@ -61,7 +60,6 @@ from app.schemas.types import (
|
||||
MUSIC_ENTITY_RECORDING,
|
||||
MediaSource,
|
||||
MediaType,
|
||||
ReplyMode,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -209,14 +207,6 @@ class TransferPlanningInput:
|
||||
raise ValueError("整理规划输入缺少源文件存储或路径")
|
||||
_canonical_json(self.to_payload())
|
||||
|
||||
@classmethod
|
||||
def legacy(cls, *, storage: str, src_path: str) -> "TransferPlanningInput":
|
||||
"""为升级前只有存储与路径的登记构造保守重规划输入。"""
|
||||
return cls(
|
||||
source_fileitem={"storage": storage, "path": src_path},
|
||||
options={"legacy_replan": True},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanningInput":
|
||||
"""从受版本约束的 JSON 对象恢复规划输入。"""
|
||||
@@ -495,6 +485,7 @@ class TransferPlanCheckpoint:
|
||||
overwrite_mode: Optional[str] = None
|
||||
preview: bool = False
|
||||
skip_reason: Optional[str] = None
|
||||
rejection_error: Optional[str] = None
|
||||
schema_version: int = TRANSFER_PLAN_CHECKPOINT_VERSION
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
@@ -551,6 +542,7 @@ class TransferPlanCheckpoint:
|
||||
or self.resolved_transfer_type
|
||||
or self.items
|
||||
or self.skip_reason
|
||||
or self.rejection_error
|
||||
):
|
||||
raise ValueError("provider_pending 检查点不得包含宿主执行计划")
|
||||
else:
|
||||
@@ -558,6 +550,12 @@ class TransferPlanCheckpoint:
|
||||
raise ValueError("整理计划检查点缺少目标身份")
|
||||
if not self.resolved_transfer_type:
|
||||
raise ValueError("整理计划检查点缺少已解析的整理方式")
|
||||
if self.rejection_error is not None and (
|
||||
not isinstance(self.rejection_error, str)
|
||||
or not self.rejection_error.strip()
|
||||
or self.items
|
||||
):
|
||||
raise ValueError("整理拒绝检查点必须包含非空错误且不得包含文件步骤")
|
||||
if tuple(item.sequence for item in self.items) != tuple(range(len(self.items))):
|
||||
raise ValueError("整理计划项必须按从零开始的连续序号保存")
|
||||
if (
|
||||
@@ -565,6 +563,7 @@ class TransferPlanCheckpoint:
|
||||
and not self.items
|
||||
and not self.preview
|
||||
and not self.skip_reason
|
||||
and not self.rejection_error
|
||||
):
|
||||
raise ValueError("非预览空计划必须记录合法跳过原因")
|
||||
_canonical_json(self.to_payload())
|
||||
@@ -574,6 +573,13 @@ class TransferPlanCheckpoint:
|
||||
"""返回该检查点是否只冻结 provider 调用、尚未生成宿主计划。"""
|
||||
return self.provider_invocation is not None
|
||||
|
||||
@property
|
||||
def fingerprint(self) -> str:
|
||||
"""返回完整冻结计划规范 JSON 的稳定 SHA-256 指纹。"""
|
||||
return hashlib.sha256(
|
||||
_canonical_json(self.to_payload()).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanCheckpoint":
|
||||
"""从受版本约束的 JSON 对象恢复完整执行检查点。"""
|
||||
@@ -634,6 +640,7 @@ class TransferPlanCheckpoint:
|
||||
overwrite_mode=payload.get("overwrite_mode"),
|
||||
preview=payload.get("preview", False),
|
||||
skip_reason=payload.get("skip_reason"),
|
||||
rejection_error=payload.get("rejection_error"),
|
||||
schema_version=payload.get("schema_version", 0),
|
||||
)
|
||||
|
||||
@@ -672,6 +679,7 @@ class TransferPlanCheckpoint:
|
||||
"overwrite_mode": self.overwrite_mode,
|
||||
"preview": self.preview,
|
||||
"skip_reason": self.skip_reason,
|
||||
"rejection_error": self.rejection_error,
|
||||
}
|
||||
|
||||
|
||||
@@ -847,9 +855,9 @@ class TransferAdmission:
|
||||
state: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
planning_input: TransferPlanningInput
|
||||
last_error: Optional[str] = None
|
||||
input_fingerprint: Optional[str] = None
|
||||
planning_input: Optional[TransferPlanningInput] = None
|
||||
checkpoint: Optional[TransferPlanCheckpoint] = None
|
||||
lease_owner: Optional[str] = None
|
||||
lease_token: Optional[str] = None
|
||||
@@ -866,7 +874,7 @@ class TransferAdmissionRepository(Protocol):
|
||||
*,
|
||||
storage: str,
|
||||
src_path: str,
|
||||
planning_input: Optional[TransferPlanningInput] = None,
|
||||
planning_input: TransferPlanningInput,
|
||||
) -> TransferAdmission:
|
||||
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
|
||||
...
|
||||
@@ -932,8 +940,8 @@ class TransferAdmissionRepository(Protocol):
|
||||
"""按 token 释放当前 claim,并可记录可恢复错误。"""
|
||||
...
|
||||
|
||||
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅允许当前 lease owner 删除已经到达终态的登记。"""
|
||||
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅允许当前 lease owner 删除确认未开始且源已消失的登记。"""
|
||||
...
|
||||
|
||||
|
||||
@@ -1929,162 +1937,3 @@ class JobManager:
|
||||
with job_lock:
|
||||
__mediaid__ = self.__get_media_id(media=media, season=season)
|
||||
return self._season_episodes.get(__mediaid__) or []
|
||||
|
||||
|
||||
class FailedRetryScheduler:
|
||||
"""
|
||||
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。
|
||||
|
||||
缓冲不提供持久化保证;关闭时会取消尚未触发的记录,由上层 durable
|
||||
工作流在后续阶段承接需要跨进程保证的重试意图。
|
||||
"""
|
||||
|
||||
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
|
||||
super().__init__()
|
||||
self._retry_transfer_buffer: dict[str, list[int]] = {}
|
||||
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
|
||||
self._retry_transfer_generations: dict[str, int] = {}
|
||||
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
|
||||
self._retry_transfer_lock = asyncio.Lock()
|
||||
self._closed = False
|
||||
|
||||
async def close(self) -> None:
|
||||
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
|
||||
async with self._retry_transfer_lock:
|
||||
self._closed = True
|
||||
timers = list(self._retry_transfer_timers.values())
|
||||
buffered_count = sum(
|
||||
len(history_ids)
|
||||
for history_ids in self._retry_transfer_buffer.values()
|
||||
)
|
||||
self._retry_transfer_timers.clear()
|
||||
self._retry_transfer_generations.clear()
|
||||
self._retry_transfer_buffer.clear()
|
||||
tasks = tuple(self._retry_transfer_tasks)
|
||||
|
||||
for timer in timers:
|
||||
timer.cancel()
|
||||
if buffered_count:
|
||||
logger.warning(
|
||||
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
|
||||
)
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
if tasks:
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
|
||||
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
|
||||
if self._closed:
|
||||
return
|
||||
task = asyncio.create_task(
|
||||
self._flush_retry_transfer(group_key, generation),
|
||||
name="transfer.failed_retry.flush",
|
||||
)
|
||||
self._retry_transfer_tasks.add(task)
|
||||
task.add_done_callback(self._observe_retry_transfer_task)
|
||||
|
||||
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
|
||||
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
|
||||
self._retry_transfer_tasks.discard(task)
|
||||
if task.cancelled():
|
||||
return
|
||||
exception = task.exception()
|
||||
if exception is not None:
|
||||
logger.error(f"智能体重试整理后台任务异常: {exception}")
|
||||
|
||||
@staticmethod
|
||||
def _build_retry_transfer_template_context(
|
||||
history_ids: list[int],
|
||||
) -> tuple[str, dict[str, int | str]]:
|
||||
"""仅负责把失败重试任务的动态数据映射成模板变量。"""
|
||||
is_batch = len(history_ids) > 1
|
||||
task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry"
|
||||
template_context: dict[str, int | str] = {
|
||||
"history_ids_csv": ", ".join(str(item) for item in history_ids),
|
||||
"history_count": len(history_ids),
|
||||
}
|
||||
if not is_batch:
|
||||
template_context["history_id"] = history_ids[0]
|
||||
return task_type, template_context
|
||||
|
||||
def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str:
|
||||
"""根据失败记录数量构建统一的重试整理后台任务提示词。"""
|
||||
task_type, template_context = self._build_retry_transfer_template_context(history_ids)
|
||||
return cast(str, get_prompt_manager().render_system_task_message(
|
||||
task_type,
|
||||
template_context=template_context,
|
||||
))
|
||||
|
||||
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
|
||||
"""
|
||||
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
|
||||
"""
|
||||
if not group_key:
|
||||
group_key = f"_default_{history_id}"
|
||||
|
||||
async with self._retry_transfer_lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
|
||||
if group_key not in self._retry_transfer_buffer:
|
||||
self._retry_transfer_buffer[group_key] = []
|
||||
if history_id not in self._retry_transfer_buffer[group_key]:
|
||||
self._retry_transfer_buffer[group_key].append(history_id)
|
||||
logger.info(
|
||||
f"智能体重试整理:记录 ID={history_id} 已加入缓冲区 "
|
||||
f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)"
|
||||
)
|
||||
|
||||
if group_key in self._retry_transfer_timers:
|
||||
self._retry_transfer_timers[group_key].cancel()
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
generation = self._retry_transfer_generations.get(group_key, 0) + 1
|
||||
self._retry_transfer_generations[group_key] = generation
|
||||
self._retry_transfer_timers[group_key] = loop.call_later(
|
||||
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
|
||||
self._start_retry_transfer_task,
|
||||
group_key,
|
||||
generation,
|
||||
)
|
||||
|
||||
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
|
||||
"""
|
||||
延迟定时器到期后,取出该分组的所有 history_id 并合并为一次 agent 调用。
|
||||
"""
|
||||
async with self._retry_transfer_lock:
|
||||
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
|
||||
if self._retry_transfer_generations.get(group_key) != generation:
|
||||
return
|
||||
history_ids = self._retry_transfer_buffer.pop(group_key, [])
|
||||
self._retry_transfer_timers.pop(group_key, None)
|
||||
self._retry_transfer_generations.pop(group_key, None)
|
||||
|
||||
if not history_ids:
|
||||
return
|
||||
|
||||
ids_str = ", ".join(str(item) for item in history_ids)
|
||||
logger.info(
|
||||
f"智能体重试整理:开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})"
|
||||
)
|
||||
|
||||
try:
|
||||
manager = get_running_agent_manager()
|
||||
if manager is None:
|
||||
logger.warning("智能助手服务未运行,跳过整理失败自动重试")
|
||||
return
|
||||
await manager.run_background_prompt(
|
||||
message=self._build_retry_transfer_prompt(history_ids),
|
||||
session_prefix="__agent_retry_transfer_batch",
|
||||
reply_mode=ReplyMode.DISPATCH,
|
||||
)
|
||||
logger.info(
|
||||
f"智能体重试整理:批量处理完成 IDs=[{ids_str}] (group={group_key})"
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}"
|
||||
)
|
||||
|
||||
+363
-373
File diff suppressed because it is too large
Load Diff
@@ -26,6 +26,7 @@ from app.application.outbox import (
|
||||
OutboxIntent,
|
||||
)
|
||||
from app.application.transfer.execution import (
|
||||
TransferExecutionCheckpoint,
|
||||
TransferExecutionConflictError,
|
||||
TransferExecutionLeaseLostError,
|
||||
TransferExecutionState,
|
||||
@@ -403,6 +404,13 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
|
||||
!= settlement.execution_fingerprint
|
||||
):
|
||||
raise TransferExecutionConflictError("整理终态与执行检查点不匹配")
|
||||
checkpoint = TransferExecutionCheckpoint.from_payload(
|
||||
pending.execution_payload,
|
||||
fingerprint=settlement.execution_fingerprint,
|
||||
)
|
||||
if pending.execution_version != checkpoint.version:
|
||||
raise TransferExecutionConflictError("整理执行检查点列版本与内容不一致")
|
||||
checkpoint.validate_settlement_outcome(settlement.outcome)
|
||||
return int(pending.settlement_revision)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -142,18 +142,14 @@ class TransactionalTransferAdmissionRepository:
|
||||
*,
|
||||
storage: str,
|
||||
src_path: str,
|
||||
planning_input: Optional[TransferPlanningInput] = None,
|
||||
planning_input: TransferPlanningInput,
|
||||
) -> TransferAdmission:
|
||||
"""按输入指纹幂等持久化准入事实,并返回跨重启稳定身份。"""
|
||||
if not storage or not src_path:
|
||||
raise ValueError("整理任务的存储与源路径不能为空")
|
||||
effective_input = planning_input or TransferPlanningInput.legacy(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if (
|
||||
effective_input.source_fileitem.get("storage") != storage
|
||||
or effective_input.source_fileitem.get("path") != src_path
|
||||
planning_input.source_fileitem.get("storage") != storage
|
||||
or planning_input.source_fileitem.get("path") != src_path
|
||||
):
|
||||
raise ValueError("整理规划输入的源文件身份与准入参数不一致")
|
||||
now_time = self._now()
|
||||
@@ -167,16 +163,16 @@ class TransactionalTransferAdmissionRepository:
|
||||
src_path=src_path,
|
||||
state=TRANSFER_ADMISSION_ACCEPTED,
|
||||
now_time=now_time,
|
||||
input_version=effective_input.schema_version,
|
||||
planning_input=effective_input.to_payload(),
|
||||
input_fingerprint=effective_input.fingerprint,
|
||||
input_version=planning_input.schema_version,
|
||||
planning_input=planning_input.to_payload(),
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
)
|
||||
if pending is None:
|
||||
raise TransferAdmissionConflictError(
|
||||
f"整理源文件已有持久终态回执: {storage}:{src_path}"
|
||||
)
|
||||
session.flush()
|
||||
self._assert_input_match(pending, effective_input)
|
||||
self._assert_input_match(pending, planning_input)
|
||||
admission = self._project(pending)
|
||||
transaction.commit()
|
||||
return admission
|
||||
@@ -192,7 +188,7 @@ class TransactionalTransferAdmissionRepository:
|
||||
)
|
||||
if pending is None:
|
||||
raise RuntimeError("并发准入冲突后未找到已提交记录") from error
|
||||
self._assert_input_match(pending, effective_input)
|
||||
self._assert_input_match(pending, planning_input)
|
||||
return self._project(pending)
|
||||
|
||||
def claim_task(
|
||||
@@ -525,14 +521,14 @@ class TransactionalTransferAdmissionRepository:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅以当前未过期 token 删除终态任务,拒绝陈旧 worker 变更。"""
|
||||
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅以当前 token 删除无执行证据的缺失源任务,拒绝陈旧 worker。"""
|
||||
if not task_id or not lease_token:
|
||||
return 0
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
deleted = TransferPendingOper(db=session).stage_discard_claimed(
|
||||
deleted = TransferPendingOper(db=session).stage_abandon_unstarted(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
|
||||
@@ -4,7 +4,8 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -27,6 +28,14 @@ from app.application.transfer.execution import (
|
||||
TransferStepIntent,
|
||||
TransferStepResult,
|
||||
TransferStepState,
|
||||
build_transfer_operation_id,
|
||||
)
|
||||
from app.application.transfer.workflow import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TransferPlanCheckpoint,
|
||||
TransferProviderInvocationSnapshot,
|
||||
TransferProviderReference,
|
||||
)
|
||||
from app.db.models.transferexecutionstep import (
|
||||
TransferExecutionStep as TransferExecutionStepModel,
|
||||
@@ -229,14 +238,332 @@ class TransactionalTransferExecutionRepository:
|
||||
))
|
||||
|
||||
@staticmethod
|
||||
def _raise_fenced_failure(
|
||||
def _plan_identity(
|
||||
pending: TransferPending,
|
||||
) -> tuple[TransferPlanCheckpoint, str]:
|
||||
"""恢复完整冻结计划,并返回其规范 payload 的稳定指纹。"""
|
||||
if pending.state not in {
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
}:
|
||||
raise TransferExecutionConflictError("整理任务尚未进入可执行规划状态")
|
||||
if (
|
||||
pending.checkpoint_version is None
|
||||
or pending.checkpoint_payload is None
|
||||
or pending.planned_at is None
|
||||
):
|
||||
raise TransferExecutionConflictError("整理任务缺少完整计划检查点")
|
||||
try:
|
||||
checkpoint = TransferPlanCheckpoint.from_payload(
|
||||
pending.checkpoint_payload
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise TransferExecutionConflictError(
|
||||
"整理任务计划检查点无法恢复"
|
||||
) from error
|
||||
if pending.checkpoint_version != checkpoint.schema_version:
|
||||
raise TransferExecutionConflictError("整理任务计划检查点版本不一致")
|
||||
if checkpoint.planning_input.fingerprint != pending.input_fingerprint:
|
||||
raise TransferExecutionConflictError("整理任务计划与准入输入指纹不一致")
|
||||
expected_state = (
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
if checkpoint.is_provider_pending
|
||||
else TRANSFER_ADMISSION_PLANNED
|
||||
)
|
||||
if pending.state != expected_state:
|
||||
raise TransferExecutionConflictError("整理任务计划类型与准入状态不一致")
|
||||
return checkpoint, checkpoint.fingerprint
|
||||
|
||||
@staticmethod
|
||||
def _provider_predecessor_checkpoint(
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
step: TransferExecutionStepModel,
|
||||
) -> Optional[TransferPlanCheckpoint]:
|
||||
"""重建 provider 回退前的冻结计划,证明序号零步骤的历史归属。"""
|
||||
if (
|
||||
not checkpoint.pre_execution_cleanup_completed
|
||||
or step.ordinal != 0
|
||||
or step.phase != "provider"
|
||||
or step.kind != "legacy_transfer_provider_sequence"
|
||||
):
|
||||
return None
|
||||
providers_payload = step.intent_payload.get("providers")
|
||||
invocation_payload = step.intent_payload.get("invocation")
|
||||
if (
|
||||
not isinstance(providers_payload, list)
|
||||
or not providers_payload
|
||||
or not all(isinstance(item, dict) for item in providers_payload)
|
||||
or not isinstance(invocation_payload, dict)
|
||||
):
|
||||
return None
|
||||
try:
|
||||
providers = tuple(
|
||||
TransferProviderReference.from_payload(item)
|
||||
for item in providers_payload
|
||||
)
|
||||
invocation = TransferProviderInvocationSnapshot.from_payload(
|
||||
invocation_payload
|
||||
)
|
||||
if invocation.fileitem != checkpoint.planning_input.source_fileitem:
|
||||
return None
|
||||
predecessor = TransferPlanCheckpoint(
|
||||
planning_input=checkpoint.planning_input,
|
||||
target_storage="",
|
||||
root_target_path="",
|
||||
final_target_path="",
|
||||
resolved_transfer_type="",
|
||||
items=(),
|
||||
resolved_meta=invocation.meta,
|
||||
resolved_meta_kind=invocation.meta_kind,
|
||||
resolved_mediainfo=invocation.mediainfo,
|
||||
resolved_mediainfo_kind=invocation.mediainfo_kind,
|
||||
resolved_episodes_info=invocation.episodes_info,
|
||||
legacy_transfer_providers=providers,
|
||||
provider_invocation=invocation,
|
||||
preview=invocation.preview,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return predecessor
|
||||
|
||||
@staticmethod
|
||||
def _intent_belongs_to_checkpoint(
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
*,
|
||||
phase: str,
|
||||
kind: str,
|
||||
payload: dict[str, Any],
|
||||
previous_steps: list[TransferExecutionStepModel],
|
||||
) -> bool:
|
||||
"""校验步骤类型及稳定参数由冻结计划或其已提交发现证据导出。"""
|
||||
if checkpoint.rejection_error:
|
||||
return (
|
||||
phase == "planning"
|
||||
and kind == "reject"
|
||||
and payload == {"error": checkpoint.rejection_error}
|
||||
)
|
||||
if checkpoint.is_provider_pending:
|
||||
invocation = checkpoint.provider_invocation
|
||||
if invocation is None:
|
||||
return False
|
||||
return (
|
||||
phase == "provider"
|
||||
and kind == "legacy_transfer_provider_sequence"
|
||||
and payload == {
|
||||
"providers": [
|
||||
provider.to_payload()
|
||||
for provider in checkpoint.legacy_transfer_providers
|
||||
],
|
||||
"invocation": invocation.to_payload(),
|
||||
}
|
||||
)
|
||||
plan_items = tuple(checkpoint.items)
|
||||
if phase == "transfer" and kind == checkpoint.resolved_transfer_type:
|
||||
return any(payload == {
|
||||
"source": item.source_fileitem.get("path"),
|
||||
"target": item.target_path,
|
||||
} for item in plan_items)
|
||||
if phase == "transfer" and kind == "materialize_target":
|
||||
return any(payload == {
|
||||
"source": item.source_fileitem,
|
||||
"target_storage": item.target_storage,
|
||||
"target_path": item.target_path,
|
||||
"transfer_type": (
|
||||
"copy"
|
||||
if (
|
||||
checkpoint.resolved_transfer_type == "move"
|
||||
and item.source_fileitem.get("storage")
|
||||
!= item.target_storage
|
||||
)
|
||||
else checkpoint.resolved_transfer_type
|
||||
),
|
||||
} for item in plan_items)
|
||||
if phase == "transfer" and kind == "delete_move_source":
|
||||
return (
|
||||
checkpoint.resolved_transfer_type == "move"
|
||||
and any(
|
||||
item.source_fileitem.get("storage") != item.target_storage
|
||||
and payload == {
|
||||
"source": item.source_fileitem,
|
||||
"target_storage": item.target_storage,
|
||||
"target_path": item.target_path,
|
||||
}
|
||||
for item in plan_items
|
||||
)
|
||||
)
|
||||
source = checkpoint.planning_input.source_fileitem
|
||||
if phase == "prepare" and kind == "cleanup_previous_destination":
|
||||
return payload == {"source_path": source.get("path")}
|
||||
if phase == "prepare" and kind == "ensure_target_directory":
|
||||
target_path = Path(checkpoint.final_target_path)
|
||||
directory_path = (
|
||||
target_path
|
||||
if source.get("type") == "dir"
|
||||
else target_path.parent
|
||||
)
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": directory_path.as_posix(),
|
||||
}
|
||||
if phase == "prepare" and kind == "delete_overwrite_target":
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": checkpoint.final_target_path,
|
||||
}
|
||||
if phase == "decision" and kind == "resolve_overwrite":
|
||||
return payload == {
|
||||
"source": source,
|
||||
"target_storage": checkpoint.target_storage,
|
||||
"target_path": checkpoint.final_target_path,
|
||||
"transfer_type": checkpoint.resolved_transfer_type,
|
||||
"overwrite_mode": checkpoint.overwrite_mode,
|
||||
"need_notify": checkpoint.need_notify,
|
||||
}
|
||||
if phase == "decision" and kind == "plugin_transfer_intercept":
|
||||
stable_payload = {
|
||||
"source": source,
|
||||
"target_storage": checkpoint.target_storage,
|
||||
"target_path": checkpoint.final_target_path,
|
||||
"transfer_type": checkpoint.resolved_transfer_type,
|
||||
}
|
||||
return payload == stable_payload or (
|
||||
set(payload) == {*stable_payload, "over_flag"}
|
||||
and all(payload[key] == value for key, value in stable_payload.items())
|
||||
and isinstance(payload.get("over_flag"), bool)
|
||||
)
|
||||
if phase == "prepare" and kind == "discover_version_targets":
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": checkpoint.final_target_path,
|
||||
}
|
||||
if phase == "prepare" and kind == "delete_version_target":
|
||||
candidate = payload.get("item")
|
||||
if (
|
||||
set(payload) != {"storage", "item"}
|
||||
or payload.get("storage") != checkpoint.target_storage
|
||||
or not isinstance(candidate, dict)
|
||||
):
|
||||
return False
|
||||
return any(
|
||||
step.phase == "prepare"
|
||||
and step.kind == "discover_version_targets"
|
||||
and step.state == TransferStepState.SUCCEEDED.value
|
||||
and isinstance(step.result_payload, dict)
|
||||
and candidate in step.result_payload.get("items", [])
|
||||
for step in previous_steps
|
||||
)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _validate_plan_steps(
|
||||
cls,
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
checkpoint_fingerprint: str,
|
||||
steps: list[TransferExecutionStepModel],
|
||||
) -> None:
|
||||
"""验证全部持久步骤属于冻结计划演进且身份与全局顺序未被篡改。"""
|
||||
if tuple(step.ordinal for step in steps) != tuple(range(len(steps))):
|
||||
raise TransferExecutionConflictError("整理步骤全局序号不连续")
|
||||
for index, step in enumerate(steps):
|
||||
if step.task_id != task_id:
|
||||
raise TransferExecutionConflictError("整理步骤绑定了错误任务")
|
||||
step_checkpoint = checkpoint
|
||||
if step.checkpoint_fingerprint != checkpoint_fingerprint:
|
||||
predecessor = cls._provider_predecessor_checkpoint(
|
||||
checkpoint,
|
||||
step,
|
||||
)
|
||||
if (
|
||||
predecessor is None
|
||||
or step.checkpoint_fingerprint != predecessor.fingerprint
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤不属于当前冻结计划或合法 provider 前驱计划"
|
||||
)
|
||||
step_checkpoint = predecessor
|
||||
if not cls._intent_belongs_to_checkpoint(
|
||||
step_checkpoint,
|
||||
phase=step.phase,
|
||||
kind=step.kind,
|
||||
payload=step.intent_payload,
|
||||
previous_steps=steps[:index],
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤类型或参数不能由冻结计划导出"
|
||||
)
|
||||
expected_operation_id = build_transfer_operation_id(
|
||||
task_id=task_id,
|
||||
checkpoint_fingerprint=step.checkpoint_fingerprint,
|
||||
ordinal=step.ordinal,
|
||||
phase=step.phase,
|
||||
kind=step.kind,
|
||||
intent_payload=step.intent_payload,
|
||||
)
|
||||
if step.operation_id != expected_operation_id:
|
||||
raise TransferExecutionConflictError("整理步骤 operation ID 与冻结意图不一致")
|
||||
|
||||
@classmethod
|
||||
def _validate_new_intent(
|
||||
cls,
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
checkpoint_fingerprint: str,
|
||||
steps: list[TransferExecutionStepModel],
|
||||
intent: TransferStepIntent,
|
||||
) -> None:
|
||||
"""验证待准备意图属于当前冻结计划,并且只追加或幂等重放既有序号。"""
|
||||
cls._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
if intent.checkpoint_fingerprint != checkpoint_fingerprint:
|
||||
raise TransferExecutionConflictError("整理步骤意图未绑定当前冻结计划指纹")
|
||||
if not cls._intent_belongs_to_checkpoint(
|
||||
checkpoint,
|
||||
phase=intent.phase,
|
||||
kind=intent.kind,
|
||||
payload=intent.payload,
|
||||
previous_steps=steps,
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤意图类型或参数不能由冻结计划导出"
|
||||
)
|
||||
expected_operation_id = build_transfer_operation_id(
|
||||
task_id=task_id,
|
||||
checkpoint_fingerprint=intent.checkpoint_fingerprint,
|
||||
ordinal=intent.ordinal,
|
||||
phase=intent.phase,
|
||||
kind=intent.kind,
|
||||
intent_payload=intent.payload,
|
||||
)
|
||||
if intent.operation_id != expected_operation_id:
|
||||
raise TransferExecutionConflictError("整理步骤意图 operation ID 不可信")
|
||||
existing = next(
|
||||
(step for step in steps if step.operation_id == intent.operation_id),
|
||||
None,
|
||||
)
|
||||
if existing is not None:
|
||||
if not cls._intent_matches(existing, task_id=task_id, intent=intent):
|
||||
raise TransferExecutionConflictError(
|
||||
"稳定 operation ID 已绑定不同步骤意图"
|
||||
)
|
||||
return
|
||||
if intent.ordinal != len(steps):
|
||||
raise TransferExecutionConflictError("整理步骤意图必须按全局序号连续追加")
|
||||
|
||||
@staticmethod
|
||||
def _require_active_lease(
|
||||
pending: Optional[TransferPending],
|
||||
*,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
detail: str,
|
||||
) -> None:
|
||||
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
|
||||
) -> TransferPending:
|
||||
"""返回仍由调用方持有的任务,并优先报告 lease fencing 失败。"""
|
||||
if (
|
||||
pending is None
|
||||
or pending.lease_token != lease_token
|
||||
@@ -244,6 +571,23 @@ class TransactionalTransferExecutionRepository:
|
||||
or pending.lease_expires_at <= now_utc
|
||||
):
|
||||
raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管")
|
||||
return pending
|
||||
|
||||
@classmethod
|
||||
def _raise_fenced_failure(
|
||||
cls,
|
||||
pending: Optional[TransferPending],
|
||||
*,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
detail: str,
|
||||
) -> None:
|
||||
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
|
||||
cls._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
raise TransferExecutionConflictError(detail)
|
||||
|
||||
def _times(self) -> tuple[str, str]:
|
||||
@@ -354,9 +698,29 @@ class TransactionalTransferExecutionRepository:
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
self._validate_new_intent(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
intent=intent,
|
||||
)
|
||||
updated = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=checkpoint.schema_version,
|
||||
checkpoint_payload=checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
@@ -367,7 +731,6 @@ class TransactionalTransferExecutionRepository:
|
||||
now_utc=now_utc,
|
||||
detail="整理任务当前状态不能准备外部步骤",
|
||||
)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
step = oper.get_by_operation_id(operation_id=intent.operation_id)
|
||||
if step is None:
|
||||
step = oper.stage_prepare(
|
||||
@@ -498,9 +861,32 @@ class TransactionalTransferExecutionRepository:
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
self._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
if operation_id not in {step.operation_id for step in steps}:
|
||||
raise TransferExecutionConflictError(
|
||||
"待恢复步骤不属于冻结计划"
|
||||
)
|
||||
pending_updated = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=checkpoint.schema_version,
|
||||
checkpoint_payload=checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
@@ -511,7 +897,6 @@ class TransactionalTransferExecutionRepository:
|
||||
now_utc=now_utc,
|
||||
detail="重试任务未到期或当前状态不能恢复",
|
||||
)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
updated = oper.stage_resume_failed_attempt(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
@@ -973,24 +1358,39 @@ class TransactionalTransferExecutionRepository:
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
plan_checkpoint, plan_fingerprint = self._plan_identity(pending)
|
||||
step_oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
step_oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
step_ids = {step.operation_id for step in steps}
|
||||
if step_ids != set(checkpoint.operation_ids):
|
||||
self._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=plan_checkpoint,
|
||||
checkpoint_fingerprint=plan_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
step_ids = tuple(step.operation_id for step in steps)
|
||||
if step_ids != checkpoint.operation_ids:
|
||||
raise TransferExecutionConflictError(
|
||||
"执行检查点引用的步骤集合与持久步骤不一致"
|
||||
"执行检查点引用的步骤顺序与持久步骤不一致"
|
||||
)
|
||||
if any(step.state != TransferStepState.SUCCEEDED.value for step in steps):
|
||||
raise TransferExecutionConflictError(
|
||||
"存在未成功步骤,不能提交执行检查点"
|
||||
)
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
running = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=plan_checkpoint.schema_version,
|
||||
checkpoint_payload=plan_checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional, cast
|
||||
from uuid import uuid4
|
||||
@@ -12,66 +10,21 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
column,
|
||||
delete,
|
||||
exists,
|
||||
func,
|
||||
or_,
|
||||
select,
|
||||
table,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
from app.db.base import Base, execute_dml, get_id_column
|
||||
|
||||
|
||||
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"source_fileitem": {"storage": storage, "path": src_path},
|
||||
"meta": None,
|
||||
"mediainfo": None,
|
||||
"target_directory": None,
|
||||
"target_storage": None,
|
||||
"target_path": None,
|
||||
"requested_transfer_type": None,
|
||||
"media_source": None,
|
||||
"media_id": None,
|
||||
"media_type": None,
|
||||
"need_scrape": False,
|
||||
"need_rename": True,
|
||||
"need_notify": True,
|
||||
"overwrite_mode": None,
|
||||
"episodes_info": [],
|
||||
"preview": False,
|
||||
"options": {"legacy_replan": True},
|
||||
}
|
||||
|
||||
|
||||
def _planning_fingerprint(payload: dict[str, Any]) -> str:
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=True,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _default_planning_payload(context: Any) -> dict[str, Any]:
|
||||
params = context.get_current_parameters()
|
||||
return _legacy_planning_payload(
|
||||
params.get("storage", ""),
|
||||
params.get("src_path", ""),
|
||||
)
|
||||
|
||||
|
||||
def _default_planning_fingerprint(context: Any) -> str:
|
||||
params = context.get_current_parameters()
|
||||
payload = params.get("planning_input") or _legacy_planning_payload(
|
||||
params.get("storage", ""),
|
||||
params.get("src_path", ""),
|
||||
)
|
||||
return _planning_fingerprint(payload)
|
||||
_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
|
||||
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
|
||||
|
||||
|
||||
class TransferPending(Base):
|
||||
@@ -112,11 +65,11 @@ class TransferPending(Base):
|
||||
input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
# 版本化规划输入 JSON
|
||||
planning_input: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, nullable=False, default=_default_planning_payload
|
||||
JSON, nullable=False
|
||||
)
|
||||
# 规划输入规范 JSON 的 SHA-256 指纹
|
||||
input_fingerprint: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default=_default_planning_fingerprint
|
||||
String(64), nullable=False
|
||||
)
|
||||
# 完整计划格式版本,尚未规划时为空
|
||||
checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
@@ -203,9 +156,9 @@ class TransferPending(Base):
|
||||
@classmethod
|
||||
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
|
||||
src_path: str, state: str,
|
||||
now_time: str, input_version: int = 1,
|
||||
planning_input: Optional[dict[str, Any]] = None,
|
||||
input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]:
|
||||
now_time: str, input_version: int,
|
||||
planning_input: dict[str, Any],
|
||||
input_fingerprint: str) -> Optional["TransferPending"]:
|
||||
"""
|
||||
在调用方会话中暂存一条持久接纳记录。
|
||||
|
||||
@@ -221,15 +174,20 @@ class TransferPending(Base):
|
||||
:param input_fingerprint: 规划输入规范 JSON 指纹
|
||||
:return: 接纳记录
|
||||
"""
|
||||
if not task_id or not storage or not src_path or not state:
|
||||
if (
|
||||
not task_id
|
||||
or not storage
|
||||
or not src_path
|
||||
or not state
|
||||
or not planning_input
|
||||
or not input_fingerprint
|
||||
):
|
||||
return None
|
||||
pending = db.execute(
|
||||
select(cls).where(cls.storage == storage, cls.src_path == src_path)
|
||||
).scalars().first()
|
||||
if pending:
|
||||
return cast("TransferPending", pending)
|
||||
effective_input = planning_input or _legacy_planning_payload(storage, src_path)
|
||||
effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input)
|
||||
pending = cls(
|
||||
task_id=task_id,
|
||||
storage=storage,
|
||||
@@ -238,8 +196,8 @@ class TransferPending(Base):
|
||||
created_at=now_time,
|
||||
updated_at=now_time,
|
||||
input_version=input_version,
|
||||
planning_input=effective_input,
|
||||
input_fingerprint=effective_fingerprint,
|
||||
planning_input=planning_input,
|
||||
input_fingerprint=input_fingerprint,
|
||||
)
|
||||
db.add(pending)
|
||||
return pending
|
||||
@@ -425,17 +383,33 @@ class TransferPending(Base):
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
admission_state: str,
|
||||
checkpoint_version: int,
|
||||
checkpoint_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约把可执行任务推进或保持为 running。"""
|
||||
if not all((task_id, lease_token, now_utc, updated_at)):
|
||||
"""仅以有效租约和完整冻结计划把任务推进或保持为 running。"""
|
||||
if not all((
|
||||
task_id,
|
||||
lease_token,
|
||||
admission_state,
|
||||
checkpoint_version,
|
||||
checkpoint_payload,
|
||||
now_utc,
|
||||
updated_at,
|
||||
)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == admission_state,
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version == checkpoint_version,
|
||||
cls.checkpoint_payload == checkpoint_payload,
|
||||
cls.planned_at.is_not(None),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
@@ -470,9 +444,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -512,6 +487,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -558,9 +537,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -598,9 +578,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -886,7 +867,7 @@ class TransferPending(Base):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def discard_claimed(
|
||||
def abandon_unstarted(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
@@ -895,7 +876,10 @@ class TransferPending(Base):
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
仅以当前且未过期的 token 删除已经到达终态的租约任务。
|
||||
仅以当前且未过期的 token 删除确认从未执行的缺失源任务。
|
||||
|
||||
任何执行状态、聚合检查点或步骤证据都意味着外部结果需要由状态机
|
||||
判定;即使源文件已经消失,也不能以此推断 move 已经安全完成。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
@@ -912,6 +896,29 @@ class TransferPending(Base):
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
cls.state == "accepted",
|
||||
cls.checkpoint_version.is_(None),
|
||||
cls.checkpoint_payload.is_(None),
|
||||
cls.planned_at.is_(None),
|
||||
cls.execution_state == "not_started",
|
||||
cls.execution_version.is_(None),
|
||||
cls.execution_payload.is_(None),
|
||||
cls.execution_fingerprint.is_(None),
|
||||
cls.retry_generation == 0,
|
||||
cls.retry_count == 0,
|
||||
cls.retry_due_at.is_(None),
|
||||
cls.settlement_revision == 0,
|
||||
cls.terminal_history_id.is_(None),
|
||||
~exists(
|
||||
select(_TRANSFER_EXECUTION_STEP.c.task_id).where(
|
||||
_TRANSFER_EXECUTION_STEP.c.task_id == cls.task_id
|
||||
)
|
||||
),
|
||||
~exists(
|
||||
select(_TRANSFER_HISTORY.c.transfer_task_id).where(
|
||||
_TRANSFER_HISTORY.c.transfer_task_id == cls.task_id
|
||||
)
|
||||
),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@@ -13,9 +13,9 @@ class TransferPendingOper(DbOper):
|
||||
"""
|
||||
|
||||
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
|
||||
state: str, now_time: str, input_version: int = 1,
|
||||
planning_input: Optional[dict[str, Any]] = None,
|
||||
input_fingerprint: Optional[str] = None) -> Optional[TransferPending]:
|
||||
state: str, now_time: str, input_version: int,
|
||||
planning_input: dict[str, Any],
|
||||
input_fingerprint: str) -> Optional[TransferPending]:
|
||||
"""
|
||||
在当前会话中暂存一条持久接纳记录。
|
||||
|
||||
@@ -287,7 +287,7 @@ class TransferPendingOper(DbOper):
|
||||
)
|
||||
)
|
||||
|
||||
def stage_discard_claimed(
|
||||
def stage_abandon_unstarted(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
@@ -295,7 +295,7 @@ class TransferPendingOper(DbOper):
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话按当前未过期 token 删除终态任务。
|
||||
使用当前会话按当前未过期 token 删除确认从未执行的缺失源任务。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
@@ -303,7 +303,7 @@ class TransferPendingOper(DbOper):
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.discard_claimed(
|
||||
lambda session: TransferPending.abandon_unstarted(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
@@ -334,6 +334,9 @@ class TransferPendingOper(DbOper):
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
admission_state: str,
|
||||
checkpoint_version: int,
|
||||
checkpoint_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
@@ -343,6 +346,9 @@ class TransferPendingOper(DbOper):
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=admission_state,
|
||||
checkpoint_version=checkpoint_version,
|
||||
checkpoint_payload=checkpoint_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
@@ -301,7 +301,6 @@ SCHEMA_EXPORTS = {
|
||||
'PluginTriggeredEventData': ('app.schemas.event', 'PluginTriggeredEventData'),
|
||||
'PluginUpdateCandidate': ('app.schemas.plugin', 'PluginUpdateCandidate'),
|
||||
'PluginWorkflowActionGroup': ('app.schemas.workflow', 'PluginWorkflowActionGroup'),
|
||||
'PrivateAttr': ('app.schemas.plugin', 'PrivateAttr'),
|
||||
'ProcessInfo': ('app.schemas.dashboard', 'ProcessInfo'),
|
||||
'ProgressKeyData': ('app.schemas.common', 'ProgressKeyData'),
|
||||
'RadarrMovie': ('app.schemas.servarr', 'RadarrMovie'),
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
from enum import Enum as _Enum
|
||||
from typing import Dict, List, Literal, Optional, Union
|
||||
|
||||
from pydantic import BaseModel, Field, PrivateAttr, RootModel, field_validator
|
||||
from pydantic import BaseModel, Field, RootModel, field_validator
|
||||
from pydantic import PrivateAttr as _PrivateAttr
|
||||
|
||||
from app.schemas.common import JsonData
|
||||
|
||||
@@ -62,7 +63,7 @@ class Plugin(BaseModel):
|
||||
"""
|
||||
插件信息
|
||||
"""
|
||||
_package_version: Optional[str] = PrivateAttr(default=None)
|
||||
_package_version: Optional[str] = _PrivateAttr(default=None)
|
||||
|
||||
id: str = None
|
||||
# 插件名称
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from uuid import uuid4
|
||||
@@ -13,6 +15,42 @@ _TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
|
||||
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
|
||||
|
||||
|
||||
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
|
||||
"""只在旧登记 ABI 内构造可由新状态机保守重规划的最小输入。"""
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"source_fileitem": {"storage": storage, "path": src_path},
|
||||
"meta": None,
|
||||
"mediainfo": None,
|
||||
"target_directory": None,
|
||||
"target_storage": None,
|
||||
"target_path": None,
|
||||
"requested_transfer_type": None,
|
||||
"media_source": None,
|
||||
"media_id": None,
|
||||
"media_type": None,
|
||||
"need_scrape": False,
|
||||
"need_rename": True,
|
||||
"need_notify": True,
|
||||
"overwrite_mode": None,
|
||||
"episodes_info": [],
|
||||
"preview": False,
|
||||
"options": {"legacy_replan": True},
|
||||
}
|
||||
|
||||
|
||||
def _planning_fingerprint(payload: dict[str, Any]) -> str:
|
||||
"""生成兼容登记输入与 canonical DTO 一致的规范指纹。"""
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=True,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _safe_legacy_delete_predicates() -> tuple[Any, ...]:
|
||||
"""只允许旧接口删除从未 claim、执行或结算的新鲜登记。"""
|
||||
return (
|
||||
@@ -61,6 +99,7 @@ class TransferPendingOper(DbOper):
|
||||
:return: 登记记录
|
||||
"""
|
||||
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
planning_input = _legacy_planning_payload(storage, src_path)
|
||||
return self._execute_sync_write(
|
||||
lambda session: _TransferPending.stage_admit(
|
||||
session,
|
||||
@@ -69,6 +108,9 @@ class TransferPendingOper(DbOper):
|
||||
src_path=src_path,
|
||||
state="accepted",
|
||||
now_time=now_time,
|
||||
input_version=1,
|
||||
planning_input=planning_input,
|
||||
input_fingerprint=_planning_fingerprint(planning_input),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user