mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
Merge upstream/v3 into codex/feat/plugin-data-query-sdk-v3
This commit is contained in:
@@ -26,6 +26,7 @@ from app.application.outbox import (
|
||||
OutboxIntent,
|
||||
)
|
||||
from app.application.transfer.execution import (
|
||||
TransferExecutionCheckpoint,
|
||||
TransferExecutionConflictError,
|
||||
TransferExecutionLeaseLostError,
|
||||
TransferExecutionState,
|
||||
@@ -403,6 +404,13 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
|
||||
!= settlement.execution_fingerprint
|
||||
):
|
||||
raise TransferExecutionConflictError("整理终态与执行检查点不匹配")
|
||||
checkpoint = TransferExecutionCheckpoint.from_payload(
|
||||
pending.execution_payload,
|
||||
fingerprint=settlement.execution_fingerprint,
|
||||
)
|
||||
if pending.execution_version != checkpoint.version:
|
||||
raise TransferExecutionConflictError("整理执行检查点列版本与内容不一致")
|
||||
checkpoint.validate_settlement_outcome(settlement.outcome)
|
||||
return int(pending.settlement_revision)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -142,18 +142,14 @@ class TransactionalTransferAdmissionRepository:
|
||||
*,
|
||||
storage: str,
|
||||
src_path: str,
|
||||
planning_input: Optional[TransferPlanningInput] = None,
|
||||
planning_input: TransferPlanningInput,
|
||||
) -> TransferAdmission:
|
||||
"""按输入指纹幂等持久化准入事实,并返回跨重启稳定身份。"""
|
||||
if not storage or not src_path:
|
||||
raise ValueError("整理任务的存储与源路径不能为空")
|
||||
effective_input = planning_input or TransferPlanningInput.legacy(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
if (
|
||||
effective_input.source_fileitem.get("storage") != storage
|
||||
or effective_input.source_fileitem.get("path") != src_path
|
||||
planning_input.source_fileitem.get("storage") != storage
|
||||
or planning_input.source_fileitem.get("path") != src_path
|
||||
):
|
||||
raise ValueError("整理规划输入的源文件身份与准入参数不一致")
|
||||
now_time = self._now()
|
||||
@@ -167,16 +163,16 @@ class TransactionalTransferAdmissionRepository:
|
||||
src_path=src_path,
|
||||
state=TRANSFER_ADMISSION_ACCEPTED,
|
||||
now_time=now_time,
|
||||
input_version=effective_input.schema_version,
|
||||
planning_input=effective_input.to_payload(),
|
||||
input_fingerprint=effective_input.fingerprint,
|
||||
input_version=planning_input.schema_version,
|
||||
planning_input=planning_input.to_payload(),
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
)
|
||||
if pending is None:
|
||||
raise TransferAdmissionConflictError(
|
||||
f"整理源文件已有持久终态回执: {storage}:{src_path}"
|
||||
)
|
||||
session.flush()
|
||||
self._assert_input_match(pending, effective_input)
|
||||
self._assert_input_match(pending, planning_input)
|
||||
admission = self._project(pending)
|
||||
transaction.commit()
|
||||
return admission
|
||||
@@ -192,7 +188,7 @@ class TransactionalTransferAdmissionRepository:
|
||||
)
|
||||
if pending is None:
|
||||
raise RuntimeError("并发准入冲突后未找到已提交记录") from error
|
||||
self._assert_input_match(pending, effective_input)
|
||||
self._assert_input_match(pending, planning_input)
|
||||
return self._project(pending)
|
||||
|
||||
def claim_task(
|
||||
@@ -525,14 +521,14 @@ class TransactionalTransferAdmissionRepository:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅以当前未过期 token 删除终态任务,拒绝陈旧 worker 变更。"""
|
||||
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅以当前 token 删除无执行证据的缺失源任务,拒绝陈旧 worker。"""
|
||||
if not task_id or not lease_token:
|
||||
return 0
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
deleted = TransferPendingOper(db=session).stage_discard_claimed(
|
||||
deleted = TransferPendingOper(db=session).stage_abandon_unstarted(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
|
||||
@@ -4,7 +4,8 @@ from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from sqlalchemy import and_, func, or_, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
@@ -27,6 +28,14 @@ from app.application.transfer.execution import (
|
||||
TransferStepIntent,
|
||||
TransferStepResult,
|
||||
TransferStepState,
|
||||
build_transfer_operation_id,
|
||||
)
|
||||
from app.application.transfer.workflow import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TransferPlanCheckpoint,
|
||||
TransferProviderInvocationSnapshot,
|
||||
TransferProviderReference,
|
||||
)
|
||||
from app.db.models.transferexecutionstep import (
|
||||
TransferExecutionStep as TransferExecutionStepModel,
|
||||
@@ -229,14 +238,332 @@ class TransactionalTransferExecutionRepository:
|
||||
))
|
||||
|
||||
@staticmethod
|
||||
def _raise_fenced_failure(
|
||||
def _plan_identity(
|
||||
pending: TransferPending,
|
||||
) -> tuple[TransferPlanCheckpoint, str]:
|
||||
"""恢复完整冻结计划,并返回其规范 payload 的稳定指纹。"""
|
||||
if pending.state not in {
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
}:
|
||||
raise TransferExecutionConflictError("整理任务尚未进入可执行规划状态")
|
||||
if (
|
||||
pending.checkpoint_version is None
|
||||
or pending.checkpoint_payload is None
|
||||
or pending.planned_at is None
|
||||
):
|
||||
raise TransferExecutionConflictError("整理任务缺少完整计划检查点")
|
||||
try:
|
||||
checkpoint = TransferPlanCheckpoint.from_payload(
|
||||
pending.checkpoint_payload
|
||||
)
|
||||
except (TypeError, ValueError) as error:
|
||||
raise TransferExecutionConflictError(
|
||||
"整理任务计划检查点无法恢复"
|
||||
) from error
|
||||
if pending.checkpoint_version != checkpoint.schema_version:
|
||||
raise TransferExecutionConflictError("整理任务计划检查点版本不一致")
|
||||
if checkpoint.planning_input.fingerprint != pending.input_fingerprint:
|
||||
raise TransferExecutionConflictError("整理任务计划与准入输入指纹不一致")
|
||||
expected_state = (
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
if checkpoint.is_provider_pending
|
||||
else TRANSFER_ADMISSION_PLANNED
|
||||
)
|
||||
if pending.state != expected_state:
|
||||
raise TransferExecutionConflictError("整理任务计划类型与准入状态不一致")
|
||||
return checkpoint, checkpoint.fingerprint
|
||||
|
||||
@staticmethod
|
||||
def _provider_predecessor_checkpoint(
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
step: TransferExecutionStepModel,
|
||||
) -> Optional[TransferPlanCheckpoint]:
|
||||
"""重建 provider 回退前的冻结计划,证明序号零步骤的历史归属。"""
|
||||
if (
|
||||
not checkpoint.pre_execution_cleanup_completed
|
||||
or step.ordinal != 0
|
||||
or step.phase != "provider"
|
||||
or step.kind != "legacy_transfer_provider_sequence"
|
||||
):
|
||||
return None
|
||||
providers_payload = step.intent_payload.get("providers")
|
||||
invocation_payload = step.intent_payload.get("invocation")
|
||||
if (
|
||||
not isinstance(providers_payload, list)
|
||||
or not providers_payload
|
||||
or not all(isinstance(item, dict) for item in providers_payload)
|
||||
or not isinstance(invocation_payload, dict)
|
||||
):
|
||||
return None
|
||||
try:
|
||||
providers = tuple(
|
||||
TransferProviderReference.from_payload(item)
|
||||
for item in providers_payload
|
||||
)
|
||||
invocation = TransferProviderInvocationSnapshot.from_payload(
|
||||
invocation_payload
|
||||
)
|
||||
if invocation.fileitem != checkpoint.planning_input.source_fileitem:
|
||||
return None
|
||||
predecessor = TransferPlanCheckpoint(
|
||||
planning_input=checkpoint.planning_input,
|
||||
target_storage="",
|
||||
root_target_path="",
|
||||
final_target_path="",
|
||||
resolved_transfer_type="",
|
||||
items=(),
|
||||
resolved_meta=invocation.meta,
|
||||
resolved_meta_kind=invocation.meta_kind,
|
||||
resolved_mediainfo=invocation.mediainfo,
|
||||
resolved_mediainfo_kind=invocation.mediainfo_kind,
|
||||
resolved_episodes_info=invocation.episodes_info,
|
||||
legacy_transfer_providers=providers,
|
||||
provider_invocation=invocation,
|
||||
preview=invocation.preview,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return predecessor
|
||||
|
||||
@staticmethod
|
||||
def _intent_belongs_to_checkpoint(
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
*,
|
||||
phase: str,
|
||||
kind: str,
|
||||
payload: dict[str, Any],
|
||||
previous_steps: list[TransferExecutionStepModel],
|
||||
) -> bool:
|
||||
"""校验步骤类型及稳定参数由冻结计划或其已提交发现证据导出。"""
|
||||
if checkpoint.rejection_error:
|
||||
return (
|
||||
phase == "planning"
|
||||
and kind == "reject"
|
||||
and payload == {"error": checkpoint.rejection_error}
|
||||
)
|
||||
if checkpoint.is_provider_pending:
|
||||
invocation = checkpoint.provider_invocation
|
||||
if invocation is None:
|
||||
return False
|
||||
return (
|
||||
phase == "provider"
|
||||
and kind == "legacy_transfer_provider_sequence"
|
||||
and payload == {
|
||||
"providers": [
|
||||
provider.to_payload()
|
||||
for provider in checkpoint.legacy_transfer_providers
|
||||
],
|
||||
"invocation": invocation.to_payload(),
|
||||
}
|
||||
)
|
||||
plan_items = tuple(checkpoint.items)
|
||||
if phase == "transfer" and kind == checkpoint.resolved_transfer_type:
|
||||
return any(payload == {
|
||||
"source": item.source_fileitem.get("path"),
|
||||
"target": item.target_path,
|
||||
} for item in plan_items)
|
||||
if phase == "transfer" and kind == "materialize_target":
|
||||
return any(payload == {
|
||||
"source": item.source_fileitem,
|
||||
"target_storage": item.target_storage,
|
||||
"target_path": item.target_path,
|
||||
"transfer_type": (
|
||||
"copy"
|
||||
if (
|
||||
checkpoint.resolved_transfer_type == "move"
|
||||
and item.source_fileitem.get("storage")
|
||||
!= item.target_storage
|
||||
)
|
||||
else checkpoint.resolved_transfer_type
|
||||
),
|
||||
} for item in plan_items)
|
||||
if phase == "transfer" and kind == "delete_move_source":
|
||||
return (
|
||||
checkpoint.resolved_transfer_type == "move"
|
||||
and any(
|
||||
item.source_fileitem.get("storage") != item.target_storage
|
||||
and payload == {
|
||||
"source": item.source_fileitem,
|
||||
"target_storage": item.target_storage,
|
||||
"target_path": item.target_path,
|
||||
}
|
||||
for item in plan_items
|
||||
)
|
||||
)
|
||||
source = checkpoint.planning_input.source_fileitem
|
||||
if phase == "prepare" and kind == "cleanup_previous_destination":
|
||||
return payload == {"source_path": source.get("path")}
|
||||
if phase == "prepare" and kind == "ensure_target_directory":
|
||||
target_path = Path(checkpoint.final_target_path)
|
||||
directory_path = (
|
||||
target_path
|
||||
if source.get("type") == "dir"
|
||||
else target_path.parent
|
||||
)
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": directory_path.as_posix(),
|
||||
}
|
||||
if phase == "prepare" and kind == "delete_overwrite_target":
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": checkpoint.final_target_path,
|
||||
}
|
||||
if phase == "decision" and kind == "resolve_overwrite":
|
||||
return payload == {
|
||||
"source": source,
|
||||
"target_storage": checkpoint.target_storage,
|
||||
"target_path": checkpoint.final_target_path,
|
||||
"transfer_type": checkpoint.resolved_transfer_type,
|
||||
"overwrite_mode": checkpoint.overwrite_mode,
|
||||
"need_notify": checkpoint.need_notify,
|
||||
}
|
||||
if phase == "decision" and kind == "plugin_transfer_intercept":
|
||||
stable_payload = {
|
||||
"source": source,
|
||||
"target_storage": checkpoint.target_storage,
|
||||
"target_path": checkpoint.final_target_path,
|
||||
"transfer_type": checkpoint.resolved_transfer_type,
|
||||
}
|
||||
return payload == stable_payload or (
|
||||
set(payload) == {*stable_payload, "over_flag"}
|
||||
and all(payload[key] == value for key, value in stable_payload.items())
|
||||
and isinstance(payload.get("over_flag"), bool)
|
||||
)
|
||||
if phase == "prepare" and kind == "discover_version_targets":
|
||||
return payload == {
|
||||
"storage": checkpoint.target_storage,
|
||||
"path": checkpoint.final_target_path,
|
||||
}
|
||||
if phase == "prepare" and kind == "delete_version_target":
|
||||
candidate = payload.get("item")
|
||||
if (
|
||||
set(payload) != {"storage", "item"}
|
||||
or payload.get("storage") != checkpoint.target_storage
|
||||
or not isinstance(candidate, dict)
|
||||
):
|
||||
return False
|
||||
return any(
|
||||
step.phase == "prepare"
|
||||
and step.kind == "discover_version_targets"
|
||||
and step.state == TransferStepState.SUCCEEDED.value
|
||||
and isinstance(step.result_payload, dict)
|
||||
and candidate in step.result_payload.get("items", [])
|
||||
for step in previous_steps
|
||||
)
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _validate_plan_steps(
|
||||
cls,
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
checkpoint_fingerprint: str,
|
||||
steps: list[TransferExecutionStepModel],
|
||||
) -> None:
|
||||
"""验证全部持久步骤属于冻结计划演进且身份与全局顺序未被篡改。"""
|
||||
if tuple(step.ordinal for step in steps) != tuple(range(len(steps))):
|
||||
raise TransferExecutionConflictError("整理步骤全局序号不连续")
|
||||
for index, step in enumerate(steps):
|
||||
if step.task_id != task_id:
|
||||
raise TransferExecutionConflictError("整理步骤绑定了错误任务")
|
||||
step_checkpoint = checkpoint
|
||||
if step.checkpoint_fingerprint != checkpoint_fingerprint:
|
||||
predecessor = cls._provider_predecessor_checkpoint(
|
||||
checkpoint,
|
||||
step,
|
||||
)
|
||||
if (
|
||||
predecessor is None
|
||||
or step.checkpoint_fingerprint != predecessor.fingerprint
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤不属于当前冻结计划或合法 provider 前驱计划"
|
||||
)
|
||||
step_checkpoint = predecessor
|
||||
if not cls._intent_belongs_to_checkpoint(
|
||||
step_checkpoint,
|
||||
phase=step.phase,
|
||||
kind=step.kind,
|
||||
payload=step.intent_payload,
|
||||
previous_steps=steps[:index],
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤类型或参数不能由冻结计划导出"
|
||||
)
|
||||
expected_operation_id = build_transfer_operation_id(
|
||||
task_id=task_id,
|
||||
checkpoint_fingerprint=step.checkpoint_fingerprint,
|
||||
ordinal=step.ordinal,
|
||||
phase=step.phase,
|
||||
kind=step.kind,
|
||||
intent_payload=step.intent_payload,
|
||||
)
|
||||
if step.operation_id != expected_operation_id:
|
||||
raise TransferExecutionConflictError("整理步骤 operation ID 与冻结意图不一致")
|
||||
|
||||
@classmethod
|
||||
def _validate_new_intent(
|
||||
cls,
|
||||
*,
|
||||
task_id: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
checkpoint_fingerprint: str,
|
||||
steps: list[TransferExecutionStepModel],
|
||||
intent: TransferStepIntent,
|
||||
) -> None:
|
||||
"""验证待准备意图属于当前冻结计划,并且只追加或幂等重放既有序号。"""
|
||||
cls._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
if intent.checkpoint_fingerprint != checkpoint_fingerprint:
|
||||
raise TransferExecutionConflictError("整理步骤意图未绑定当前冻结计划指纹")
|
||||
if not cls._intent_belongs_to_checkpoint(
|
||||
checkpoint,
|
||||
phase=intent.phase,
|
||||
kind=intent.kind,
|
||||
payload=intent.payload,
|
||||
previous_steps=steps,
|
||||
):
|
||||
raise TransferExecutionConflictError(
|
||||
"整理步骤意图类型或参数不能由冻结计划导出"
|
||||
)
|
||||
expected_operation_id = build_transfer_operation_id(
|
||||
task_id=task_id,
|
||||
checkpoint_fingerprint=intent.checkpoint_fingerprint,
|
||||
ordinal=intent.ordinal,
|
||||
phase=intent.phase,
|
||||
kind=intent.kind,
|
||||
intent_payload=intent.payload,
|
||||
)
|
||||
if intent.operation_id != expected_operation_id:
|
||||
raise TransferExecutionConflictError("整理步骤意图 operation ID 不可信")
|
||||
existing = next(
|
||||
(step for step in steps if step.operation_id == intent.operation_id),
|
||||
None,
|
||||
)
|
||||
if existing is not None:
|
||||
if not cls._intent_matches(existing, task_id=task_id, intent=intent):
|
||||
raise TransferExecutionConflictError(
|
||||
"稳定 operation ID 已绑定不同步骤意图"
|
||||
)
|
||||
return
|
||||
if intent.ordinal != len(steps):
|
||||
raise TransferExecutionConflictError("整理步骤意图必须按全局序号连续追加")
|
||||
|
||||
@staticmethod
|
||||
def _require_active_lease(
|
||||
pending: Optional[TransferPending],
|
||||
*,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
detail: str,
|
||||
) -> None:
|
||||
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
|
||||
) -> TransferPending:
|
||||
"""返回仍由调用方持有的任务,并优先报告 lease fencing 失败。"""
|
||||
if (
|
||||
pending is None
|
||||
or pending.lease_token != lease_token
|
||||
@@ -244,6 +571,23 @@ class TransactionalTransferExecutionRepository:
|
||||
or pending.lease_expires_at <= now_utc
|
||||
):
|
||||
raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管")
|
||||
return pending
|
||||
|
||||
@classmethod
|
||||
def _raise_fenced_failure(
|
||||
cls,
|
||||
pending: Optional[TransferPending],
|
||||
*,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
detail: str,
|
||||
) -> None:
|
||||
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
|
||||
cls._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
raise TransferExecutionConflictError(detail)
|
||||
|
||||
def _times(self) -> tuple[str, str]:
|
||||
@@ -354,9 +698,29 @@ class TransactionalTransferExecutionRepository:
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
self._validate_new_intent(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
intent=intent,
|
||||
)
|
||||
updated = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=checkpoint.schema_version,
|
||||
checkpoint_payload=checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
@@ -367,7 +731,6 @@ class TransactionalTransferExecutionRepository:
|
||||
now_utc=now_utc,
|
||||
detail="整理任务当前状态不能准备外部步骤",
|
||||
)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
step = oper.get_by_operation_id(operation_id=intent.operation_id)
|
||||
if step is None:
|
||||
step = oper.stage_prepare(
|
||||
@@ -498,9 +861,32 @@ class TransactionalTransferExecutionRepository:
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
self._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=checkpoint,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
if operation_id not in {step.operation_id for step in steps}:
|
||||
raise TransferExecutionConflictError(
|
||||
"待恢复步骤不属于冻结计划"
|
||||
)
|
||||
pending_updated = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=checkpoint.schema_version,
|
||||
checkpoint_payload=checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
@@ -511,7 +897,6 @@ class TransactionalTransferExecutionRepository:
|
||||
now_utc=now_utc,
|
||||
detail="重试任务未到期或当前状态不能恢复",
|
||||
)
|
||||
oper = TransferExecutionStepOper(session)
|
||||
updated = oper.stage_resume_failed_attempt(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
@@ -973,24 +1358,39 @@ class TransactionalTransferExecutionRepository:
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
pending = self._require_active_lease(
|
||||
pending,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
)
|
||||
plan_checkpoint, plan_fingerprint = self._plan_identity(pending)
|
||||
step_oper = TransferExecutionStepOper(session)
|
||||
steps = self._execution_steps(
|
||||
step_oper.list_by_task_id(task_id=task_id)
|
||||
)
|
||||
step_ids = {step.operation_id for step in steps}
|
||||
if step_ids != set(checkpoint.operation_ids):
|
||||
self._validate_plan_steps(
|
||||
task_id=task_id,
|
||||
checkpoint=plan_checkpoint,
|
||||
checkpoint_fingerprint=plan_fingerprint,
|
||||
steps=steps,
|
||||
)
|
||||
step_ids = tuple(step.operation_id for step in steps)
|
||||
if step_ids != checkpoint.operation_ids:
|
||||
raise TransferExecutionConflictError(
|
||||
"执行检查点引用的步骤集合与持久步骤不一致"
|
||||
"执行检查点引用的步骤顺序与持久步骤不一致"
|
||||
)
|
||||
if any(step.state != TransferStepState.SUCCEEDED.value for step in steps):
|
||||
raise TransferExecutionConflictError(
|
||||
"存在未成功步骤,不能提交执行检查点"
|
||||
)
|
||||
pending_oper = TransferPendingOper(session)
|
||||
pending = pending_oper.get_by_task_id(task_id=task_id)
|
||||
running = pending_oper.stage_execution_running(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=pending.state,
|
||||
checkpoint_version=plan_checkpoint.schema_version,
|
||||
checkpoint_payload=plan_checkpoint.to_payload(),
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import hashlib
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional, cast
|
||||
from uuid import uuid4
|
||||
@@ -12,66 +10,21 @@ from sqlalchemy import (
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
column,
|
||||
delete,
|
||||
exists,
|
||||
func,
|
||||
or_,
|
||||
select,
|
||||
table,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
from app.db.base import Base, execute_dml, get_id_column
|
||||
|
||||
|
||||
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"source_fileitem": {"storage": storage, "path": src_path},
|
||||
"meta": None,
|
||||
"mediainfo": None,
|
||||
"target_directory": None,
|
||||
"target_storage": None,
|
||||
"target_path": None,
|
||||
"requested_transfer_type": None,
|
||||
"media_source": None,
|
||||
"media_id": None,
|
||||
"media_type": None,
|
||||
"need_scrape": False,
|
||||
"need_rename": True,
|
||||
"need_notify": True,
|
||||
"overwrite_mode": None,
|
||||
"episodes_info": [],
|
||||
"preview": False,
|
||||
"options": {"legacy_replan": True},
|
||||
}
|
||||
|
||||
|
||||
def _planning_fingerprint(payload: dict[str, Any]) -> str:
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=True,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
allow_nan=False,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _default_planning_payload(context: Any) -> dict[str, Any]:
|
||||
params = context.get_current_parameters()
|
||||
return _legacy_planning_payload(
|
||||
params.get("storage", ""),
|
||||
params.get("src_path", ""),
|
||||
)
|
||||
|
||||
|
||||
def _default_planning_fingerprint(context: Any) -> str:
|
||||
params = context.get_current_parameters()
|
||||
payload = params.get("planning_input") or _legacy_planning_payload(
|
||||
params.get("storage", ""),
|
||||
params.get("src_path", ""),
|
||||
)
|
||||
return _planning_fingerprint(payload)
|
||||
_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
|
||||
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
|
||||
|
||||
|
||||
class TransferPending(Base):
|
||||
@@ -112,11 +65,11 @@ class TransferPending(Base):
|
||||
input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
|
||||
# 版本化规划输入 JSON
|
||||
planning_input: Mapped[dict[str, Any]] = mapped_column(
|
||||
JSON, nullable=False, default=_default_planning_payload
|
||||
JSON, nullable=False
|
||||
)
|
||||
# 规划输入规范 JSON 的 SHA-256 指纹
|
||||
input_fingerprint: Mapped[str] = mapped_column(
|
||||
String(64), nullable=False, default=_default_planning_fingerprint
|
||||
String(64), nullable=False
|
||||
)
|
||||
# 完整计划格式版本,尚未规划时为空
|
||||
checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
@@ -203,9 +156,9 @@ class TransferPending(Base):
|
||||
@classmethod
|
||||
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
|
||||
src_path: str, state: str,
|
||||
now_time: str, input_version: int = 1,
|
||||
planning_input: Optional[dict[str, Any]] = None,
|
||||
input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]:
|
||||
now_time: str, input_version: int,
|
||||
planning_input: dict[str, Any],
|
||||
input_fingerprint: str) -> Optional["TransferPending"]:
|
||||
"""
|
||||
在调用方会话中暂存一条持久接纳记录。
|
||||
|
||||
@@ -221,15 +174,20 @@ class TransferPending(Base):
|
||||
:param input_fingerprint: 规划输入规范 JSON 指纹
|
||||
:return: 接纳记录
|
||||
"""
|
||||
if not task_id or not storage or not src_path or not state:
|
||||
if (
|
||||
not task_id
|
||||
or not storage
|
||||
or not src_path
|
||||
or not state
|
||||
or not planning_input
|
||||
or not input_fingerprint
|
||||
):
|
||||
return None
|
||||
pending = db.execute(
|
||||
select(cls).where(cls.storage == storage, cls.src_path == src_path)
|
||||
).scalars().first()
|
||||
if pending:
|
||||
return cast("TransferPending", pending)
|
||||
effective_input = planning_input or _legacy_planning_payload(storage, src_path)
|
||||
effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input)
|
||||
pending = cls(
|
||||
task_id=task_id,
|
||||
storage=storage,
|
||||
@@ -238,8 +196,8 @@ class TransferPending(Base):
|
||||
created_at=now_time,
|
||||
updated_at=now_time,
|
||||
input_version=input_version,
|
||||
planning_input=effective_input,
|
||||
input_fingerprint=effective_fingerprint,
|
||||
planning_input=planning_input,
|
||||
input_fingerprint=input_fingerprint,
|
||||
)
|
||||
db.add(pending)
|
||||
return pending
|
||||
@@ -425,17 +383,33 @@ class TransferPending(Base):
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
admission_state: str,
|
||||
checkpoint_version: int,
|
||||
checkpoint_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约把可执行任务推进或保持为 running。"""
|
||||
if not all((task_id, lease_token, now_utc, updated_at)):
|
||||
"""仅以有效租约和完整冻结计划把任务推进或保持为 running。"""
|
||||
if not all((
|
||||
task_id,
|
||||
lease_token,
|
||||
admission_state,
|
||||
checkpoint_version,
|
||||
checkpoint_payload,
|
||||
now_utc,
|
||||
updated_at,
|
||||
)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == admission_state,
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version == checkpoint_version,
|
||||
cls.checkpoint_payload == checkpoint_payload,
|
||||
cls.planned_at.is_not(None),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
@@ -470,9 +444,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -512,6 +487,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -558,9 +537,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -598,9 +578,10 @@ class TransferPending(Base):
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.state.in_(("planned", "provider_pending")),
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.planned_at.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
@@ -886,7 +867,7 @@ class TransferPending(Base):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def discard_claimed(
|
||||
def abandon_unstarted(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
@@ -895,7 +876,10 @@ class TransferPending(Base):
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
仅以当前且未过期的 token 删除已经到达终态的租约任务。
|
||||
仅以当前且未过期的 token 删除确认从未执行的缺失源任务。
|
||||
|
||||
任何执行状态、聚合检查点或步骤证据都意味着外部结果需要由状态机
|
||||
判定;即使源文件已经消失,也不能以此推断 move 已经安全完成。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
@@ -912,6 +896,29 @@ class TransferPending(Base):
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
cls.state == "accepted",
|
||||
cls.checkpoint_version.is_(None),
|
||||
cls.checkpoint_payload.is_(None),
|
||||
cls.planned_at.is_(None),
|
||||
cls.execution_state == "not_started",
|
||||
cls.execution_version.is_(None),
|
||||
cls.execution_payload.is_(None),
|
||||
cls.execution_fingerprint.is_(None),
|
||||
cls.retry_generation == 0,
|
||||
cls.retry_count == 0,
|
||||
cls.retry_due_at.is_(None),
|
||||
cls.settlement_revision == 0,
|
||||
cls.terminal_history_id.is_(None),
|
||||
~exists(
|
||||
select(_TRANSFER_EXECUTION_STEP.c.task_id).where(
|
||||
_TRANSFER_EXECUTION_STEP.c.task_id == cls.task_id
|
||||
)
|
||||
),
|
||||
~exists(
|
||||
select(_TRANSFER_HISTORY.c.transfer_task_id).where(
|
||||
_TRANSFER_HISTORY.c.transfer_task_id == cls.task_id
|
||||
)
|
||||
),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@@ -13,9 +13,9 @@ class TransferPendingOper(DbOper):
|
||||
"""
|
||||
|
||||
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
|
||||
state: str, now_time: str, input_version: int = 1,
|
||||
planning_input: Optional[dict[str, Any]] = None,
|
||||
input_fingerprint: Optional[str] = None) -> Optional[TransferPending]:
|
||||
state: str, now_time: str, input_version: int,
|
||||
planning_input: dict[str, Any],
|
||||
input_fingerprint: str) -> Optional[TransferPending]:
|
||||
"""
|
||||
在当前会话中暂存一条持久接纳记录。
|
||||
|
||||
@@ -287,7 +287,7 @@ class TransferPendingOper(DbOper):
|
||||
)
|
||||
)
|
||||
|
||||
def stage_discard_claimed(
|
||||
def stage_abandon_unstarted(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
@@ -295,7 +295,7 @@ class TransferPendingOper(DbOper):
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话按当前未过期 token 删除终态任务。
|
||||
使用当前会话按当前未过期 token 删除确认从未执行的缺失源任务。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
@@ -303,7 +303,7 @@ class TransferPendingOper(DbOper):
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.discard_claimed(
|
||||
lambda session: TransferPending.abandon_unstarted(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
@@ -334,6 +334,9 @@ class TransferPendingOper(DbOper):
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
admission_state: str,
|
||||
checkpoint_version: int,
|
||||
checkpoint_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
@@ -343,6 +346,9 @@ class TransferPendingOper(DbOper):
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
admission_state=admission_state,
|
||||
checkpoint_version=checkpoint_version,
|
||||
checkpoint_payload=checkpoint_payload,
|
||||
now_utc=now_utc,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user