Merge upstream/v3 into codex/feat/plugin-data-query-sdk-v3

This commit is contained in:
InfinityPacer
2026-08-28 00:43:57 +08:00
49 changed files with 5423 additions and 1143 deletions
+74 -7
View File
@@ -43,6 +43,14 @@ class TransferTerminalState(StrEnum):
FAILED = "failed"
class TransferExecutionOutcome(StrEnum):
"""描述执行检查点可裁决的业务结果。"""
SUCCEEDED = "succeeded"
FAILED = "failed"
OVERWRITE_SKIPPED = "overwrite_skipped"
class TransferOperationObservationState(StrEnum):
"""描述重启后对遗留 STARTED 外部操作的严格探测结论。"""
@@ -249,6 +257,29 @@ class TransferExecutionCheckpoint:
raise ValueError("整理执行检查点包含无效操作标识")
if not self.operation_ids and not self.skip_reason:
raise ValueError("零副作用整理执行检查点必须记录 skip_reason")
raw_outcome = self.payload.get("outcome")
try:
outcome = TransferExecutionOutcome(
raw_outcome if isinstance(raw_outcome, str) else ""
)
except (TypeError, ValueError) as error:
raise ValueError("整理执行检查点 outcome 无效") from error
transferinfo = self.payload.get("transferinfo")
if transferinfo is not None:
if not isinstance(transferinfo, dict):
raise ValueError("整理执行检查点 TransferInfo 必须是 JSON 对象")
expected_success = outcome is TransferExecutionOutcome.SUCCEEDED
expected_overwrite_skip = (
outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED
)
if (
bool(transferinfo.get("success")) != expected_success
or bool(transferinfo.get("overwrite_skipped"))
!= expected_overwrite_skip
):
raise ValueError("整理执行 checkpoint outcome 与 TransferInfo 不一致")
elif outcome is TransferExecutionOutcome.OVERWRITE_SKIPPED:
raise ValueError("覆盖跳过执行检查点缺少冻结 TransferInfo")
_canonical_json(self.payload)
if build_transfer_checkpoint_fingerprint(self.to_payload()) != self.fingerprint:
raise ValueError("整理执行检查点内容与指纹不一致")
@@ -285,6 +316,35 @@ class TransferExecutionCheckpoint:
"skip_reason": self.skip_reason,
}
def validate_settlement_outcome(
self,
settlement_outcome: str,
) -> TransferTerminalState:
"""解析执行事实,并校验同事务历史裁决给出的结算方向。"""
raw_outcome = self.payload.get("outcome")
try:
outcome = TransferExecutionOutcome(
raw_outcome if isinstance(raw_outcome, str) else ""
)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理执行检查点 outcome 无效"
) from error
try:
terminal_state = TransferTerminalState(settlement_outcome)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理结算 outcome 无效"
) from error
if (
outcome != TransferExecutionOutcome.OVERWRITE_SKIPPED
and outcome.value != terminal_state.value
):
raise TransferExecutionConflictError(
"整理结算 outcome 与执行检查点不一致"
)
return terminal_state
@classmethod
def from_payload(
cls,
@@ -293,6 +353,8 @@ class TransferExecutionCheckpoint:
fingerprint: str,
) -> "TransferExecutionCheckpoint":
"""解析并校验数据库中的执行检查点版本与稳定指纹。"""
if not isinstance(payload, Mapping):
raise TransferExecutionConflictError("整理执行检查点 JSON 结构无效")
serialized = dict(payload)
if build_transfer_checkpoint_fingerprint(serialized) != fingerprint:
raise TransferExecutionConflictError("整理执行检查点 JSON 与指纹不一致")
@@ -308,13 +370,18 @@ class TransferExecutionCheckpoint:
skip_reason = serialized.get("skip_reason")
if skip_reason is not None and not isinstance(skip_reason, str):
raise TransferExecutionConflictError("整理执行跳过原因类型无效")
return cls(
fingerprint=fingerprint,
payload=result_payload,
operation_ids=tuple(operation_ids),
skip_reason=skip_reason,
version=version,
)
try:
return cls(
fingerprint=fingerprint,
payload=result_payload,
operation_ids=tuple(operation_ids),
skip_reason=skip_reason,
version=version,
)
except ValueError as error:
raise TransferExecutionConflictError(
"整理执行检查点内容无效"
) from error
@dataclass(frozen=True, slots=True)
+22 -173
View File
@@ -38,7 +38,6 @@ from typing import (
from pydantic import BaseModel, ConfigDict, PrivateAttr
from app.adapters.system.host import SystemUtils
from app.application.agent import get_prompt_manager, get_running_agent_manager
from app.application.transfer.execution import TransferExecutionCheckpoint
from app.domain.context import MediaInfo, MusicInfo
from app.domain.media import normalize_music_type
@@ -61,7 +60,6 @@ from app.schemas.types import (
MUSIC_ENTITY_RECORDING,
MediaSource,
MediaType,
ReplyMode,
)
if TYPE_CHECKING:
@@ -209,14 +207,6 @@ class TransferPlanningInput:
raise ValueError("整理规划输入缺少源文件存储或路径")
_canonical_json(self.to_payload())
@classmethod
def legacy(cls, *, storage: str, src_path: str) -> "TransferPlanningInput":
"""为升级前只有存储与路径的登记构造保守重规划输入。"""
return cls(
source_fileitem={"storage": storage, "path": src_path},
options={"legacy_replan": True},
)
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanningInput":
"""从受版本约束的 JSON 对象恢复规划输入。"""
@@ -495,6 +485,7 @@ class TransferPlanCheckpoint:
overwrite_mode: Optional[str] = None
preview: bool = False
skip_reason: Optional[str] = None
rejection_error: Optional[str] = None
schema_version: int = TRANSFER_PLAN_CHECKPOINT_VERSION
def __post_init__(self) -> None:
@@ -551,6 +542,7 @@ class TransferPlanCheckpoint:
or self.resolved_transfer_type
or self.items
or self.skip_reason
or self.rejection_error
):
raise ValueError("provider_pending 检查点不得包含宿主执行计划")
else:
@@ -558,6 +550,12 @@ class TransferPlanCheckpoint:
raise ValueError("整理计划检查点缺少目标身份")
if not self.resolved_transfer_type:
raise ValueError("整理计划检查点缺少已解析的整理方式")
if self.rejection_error is not None and (
not isinstance(self.rejection_error, str)
or not self.rejection_error.strip()
or self.items
):
raise ValueError("整理拒绝检查点必须包含非空错误且不得包含文件步骤")
if tuple(item.sequence for item in self.items) != tuple(range(len(self.items))):
raise ValueError("整理计划项必须按从零开始的连续序号保存")
if (
@@ -565,6 +563,7 @@ class TransferPlanCheckpoint:
and not self.items
and not self.preview
and not self.skip_reason
and not self.rejection_error
):
raise ValueError("非预览空计划必须记录合法跳过原因")
_canonical_json(self.to_payload())
@@ -574,6 +573,13 @@ class TransferPlanCheckpoint:
"""返回该检查点是否只冻结 provider 调用、尚未生成宿主计划。"""
return self.provider_invocation is not None
@property
def fingerprint(self) -> str:
"""返回完整冻结计划规范 JSON 的稳定 SHA-256 指纹。"""
return hashlib.sha256(
_canonical_json(self.to_payload()).encode("utf-8")
).hexdigest()
@classmethod
def from_payload(cls, payload: dict[str, Any]) -> "TransferPlanCheckpoint":
"""从受版本约束的 JSON 对象恢复完整执行检查点。"""
@@ -634,6 +640,7 @@ class TransferPlanCheckpoint:
overwrite_mode=payload.get("overwrite_mode"),
preview=payload.get("preview", False),
skip_reason=payload.get("skip_reason"),
rejection_error=payload.get("rejection_error"),
schema_version=payload.get("schema_version", 0),
)
@@ -672,6 +679,7 @@ class TransferPlanCheckpoint:
"overwrite_mode": self.overwrite_mode,
"preview": self.preview,
"skip_reason": self.skip_reason,
"rejection_error": self.rejection_error,
}
@@ -847,9 +855,9 @@ class TransferAdmission:
state: str
created_at: str
updated_at: str
planning_input: TransferPlanningInput
last_error: Optional[str] = None
input_fingerprint: Optional[str] = None
planning_input: Optional[TransferPlanningInput] = None
checkpoint: Optional[TransferPlanCheckpoint] = None
lease_owner: Optional[str] = None
lease_token: Optional[str] = None
@@ -866,7 +874,7 @@ class TransferAdmissionRepository(Protocol):
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
planning_input: TransferPlanningInput,
) -> TransferAdmission:
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
...
@@ -932,8 +940,8 @@ class TransferAdmissionRepository(Protocol):
"""按 token 释放当前 claim,并可记录可恢复错误。"""
...
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅允许当前 lease owner 删除已经到达终态的登记。"""
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
"""仅允许当前 lease owner 删除确认未开始且源已消失的登记。"""
...
@@ -1929,162 +1937,3 @@ class JobManager:
with job_lock:
__mediaid__ = self.__get_media_id(media=media, season=season)
return self._season_episodes.get(__mediaid__) or []
class FailedRetryScheduler:
"""
负责失败整理记录的进程内 debounce 聚合与 AI 重试调度。
缓冲不提供持久化保证;关闭时会取消尚未触发的记录,由上层 durable
工作流在后续阶段承接需要跨进程保证的重试意图。
"""
RETRY_TRANSFER_DEBOUNCE_SECONDS = 300
def __init__(self) -> None:
"""初始化重试缓冲、定时器、活跃任务集合与关闭状态。"""
super().__init__()
self._retry_transfer_buffer: dict[str, list[int]] = {}
self._retry_transfer_timers: dict[str, asyncio.TimerHandle] = {}
self._retry_transfer_generations: dict[str, int] = {}
self._retry_transfer_tasks: set[asyncio.Task[None]] = set()
self._retry_transfer_lock = asyncio.Lock()
self._closed = False
async def close(self) -> None:
"""停止接收重试,取消定时器,并等待活跃 flush 任务完成取消。"""
async with self._retry_transfer_lock:
self._closed = True
timers = list(self._retry_transfer_timers.values())
buffered_count = sum(
len(history_ids)
for history_ids in self._retry_transfer_buffer.values()
)
self._retry_transfer_timers.clear()
self._retry_transfer_generations.clear()
self._retry_transfer_buffer.clear()
tasks = tuple(self._retry_transfer_tasks)
for timer in timers:
timer.cancel()
if buffered_count:
logger.warning(
f"智能体重试整理调度器关闭,取消 {buffered_count} 条未持久化缓冲记录"
)
for task in tasks:
task.cancel()
if tasks:
await asyncio.gather(*tasks, return_exceptions=True)
def _start_retry_transfer_task(self, group_key: str, generation: int) -> None:
"""把定时器到期后的 flush 建为具名且受本调度器管理的任务。"""
if self._closed:
return
task = asyncio.create_task(
self._flush_retry_transfer(group_key, generation),
name="transfer.failed_retry.flush",
)
self._retry_transfer_tasks.add(task)
task.add_done_callback(self._observe_retry_transfer_task)
def _observe_retry_transfer_task(self, task: asyncio.Task[None]) -> None:
"""移除已结束任务,并观察未被 flush 逻辑处理的异常。"""
self._retry_transfer_tasks.discard(task)
if task.cancelled():
return
exception = task.exception()
if exception is not None:
logger.error(f"智能体重试整理后台任务异常: {exception}")
@staticmethod
def _build_retry_transfer_template_context(
history_ids: list[int],
) -> tuple[str, dict[str, int | str]]:
"""仅负责把失败重试任务的动态数据映射成模板变量。"""
is_batch = len(history_ids) > 1
task_type = "batch_transfer_failed_retry" if is_batch else "transfer_failed_retry"
template_context: dict[str, int | str] = {
"history_ids_csv": ", ".join(str(item) for item in history_ids),
"history_count": len(history_ids),
}
if not is_batch:
template_context["history_id"] = history_ids[0]
return task_type, template_context
def _build_retry_transfer_prompt(self, history_ids: list[int]) -> str:
"""根据失败记录数量构建统一的重试整理后台任务提示词。"""
task_type, template_context = self._build_retry_transfer_template_context(history_ids)
return cast(str, get_prompt_manager().render_system_task_message(
task_type,
template_context=template_context,
))
async def schedule_retry(self, history_id: int, group_key: str = "") -> None:
"""
同一 group_key 的失败记录会在缓冲期内合并为一次 agent 调用。
"""
if not group_key:
group_key = f"_default_{history_id}"
async with self._retry_transfer_lock:
if self._closed:
raise RuntimeError("智能体重试整理调度器正在关闭,不能再接收任务")
if group_key not in self._retry_transfer_buffer:
self._retry_transfer_buffer[group_key] = []
if history_id not in self._retry_transfer_buffer[group_key]:
self._retry_transfer_buffer[group_key].append(history_id)
logger.info(
f"智能体重试整理:记录 ID={history_id} 已加入缓冲区 "
f"(group={group_key}, 当前{len(self._retry_transfer_buffer[group_key])}条)"
)
if group_key in self._retry_transfer_timers:
self._retry_transfer_timers[group_key].cancel()
loop = asyncio.get_running_loop()
generation = self._retry_transfer_generations.get(group_key, 0) + 1
self._retry_transfer_generations[group_key] = generation
self._retry_transfer_timers[group_key] = loop.call_later(
self.RETRY_TRANSFER_DEBOUNCE_SECONDS,
self._start_retry_transfer_task,
group_key,
generation,
)
async def _flush_retry_transfer(self, group_key: str, generation: int) -> None:
"""
延迟定时器到期后,取出该分组的所有 history_id 并合并为一次 agent 调用。
"""
async with self._retry_transfer_lock:
# callback 到期与真正取得锁之间可能有新记录续期;旧代不能提前取走新批次。
if self._retry_transfer_generations.get(group_key) != generation:
return
history_ids = self._retry_transfer_buffer.pop(group_key, [])
self._retry_transfer_timers.pop(group_key, None)
self._retry_transfer_generations.pop(group_key, None)
if not history_ids:
return
ids_str = ", ".join(str(item) for item in history_ids)
logger.info(
f"智能体重试整理:开始批量处理失败记录 IDs=[{ids_str}] (group={group_key})"
)
try:
manager = get_running_agent_manager()
if manager is None:
logger.warning("智能助手服务未运行,跳过整理失败自动重试")
return
await manager.run_background_prompt(
message=self._build_retry_transfer_prompt(history_ids),
session_prefix="__agent_retry_transfer_batch",
reply_mode=ReplyMode.DISPATCH,
)
logger.info(
f"智能体重试整理:批量处理完成 IDs=[{ids_str}] (group={group_key})"
)
except Exception as err:
logger.error(
f"智能体重试整理失败 (IDs=[{ids_str}], group={group_key}): {err}"
)
+363 -373
View File
File diff suppressed because it is too large Load Diff
+8
View File
@@ -26,6 +26,7 @@ from app.application.outbox import (
OutboxIntent,
)
from app.application.transfer.execution import (
TransferExecutionCheckpoint,
TransferExecutionConflictError,
TransferExecutionLeaseLostError,
TransferExecutionState,
@@ -403,6 +404,13 @@ class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
!= settlement.execution_fingerprint
):
raise TransferExecutionConflictError("整理终态与执行检查点不匹配")
checkpoint = TransferExecutionCheckpoint.from_payload(
pending.execution_payload,
fingerprint=settlement.execution_fingerprint,
)
if pending.execution_version != checkpoint.version:
raise TransferExecutionConflictError("整理执行检查点列版本与内容不一致")
checkpoint.validate_settlement_outcome(settlement.outcome)
return int(pending.settlement_revision)
@staticmethod
+11 -15
View File
@@ -142,18 +142,14 @@ class TransactionalTransferAdmissionRepository:
*,
storage: str,
src_path: str,
planning_input: Optional[TransferPlanningInput] = None,
planning_input: TransferPlanningInput,
) -> TransferAdmission:
"""按输入指纹幂等持久化准入事实,并返回跨重启稳定身份。"""
if not storage or not src_path:
raise ValueError("整理任务的存储与源路径不能为空")
effective_input = planning_input or TransferPlanningInput.legacy(
storage=storage,
src_path=src_path,
)
if (
effective_input.source_fileitem.get("storage") != storage
or effective_input.source_fileitem.get("path") != src_path
planning_input.source_fileitem.get("storage") != storage
or planning_input.source_fileitem.get("path") != src_path
):
raise ValueError("整理规划输入的源文件身份与准入参数不一致")
now_time = self._now()
@@ -167,16 +163,16 @@ class TransactionalTransferAdmissionRepository:
src_path=src_path,
state=TRANSFER_ADMISSION_ACCEPTED,
now_time=now_time,
input_version=effective_input.schema_version,
planning_input=effective_input.to_payload(),
input_fingerprint=effective_input.fingerprint,
input_version=planning_input.schema_version,
planning_input=planning_input.to_payload(),
input_fingerprint=planning_input.fingerprint,
)
if pending is None:
raise TransferAdmissionConflictError(
f"整理源文件已有持久终态回执: {storage}:{src_path}"
)
session.flush()
self._assert_input_match(pending, effective_input)
self._assert_input_match(pending, planning_input)
admission = self._project(pending)
transaction.commit()
return admission
@@ -192,7 +188,7 @@ class TransactionalTransferAdmissionRepository:
)
if pending is None:
raise RuntimeError("并发准入冲突后未找到已提交记录") from error
self._assert_input_match(pending, effective_input)
self._assert_input_match(pending, planning_input)
return self._project(pending)
def claim_task(
@@ -525,14 +521,14 @@ class TransactionalTransferAdmissionRepository:
transaction.rollback()
raise
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前未过期 token 删除终态任务,拒绝陈旧 worker 变更"""
def abandon_unstarted(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前 token 删除无执行证据的缺失源任务,拒绝陈旧 worker。"""
if not task_id or not lease_token:
return 0
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
deleted = TransferPendingOper(db=session).stage_discard_claimed(
deleted = TransferPendingOper(db=session).stage_abandon_unstarted(
task_id=task_id,
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
+412 -12
View File
@@ -4,7 +4,8 @@ from __future__ import annotations
from collections.abc import Callable
from datetime import datetime, timezone
from typing import Optional
from pathlib import Path
from typing import Any, Optional
from sqlalchemy import and_, func, or_, select
from sqlalchemy.exc import IntegrityError
@@ -27,6 +28,14 @@ from app.application.transfer.execution import (
TransferStepIntent,
TransferStepResult,
TransferStepState,
build_transfer_operation_id,
)
from app.application.transfer.workflow import (
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferPlanCheckpoint,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.models.transferexecutionstep import (
TransferExecutionStep as TransferExecutionStepModel,
@@ -229,14 +238,332 @@ class TransactionalTransferExecutionRepository:
))
@staticmethod
def _raise_fenced_failure(
def _plan_identity(
pending: TransferPending,
) -> tuple[TransferPlanCheckpoint, str]:
"""恢复完整冻结计划,并返回其规范 payload 的稳定指纹。"""
if pending.state not in {
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
}:
raise TransferExecutionConflictError("整理任务尚未进入可执行规划状态")
if (
pending.checkpoint_version is None
or pending.checkpoint_payload is None
or pending.planned_at is None
):
raise TransferExecutionConflictError("整理任务缺少完整计划检查点")
try:
checkpoint = TransferPlanCheckpoint.from_payload(
pending.checkpoint_payload
)
except (TypeError, ValueError) as error:
raise TransferExecutionConflictError(
"整理任务计划检查点无法恢复"
) from error
if pending.checkpoint_version != checkpoint.schema_version:
raise TransferExecutionConflictError("整理任务计划检查点版本不一致")
if checkpoint.planning_input.fingerprint != pending.input_fingerprint:
raise TransferExecutionConflictError("整理任务计划与准入输入指纹不一致")
expected_state = (
TRANSFER_ADMISSION_PROVIDER_PENDING
if checkpoint.is_provider_pending
else TRANSFER_ADMISSION_PLANNED
)
if pending.state != expected_state:
raise TransferExecutionConflictError("整理任务计划类型与准入状态不一致")
return checkpoint, checkpoint.fingerprint
@staticmethod
def _provider_predecessor_checkpoint(
checkpoint: TransferPlanCheckpoint,
step: TransferExecutionStepModel,
) -> Optional[TransferPlanCheckpoint]:
"""重建 provider 回退前的冻结计划,证明序号零步骤的历史归属。"""
if (
not checkpoint.pre_execution_cleanup_completed
or step.ordinal != 0
or step.phase != "provider"
or step.kind != "legacy_transfer_provider_sequence"
):
return None
providers_payload = step.intent_payload.get("providers")
invocation_payload = step.intent_payload.get("invocation")
if (
not isinstance(providers_payload, list)
or not providers_payload
or not all(isinstance(item, dict) for item in providers_payload)
or not isinstance(invocation_payload, dict)
):
return None
try:
providers = tuple(
TransferProviderReference.from_payload(item)
for item in providers_payload
)
invocation = TransferProviderInvocationSnapshot.from_payload(
invocation_payload
)
if invocation.fileitem != checkpoint.planning_input.source_fileitem:
return None
predecessor = TransferPlanCheckpoint(
planning_input=checkpoint.planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
resolved_episodes_info=invocation.episodes_info,
legacy_transfer_providers=providers,
provider_invocation=invocation,
preview=invocation.preview,
)
except (TypeError, ValueError):
return None
return predecessor
@staticmethod
def _intent_belongs_to_checkpoint(
checkpoint: TransferPlanCheckpoint,
*,
phase: str,
kind: str,
payload: dict[str, Any],
previous_steps: list[TransferExecutionStepModel],
) -> bool:
"""校验步骤类型及稳定参数由冻结计划或其已提交发现证据导出。"""
if checkpoint.rejection_error:
return (
phase == "planning"
and kind == "reject"
and payload == {"error": checkpoint.rejection_error}
)
if checkpoint.is_provider_pending:
invocation = checkpoint.provider_invocation
if invocation is None:
return False
return (
phase == "provider"
and kind == "legacy_transfer_provider_sequence"
and payload == {
"providers": [
provider.to_payload()
for provider in checkpoint.legacy_transfer_providers
],
"invocation": invocation.to_payload(),
}
)
plan_items = tuple(checkpoint.items)
if phase == "transfer" and kind == checkpoint.resolved_transfer_type:
return any(payload == {
"source": item.source_fileitem.get("path"),
"target": item.target_path,
} for item in plan_items)
if phase == "transfer" and kind == "materialize_target":
return any(payload == {
"source": item.source_fileitem,
"target_storage": item.target_storage,
"target_path": item.target_path,
"transfer_type": (
"copy"
if (
checkpoint.resolved_transfer_type == "move"
and item.source_fileitem.get("storage")
!= item.target_storage
)
else checkpoint.resolved_transfer_type
),
} for item in plan_items)
if phase == "transfer" and kind == "delete_move_source":
return (
checkpoint.resolved_transfer_type == "move"
and any(
item.source_fileitem.get("storage") != item.target_storage
and payload == {
"source": item.source_fileitem,
"target_storage": item.target_storage,
"target_path": item.target_path,
}
for item in plan_items
)
)
source = checkpoint.planning_input.source_fileitem
if phase == "prepare" and kind == "cleanup_previous_destination":
return payload == {"source_path": source.get("path")}
if phase == "prepare" and kind == "ensure_target_directory":
target_path = Path(checkpoint.final_target_path)
directory_path = (
target_path
if source.get("type") == "dir"
else target_path.parent
)
return payload == {
"storage": checkpoint.target_storage,
"path": directory_path.as_posix(),
}
if phase == "prepare" and kind == "delete_overwrite_target":
return payload == {
"storage": checkpoint.target_storage,
"path": checkpoint.final_target_path,
}
if phase == "decision" and kind == "resolve_overwrite":
return payload == {
"source": source,
"target_storage": checkpoint.target_storage,
"target_path": checkpoint.final_target_path,
"transfer_type": checkpoint.resolved_transfer_type,
"overwrite_mode": checkpoint.overwrite_mode,
"need_notify": checkpoint.need_notify,
}
if phase == "decision" and kind == "plugin_transfer_intercept":
stable_payload = {
"source": source,
"target_storage": checkpoint.target_storage,
"target_path": checkpoint.final_target_path,
"transfer_type": checkpoint.resolved_transfer_type,
}
return payload == stable_payload or (
set(payload) == {*stable_payload, "over_flag"}
and all(payload[key] == value for key, value in stable_payload.items())
and isinstance(payload.get("over_flag"), bool)
)
if phase == "prepare" and kind == "discover_version_targets":
return payload == {
"storage": checkpoint.target_storage,
"path": checkpoint.final_target_path,
}
if phase == "prepare" and kind == "delete_version_target":
candidate = payload.get("item")
if (
set(payload) != {"storage", "item"}
or payload.get("storage") != checkpoint.target_storage
or not isinstance(candidate, dict)
):
return False
return any(
step.phase == "prepare"
and step.kind == "discover_version_targets"
and step.state == TransferStepState.SUCCEEDED.value
and isinstance(step.result_payload, dict)
and candidate in step.result_payload.get("items", [])
for step in previous_steps
)
return False
@classmethod
def _validate_plan_steps(
cls,
*,
task_id: str,
checkpoint: TransferPlanCheckpoint,
checkpoint_fingerprint: str,
steps: list[TransferExecutionStepModel],
) -> None:
"""验证全部持久步骤属于冻结计划演进且身份与全局顺序未被篡改。"""
if tuple(step.ordinal for step in steps) != tuple(range(len(steps))):
raise TransferExecutionConflictError("整理步骤全局序号不连续")
for index, step in enumerate(steps):
if step.task_id != task_id:
raise TransferExecutionConflictError("整理步骤绑定了错误任务")
step_checkpoint = checkpoint
if step.checkpoint_fingerprint != checkpoint_fingerprint:
predecessor = cls._provider_predecessor_checkpoint(
checkpoint,
step,
)
if (
predecessor is None
or step.checkpoint_fingerprint != predecessor.fingerprint
):
raise TransferExecutionConflictError(
"整理步骤不属于当前冻结计划或合法 provider 前驱计划"
)
step_checkpoint = predecessor
if not cls._intent_belongs_to_checkpoint(
step_checkpoint,
phase=step.phase,
kind=step.kind,
payload=step.intent_payload,
previous_steps=steps[:index],
):
raise TransferExecutionConflictError(
"整理步骤类型或参数不能由冻结计划导出"
)
expected_operation_id = build_transfer_operation_id(
task_id=task_id,
checkpoint_fingerprint=step.checkpoint_fingerprint,
ordinal=step.ordinal,
phase=step.phase,
kind=step.kind,
intent_payload=step.intent_payload,
)
if step.operation_id != expected_operation_id:
raise TransferExecutionConflictError("整理步骤 operation ID 与冻结意图不一致")
@classmethod
def _validate_new_intent(
cls,
*,
task_id: str,
checkpoint: TransferPlanCheckpoint,
checkpoint_fingerprint: str,
steps: list[TransferExecutionStepModel],
intent: TransferStepIntent,
) -> None:
"""验证待准备意图属于当前冻结计划,并且只追加或幂等重放既有序号。"""
cls._validate_plan_steps(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
)
if intent.checkpoint_fingerprint != checkpoint_fingerprint:
raise TransferExecutionConflictError("整理步骤意图未绑定当前冻结计划指纹")
if not cls._intent_belongs_to_checkpoint(
checkpoint,
phase=intent.phase,
kind=intent.kind,
payload=intent.payload,
previous_steps=steps,
):
raise TransferExecutionConflictError(
"整理步骤意图类型或参数不能由冻结计划导出"
)
expected_operation_id = build_transfer_operation_id(
task_id=task_id,
checkpoint_fingerprint=intent.checkpoint_fingerprint,
ordinal=intent.ordinal,
phase=intent.phase,
kind=intent.kind,
intent_payload=intent.payload,
)
if intent.operation_id != expected_operation_id:
raise TransferExecutionConflictError("整理步骤意图 operation ID 不可信")
existing = next(
(step for step in steps if step.operation_id == intent.operation_id),
None,
)
if existing is not None:
if not cls._intent_matches(existing, task_id=task_id, intent=intent):
raise TransferExecutionConflictError(
"稳定 operation ID 已绑定不同步骤意图"
)
return
if intent.ordinal != len(steps):
raise TransferExecutionConflictError("整理步骤意图必须按全局序号连续追加")
@staticmethod
def _require_active_lease(
pending: Optional[TransferPending],
*,
lease_token: str,
now_utc: str,
detail: str,
) -> None:
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
) -> TransferPending:
"""返回仍由调用方持有的任务,并优先报告 lease fencing 失败。"""
if (
pending is None
or pending.lease_token != lease_token
@@ -244,6 +571,23 @@ class TransactionalTransferExecutionRepository:
or pending.lease_expires_at <= now_utc
):
raise TransferExecutionLeaseLostError("整理任务租约已失效或被接管")
return pending
@classmethod
def _raise_fenced_failure(
cls,
pending: Optional[TransferPending],
*,
lease_token: str,
now_utc: str,
detail: str,
) -> None:
"""区分租约丢失与同租约内的状态或 attempt 冲突。"""
cls._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
raise TransferExecutionConflictError(detail)
def _times(self) -> tuple[str, str]:
@@ -354,9 +698,29 @@ class TransactionalTransferExecutionRepository:
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
oper.list_by_task_id(task_id=task_id)
)
self._validate_new_intent(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
intent=intent,
)
updated = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
@@ -367,7 +731,6 @@ class TransactionalTransferExecutionRepository:
now_utc=now_utc,
detail="整理任务当前状态不能准备外部步骤",
)
oper = TransferExecutionStepOper(session)
step = oper.get_by_operation_id(operation_id=intent.operation_id)
if step is None:
step = oper.stage_prepare(
@@ -498,9 +861,32 @@ class TransactionalTransferExecutionRepository:
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
checkpoint, checkpoint_fingerprint = self._plan_identity(pending)
oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
oper.list_by_task_id(task_id=task_id)
)
self._validate_plan_steps(
task_id=task_id,
checkpoint=checkpoint,
checkpoint_fingerprint=checkpoint_fingerprint,
steps=steps,
)
if operation_id not in {step.operation_id for step in steps}:
raise TransferExecutionConflictError(
"待恢复步骤不属于冻结计划"
)
pending_updated = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=checkpoint.schema_version,
checkpoint_payload=checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
@@ -511,7 +897,6 @@ class TransactionalTransferExecutionRepository:
now_utc=now_utc,
detail="重试任务未到期或当前状态不能恢复",
)
oper = TransferExecutionStepOper(session)
updated = oper.stage_resume_failed_attempt(
task_id=task_id,
lease_token=lease_token,
@@ -973,24 +1358,39 @@ class TransactionalTransferExecutionRepository:
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
pending = self._require_active_lease(
pending,
lease_token=lease_token,
now_utc=now_utc,
)
plan_checkpoint, plan_fingerprint = self._plan_identity(pending)
step_oper = TransferExecutionStepOper(session)
steps = self._execution_steps(
step_oper.list_by_task_id(task_id=task_id)
)
step_ids = {step.operation_id for step in steps}
if step_ids != set(checkpoint.operation_ids):
self._validate_plan_steps(
task_id=task_id,
checkpoint=plan_checkpoint,
checkpoint_fingerprint=plan_fingerprint,
steps=steps,
)
step_ids = tuple(step.operation_id for step in steps)
if step_ids != checkpoint.operation_ids:
raise TransferExecutionConflictError(
"执行检查点引用的步骤集合与持久步骤不一致"
"执行检查点引用的步骤顺序与持久步骤不一致"
)
if any(step.state != TransferStepState.SUCCEEDED.value for step in steps):
raise TransferExecutionConflictError(
"存在未成功步骤,不能提交执行检查点"
)
pending_oper = TransferPendingOper(session)
pending = pending_oper.get_by_task_id(task_id=task_id)
running = pending_oper.stage_execution_running(
task_id=task_id,
lease_token=lease_token,
admission_state=pending.state,
checkpoint_version=plan_checkpoint.schema_version,
checkpoint_payload=plan_checkpoint.to_payload(),
now_utc=now_utc,
updated_at=updated_at,
)
+76 -69
View File
@@ -1,5 +1,3 @@
import hashlib
import json
from datetime import datetime
from typing import Any, List, Optional, cast
from uuid import uuid4
@@ -12,66 +10,21 @@ from sqlalchemy import (
Text,
UniqueConstraint,
and_,
column,
delete,
exists,
func,
or_,
select,
table,
update,
)
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import Base, execute_dml, get_id_column
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _planning_fingerprint(payload: dict[str, Any]) -> str:
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _default_planning_payload(context: Any) -> dict[str, Any]:
params = context.get_current_parameters()
return _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
def _default_planning_fingerprint(context: Any) -> str:
params = context.get_current_parameters()
payload = params.get("planning_input") or _legacy_planning_payload(
params.get("storage", ""),
params.get("src_path", ""),
)
return _planning_fingerprint(payload)
_TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
class TransferPending(Base):
@@ -112,11 +65,11 @@ class TransferPending(Base):
input_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1)
# 版本化规划输入 JSON
planning_input: Mapped[dict[str, Any]] = mapped_column(
JSON, nullable=False, default=_default_planning_payload
JSON, nullable=False
)
# 规划输入规范 JSON 的 SHA-256 指纹
input_fingerprint: Mapped[str] = mapped_column(
String(64), nullable=False, default=_default_planning_fingerprint
String(64), nullable=False
)
# 完整计划格式版本,尚未规划时为空
checkpoint_version: Mapped[Optional[int]] = mapped_column(Integer)
@@ -203,9 +156,9 @@ class TransferPending(Base):
@classmethod
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
src_path: str, state: str,
now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional["TransferPending"]:
now_time: str, input_version: int,
planning_input: dict[str, Any],
input_fingerprint: str) -> Optional["TransferPending"]:
"""
在调用方会话中暂存一条持久接纳记录
@@ -221,15 +174,20 @@ class TransferPending(Base):
:param input_fingerprint: 规划输入规范 JSON 指纹
:return: 接纳记录
"""
if not task_id or not storage or not src_path or not state:
if (
not task_id
or not storage
or not src_path
or not state
or not planning_input
or not input_fingerprint
):
return None
pending = db.execute(
select(cls).where(cls.storage == storage, cls.src_path == src_path)
).scalars().first()
if pending:
return cast("TransferPending", pending)
effective_input = planning_input or _legacy_planning_payload(storage, src_path)
effective_fingerprint = input_fingerprint or _planning_fingerprint(effective_input)
pending = cls(
task_id=task_id,
storage=storage,
@@ -238,8 +196,8 @@ class TransferPending(Base):
created_at=now_time,
updated_at=now_time,
input_version=input_version,
planning_input=effective_input,
input_fingerprint=effective_fingerprint,
planning_input=planning_input,
input_fingerprint=input_fingerprint,
)
db.add(pending)
return pending
@@ -425,17 +383,33 @@ class TransferPending(Base):
*,
task_id: str,
lease_token: str,
admission_state: str,
checkpoint_version: int,
checkpoint_payload: dict[str, Any],
now_utc: str,
updated_at: str,
) -> int:
"""以有效租约把可执行任务推进或保持为 running。"""
if not all((task_id, lease_token, now_utc, updated_at)):
"""以有效租约和完整冻结计划把任务推进或保持为 running。"""
if not all((
task_id,
lease_token,
admission_state,
checkpoint_version,
checkpoint_payload,
now_utc,
updated_at,
)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state == admission_state,
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version == checkpoint_version,
cls.checkpoint_payload == checkpoint_payload,
cls.planned_at.is_not(None),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_utc,
@@ -470,9 +444,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -512,6 +487,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state.in_(("not_started", "running", "retry_wait")),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -558,9 +537,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -598,9 +578,10 @@ class TransferPending(Base):
update(cls)
.where(
cls.task_id == task_id,
cls.state == "planned",
cls.state.in_(("planned", "provider_pending")),
cls.checkpoint_version.is_not(None),
cls.checkpoint_payload.is_not(None),
cls.planned_at.is_not(None),
cls.execution_state == "running",
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
@@ -886,7 +867,7 @@ class TransferPending(Base):
)
@classmethod
def discard_claimed(
def abandon_unstarted(
cls,
db: Session,
*,
@@ -895,7 +876,10 @@ class TransferPending(Base):
now_time: str,
) -> int:
"""
仅以当前且未过期的 token 删除已经到达终态的租约任务
仅以当前且未过期的 token 删除确认从未执行的缺失源任务
任何执行状态聚合检查点或步骤证据都意味着外部结果需要由状态机
判定即使源文件已经消失也不能以此推断 move 已经安全完成
:param db: 数据库会话
:param task_id: 稳定任务标识
@@ -912,6 +896,29 @@ class TransferPending(Base):
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
cls.state == "accepted",
cls.checkpoint_version.is_(None),
cls.checkpoint_payload.is_(None),
cls.planned_at.is_(None),
cls.execution_state == "not_started",
cls.execution_version.is_(None),
cls.execution_payload.is_(None),
cls.execution_fingerprint.is_(None),
cls.retry_generation == 0,
cls.retry_count == 0,
cls.retry_due_at.is_(None),
cls.settlement_revision == 0,
cls.terminal_history_id.is_(None),
~exists(
select(_TRANSFER_EXECUTION_STEP.c.task_id).where(
_TRANSFER_EXECUTION_STEP.c.task_id == cls.task_id
)
),
~exists(
select(_TRANSFER_HISTORY.c.transfer_task_id).where(
_TRANSFER_HISTORY.c.transfer_task_id == cls.task_id
)
),
),
execution_options={"synchronize_session": False},
)
+12 -6
View File
@@ -13,9 +13,9 @@ class TransferPendingOper(DbOper):
"""
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
state: str, now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
input_fingerprint: Optional[str] = None) -> Optional[TransferPending]:
state: str, now_time: str, input_version: int,
planning_input: dict[str, Any],
input_fingerprint: str) -> Optional[TransferPending]:
"""
在当前会话中暂存一条持久接纳记录
@@ -287,7 +287,7 @@ class TransferPendingOper(DbOper):
)
)
def stage_discard_claimed(
def stage_abandon_unstarted(
self,
*,
task_id: str,
@@ -295,7 +295,7 @@ class TransferPendingOper(DbOper):
now_time: str,
) -> int:
"""
使用当前会话按当前未过期 token 删除终态任务
使用当前会话按当前未过期 token 删除确认从未执行的缺失源任务
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
@@ -303,7 +303,7 @@ class TransferPendingOper(DbOper):
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_claimed(
lambda session: TransferPending.abandon_unstarted(
session,
task_id=task_id,
lease_token=lease_token,
@@ -334,6 +334,9 @@ class TransferPendingOper(DbOper):
*,
task_id: str,
lease_token: str,
admission_state: str,
checkpoint_version: int,
checkpoint_payload: dict[str, Any],
now_utc: str,
updated_at: str,
) -> int:
@@ -343,6 +346,9 @@ class TransferPendingOper(DbOper):
session,
task_id=task_id,
lease_token=lease_token,
admission_state=admission_state,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
now_utc=now_utc,
updated_at=updated_at,
)
-1
View File
@@ -301,7 +301,6 @@ SCHEMA_EXPORTS = {
'PluginTriggeredEventData': ('app.schemas.event', 'PluginTriggeredEventData'),
'PluginUpdateCandidate': ('app.schemas.plugin', 'PluginUpdateCandidate'),
'PluginWorkflowActionGroup': ('app.schemas.workflow', 'PluginWorkflowActionGroup'),
'PrivateAttr': ('app.schemas.plugin', 'PrivateAttr'),
'ProcessInfo': ('app.schemas.dashboard', 'ProcessInfo'),
'ProgressKeyData': ('app.schemas.common', 'ProgressKeyData'),
'RadarrMovie': ('app.schemas.servarr', 'RadarrMovie'),
+3 -2
View File
@@ -1,7 +1,8 @@
from enum import Enum as _Enum
from typing import Dict, List, Literal, Optional, Union
from pydantic import BaseModel, Field, PrivateAttr, RootModel, field_validator
from pydantic import BaseModel, Field, RootModel, field_validator
from pydantic import PrivateAttr as _PrivateAttr
from app.schemas.common import JsonData
@@ -62,7 +63,7 @@ class Plugin(BaseModel):
"""
插件信息
"""
_package_version: Optional[str] = PrivateAttr(default=None)
_package_version: Optional[str] = _PrivateAttr(default=None)
id: str = None
# 插件名称
+42
View File
@@ -1,5 +1,7 @@
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
import hashlib
import json
from datetime import datetime
from typing import Any, List, Optional, Tuple
from uuid import uuid4
@@ -13,6 +15,42 @@ _TRANSFER_EXECUTION_STEP = table("transferexecutionstep", column("task_id"))
_TRANSFER_HISTORY = table("transferhistory", column("transfer_task_id"))
def _legacy_planning_payload(storage: str, src_path: str) -> dict[str, Any]:
"""只在旧登记 ABI 内构造可由新状态机保守重规划的最小输入。"""
return {
"schema_version": 1,
"source_fileitem": {"storage": storage, "path": src_path},
"meta": None,
"mediainfo": None,
"target_directory": None,
"target_storage": None,
"target_path": None,
"requested_transfer_type": None,
"media_source": None,
"media_id": None,
"media_type": None,
"need_scrape": False,
"need_rename": True,
"need_notify": True,
"overwrite_mode": None,
"episodes_info": [],
"preview": False,
"options": {"legacy_replan": True},
}
def _planning_fingerprint(payload: dict[str, Any]) -> str:
"""生成兼容登记输入与 canonical DTO 一致的规范指纹。"""
canonical = json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
allow_nan=False,
)
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _safe_legacy_delete_predicates() -> tuple[Any, ...]:
"""只允许旧接口删除从未 claim、执行或结算的新鲜登记。"""
return (
@@ -61,6 +99,7 @@ class TransferPendingOper(DbOper):
:return: 登记记录
"""
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
planning_input = _legacy_planning_payload(storage, src_path)
return self._execute_sync_write(
lambda session: _TransferPending.stage_admit(
session,
@@ -69,6 +108,9 @@ class TransferPendingOper(DbOper):
src_path=src_path,
state="accepted",
now_time=now_time,
input_version=1,
planning_input=planning_input,
input_fingerprint=_planning_fingerprint(planning_input),
)
)