mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
refactor: complete durable transfer execution settlement
This commit is contained in:
jxxghp
@@ -37,6 +37,14 @@ _MODEL_EXPORTS = {
|
||||
),
|
||||
"SystemConfig": ("app.db.models.systemconfig", "SystemConfig"),
|
||||
"TransferHistory": ("app.db.models.transferhistory", "TransferHistory"),
|
||||
"TransferExecutionStep": (
|
||||
"app.db.models.transferexecutionstep",
|
||||
"TransferExecutionStep",
|
||||
),
|
||||
"TransferSettlementReceipt": (
|
||||
"app.db.models.transfersettlementreceipt",
|
||||
"TransferSettlementReceipt",
|
||||
),
|
||||
"TransferPending": ("app.db.models.transferpending", "TransferPending"),
|
||||
"User": ("app.db.models.user", "User"),
|
||||
"UserConfig": ("app.db.models.userconfig", "UserConfig"),
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
"""整理任务外部操作步骤的持久化模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
delete,
|
||||
exists,
|
||||
or_,
|
||||
select,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
from sqlalchemy.sql.selectable import Exists
|
||||
|
||||
from app.db.base import Base, execute_dml, get_id_column
|
||||
from app.db.models.transferpending import TransferPending
|
||||
|
||||
|
||||
class TransferExecutionStep(Base):
|
||||
"""保存一次稳定外部操作的意图、尝试身份与结果证据。"""
|
||||
|
||||
id = get_id_column()
|
||||
task_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("transferpending.task_id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
)
|
||||
operation_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
checkpoint_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
ordinal: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
phase: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
state: Mapped[str] = mapped_column(String(32), nullable=False, default="prepared")
|
||||
attempt_token: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
intent_version: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
intent_payload: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False)
|
||||
result_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
result_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
|
||||
last_error: Mapped[Optional[str]] = mapped_column(Text)
|
||||
prepared_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
started_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
completed_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("operation_id", name="uq_transferexecutionstep_operation_id"),
|
||||
UniqueConstraint(
|
||||
"task_id",
|
||||
"ordinal",
|
||||
name="uq_transferexecutionstep_task_ordinal",
|
||||
),
|
||||
Index(
|
||||
"ix_transferexecutionstep_task_state_ordinal",
|
||||
"task_id",
|
||||
"state",
|
||||
"ordinal",
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_by_operation_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
operation_id: str,
|
||||
) -> Optional["TransferExecutionStep"]:
|
||||
"""按稳定操作标识读取步骤。"""
|
||||
if not operation_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferExecutionStep"],
|
||||
db.execute(
|
||||
select(cls).where(cls.operation_id == operation_id)
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_by_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> list["TransferExecutionStep"]:
|
||||
"""按全局序号读取任务的全部外部操作步骤。"""
|
||||
if not task_id:
|
||||
return []
|
||||
return list(
|
||||
db.execute(
|
||||
select(cls)
|
||||
.where(cls.task_id == task_id)
|
||||
.order_by(cls.ordinal.asc())
|
||||
).scalars().all()
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_prepare(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
checkpoint_fingerprint: str,
|
||||
ordinal: int,
|
||||
phase: str,
|
||||
kind: str,
|
||||
intent_version: int,
|
||||
intent_payload: dict[str, Any],
|
||||
now_time: str,
|
||||
) -> "TransferExecutionStep":
|
||||
"""在调用方事务中暂存尚未执行的稳定步骤意图。"""
|
||||
step = cls(
|
||||
task_id=task_id,
|
||||
operation_id=operation_id,
|
||||
checkpoint_fingerprint=checkpoint_fingerprint,
|
||||
ordinal=ordinal,
|
||||
phase=phase,
|
||||
kind=kind,
|
||||
state="prepared",
|
||||
attempt_count=0,
|
||||
intent_version=intent_version,
|
||||
intent_payload=intent_payload,
|
||||
prepared_at=now_time,
|
||||
updated_at=now_time,
|
||||
)
|
||||
db.add(step)
|
||||
return step
|
||||
|
||||
@classmethod
|
||||
def start_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效任务租约 CAS 开始一次新的步骤尝试。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "prepared",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="started",
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def restart_after_not_applied(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
previous_attempt_token: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以 NOT_APPLIED 证据和旧 attempt token CAS 重启遗留步骤。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == previous_attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resume_failed_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以重试调度的新 lease CAS 恢复 FAILED 步骤并保留失败证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "failed",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="started",
|
||||
attempt_token=attempt_token,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
started_at=updated_at,
|
||||
completed_at=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def complete_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
result_version: int,
|
||||
result_payload: dict[str, Any],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以租约与 attempt 双 CAS 提交步骤成功证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="succeeded",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=None,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def fail_attempt(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: str,
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以租约与 attempt 双 CAS 提交已知失败证据。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "started",
|
||||
cls.attempt_token == attempt_token,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="failed",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=error,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def mark_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
operation_id: str,
|
||||
attempt_token: Optional[str],
|
||||
error: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以当前尝试身份隔离外部结果不可判定的步骤。"""
|
||||
attempt_match = (
|
||||
cls.attempt_token == attempt_token
|
||||
if attempt_token is not None
|
||||
else cls.attempt_token.is_(None)
|
||||
)
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state.in_(("prepared", "started", "failed")),
|
||||
attempt_match,
|
||||
cls._active_lease_exists(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_utc=now_utc,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
state="manual_review",
|
||||
attempt_token=None,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=error,
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resolve_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
operation_id: str,
|
||||
target_state: str,
|
||||
reason: str,
|
||||
result_version: Optional[int],
|
||||
result_payload: Optional[dict[str, Any]],
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""仅在 pending 同为无租约人工态时 CAS 提交步骤判定。"""
|
||||
if target_state not in {"failed", "succeeded"}:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.operation_id == operation_id,
|
||||
cls.state == "manual_review",
|
||||
cls.attempt_token.is_(None),
|
||||
cls._manual_review_pending_exists(task_id=task_id),
|
||||
)
|
||||
.values(
|
||||
state=target_state,
|
||||
result_version=result_version,
|
||||
result_payload=result_payload,
|
||||
last_error=(reason if target_state == "failed" else None),
|
||||
completed_at=updated_at,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def delete_by_task_id(cls, db: Session, *, task_id: str) -> int:
|
||||
"""在终态成功结算事务中删除任务的步骤证据。"""
|
||||
if not task_id:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
delete(cls).where(cls.task_id == task_id),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _active_lease_exists(
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
) -> Exists:
|
||||
"""构造关联 pending 行仍持有当前有效租约的 SQL 谓词。"""
|
||||
return exists(
|
||||
select(TransferPending.id).where(
|
||||
and_(
|
||||
TransferPending.task_id == task_id,
|
||||
TransferPending.lease_token == lease_token,
|
||||
TransferPending.lease_expires_at.is_not(None),
|
||||
TransferPending.lease_expires_at > now_utc,
|
||||
or_(
|
||||
TransferPending.execution_state == "running",
|
||||
TransferPending.execution_state == "not_started",
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _manual_review_pending_exists(*, task_id: str) -> Exists:
|
||||
"""构造关联 pending 行处于无租约人工复核态的 SQL 谓词。"""
|
||||
return exists(
|
||||
select(TransferPending.id).where(
|
||||
TransferPending.task_id == task_id,
|
||||
TransferPending.execution_state == "manual_review",
|
||||
TransferPending.lease_token.is_(None),
|
||||
TransferPending.lease_owner.is_(None),
|
||||
)
|
||||
)
|
||||
@@ -1,9 +1,9 @@
|
||||
import re
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Optional
|
||||
from typing import Any, List, Optional, cast
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, JSON, String, delete, func, or_, select, update
|
||||
from sqlalchemy import JSON, Boolean, Index, Integer, String, delete, func, or_, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
@@ -24,6 +24,10 @@ class TransferHistory(Base):
|
||||
整理记录
|
||||
"""
|
||||
id = get_id_column()
|
||||
# 失败 pending 当前映射使用的稳定整理任务标识
|
||||
transfer_task_id: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
# 失败 pending 当前映射对应的结算版本
|
||||
transfer_settlement_revision: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 源路径
|
||||
src: Mapped[Optional[str]] = mapped_column(String, index=True)
|
||||
# 源存储
|
||||
@@ -90,6 +94,11 @@ class TransferHistory(Base):
|
||||
Index('ix_transferhistory_date_id', 'date', 'id'),
|
||||
Index('ix_transferhistory_media_identity', 'media_source', 'media_id'),
|
||||
Index('ux_transferhistory_src_storage', 'src', 'src_storage', unique=True),
|
||||
Index(
|
||||
'ux_transferhistory_transfer_task_id',
|
||||
'transfer_task_id',
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -208,6 +217,82 @@ class TransferHistory(Base):
|
||||
statement = statement.where(cls.src_storage == storage)
|
||||
return db.execute(statement.order_by(cls.id.desc())).scalars().first()
|
||||
|
||||
@classmethod
|
||||
def get_by_transfer_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional["TransferHistory"]:
|
||||
"""按稳定整理任务标识读取终态结算历史。"""
|
||||
if not task_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferHistory"],
|
||||
db.execute(
|
||||
select(cls).where(cls.transfer_task_id == task_id)
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def upsert_by_transfer_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
settlement_revision: int,
|
||||
retain_task_mapping: bool,
|
||||
payload: dict[str, Any],
|
||||
) -> "TransferHistory":
|
||||
"""按任务幂等写历史，并维持同源存储仅保留一条的既有约束。"""
|
||||
if not task_id or settlement_revision <= 0:
|
||||
raise ValueError("整理历史结算缺少稳定任务或正向版本")
|
||||
column_names = {column.name for column in cls.__table__.columns}
|
||||
values = {
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key in column_names and key not in {
|
||||
"id",
|
||||
"transfer_task_id",
|
||||
"transfer_settlement_revision",
|
||||
}
|
||||
}
|
||||
src = values.get("src")
|
||||
if not src:
|
||||
raise ValueError("整理历史结算缺少源路径")
|
||||
src_storage = values.get("src_storage") or "local"
|
||||
values["src_storage"] = src_storage
|
||||
history = cls.get_by_transfer_task_id(db, task_id=task_id)
|
||||
if history is None:
|
||||
history = db.execute(
|
||||
select(cls).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
)
|
||||
).scalars().first()
|
||||
if history is None:
|
||||
history = cls(
|
||||
transfer_task_id=(task_id if retain_task_mapping else None),
|
||||
transfer_settlement_revision=(
|
||||
settlement_revision if retain_task_mapping else None
|
||||
),
|
||||
**values,
|
||||
)
|
||||
db.add(history)
|
||||
else:
|
||||
if (history.transfer_task_id == task_id
|
||||
and history.transfer_settlement_revision is not None
|
||||
and settlement_revision <= history.transfer_settlement_revision):
|
||||
raise ValueError("整理历史结算版本必须单调递增")
|
||||
history.transfer_task_id = task_id if retain_task_mapping else None
|
||||
history.transfer_settlement_revision = (
|
||||
settlement_revision if retain_task_mapping else None
|
||||
)
|
||||
for key, value in values.items():
|
||||
setattr(history, key, value)
|
||||
db.flush()
|
||||
return history
|
||||
|
||||
@classmethod
|
||||
def get_success_by_src(
|
||||
cls, db: Session, src: str,
|
||||
@@ -557,10 +642,20 @@ class TransferHistory(Base):
|
||||
src_storage = kwargs.get("src_storage") or "local"
|
||||
kwargs["src_storage"] = src_storage
|
||||
if src:
|
||||
durable = db.execute(
|
||||
select(cls.id).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
cls.transfer_task_id.is_not(None),
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if durable is not None:
|
||||
raise ValueError("持久整理回执不能由旧历史写入口覆盖")
|
||||
db.execute(
|
||||
delete(cls).where(
|
||||
cls.src == src,
|
||||
cls.src_storage == src_storage,
|
||||
cls.transfer_task_id.is_(None),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
@@ -590,7 +685,10 @@ class TransferHistory(Base):
|
||||
"""
|
||||
ids = db.execute(
|
||||
select(cls.id)
|
||||
.where(cls.date < before_time)
|
||||
.where(
|
||||
cls.date < before_time,
|
||||
cls.transfer_task_id.is_(None),
|
||||
)
|
||||
.order_by(cls.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all()
|
||||
|
||||
@@ -134,6 +134,42 @@ class TransferPending(Base):
|
||||
heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 真正取得新 token 的累计次数
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 与规划状态正交的执行状态
|
||||
execution_state: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="not_started"
|
||||
)
|
||||
# 聚合执行检查点格式版本
|
||||
execution_version: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 可独立重放终态结算的聚合执行结果
|
||||
execution_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
|
||||
# 聚合执行结果规范 JSON 的 SHA-256 指纹
|
||||
execution_fingerprint: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
# 每次进入 retry_wait 都递增的调度世代
|
||||
retry_generation: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 已持久提交的步骤重试次数
|
||||
retry_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 下一次允许 claim 的 UTC 时间
|
||||
retry_due_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 最近一次终态失败重试请求身份
|
||||
retry_requested_by: Mapped[Optional[str]] = mapped_column(String(128))
|
||||
# 最近一次终态失败重试请求原因
|
||||
retry_reason: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 已完成终态结算的单调版本
|
||||
settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
# 失败终态保留的整理历史标识
|
||||
terminal_history_id: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
# 人工判定的单调审计版本
|
||||
manual_review_revision: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=0
|
||||
)
|
||||
# 最近一次人工判定时间
|
||||
reviewed_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 最近一次人工判定操作者
|
||||
reviewed_by: Mapped[Optional[str]] = mapped_column(String(128))
|
||||
# 最近一次人工判定原因
|
||||
review_reason: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 最近一次人工判定结论
|
||||
review_decision: Mapped[Optional[str]] = mapped_column(String(32))
|
||||
|
||||
__table_args__ = (
|
||||
# 同一个文件重复入队只保留一条，回放时不会重复送入整理链
|
||||
@@ -153,6 +189,14 @@ class TransferPending(Base):
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
Index(
|
||||
"ix_transferpending_execution_due",
|
||||
"execution_state",
|
||||
"retry_due_at",
|
||||
"state",
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
UniqueConstraint("task_id", name="uq_transferpending_task_id"),
|
||||
)
|
||||
|
||||
@@ -264,6 +308,19 @@ class TransferPending(Base):
|
||||
cursor_created_at = func.coalesce(cls.created_at, "")
|
||||
statement = select(cls.task_id, cursor_created_at, cls.id).where(
|
||||
cls.state.in_(states),
|
||||
cls.execution_state.in_((
|
||||
"not_started",
|
||||
"running",
|
||||
"retry_wait",
|
||||
"settling",
|
||||
)),
|
||||
or_(
|
||||
cls.execution_state != "retry_wait",
|
||||
and_(
|
||||
cls.retry_due_at.is_not(None),
|
||||
cls.retry_due_at <= now_time,
|
||||
),
|
||||
),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
@@ -331,6 +388,19 @@ class TransferPending(Base):
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(states),
|
||||
cls.execution_state.in_((
|
||||
"not_started",
|
||||
"running",
|
||||
"retry_wait",
|
||||
"settling",
|
||||
)),
|
||||
or_(
|
||||
cls.execution_state != "retry_wait",
|
||||
and_(
|
||||
cls.retry_due_at.is_not(None),
|
||||
cls.retry_due_at <= now_time,
|
||||
),
|
||||
),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
@@ -348,6 +418,345 @@ class TransferPending(Base):
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_execution_running(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约把可执行任务推进或保持为 running。"""
|
||||
if not all((task_id, lease_token, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
)
|
||||
.values(
|
||||
execution_state="running",
|
||||
retry_due_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def defer_execution(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
retry_due_at: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约进入 retry_wait，并原子释放当前租约。"""
|
||||
if not all((task_id, lease_token, error, retry_due_at, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_count=cls.retry_count + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def mark_execution_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约隔离执行结果未知的任务，并释放自动调度租约。"""
|
||||
if not all((task_id, lease_token, error, now_utc, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state.in_(("not_started", "running", "retry_wait")),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="manual_review",
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def checkpoint_execution(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效租约保存可重放执行检查点并进入 settling。"""
|
||||
if not all((
|
||||
task_id,
|
||||
lease_token,
|
||||
execution_version,
|
||||
execution_payload,
|
||||
execution_fingerprint,
|
||||
now_utc,
|
||||
updated_at,
|
||||
)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="settling",
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
retry_due_at=None,
|
||||
last_error=None,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def checkpoint_exhausted_failure(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_version: int,
|
||||
execution_payload: dict[str, Any],
|
||||
execution_fingerprint: str,
|
||||
error: str,
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以有效 lease 保存预算耗尽失败检查点并保持租约进入 settling。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state == "planned",
|
||||
cls.checkpoint_version.is_not(None),
|
||||
cls.checkpoint_payload.is_not(None),
|
||||
cls.execution_state == "running",
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="settling",
|
||||
execution_version=execution_version,
|
||||
execution_payload=execution_payload,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
retry_due_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def request_execution_retry(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
reason: str,
|
||||
requested_by: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""仅将无租约 FAILED 任务 CAS 为立即到期的 retry_wait。"""
|
||||
if not all((task_id, reason, requested_by, retry_due_at, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "failed",
|
||||
cls.lease_token.is_(None),
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
retry_requested_by=requested_by,
|
||||
retry_reason=reason,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def resolve_manual_review(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
decision: str,
|
||||
actor: str,
|
||||
reason: str,
|
||||
retry_due_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""无 lease 地 CAS 提交人工判定审计并交回 retry_wait 调度。"""
|
||||
if decision not in {"not_applied", "applied"}:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "manual_review",
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_owner.is_(None),
|
||||
)
|
||||
.values(
|
||||
execution_state="retry_wait",
|
||||
retry_generation=cls.retry_generation + 1,
|
||||
retry_due_at=retry_due_at,
|
||||
manual_review_revision=cls.manual_review_revision + 1,
|
||||
reviewed_at=updated_at,
|
||||
reviewed_by=actor,
|
||||
review_reason=reason,
|
||||
review_decision=decision,
|
||||
last_error=(reason if decision == "not_applied" else None),
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_terminal_failure(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
history_id: int,
|
||||
error: Optional[str],
|
||||
now_utc: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本 CAS 保留失败终态及其历史。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "settling",
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.settlement_revision == expected_revision,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
)
|
||||
.values(
|
||||
execution_state="failed",
|
||||
settlement_revision=cls.settlement_revision + 1,
|
||||
terminal_history_id=history_id,
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def delete_terminal_success(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
execution_fingerprint: str,
|
||||
expected_revision: int,
|
||||
now_utc: str,
|
||||
) -> int:
|
||||
"""以执行指纹和结算版本 CAS 删除已成功结算的 pending。"""
|
||||
return execute_dml(
|
||||
db,
|
||||
delete(cls).where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_state == "settling",
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.settlement_revision == expected_revision,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_utc,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def record_projection_failure(
|
||||
cls,
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""整理任务终态结算回执模型。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, cast
|
||||
|
||||
from sqlalchemy import Boolean, Index, Integer, String, Text, UniqueConstraint, desc, select
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
from app.db.base import Base, get_id_column
|
||||
|
||||
|
||||
class TransferSettlementReceipt(Base):
|
||||
"""按任务保存独立于最新历史投影的 durable 终态证据。"""
|
||||
|
||||
id = get_id_column()
|
||||
task_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
history_id: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
settlement_revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
outcome: Mapped[str] = mapped_column(String(16), nullable=False)
|
||||
execution_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
lease_token: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
history_status: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
src: Mapped[Optional[str]] = mapped_column(String)
|
||||
src_storage: Mapped[Optional[str]] = mapped_column(String)
|
||||
pending_deleted: Mapped[bool] = mapped_column(Boolean, nullable=False)
|
||||
error: Mapped[Optional[str]] = mapped_column(Text)
|
||||
created_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
updated_at: Mapped[str] = mapped_column(String(40), nullable=False)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"task_id",
|
||||
"settlement_revision",
|
||||
name="uq_transfersettlementreceipt_task_revision",
|
||||
),
|
||||
Index(
|
||||
"ix_transfersettlementreceipt_task_revision",
|
||||
"task_id",
|
||||
"settlement_revision",
|
||||
),
|
||||
Index(
|
||||
"ix_transfersettlementreceipt_history_id",
|
||||
"history_id",
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_latest_by_task_id(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
) -> Optional["TransferSettlementReceipt"]:
|
||||
"""按稳定任务标识读取最新已提交结算回执。"""
|
||||
if not task_id:
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferSettlementReceipt"],
|
||||
db.execute(
|
||||
select(cls)
|
||||
.where(cls.task_id == task_id)
|
||||
.order_by(desc(cls.settlement_revision))
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_by_identity(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
outcome: str,
|
||||
) -> Optional["TransferSettlementReceipt"]:
|
||||
"""按原始执行身份读取不可变结算回执。"""
|
||||
if not all((task_id, execution_fingerprint, lease_token, outcome)):
|
||||
return None
|
||||
return cast(
|
||||
Optional["TransferSettlementReceipt"],
|
||||
db.execute(
|
||||
select(cls).where(
|
||||
cls.task_id == task_id,
|
||||
cls.execution_fingerprint == execution_fingerprint,
|
||||
cls.lease_token == lease_token,
|
||||
cls.outcome == outcome,
|
||||
).order_by(desc(cls.settlement_revision))
|
||||
).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def stage_append(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
history_id: int,
|
||||
settlement_revision: int,
|
||||
outcome: str,
|
||||
execution_fingerprint: str,
|
||||
lease_token: str,
|
||||
history_status: bool,
|
||||
src: Optional[str],
|
||||
src_storage: Optional[str],
|
||||
pending_deleted: bool,
|
||||
error: Optional[str],
|
||||
settled_at: str,
|
||||
) -> "TransferSettlementReceipt":
|
||||
"""按连续修订追加任务回执，旧修订证据永不覆盖。"""
|
||||
if not all((task_id, history_id, settlement_revision, outcome,
|
||||
execution_fingerprint, lease_token, settled_at)):
|
||||
raise ValueError("整理结算回执缺少稳定身份或结果证据")
|
||||
if outcome not in {"succeeded", "failed"}:
|
||||
raise ValueError(f"不支持的整理结算结果：{outcome}")
|
||||
if cls.get_by_identity(
|
||||
db,
|
||||
task_id=task_id,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
outcome=outcome,
|
||||
) is not None:
|
||||
raise ValueError("同一整理执行身份不能追加多个结算修订")
|
||||
latest = cls.get_latest_by_task_id(db, task_id=task_id)
|
||||
if latest is None:
|
||||
if settlement_revision != 1:
|
||||
raise ValueError("整理结算回执必须从修订 1 开始")
|
||||
elif settlement_revision != latest.settlement_revision + 1:
|
||||
raise ValueError("整理结算回执修订必须连续递增")
|
||||
receipt = cls(
|
||||
task_id=task_id,
|
||||
history_id=history_id,
|
||||
settlement_revision=settlement_revision,
|
||||
outcome=outcome,
|
||||
execution_fingerprint=execution_fingerprint,
|
||||
lease_token=lease_token,
|
||||
history_status=history_status,
|
||||
src=src,
|
||||
src_storage=src_storage,
|
||||
pending_deleted=pending_deleted,
|
||||
error=error,
|
||||
created_at=settled_at,
|
||||
updated_at=settled_at,
|
||||
)
|
||||
db.add(receipt)
|
||||
db.flush()
|
||||
return receipt
|
||||
Reference in New Issue
Block a user