GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-05 23:47:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: add fenced transfer recovery leases

Browse Source
This commit is contained in:
jxxghp
2026-08-27 14:39:38 +08:00
parent 22865cb35b
commit a62c541ec0
32 changed files with 4135 additions and 650 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/db/adapters/transfer.py
+290 -26
View File
@@ -1,7 +1,9 @@
"""整理任务持久准入端口的 SQLAlchemy 适配器。"""
import logging
from collections.abc import Callable
from datetime import datetime
from datetime import datetime, timedelta, timezone
from json import JSONDecodeError
from typing import Optional
from uuid import uuid4
@@ -14,6 +16,8 @@ from app.application.transfer import (
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferAdmission,
TransferAdmissionConflictError,
TransferAdmissionProjectionError,
TransferLeaseLostError,
TransferPlanCheckpoint,
TransferPlanningInput,
TransferPlanningStateError,
@@ -22,10 +26,14 @@ from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
from app.db.uow import SqlAlchemyUnitOfWork
_diagnostic_logger = logging.getLogger(__name__)
class TransactionalTransferAdmissionRepository:
"""以短生命周期 Session 实现整理任务持久准入端口。"""
_MAX_RECOVERY_SCAN_TASKS = 5000
def __init__(self, session_factory: Callable[[], Session]) -> None:
"""保存由组合根提供的同步会话工厂。"""
self._session_factory = session_factory
@@ -35,6 +43,33 @@ class TransactionalTransferAdmissionRepository:
"""生成与历史登记时间可按字典序比较的当前时间。"""
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@staticmethod
def _lease_now() -> datetime:
"""生成不受宿主时区影响的当前 UTC 租约时间。"""
return datetime.now(timezone.utc)
@staticmethod
def _format_lease_time(value: datetime) -> str:
"""把 UTC 时间编码为可稳定排序的固定宽度字符串。"""
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
@staticmethod
def _validate_claim_arguments(*, owner_id: str, lease_seconds: int) -> None:
"""拒绝无法建立有效租约身份或正向期限的调用。"""
if not owner_id:
raise ValueError("整理任务 claim 缺少 owner_id")
if lease_seconds <= 0:
raise ValueError("整理任务 lease_seconds 必须大于零")
@staticmethod
def _recoverable_states() -> tuple[str, ...]:
"""返回允许被 worker claim 的稳定业务状态。"""
return (
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
)
@staticmethod
def _project(pending: TransferPending) -> TransferAdmission:
"""在 Session 有效期内把 ORM 行冻结为应用层 DTO。"""
@@ -84,6 +119,11 @@ class TransactionalTransferAdmissionRepository:
input_fingerprint=pending.input_fingerprint,
planning_input=planning_input,
checkpoint=checkpoint,
lease_owner=pending.lease_owner,
lease_token=pending.lease_token,
lease_expires_at=pending.lease_expires_at,
heartbeat_at=pending.heartbeat_at,
attempt_count=pending.attempt_count,
)
@staticmethod
@@ -151,27 +191,225 @@ class TransactionalTransferAdmissionRepository:
self._assert_input_match(pending, effective_input)
return self._project(pending)
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
"""在独立只读会话中投影等待恢复或执行的准入记录。"""
def claim_task(
self,
*,
task_id: str,
owner_id: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""原子 claim 指定任务，任何已存在的有效租约都拒绝重复领取。"""
self._validate_claim_arguments(
owner_id=owner_id,
lease_seconds=lease_seconds,
)
if not task_id:
raise ValueError("整理任务 claim 缺少 task_id")
now = self._lease_now()
now_time = self._format_lease_time(now)
lease_expires_at = self._format_lease_time(
now + timedelta(seconds=lease_seconds)
)
with self._session_factory() as session:
pending_items = TransferPendingOper(db=session).list_by_state(
state=TRANSFER_ADMISSION_ACCEPTED,
limit=limit,
)
return [self._project(pending) for pending in pending_items]
transaction = SqlAlchemyUnitOfWork(session)
try:
oper = TransferPendingOper(db=session)
lease_token = uuid4().hex
updated = oper.stage_claim_task(
task_id=task_id,
states=self._recoverable_states(),
owner_id=owner_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
updated_at=self._now(),
)
session.flush()
session.expire_all()
try:
pending = oper.get_by_task_id(task_id=task_id)
except JSONDecodeError as error:
raise TransferAdmissionProjectionError(
f"整理任务持久 JSON 无法解码: {task_id} - {error}"
) from error
if updated:
if pending is None:
raise TransferPlanningStateError(
f"claim 后未找到整理任务: {task_id}"
)
try:
admission = self._project(pending)
except (
TransferAdmissionConflictError,
TransferPlanningStateError,
TypeError,
ValueError,
) as error:
raise TransferAdmissionProjectionError(
f"整理任务持久投影损坏: {task_id} - {error}"
) from error
transaction.commit()
return admission
transaction.commit()
return None
except Exception:
transaction.rollback()
raise
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
"""投影接纳、provider 待执行或已规划的全部可恢复任务。"""
with self._session_factory() as session:
pending_items = TransferPendingOper(db=session).list_by_states(
states=(
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
),
limit=limit,
def claim_recoverable(
self,
*,
owner_id: str,
limit: int,
lease_seconds: int,
) -> list[TransferAdmission]:
"""按登记顺序逐条 CAS claim 未租用或租约已过期的恢复任务。"""
self._validate_claim_arguments(
owner_id=owner_id,
lease_seconds=lease_seconds,
)
if limit <= 0:
return []
claimed: list[TransferAdmission] = []
after_cursor: Optional[tuple[str, int]] = None
scanned_count = 0
scan_limit = self._MAX_RECOVERY_SCAN_TASKS
while len(claimed) < limit and scanned_count < scan_limit:
candidate_limit = min(
limit - len(claimed),
scan_limit - scanned_count,
)
return [self._project(pending) for pending in pending_items]
now_time = self._format_lease_time(self._lease_now())
with self._session_factory() as session:
candidates = TransferPendingOper(db=session).list_claimable_candidates(
states=self._recoverable_states(),
now_time=now_time,
limit=candidate_limit,
after_cursor=after_cursor,
)
if not candidates:
break
scanned_count += len(candidates)
_, cursor_created_at, cursor_id = candidates[-1]
after_cursor = (cursor_created_at, cursor_id)
for task_id, _, _ in candidates:
try:
admission = self.claim_task(
task_id=task_id,
owner_id=owner_id,
lease_seconds=lease_seconds,
)
except TransferAdmissionProjectionError as error:
self._record_projection_failure(
task_id=task_id,
error=error,
)
continue
if admission is not None:
claimed.append(admission)
if len(claimed) >= limit:
break
return claimed
def _record_projection_failure(
self,
*,
task_id: str,
error: TransferAdmissionProjectionError,
) -> bool:
"""以独立 CAS 留存变化后的投影错误，并仅为新诊断记一次运行日志。"""
diagnostic = f"恢复投影失败: {error}"
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
recorded = TransferPendingOper(
db=session
).stage_record_projection_failure(
task_id=task_id,
states=self._recoverable_states(),
error=diagnostic,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
transaction.commit()
except Exception:
transaction.rollback()
raise
if recorded:
_diagnostic_logger.error(
f"整理恢复任务投影损坏：task_id={task_id}, error={error}"
)
return bool(recorded)
def heartbeat(
self,
*,
task_id: str,
lease_token: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""仅以当前且未过期的 token 续租，禁止陈旧 worker 复活租约。"""
if not task_id or not lease_token:
raise ValueError("整理任务 heartbeat 缺少任务或租约身份")
if lease_seconds <= 0:
raise ValueError("整理任务 lease_seconds 必须大于零")
now = self._lease_now()
now_time = self._format_lease_time(now)
lease_expires_at = self._format_lease_time(
now + timedelta(seconds=lease_seconds)
)
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
oper = TransferPendingOper(db=session)
updated = oper.stage_heartbeat(
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
)
if not updated:
transaction.commit()
return None
session.flush()
session.expire_all()
pending = oper.get_by_task_id(task_id=task_id)
if pending is None:
raise TransferPlanningStateError(
f"heartbeat 后未找到整理任务: {task_id}"
)
admission = self._project(pending)
transaction.commit()
return admission
except Exception:
transaction.rollback()
raise
def release_claim(
self,
*,
task_id: str,
lease_token: str,
error: Optional[str] = None,
) -> bool:
"""仅以当前未过期 token 释放租约，陈旧 worker 不得改变任务。"""
if not task_id or not lease_token:
return False
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
released = TransferPendingOper(db=session).stage_release_claim(
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
transaction.commit()
return bool(released)
except Exception:
transaction.rollback()
raise
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
"""独立提交最近一次入队失败，保留准入记录供后续恢复。"""
@@ -192,6 +430,7 @@ class TransactionalTransferAdmissionRepository:
self,
*,
task_id: str,
lease_token: str,
input_fingerprint: str,
checkpoint: TransferPlanCheckpoint,
) -> TransferAdmission:
@@ -223,7 +462,9 @@ class TransactionalTransferAdmissionRepository:
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
now_time=self._now(),
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
session.flush()
session.expire_all()
@@ -232,6 +473,13 @@ class TransactionalTransferAdmissionRepository:
raise TransferPlanningStateError(f"未找到整理任务: {task_id}")
if pending.input_fingerprint != input_fingerprint:
raise TransferAdmissionConflictError("整理任务输入指纹已经改变")
now_time = self._format_lease_time(self._lease_now())
if (
pending.lease_token != lease_token
or not pending.lease_expires_at
or pending.lease_expires_at <= now_time
):
raise TransferLeaseLostError("整理任务租约已过期或已被其他 worker 接管")
if not updated and not (
pending.state == target_state
and pending.checkpoint_payload == checkpoint_payload
@@ -246,28 +494,44 @@ class TransactionalTransferAdmissionRepository:
transaction.rollback()
raise
def record_planning_failure(self, *, task_id: str, error: str) -> None:
def record_planning_failure(
self,
*,
task_id: str,
lease_token: str,
error: str,
) -> None:
"""独立提交规划错误并保持任务处于接纳态供恢复重试。"""
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
TransferPendingOper(db=session).stage_record_planning_failure(
updated = TransferPendingOper(db=session).stage_record_planning_failure(
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=self._now(),
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
if not updated:
raise TransferLeaseLostError(
"整理任务租约已过期或已被其他 worker 接管"
)
transaction.commit()
except Exception:
transaction.rollback()
raise
def discard_task(self, *, task_id: str) -> int:
"""在独立事务中按稳定任务标识删除已到终态的准入记录。"""
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前未过期 token 删除终态任务，拒绝陈旧 worker 变更。"""
if not task_id or not lease_token:
return 0
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
deleted = TransferPendingOper(db=session).stage_discard_task(
deleted = TransferPendingOper(db=session).stage_discard_claimed(
task_id=task_id,
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
)
transaction.commit()
return deleted
app/db/models/transferpending.py
+338 -148
View File
@@ -4,7 +4,20 @@ from datetime import datetime
from typing import Any, List, Optional, cast
from uuid import uuid4
from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update
from sqlalchemy import (
JSON,
Index,
Integer,
String,
Text,
UniqueConstraint,
and_,
delete,
func,
or_,
select,
update,
)
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import Base, execute_dml, get_id_column
@@ -72,7 +85,7 @@ class TransferPending(Base):
准入时保存版本化规划输入和指纹；纯规划完成后以同一行原子保存完整有序计划并
推进到 planned。重启恢复可直接消费已规划路径，避免再次触发 rename 等插件事件。
旧路径登记接口仍生成最小 legacy_replan 输入，供插件兼容调用方继续使用。
所有执行期 mutation 都以稳定任务身份和租约 token 进行 CAS fencing。
"""
id = get_id_column()
@@ -111,6 +124,16 @@ class TransferPending(Base):
checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
# 规划完成时间
planned_at: Mapped[Optional[str]] = mapped_column(String(40))
# 当前租约拥有者
lease_owner: Mapped[Optional[str]] = mapped_column(String(128))
# 当前租约的唯一防陈旧令牌
lease_token: Mapped[Optional[str]] = mapped_column(String(64))
# 当前租约的 UTC 到期时间
lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40))
# 最近一次成功 claim 或 heartbeat 的 UTC 时间
heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40))
# 真正取得新 token 的累计次数
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (
# 同一个文件重复入队只保留一条，回放时不会重复送入整理链
@@ -122,41 +145,17 @@ class TransferPending(Base):
"created_at",
"id",
),
# 恢复调度按业务状态和租约到期时间筛选可接管任务
Index(
"ix_transferpending_recovery_lease",
"state",
"lease_expires_at",
"created_at",
"id",
),
UniqueConstraint("task_id", name="uq_transferpending_task_id"),
)
@classmethod
def register(cls, db: Session, storage: str, src_path: str,
now_time: str) -> Optional["TransferPending"]:
"""
登记一个待整理文件，已存在时保持原登记时间不变。
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:param now_time: 当前时间
:return: 登记记录
"""
if not storage or not src_path:
return None
pending = db.execute(
select(cls).where(cls.storage == storage, cls.src_path == src_path)
).scalars().first()
if pending:
return cast("TransferPending", pending)
planning_input = _legacy_planning_payload(storage, src_path)
pending = cls(
storage=storage,
src_path=src_path,
state="accepted",
created_at=now_time,
updated_at=now_time,
input_version=1,
planning_input=planning_input,
input_fingerprint=_planning_fingerprint(planning_input),
)
db.add(pending)
return pending
@classmethod
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
src_path: str, state: str,
@@ -201,44 +200,6 @@ class TransferPending(Base):
db.add(pending)
return pending
@classmethod
def list_by_state(cls, db: Session, *, state: str,
limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
按登记顺序列出指定持久状态的接纳记录。
:param db: 数据库会话
:param state: 持久状态
:param limit: 单次读取上限
:return: 接纳记录列表
"""
if not state:
return []
return list(db.execute(
select(cls)
.where(cls.state == state)
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def list_by_states(cls, db: Session, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
按登记顺序列出多个可恢复持久状态的记录。
:param db: 数据库会话
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: 接纳记录列表
"""
if not states:
return []
return list(db.execute(
select(cls)
.where(cls.state.in_(states))
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def get_by_identity(cls, db: Session, *, storage: str,
src_path: str) -> Optional["TransferPending"]:
@@ -276,12 +237,283 @@ class TransferPending(Base):
db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(),
)
@classmethod
def list_claimable_candidates(
cls,
db: Session,
*,
states: tuple[str, ...],
now_time: str,
limit: int,
after_cursor: Optional[tuple[str, int]] = None,
) -> List[tuple[str, str, int]]:
"""
按稳定游标列出未租用或租约已过期的候选任务。
返回候选不等于取得租约；调用方必须继续执行带相同过期条件的 claim CAS，
并以受影响行数决定竞争结果。
:param db: 数据库会话
:param states: 可恢复业务状态
:param now_time: 当前 UTC 时间
:param limit: 候选数量上限
:param after_cursor: 上一页最后一条的规范登记时间与主键
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
"""
if not states or not now_time or limit <= 0:
return []
cursor_created_at = func.coalesce(cls.created_at, "")
statement = select(cls.task_id, cursor_created_at, cls.id).where(
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
)
if after_cursor is not None:
after_created_at, after_id = after_cursor
statement = statement.where(or_(
cursor_created_at > after_created_at,
and_(
cursor_created_at == after_created_at,
cls.id > after_id,
),
))
rows = db.execute(
statement
.order_by(cursor_created_at.asc(), cls.id.asc())
.limit(limit)
).all()
return [
(task_id, created_at or "", int(row_id))
for task_id, created_at, row_id in rows
]
@classmethod
def claim_task(
cls,
db: Session,
*,
task_id: str,
states: tuple[str, ...],
owner_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
updated_at: str,
) -> int:
"""
以未租用或租约已过期为条件原子取得任务租约。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param states: 允许 claim 的业务状态
:param owner_id: 新租约拥有者
:param lease_token: 新租约唯一令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数，1 表示赢得竞争
"""
if not all((
task_id,
states,
owner_id,
lease_token,
now_time,
lease_expires_at,
updated_at,
)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
)
.values(
lease_owner=owner_id,
lease_token=lease_token,
lease_expires_at=lease_expires_at,
heartbeat_at=now_time,
attempt_count=cls.attempt_count + 1,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def record_projection_failure(
cls,
db: Session,
*,
task_id: str,
states: tuple[str, ...],
error: str,
now_time: str,
updated_at: str,
) -> int:
"""
在没有有效租约且诊断发生变化时原子记录恢复投影损坏。
claim 的投影失败会先回滚，因此这里不得重新占用租约。CAS 同时保护
已被其他 worker 领取的任务，并避免周期恢复反复刷新相同错误。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param error: 可持久化的稳定诊断文本
:param now_time: 当前 UTC 租约时间
:param updated_at: 宿主本地业务审计时间
:return: 更新的记录数，1 表示首次或变化后的诊断被记录
"""
if not all((task_id, states, error, now_time, updated_at)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
cls.last_error.is_distinct_from(error),
)
.values(
last_error=error,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def heartbeat(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
) -> int:
"""
仅以当前且未过期的 token 原子延长任务租约。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:return: 更新的记录数
"""
if not all((task_id, lease_token, now_time, lease_expires_at)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
lease_expires_at=lease_expires_at,
heartbeat_at=now_time,
),
execution_options={"synchronize_session": False},
)
@classmethod
def release_claim(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
error: Optional[str],
now_time: str,
updated_at: str,
) -> int:
"""
仅以当前且未过期的 token 释放租约并保存本次执行错误。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param error: 本次执行错误，成功释放时为空
:param now_time: 当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if not task_id or not lease_token or not now_time or not updated_at:
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
lease_owner=None,
lease_token=None,
lease_expires_at=None,
heartbeat_at=None,
last_error=error,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def discard_claimed(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
now_time: str,
) -> int:
"""
仅以当前且未过期的 token 删除已经到达终态的租约任务。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:return: 删除的记录数
"""
if not task_id or not lease_token or not now_time:
return 0
return execute_dml(
db,
delete(cls).where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
),
execution_options={"synchronize_session": False},
)
@classmethod
def checkpoint_plan(cls, db: Session, *, task_id: str,
input_fingerprint: str, checkpoint_version: int,
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...], target_state: str,
now_time: str) -> int:
lease_token: str, now_time: str,
updated_at: str) -> int:
"""
以输入指纹为 CAS 条件原子保存计划并推进到已规划。
:param db: 数据库会话
@@ -291,7 +523,9 @@ class TransferPending(Base):
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许推进检查点的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:param lease_token: 当前且未过期的租约令牌
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if (
@@ -300,8 +534,19 @@ class TransferPending(Base):
or not checkpoint_payload
or not source_states
or not target_state
or not lease_token
or not updated_at
):
return 0
values: dict[str, Any] = {
"state": target_state,
"checkpoint_version": checkpoint_version,
"checkpoint_payload": checkpoint_payload,
"last_error": None,
"updated_at": updated_at,
}
if target_state == "planned":
values["planned_at"] = updated_at
return execute_dml(
db,
update(cls)
@@ -309,30 +554,29 @@ class TransferPending(Base):
cls.task_id == task_id,
cls.state.in_(source_states),
cls.input_fingerprint == input_fingerprint,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
state=target_state,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
planned_at=now_time,
last_error=None,
updated_at=now_time,
),
.values(**values),
execution_options={"synchronize_session": False},
)
@classmethod
def record_planning_failure(cls, db: Session, *, task_id: str,
error: str, now_time: str) -> int:
lease_token: str, error: str,
now_time: str, updated_at: str) -> int:
"""
为接纳态或 provider 待执行任务记录规划失败，不改变其恢复状态。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前且未过期的租约令牌
:param error: 失败原因
:param now_time: 当前时间
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if not task_id:
if not task_id or not lease_token or not now_time or not updated_at:
return 0
return execute_dml(
db,
@@ -340,8 +584,11 @@ class TransferPending(Base):
.where(
cls.task_id == task_id,
cls.state.in_(("accepted", "provider_pending")),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(last_error=error, updated_at=now_time),
.values(last_error=error, updated_at=updated_at),
execution_options={"synchronize_session": False},
)
@@ -361,67 +608,10 @@ class TransferPending(Base):
return execute_dml(
db,
update(cls)
.where(cls.task_id == task_id)
.where(
cls.task_id == task_id,
cls.lease_token.is_(None),
)
.values(last_error=error, updated_at=now_time),
execution_options={"synchronize_session": False},
)
@classmethod
def discard_task(cls, db: Session, *, task_id: str) -> int:
"""
在调用方会话中按任务标识删除接纳记录。
:param db: 数据库会话
:param task_id: 任务标识
:return: 删除的记录数
"""
if not task_id:
return 0
return execute_dml(
db, delete(cls).where(cls.task_id == task_id),
execution_options={"synchronize_session": False},
)
@classmethod
def discard(cls, db: Session, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记，整理到达终态（成功或失败）时调用。
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
if not storage or not src_path:
return 0
return execute_dml(
db, delete(cls).where(cls.storage == storage, cls.src_path == src_path),
execution_options={"synchronize_session": False},
)
@classmethod
def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
列出全部待整理登记，供启动回放使用。
按登记时间升序回放，保持与原入队顺序一致；上限避免异常积压时
一次性把整理链压垮。
:param db: 数据库会话
:param limit: 单次回放上限
:return: 待整理登记列表
"""
return list(db.execute(
select(cls)
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def clear(cls, db: Session) -> int:
"""
清空全部待整理登记。
:param db: 数据库会话
:return: 删除的记录数
"""
return execute_dml(
db, delete(cls),
execution_options={"synchronize_session": False},
execution_options={"synchronize_session": "fetch"},
)
app/db/oper/transferpending.py
+191 -110
View File
@@ -1,5 +1,4 @@
from datetime import datetime
from typing import Any, List, Optional, Tuple
from typing import Any, Optional
from app.db.base import DbOper
from app.db.models.transferpending import TransferPending
@@ -10,27 +9,9 @@ class TransferPendingOper(DbOper):
待整理文件登记管理。
保存稳定任务身份、存储、源文件路径和准入状态，用于在进程重启后把没走完
整理链的文件重新送回去，避免挂载故障重启后永久漏件。旧版路径登记接口继续
保留，供插件和兼容调用方使用。
整理链的文件重新送回去，避免挂载故障重启后永久漏件。
"""
def register(self, storage: str, src_path: str) -> Optional[TransferPending]:
"""
登记一个待整理文件。
:param storage: 存储
:param src_path: 源文件路径
:return: 登记记录
"""
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return self._execute_sync_write(
lambda session: TransferPending.register(
session,
storage=storage,
src_path=src_path,
now_time=now_time,
)
)
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
state: str, now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
@@ -63,38 +44,6 @@ class TransferPendingOper(DbOper):
)
)
def list_by_state(self, *, state: str,
limit: Optional[int] = 5000) -> List[TransferPending]:
"""
使用当前会话列出指定状态记录。
:param state: 持久状态
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_by_state(
session,
state=state,
limit=limit,
)
) or []
def list_by_states(self, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List[TransferPending]:
"""
使用当前会话列出多个可恢复状态的记录。
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_by_states(
session,
states=states,
limit=limit,
)
) or []
def get_by_identity(self, *, storage: str,
src_path: str) -> Optional[TransferPending]:
"""
@@ -133,7 +82,9 @@ class TransferPendingOper(DbOper):
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...],
target_state: str,
lease_token: str,
now_time: str,
updated_at: str,
) -> int:
"""
在当前会话中以输入指纹为条件暂存完整计划检查点。
@@ -143,7 +94,9 @@ class TransferPendingOper(DbOper):
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许执行 CAS 的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:param lease_token: 当前且未过期的租约令牌
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
@@ -155,25 +108,206 @@ class TransferPendingOper(DbOper):
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
lease_token=lease_token,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_record_planning_failure(self, *, task_id: str, error: str,
now_time: str) -> int:
def stage_record_planning_failure(self, *, task_id: str, lease_token: str,
error: str, now_time: str,
updated_at: str) -> int:
"""
在当前会话中记录规划失败并保持任务处于接纳态。
:param task_id: 稳定任务标识
:param lease_token: 当前且未过期的租约令牌
:param error: 失败原因
:param now_time: 当前时间
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.record_planning_failure(
session,
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def list_claimable_candidates(
self,
*,
states: tuple[str, ...],
now_time: str,
limit: int,
after_cursor: Optional[tuple[str, int]] = None,
) -> list[tuple[str, str, int]]:
"""
使用当前会话按稳定游标读取未租用或已过期的恢复候选。
:param states: 可恢复业务状态
:param now_time: 当前 UTC 时间
:param limit: 候选数量上限
:param after_cursor: 上一页最后一条的规范登记时间与主键
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_claimable_candidates(
session,
states=states,
now_time=now_time,
limit=limit,
after_cursor=after_cursor,
)
) or []
def stage_claim_task(
self,
*,
task_id: str,
states: tuple[str, ...],
owner_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
updated_at: str,
) -> int:
"""
使用当前会话以未租或过期条件竞争一个新租约。
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param owner_id: 新租约拥有者
:param lease_token: 新租约唯一令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.claim_task(
session,
task_id=task_id,
states=states,
owner_id=owner_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
updated_at=updated_at,
)
)
def stage_record_projection_failure(
self,
*,
task_id: str,
states: tuple[str, ...],
error: str,
now_time: str,
updated_at: str,
) -> int:
"""
使用当前会话按无有效租约和诊断变化条件记录投影损坏。
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param error: 可持久化的稳定诊断文本
:param now_time: 当前 UTC 租约时间
:param updated_at: 宿主本地业务审计时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.record_projection_failure(
session,
task_id=task_id,
states=states,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_heartbeat(
self,
*,
task_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
) -> int:
"""
使用当前会话以当前未过期 token 延长租约。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.heartbeat(
session,
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
)
)
def stage_release_claim(
self,
*,
task_id: str,
lease_token: str,
error: Optional[str],
now_time: str,
updated_at: str,
) -> int:
"""
使用当前会话按未过期 token 释放租约并记录本次错误。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param error: 本次执行错误，成功释放时为空
:param now_time: 当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.release_claim(
session,
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_discard_claimed(
self,
*,
task_id: str,
lease_token: str,
now_time: str,
) -> int:
"""
使用当前会话按当前未过期 token 删除终态任务。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_claimed(
session,
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
)
)
@@ -194,56 +328,3 @@ class TransferPendingOper(DbOper):
now_time=now_time,
)
)
def stage_discard_task(self, *, task_id: str) -> int:
"""
在当前会话中暂存按任务标识删除接纳记录。
:param task_id: 任务标识
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_task(
session,
task_id=task_id,
)
)
def discard(self, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记。
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard(
session,
storage=storage,
src_path=src_path,
)
)
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
"""
列出全部待整理登记，供启动回放使用。
返回纯元组而不是 ORM 实例：回放发生在会话之外，ORM 实例脱离 session
后访问属性会触发 DetachedInstanceError。
:param limit: 单次回放上限
:return: (存储, 源文件路径) 列表
"""
items = self._execute_sync_query(
lambda session: TransferPending.list_all(session, limit=limit)
)
return [
(item.storage, item.src_path)
for item in items or []
if item and item.storage and item.src_path
]
def clear(self) -> int:
"""
清空全部待整理登记。
:return: 删除的记录数
"""
return self._execute_sync_write(TransferPending.clear)