mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-31 04:57:23 +08:00
refactor: add fenced transfer recovery leases
This commit is contained in:
+78
-11
@@ -627,6 +627,14 @@ class TransferPlanningStateError(RuntimeError):
|
||||
"""计划检查点无法从当前持久状态推进时抛出的状态错误。"""
|
||||
|
||||
|
||||
class TransferAdmissionProjectionError(RuntimeError):
|
||||
"""持久登记无法安全恢复为应用层整理准入投影时抛出的错误。"""
|
||||
|
||||
|
||||
class TransferLeaseLostError(TransferPlanningStateError):
|
||||
"""整理 worker 已失去持久租约、不得继续推进任务时抛出的错误。"""
|
||||
|
||||
|
||||
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
"""
|
||||
文件整理任务。
|
||||
@@ -661,6 +669,8 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
_planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None)
|
||||
_plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None)
|
||||
_planning_context_restored: bool = PrivateAttr(default=False)
|
||||
_lease_owner: Optional[str] = PrivateAttr(default=None)
|
||||
_lease_token: Optional[str] = PrivateAttr(default=None)
|
||||
|
||||
@property
|
||||
def admission_task_id(self) -> Optional[str]:
|
||||
@@ -698,6 +708,23 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
|
||||
"""标记领域上下文已离线恢复,禁止旧流程再次在线补充。"""
|
||||
self._planning_context_restored = True
|
||||
|
||||
@property
|
||||
def lease_owner(self) -> Optional[str]:
|
||||
"""返回当前任务绑定的进程级 worker owner。"""
|
||||
return self._lease_owner
|
||||
|
||||
@property
|
||||
def lease_token(self) -> Optional[str]:
|
||||
"""返回当前任务绑定的持久租约令牌。"""
|
||||
return self._lease_token
|
||||
|
||||
def bind_execution_lease(self, *, owner_id: str, lease_token: str) -> None:
|
||||
"""绑定持久 claim 结果,且不改变插件可见的旧任务序列化字段。"""
|
||||
if not owner_id or not lease_token:
|
||||
raise ValueError("整理执行租约缺少 owner 或 token")
|
||||
self._lease_owner = owner_id
|
||||
self._lease_token = lease_token
|
||||
|
||||
def to_dict(self):
|
||||
"""
|
||||
返回字典。
|
||||
@@ -743,6 +770,11 @@ class TransferAdmission:
|
||||
input_fingerprint: Optional[str] = None
|
||||
planning_input: Optional[TransferPlanningInput] = None
|
||||
checkpoint: Optional[TransferPlanCheckpoint] = None
|
||||
lease_owner: Optional[str] = None
|
||||
lease_token: Optional[str] = None
|
||||
lease_expires_at: Optional[str] = None
|
||||
heartbeat_at: Optional[str] = None
|
||||
attempt_count: int = 0
|
||||
|
||||
|
||||
class TransferAdmissionRepository(Protocol):
|
||||
@@ -758,10 +790,6 @@ class TransferAdmissionRepository(Protocol):
|
||||
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
|
||||
...
|
||||
|
||||
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
|
||||
"""按登记顺序返回等待恢复或执行的任务。"""
|
||||
...
|
||||
|
||||
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
|
||||
"""记录内存队列接收失败,保留任务供后续恢复。"""
|
||||
...
|
||||
@@ -770,22 +798,61 @@ class TransferAdmissionRepository(Protocol):
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
input_fingerprint: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
) -> TransferAdmission:
|
||||
"""原子保存完整计划并将匹配输入的任务推进到已规划。"""
|
||||
"""按有效租约原子保存完整计划并推进匹配输入的任务。"""
|
||||
...
|
||||
|
||||
def record_planning_failure(self, *, task_id: str, error: str) -> None:
|
||||
"""记录规划失败但保留接纳状态供下次恢复重试。"""
|
||||
def record_planning_failure(
|
||||
self, *, task_id: str, lease_token: str, error: str
|
||||
) -> None:
|
||||
"""按有效租约记录规划失败,保留业务状态供恢复重试。"""
|
||||
...
|
||||
|
||||
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
|
||||
"""按登记顺序返回接纳或已规划的可恢复任务。"""
|
||||
def claim_task(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
owner_id: str,
|
||||
lease_seconds: int,
|
||||
) -> Optional[TransferAdmission]:
|
||||
"""为指定任务取得唯一执行租约;已有有效租约时返回 None。"""
|
||||
...
|
||||
|
||||
def discard_task(self, *, task_id: str) -> int:
|
||||
"""按稳定任务身份删除已经到达终态的登记。"""
|
||||
def claim_recoverable(
|
||||
self,
|
||||
*,
|
||||
owner_id: str,
|
||||
limit: int,
|
||||
lease_seconds: int,
|
||||
) -> list[TransferAdmission]:
|
||||
"""按登记顺序原子取得可恢复任务的唯一执行租约。"""
|
||||
...
|
||||
|
||||
def heartbeat(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
lease_seconds: int,
|
||||
) -> Optional[TransferAdmission]:
|
||||
"""仅为当前且未过期的 token 延长租约;过期 token 不得复活。"""
|
||||
...
|
||||
|
||||
def release_claim(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""按 token 释放当前 claim,并可记录可恢复错误。"""
|
||||
...
|
||||
|
||||
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅允许当前 lease owner 删除已经到达终态的登记。"""
|
||||
...
|
||||
|
||||
|
||||
|
||||
+644
-42
@@ -46,6 +46,7 @@ from app.application.transfer import (
|
||||
TransferAdmission,
|
||||
TransferFailureNotification,
|
||||
TransferFailureNotificationAggregator,
|
||||
TransferLeaseLostError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanningInput,
|
||||
TransferPlanningStateError,
|
||||
@@ -146,6 +147,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
|
||||
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
|
||||
_QUEUE_STOP_SENTINEL = object()
|
||||
_WORKER_LEASE_SECONDS = 120
|
||||
_LEASE_HEARTBEAT_INTERVAL_SECONDS = 30.0
|
||||
_RECOVERY_POLL_INTERVAL_SECONDS = 15.0
|
||||
_RECOVERY_CLAIM_LIMIT = 100
|
||||
|
||||
@staticmethod
|
||||
def _transfer_result_payload(
|
||||
@@ -254,9 +259,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._worker_lifecycle_lock = threading.RLock()
|
||||
self._worker_state_lock = threading.RLock()
|
||||
self._closing = False
|
||||
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
|
||||
# 恢复调度与租约续期均由整理链持有,关闭时与 worker 一并收敛。
|
||||
self._worker_owner_id = uuid.uuid4().hex
|
||||
self._owned_leases: Dict[str, Tuple[str, float]] = {}
|
||||
self._queued_lease_tokens: set[Tuple[str, str]] = set()
|
||||
self._replay_thread: Optional[threading.Thread] = None
|
||||
self._replay_stop_event = threading.Event()
|
||||
self._recovery_wakeup_event = threading.Event()
|
||||
self._lease_heartbeat_thread: Optional[threading.Thread] = None
|
||||
self._lease_heartbeat_stop_event = threading.Event()
|
||||
self._lease_release_thread: Optional[threading.Thread] = None
|
||||
self._active_tasks = 0
|
||||
self._processed_num = 0
|
||||
self._fail_num = 0
|
||||
@@ -366,6 +378,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
:param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数
|
||||
:return: 全部后台线程均已收敛时返回 True,否则返回 False
|
||||
"""
|
||||
self.__ensure_lease_runtime_state()
|
||||
deadline = time.monotonic() + max(0.0, timeout_seconds)
|
||||
if not self.__acquire_worker_lifecycle_lock(deadline):
|
||||
logger.error(
|
||||
@@ -378,7 +391,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._closing = True
|
||||
worker_threads = self.__request_worker_stop()
|
||||
replay_thread = self._replay_thread
|
||||
heartbeat_thread = self._lease_heartbeat_thread
|
||||
self._replay_stop_event.set()
|
||||
self._recovery_wakeup_event.set()
|
||||
|
||||
alive_workers = self.__join_threads(worker_threads, deadline)
|
||||
alive_replays = self.__join_threads(
|
||||
@@ -387,7 +402,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
with self._worker_state_lock:
|
||||
self._threads = []
|
||||
self._retiring_threads = alive_workers
|
||||
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
|
||||
if (
|
||||
replay_thread
|
||||
and replay_thread not in alive_replays
|
||||
and self._replay_thread is replay_thread
|
||||
):
|
||||
self._replay_thread = None
|
||||
|
||||
alive_threads = [*alive_workers, *alive_replays]
|
||||
@@ -398,6 +417,38 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
", ".join(thread.name for thread in alive_threads),
|
||||
)
|
||||
return False
|
||||
with self._worker_state_lock:
|
||||
release_thread = self.__start_lease_release_owner_locked(
|
||||
error="整理宿主关闭,释放未结算任务租约"
|
||||
)
|
||||
alive_releases = self.__join_threads(
|
||||
[release_thread] if release_thread else [], deadline
|
||||
)
|
||||
if alive_releases:
|
||||
logger.error(
|
||||
"整理租约释放线程未在 %.1f 秒内收敛,heartbeat 保持运行:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_releases),
|
||||
)
|
||||
return False
|
||||
with self._worker_state_lock:
|
||||
if self._lease_release_thread is release_thread:
|
||||
self._lease_release_thread = None
|
||||
self._lease_heartbeat_stop_event.set()
|
||||
alive_heartbeats = self.__join_threads(
|
||||
[heartbeat_thread] if heartbeat_thread else [], deadline
|
||||
)
|
||||
if heartbeat_thread and not alive_heartbeats:
|
||||
with self._worker_state_lock:
|
||||
if self._lease_heartbeat_thread is heartbeat_thread:
|
||||
self._lease_heartbeat_thread = None
|
||||
if alive_heartbeats:
|
||||
logger.error(
|
||||
"整理租约续期线程未在 %.1f 秒内收敛:%s",
|
||||
max(0.0, timeout_seconds),
|
||||
", ".join(thread.name for thread in alive_heartbeats),
|
||||
)
|
||||
return False
|
||||
logger.info("文件整理 worker 与待处理回放线程已关闭")
|
||||
return True
|
||||
finally:
|
||||
@@ -885,6 +936,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
if self._closing:
|
||||
logger.warning("文件整理链已关闭,拒绝新的队列任务")
|
||||
return False
|
||||
if isinstance(task.lease_token, str) and task.lease_token:
|
||||
return self.__enqueue_claimed_task(task)
|
||||
return self._transfer_queue_service().put(task, self.__default_callback)
|
||||
|
||||
def _transfer_queue_service(self) -> TransferQueueService:
|
||||
@@ -902,39 +955,369 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
|
||||
def replay_pending(self) -> None:
|
||||
"""
|
||||
回放上次进程退出时仍未整理完的文件。
|
||||
启动唯一恢复调度 owner,并唤醒一次即时恢复扫描。
|
||||
|
||||
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
|
||||
挂死状态,同步执行会把整个启动流程堵住。
|
||||
启动回放、同进程入队补偿和租约过期接管都经由这个入口。调度线程只负责
|
||||
claim 和重新入队;实际业务仍由普通整理 worker 执行。
|
||||
"""
|
||||
self.__ensure_recovery_scheduler(immediate=True)
|
||||
|
||||
def __ensure_recovery_scheduler(self, *, immediate: bool) -> None:
|
||||
"""确保唯一恢复调度 owner 存在,并只为显式请求执行即时扫描。"""
|
||||
self.__ensure_lease_runtime_state()
|
||||
with self._worker_state_lock:
|
||||
if self._closing:
|
||||
logger.info("文件整理链正在关闭,跳过待处理文件回放")
|
||||
return
|
||||
self.__start_lease_heartbeat_owner_locked()
|
||||
if self._replay_thread and self._replay_thread.is_alive():
|
||||
logger.info("待处理文件回放已在运行,跳过重复启动")
|
||||
if immediate:
|
||||
self._recovery_wakeup_event.set()
|
||||
return
|
||||
stop_event = threading.Event()
|
||||
thread = threading.Thread(
|
||||
target=self.__run_replay_pending,
|
||||
args=(stop_event,),
|
||||
args=(self._replay_stop_event, immediate),
|
||||
name="MoviePilot-TransferReplay",
|
||||
daemon=True,
|
||||
)
|
||||
self._replay_stop_event = stop_event
|
||||
self._replay_thread = thread
|
||||
if immediate:
|
||||
self._recovery_wakeup_event.set()
|
||||
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
|
||||
thread.start()
|
||||
|
||||
def __run_replay_pending(self, stop_event: threading.Event) -> None:
|
||||
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
|
||||
def __run_replay_pending(
|
||||
self,
|
||||
stop_event: threading.Event,
|
||||
initial_immediate: bool = True,
|
||||
) -> None:
|
||||
"""按初次扫描意图持续恢复,失败兜底新建 owner 时先等待固定轮询。"""
|
||||
try:
|
||||
self.__replay_pending(stop_event)
|
||||
if not initial_immediate:
|
||||
self._recovery_wakeup_event.wait(
|
||||
timeout=self._RECOVERY_POLL_INTERVAL_SECONDS
|
||||
)
|
||||
while not stop_event.is_set():
|
||||
self._recovery_wakeup_event.clear()
|
||||
self.__replay_pending(stop_event)
|
||||
self._recovery_wakeup_event.wait(
|
||||
timeout=self._RECOVERY_POLL_INTERVAL_SECONDS
|
||||
)
|
||||
finally:
|
||||
with self._worker_state_lock:
|
||||
if self._replay_thread is threading.current_thread():
|
||||
self._replay_thread = None
|
||||
|
||||
def __ensure_lease_runtime_state(self) -> None:
|
||||
"""为绕过构造器的兼容调用补齐进程 owner 与租约线程状态。"""
|
||||
if not hasattr(self, "_worker_owner_id"):
|
||||
self._worker_owner_id = uuid.uuid4().hex
|
||||
if not hasattr(self, "_owned_leases"):
|
||||
self._owned_leases = {}
|
||||
if not hasattr(self, "_queued_lease_tokens"):
|
||||
self._queued_lease_tokens = set()
|
||||
if not hasattr(self, "_recovery_wakeup_event"):
|
||||
self._recovery_wakeup_event = threading.Event()
|
||||
if not hasattr(self, "_lease_heartbeat_thread"):
|
||||
self._lease_heartbeat_thread = None
|
||||
if not hasattr(self, "_lease_heartbeat_stop_event"):
|
||||
self._lease_heartbeat_stop_event = threading.Event()
|
||||
if not hasattr(self, "_lease_release_thread"):
|
||||
self._lease_release_thread = None
|
||||
if not hasattr(self, "_replay_thread"):
|
||||
self._replay_thread = None
|
||||
if not hasattr(self, "_replay_stop_event"):
|
||||
self._replay_stop_event = threading.Event()
|
||||
if not hasattr(self, "_worker_state_lock"):
|
||||
self._worker_state_lock = threading.RLock()
|
||||
if not hasattr(self, "_closing"):
|
||||
self._closing = False
|
||||
|
||||
def __start_lease_heartbeat_owner_locked(self) -> None:
|
||||
"""在状态锁内确保当前进程只有一个租约续期线程。"""
|
||||
heartbeat_thread = self._lease_heartbeat_thread
|
||||
if heartbeat_thread and heartbeat_thread.is_alive():
|
||||
return
|
||||
heartbeat_thread = threading.Thread(
|
||||
target=self.__run_lease_heartbeat,
|
||||
args=(self._lease_heartbeat_stop_event,),
|
||||
name="MoviePilot-TransferLeaseHeartbeat",
|
||||
daemon=True,
|
||||
)
|
||||
self._lease_heartbeat_thread = heartbeat_thread
|
||||
heartbeat_thread.start()
|
||||
|
||||
def __ensure_lease_heartbeat_owner(self) -> None:
|
||||
"""按需启动进程级租约续期 owner,供启动前到达的普通任务使用。"""
|
||||
self.__ensure_lease_runtime_state()
|
||||
with self._worker_state_lock:
|
||||
if not self._closing:
|
||||
self.__start_lease_heartbeat_owner_locked()
|
||||
|
||||
def __run_lease_heartbeat(self, stop_event: threading.Event) -> None:
|
||||
"""按固定周期续期本进程已 claim 且尚未结算的任务。"""
|
||||
try:
|
||||
while not stop_event.wait(self._LEASE_HEARTBEAT_INTERVAL_SECONDS):
|
||||
self.__heartbeat_owned_leases()
|
||||
finally:
|
||||
with self._worker_state_lock:
|
||||
if self._lease_heartbeat_thread is threading.current_thread():
|
||||
self._lease_heartbeat_thread = None
|
||||
|
||||
def __heartbeat_owned_leases(self) -> None:
|
||||
"""经 Application Port 续期所有排队中或执行中的任务租约。"""
|
||||
self.__ensure_lease_runtime_state()
|
||||
with self._worker_state_lock:
|
||||
owned_leases = list(self._owned_leases.items())
|
||||
for task_id, (lease_token, deadline) in owned_leases:
|
||||
try:
|
||||
admission = self._transfer_admissions.heartbeat(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
lease_seconds=self._WORKER_LEASE_SECONDS,
|
||||
)
|
||||
except Exception as err:
|
||||
if time.monotonic() >= deadline:
|
||||
self.__forget_owned_lease(task_id, lease_token)
|
||||
logger.error(
|
||||
"整理任务租约续期持续失败并已超过本地期限:%s - %s",
|
||||
task_id,
|
||||
err,
|
||||
)
|
||||
else:
|
||||
logger.error(f"整理任务租约续期失败:{task_id} - {err}")
|
||||
continue
|
||||
if (
|
||||
admission is None
|
||||
or admission.lease_owner != self._worker_owner_id
|
||||
or admission.lease_token != lease_token
|
||||
):
|
||||
self.__forget_owned_lease(task_id, lease_token)
|
||||
logger.error(f"整理任务租约已失效或被接管:{task_id}")
|
||||
continue
|
||||
with self._worker_state_lock:
|
||||
current = self._owned_leases.get(task_id)
|
||||
if current and current[0] == lease_token:
|
||||
self._owned_leases[task_id] = (
|
||||
lease_token,
|
||||
time.monotonic() + self._WORKER_LEASE_SECONDS,
|
||||
)
|
||||
|
||||
def __bind_claimed_admission(
|
||||
self,
|
||||
task: TransferTask,
|
||||
admission: TransferAdmission,
|
||||
) -> None:
|
||||
"""校验仓储 claim 投影并把 token 私有绑定到执行任务。"""
|
||||
if (
|
||||
admission.task_id != task.admission_task_id
|
||||
):
|
||||
raise TransferLeaseLostError(
|
||||
f"整理任务 claim 投影无效:{admission.task_id}"
|
||||
)
|
||||
self.__register_claimed_admission(admission)
|
||||
assert admission.lease_owner is not None
|
||||
assert admission.lease_token is not None
|
||||
task.bind_execution_lease(
|
||||
owner_id=admission.lease_owner,
|
||||
lease_token=admission.lease_token,
|
||||
)
|
||||
|
||||
def __register_claimed_admission(
|
||||
self,
|
||||
admission: TransferAdmission,
|
||||
) -> None:
|
||||
"""把仓储 claim 加入续期集合,覆盖恢复构造阶段可能发生的同步 I/O。"""
|
||||
if (
|
||||
admission.lease_owner != self._worker_owner_id
|
||||
or not admission.lease_token
|
||||
):
|
||||
raise TransferLeaseLostError(
|
||||
f"整理任务 claim 投影无效:{admission.task_id}"
|
||||
)
|
||||
with self._worker_state_lock:
|
||||
self._owned_leases[admission.task_id] = (
|
||||
admission.lease_token,
|
||||
time.monotonic() + self._WORKER_LEASE_SECONDS,
|
||||
)
|
||||
self.__ensure_lease_heartbeat_owner()
|
||||
|
||||
def __forget_owned_lease(self, task_id: str, lease_token: str) -> None:
|
||||
"""仅在 token 仍匹配时移除本进程租约镜像,避免删掉新接管记录。"""
|
||||
with self._worker_state_lock:
|
||||
current = self._owned_leases.get(task_id)
|
||||
if current and current[0] == lease_token:
|
||||
self._owned_leases.pop(task_id, None)
|
||||
self._queued_lease_tokens.discard((task_id, lease_token))
|
||||
|
||||
def __is_claimed_task_enqueued(self, task_id: str, lease_token: str) -> bool:
|
||||
"""返回指定 claim 是否已经成功进入普通 worker 队列。"""
|
||||
with self._worker_state_lock:
|
||||
return (task_id, lease_token) in self._queued_lease_tokens
|
||||
|
||||
def __owns_lease(self, task_id: str, lease_token: Optional[str]) -> bool:
|
||||
"""返回本地续期镜像是否仍持有指定 token。"""
|
||||
if not lease_token:
|
||||
return False
|
||||
with self._worker_state_lock:
|
||||
current = self._owned_leases.get(task_id)
|
||||
return bool(current and current[0] == lease_token)
|
||||
|
||||
def __assert_owned_lease(self, task: TransferTask) -> None:
|
||||
"""拒绝无 token、已过本地期限或已被 heartbeat 判失效的任务推进。"""
|
||||
if task.preview:
|
||||
return
|
||||
task_id = task.admission_task_id
|
||||
lease_token = task.lease_token
|
||||
if (
|
||||
not task_id
|
||||
or not lease_token
|
||||
or task.lease_owner != self._worker_owner_id
|
||||
):
|
||||
raise TransferLeaseLostError("整理任务缺少当前进程的有效执行租约")
|
||||
with self._worker_state_lock:
|
||||
current = self._owned_leases.get(task_id)
|
||||
if (
|
||||
current is None
|
||||
or current[0] != lease_token
|
||||
or current[1] <= time.monotonic()
|
||||
):
|
||||
if current and current[0] == lease_token:
|
||||
self._owned_leases.pop(task_id, None)
|
||||
raise TransferLeaseLostError(f"整理任务租约已经失效:{task_id}")
|
||||
|
||||
def __claim_task_for_execution(self, task: TransferTask) -> None:
|
||||
"""让普通队列任务在业务执行前取得唯一租约,恢复任务复用既有 token。"""
|
||||
if task.preview:
|
||||
return
|
||||
self.__ensure_lease_runtime_state()
|
||||
if task.lease_token:
|
||||
self.__assert_owned_lease(task)
|
||||
return
|
||||
if not task.admission_task_id:
|
||||
admitted = self.__admit_transfer(task)
|
||||
task.bind_admission_task_id(admitted.task_id)
|
||||
task_id = task.admission_task_id
|
||||
if task_id is None:
|
||||
raise TransferLeaseLostError("整理任务准入后仍缺少 durable 身份")
|
||||
claimed = self._transfer_admissions.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id=self._worker_owner_id,
|
||||
lease_seconds=self._WORKER_LEASE_SECONDS,
|
||||
)
|
||||
if claimed is None:
|
||||
raise TransferLeaseLostError(
|
||||
f"整理任务已由其他 worker claim:{task_id}"
|
||||
)
|
||||
try:
|
||||
self.__bind_claimed_admission(task, claimed)
|
||||
except Exception as err:
|
||||
self.__release_admission_claim(claimed, error=str(err))
|
||||
raise
|
||||
|
||||
def __release_task_claim(
|
||||
self,
|
||||
task: TransferTask,
|
||||
*,
|
||||
error: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""按 task token 释放未到终态的 claim,并按固定轮询等待恢复。"""
|
||||
task_id = task.admission_task_id if task else None
|
||||
lease_token = task.lease_token if task else None
|
||||
if not task_id or not lease_token:
|
||||
return False
|
||||
try:
|
||||
return self._transfer_admissions.release_claim(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"释放整理任务租约失败:{task_id} - {err}")
|
||||
return False
|
||||
finally:
|
||||
self.__forget_owned_lease(task_id, lease_token)
|
||||
self.__ensure_recovery_scheduler(immediate=False)
|
||||
|
||||
def __release_admission_claim(
|
||||
self,
|
||||
admission: TransferAdmission,
|
||||
*,
|
||||
error: Optional[str] = None,
|
||||
) -> None:
|
||||
"""释放尚未绑定或成功入队的恢复 claim。"""
|
||||
if not admission.lease_token:
|
||||
return
|
||||
try:
|
||||
self._transfer_admissions.release_claim(
|
||||
task_id=admission.task_id,
|
||||
lease_token=admission.lease_token,
|
||||
error=error,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"释放恢复任务租约失败:{admission.task_id} - {err}")
|
||||
finally:
|
||||
self.__forget_owned_lease(admission.task_id, admission.lease_token)
|
||||
|
||||
def __release_all_owned_leases(self, *, error: str) -> None:
|
||||
"""在执行 owner 全部收敛后释放剩余 claim,避免关停后等待租约自然过期。"""
|
||||
with self._worker_state_lock:
|
||||
owned_leases = list(self._owned_leases.items())
|
||||
for task_id, (lease_token, _deadline) in owned_leases:
|
||||
try:
|
||||
self._transfer_admissions.release_claim(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"关闭时释放整理任务租约失败:{task_id} - {err}")
|
||||
finally:
|
||||
self.__forget_owned_lease(task_id, lease_token)
|
||||
|
||||
def __start_lease_release_owner_locked(
|
||||
self, *, error: str
|
||||
) -> Optional[threading.Thread]:
|
||||
"""启动并持有唯一租约释放线程,使同步数据库阻塞不突破关闭预算。"""
|
||||
release_thread = self._lease_release_thread
|
||||
if release_thread is not None:
|
||||
return release_thread
|
||||
if not self._owned_leases:
|
||||
return None
|
||||
release_thread = threading.Thread(
|
||||
target=self.__release_all_owned_leases,
|
||||
kwargs={"error": error},
|
||||
name="MoviePilot-TransferLeaseRelease",
|
||||
daemon=True,
|
||||
)
|
||||
self._lease_release_thread = release_thread
|
||||
release_thread.start()
|
||||
return release_thread
|
||||
|
||||
def __enqueue_claimed_task(self, task: TransferTask) -> bool:
|
||||
"""把已 claim 的恢复任务送入普通队列,禁止再次准入或二次 claim。"""
|
||||
self.__assert_owned_lease(task)
|
||||
if not self.__put_to_jobview(task):
|
||||
return False
|
||||
try:
|
||||
self._register_scrape_batch_task(task)
|
||||
assert task.admission_task_id is not None
|
||||
assert task.lease_token is not None
|
||||
with self._worker_state_lock:
|
||||
self._queued_lease_tokens.add(
|
||||
(task.admission_task_id, task.lease_token)
|
||||
)
|
||||
self._queue.put(
|
||||
TransferQueue(task=task, callback=self.__default_callback)
|
||||
)
|
||||
except Exception as err:
|
||||
try:
|
||||
self.__record_enqueue_failure(task, err)
|
||||
finally:
|
||||
self.jobview.remove_task(task.fileitem)
|
||||
raise
|
||||
return True
|
||||
|
||||
def __replay_pending(
|
||||
self, stop_event: Optional[threading.Event] = None
|
||||
) -> None:
|
||||
@@ -948,17 +1331,35 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
stop_event = stop_event or threading.Event()
|
||||
if stop_event.is_set():
|
||||
return
|
||||
self.__ensure_lease_runtime_state()
|
||||
try:
|
||||
pendings = self._transfer_admissions.list_recoverable()
|
||||
pendings = self._transfer_admissions.claim_recoverable(
|
||||
owner_id=self._worker_owner_id,
|
||||
limit=self._RECOVERY_CLAIM_LIMIT,
|
||||
lease_seconds=self._WORKER_LEASE_SECONDS,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"读取待整理文件登记失败:{err}")
|
||||
return
|
||||
if not pendings:
|
||||
return
|
||||
try:
|
||||
for admission in pendings:
|
||||
self.__register_claimed_admission(admission)
|
||||
except Exception as err:
|
||||
logger.error(f"登记恢复任务租约失败:{err}")
|
||||
for claimed in pendings:
|
||||
self.__release_admission_claim(claimed, error=str(err))
|
||||
return
|
||||
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
|
||||
replayed = 0
|
||||
for admission in pendings:
|
||||
for index, admission in enumerate(pendings):
|
||||
if stop_event.is_set():
|
||||
for unprocessed in pendings[index:]:
|
||||
self.__release_admission_claim(
|
||||
unprocessed,
|
||||
error="整理宿主关闭,恢复任务尚未入队",
|
||||
)
|
||||
break
|
||||
storage = admission.storage
|
||||
src_path = admission.src_path
|
||||
@@ -970,17 +1371,50 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
)
|
||||
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
|
||||
if stop_event.is_set():
|
||||
self.__release_admission_claim(
|
||||
admission,
|
||||
error="整理宿主关闭,恢复任务尚未入队",
|
||||
)
|
||||
for unprocessed in pendings[index + 1:]:
|
||||
self.__release_admission_claim(
|
||||
unprocessed,
|
||||
error="整理宿主关闭,恢复任务尚未入队",
|
||||
)
|
||||
break
|
||||
if not fileitem:
|
||||
if should_discard:
|
||||
# 源文件确认已消失,注销登记避免每次启动重复回放
|
||||
self._transfer_admissions.discard_task(
|
||||
task_id=admission.task_id
|
||||
lease_token = admission.lease_token
|
||||
if lease_token is None:
|
||||
raise TransferLeaseLostError(
|
||||
f"恢复任务缺少 lease token:{admission.task_id}"
|
||||
)
|
||||
discarded = self._transfer_admissions.discard_claimed(
|
||||
task_id=admission.task_id,
|
||||
lease_token=lease_token,
|
||||
)
|
||||
if not discarded:
|
||||
logger.warning(
|
||||
f"恢复任务终态注销被 CAS 拒绝:{admission.task_id}"
|
||||
)
|
||||
self.__forget_owned_lease(
|
||||
admission.task_id,
|
||||
lease_token,
|
||||
)
|
||||
else:
|
||||
self.__release_admission_claim(
|
||||
admission,
|
||||
error="恢复源文件暂时不可读取",
|
||||
)
|
||||
continue
|
||||
if admission.checkpoint:
|
||||
if self.__queue_planned_replay(fileitem, admission):
|
||||
replayed += 1
|
||||
else:
|
||||
self.__release_admission_claim(
|
||||
admission,
|
||||
error="恢复任务未进入内存队列",
|
||||
)
|
||||
continue
|
||||
planning_input = admission.planning_input
|
||||
if planning_input and (
|
||||
@@ -990,6 +1424,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
):
|
||||
if self.__queue_accepted_replay(fileitem, admission):
|
||||
replayed += 1
|
||||
else:
|
||||
self.__release_admission_claim(
|
||||
admission,
|
||||
error="恢复任务未进入内存队列",
|
||||
)
|
||||
continue
|
||||
replay_kwargs = self.__build_replay_kwargs(planning_input)
|
||||
self._execute_transfer(
|
||||
@@ -997,9 +1436,21 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
recovery_admission=admission,
|
||||
**replay_kwargs,
|
||||
)
|
||||
replayed += 1
|
||||
assert admission.lease_token is not None
|
||||
if self.__is_claimed_task_enqueued(
|
||||
admission.task_id,
|
||||
admission.lease_token,
|
||||
):
|
||||
replayed += 1
|
||||
else:
|
||||
self.__release_admission_claim(
|
||||
admission,
|
||||
error="旧恢复入口未产生可执行队列任务",
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
|
||||
if self.__owns_lease(admission.task_id, admission.lease_token):
|
||||
self.__release_admission_claim(admission, error=str(err))
|
||||
if stop_event.is_set():
|
||||
logger.info(
|
||||
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
|
||||
@@ -1116,6 +1567,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
preview=False,
|
||||
)
|
||||
task.bind_admission_task_id(admission.task_id)
|
||||
self.__bind_claimed_admission(task, admission)
|
||||
task.bind_planning_input(planning_input)
|
||||
if planning_input.mediainfo:
|
||||
task.mark_planning_context_restored()
|
||||
@@ -1168,6 +1620,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
preview=False,
|
||||
)
|
||||
task.bind_admission_task_id(admission.task_id)
|
||||
self.__bind_claimed_admission(task, admission)
|
||||
task.bind_planning_input(checkpoint.planning_input)
|
||||
task.bind_plan_checkpoint(checkpoint)
|
||||
self.__restore_planned_task(task)
|
||||
@@ -1442,11 +1895,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
error: object,
|
||||
) -> None:
|
||||
"""记录 checkpoint 前失败,保留 accepted 任务供后续重新规划。"""
|
||||
if task.preview or task.plan_checkpoint or not task.admission_task_id:
|
||||
if (
|
||||
task.preview
|
||||
or task.plan_checkpoint
|
||||
or not task.admission_task_id
|
||||
or not task.lease_token
|
||||
):
|
||||
return
|
||||
try:
|
||||
self._transfer_admissions.record_planning_failure(
|
||||
task_id=task.admission_task_id,
|
||||
lease_token=task.lease_token,
|
||||
error=str(error),
|
||||
)
|
||||
except Exception as record_error:
|
||||
@@ -1488,9 +1947,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
"""先提交冻结 provider 调用,空结果时再提交并执行宿主计划。"""
|
||||
planning_input = task.planning_input or self.__build_planning_input(task)
|
||||
task.bind_planning_input(planning_input)
|
||||
if not task.preview and not task.admission_task_id:
|
||||
admission = self.__admit_transfer(task)
|
||||
task.bind_admission_task_id(admission.task_id)
|
||||
if not task.preview:
|
||||
self.__claim_task_for_execution(task)
|
||||
self.__assert_owned_lease(task)
|
||||
|
||||
checkpoint = task.plan_checkpoint
|
||||
if checkpoint is None:
|
||||
@@ -1540,6 +1999,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
raise
|
||||
|
||||
self.__restore_planned_task(task)
|
||||
self.__assert_owned_lease(task)
|
||||
|
||||
legacy_result = self.__execute_legacy_transfer_providers(
|
||||
task,
|
||||
@@ -1571,6 +2031,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self.__record_checkpoint_failure(task, error)
|
||||
raise
|
||||
|
||||
self.__assert_owned_lease(task)
|
||||
result = self.execute_transfer_plan(
|
||||
checkpoint,
|
||||
meta=task.meta,
|
||||
@@ -1623,8 +2084,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
return checkpoint
|
||||
if not task.admission_task_id:
|
||||
raise RuntimeError("整理计划提交前缺少持久任务身份")
|
||||
self.__assert_owned_lease(task)
|
||||
assert task.lease_token is not None
|
||||
persisted = self._transfer_admissions.checkpoint_plan(
|
||||
task_id=task.admission_task_id,
|
||||
lease_token=task.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -1638,11 +2102,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""记录当前 checkpoint 阶段失败并保留原状态供重启恢复。"""
|
||||
if task.preview or not task.admission_task_id:
|
||||
if task.preview or not task.admission_task_id or not task.lease_token:
|
||||
return
|
||||
try:
|
||||
self._transfer_admissions.record_planning_failure(
|
||||
task_id=task.admission_task_id,
|
||||
lease_token=task.lease_token,
|
||||
error=str(error),
|
||||
)
|
||||
setattr(error, "_transfer_planning_failure_recorded", True)
|
||||
@@ -1790,6 +2255,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error(f"旧整理兼容命令执行失败:{error}")
|
||||
self.__release_task_claim(task, error=str(error))
|
||||
return TransferInfo(
|
||||
success=False,
|
||||
message=str(error),
|
||||
@@ -1797,7 +2263,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
)
|
||||
self.__discard_pending(task)
|
||||
if not self.__discard_pending(task):
|
||||
message = "旧整理兼容命令已执行,但 durable 终态结算失去租约"
|
||||
logger.error(message)
|
||||
return TransferInfo(
|
||||
success=False,
|
||||
message=message,
|
||||
fileitem=fileitem,
|
||||
fail_list=[fileitem.path],
|
||||
transfer_type=transfer_type,
|
||||
)
|
||||
return result
|
||||
|
||||
def __record_enqueue_failure(
|
||||
@@ -1805,9 +2280,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
task: TransferTask,
|
||||
error: Exception,
|
||||
) -> None:
|
||||
"""记录内存入队失败并撤销该任务的批次占位。"""
|
||||
"""按是否已经 claim 选择 fencing 失败记录,并撤销批次占位。"""
|
||||
try:
|
||||
if task.admission_task_id:
|
||||
if task.lease_token:
|
||||
self.__release_task_claim(task, error=str(error))
|
||||
elif task.admission_task_id:
|
||||
self._transfer_admissions.record_enqueue_failure(
|
||||
task_id=task.admission_task_id,
|
||||
error=str(error),
|
||||
@@ -1819,8 +2296,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
)
|
||||
finally:
|
||||
self._finish_scrape_batch_task(task)
|
||||
self.__ensure_recovery_scheduler(immediate=False)
|
||||
|
||||
def __discard_pending(self, task: TransferTask):
|
||||
def __discard_pending(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
|
||||
|
||||
@@ -1829,16 +2307,34 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
:param task: 任务信息
|
||||
"""
|
||||
if not task or not task.admission_task_id:
|
||||
return
|
||||
try:
|
||||
self._transfer_admissions.discard_task(
|
||||
task_id=task.admission_task_id
|
||||
return True
|
||||
if not task.lease_token:
|
||||
logger.error(
|
||||
f"整理任务缺少 lease token,拒绝伪装 durable 终态:"
|
||||
f"{task.admission_task_id}"
|
||||
)
|
||||
return False
|
||||
discarded = 0
|
||||
try:
|
||||
discarded = self._transfer_admissions.discard_claimed(
|
||||
task_id=task.admission_task_id,
|
||||
lease_token=task.lease_token,
|
||||
)
|
||||
if not discarded:
|
||||
logger.error(
|
||||
f"整理任务终态注销被 CAS 拒绝:{task.admission_task_id}"
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
"注销整理任务 durable admission 失败: "
|
||||
f"{task.admission_task_id} - {err}"
|
||||
)
|
||||
finally:
|
||||
self.__forget_owned_lease(
|
||||
task.admission_task_id,
|
||||
task.lease_token,
|
||||
)
|
||||
return bool(discarded)
|
||||
|
||||
def __put_to_jobview(self, task: TransferTask) -> bool:
|
||||
"""
|
||||
@@ -1905,13 +2401,23 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
if marker:
|
||||
marker(task)
|
||||
|
||||
def __finish_job_execution(self, task: TransferTask, *, terminal: bool = True):
|
||||
"""结束内存执行;仅确定到达业务终态时注销 durable 记录。"""
|
||||
def __finish_job_execution(
|
||||
self,
|
||||
task: TransferTask,
|
||||
*,
|
||||
terminal: bool = True,
|
||||
terminal_settlement: Optional[bool] = None,
|
||||
) -> bool:
|
||||
"""结束内存执行,并复用成功回调前已经提交的 durable 终态结果。"""
|
||||
marker = getattr(self.jobview, "finish_execution", None)
|
||||
if marker:
|
||||
marker(task)
|
||||
if terminal:
|
||||
self.__discard_pending(task)
|
||||
if terminal_settlement is not None:
|
||||
return terminal_settlement
|
||||
return self.__discard_pending(task)
|
||||
self.__release_task_claim(task)
|
||||
return True
|
||||
|
||||
def __expire_stale_transfer_tasks(self):
|
||||
"""清理外部接管后失去状态心跳的运行中整理任务。"""
|
||||
@@ -2001,6 +2507,32 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
continue
|
||||
|
||||
if task.admission_task_id and task.lease_token:
|
||||
with self._worker_state_lock:
|
||||
self._queued_lease_tokens.discard(
|
||||
(task.admission_task_id, task.lease_token)
|
||||
)
|
||||
try:
|
||||
self.__claim_task_for_execution(task)
|
||||
except TransferLeaseLostError as err:
|
||||
logger.info(f"跳过未取得执行租约的整理任务:{err}")
|
||||
self.__release_task_claim(task, error=str(err))
|
||||
self.jobview.try_remove_job(task)
|
||||
self._finish_scrape_batch_task(task)
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
continue
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"整理任务 claim 失败,保留 durable admission:{err}"
|
||||
)
|
||||
self.jobview.try_remove_job(task)
|
||||
self._finish_scrape_batch_task(task)
|
||||
self._queue.task_done()
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
self.__ensure_recovery_scheduler(immediate=False)
|
||||
continue
|
||||
|
||||
# 文件信息
|
||||
fileitem = task.fileitem
|
||||
|
||||
@@ -2030,6 +2562,29 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._active_tasks += 1
|
||||
|
||||
terminal = False
|
||||
terminal_settlement: Optional[bool] = None
|
||||
state = False
|
||||
err_msg = ""
|
||||
|
||||
def callback_after_terminal_settlement(
|
||||
callback_task: TransferTask,
|
||||
transferinfo: TransferInfo,
|
||||
) -> Tuple[bool, str]:
|
||||
"""成功结果先提交 durable 终态,再执行历史、事件和通知回调。"""
|
||||
nonlocal terminal_settlement
|
||||
if transferinfo.success and not callback_task.preview:
|
||||
terminal_settlement = self.__discard_pending(callback_task)
|
||||
if not terminal_settlement:
|
||||
self.__fail_transfer_task(callback_task)
|
||||
return False, "整理任务 durable 终态结算失去租约"
|
||||
if item.callback:
|
||||
callback_result: Tuple[bool, str] = item.callback(
|
||||
callback_task,
|
||||
transferinfo,
|
||||
)
|
||||
return callback_result
|
||||
return transferinfo.success, transferinfo.message or ""
|
||||
|
||||
try:
|
||||
self.__start_job_execution(task)
|
||||
# 更新进度
|
||||
@@ -2044,7 +2599,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
)
|
||||
# 整理
|
||||
state, err_msg = self.__handle_transfer(
|
||||
task=task, callback=item.callback
|
||||
task=task,
|
||||
callback=callback_after_terminal_settlement,
|
||||
)
|
||||
terminal = task.plan_checkpoint is not None
|
||||
|
||||
@@ -2063,6 +2619,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
text=__process_msg,
|
||||
)
|
||||
except Exception as e:
|
||||
if terminal_settlement is not None:
|
||||
terminal = True
|
||||
logger.error(
|
||||
f"{fileitem.name} 整理任务处理出现错误:{e} - {traceback.format_exc()}"
|
||||
)
|
||||
@@ -2071,12 +2629,25 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self._processed_num += 1
|
||||
self._fail_num += 1
|
||||
finally:
|
||||
self.__finish_job_execution(task, terminal=terminal)
|
||||
self._queue.task_done()
|
||||
with task_lock:
|
||||
# 减少运行中的任务数
|
||||
self._active_tasks -= 1
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
durable_settled = False
|
||||
try:
|
||||
durable_settled = self.__finish_job_execution(
|
||||
task,
|
||||
terminal=terminal,
|
||||
terminal_settlement=terminal_settlement,
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"整理任务终态结算异常:{task.admission_task_id} - {err}"
|
||||
)
|
||||
finally:
|
||||
self._queue.task_done()
|
||||
with task_lock:
|
||||
# 减少运行中的任务数
|
||||
self._active_tasks -= 1
|
||||
if terminal and state and not durable_settled:
|
||||
self._fail_num += 1
|
||||
self.__settle_transfer_progress_if_idle()
|
||||
|
||||
except queue.Empty:
|
||||
# 即使队列空了,如果还有任务在运行,也不应该结束进度
|
||||
@@ -3548,6 +4119,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
and file_item.path == recovery_admission.src_path
|
||||
):
|
||||
transfer_task.bind_admission_task_id(recovery_admission.task_id)
|
||||
self.__bind_claimed_admission(
|
||||
transfer_task,
|
||||
recovery_admission,
|
||||
)
|
||||
if recovery_admission.planning_input:
|
||||
transfer_task.bind_planning_input(
|
||||
recovery_admission.planning_input
|
||||
@@ -3651,14 +4226,37 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
},
|
||||
)
|
||||
terminal = False
|
||||
terminal_settlement: Optional[bool] = None
|
||||
|
||||
def callback_after_terminal_settlement(
|
||||
callback_task: TransferTask,
|
||||
transferinfo: TransferInfo,
|
||||
) -> Tuple[bool, str]:
|
||||
"""同步成功结果先结算 durable 行,再执行兼容回调副作用。"""
|
||||
nonlocal terminal_settlement
|
||||
if transferinfo.success and not callback_task.preview:
|
||||
terminal_settlement = self.__discard_pending(callback_task)
|
||||
if not terminal_settlement:
|
||||
self.__fail_transfer_task(callback_task)
|
||||
return False, "整理任务 durable 终态结算失去租约"
|
||||
callback = (
|
||||
_preview_callback
|
||||
if preview
|
||||
else self.__default_callback
|
||||
)
|
||||
return callback(callback_task, transferinfo)
|
||||
|
||||
try:
|
||||
self.__claim_task_for_execution(transfer_task)
|
||||
self.__start_job_execution(transfer_task)
|
||||
state, err_msg = self.__handle_transfer(
|
||||
task=transfer_task,
|
||||
callback=_preview_callback if preview else self.__default_callback,
|
||||
callback=callback_after_terminal_settlement,
|
||||
)
|
||||
terminal = bool(preview or transfer_task.plan_checkpoint is not None)
|
||||
except Exception as e:
|
||||
if terminal_settlement is not None:
|
||||
terminal = True
|
||||
logger.error(
|
||||
f"{transfer_task.fileitem.name} 整理任务处理出现错误:"
|
||||
f"{e} - {traceback.format_exc()}"
|
||||
@@ -3667,10 +4265,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
|
||||
self.__fail_transfer_task(transfer_task)
|
||||
state, err_msg = False, str(e)
|
||||
finally:
|
||||
self.__finish_job_execution(
|
||||
durable_settled = self.__finish_job_execution(
|
||||
transfer_task,
|
||||
terminal=terminal,
|
||||
terminal_settlement=terminal_settlement,
|
||||
)
|
||||
if terminal and not durable_settled:
|
||||
state = False
|
||||
err_msg = "整理任务 durable 终态结算失去租约"
|
||||
if not state:
|
||||
all_success = False
|
||||
logger.warn(f"{transfer_task.fileitem.name} {err_msg}")
|
||||
|
||||
+290
-26
@@ -1,7 +1,9 @@
|
||||
"""整理任务持久准入端口的 SQLAlchemy 适配器。"""
|
||||
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from datetime import datetime
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from json import JSONDecodeError
|
||||
from typing import Optional
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -14,6 +16,8 @@ from app.application.transfer import (
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TransferAdmission,
|
||||
TransferAdmissionConflictError,
|
||||
TransferAdmissionProjectionError,
|
||||
TransferLeaseLostError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanningInput,
|
||||
TransferPlanningStateError,
|
||||
@@ -22,10 +26,14 @@ from app.db.models.transferpending import TransferPending
|
||||
from app.db.oper.transferpending import TransferPendingOper
|
||||
from app.db.uow import SqlAlchemyUnitOfWork
|
||||
|
||||
_diagnostic_logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TransactionalTransferAdmissionRepository:
|
||||
"""以短生命周期 Session 实现整理任务持久准入端口。"""
|
||||
|
||||
_MAX_RECOVERY_SCAN_TASKS = 5000
|
||||
|
||||
def __init__(self, session_factory: Callable[[], Session]) -> None:
|
||||
"""保存由组合根提供的同步会话工厂。"""
|
||||
self._session_factory = session_factory
|
||||
@@ -35,6 +43,33 @@ class TransactionalTransferAdmissionRepository:
|
||||
"""生成与历史登记时间可按字典序比较的当前时间。"""
|
||||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@staticmethod
|
||||
def _lease_now() -> datetime:
|
||||
"""生成不受宿主时区影响的当前 UTC 租约时间。"""
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
@staticmethod
|
||||
def _format_lease_time(value: datetime) -> str:
|
||||
"""把 UTC 时间编码为可稳定排序的固定宽度字符串。"""
|
||||
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
|
||||
@staticmethod
|
||||
def _validate_claim_arguments(*, owner_id: str, lease_seconds: int) -> None:
|
||||
"""拒绝无法建立有效租约身份或正向期限的调用。"""
|
||||
if not owner_id:
|
||||
raise ValueError("整理任务 claim 缺少 owner_id")
|
||||
if lease_seconds <= 0:
|
||||
raise ValueError("整理任务 lease_seconds 必须大于零")
|
||||
|
||||
@staticmethod
|
||||
def _recoverable_states() -> tuple[str, ...]:
|
||||
"""返回允许被 worker claim 的稳定业务状态。"""
|
||||
return (
|
||||
TRANSFER_ADMISSION_ACCEPTED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _project(pending: TransferPending) -> TransferAdmission:
|
||||
"""在 Session 有效期内把 ORM 行冻结为应用层 DTO。"""
|
||||
@@ -84,6 +119,11 @@ class TransactionalTransferAdmissionRepository:
|
||||
input_fingerprint=pending.input_fingerprint,
|
||||
planning_input=planning_input,
|
||||
checkpoint=checkpoint,
|
||||
lease_owner=pending.lease_owner,
|
||||
lease_token=pending.lease_token,
|
||||
lease_expires_at=pending.lease_expires_at,
|
||||
heartbeat_at=pending.heartbeat_at,
|
||||
attempt_count=pending.attempt_count,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -151,27 +191,225 @@ class TransactionalTransferAdmissionRepository:
|
||||
self._assert_input_match(pending, effective_input)
|
||||
return self._project(pending)
|
||||
|
||||
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
|
||||
"""在独立只读会话中投影等待恢复或执行的准入记录。"""
|
||||
def claim_task(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
owner_id: str,
|
||||
lease_seconds: int,
|
||||
) -> Optional[TransferAdmission]:
|
||||
"""原子 claim 指定任务,任何已存在的有效租约都拒绝重复领取。"""
|
||||
self._validate_claim_arguments(
|
||||
owner_id=owner_id,
|
||||
lease_seconds=lease_seconds,
|
||||
)
|
||||
if not task_id:
|
||||
raise ValueError("整理任务 claim 缺少 task_id")
|
||||
now = self._lease_now()
|
||||
now_time = self._format_lease_time(now)
|
||||
lease_expires_at = self._format_lease_time(
|
||||
now + timedelta(seconds=lease_seconds)
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
pending_items = TransferPendingOper(db=session).list_by_state(
|
||||
state=TRANSFER_ADMISSION_ACCEPTED,
|
||||
limit=limit,
|
||||
)
|
||||
return [self._project(pending) for pending in pending_items]
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
oper = TransferPendingOper(db=session)
|
||||
lease_token = uuid4().hex
|
||||
updated = oper.stage_claim_task(
|
||||
task_id=task_id,
|
||||
states=self._recoverable_states(),
|
||||
owner_id=owner_id,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
lease_expires_at=lease_expires_at,
|
||||
updated_at=self._now(),
|
||||
)
|
||||
session.flush()
|
||||
session.expire_all()
|
||||
try:
|
||||
pending = oper.get_by_task_id(task_id=task_id)
|
||||
except JSONDecodeError as error:
|
||||
raise TransferAdmissionProjectionError(
|
||||
f"整理任务持久 JSON 无法解码: {task_id} - {error}"
|
||||
) from error
|
||||
if updated:
|
||||
if pending is None:
|
||||
raise TransferPlanningStateError(
|
||||
f"claim 后未找到整理任务: {task_id}"
|
||||
)
|
||||
try:
|
||||
admission = self._project(pending)
|
||||
except (
|
||||
TransferAdmissionConflictError,
|
||||
TransferPlanningStateError,
|
||||
TypeError,
|
||||
ValueError,
|
||||
) as error:
|
||||
raise TransferAdmissionProjectionError(
|
||||
f"整理任务持久投影损坏: {task_id} - {error}"
|
||||
) from error
|
||||
transaction.commit()
|
||||
return admission
|
||||
transaction.commit()
|
||||
return None
|
||||
except Exception:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
|
||||
"""投影接纳、provider 待执行或已规划的全部可恢复任务。"""
|
||||
with self._session_factory() as session:
|
||||
pending_items = TransferPendingOper(db=session).list_by_states(
|
||||
states=(
|
||||
TRANSFER_ADMISSION_ACCEPTED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
),
|
||||
limit=limit,
|
||||
def claim_recoverable(
|
||||
self,
|
||||
*,
|
||||
owner_id: str,
|
||||
limit: int,
|
||||
lease_seconds: int,
|
||||
) -> list[TransferAdmission]:
|
||||
"""按登记顺序逐条 CAS claim 未租用或租约已过期的恢复任务。"""
|
||||
self._validate_claim_arguments(
|
||||
owner_id=owner_id,
|
||||
lease_seconds=lease_seconds,
|
||||
)
|
||||
if limit <= 0:
|
||||
return []
|
||||
claimed: list[TransferAdmission] = []
|
||||
after_cursor: Optional[tuple[str, int]] = None
|
||||
scanned_count = 0
|
||||
scan_limit = self._MAX_RECOVERY_SCAN_TASKS
|
||||
while len(claimed) < limit and scanned_count < scan_limit:
|
||||
candidate_limit = min(
|
||||
limit - len(claimed),
|
||||
scan_limit - scanned_count,
|
||||
)
|
||||
return [self._project(pending) for pending in pending_items]
|
||||
now_time = self._format_lease_time(self._lease_now())
|
||||
with self._session_factory() as session:
|
||||
candidates = TransferPendingOper(db=session).list_claimable_candidates(
|
||||
states=self._recoverable_states(),
|
||||
now_time=now_time,
|
||||
limit=candidate_limit,
|
||||
after_cursor=after_cursor,
|
||||
)
|
||||
if not candidates:
|
||||
break
|
||||
scanned_count += len(candidates)
|
||||
_, cursor_created_at, cursor_id = candidates[-1]
|
||||
after_cursor = (cursor_created_at, cursor_id)
|
||||
for task_id, _, _ in candidates:
|
||||
try:
|
||||
admission = self.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id=owner_id,
|
||||
lease_seconds=lease_seconds,
|
||||
)
|
||||
except TransferAdmissionProjectionError as error:
|
||||
self._record_projection_failure(
|
||||
task_id=task_id,
|
||||
error=error,
|
||||
)
|
||||
continue
|
||||
if admission is not None:
|
||||
claimed.append(admission)
|
||||
if len(claimed) >= limit:
|
||||
break
|
||||
return claimed
|
||||
|
||||
def _record_projection_failure(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
error: TransferAdmissionProjectionError,
|
||||
) -> bool:
|
||||
"""以独立 CAS 留存变化后的投影错误,并仅为新诊断记一次运行日志。"""
|
||||
diagnostic = f"恢复投影失败: {error}"
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
recorded = TransferPendingOper(
|
||||
db=session
|
||||
).stage_record_projection_failure(
|
||||
task_id=task_id,
|
||||
states=self._recoverable_states(),
|
||||
error=diagnostic,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
updated_at=self._now(),
|
||||
)
|
||||
transaction.commit()
|
||||
except Exception:
|
||||
transaction.rollback()
|
||||
raise
|
||||
if recorded:
|
||||
_diagnostic_logger.error(
|
||||
f"整理恢复任务投影损坏:task_id={task_id}, error={error}"
|
||||
)
|
||||
return bool(recorded)
|
||||
|
||||
def heartbeat(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
lease_seconds: int,
|
||||
) -> Optional[TransferAdmission]:
|
||||
"""仅以当前且未过期的 token 续租,禁止陈旧 worker 复活租约。"""
|
||||
if not task_id or not lease_token:
|
||||
raise ValueError("整理任务 heartbeat 缺少任务或租约身份")
|
||||
if lease_seconds <= 0:
|
||||
raise ValueError("整理任务 lease_seconds 必须大于零")
|
||||
now = self._lease_now()
|
||||
now_time = self._format_lease_time(now)
|
||||
lease_expires_at = self._format_lease_time(
|
||||
now + timedelta(seconds=lease_seconds)
|
||||
)
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
oper = TransferPendingOper(db=session)
|
||||
updated = oper.stage_heartbeat(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
lease_expires_at=lease_expires_at,
|
||||
)
|
||||
if not updated:
|
||||
transaction.commit()
|
||||
return None
|
||||
session.flush()
|
||||
session.expire_all()
|
||||
pending = oper.get_by_task_id(task_id=task_id)
|
||||
if pending is None:
|
||||
raise TransferPlanningStateError(
|
||||
f"heartbeat 后未找到整理任务: {task_id}"
|
||||
)
|
||||
admission = self._project(pending)
|
||||
transaction.commit()
|
||||
return admission
|
||||
except Exception:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def release_claim(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""仅以当前未过期 token 释放租约,陈旧 worker 不得改变任务。"""
|
||||
if not task_id or not lease_token:
|
||||
return False
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
released = TransferPendingOper(db=session).stage_release_claim(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
updated_at=self._now(),
|
||||
)
|
||||
transaction.commit()
|
||||
return bool(released)
|
||||
except Exception:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
|
||||
"""独立提交最近一次入队失败,保留准入记录供后续恢复。"""
|
||||
@@ -192,6 +430,7 @@ class TransactionalTransferAdmissionRepository:
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
input_fingerprint: str,
|
||||
checkpoint: TransferPlanCheckpoint,
|
||||
) -> TransferAdmission:
|
||||
@@ -223,7 +462,9 @@ class TransactionalTransferAdmissionRepository:
|
||||
checkpoint_payload=checkpoint_payload,
|
||||
source_states=source_states,
|
||||
target_state=target_state,
|
||||
now_time=self._now(),
|
||||
lease_token=lease_token,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
updated_at=self._now(),
|
||||
)
|
||||
session.flush()
|
||||
session.expire_all()
|
||||
@@ -232,6 +473,13 @@ class TransactionalTransferAdmissionRepository:
|
||||
raise TransferPlanningStateError(f"未找到整理任务: {task_id}")
|
||||
if pending.input_fingerprint != input_fingerprint:
|
||||
raise TransferAdmissionConflictError("整理任务输入指纹已经改变")
|
||||
now_time = self._format_lease_time(self._lease_now())
|
||||
if (
|
||||
pending.lease_token != lease_token
|
||||
or not pending.lease_expires_at
|
||||
or pending.lease_expires_at <= now_time
|
||||
):
|
||||
raise TransferLeaseLostError("整理任务租约已过期或已被其他 worker 接管")
|
||||
if not updated and not (
|
||||
pending.state == target_state
|
||||
and pending.checkpoint_payload == checkpoint_payload
|
||||
@@ -246,28 +494,44 @@ class TransactionalTransferAdmissionRepository:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def record_planning_failure(self, *, task_id: str, error: str) -> None:
|
||||
def record_planning_failure(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: str,
|
||||
) -> None:
|
||||
"""独立提交规划错误并保持任务处于接纳态供恢复重试。"""
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
TransferPendingOper(db=session).stage_record_planning_failure(
|
||||
updated = TransferPendingOper(db=session).stage_record_planning_failure(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
now_time=self._now(),
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
updated_at=self._now(),
|
||||
)
|
||||
if not updated:
|
||||
raise TransferLeaseLostError(
|
||||
"整理任务租约已过期或已被其他 worker 接管"
|
||||
)
|
||||
transaction.commit()
|
||||
except Exception:
|
||||
transaction.rollback()
|
||||
raise
|
||||
|
||||
def discard_task(self, *, task_id: str) -> int:
|
||||
"""在独立事务中按稳定任务标识删除已到终态的准入记录。"""
|
||||
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
|
||||
"""仅以当前未过期 token 删除终态任务,拒绝陈旧 worker 变更。"""
|
||||
if not task_id or not lease_token:
|
||||
return 0
|
||||
with self._session_factory() as session:
|
||||
transaction = SqlAlchemyUnitOfWork(session)
|
||||
try:
|
||||
deleted = TransferPendingOper(db=session).stage_discard_task(
|
||||
deleted = TransferPendingOper(db=session).stage_discard_claimed(
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=self._format_lease_time(self._lease_now()),
|
||||
)
|
||||
transaction.commit()
|
||||
return deleted
|
||||
|
||||
+338
-148
@@ -4,7 +4,20 @@ from datetime import datetime
|
||||
from typing import Any, List, Optional, cast
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update
|
||||
from sqlalchemy import (
|
||||
JSON,
|
||||
Index,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
and_,
|
||||
delete,
|
||||
func,
|
||||
or_,
|
||||
select,
|
||||
update,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, Session, mapped_column
|
||||
|
||||
from app.db.base import Base, execute_dml, get_id_column
|
||||
@@ -72,7 +85,7 @@ class TransferPending(Base):
|
||||
|
||||
准入时保存版本化规划输入和指纹;纯规划完成后以同一行原子保存完整有序计划并
|
||||
推进到 planned。重启恢复可直接消费已规划路径,避免再次触发 rename 等插件事件。
|
||||
旧路径登记接口仍生成最小 legacy_replan 输入,供插件兼容调用方继续使用。
|
||||
所有执行期 mutation 都以稳定任务身份和租约 token 进行 CAS fencing。
|
||||
"""
|
||||
|
||||
id = get_id_column()
|
||||
@@ -111,6 +124,16 @@ class TransferPending(Base):
|
||||
checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
|
||||
# 规划完成时间
|
||||
planned_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 当前租约拥有者
|
||||
lease_owner: Mapped[Optional[str]] = mapped_column(String(128))
|
||||
# 当前租约的唯一防陈旧令牌
|
||||
lease_token: Mapped[Optional[str]] = mapped_column(String(64))
|
||||
# 当前租约的 UTC 到期时间
|
||||
lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 最近一次成功 claim 或 heartbeat 的 UTC 时间
|
||||
heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40))
|
||||
# 真正取得新 token 的累计次数
|
||||
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
|
||||
__table_args__ = (
|
||||
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
|
||||
@@ -122,41 +145,17 @@ class TransferPending(Base):
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
# 恢复调度按业务状态和租约到期时间筛选可接管任务
|
||||
Index(
|
||||
"ix_transferpending_recovery_lease",
|
||||
"state",
|
||||
"lease_expires_at",
|
||||
"created_at",
|
||||
"id",
|
||||
),
|
||||
UniqueConstraint("task_id", name="uq_transferpending_task_id"),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def register(cls, db: Session, storage: str, src_path: str,
|
||||
now_time: str) -> Optional["TransferPending"]:
|
||||
"""
|
||||
登记一个待整理文件,已存在时保持原登记时间不变。
|
||||
:param db: 数据库会话
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:param now_time: 当前时间
|
||||
:return: 登记记录
|
||||
"""
|
||||
if not storage or not src_path:
|
||||
return None
|
||||
pending = db.execute(
|
||||
select(cls).where(cls.storage == storage, cls.src_path == src_path)
|
||||
).scalars().first()
|
||||
if pending:
|
||||
return cast("TransferPending", pending)
|
||||
planning_input = _legacy_planning_payload(storage, src_path)
|
||||
pending = cls(
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
state="accepted",
|
||||
created_at=now_time,
|
||||
updated_at=now_time,
|
||||
input_version=1,
|
||||
planning_input=planning_input,
|
||||
input_fingerprint=_planning_fingerprint(planning_input),
|
||||
)
|
||||
db.add(pending)
|
||||
return pending
|
||||
|
||||
@classmethod
|
||||
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
|
||||
src_path: str, state: str,
|
||||
@@ -201,44 +200,6 @@ class TransferPending(Base):
|
||||
db.add(pending)
|
||||
return pending
|
||||
|
||||
@classmethod
|
||||
def list_by_state(cls, db: Session, *, state: str,
|
||||
limit: Optional[int] = 5000) -> List["TransferPending"]:
|
||||
"""
|
||||
按登记顺序列出指定持久状态的接纳记录。
|
||||
:param db: 数据库会话
|
||||
:param state: 持久状态
|
||||
:param limit: 单次读取上限
|
||||
:return: 接纳记录列表
|
||||
"""
|
||||
if not state:
|
||||
return []
|
||||
return list(db.execute(
|
||||
select(cls)
|
||||
.where(cls.state == state)
|
||||
.order_by(cls.created_at.asc(), cls.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all())
|
||||
|
||||
@classmethod
|
||||
def list_by_states(cls, db: Session, *, states: tuple[str, ...],
|
||||
limit: Optional[int] = 5000) -> List["TransferPending"]:
|
||||
"""
|
||||
按登记顺序列出多个可恢复持久状态的记录。
|
||||
:param db: 数据库会话
|
||||
:param states: 允许恢复的状态集合
|
||||
:param limit: 单次读取上限
|
||||
:return: 接纳记录列表
|
||||
"""
|
||||
if not states:
|
||||
return []
|
||||
return list(db.execute(
|
||||
select(cls)
|
||||
.where(cls.state.in_(states))
|
||||
.order_by(cls.created_at.asc(), cls.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all())
|
||||
|
||||
@classmethod
|
||||
def get_by_identity(cls, db: Session, *, storage: str,
|
||||
src_path: str) -> Optional["TransferPending"]:
|
||||
@@ -276,12 +237,283 @@ class TransferPending(Base):
|
||||
db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_claimable_candidates(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
states: tuple[str, ...],
|
||||
now_time: str,
|
||||
limit: int,
|
||||
after_cursor: Optional[tuple[str, int]] = None,
|
||||
) -> List[tuple[str, str, int]]:
|
||||
"""
|
||||
按稳定游标列出未租用或租约已过期的候选任务。
|
||||
|
||||
返回候选不等于取得租约;调用方必须继续执行带相同过期条件的 claim CAS,
|
||||
并以受影响行数决定竞争结果。
|
||||
:param db: 数据库会话
|
||||
:param states: 可恢复业务状态
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param limit: 候选数量上限
|
||||
:param after_cursor: 上一页最后一条的规范登记时间与主键
|
||||
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
|
||||
"""
|
||||
if not states or not now_time or limit <= 0:
|
||||
return []
|
||||
cursor_created_at = func.coalesce(cls.created_at, "")
|
||||
statement = select(cls.task_id, cursor_created_at, cls.id).where(
|
||||
cls.state.in_(states),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
cls.lease_expires_at <= now_time,
|
||||
),
|
||||
)
|
||||
if after_cursor is not None:
|
||||
after_created_at, after_id = after_cursor
|
||||
statement = statement.where(or_(
|
||||
cursor_created_at > after_created_at,
|
||||
and_(
|
||||
cursor_created_at == after_created_at,
|
||||
cls.id > after_id,
|
||||
),
|
||||
))
|
||||
rows = db.execute(
|
||||
statement
|
||||
.order_by(cursor_created_at.asc(), cls.id.asc())
|
||||
.limit(limit)
|
||||
).all()
|
||||
return [
|
||||
(task_id, created_at or "", int(row_id))
|
||||
for task_id, created_at, row_id in rows
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def claim_task(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
states: tuple[str, ...],
|
||||
owner_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
lease_expires_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
以未租用或租约已过期为条件原子取得任务租约。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param states: 允许 claim 的业务状态
|
||||
:param owner_id: 新租约拥有者
|
||||
:param lease_token: 新租约唯一令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param lease_expires_at: 新租约到期时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数,1 表示赢得竞争
|
||||
"""
|
||||
if not all((
|
||||
task_id,
|
||||
states,
|
||||
owner_id,
|
||||
lease_token,
|
||||
now_time,
|
||||
lease_expires_at,
|
||||
updated_at,
|
||||
)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(states),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
cls.lease_expires_at <= now_time,
|
||||
),
|
||||
)
|
||||
.values(
|
||||
lease_owner=owner_id,
|
||||
lease_token=lease_token,
|
||||
lease_expires_at=lease_expires_at,
|
||||
heartbeat_at=now_time,
|
||||
attempt_count=cls.attempt_count + 1,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def record_projection_failure(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
states: tuple[str, ...],
|
||||
error: str,
|
||||
now_time: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
在没有有效租约且诊断发生变化时原子记录恢复投影损坏。
|
||||
|
||||
claim 的投影失败会先回滚,因此这里不得重新占用租约。CAS 同时保护
|
||||
已被其他 worker 领取的任务,并避免周期恢复反复刷新相同错误。
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param states: 可恢复业务状态
|
||||
:param error: 可持久化的稳定诊断文本
|
||||
:param now_time: 当前 UTC 租约时间
|
||||
:param updated_at: 宿主本地业务审计时间
|
||||
:return: 更新的记录数,1 表示首次或变化后的诊断被记录
|
||||
"""
|
||||
if not all((task_id, states, error, now_time, updated_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(states),
|
||||
or_(
|
||||
cls.lease_token.is_(None),
|
||||
cls.lease_expires_at.is_(None),
|
||||
cls.lease_expires_at <= now_time,
|
||||
),
|
||||
cls.last_error.is_distinct_from(error),
|
||||
)
|
||||
.values(
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def heartbeat(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
lease_expires_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
仅以当前且未过期的 token 原子延长任务租约。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param lease_expires_at: 新租约到期时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
if not all((task_id, lease_token, now_time, lease_expires_at)):
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
)
|
||||
.values(
|
||||
lease_expires_at=lease_expires_at,
|
||||
heartbeat_at=now_time,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def release_claim(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: Optional[str],
|
||||
now_time: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
仅以当前且未过期的 token 释放租约并保存本次执行错误。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param error: 本次执行错误,成功释放时为空
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
if not task_id or not lease_token or not now_time or not updated_at:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
)
|
||||
.values(
|
||||
lease_owner=None,
|
||||
lease_token=None,
|
||||
lease_expires_at=None,
|
||||
heartbeat_at=None,
|
||||
last_error=error,
|
||||
updated_at=updated_at,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def discard_claimed(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
仅以当前且未过期的 token 删除已经到达终态的租约任务。
|
||||
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
if not task_id or not lease_token or not now_time:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
delete(cls).where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def checkpoint_plan(cls, db: Session, *, task_id: str,
|
||||
input_fingerprint: str, checkpoint_version: int,
|
||||
checkpoint_payload: dict[str, Any],
|
||||
source_states: tuple[str, ...], target_state: str,
|
||||
now_time: str) -> int:
|
||||
lease_token: str, now_time: str,
|
||||
updated_at: str) -> int:
|
||||
"""
|
||||
以输入指纹为 CAS 条件原子保存计划并推进到已规划。
|
||||
:param db: 数据库会话
|
||||
@@ -291,7 +523,9 @@ class TransferPending(Base):
|
||||
:param checkpoint_payload: 完整有序计划 JSON
|
||||
:param source_states: 允许推进检查点的起始状态
|
||||
:param target_state: 检查点提交后的目标状态
|
||||
:param now_time: 当前时间
|
||||
:param lease_token: 当前且未过期的租约令牌
|
||||
:param now_time: 用于租约 fencing 的当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
if (
|
||||
@@ -300,8 +534,19 @@ class TransferPending(Base):
|
||||
or not checkpoint_payload
|
||||
or not source_states
|
||||
or not target_state
|
||||
or not lease_token
|
||||
or not updated_at
|
||||
):
|
||||
return 0
|
||||
values: dict[str, Any] = {
|
||||
"state": target_state,
|
||||
"checkpoint_version": checkpoint_version,
|
||||
"checkpoint_payload": checkpoint_payload,
|
||||
"last_error": None,
|
||||
"updated_at": updated_at,
|
||||
}
|
||||
if target_state == "planned":
|
||||
values["planned_at"] = updated_at
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
@@ -309,30 +554,29 @@ class TransferPending(Base):
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(source_states),
|
||||
cls.input_fingerprint == input_fingerprint,
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
)
|
||||
.values(
|
||||
state=target_state,
|
||||
checkpoint_version=checkpoint_version,
|
||||
checkpoint_payload=checkpoint_payload,
|
||||
planned_at=now_time,
|
||||
last_error=None,
|
||||
updated_at=now_time,
|
||||
),
|
||||
.values(**values),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def record_planning_failure(cls, db: Session, *, task_id: str,
|
||||
error: str, now_time: str) -> int:
|
||||
lease_token: str, error: str,
|
||||
now_time: str, updated_at: str) -> int:
|
||||
"""
|
||||
为接纳态或 provider 待执行任务记录规划失败,不改变其恢复状态。
|
||||
:param db: 数据库会话
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前且未过期的租约令牌
|
||||
:param error: 失败原因
|
||||
:param now_time: 当前时间
|
||||
:param now_time: 用于租约 fencing 的当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
if not task_id:
|
||||
if not task_id or not lease_token or not now_time or not updated_at:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db,
|
||||
@@ -340,8 +584,11 @@ class TransferPending(Base):
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.state.in_(("accepted", "provider_pending")),
|
||||
cls.lease_token == lease_token,
|
||||
cls.lease_expires_at.is_not(None),
|
||||
cls.lease_expires_at > now_time,
|
||||
)
|
||||
.values(last_error=error, updated_at=now_time),
|
||||
.values(last_error=error, updated_at=updated_at),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@@ -361,67 +608,10 @@ class TransferPending(Base):
|
||||
return execute_dml(
|
||||
db,
|
||||
update(cls)
|
||||
.where(cls.task_id == task_id)
|
||||
.where(
|
||||
cls.task_id == task_id,
|
||||
cls.lease_token.is_(None),
|
||||
)
|
||||
.values(last_error=error, updated_at=now_time),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def discard_task(cls, db: Session, *, task_id: str) -> int:
|
||||
"""
|
||||
在调用方会话中按任务标识删除接纳记录。
|
||||
:param db: 数据库会话
|
||||
:param task_id: 任务标识
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
if not task_id:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db, delete(cls).where(cls.task_id == task_id),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def discard(cls, db: Session, storage: str, src_path: str) -> int:
|
||||
"""
|
||||
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
|
||||
:param db: 数据库会话
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
if not storage or not src_path:
|
||||
return 0
|
||||
return execute_dml(
|
||||
db, delete(cls).where(cls.storage == storage, cls.src_path == src_path),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]:
|
||||
"""
|
||||
列出全部待整理登记,供启动回放使用。
|
||||
|
||||
按登记时间升序回放,保持与原入队顺序一致;上限避免异常积压时
|
||||
一次性把整理链压垮。
|
||||
:param db: 数据库会话
|
||||
:param limit: 单次回放上限
|
||||
:return: 待整理登记列表
|
||||
"""
|
||||
return list(db.execute(
|
||||
select(cls)
|
||||
.order_by(cls.created_at.asc(), cls.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all())
|
||||
|
||||
@classmethod
|
||||
def clear(cls, db: Session) -> int:
|
||||
"""
|
||||
清空全部待整理登记。
|
||||
:param db: 数据库会话
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return execute_dml(
|
||||
db, delete(cls),
|
||||
execution_options={"synchronize_session": False},
|
||||
execution_options={"synchronize_session": "fetch"},
|
||||
)
|
||||
|
||||
+191
-110
@@ -1,5 +1,4 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, List, Optional, Tuple
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.db.base import DbOper
|
||||
from app.db.models.transferpending import TransferPending
|
||||
@@ -10,27 +9,9 @@ class TransferPendingOper(DbOper):
|
||||
待整理文件登记管理。
|
||||
|
||||
保存稳定任务身份、存储、源文件路径和准入状态,用于在进程重启后把没走完
|
||||
整理链的文件重新送回去,避免挂载故障重启后永久漏件。旧版路径登记接口继续
|
||||
保留,供插件和兼容调用方使用。
|
||||
整理链的文件重新送回去,避免挂载故障重启后永久漏件。
|
||||
"""
|
||||
|
||||
def register(self, storage: str, src_path: str) -> Optional[TransferPending]:
|
||||
"""
|
||||
登记一个待整理文件。
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 登记记录
|
||||
"""
|
||||
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.register(
|
||||
session,
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
now_time=now_time,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
|
||||
state: str, now_time: str, input_version: int = 1,
|
||||
planning_input: Optional[dict[str, Any]] = None,
|
||||
@@ -63,38 +44,6 @@ class TransferPendingOper(DbOper):
|
||||
)
|
||||
)
|
||||
|
||||
def list_by_state(self, *, state: str,
|
||||
limit: Optional[int] = 5000) -> List[TransferPending]:
|
||||
"""
|
||||
使用当前会话列出指定状态记录。
|
||||
:param state: 持久状态
|
||||
:param limit: 单次读取上限
|
||||
:return: ORM 接纳记录列表
|
||||
"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: TransferPending.list_by_state(
|
||||
session,
|
||||
state=state,
|
||||
limit=limit,
|
||||
)
|
||||
) or []
|
||||
|
||||
def list_by_states(self, *, states: tuple[str, ...],
|
||||
limit: Optional[int] = 5000) -> List[TransferPending]:
|
||||
"""
|
||||
使用当前会话列出多个可恢复状态的记录。
|
||||
:param states: 允许恢复的状态集合
|
||||
:param limit: 单次读取上限
|
||||
:return: ORM 接纳记录列表
|
||||
"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: TransferPending.list_by_states(
|
||||
session,
|
||||
states=states,
|
||||
limit=limit,
|
||||
)
|
||||
) or []
|
||||
|
||||
def get_by_identity(self, *, storage: str,
|
||||
src_path: str) -> Optional[TransferPending]:
|
||||
"""
|
||||
@@ -133,7 +82,9 @@ class TransferPendingOper(DbOper):
|
||||
checkpoint_payload: dict[str, Any],
|
||||
source_states: tuple[str, ...],
|
||||
target_state: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
在当前会话中以输入指纹为条件暂存完整计划检查点。
|
||||
@@ -143,7 +94,9 @@ class TransferPendingOper(DbOper):
|
||||
:param checkpoint_payload: 完整有序计划 JSON
|
||||
:param source_states: 允许执行 CAS 的起始状态
|
||||
:param target_state: 检查点提交后的目标状态
|
||||
:param now_time: 当前时间
|
||||
:param lease_token: 当前且未过期的租约令牌
|
||||
:param now_time: 用于租约 fencing 的当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
@@ -155,25 +108,206 @@ class TransferPendingOper(DbOper):
|
||||
checkpoint_payload=checkpoint_payload,
|
||||
source_states=source_states,
|
||||
target_state=target_state,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_record_planning_failure(self, *, task_id: str, error: str,
|
||||
now_time: str) -> int:
|
||||
def stage_record_planning_failure(self, *, task_id: str, lease_token: str,
|
||||
error: str, now_time: str,
|
||||
updated_at: str) -> int:
|
||||
"""
|
||||
在当前会话中记录规划失败并保持任务处于接纳态。
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前且未过期的租约令牌
|
||||
:param error: 失败原因
|
||||
:param now_time: 当前时间
|
||||
:param now_time: 用于租约 fencing 的当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.record_planning_failure(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
now_time=now_time,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def list_claimable_candidates(
|
||||
self,
|
||||
*,
|
||||
states: tuple[str, ...],
|
||||
now_time: str,
|
||||
limit: int,
|
||||
after_cursor: Optional[tuple[str, int]] = None,
|
||||
) -> list[tuple[str, str, int]]:
|
||||
"""
|
||||
使用当前会话按稳定游标读取未租用或已过期的恢复候选。
|
||||
|
||||
:param states: 可恢复业务状态
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param limit: 候选数量上限
|
||||
:param after_cursor: 上一页最后一条的规范登记时间与主键
|
||||
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
|
||||
"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: TransferPending.list_claimable_candidates(
|
||||
session,
|
||||
states=states,
|
||||
now_time=now_time,
|
||||
limit=limit,
|
||||
after_cursor=after_cursor,
|
||||
)
|
||||
) or []
|
||||
|
||||
def stage_claim_task(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
states: tuple[str, ...],
|
||||
owner_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
lease_expires_at: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话以未租或过期条件竞争一个新租约。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param states: 可恢复业务状态
|
||||
:param owner_id: 新租约拥有者
|
||||
:param lease_token: 新租约唯一令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param lease_expires_at: 新租约到期时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.claim_task(
|
||||
session,
|
||||
task_id=task_id,
|
||||
states=states,
|
||||
owner_id=owner_id,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
lease_expires_at=lease_expires_at,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_record_projection_failure(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
states: tuple[str, ...],
|
||||
error: str,
|
||||
now_time: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话按无有效租约和诊断变化条件记录投影损坏。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param states: 可恢复业务状态
|
||||
:param error: 可持久化的稳定诊断文本
|
||||
:param now_time: 当前 UTC 租约时间
|
||||
:param updated_at: 宿主本地业务审计时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.record_projection_failure(
|
||||
session,
|
||||
task_id=task_id,
|
||||
states=states,
|
||||
error=error,
|
||||
now_time=now_time,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_heartbeat(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
lease_expires_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话以当前未过期 token 延长租约。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param lease_expires_at: 新租约到期时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.heartbeat(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
lease_expires_at=lease_expires_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_release_claim(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
error: Optional[str],
|
||||
now_time: str,
|
||||
updated_at: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话按未过期 token 释放租约并记录本次错误。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param error: 本次执行错误,成功释放时为空
|
||||
:param now_time: 当前 UTC 时间
|
||||
:param updated_at: 与既有业务审计字段一致的宿主本地时间
|
||||
:return: 更新的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.release_claim(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
error=error,
|
||||
now_time=now_time,
|
||||
updated_at=updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_discard_claimed(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
lease_token: str,
|
||||
now_time: str,
|
||||
) -> int:
|
||||
"""
|
||||
使用当前会话按当前未过期 token 删除终态任务。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:param lease_token: 当前租约令牌
|
||||
:param now_time: 当前 UTC 时间
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.discard_claimed(
|
||||
session,
|
||||
task_id=task_id,
|
||||
lease_token=lease_token,
|
||||
now_time=now_time,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -194,56 +328,3 @@ class TransferPendingOper(DbOper):
|
||||
now_time=now_time,
|
||||
)
|
||||
)
|
||||
|
||||
def stage_discard_task(self, *, task_id: str) -> int:
|
||||
"""
|
||||
在当前会话中暂存按任务标识删除接纳记录。
|
||||
:param task_id: 任务标识
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.discard_task(
|
||||
session,
|
||||
task_id=task_id,
|
||||
)
|
||||
)
|
||||
|
||||
def discard(self, storage: str, src_path: str) -> int:
|
||||
"""
|
||||
注销一个待整理文件登记。
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: TransferPending.discard(
|
||||
session,
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
)
|
||||
|
||||
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
列出全部待整理登记,供启动回放使用。
|
||||
|
||||
返回纯元组而不是 ORM 实例:回放发生在会话之外,ORM 实例脱离 session
|
||||
后访问属性会触发 DetachedInstanceError。
|
||||
:param limit: 单次回放上限
|
||||
:return: (存储, 源文件路径) 列表
|
||||
"""
|
||||
items = self._execute_sync_query(
|
||||
lambda session: TransferPending.list_all(session, limit=limit)
|
||||
)
|
||||
return [
|
||||
(item.storage, item.src_path)
|
||||
for item in items or []
|
||||
if item and item.storage and item.src_path
|
||||
]
|
||||
|
||||
def clear(self) -> int:
|
||||
"""
|
||||
清空全部待整理登记。
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(TransferPending.clear)
|
||||
|
||||
@@ -135,10 +135,10 @@ MODULE_ALIASES: Dict[str, ModuleAlias] = {
|
||||
owner="sdk",
|
||||
),
|
||||
"app.db.transferpending_oper": ModuleAlias(
|
||||
target="app.db.oper.transferpending",
|
||||
replacement="app.db.oper.transferpending",
|
||||
target="app.sdk._legacy.transferpending",
|
||||
replacement="app.application.transfer",
|
||||
introduced="v3.0.0",
|
||||
owner="db",
|
||||
owner="sdk",
|
||||
),
|
||||
"app.db.user_oper": ModuleAlias(
|
||||
target="app.sdk._legacy.user",
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Tuple
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from app.db.base import DbOper, execute_dml
|
||||
from app.db.models.transferpending import TransferPending as _TransferPending
|
||||
|
||||
|
||||
class TransferPendingOper(DbOper):
|
||||
"""
|
||||
保留旧待整理登记 ABI,并对执行租约实施兼容写 fencing。
|
||||
|
||||
本类只由精确旧导入映射加载。宿主整理链仍使用 Application Port 和显式
|
||||
Session 的 canonical Oper;旧删除入口只能处理从未取得租约的记录。
|
||||
"""
|
||||
|
||||
def register(self, storage: str, src_path: str) -> Optional[_TransferPending]:
|
||||
"""
|
||||
按旧签名登记待整理文件,重复登记保持原任务和租约不变。
|
||||
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 登记记录
|
||||
"""
|
||||
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
return self._execute_sync_write(
|
||||
lambda session: _TransferPending.stage_admit(
|
||||
session,
|
||||
task_id=uuid4().hex,
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
state="accepted",
|
||||
now_time=now_time,
|
||||
)
|
||||
)
|
||||
|
||||
def list_by_state(
|
||||
self,
|
||||
*,
|
||||
state: str,
|
||||
limit: Optional[int] = 5000,
|
||||
) -> List[_TransferPending]:
|
||||
"""
|
||||
按旧签名和登记顺序列出指定状态记录。
|
||||
|
||||
:param state: 持久状态
|
||||
:param limit: 单次读取上限
|
||||
:return: ORM 接纳记录列表
|
||||
"""
|
||||
if not state:
|
||||
return []
|
||||
return self._execute_sync_query(
|
||||
lambda session: list(session.execute(
|
||||
select(_TransferPending)
|
||||
.where(_TransferPending.state == state)
|
||||
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all())
|
||||
)
|
||||
|
||||
def list_by_states(
|
||||
self,
|
||||
*,
|
||||
states: tuple[str, ...],
|
||||
limit: Optional[int] = 5000,
|
||||
) -> List[_TransferPending]:
|
||||
"""
|
||||
按旧签名和登记顺序列出多个状态记录。
|
||||
|
||||
:param states: 持久状态集合
|
||||
:param limit: 单次读取上限
|
||||
:return: ORM 接纳记录列表
|
||||
"""
|
||||
if not states:
|
||||
return []
|
||||
return self._execute_sync_query(
|
||||
lambda session: list(session.execute(
|
||||
select(_TransferPending)
|
||||
.where(_TransferPending.state.in_(states))
|
||||
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
|
||||
.limit(limit)
|
||||
).scalars().all())
|
||||
)
|
||||
|
||||
def get_by_identity(
|
||||
self,
|
||||
*,
|
||||
storage: str,
|
||||
src_path: str,
|
||||
) -> Optional[_TransferPending]:
|
||||
"""
|
||||
按旧签名查询指定存储与源路径的登记。
|
||||
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 登记记录
|
||||
"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: _TransferPending.get_by_identity(
|
||||
session,
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
)
|
||||
)
|
||||
|
||||
def get_by_task_id(self, *, task_id: str) -> Optional[_TransferPending]:
|
||||
"""
|
||||
按旧签名查询稳定任务标识对应的登记。
|
||||
|
||||
:param task_id: 稳定任务标识
|
||||
:return: 登记记录
|
||||
"""
|
||||
return self._execute_sync_query(
|
||||
lambda session: _TransferPending.get_by_task_id(
|
||||
session,
|
||||
task_id=task_id,
|
||||
)
|
||||
)
|
||||
|
||||
def discard(self, storage: str, src_path: str) -> int:
|
||||
"""
|
||||
按旧签名删除未 claim 的指定登记。
|
||||
|
||||
任何带 token 的记录都由当前租约拥有者通过 fenced canonical API 收口;
|
||||
即使租约已经过期,旧插件也不得越权代替恢复调度器删除。
|
||||
:param storage: 存储
|
||||
:param src_path: 源文件路径
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
if not storage or not src_path:
|
||||
return 0
|
||||
return self._execute_sync_write(
|
||||
lambda session: execute_dml(
|
||||
session,
|
||||
delete(_TransferPending).where(
|
||||
_TransferPending.storage == storage,
|
||||
_TransferPending.src_path == src_path,
|
||||
_TransferPending.lease_token.is_(None),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
)
|
||||
|
||||
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
|
||||
"""
|
||||
按旧返回形态列出全部待整理路径。
|
||||
|
||||
:param limit: 单次读取上限
|
||||
:return: ``(存储, 源文件路径)`` 列表
|
||||
"""
|
||||
rows = self._execute_sync_query(
|
||||
lambda session: session.execute(
|
||||
select(_TransferPending.storage, _TransferPending.src_path)
|
||||
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
|
||||
.limit(limit)
|
||||
).all()
|
||||
)
|
||||
return [
|
||||
(storage, src_path)
|
||||
for storage, src_path in rows
|
||||
if storage and src_path
|
||||
]
|
||||
|
||||
def clear(self) -> int:
|
||||
"""
|
||||
清空全部未 claim 登记,保留任何带租约 token 的任务。
|
||||
|
||||
:return: 删除的记录数
|
||||
"""
|
||||
return self._execute_sync_write(
|
||||
lambda session: execute_dml(
|
||||
session,
|
||||
delete(_TransferPending).where(
|
||||
_TransferPending.lease_token.is_(None),
|
||||
),
|
||||
execution_options={"synchronize_session": False},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["TransferPendingOper"]
|
||||
@@ -3,12 +3,12 @@ from app.chain.transfer import TransferChain
|
||||
|
||||
def replay_pending_transfers():
|
||||
"""
|
||||
回放上次进程退出时仍未整理完的文件。
|
||||
启动唯一整理恢复调度器。
|
||||
|
||||
整理队列是纯内存的,挂载挂死后的人工重启、版本升级、OOM、宿主重启都会让
|
||||
队列连同「这些文件还没整理」这个事实一起蒸发;而已稳定落地的文件不会再产生
|
||||
任何监控事件,也不会有新的补偿扫描起点,结果就是永久漏件。
|
||||
回放本身在后台线程执行,不阻塞启动流程。
|
||||
启动回放、同进程补偿和过期租约接管共用同一个后台调度入口,不阻塞启动流程。
|
||||
"""
|
||||
TransferChain().replay_pending()
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
"""3.0.15 为整理恢复任务增加强 CAS 租约字段。
|
||||
|
||||
Revision ID: d3a9e5f7b2c4
|
||||
Revises: c2f8a4d6e1b3
|
||||
Create Date: 2026-08-27
|
||||
"""
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "d3a9e5f7b2c4"
|
||||
down_revision = "c2f8a4d6e1b3"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
_TABLE_NAME = "transferpending"
|
||||
_LEASE_INDEX = "ix_transferpending_recovery_lease"
|
||||
_LEASE_COLUMNS = {
|
||||
"lease_owner",
|
||||
"lease_token",
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
"attempt_count",
|
||||
}
|
||||
|
||||
|
||||
def _column_names() -> set[str]:
|
||||
"""返回当前待整理登记表的字段集合。"""
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if _TABLE_NAME not in inspector.get_table_names():
|
||||
return set()
|
||||
return {
|
||||
column["name"]
|
||||
for column in inspector.get_columns(_TABLE_NAME)
|
||||
}
|
||||
|
||||
|
||||
def _index_names() -> set[str]:
|
||||
"""返回当前待整理登记表的索引名称集合。"""
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if _TABLE_NAME not in inspector.get_table_names():
|
||||
return set()
|
||||
return {
|
||||
index["name"]
|
||||
for index in inspector.get_indexes(_TABLE_NAME)
|
||||
if index.get("name")
|
||||
}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""增加租约身份、到期、心跳和接管次数,并支持中断后重跑。"""
|
||||
columns = _column_names()
|
||||
if not columns:
|
||||
return
|
||||
additions = (
|
||||
(
|
||||
"lease_owner",
|
||||
sa.Column("lease_owner", sa.String(length=128), nullable=True),
|
||||
),
|
||||
(
|
||||
"lease_token",
|
||||
sa.Column("lease_token", sa.String(length=64), nullable=True),
|
||||
),
|
||||
(
|
||||
"lease_expires_at",
|
||||
sa.Column("lease_expires_at", sa.String(length=40), nullable=True),
|
||||
),
|
||||
(
|
||||
"heartbeat_at",
|
||||
sa.Column("heartbeat_at", sa.String(length=40), nullable=True),
|
||||
),
|
||||
(
|
||||
"attempt_count",
|
||||
sa.Column("attempt_count", sa.Integer(), nullable=True),
|
||||
),
|
||||
)
|
||||
for column_name, column in additions:
|
||||
if column_name not in columns:
|
||||
op.add_column(_TABLE_NAME, column)
|
||||
|
||||
columns = _column_names()
|
||||
if "attempt_count" in columns:
|
||||
pending = sa.table(
|
||||
_TABLE_NAME,
|
||||
sa.column("attempt_count", sa.Integer()),
|
||||
)
|
||||
op.get_bind().execute(
|
||||
pending.update()
|
||||
.where(pending.c.attempt_count.is_(None))
|
||||
.values(attempt_count=0)
|
||||
)
|
||||
with op.batch_alter_table(_TABLE_NAME) as batch_op:
|
||||
batch_op.alter_column(
|
||||
"attempt_count",
|
||||
existing_type=sa.Integer(),
|
||||
nullable=False,
|
||||
)
|
||||
|
||||
if _LEASE_INDEX not in _index_names():
|
||||
op.create_index(
|
||||
_LEASE_INDEX,
|
||||
_TABLE_NAME,
|
||||
["state", "lease_expires_at", "created_at", "id"],
|
||||
unique=False,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""移除租约索引和字段,保留原业务状态及规划检查点。"""
|
||||
columns = _column_names()
|
||||
if not columns or not (_LEASE_COLUMNS & columns):
|
||||
return
|
||||
if _LEASE_INDEX in _index_names():
|
||||
op.drop_index(_LEASE_INDEX, table_name=_TABLE_NAME)
|
||||
with op.batch_alter_table(_TABLE_NAME) as batch_op:
|
||||
for column_name in (
|
||||
"attempt_count",
|
||||
"heartbeat_at",
|
||||
"lease_expires_at",
|
||||
"lease_token",
|
||||
"lease_owner",
|
||||
):
|
||||
if column_name in columns:
|
||||
batch_op.drop_column(column_name)
|
||||
@@ -69,7 +69,7 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
|
||||
|
||||
| 指标 | 当前值 | 解释 |
|
||||
|---|---:|---|
|
||||
| 宿主 Python 模块 / 内部依赖边 | 836 / 6,832 | `dependency-baseline.json` 当前快照 |
|
||||
| 宿主 Python 模块 / 内部依赖边 | 837 / 6,836 | `dependency-baseline.json` 当前快照 |
|
||||
| 非平凡 SCC | 2 | 新增 Chain 包根环;另一个是隔离的 29 模块 TMDB 移植包环 |
|
||||
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
|
||||
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
|
||||
@@ -192,7 +192,8 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
|
||||
- `S1-L1.2 Planning checkpoint`:`VERIFIED`。版本化请求与指纹先准入;无 legacy provider 时通过
|
||||
`accepted -> planned` CAS 提交完整目标和有序操作,有 provider 时先提交 `provider_pending`,全部
|
||||
返回空后再以第二次 CAS 提交 `planned`;planned 重放只消费冻结上下文和目标。
|
||||
- `S1-L1.3 Lease 与恢复调度`:`PLANNED`。交付 claim/lease/heartbeat/attempt、过期接管与唯一恢复入口。
|
||||
- `S1-L1.3 Lease 与恢复调度`:`VERIFIED`。已交付 token fencing 的
|
||||
claim/lease/heartbeat/attempt、过期接管、固定退避的唯一恢复入口和有界关闭 owner。
|
||||
- `S1-L1.4 幂等执行与终态结算`:`PLANNED`。交付文件/历史幂等、唯一 retry owner 和
|
||||
`manual_review` 语义。
|
||||
- `S1-L1.5 E3 全链收口`:`PLANNED`。完成崩溃矩阵、兼容验收与旧路径删除。此叶交付前,
|
||||
@@ -210,9 +211,13 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
|
||||
解析并严格执行,缺失或异常不 fallback。全部返回空后才生成宿主计划,并以第二次 CAS 提升为
|
||||
`planned` 后执行。旧 caller 只经 `ChainBase.transfer` 注入式兼容门面进入同一 durable command,
|
||||
宿主 FileManager/TransHandler 的旧执行入口已删除。
|
||||
- `TransferPending` 现在可区分 `accepted/provider_pending/planned`,但尚无
|
||||
claim/lease/heartbeat/attempt、逐步骤执行结果和 `manual_review`,仍无法判定“文件已移动、历史未提交”
|
||||
等后续中间态。
|
||||
- `S1-L1.3` 已把执行所有权与 planning phase 正交:恢复任务入队前原子 claim,普通任务在任何业务
|
||||
副作用前 claim;heartbeat、checkpoint、失败留痕、release 和终态删除均受当前未过期 token
|
||||
fencing。启动和同进程恢复共享唯一 scheduler,确定性失败按固定轮询退避,关闭时 worker、replay、
|
||||
lease release 和 heartbeat 都由有界生命周期 owner 持有。损坏投影以无有效租约 CAS 留痕,同错不
|
||||
重复刷写,且不会阻塞后续健康任务。
|
||||
- `TransferPending` 仍缺少逐步骤执行结果和 `manual_review`,因此还不能判定“文件已移动、历史未提交”
|
||||
等外部结果未知的后续中间态。
|
||||
- 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、
|
||||
lease/heartbeat 和人工恢复要求不一致。
|
||||
|
||||
@@ -222,7 +227,7 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
|
||||
原子提交,入队失败时必须保留 pending 供重放。
|
||||
- [x] 初始登记保存稳定源身份、版本化请求和状态;目标与有序操作在纯规划完成后以 planning
|
||||
checkpoint 原子更新,任何文件副作用不得早于该提交。
|
||||
- [ ] 增加 claim/lease/heartbeat/attempt 与过期接管,同一任务同时只能有一个 worker owner。
|
||||
- [x] 增加 claim/lease/heartbeat/attempt 与过期接管,同一任务同时只能有一个 worker owner。
|
||||
- [ ] 设计幂等文件操作和历史提交;只有所有必要步骤达到持久终态后才能删除记录。
|
||||
- [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner,定义旧记录迁移和兼容规则。
|
||||
- [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界,不直接套用 E2
|
||||
|
||||
@@ -704,8 +704,8 @@ flowchart LR
|
||||
|
||||
| 指标 | 当前值 |
|
||||
|---|---:|
|
||||
| Python 模块 | 836 |
|
||||
| 内部导入边 | 6,832 |
|
||||
| Python 模块 | 837 |
|
||||
| 内部导入边 | 6,836 |
|
||||
| 非平凡 SCC | 2(`ARCH-107` 临时 Chain 包根环;精确 containment 的 TMDB 移植包环) |
|
||||
| Direct egress | 66(12 条待迁移债务,54 条精确 containment) |
|
||||
| Module Contract V2 spec | 217(其中 215 个进入 `run_module` 观察面) |
|
||||
|
||||
@@ -94,7 +94,7 @@ G-ARCH 只有在以下条件全部满足后才可完成:
|
||||
|---|---|---|---|
|
||||
| S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地;先持久 commit 再入队,入队失败保留可恢复记录;宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission |
|
||||
| S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化;无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划,有 provider 时先提交 `provider_pending`,全部返回空后再以第二次 CAS 提交 `planned`;重放只执行冻结目标,所有文件副作用晚于对应 checkpoint commit |
|
||||
| S1-L1.3 Lease 与恢复调度 | `PLANNED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地;启动回放和同进程恢复共用唯一调度入口,同一任务同时只有一个 worker owner |
|
||||
| S1-L1.3 Lease 与恢复调度 | `VERIFIED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地;启动回放和同进程恢复共用唯一调度入口,同一任务同时只有一个 worker owner |
|
||||
| S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放;唯一 retry owner 生效,未知外部结果进入 `manual_review`,仅完整终态删除 pending |
|
||||
| S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整;旧 fail-open、重复状态与兼容层外旧入口删除,ARCH-102 债务归零 |
|
||||
| S1-L2 Workflow typed query | `PLANNED` | S0 | Workflow Application Port 不返回 `Any`/ORM,Session 内投影 DTO,正式调用方全部切换 |
|
||||
@@ -274,3 +274,83 @@ git diff --check
|
||||
- failure injection 覆盖 commit 前零文件副作用、commit 后崩溃重放、离线 resolved context 恢复、
|
||||
配置漂移仍使用冻结 target storage、规划失败留痕、旧 provider 提交后短路、严格异常、空结果两阶段
|
||||
fallback、缺失引用零 cleanup,以及 cleanup 顺序/幂等/瞬时失败。
|
||||
|
||||
### S1-L1.3 Lease 与恢复调度
|
||||
|
||||
**Status:** `VERIFIED`
|
||||
|
||||
**Outcome**
|
||||
|
||||
为 durable transfer 增加可过期、可接管且带 fencing token 的执行租约。`owner` 只用于标识执行者,
|
||||
每次有效 claim 生成的新 token 才是后续 heartbeat、checkpoint、失败留痕和终态删除的授权;所有写入
|
||||
必须同时匹配当前且未过期的 token,过期 worker 即使稍后恢复也不能修改新 owner 的记录。
|
||||
|
||||
`accepted`、`provider_pending`、`planned` 仍是业务 planning phase,lease 与其正交:claim、heartbeat
|
||||
和 takeover 不改变 phase,恢复 worker 继续按冻结 checkpoint 决定执行路径。启动回放与同进程恢复共用
|
||||
唯一调度入口:恢复任务在入队前完成原子 claim 并绑定 token,新 admission 由 worker 在任何副作用前
|
||||
claim。已经 claim 的恢复任务不在 worker 内二次 claim。
|
||||
|
||||
**State machine**
|
||||
|
||||
| 当前租约 | 操作 | 结果与 fencing 约束 |
|
||||
|---|---|---|
|
||||
| 无租约 | `claim` | 生成新 token、设置 owner/到期时间并递增 attempt;phase 不变 |
|
||||
| 当前租约未过期 | 同 token `heartbeat` | 只延长当前租约;owner、token、attempt 和 phase 不变 |
|
||||
| 当前租约未过期 | 任意再次 `claim`,包括同 owner | 原子拒绝,不递增 attempt、不入队、不执行副作用 |
|
||||
| 当前租约已过期 | `heartbeat` 或旧 token 写入 | 原子拒绝;旧租约不可复活 |
|
||||
| 当前租约已过期 | 新 worker `claim` | 生成新 token 并递增 attempt;旧 token 永久失效,phase 不变 |
|
||||
| 当前 token 有效 | checkpoint、失败留痕或终态删除 | 仅精确 token CAS 成功;状态提交后不得由旧 worker 覆盖 |
|
||||
|
||||
任意时刻一条 pending 记录最多只有一个数据库认可的有效 fencing owner。该保证不等同于外部文件、
|
||||
插件或历史副作用 exactly-once;租约在不可中断调用期间过期时,旧调用的外部结果仍可能未知。
|
||||
|
||||
**Scheduling and shutdown**
|
||||
|
||||
- 启动回放和同进程恢复只调用一个 claim-and-schedule 入口;禁止另建 list-then-enqueue 回放路径或
|
||||
第二个 retry owner。批量恢复必须先原子 claim,再把已绑定 token 的任务交给唯一 worker 队列。
|
||||
- heartbeat 由 Transfer worker 生命周期拥有,不创建游离后台 owner;停止时先封口新 claim 和调度,
|
||||
再通知并有限等待 worker,heartbeat 在 worker 存活期间继续维持其租约。worker 收敛后才停止
|
||||
heartbeat;超时时保留仍存活的 worker/heartbeat owner 并返回失败。
|
||||
- checkpoint、失败留痕和 pending 删除在同次持久化写入中执行 token CAS;租约丢失后 worker 停止
|
||||
继续提交状态,不以预查询替代 fencing,也不把 stale mutation 伪装成成功。
|
||||
- 确定性失败只确保唯一 scheduler 存在,不即时唤醒;新建的失败恢复 owner 首次扫描先等待固定轮询
|
||||
周期。损坏持久投影按单任务事务回滚,再以无有效租约 CAS 留下去重诊断,不能饿死后续健康记录。
|
||||
- 精确旧 `app.db.transferpending_oper` 导入只解析到 `app/sdk/_legacy` 门面;canonical Model、Oper、
|
||||
Application Port 不保留旧 list-then-act 或无 fencing mutation。兼容 `discard/clear` 也不得删除任何
|
||||
带 token 的有效或过期 claim。
|
||||
|
||||
**Excluded**
|
||||
|
||||
- 本叶不承诺文件、legacy provider、历史写入或其他外部副作用 exactly-once,也不以延长 lease
|
||||
掩盖不可判定结果。
|
||||
- 文件步骤幂等键、逐步结果 checkpoint、崩溃后未知结果判定、唯一 retry owner 和
|
||||
`manual_review` 终态由 `S1-L1.4` 完整交付。
|
||||
- 不增加 `processing` 等与 planning phase 重复的业务状态;不修改插件公开 Transfer ABI,不扫描或
|
||||
修改 `app/plugins/**`,不恢复宿主旧 pending 入口或重复导出。
|
||||
|
||||
**Acceptance matrix**
|
||||
|
||||
| 场景 | 必须证明 |
|
||||
|---|---|
|
||||
| 新 admission 与恢复记录竞争 | 只有 claim 成功者入队并执行;失败者零文件副作用 |
|
||||
| 两进程同时 claim 同一记录 | 仅一个新 token 成功,attempt 只按真实新 claim 增长 |
|
||||
| heartbeat 与 takeover 竞争 | 未过期 heartbeat 可续租;过期租约不可复活,接管 token 唯一有效 |
|
||||
| 旧 worker 延迟提交 | checkpoint、失败留痕和删除均被 token CAS 拒绝,不覆盖新 owner |
|
||||
| 三种 planning phase 恢复 | claim/续租/接管保持 phase,并消费各自冻结输入或 checkpoint |
|
||||
| 启动与同进程恢复同时触发 | 只经过唯一 scheduler 入口,同一任务只有一个 worker owner |
|
||||
| 正常与超时关闭 | 先封口后有限等待;存活 scheduler/heartbeat/worker 不丢失 owner、不报告成功 |
|
||||
|
||||
**Failure injection matrix**
|
||||
|
||||
| 注入点 | 预期持久结果 |
|
||||
|---|---|
|
||||
| claim commit 前崩溃 | 无新 token、attempt 不变,可由后续 worker claim |
|
||||
| claim commit 后、入队前崩溃 | 租约到期后可接管,新 token fencing 旧 worker |
|
||||
| worker 执行前 lease 丢失 | 不执行文件副作用,不提交失败或终态状态 |
|
||||
| heartbeat commit 前后崩溃 | 仅已提交到期时间生效;过期后不可用旧 token 续租 |
|
||||
| checkpoint/失败留痕提交时被接管 | stale CAS 失败,新 owner 的 phase、错误和 token 不被覆盖 |
|
||||
| 终态删除提交时被接管 | stale delete 为零行,pending 保留给当前 owner |
|
||||
| shutdown 时 scheduler、worker 或 lease release 阻塞 | 有限等待返回失败并保留 owner/heartbeat,禁止清句柄后重建重复 owner |
|
||||
|
||||
本叶验收还必须运行 lease persistence、replay/worker、startup lifecycle、migration、架构与兼容聚焦
|
||||
测试,以及锁定全量测试和 scoped Pylint;插件 ABI 只经统一 Compat/SDK 验证。
|
||||
|
||||
@@ -484,8 +484,30 @@ result permits host planning and a second CAS to `planned`. Host-only `plan_tran
|
||||
is the sole legacy caller facade and delegates the startup-injected durable
|
||||
command; `FileManagerModule.transfer` and `TransHandler.transfer_media` must not
|
||||
be recreated.
|
||||
Canonical host chains never obtain `TransferPendingOper`; its no-Session API
|
||||
remains only for the exact legacy plugin import contract.
|
||||
|
||||
Transfer execution ownership is orthogonal to those planning phases.
|
||||
`app/application/transfer.py` defines the claim, heartbeat, release and fenced
|
||||
mutation Port; `app/db/adapters/transfer.py` implements each operation in a
|
||||
short UoW with a unique lease token. Any active lease rejects another claim,
|
||||
including one from the same process owner. Expired leases may be taken over with
|
||||
a new token and incremented attempt count, while the stale token cannot renew,
|
||||
checkpoint, record failure, release or delete the task. Startup replay and
|
||||
same-process recovery use the single scheduler owned by `TransferChain`; the
|
||||
scheduler claims before enqueueing, and queued or executing claims are renewed
|
||||
by one lifecycle-managed heartbeat owner. Lease ownership guarantees one
|
||||
database-authorized worker, not physical exactly-once behavior for an already
|
||||
issued file or legacy-plugin side effect; step idempotency and unknown outcomes
|
||||
remain explicit execution concerns.
|
||||
|
||||
Canonical host chains never obtain `TransferPendingOper`. The canonical Model,
|
||||
Oper and Application Port do not retain the historical `register`, `list_all`,
|
||||
`discard`, `clear` or `list_by_*` surface. The exact
|
||||
`app.db.transferpending_oper` mapping resolves instead to the private
|
||||
`app/sdk/_legacy/transferpending.py` facade, which preserves the old no-Session
|
||||
query ABI without becoming a host implementation. Its `register` delegates to
|
||||
canonical durable admission without overwriting an existing row, while
|
||||
`discard` and `clear` delete only rows whose `lease_token` is null; an active or
|
||||
expired claimed task remains exclusively owned by fenced recovery APIs.
|
||||
|
||||
## Composition and Compatibility Boundaries
|
||||
|
||||
|
||||
+8
-3
@@ -1441,8 +1441,8 @@
|
||||
"runtime_only": true
|
||||
}
|
||||
},
|
||||
"edge_count": 6832,
|
||||
"edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39",
|
||||
"edge_count": 6836,
|
||||
"edge_sha256": "3709e09a49257075f48a44aba353be1f25db610471ec756e09cbbfd7070acb9a",
|
||||
"edges": [
|
||||
"app -> app.runtime",
|
||||
"app -> app.runtime.compat",
|
||||
@@ -7625,6 +7625,10 @@
|
||||
"app.sdk._legacy.subscribe -> app.domain.context",
|
||||
"app.sdk._legacy.transfer -> app.application",
|
||||
"app.sdk._legacy.transfer -> app.application.transfer",
|
||||
"app.sdk._legacy.transferpending -> app.db",
|
||||
"app.sdk._legacy.transferpending -> app.db.base",
|
||||
"app.sdk._legacy.transferpending -> app.db.models",
|
||||
"app.sdk._legacy.transferpending -> app.db.models.transferpending",
|
||||
"app.sdk._legacy.user -> app.api",
|
||||
"app.sdk._legacy.user -> app.api.deps",
|
||||
"app.sdk._legacy.user -> app.db",
|
||||
@@ -8277,7 +8281,7 @@
|
||||
"app.workflow.actions.transfer_file -> app.workflow",
|
||||
"app.workflow.actions.transfer_file -> app.workflow.actions"
|
||||
],
|
||||
"module_count": 836,
|
||||
"module_count": 837,
|
||||
"modules": [
|
||||
"app",
|
||||
"app.adapters",
|
||||
@@ -9058,6 +9062,7 @@
|
||||
"app.sdk._legacy.history",
|
||||
"app.sdk._legacy.subscribe",
|
||||
"app.sdk._legacy.transfer",
|
||||
"app.sdk._legacy.transferpending",
|
||||
"app.sdk._legacy.user",
|
||||
"app.sdk.browser",
|
||||
"app.sdk.cache",
|
||||
|
||||
+1
-1
@@ -1594,7 +1594,7 @@
|
||||
"misc": 2,
|
||||
"no-any-return": 3,
|
||||
"no-untyped-call": 8,
|
||||
"no-untyped-def": 14,
|
||||
"no-untyped-def": 12,
|
||||
"operator": 3,
|
||||
"return-value": 2,
|
||||
"truthy-function": 5,
|
||||
|
||||
@@ -249,9 +249,9 @@
|
||||
"app.db.transferpending_oper": {
|
||||
"introduced": "v3.0.0",
|
||||
"is_package": false,
|
||||
"owner": "db",
|
||||
"replacement": "app.db.oper.transferpending",
|
||||
"target": "app.db.oper.transferpending"
|
||||
"owner": "sdk",
|
||||
"replacement": "app.application.transfer",
|
||||
"target": "app.sdk._legacy.transferpending"
|
||||
},
|
||||
"app.db.user_oper": {
|
||||
"introduced": "v3.0.0",
|
||||
|
||||
@@ -828,15 +828,19 @@ def test_models_and_base_require_explicit_database_sessions():
|
||||
|
||||
|
||||
def test_plugin_sdk_does_not_import_or_export_host_models():
|
||||
"""插件 SDK 只能暴露 Oper,不得把宿主 ORM Model 作为插件接口。"""
|
||||
"""插件 SDK 不得暴露 ORM Model,只有精确旧 ABI 门面可在内部访问。"""
|
||||
internal_compat_imports = {
|
||||
("app/sdk/_legacy/transferpending.py", "app.db.models.transferpending"),
|
||||
}
|
||||
violations: list[str] = []
|
||||
for path in (APP_ROOT / "sdk").rglob("*.py"):
|
||||
relative = path.relative_to(PROJECT_ROOT).as_posix()
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module and (
|
||||
node.module == "app.db.models"
|
||||
or node.module.startswith("app.db.models.")
|
||||
):
|
||||
) and (relative, node.module) not in internal_compat_imports:
|
||||
violations.append(
|
||||
f"{path.relative_to(PROJECT_ROOT)}:{node.lineno}:{node.module}"
|
||||
)
|
||||
|
||||
@@ -158,8 +158,18 @@ def test_tables_without_identity_columns_are_untouched(db):
|
||||
不带身份列的表不受影响——事件挂在 Mapper 上覆盖全部映射,必须靠列名检查收窄,
|
||||
否则会去动一张根本没有这两列的表。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-14 10:00:00")
|
||||
TransferPending.stage_admit(
|
||||
db.session,
|
||||
task_id="identity-free-table",
|
||||
storage="local",
|
||||
src_path="/mnt/a.mkv",
|
||||
state="accepted",
|
||||
now_time="2026-08-14 10:00:00",
|
||||
)
|
||||
|
||||
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/mnt/a.mkv"]
|
||||
assert len(rows) == 1
|
||||
row = TransferPending.get_by_identity(
|
||||
db.session,
|
||||
storage="local",
|
||||
src_path="/mnt/a.mkv",
|
||||
)
|
||||
assert row.task_id == "identity-free-table"
|
||||
|
||||
@@ -5,8 +5,10 @@
|
||||
任何一件出偏差,都直接表现为文件被漏整理或被重复整理,而不是一个可见的报错。
|
||||
因此这里对着真实数据库断言查回的内容,而不是断言调用了什么。
|
||||
"""
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.db import base as db_base
|
||||
@@ -21,185 +23,6 @@ def _track(db):
|
||||
db.watermark(TransferPending)
|
||||
|
||||
|
||||
def test_register_is_idempotent_and_keeps_first_time(db):
|
||||
"""
|
||||
同一文件重复登记只保留一条,且登记时间保持首次的值。
|
||||
|
||||
监控在挂载抖动时会对同一个文件反复触发事件,若每次都新增一条,回放时同一个
|
||||
文件会被送进整理链多次。保留首次时间则保证回放顺序仍是「最早发现」的顺序。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 12:00:00")
|
||||
|
||||
rows = TransferPending.list_all(db.session)
|
||||
same_path = [r for r in rows if r.src_path == "/mnt/a.mkv"]
|
||||
assert len(same_path) == 1
|
||||
assert same_path[0].created_at == "2026-08-13 10:00:00"
|
||||
|
||||
|
||||
def test_register_scopes_by_storage(db):
|
||||
"""
|
||||
存储不同即为不同文件——路径相同但分属不同存储时不能互相去重。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/data/x.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="alist", src_path="/data/x.mkv",
|
||||
now_time="2026-08-13 10:00:01")
|
||||
|
||||
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/data/x.mkv"]
|
||||
assert {r.storage for r in rows} == {"local", "alist"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", ""), ("", "")])
|
||||
def test_register_rejects_incomplete_identity(db, storage, src_path):
|
||||
"""
|
||||
缺少存储或路径的登记必须直接丢弃,不能写入半条记录。
|
||||
|
||||
半条记录回放时既定位不到文件、也无法被 discard 匹配,会永久留在表里。
|
||||
"""
|
||||
assert TransferPending.register(db.session, storage=storage, src_path=src_path,
|
||||
now_time="2026-08-13 10:00:00") is None
|
||||
|
||||
|
||||
def test_list_all_replays_in_registration_order(db):
|
||||
"""
|
||||
回放顺序必须是登记时间升序、同时间按主键升序。
|
||||
|
||||
乱序回放会让后发现的文件先进整理链,与原入队顺序不一致。
|
||||
"""
|
||||
for path, moment in [("/mnt/c.mkv", "2026-08-13 12:00:00"),
|
||||
("/mnt/a.mkv", "2026-08-13 10:00:00"),
|
||||
("/mnt/b.mkv", "2026-08-13 11:00:00")]:
|
||||
TransferPending.register(db.session, storage="local", src_path=path, now_time=moment)
|
||||
|
||||
ordered = [r.src_path for r in TransferPending.list_all(db.session)
|
||||
if r.src_path.startswith("/mnt/")]
|
||||
assert ordered == ["/mnt/a.mkv", "/mnt/b.mkv", "/mnt/c.mkv"]
|
||||
|
||||
|
||||
def test_list_all_honours_limit(db):
|
||||
"""
|
||||
回放上限必须生效——异常积压时一次性全放会把整理链直接压垮。
|
||||
"""
|
||||
for index in range(5):
|
||||
TransferPending.register(db.session, storage="local", src_path=f"/mnt/{index}.mkv",
|
||||
now_time=f"2026-08-13 10:00:0{index}")
|
||||
|
||||
assert len(TransferPending.list_all(db.session, limit=3)) == 3
|
||||
|
||||
|
||||
def test_discard_removes_only_the_matching_row(db):
|
||||
"""
|
||||
注销只应删除匹配的那一条,并返回删除条数。
|
||||
|
||||
整理到达终态时按「存储 + 路径」注销,误删其他登记等于把别的文件也判成已完成。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/b.mkv",
|
||||
now_time="2026-08-13 10:00:01")
|
||||
|
||||
assert TransferPending.discard(db.session, storage="local", src_path="/mnt/a.mkv") == 1
|
||||
|
||||
remaining = [r.src_path for r in TransferPending.list_all(db.session)
|
||||
if r.src_path.startswith("/mnt/")]
|
||||
assert remaining == ["/mnt/b.mkv"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", "")])
|
||||
def test_discard_rejects_incomplete_identity(db, storage, src_path):
|
||||
"""
|
||||
身份不全时必须直接返回 0,不能退化成「条件为空」的全表删除。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
|
||||
assert TransferPending.discard(db.session, storage=storage, src_path=src_path) == 0
|
||||
assert TransferPending.list_all(db.session)
|
||||
|
||||
|
||||
def test_discard_returns_zero_when_absent(db):
|
||||
"""
|
||||
注销不存在的登记返回 0,不抛异常——整理链的终态回调不应因此中断。
|
||||
"""
|
||||
assert TransferPending.discard(db.session, storage="local", src_path="/nope.mkv") == 0
|
||||
|
||||
|
||||
def test_clear_empties_the_table(db):
|
||||
"""
|
||||
清空返回删除条数且表内不再有登记。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
|
||||
assert TransferPending.clear(db.session) >= 1
|
||||
assert TransferPending.list_all(db.session) == []
|
||||
|
||||
|
||||
def test_oper_returns_plain_tuples_not_orm_instances(db):
|
||||
"""
|
||||
回放接口必须返回纯元组。
|
||||
|
||||
回放发生在会话之外,ORM 实例脱离 session 后访问属性会抛
|
||||
DetachedInstanceError——那时启动流程已经在跑,报错等于整批漏件。
|
||||
"""
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/a.mkv")
|
||||
|
||||
listed = oper.list_all()
|
||||
|
||||
assert ("local", "/mnt/a.mkv") in listed
|
||||
assert all(isinstance(item, tuple) for item in listed)
|
||||
|
||||
|
||||
def test_oper_reuses_explicit_query_session(db, monkeypatch):
|
||||
"""TransferPendingOper 查询必须复用调用方会话。"""
|
||||
db.add(TransferPending(
|
||||
storage="local",
|
||||
src_path="/mnt/explicit.mkv",
|
||||
created_at="2026-08-13 10:00:00",
|
||||
))
|
||||
monkeypatch.setattr(
|
||||
db_base,
|
||||
"run_sync_transaction",
|
||||
lambda _operation: (_ for _ in ()).throw(
|
||||
AssertionError("不应创建额外同步事务")
|
||||
),
|
||||
)
|
||||
|
||||
assert ("local", "/mnt/explicit.mkv") in TransferPendingOper(db.session).list_all()
|
||||
|
||||
|
||||
def test_oper_drops_rows_with_missing_fields(db):
|
||||
"""
|
||||
回放时必须跳过字段残缺的历史遗留行,不能把空存储送进整理链。
|
||||
|
||||
列上有 NOT NULL 约束,残缺只可能表现为空串;直接绕过 register 写入,
|
||||
模拟历史数据或外部写库留下的半条记录。
|
||||
"""
|
||||
db.add(TransferPending(storage="", src_path="/mnt/broken.mkv",
|
||||
created_at="2026-08-13 10:00:00"))
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/ok.mkv")
|
||||
|
||||
assert oper.list_all() == [("local", "/mnt/ok.mkv")]
|
||||
|
||||
|
||||
def test_oper_discard_and_clear_report_counts(db):
|
||||
"""
|
||||
注销与清空都要如实返回条数,调用方据此判断是否真的清理掉了。
|
||||
"""
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/a.mkv")
|
||||
oper.register(storage="local", src_path="/mnt/b.mkv")
|
||||
|
||||
assert oper.discard(storage="local", src_path="/mnt/a.mkv") == 1
|
||||
assert oper.clear() >= 1
|
||||
assert oper.list_all() == []
|
||||
|
||||
|
||||
def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
|
||||
"""显式准入重复执行时必须复用首个稳定任务标识。"""
|
||||
first = TransferPending.stage_admit(
|
||||
@@ -224,8 +47,32 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
|
||||
assert second.updated_at == "2026-08-27 10:00:00"
|
||||
|
||||
|
||||
def test_state_queries_failure_record_and_task_discard(db):
|
||||
"""状态查询、失败留痕和按任务删除应共享同一稳定身份。"""
|
||||
def test_unfenced_legacy_mutation_apis_are_absent() -> None:
|
||||
"""持久整理表不得重新暴露绕过稳定任务身份和租约的旧写入口。"""
|
||||
for owner in (TransferPending, TransferPendingOper):
|
||||
for method_name in (
|
||||
"register",
|
||||
"discard",
|
||||
"list_all",
|
||||
"list_by_state",
|
||||
"list_by_states",
|
||||
"clear",
|
||||
):
|
||||
assert not hasattr(owner, method_name)
|
||||
|
||||
enqueue_failure_source = inspect.getsource(
|
||||
TransferPending.record_enqueue_failure
|
||||
)
|
||||
assert "cls.lease_token.is_(None)" in enqueue_failure_source
|
||||
claimable_source = inspect.getsource(
|
||||
TransferPending.list_claimable_candidates
|
||||
)
|
||||
assert "after_cursor" in claimable_source
|
||||
assert ".not_in(" not in claimable_source
|
||||
|
||||
|
||||
def test_state_queries_and_failure_record_share_stable_identity(db):
|
||||
"""状态查询与失败留痕应共享同一稳定身份。"""
|
||||
TransferPending.stage_admit(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
@@ -244,11 +91,11 @@ def test_state_queries_failure_record_and_task_discard(db):
|
||||
))
|
||||
db.session.flush()
|
||||
|
||||
accepted = TransferPending.list_by_state(
|
||||
accepted = TransferPending.get_by_task_id(
|
||||
db.session,
|
||||
state="accepted",
|
||||
task_id="task-accepted",
|
||||
)
|
||||
assert [item.task_id for item in accepted] == ["task-accepted"]
|
||||
assert accepted.state == "accepted"
|
||||
assert TransferPending.record_enqueue_failure(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
@@ -263,10 +110,6 @@ def test_state_queries_failure_record_and_task_discard(db):
|
||||
)
|
||||
assert failed.last_error == "queue full"
|
||||
assert failed.updated_at == "2026-08-27 10:01:00"
|
||||
assert TransferPending.discard_task(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
) == 1
|
||||
|
||||
|
||||
def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
|
||||
@@ -288,15 +131,13 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
|
||||
now_time="2026-08-27 10:00:00",
|
||||
)
|
||||
assert pending.task_id == "task-explicit"
|
||||
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
|
||||
"task-explicit"
|
||||
]
|
||||
assert oper.get_by_task_id(task_id="task-explicit").state == "accepted"
|
||||
assert oper.stage_record_enqueue_failure(
|
||||
task_id="task-explicit",
|
||||
error="queue full",
|
||||
now_time="2026-08-27 10:01:00",
|
||||
) == 1
|
||||
assert oper.stage_discard_task(task_id="task-explicit") == 1
|
||||
assert oper.get_by_task_id(task_id="task-explicit").last_error == "queue full"
|
||||
|
||||
|
||||
def test_transactional_repository_commits_frozen_projections(tmp_path):
|
||||
@@ -317,16 +158,34 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
|
||||
assert repeated == admitted
|
||||
assert admitted.task_id
|
||||
assert admitted.state == "accepted"
|
||||
assert repository.list_accepted() == [admitted]
|
||||
|
||||
repository.record_enqueue_failure(
|
||||
task_id=admitted.task_id,
|
||||
error="queue full",
|
||||
)
|
||||
failed = repository.list_accepted()[0]
|
||||
assert failed.last_error == "queue full"
|
||||
assert repository.discard_task(task_id=admitted.task_id) == 1
|
||||
assert repository.list_accepted() == []
|
||||
with factory() as session:
|
||||
failed = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert failed.last_error == "queue full"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="repository-test-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) == 1
|
||||
with factory() as session:
|
||||
assert session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one_or_none() is None
|
||||
engine.dispose()
|
||||
|
||||
|
||||
|
||||
@@ -305,10 +305,16 @@ def test_legacy_transfer_task_hides_internal_admission_identity():
|
||||
public_fields = set(task.to_dict())
|
||||
|
||||
task.bind_admission_task_id("internal-task-id")
|
||||
task.bind_execution_lease(
|
||||
owner_id="internal-worker",
|
||||
lease_token="internal-lease-token",
|
||||
)
|
||||
|
||||
assert set(task.to_dict()) == public_fields
|
||||
assert "task_id" not in task.to_dict()
|
||||
assert "admission_task_id" not in task.to_dict()
|
||||
assert "lease_owner" not in task.to_dict()
|
||||
assert "lease_token" not in task.to_dict()
|
||||
|
||||
|
||||
def test_chain_media_legacy_scraping_symbols_resolve_to_scraping_chain():
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -8,7 +9,11 @@ from app.application.history import (
|
||||
failed_retry_count,
|
||||
record_transfer_failure,
|
||||
)
|
||||
from app.application.transfer import TransferPlanningInput, TransferTask
|
||||
from app.application.transfer import (
|
||||
TransferAdmission,
|
||||
TransferPlanningInput,
|
||||
TransferTask,
|
||||
)
|
||||
from app.chain.transfer import JobManager, TransferChain
|
||||
from app.domain.context import MediaInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
@@ -151,6 +156,7 @@ def make_task(episode: int, season: int = 1) -> TransferTask:
|
||||
|
||||
|
||||
def make_transfer_chain() -> TransferChain:
|
||||
"""构造带内存 durable admission 契约的整理链测试骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.jobview = JobManager()
|
||||
chain._media_exts = settings.RMT_MEDIAEXT
|
||||
@@ -161,6 +167,66 @@ def make_transfer_chain() -> TransferChain:
|
||||
)
|
||||
chain._success_target_files = {}
|
||||
chain._scrape_batches = {}
|
||||
admissions = MagicMock()
|
||||
admissions_by_identity = {}
|
||||
admissions_by_id = {}
|
||||
|
||||
def admit(*, storage, src_path, planning_input=None):
|
||||
"""按源身份幂等返回测试用 durable admission。"""
|
||||
identity = storage, src_path
|
||||
existing = admissions_by_identity.get(identity)
|
||||
if existing is not None:
|
||||
return existing
|
||||
admission = TransferAdmission(
|
||||
task_id=f"test-task-{len(admissions_by_id) + 1}",
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
admissions_by_identity[identity] = admission
|
||||
admissions_by_id[admission.task_id] = admission
|
||||
return admission
|
||||
|
||||
def claim_task(*, task_id, owner_id, lease_seconds):
|
||||
"""为测试任务返回唯一 token,并保留正式 claim 的参数约束。"""
|
||||
assert lease_seconds > 0
|
||||
admission = admissions_by_id[task_id]
|
||||
claimed = replace(
|
||||
admission,
|
||||
lease_owner=owner_id,
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=admission.attempt_count + 1,
|
||||
)
|
||||
admissions_by_identity[(claimed.storage, claimed.src_path)] = claimed
|
||||
admissions_by_id[task_id] = claimed
|
||||
return claimed
|
||||
|
||||
def checkpoint_plan(*, task_id, lease_token, input_fingerprint, checkpoint):
|
||||
"""回读带检查点的持久投影,供同步整理测试执行真实编排。"""
|
||||
del input_fingerprint
|
||||
admission = admissions_by_id[task_id]
|
||||
assert admission.lease_token == lease_token
|
||||
planned = replace(
|
||||
admission,
|
||||
state="provider_pending" if checkpoint.is_provider_pending else "planned",
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
admissions_by_identity[(planned.storage, planned.src_path)] = planned
|
||||
admissions_by_id[task_id] = planned
|
||||
return planned
|
||||
|
||||
admissions.admit.side_effect = admit
|
||||
admissions.claim_task.side_effect = claim_task
|
||||
admissions.checkpoint_plan.side_effect = checkpoint_plan
|
||||
admissions.discard_claimed.return_value = 1
|
||||
admissions.release_claim.return_value = True
|
||||
chain._transfer_admissions = admissions
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""整理恢复租约字段的 Alembic 可逆迁移测试。"""
|
||||
|
||||
import importlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
|
||||
from app.db.models.transferpending import TransferPending
|
||||
|
||||
MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
|
||||
LEASE_COLUMNS = {
|
||||
"lease_owner",
|
||||
"lease_token",
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
"attempt_count",
|
||||
}
|
||||
|
||||
|
||||
def _bind_migration(monkeypatch, connection):
|
||||
"""把 3.0.15 迁移绑定到隔离数据库连接。"""
|
||||
migration = importlib.import_module(MIGRATION)
|
||||
monkeypatch.setattr(
|
||||
migration,
|
||||
"op",
|
||||
Operations(MigrationContext.configure(connection)),
|
||||
)
|
||||
return migration
|
||||
|
||||
|
||||
def _create_planning_table(connection) -> None:
|
||||
"""创建 3.0.14 时代包含完整规划检查点的登记表。"""
|
||||
metadata = sa.MetaData()
|
||||
table = sa.Table(
|
||||
"transferpending",
|
||||
metadata,
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("task_id", sa.String(64), nullable=False),
|
||||
sa.Column("storage", sa.String(), nullable=False),
|
||||
sa.Column("src_path", sa.String(), nullable=False),
|
||||
sa.Column("created_at", sa.String(), nullable=True),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("updated_at", sa.String(40), nullable=False),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("input_version", sa.Integer(), nullable=False),
|
||||
sa.Column("planning_input", sa.JSON(), nullable=False),
|
||||
sa.Column("input_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("checkpoint_version", sa.Integer(), nullable=True),
|
||||
sa.Column("checkpoint_payload", sa.JSON(), nullable=True),
|
||||
sa.Column("planned_at", sa.String(40), nullable=True),
|
||||
sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"),
|
||||
)
|
||||
sa.Index(
|
||||
"ux_transferpending_storage_path",
|
||||
table.c.storage,
|
||||
table.c.src_path,
|
||||
unique=True,
|
||||
)
|
||||
sa.Index(
|
||||
"ix_transferpending_state_created",
|
||||
table.c.state,
|
||||
table.c.created_at,
|
||||
table.c.id,
|
||||
)
|
||||
metadata.create_all(connection)
|
||||
connection.execute(
|
||||
table.insert(),
|
||||
{
|
||||
"id": 1,
|
||||
"task_id": "stable-task",
|
||||
"storage": "local",
|
||||
"src_path": "/downloads/Movie.mkv",
|
||||
"created_at": "2026-08-27 10:00:00",
|
||||
"state": "planned",
|
||||
"updated_at": "2026-08-27 10:00:00",
|
||||
"last_error": None,
|
||||
"input_version": 1,
|
||||
"planning_input": {
|
||||
"schema_version": 1,
|
||||
"source_fileitem": {
|
||||
"storage": "local",
|
||||
"path": "/downloads/Movie.mkv",
|
||||
},
|
||||
},
|
||||
"input_fingerprint": "0" * 64,
|
||||
"checkpoint_version": 1,
|
||||
"checkpoint_payload": {"schema_version": 1},
|
||||
"planned_at": "2026-08-27 10:00:00",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None:
|
||||
"""SQLite 应支持租约字段重复升级、降级和再次升级。"""
|
||||
engine = sa.create_engine("sqlite://")
|
||||
with engine.begin() as connection:
|
||||
_create_planning_table(connection)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
|
||||
migration.upgrade()
|
||||
migration.upgrade()
|
||||
|
||||
inspector = sa.inspect(connection)
|
||||
assert {
|
||||
column["name"]
|
||||
for column in inspector.get_columns("transferpending")
|
||||
} == {column.name for column in TransferPending.__table__.columns}
|
||||
assert "ix_transferpending_recovery_lease" in {
|
||||
index["name"]
|
||||
for index in inspector.get_indexes("transferpending")
|
||||
}
|
||||
row = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, lease_expires_at, "
|
||||
"heartbeat_at, attempt_count FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(row) == {
|
||||
"lease_owner": None,
|
||||
"lease_token": None,
|
||||
"lease_expires_at": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"UPDATE transferpending SET lease_owner = 'worker', "
|
||||
"lease_token = 'token', "
|
||||
"lease_expires_at = '2026-08-27 10:05:00.000000', "
|
||||
"heartbeat_at = '2026-08-27 10:00:00.000000', "
|
||||
"attempt_count = 3 WHERE id = 1"
|
||||
)
|
||||
)
|
||||
migration.downgrade()
|
||||
|
||||
downgraded = sa.inspect(connection)
|
||||
assert LEASE_COLUMNS.isdisjoint({
|
||||
column["name"]
|
||||
for column in downgraded.get_columns("transferpending")
|
||||
})
|
||||
assert "ix_transferpending_recovery_lease" not in {
|
||||
index["name"]
|
||||
for index in downgraded.get_indexes("transferpending")
|
||||
}
|
||||
assert connection.execute(
|
||||
sa.text("SELECT state FROM transferpending WHERE id = 1")
|
||||
).scalar_one() == "planned"
|
||||
|
||||
migration.upgrade()
|
||||
reupgraded = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, attempt_count "
|
||||
"FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(reupgraded) == {
|
||||
"lease_owner": None,
|
||||
"lease_token": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_partial_transfer_lease_upgrade_preserves_existing_owner(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""迁移中断后重跑应补齐字段且不得覆盖已经写入的租约拥有者。"""
|
||||
engine = sa.create_engine("sqlite://")
|
||||
with engine.begin() as connection:
|
||||
_create_planning_table(connection)
|
||||
connection.execute(
|
||||
sa.text("ALTER TABLE transferpending ADD COLUMN lease_owner VARCHAR(128)")
|
||||
)
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"UPDATE transferpending SET lease_owner = 'preserved-worker' "
|
||||
"WHERE id = 1"
|
||||
)
|
||||
)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
|
||||
migration.upgrade()
|
||||
|
||||
row = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, attempt_count "
|
||||
"FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(row) == {
|
||||
"lease_owner": "preserved-worker",
|
||||
"lease_token": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,801 @@
|
||||
"""整理恢复租约的原子 claim、续租和陈旧 token 防护测试。"""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Barrier, Lock
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application.transfer import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TransferAdmission,
|
||||
TransferAdmissionProjectionError,
|
||||
TransferLeaseLostError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanningInput,
|
||||
TransferProviderInvocationSnapshot,
|
||||
TransferProviderReference,
|
||||
)
|
||||
from app.db.adapters.transfer import TransactionalTransferAdmissionRepository
|
||||
from app.db.models.transferpending import TransferPending
|
||||
from app.db.oper.transferpending import TransferPendingOper
|
||||
|
||||
|
||||
def _planning_input(path: str) -> TransferPlanningInput:
|
||||
"""构造与测试源路径绑定的最小规划输入。"""
|
||||
return TransferPlanningInput(
|
||||
source_fileitem={
|
||||
"storage": "local",
|
||||
"path": path,
|
||||
"type": "file",
|
||||
"name": path.rsplit("/", maxsplit=1)[-1],
|
||||
},
|
||||
meta={"name": "Movie"},
|
||||
mediainfo={"title": "Movie"},
|
||||
)
|
||||
|
||||
|
||||
def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
|
||||
"""构造无需文件副作用的合法宿主跳过检查点。"""
|
||||
return TransferPlanCheckpoint(
|
||||
planning_input=planning_input,
|
||||
target_storage="local",
|
||||
root_target_path="/library",
|
||||
final_target_path="/library",
|
||||
resolved_transfer_type="copy",
|
||||
items=(),
|
||||
skip_reason="测试跳过计划",
|
||||
)
|
||||
|
||||
|
||||
def _provider_checkpoint(
|
||||
planning_input: TransferPlanningInput,
|
||||
) -> TransferPlanCheckpoint:
|
||||
"""构造只冻结 provider ABI、尚未完成宿主规划的检查点。"""
|
||||
invocation = TransferProviderInvocationSnapshot(
|
||||
fileitem=planning_input.source_fileitem,
|
||||
meta=planning_input.meta,
|
||||
meta_kind="MetaVideo",
|
||||
mediainfo=planning_input.mediainfo,
|
||||
mediainfo_kind="MediaInfo",
|
||||
)
|
||||
return TransferPlanCheckpoint(
|
||||
planning_input=planning_input,
|
||||
target_storage="",
|
||||
root_target_path="",
|
||||
final_target_path="",
|
||||
resolved_transfer_type="",
|
||||
items=(),
|
||||
resolved_meta=invocation.meta,
|
||||
resolved_meta_kind=invocation.meta_kind,
|
||||
resolved_mediainfo=invocation.mediainfo,
|
||||
resolved_mediainfo_kind=invocation.mediainfo_kind,
|
||||
legacy_transfer_providers=(
|
||||
TransferProviderReference(
|
||||
plugin_id="provider-a",
|
||||
plugin_name="Provider A",
|
||||
),
|
||||
),
|
||||
provider_invocation=invocation,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repository_factory(tmp_path):
|
||||
"""创建允许多线程独立 Session 竞争的 SQLite 租约仓储工厂。"""
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'transfer-lease.db'}",
|
||||
connect_args={"check_same_thread": False, "timeout": 10},
|
||||
)
|
||||
TransferPending.__table__.create(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
yield lambda: TransactionalTransferAdmissionRepository(factory)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lease_clock(monkeypatch):
|
||||
"""为所有仓储实例提供可推进的固定 UTC 时钟。"""
|
||||
clock = {"now": datetime(2026, 8, 27, 10, 0, tzinfo=timezone.utc)}
|
||||
monkeypatch.setattr(
|
||||
TransactionalTransferAdmissionRepository,
|
||||
"_lease_now",
|
||||
staticmethod(lambda: clock["now"]),
|
||||
)
|
||||
return clock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def business_clock(monkeypatch):
|
||||
"""为业务审计字段提供与 UTC 租约时钟明确分离的宿主本地时间。"""
|
||||
clock = {"now": "2026-08-27 18:00:00"}
|
||||
monkeypatch.setattr(
|
||||
TransactionalTransferAdmissionRepository,
|
||||
"_now",
|
||||
staticmethod(lambda: clock["now"]),
|
||||
)
|
||||
return clock
|
||||
|
||||
|
||||
def _admit(
|
||||
repository: TransactionalTransferAdmissionRepository,
|
||||
path: str,
|
||||
) -> TransferAdmission:
|
||||
"""登记一个带完整版本化输入的测试任务。"""
|
||||
return repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=_planning_input(path),
|
||||
)
|
||||
|
||||
|
||||
def _pending_snapshot(
|
||||
repository: TransactionalTransferAdmissionRepository,
|
||||
task_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""在 Session 关闭前冻结测试需要检查的持久登记字段。"""
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(TransferPending.task_id == task_id)
|
||||
).scalar_one()
|
||||
return {
|
||||
"lease_owner": pending.lease_owner,
|
||||
"lease_token": pending.lease_token,
|
||||
"lease_expires_at": pending.lease_expires_at,
|
||||
"heartbeat_at": pending.heartbeat_at,
|
||||
"last_error": pending.last_error,
|
||||
"attempt_count": pending.attempt_count,
|
||||
"updated_at": pending.updated_at,
|
||||
"planned_at": pending.planned_at,
|
||||
}
|
||||
|
||||
|
||||
def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""新 token 才增加 attempt,过期 token 不能续租、释放或删除接管者。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/movie.mkv")
|
||||
assert admitted.created_at == "2026-08-27 18:00:00"
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
|
||||
first = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert first is not None
|
||||
assert first.lease_owner == "worker-a"
|
||||
assert first.lease_token
|
||||
assert first.attempt_count == 1
|
||||
assert first.updated_at == "2026-08-27 18:01:00"
|
||||
|
||||
assert repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
assert repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=30)
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
renewed = repository.heartbeat(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert renewed is not None
|
||||
assert renewed.attempt_count == 1
|
||||
assert renewed.heartbeat_at == "2026-08-27 10:00:30.000000"
|
||||
assert renewed.updated_at == first.updated_at
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=61)
|
||||
business_clock["now"] = "2026-08-27 18:03:00"
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="expired worker",
|
||||
) is False
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
) == 0
|
||||
takeover = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert takeover is not None
|
||||
assert takeover.lease_token != first.lease_token
|
||||
assert takeover.attempt_count == 2
|
||||
assert takeover.updated_at == "2026-08-27 18:03:00"
|
||||
assert repository.heartbeat(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
business_clock["now"] = "2026-08-27 18:04:00"
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="stale worker",
|
||||
) is False
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
) == 0
|
||||
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
error="retry later",
|
||||
) is True
|
||||
released = _pending_snapshot(repository, admitted.task_id)
|
||||
assert released["lease_owner"] is None
|
||||
assert released["lease_token"] is None
|
||||
assert released["lease_expires_at"] is None
|
||||
assert released["heartbeat_at"] is None
|
||||
assert released["last_error"] == "retry later"
|
||||
assert released["attempt_count"] == 2
|
||||
assert released["updated_at"] == "2026-08-27 18:04:00"
|
||||
|
||||
third = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-c",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert third is not None
|
||||
assert third.attempt_count == 3
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=third.lease_token,
|
||||
) == 1
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
assert session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one_or_none() is None
|
||||
|
||||
|
||||
def test_claim_recoverable_respects_order_limit_and_active_lease(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""批量恢复跳过有效租约,并按登记顺序逐条 CAS 到请求上限。"""
|
||||
repository = repository_factory()
|
||||
first = _admit(repository, "/downloads/a.mkv")
|
||||
second = _admit(repository, "/downloads/b.mkv")
|
||||
third = _admit(repository, "/downloads/c.mkv")
|
||||
active = repository.claim_task(
|
||||
task_id=first.task_id,
|
||||
owner_id="active-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert active is not None
|
||||
|
||||
claimed = repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
assert [item.task_id for item in claimed] == [second.task_id, third.task_id]
|
||||
assert all(item.lease_owner == "recovery-worker" for item in claimed)
|
||||
assert all(item.attempt_count == 1 for item in claimed)
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="other-worker",
|
||||
limit=10,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=61)
|
||||
reclaimed = repository.claim_recoverable(
|
||||
owner_id="takeover-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert [item.task_id for item in reclaimed] == [first.task_id, second.task_id]
|
||||
assert reclaimed[0].attempt_count == 2
|
||||
assert reclaimed[1].attempt_count == 2
|
||||
|
||||
|
||||
def test_claim_recoverable_skips_corrupt_projection_and_claims_later_tasks(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""毒行应留下单次诊断但不持有租约或饿死后续健康任务。"""
|
||||
repository = repository_factory()
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
corrupt = _admit(repository, "/downloads/a-corrupt.mkv")
|
||||
healthy = [
|
||||
_admit(repository, "/downloads/b-healthy.mkv"),
|
||||
_admit(repository, "/downloads/c-healthy.mkv"),
|
||||
]
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == corrupt.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
claimed = repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
assert [item.task_id for item in claimed] == [item.task_id for item in healthy]
|
||||
corrupt_snapshot = _pending_snapshot(repository, corrupt.task_id)
|
||||
assert corrupt_snapshot["lease_token"] is None
|
||||
assert corrupt_snapshot["attempt_count"] == 0
|
||||
assert corrupt_snapshot["last_error"].startswith("恢复投影失败:")
|
||||
assert corrupt_snapshot["updated_at"] == "2026-08-27 18:00:00"
|
||||
assert len(messages) == 1
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="second-recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
assert _pending_snapshot(repository, corrupt.task_id) == corrupt_snapshot
|
||||
assert len(messages) == 1
|
||||
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == corrupt.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = _planning_input(
|
||||
"/downloads/a-corrupt.mkv"
|
||||
).fingerprint
|
||||
session.commit()
|
||||
|
||||
repaired = repository.claim_recoverable(
|
||||
owner_id="repaired-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert [item.task_id for item in repaired] == [corrupt.task_id]
|
||||
|
||||
|
||||
def test_projection_diagnostic_changes_are_recorded_once_each(
|
||||
repository_factory,
|
||||
business_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""相同投影错误不重复写库,错误类型变化时才更新诊断并再次告警。"""
|
||||
repository = repository_factory()
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
admitted = _admit(repository, "/downloads/changing-corrupt.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-a",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
first = _pending_snapshot(repository, admitted.task_id)
|
||||
assert len(messages) == 1
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-b",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
assert _pending_snapshot(repository, admitted.task_id) == first
|
||||
assert len(messages) == 1
|
||||
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = _planning_input(
|
||||
"/downloads/changing-corrupt.mkv"
|
||||
).fingerprint
|
||||
pending.input_version = 999
|
||||
session.commit()
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-c",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
changed = _pending_snapshot(repository, admitted.task_id)
|
||||
assert changed["last_error"] != first["last_error"]
|
||||
assert changed["updated_at"] == "2026-08-27 18:02:00"
|
||||
assert len(messages) == 2
|
||||
|
||||
|
||||
def test_projection_diagnostic_cas_is_concurrency_safe(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""并发恢复观察到同一损坏时只允许一个诊断写入者和一条运行日志。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/concurrent-corrupt.mkv")
|
||||
messages: list[str] = []
|
||||
message_lock = Lock()
|
||||
|
||||
def capture(message: str) -> None:
|
||||
"""并发安全收集错误日志。"""
|
||||
with message_lock:
|
||||
messages.append(message)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
capture,
|
||||
)
|
||||
barrier = Barrier(2)
|
||||
projection_error = TransferAdmissionProjectionError("same corruption")
|
||||
|
||||
def record(_: int) -> bool:
|
||||
"""让两个独立 Session 同时竞争同一诊断 CAS。"""
|
||||
barrier.wait(timeout=5)
|
||||
return repository_factory()._record_projection_failure( # noqa: SLF001
|
||||
task_id=admitted.task_id,
|
||||
error=projection_error,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(record, range(2)))
|
||||
|
||||
assert sorted(results) == [False, True]
|
||||
assert len(messages) == 1
|
||||
snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert snapshot["last_error"] == "恢复投影失败: same corruption"
|
||||
|
||||
|
||||
def test_projection_diagnostic_does_not_overwrite_active_lease(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""诊断 CAS 不得覆盖已经由健康 worker 取得有效租约的任务。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/active-lease.mkv")
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="active-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
before = _pending_snapshot(repository, admitted.task_id)
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
|
||||
recorded = repository._record_projection_failure( # noqa: SLF001
|
||||
task_id=admitted.task_id,
|
||||
error=TransferAdmissionProjectionError("stale observation"),
|
||||
)
|
||||
|
||||
assert recorded is False
|
||||
assert _pending_snapshot(repository, admitted.task_id) == before
|
||||
assert messages == []
|
||||
|
||||
|
||||
def test_projection_diagnostic_database_failure_propagates(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""诊断留痕的数据库基础设施异常必须向上游传播而非静默跳过。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/db-error-corrupt.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
def fail_diagnostic(*_args, **_kwargs):
|
||||
"""模拟诊断短事务的底层数据库写入失败。"""
|
||||
raise RuntimeError("database unavailable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
TransferPendingOper,
|
||||
"stage_record_projection_failure",
|
||||
fail_diagnostic,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="database unavailable"):
|
||||
repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
|
||||
def test_claim_task_wraps_persisted_json_decode_failure(
|
||||
repository_factory,
|
||||
) -> None:
|
||||
"""持久 JSON 解码错误应归类为投影损坏,而不是数据库基础设施故障。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/invalid-json.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
session.execute(
|
||||
text(
|
||||
"UPDATE transferpending SET planning_input = 'not-json' "
|
||||
"WHERE task_id = :task_id"
|
||||
),
|
||||
{"task_id": admitted.task_id},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="JSON"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="recovery-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
snapshot = session.execute(
|
||||
text(
|
||||
"SELECT lease_token, attempt_count FROM transferpending "
|
||||
"WHERE task_id = :task_id"
|
||||
),
|
||||
{"task_id": admitted.task_id},
|
||||
).mappings().one()
|
||||
assert snapshot["lease_token"] is None
|
||||
assert snapshot["attempt_count"] == 0
|
||||
|
||||
|
||||
def test_concurrent_recovery_callers_scan_past_lost_candidates(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""并发 caller 竞争同一首批后应继续向后扫描并各自填满限额。"""
|
||||
setup_repository = repository_factory()
|
||||
admitted = [
|
||||
_admit(setup_repository, f"/downloads/concurrent-{index}.mkv")
|
||||
for index in range(4)
|
||||
]
|
||||
barrier = Barrier(2)
|
||||
barrier_lock = Lock()
|
||||
initial_scans = 0
|
||||
original = TransferPending.list_claimable_candidates.__func__
|
||||
|
||||
def synchronized_candidates(cls, db, **kwargs):
|
||||
"""强制两个 caller 在取得相同首批候选后再进入逐任务 CAS。"""
|
||||
nonlocal initial_scans
|
||||
candidates = original(cls, db, **kwargs)
|
||||
if kwargs.get("after_cursor") is None:
|
||||
with barrier_lock:
|
||||
initial_scans += 1
|
||||
barrier.wait(timeout=5)
|
||||
return candidates
|
||||
|
||||
monkeypatch.setattr(
|
||||
TransferPending,
|
||||
"list_claimable_candidates",
|
||||
classmethod(synchronized_candidates),
|
||||
)
|
||||
|
||||
def recover(owner_id: str) -> list[TransferAdmission]:
|
||||
"""使用独立仓储与 Session 执行一次有界恢复扫描。"""
|
||||
return repository_factory().claim_recoverable(
|
||||
owner_id=owner_id,
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(recover, ("worker-a", "worker-b")))
|
||||
|
||||
assert initial_scans == 2
|
||||
assert [len(result) for result in results] == [2, 2]
|
||||
claimed_task_ids = [item.task_id for result in results for item in result]
|
||||
assert len(set(claimed_task_ids)) == 4
|
||||
assert set(claimed_task_ids) == {item.task_id for item in admitted}
|
||||
|
||||
|
||||
def test_unclaimed_enqueue_failure_cannot_overwrite_claimed_task(
|
||||
repository_factory,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""task-id-only 入队失败入口不得改写已经由 worker claim 的登记。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/claimed.mkv")
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
before = _pending_snapshot(repository, admitted.task_id)
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
repository.record_enqueue_failure(
|
||||
task_id=admitted.task_id,
|
||||
error="stale queue failure",
|
||||
)
|
||||
|
||||
after = _pending_snapshot(repository, admitted.task_id)
|
||||
assert after == before
|
||||
|
||||
|
||||
def test_planning_writes_require_current_unexpired_lease(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""checkpoint 和规划错误均不得由已过期或已被接管的 worker 写入。"""
|
||||
repository = repository_factory()
|
||||
path = "/downloads/planning.mkv"
|
||||
planning_input = _planning_input(path)
|
||||
admitted = repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=planning_input,
|
||||
)
|
||||
first = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=30,
|
||||
)
|
||||
assert first is not None
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=31)
|
||||
takeover = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert takeover is not None
|
||||
|
||||
with pytest.raises(TransferLeaseLostError, match="租约"):
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="stale planning failure",
|
||||
)
|
||||
with pytest.raises(TransferLeaseLostError, match="租约"):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
error="retryable planning failure",
|
||||
)
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
assert planned.state == TRANSFER_ADMISSION_PLANNED
|
||||
assert planned.last_error is None
|
||||
assert repeated == planned
|
||||
assert planned.attempt_count == 2
|
||||
|
||||
|
||||
def test_provider_checkpoint_sets_planned_time_only_after_host_plan(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""provider 快照不是规划完成,planned_at 只记录首次宿主完整计划。"""
|
||||
repository = repository_factory()
|
||||
path = "/downloads/provider-plan.mkv"
|
||||
planning_input = _planning_input(path)
|
||||
admitted = repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=planning_input,
|
||||
)
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_provider_checkpoint(planning_input),
|
||||
)
|
||||
provider_snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert provider_snapshot["planned_at"] is None
|
||||
assert provider_snapshot["updated_at"] == "2026-08-27 18:02:00"
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:03:00"
|
||||
checkpoint = _checkpoint(planning_input)
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
planned_snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert planned_snapshot["planned_at"] == "2026-08-27 18:03:00"
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:04:00"
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
assert _pending_snapshot(repository, admitted.task_id) == planned_snapshot
|
||||
|
||||
|
||||
def test_concurrent_claim_uses_rowcount_as_single_winner(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""并发 worker 即使读取同一任务,也只能有一个 CAS 更新获胜。"""
|
||||
setup_repository = repository_factory()
|
||||
admitted = _admit(setup_repository, "/downloads/concurrent.mkv")
|
||||
barrier = Barrier(2)
|
||||
|
||||
def claim(owner_id: str):
|
||||
"""等待竞争者就绪后使用独立 Session claim 同一任务。"""
|
||||
repository = repository_factory()
|
||||
barrier.wait()
|
||||
return repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id=owner_id,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(claim, ("worker-a", "worker-b")))
|
||||
|
||||
winners = [result for result in results if result is not None]
|
||||
assert len(winners) == 1
|
||||
assert winners[0].attempt_count == 1
|
||||
@@ -0,0 +1,158 @@
|
||||
"""旧待整理 Oper 的精确插件兼容与租约 fencing 测试。"""
|
||||
|
||||
import importlib
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.db import base as db_base
|
||||
from app.db.models.transferpending import TransferPending
|
||||
from app.db.oper.transferpending import TransferPendingOper as CanonicalTransferPendingOper
|
||||
from app.runtime.compat.manifest import MODULE_ALIASES
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_session_factory(tmp_path, monkeypatch):
|
||||
"""为无 Session 兼容 Oper 提供独占事务,并在提交后保留返回快照。"""
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'legacy-transferpending.db'}")
|
||||
TransferPending.__table__.create(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
|
||||
def run_transaction(operation: Callable[[Session], Any]) -> Any:
|
||||
"""按生产组合根语义执行一次同步兼容事务。"""
|
||||
with factory() as session:
|
||||
session.expire_on_commit = False
|
||||
try:
|
||||
result = operation(session)
|
||||
session.commit()
|
||||
return result
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(db_base, "run_sync_transaction", run_transaction)
|
||||
yield factory
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_legacy_import_targets_private_sdk_facade() -> None:
|
||||
"""旧模块路径只能解析到私有 SDK 门面,不能回退到 canonical Oper。"""
|
||||
alias = MODULE_ALIASES["app.db.transferpending_oper"]
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
|
||||
assert alias.target == "app.sdk._legacy.transferpending"
|
||||
assert alias.owner == "sdk"
|
||||
assert alias.replacement == "app.application.transfer"
|
||||
assert legacy is importlib.import_module(alias.target)
|
||||
assert legacy.__all__ == ["TransferPendingOper"]
|
||||
assert not hasattr(legacy, "TransferPending")
|
||||
assert legacy.TransferPendingOper is not CanonicalTransferPendingOper
|
||||
for internal_method in (
|
||||
"stage_admit",
|
||||
"stage_claim_task",
|
||||
"stage_discard_claimed",
|
||||
"stage_checkpoint_plan",
|
||||
):
|
||||
assert not hasattr(legacy.TransferPendingOper, internal_method)
|
||||
|
||||
|
||||
def test_legacy_no_session_queries_preserve_historical_shapes(
|
||||
legacy_session_factory,
|
||||
) -> None:
|
||||
"""旧插件无需 Session 即可登记和查询,返回形态与原 ABI 一致。"""
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
oper = legacy.TransferPendingOper()
|
||||
|
||||
first = oper.register("local", "/downloads/first.mkv")
|
||||
repeated = oper.register("local", "/downloads/first.mkv")
|
||||
second = oper.register("remote", "/downloads/second.mkv")
|
||||
|
||||
assert first is not None
|
||||
assert repeated is not None
|
||||
assert second is not None
|
||||
assert repeated.task_id == first.task_id
|
||||
assert oper.list_all() == [
|
||||
("local", "/downloads/first.mkv"),
|
||||
("remote", "/downloads/second.mkv"),
|
||||
]
|
||||
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
|
||||
first.task_id,
|
||||
second.task_id,
|
||||
]
|
||||
assert [
|
||||
item.task_id
|
||||
for item in oper.list_by_states(states=("accepted", "planned"), limit=1)
|
||||
] == [first.task_id]
|
||||
assert oper.get_by_identity(
|
||||
storage="local",
|
||||
src_path="/downloads/first.mkv",
|
||||
).task_id == first.task_id
|
||||
assert oper.get_by_task_id(task_id=second.task_id).src_path == second.src_path
|
||||
assert oper.register("", "/downloads/invalid.mkv") is None
|
||||
assert oper.list_by_state(state="") == []
|
||||
assert oper.list_by_states(states=()) == []
|
||||
|
||||
|
||||
def test_legacy_mutations_never_delete_claimed_rows(
|
||||
legacy_session_factory,
|
||||
) -> None:
|
||||
"""旧 discard/clear 只处理未 claim 行,有效或过期 token 均受保护。"""
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
oper = legacy.TransferPendingOper()
|
||||
active = oper.register("local", "/downloads/active.mkv")
|
||||
expired = oper.register("local", "/downloads/expired.mkv")
|
||||
free = oper.register("local", "/downloads/free.mkv")
|
||||
assert active is not None
|
||||
assert expired is not None
|
||||
assert free is not None
|
||||
|
||||
with legacy_session_factory() as session:
|
||||
assert TransferPending.claim_task(
|
||||
session,
|
||||
task_id=active.task_id,
|
||||
states=("accepted",),
|
||||
owner_id="active-worker",
|
||||
lease_token="active-token",
|
||||
now_time="2026-08-27 10:00:00.000000",
|
||||
lease_expires_at="2026-08-27 10:01:00.000000",
|
||||
updated_at="2026-08-27 18:00:00",
|
||||
) == 1
|
||||
assert TransferPending.claim_task(
|
||||
session,
|
||||
task_id=expired.task_id,
|
||||
states=("accepted",),
|
||||
owner_id="expired-worker",
|
||||
lease_token="expired-token",
|
||||
now_time="2026-08-27 10:00:00.000000",
|
||||
lease_expires_at="2026-08-27 09:59:00.000000",
|
||||
updated_at="2026-08-27 18:00:00",
|
||||
) == 1
|
||||
session.commit()
|
||||
|
||||
repeated = oper.register("local", "/downloads/active.mkv")
|
||||
assert repeated is not None
|
||||
assert repeated.task_id == active.task_id
|
||||
assert repeated.lease_token == "active-token"
|
||||
assert oper.discard("local", "/downloads/active.mkv") == 0
|
||||
assert oper.discard("local", "/downloads/expired.mkv") == 0
|
||||
assert oper.discard("local", "/downloads/free.mkv") == 1
|
||||
|
||||
removable = oper.register("local", "/downloads/removable.mkv")
|
||||
assert removable is not None
|
||||
assert oper.clear() == 1
|
||||
assert oper.list_all() == [
|
||||
("local", "/downloads/active.mkv"),
|
||||
("local", "/downloads/expired.mkv"),
|
||||
]
|
||||
|
||||
with legacy_session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(TransferPending).order_by(TransferPending.id.asc())
|
||||
).scalars().all()
|
||||
assert [(row.task_id, row.lease_token) for row in rows] == [
|
||||
(active.task_id, "active-token"),
|
||||
(expired.task_id, "expired-token"),
|
||||
]
|
||||
@@ -8,6 +8,8 @@
|
||||
这些测试固定三项不变量:入队即落盘登记、终态即注销、重启能回放。
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -24,6 +26,17 @@ def _build_chain(admissions) -> TransferChain:
|
||||
"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = admissions
|
||||
chain._worker_owner_id = "test-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._replay_stop_event = threading.Event()
|
||||
chain._lease_heartbeat_stop_event = threading.Event()
|
||||
chain._lease_heartbeat_thread = None
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
@@ -36,6 +49,11 @@ def _admission(path: str, task_id: str = "task-1") -> TransferAdmission:
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner="test-owner",
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
|
||||
@@ -87,10 +105,19 @@ def test_discard_pending_on_terminal_state():
|
||||
chain = _build_chain(admissions)
|
||||
task = _task("/mnt/cd2/downloads/Movie.2024.mkv")
|
||||
task.bind_admission_task_id("task-1")
|
||||
task.bind_execution_lease(owner_id="test-owner", lease_token="lease-task-1")
|
||||
chain._worker_owner_id = "test-owner"
|
||||
chain._owned_leases = {
|
||||
"task-1": ("lease-task-1", time.monotonic() + 120)
|
||||
}
|
||||
admissions.discard_claimed.return_value = 1
|
||||
|
||||
chain._TransferChain__discard_pending(task)
|
||||
assert chain._TransferChain__discard_pending(task) is True
|
||||
|
||||
admissions.discard_task.assert_called_once_with(task_id="task-1")
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
|
||||
@@ -101,7 +128,7 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
|
||||
media.write_bytes(b"x" * 10)
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(media))]
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
transferred = []
|
||||
@@ -128,14 +155,18 @@ def test_replay_discards_vanished_files(tmp_path):
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
missing = tmp_path / "gone.mkv"
|
||||
admissions.list_recoverable.return_value = [_admission(str(missing))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(missing))]
|
||||
admissions.discard_claimed.return_value = 1
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
chain._execute_transfer.assert_not_called()
|
||||
admissions.discard_task.assert_called_once_with(task_id="task-1")
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
@@ -148,7 +179,8 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
media.write_bytes(b"x")
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -163,7 +195,12 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
chain._execute_transfer.assert_not_called()
|
||||
admissions.discard_task.assert_not_called()
|
||||
admissions.discard_claimed.assert_not_called()
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
error="恢复源文件暂时不可读取",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
|
||||
@@ -175,7 +212,7 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
|
||||
src_path = f"{bluray.as_posix()}/"
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(src_path)]
|
||||
admissions.claim_recoverable.return_value = [_admission(src_path)]
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
transferred = []
|
||||
@@ -197,7 +234,7 @@ def test_replay_is_noop_without_registrations():
|
||||
没有登记时回放不应触碰整理链。
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = []
|
||||
admissions.claim_recoverable.return_value = []
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -211,7 +248,7 @@ def test_replay_survives_db_failure():
|
||||
读取登记失败不能让启动流程报错。
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.side_effect = RuntimeError("db gone")
|
||||
admissions.claim_recoverable.side_effect = RuntimeError("db gone")
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -230,7 +267,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
|
||||
item.write_bytes(b"x")
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [
|
||||
admissions.claim_recoverable.return_value = [
|
||||
_admission(str(first), "task-1"),
|
||||
_admission(str(second), "task-2"),
|
||||
]
|
||||
@@ -261,7 +298,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
|
||||
first.write_bytes(b"x")
|
||||
missing_second = tmp_path / "gone.mkv"
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [
|
||||
admissions.claim_recoverable.return_value = [
|
||||
_admission(str(first), "task-1"),
|
||||
_admission(str(missing_second), "task-2"),
|
||||
]
|
||||
@@ -279,4 +316,94 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
|
||||
chain._TransferChain__replay_pending(stop_event)
|
||||
|
||||
assert transferred == [first.as_posix()]
|
||||
admissions.discard_task.assert_not_called()
|
||||
admissions.discard_claimed.assert_not_called()
|
||||
assert admissions.release_claim.call_count == 2
|
||||
|
||||
|
||||
def test_replay_registers_entire_claimed_batch_before_first_source_stat(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""批量 claim 返回后必须先把全部 token 交给 heartbeat,再做逐条同步 I/O。"""
|
||||
first = _admission(str(tmp_path / "A.mkv"), "task-1")
|
||||
second = _admission(str(tmp_path / "B.mkv"), "task-2")
|
||||
admissions = MagicMock()
|
||||
admissions.claim_recoverable.return_value = [first, second]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
def observe_owned_batch(*_args, **_kwargs):
|
||||
"""首个 stat 前观察两个 token 已同时进入续期集合。"""
|
||||
assert set(chain._owned_leases) == {"task-1", "task-2"}
|
||||
return None, False
|
||||
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"_TransferChain__build_replay_fileitem",
|
||||
observe_owned_batch,
|
||||
)
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
assert admissions.release_claim.call_count == 2
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
|
||||
def test_replay_releases_claim_when_jobview_rejects_recovered_task(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""恢复任务未进入队列时必须立即 release,不能靠租约自然过期。"""
|
||||
media = tmp_path / "Movie.2024.mkv"
|
||||
media.write_bytes(b"x")
|
||||
admission = _admission(str(media))
|
||||
admission = replace(admission, checkpoint=MagicMock())
|
||||
admissions = MagicMock()
|
||||
admissions.claim_recoverable.return_value = [admission]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"_TransferChain__queue_planned_replay",
|
||||
MagicMock(return_value=False),
|
||||
)
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
error="恢复任务未进入内存队列",
|
||||
)
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
|
||||
def test_claimed_enqueue_failure_never_uses_unfenced_error_writer() -> None:
|
||||
"""陈旧 token 入队失败只能 release_claim,不能覆盖新 owner 的 last_error。"""
|
||||
admissions = MagicMock()
|
||||
admissions.release_claim.return_value = False
|
||||
chain = _build_chain(admissions)
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain.replay_pending = MagicMock()
|
||||
task = _task("/downloads/stale-enqueue.mkv")
|
||||
task.bind_admission_task_id("stale-task")
|
||||
task.bind_execution_lease(
|
||||
owner_id="test-owner",
|
||||
lease_token="stale-token",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"stale-task": ("stale-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
chain._TransferChain__record_enqueue_failure(
|
||||
task,
|
||||
RuntimeError("queue closed"),
|
||||
)
|
||||
|
||||
admissions.record_enqueue_failure.assert_not_called()
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="stale-task",
|
||||
lease_token="stale-token",
|
||||
error="queue closed",
|
||||
)
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application import transfer as transfer_application
|
||||
@@ -202,6 +203,31 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
|
||||
"""构造只保留规划编排依赖的 TransferChain 骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = repository or Mock()
|
||||
chain._worker_owner_id = "planning-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
|
||||
|
||||
def claim_task(**kwargs):
|
||||
"""为规划测试返回与进程 owner 匹配的稳定 claim。"""
|
||||
return transfer_application.TransferAdmission(
|
||||
task_id=kwargs["task_id"],
|
||||
storage="local",
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner=kwargs["owner_id"],
|
||||
lease_token=f"lease-{kwargs['task_id']}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
chain._transfer_admissions.claim_task.side_effect = claim_task
|
||||
chain._module_dispatcher = Mock()
|
||||
chain._module_dispatcher.freeze_plugin_providers.return_value = ()
|
||||
chain.eventmanager = Mock()
|
||||
@@ -217,6 +243,20 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
|
||||
return chain
|
||||
|
||||
|
||||
def _replay_chain(repository) -> TransferChain:
|
||||
"""构造绑定固定恢复 owner 且不启动真实 heartbeat 线程的测试链。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = repository
|
||||
chain._worker_owner_id = "replay-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
|
||||
return chain
|
||||
|
||||
|
||||
def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher:
|
||||
"""构造使用真实冻结解析与执行内核的内存插件调度器。"""
|
||||
plugin_catalog = Mock()
|
||||
@@ -725,6 +765,7 @@ def test_provider_empty_fallback_cas_failure_blocks_host_execution():
|
||||
chain.execute_transfer_plan.assert_not_called()
|
||||
repository.record_planning_failure.assert_called_once_with(
|
||||
task_id="task-provider-cas-failure",
|
||||
lease_token="lease-task-provider-cas-failure",
|
||||
error="CAS failed",
|
||||
)
|
||||
|
||||
@@ -804,7 +845,7 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
|
||||
return SimpleNamespace(checkpoint=kwargs["checkpoint"])
|
||||
|
||||
repository.checkpoint_plan.side_effect = checkpoint_plan
|
||||
repository.discard_task.side_effect = (
|
||||
repository.discard_claimed.side_effect = (
|
||||
lambda **_kwargs: calls.append("discard") or 1
|
||||
)
|
||||
result = TransferInfo(
|
||||
@@ -830,8 +871,9 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
|
||||
|
||||
assert returned is result
|
||||
assert calls == ["admit", "checkpoint", "execute", "discard"]
|
||||
repository.discard_task.assert_called_once_with(
|
||||
task_id="task-legacy-command"
|
||||
repository.discard_claimed.assert_called_once_with(
|
||||
task_id="task-legacy-command",
|
||||
lease_token="lease-task-legacy-command",
|
||||
)
|
||||
|
||||
|
||||
@@ -898,16 +940,32 @@ def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=accepted_input,
|
||||
)
|
||||
claimed = repository.claim_task(
|
||||
task_id=admission.task_id,
|
||||
owner_id="fingerprint-test",
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token is not None
|
||||
mismatched = _checkpoint(target_path="/library/B/Movie.mkv")
|
||||
|
||||
with pytest.raises(ValueError, match="指纹|fingerprint|规划输入"):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=accepted_input.fingerprint,
|
||||
checkpoint=mismatched,
|
||||
)
|
||||
|
||||
recovered = repository.list_recoverable()
|
||||
assert repository.release_claim(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) is True
|
||||
recovered = repository.claim_recoverable(
|
||||
owner_id="fingerprint-recovery",
|
||||
limit=10,
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == "accepted"
|
||||
assert recovered[0].checkpoint is None
|
||||
@@ -926,23 +984,35 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
|
||||
planning_input=planning_input,
|
||||
)
|
||||
|
||||
accepted = repository.list_recoverable()
|
||||
assert len(accepted) == 1
|
||||
assert accepted[0].state == "accepted"
|
||||
assert accepted[0].planning_input == planning_input
|
||||
assert accepted[0].checkpoint is None
|
||||
claimed = repository.claim_task(
|
||||
task_id=admission.task_id,
|
||||
owner_id="roundtrip-owner",
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token is not None
|
||||
assert claimed.state == "accepted"
|
||||
assert claimed.planning_input == planning_input
|
||||
assert claimed.checkpoint is None
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="rename unavailable",
|
||||
)
|
||||
retryable = repository.list_recoverable()[0]
|
||||
assert retryable.state == "accepted"
|
||||
assert retryable.last_error == "rename unavailable"
|
||||
with sessionmaker(bind=engine)() as session:
|
||||
retryable = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admission.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert retryable.state == "accepted"
|
||||
assert retryable.last_error == "rename unavailable"
|
||||
|
||||
checkpoint = _checkpoint()
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -950,7 +1020,18 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
|
||||
assert planned.state == "planned"
|
||||
assert planned.checkpoint == checkpoint
|
||||
assert planned.last_error is None
|
||||
assert repository.list_recoverable() == [planned]
|
||||
assert repository.release_claim(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) is True
|
||||
recovered = repository.claim_recoverable(
|
||||
owner_id="roundtrip-recovery",
|
||||
limit=10,
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == "planned"
|
||||
assert recovered[0].checkpoint == checkpoint
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -982,6 +1063,10 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
|
||||
repository.admit.side_effect = admit
|
||||
repository.checkpoint_plan.side_effect = checkpoint_plan
|
||||
chain = _chain(repository=repository, checkpoint=checkpoint, result=result)
|
||||
claim_task = repository.claim_task.side_effect
|
||||
repository.claim_task.side_effect = lambda **kwargs: (
|
||||
order.append("claim") or claim_task(**kwargs)
|
||||
)
|
||||
chain.plan_transfer.side_effect = lambda *_args, **_kwargs: (
|
||||
order.append("plan") or checkpoint
|
||||
)
|
||||
@@ -992,12 +1077,13 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
|
||||
returned = chain._plan_checkpoint_and_execute(task)
|
||||
|
||||
assert returned is result
|
||||
assert order == ["admit", "plan", "commit-checkpoint", "execute"]
|
||||
assert order == ["admit", "claim", "plan", "commit-checkpoint", "execute"]
|
||||
assert task.admission_task_id == "task-sync"
|
||||
repository.admit.assert_called_once()
|
||||
assert repository.admit.call_args.kwargs["planning_input"] is planning_input
|
||||
repository.checkpoint_plan.assert_called_once_with(
|
||||
task_id="task-sync",
|
||||
lease_token="lease-task-sync",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -1192,11 +1278,15 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
planning_input=planning_input,
|
||||
lease_owner="replay-owner",
|
||||
lease_token="lease-task-accepted-offline",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
replay_repository = Mock()
|
||||
replay_repository.list_recoverable.return_value = [admission]
|
||||
replay_chain = object.__new__(TransferChain)
|
||||
replay_chain._transfer_admissions = replay_repository
|
||||
replay_repository.claim_recoverable.return_value = [admission]
|
||||
replay_chain = _replay_chain(replay_repository)
|
||||
replay_chain._execute_transfer = Mock()
|
||||
queued_tasks = []
|
||||
replay_chain.put_to_queue = Mock(
|
||||
@@ -1230,6 +1320,10 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
|
||||
repository=execution_repository,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
execution_chain._worker_owner_id = "replay-owner"
|
||||
execution_chain._owned_leases = {
|
||||
admission.task_id: (str(admission.lease_token), float("inf"))
|
||||
}
|
||||
execution_chain.jobview = Mock()
|
||||
execution_chain.eventmanager = Mock()
|
||||
execution_chain.eventmanager.send_event.return_value = None
|
||||
@@ -1292,11 +1386,15 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
planning_input=planning_input,
|
||||
lease_owner="replay-owner",
|
||||
lease_token="lease-task-accepted-empty-episodes",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
replay_repository = Mock()
|
||||
replay_repository.list_recoverable.return_value = [admission]
|
||||
replay_chain = object.__new__(TransferChain)
|
||||
replay_chain._transfer_admissions = replay_repository
|
||||
replay_repository.claim_recoverable.return_value = [admission]
|
||||
replay_chain = _replay_chain(replay_repository)
|
||||
queued_tasks = []
|
||||
replay_chain.put_to_queue = Mock(
|
||||
side_effect=lambda task: queued_tasks.append(task) or True
|
||||
@@ -1319,6 +1417,10 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
|
||||
repository=execution_repository,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
execution_chain._worker_owner_id = "replay-owner"
|
||||
execution_chain._owned_leases = {
|
||||
admission.task_id: (str(admission.lease_token), float("inf"))
|
||||
}
|
||||
execution_chain.jobview = Mock()
|
||||
execution_chain.eventmanager = Mock()
|
||||
execution_chain.eventmanager.send_event.return_value = None
|
||||
@@ -1425,8 +1527,20 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
|
||||
task = _task()
|
||||
task.meta = MetaBase("Unrecognized.Movie.2026.mkv")
|
||||
task.bind_admission_task_id("task-before-checkpoint")
|
||||
task.bind_execution_lease(
|
||||
owner_id="recognition-owner",
|
||||
lease_token="lease-task-before-checkpoint",
|
||||
)
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = Mock()
|
||||
chain._worker_owner_id = "recognition-owner"
|
||||
chain._owned_leases = {
|
||||
"task-before-checkpoint": (
|
||||
"lease-task-before-checkpoint",
|
||||
float("inf"),
|
||||
)
|
||||
}
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain.jobview = Mock()
|
||||
chain.queue_failed_transfer_notification = Mock()
|
||||
chain.runtime_config = SimpleNamespace(
|
||||
@@ -1449,6 +1563,7 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
|
||||
assert result == (False, "未识别到媒体信息")
|
||||
chain._transfer_admissions.record_planning_failure.assert_called_once_with(
|
||||
task_id="task-before-checkpoint",
|
||||
lease_token="lease-task-before-checkpoint",
|
||||
error="未识别到媒体信息",
|
||||
)
|
||||
|
||||
|
||||
@@ -57,6 +57,8 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
|
||||
task.bind_admission_task_id("task-stable")
|
||||
|
||||
values = task.to_dict()
|
||||
task.bind_execution_lease(owner_id="worker-owner", lease_token="lease-token")
|
||||
leased_values = task.to_dict()
|
||||
|
||||
assert set(values) == {
|
||||
"fileitem",
|
||||
@@ -84,9 +86,12 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
|
||||
}
|
||||
assert values["fileitem"] == task.fileitem.model_dump()
|
||||
assert values["target_path"] == Path("/library/Movie (2026)")
|
||||
assert leased_values == values
|
||||
assert "admission_task_id" not in values
|
||||
assert "planning_input" not in values
|
||||
assert "plan_checkpoint" not in values
|
||||
assert "lease_owner" not in values
|
||||
assert "lease_token" not in values
|
||||
|
||||
|
||||
def test_transfer_chain_do_transfer_keeps_legacy_signature():
|
||||
|
||||
@@ -24,12 +24,13 @@ except ModuleNotFoundError:
|
||||
|
||||
POSTGRESQL_DIALECT = "postgresql+psycopg"
|
||||
|
||||
MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
|
||||
PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
|
||||
LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
|
||||
|
||||
|
||||
def _bind_migration(monkeypatch, connection):
|
||||
def _bind_migration(monkeypatch, connection, module_name=PLANNING_MIGRATION):
|
||||
"""把迁移绑定到隔离数据库连接。"""
|
||||
migration = importlib.import_module(MIGRATION)
|
||||
migration = importlib.import_module(module_name)
|
||||
monkeypatch.setattr(
|
||||
migration,
|
||||
"op",
|
||||
@@ -91,10 +92,13 @@ def _planning_row(connection) -> dict[str, object]:
|
||||
def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
"""断言规划迁移在当前隔离连接上的完整可逆生命周期。"""
|
||||
_create_admission_table(connection)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
planning_migration = _bind_migration(monkeypatch, connection)
|
||||
lease_migration = _bind_migration(monkeypatch, connection, LEASE_MIGRATION)
|
||||
|
||||
migration.upgrade()
|
||||
migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
|
||||
inspector = sa.inspect(connection)
|
||||
assert {
|
||||
@@ -133,7 +137,9 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
planned_at="2026-08-27 11:00:00",
|
||||
)
|
||||
)
|
||||
migration.downgrade()
|
||||
lease_migration.downgrade()
|
||||
lease_migration.downgrade()
|
||||
planning_migration.downgrade()
|
||||
|
||||
downgraded = sa.inspect(connection)
|
||||
assert {
|
||||
@@ -152,12 +158,17 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
for index in downgraded.get_indexes("transferpending")
|
||||
} == {"ix_transferpending_state_created", "ux_transferpending_storage_path"}
|
||||
|
||||
migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
reupgraded = _planning_row(connection)
|
||||
assert reupgraded["task_id"] == "stable-task"
|
||||
assert reupgraded["state"] == "accepted"
|
||||
assert reupgraded["checkpoint_payload"] is None
|
||||
assert reupgraded["input_fingerprint"] == planning_input.fingerprint
|
||||
assert {
|
||||
column["name"]
|
||||
for column in sa.inspect(connection).get_columns("transferpending")
|
||||
} == {column.name for column in TransferPending.__table__.columns}
|
||||
|
||||
|
||||
def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None:
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.application.transfer import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TransferAdmissionConflictError,
|
||||
TransferAdmissionProjectionError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanItem,
|
||||
TransferPlanningInput,
|
||||
@@ -30,6 +31,31 @@ def repository(tmp_path):
|
||||
return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
|
||||
|
||||
|
||||
def _claim(repository, task_id: str):
|
||||
"""为需要变更规划状态的测试取得独占租约。"""
|
||||
claimed = repository.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id="planning-test-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token
|
||||
return claimed
|
||||
|
||||
|
||||
def _pending_snapshot(repository, task_id: str) -> dict[str, object]:
|
||||
"""使用隔离 Session 冻结测试所需的持久状态字段。"""
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(TransferPending.task_id == task_id)
|
||||
).scalar_one()
|
||||
return {
|
||||
"state": pending.state,
|
||||
"last_error": pending.last_error,
|
||||
"checkpoint_payload": pending.checkpoint_payload,
|
||||
}
|
||||
|
||||
|
||||
def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput:
|
||||
"""构造包含恢复所需媒体上下文的完整规划输入。"""
|
||||
return TransferPlanningInput(
|
||||
@@ -264,6 +290,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
checkpoint = replace(
|
||||
_checkpoint(planning_input),
|
||||
resolved_meta={"name": "Resolved Movie", "year": 2026},
|
||||
@@ -275,6 +302,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -287,6 +315,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=replace(
|
||||
checkpoint,
|
||||
@@ -326,15 +355,18 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
checkpoint = _checkpoint(planning_input)
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -350,8 +382,9 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
|
||||
"plugin-provider-a",
|
||||
)
|
||||
assert repeated == planned
|
||||
assert repository.list_accepted() == []
|
||||
assert repository.list_recoverable() == [planned]
|
||||
assert _pending_snapshot(repository, admitted.task_id)["state"] == (
|
||||
TRANSFER_ADMISSION_PLANNED
|
||||
)
|
||||
|
||||
|
||||
def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None:
|
||||
@@ -362,23 +395,34 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
provider_checkpoint = _provider_checkpoint(planning_input)
|
||||
|
||||
provider_pending = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=provider_checkpoint,
|
||||
)
|
||||
|
||||
assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
assert provider_pending.checkpoint == provider_checkpoint
|
||||
assert repository.list_recoverable() == [provider_pending]
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="host planning unavailable",
|
||||
)
|
||||
failed = repository.list_recoverable()[0]
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="host planning unavailable",
|
||||
)
|
||||
failed = repository.claim_recoverable(
|
||||
owner_id="planning-recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=3600,
|
||||
)[0]
|
||||
assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
assert failed.checkpoint == provider_checkpoint
|
||||
assert failed.last_error == "host planning unavailable"
|
||||
@@ -386,11 +430,13 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
host_checkpoint = _checkpoint(planning_input)
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=host_checkpoint,
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=host_checkpoint,
|
||||
)
|
||||
@@ -402,6 +448,7 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=provider_checkpoint,
|
||||
)
|
||||
@@ -415,18 +462,19 @@ def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> Non
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
|
||||
with pytest.raises(TransferAdmissionConflictError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint="0" * 64,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
recovered = repository.list_recoverable()
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert recovered[0].checkpoint is None
|
||||
recovered = _pending_snapshot(repository, admitted.task_id)
|
||||
assert recovered["state"] == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert recovered["checkpoint_payload"] is None
|
||||
|
||||
|
||||
def test_planning_failure_stays_accepted_until_success(repository) -> None:
|
||||
@@ -437,15 +485,21 @@ def test_planning_failure_stays_accepted_until_success(repository) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
|
||||
repository.record_planning_failure(task_id=admitted.task_id, error="rename failed")
|
||||
failed = repository.list_recoverable()[0]
|
||||
assert failed.state == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert failed.last_error == "rename failed"
|
||||
assert failed.checkpoint is None
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="rename failed",
|
||||
)
|
||||
failed = _pending_snapshot(repository, admitted.task_id)
|
||||
assert failed["state"] == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert failed["last_error"] == "rename failed"
|
||||
assert failed["checkpoint_payload"] is None
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -459,6 +513,7 @@ def test_checkpoint_rejects_missing_task(repository) -> None:
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id="missing",
|
||||
lease_token="missing-token",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -470,17 +525,25 @@ def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
|
||||
factory = sessionmaker(bind=engine)
|
||||
TransferPending.__table__.create(engine)
|
||||
with factory() as session:
|
||||
session.add(TransferPending(
|
||||
pending = TransferPending(
|
||||
storage="local",
|
||||
src_path="/downloads/legacy.mkv",
|
||||
state=TRANSFER_ADMISSION_ACCEPTED,
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
))
|
||||
)
|
||||
session.add(pending)
|
||||
session.commit()
|
||||
task_id = pending.task_id
|
||||
|
||||
admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0]
|
||||
repository = TransactionalTransferAdmissionRepository(factory)
|
||||
admitted = repository.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id="legacy-projection-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
|
||||
assert admitted is not None
|
||||
assert admitted.planning_input == TransferPlanningInput.legacy(
|
||||
storage="local",
|
||||
src_path="/downloads/legacy.mkv",
|
||||
@@ -508,8 +571,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
|
||||
row.input_version = 2
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferPlanningStateError, match="版本"):
|
||||
repository.list_accepted()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="corruption-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
|
||||
with factory() as session:
|
||||
row = session.execute(
|
||||
@@ -521,8 +588,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
|
||||
row.planning_input = corrupted
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferAdmissionConflictError, match="指纹"):
|
||||
repository.list_accepted()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="指纹"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="corruption-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -538,8 +609,10 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -548,8 +621,18 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
|
||||
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
|
||||
).scalar_one()
|
||||
row.checkpoint_version = 2
|
||||
row.lease_expires_at = "2000-01-01 00:00:00.000000"
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferPlanningStateError, match="版本"):
|
||||
repository.list_recoverable()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="direct-corruption-test-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="batch-corruption-test-worker",
|
||||
limit=1,
|
||||
lease_seconds=3600,
|
||||
) == []
|
||||
engine.dispose()
|
||||
|
||||
@@ -2,7 +2,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application.transfer import TransferAdmission, TransferQueueService
|
||||
@@ -118,7 +118,8 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
|
||||
"""真实仓储已提交后即使内存入队失败,任务也必须带原因留待恢复。"""
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'durable-admission.db'}")
|
||||
TransferPending.__table__.create(engine)
|
||||
repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
|
||||
factory = sessionmaker(bind=engine)
|
||||
repository = TransactionalTransferAdmissionRepository(factory)
|
||||
task = make_task(1)
|
||||
service, _ = _service(
|
||||
admit_task=lambda item: repository.admit(
|
||||
@@ -135,10 +136,13 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
|
||||
with pytest.raises(RuntimeError, match="queue closed"):
|
||||
service.put(task, Mock())
|
||||
|
||||
admissions = repository.list_accepted()
|
||||
assert len(admissions) == 1
|
||||
assert admissions[0].task_id == task.admission_task_id
|
||||
assert admissions[0].last_error == "queue closed"
|
||||
with factory() as session:
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == task.admission_task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert pending.last_error == "queue closed"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.chain.transfer import JobManager, TransferChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.runtime.config import settings
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.transfer import EpisodeFormat
|
||||
from app.schemas.types import MediaType
|
||||
from tests.test_transfer_job_manager import (
|
||||
make_transfer_chain as make_base_transfer_chain,
|
||||
)
|
||||
|
||||
|
||||
class FakeMeta(MetaBase):
|
||||
@@ -46,16 +49,8 @@ def make_transfer_chain() -> TransferChain:
|
||||
"""
|
||||
构造不启动后台线程的整理链实例。
|
||||
"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.jobview = JobManager()
|
||||
chain._media_exts = settings.RMT_MEDIAEXT
|
||||
chain._subtitle_exts = settings.RMT_SUBEXT
|
||||
chain._audio_exts = settings.RMT_AUDIOEXT
|
||||
chain._allowed_exts = (
|
||||
chain._media_exts + chain._audio_exts + chain._subtitle_exts
|
||||
)
|
||||
chain._success_target_files = {}
|
||||
chain._scrape_batches = {}
|
||||
chain = make_base_transfer_chain()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
|
||||
@@ -15,13 +15,17 @@ from app.chain.transfer import TransferChain
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.config import global_vars
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.transfer import TransferInfo
|
||||
from app.startup.initializers import transfer as transfer_initializer
|
||||
|
||||
|
||||
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
|
||||
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads)
|
||||
chain.runtime_config = SimpleNamespace(
|
||||
transfer_threads=transfer_threads,
|
||||
transfer_task_timeout=0,
|
||||
)
|
||||
chain._queue = queue.Queue()
|
||||
chain._transfer_interval = 0.1
|
||||
chain._threads = []
|
||||
@@ -33,9 +37,59 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
|
||||
chain._closing = False
|
||||
chain._replay_thread = None
|
||||
chain._replay_stop_event = threading.Event()
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._lease_heartbeat_thread = None
|
||||
chain._lease_heartbeat_stop_event = threading.Event()
|
||||
chain._worker_owner_id = "worker-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
admissions = MagicMock()
|
||||
admissions.admit.side_effect = lambda **kwargs: TransferAdmission(
|
||||
task_id="admitted-task",
|
||||
storage=kwargs["storage"],
|
||||
src_path=kwargs["src_path"],
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
)
|
||||
admissions.claim_task.side_effect = lambda **kwargs: TransferAdmission(
|
||||
task_id=kwargs["task_id"],
|
||||
storage="local",
|
||||
src_path="/downloads/test.mkv",
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner=kwargs["owner_id"],
|
||||
lease_token=f"lease-{kwargs['task_id']}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
admissions.discard_claimed.return_value = 1
|
||||
admissions.release_claim.return_value = True
|
||||
chain._transfer_admissions = admissions
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
def _claimed_admission(task: TransferTask, task_id: str) -> TransferAdmission:
|
||||
"""构造属于测试进程 owner 的有效 claim 投影。"""
|
||||
return TransferAdmission(
|
||||
task_id=task_id,
|
||||
storage=task.fileitem.storage,
|
||||
src_path=task.fileitem.path,
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner="worker-owner",
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
|
||||
def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None:
|
||||
"""热更新应等待旧 worker 收敛,再启动使用独立停止信号的新一代。"""
|
||||
chain = _build_chain(transfer_threads=1)
|
||||
@@ -433,7 +487,11 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
)
|
||||
admissions.discard_task.side_effect = (
|
||||
admissions.claim_task.return_value = _claimed_admission(
|
||||
task,
|
||||
"durable-task-id",
|
||||
)
|
||||
admissions.discard_claimed.side_effect = (
|
||||
lambda **_kwargs: discarded.set() or 1
|
||||
)
|
||||
chain._transfer_admissions = admissions
|
||||
@@ -471,7 +529,15 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
assert task.admission_task_id == "durable-task-id"
|
||||
admissions.discard_task.assert_called_once_with(task_id="durable-task-id")
|
||||
admissions.claim_task.assert_called_once_with(
|
||||
task_id="durable-task-id",
|
||||
owner_id="worker-owner",
|
||||
lease_seconds=120,
|
||||
)
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="durable-task-id",
|
||||
lease_token="lease-durable-task-id",
|
||||
)
|
||||
|
||||
|
||||
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
|
||||
@@ -527,6 +593,7 @@ def test_claimed_task_prevents_progress_settlement_before_active_registration()
|
||||
def test_replay_has_single_owner_and_close_waits_for_it() -> None:
|
||||
"""重复回放只保留一个线程,关闭会通知并等待该线程退出。"""
|
||||
chain = _build_chain()
|
||||
del chain._TransferChain__ensure_recovery_scheduler
|
||||
replay_started = threading.Event()
|
||||
replay_calls = []
|
||||
|
||||
@@ -548,3 +615,355 @@ def test_replay_has_single_owner_and_close_waits_for_it() -> None:
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert replay_thread.is_alive() is False
|
||||
assert chain._replay_thread is None
|
||||
|
||||
|
||||
def test_recovered_worker_reuses_claimed_token_without_second_claim(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""恢复任务携带 token 入队后,普通 worker 必须直接执行而非二次 claim。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/recovered.mkv",
|
||||
type="file",
|
||||
name="recovered.mkv",
|
||||
basename="recovered",
|
||||
extension="mkv",
|
||||
))
|
||||
task.bind_admission_task_id("recovered-task")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="lease-recovered-task",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"recovered-task": ("lease-recovered-task", time.monotonic() + 120)
|
||||
}
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 1
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_recovery(*, task, callback):
|
||||
"""模拟恢复任务成功提交检查点并让 worker 在本项后退出。"""
|
||||
del callback
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_recovery
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._transfer_admissions.claim_task.assert_not_called()
|
||||
chain._transfer_admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="recovered-task",
|
||||
lease_token="lease-recovered-task",
|
||||
)
|
||||
|
||||
|
||||
def test_heartbeat_refreshes_current_token_and_forgets_lost_lease() -> None:
|
||||
"""heartbeat 成功应刷新本地期限,CAS 拒绝后必须立即停止本地推进资格。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/heartbeat.mkv",
|
||||
type="file",
|
||||
))
|
||||
current = _claimed_admission(task, "heartbeat-task")
|
||||
initial_deadline = time.monotonic() + 1
|
||||
chain._owned_leases = {
|
||||
"heartbeat-task": ("lease-heartbeat-task", initial_deadline)
|
||||
}
|
||||
chain._transfer_admissions.heartbeat.return_value = current
|
||||
|
||||
chain._TransferChain__heartbeat_owned_leases()
|
||||
|
||||
assert chain._owned_leases["heartbeat-task"][1] > initial_deadline
|
||||
chain._transfer_admissions.heartbeat.return_value = None
|
||||
|
||||
chain._TransferChain__heartbeat_owned_leases()
|
||||
|
||||
assert "heartbeat-task" not in chain._owned_leases
|
||||
|
||||
|
||||
def test_close_timeout_keeps_heartbeat_alive_until_blocked_worker_converges() -> None:
|
||||
"""阻塞 worker 未退出时关闭不得停止 heartbeat 或允许租约过期接管。"""
|
||||
chain = _build_chain()
|
||||
worker_release = threading.Event()
|
||||
worker = threading.Thread(
|
||||
target=worker_release.wait,
|
||||
name="transfer-blocked-owner",
|
||||
daemon=True,
|
||||
)
|
||||
heartbeat = threading.Thread(
|
||||
target=chain._lease_heartbeat_stop_event.wait,
|
||||
name="transfer-heartbeat-owner",
|
||||
daemon=True,
|
||||
)
|
||||
chain._threads = [worker]
|
||||
chain._lease_heartbeat_thread = heartbeat
|
||||
chain._owned_leases = {
|
||||
"blocked-task": ("blocked-token", time.monotonic() + 120)
|
||||
}
|
||||
worker.start()
|
||||
heartbeat.start()
|
||||
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert heartbeat.is_alive() is True
|
||||
assert chain._lease_heartbeat_stop_event.is_set() is False
|
||||
|
||||
worker_release.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert worker.is_alive() is False
|
||||
assert heartbeat.is_alive() is False
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="blocked-task",
|
||||
lease_token="blocked-token",
|
||||
error="整理宿主关闭,释放未结算任务租约",
|
||||
)
|
||||
|
||||
|
||||
def test_worker_reports_failed_settlement_without_skipping_queue_bookkeeping(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""终态 CAS=0 必须计为失败,同时仍完成 task_done 与 active 归零。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/stale.mkv",
|
||||
type="file",
|
||||
name="stale.mkv",
|
||||
basename="stale",
|
||||
extension="mkv",
|
||||
))
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.add_task.return_value = True
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._register_scrape_batch_task = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 0
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_with_stale_lease(*, task, callback):
|
||||
"""模拟文件副作用完成后终态 token 已被新 owner 接管。"""
|
||||
del callback
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_with_stale_lease
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
assert chain.put_to_queue(task) is True
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
assert chain._active_tasks == 0
|
||||
assert chain._fail_num == 1
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
|
||||
|
||||
def test_failed_claim_release_waits_for_fixed_recovery_poll() -> None:
|
||||
"""失败释放不得即时唤醒恢复线程,避免确定性错误形成热重试。"""
|
||||
chain = _build_chain()
|
||||
del chain._TransferChain__ensure_recovery_scheduler
|
||||
chain._RECOVERY_POLL_INTERVAL_SECONDS = 0.05
|
||||
chain._TransferChain__replay_pending = MagicMock()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/retry-later.mkv",
|
||||
type="file",
|
||||
))
|
||||
task.bind_admission_task_id("retry-later")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="retry-token",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"retry-later": ("retry-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
assert chain._TransferChain__release_task_claim(
|
||||
task,
|
||||
error="planning failed",
|
||||
) is True
|
||||
|
||||
assert chain._recovery_wakeup_event.is_set() is False
|
||||
assert chain._replay_thread is not None
|
||||
time.sleep(0.01)
|
||||
chain._TransferChain__replay_pending.assert_not_called()
|
||||
deadline = time.monotonic() + 0.5
|
||||
while (
|
||||
not chain._TransferChain__replay_pending.called
|
||||
and time.monotonic() < deadline
|
||||
):
|
||||
time.sleep(0.01)
|
||||
chain._TransferChain__replay_pending.assert_called()
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="retry-later",
|
||||
lease_token="retry-token",
|
||||
error="planning failed",
|
||||
)
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
|
||||
|
||||
def test_worker_fenced_releases_lost_lease_and_completes_queue_bookkeeping(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""本地租约失效时仍尝试 token CAS release,并完整结算内存队列。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/lost-lease.mkv",
|
||||
type="file",
|
||||
))
|
||||
task.bind_admission_task_id("lost-lease")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="lost-token",
|
||||
)
|
||||
chain.jobview = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
stop_event = threading.Event()
|
||||
chain._transfer_admissions.release_claim.side_effect = (
|
||||
lambda **_kwargs: stop_event.set() or True
|
||||
)
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="lost-lease",
|
||||
lease_token="lost-token",
|
||||
error="整理任务租约已经失效:lost-lease",
|
||||
)
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
assert chain._recovery_wakeup_event.is_set() is False
|
||||
|
||||
|
||||
def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) -> None:
|
||||
"""终态 CAS 被拒绝时不得写成功历史、事件或通知。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/fenced-success.mkv",
|
||||
type="file",
|
||||
name="fenced-success.mkv",
|
||||
))
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.add_task.return_value = True
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._register_scrape_batch_task = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 0
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
success_callback = MagicMock(return_value=(True, ""))
|
||||
chain._TransferChain__default_callback = success_callback
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_with_success(*, task, callback):
|
||||
"""模拟文件成功后进入受 durable 终态保护的回调。"""
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return callback(
|
||||
task,
|
||||
TransferInfo(success=True, fileitem=task.fileitem),
|
||||
)
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_with_success
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
assert chain.put_to_queue(task) is True
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
success_callback.assert_not_called()
|
||||
chain.jobview.fail_unfinished_task.assert_called_once_with(task)
|
||||
assert chain._fail_num == 1
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
|
||||
|
||||
def test_close_release_db_block_respects_deadline_and_keeps_heartbeat() -> None:
|
||||
"""关闭租约释放被数据库阻塞时应按预算返回,并继续 heartbeat。"""
|
||||
chain = _build_chain()
|
||||
release_started = threading.Event()
|
||||
release_db = threading.Event()
|
||||
heartbeat = threading.Thread(
|
||||
target=chain._lease_heartbeat_stop_event.wait,
|
||||
name="transfer-heartbeat-release-test",
|
||||
daemon=True,
|
||||
)
|
||||
chain._lease_heartbeat_thread = heartbeat
|
||||
chain._owned_leases = {
|
||||
"blocked-release": ("blocked-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
def block_release(**_kwargs):
|
||||
"""模拟数据库锁住 release_claim,直到测试显式放行。"""
|
||||
release_started.set()
|
||||
release_db.wait()
|
||||
return True
|
||||
|
||||
chain._transfer_admissions.release_claim.side_effect = block_release
|
||||
heartbeat.start()
|
||||
|
||||
started_at = time.monotonic()
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert time.monotonic() - started_at < 0.5
|
||||
assert release_started.is_set()
|
||||
assert heartbeat.is_alive() is True
|
||||
assert chain._lease_heartbeat_stop_event.is_set() is False
|
||||
assert chain._lease_release_thread is not None
|
||||
assert chain._lease_release_thread.is_alive() is True
|
||||
|
||||
release_db.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert heartbeat.is_alive() is False
|
||||
assert chain._lease_release_thread is None
|
||||
|
||||
Reference in New Issue
Block a user