refactor: add fenced transfer recovery leases

This commit is contained in:
jxxghp
2026-08-27 14:39:38 +08:00
parent 22865cb35b
commit a62c541ec0
32 changed files with 4135 additions and 650 deletions
+78 -11
View File
@@ -627,6 +627,14 @@ class TransferPlanningStateError(RuntimeError):
"""计划检查点无法从当前持久状态推进时抛出的状态错误。"""
class TransferAdmissionProjectionError(RuntimeError):
"""持久登记无法安全恢复为应用层整理准入投影时抛出的错误。"""
class TransferLeaseLostError(TransferPlanningStateError):
"""整理 worker 已失去持久租约、不得继续推进任务时抛出的错误。"""
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""
文件整理任务。
@@ -661,6 +669,8 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
_planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None)
_plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None)
_planning_context_restored: bool = PrivateAttr(default=False)
_lease_owner: Optional[str] = PrivateAttr(default=None)
_lease_token: Optional[str] = PrivateAttr(default=None)
@property
def admission_task_id(self) -> Optional[str]:
@@ -698,6 +708,23 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""标记领域上下文已离线恢复,禁止旧流程再次在线补充。"""
self._planning_context_restored = True
@property
def lease_owner(self) -> Optional[str]:
"""返回当前任务绑定的进程级 worker owner。"""
return self._lease_owner
@property
def lease_token(self) -> Optional[str]:
"""返回当前任务绑定的持久租约令牌。"""
return self._lease_token
def bind_execution_lease(self, *, owner_id: str, lease_token: str) -> None:
"""绑定持久 claim 结果,且不改变插件可见的旧任务序列化字段。"""
if not owner_id or not lease_token:
raise ValueError("整理执行租约缺少 owner 或 token")
self._lease_owner = owner_id
self._lease_token = lease_token
def to_dict(self):
"""
返回字典。
@@ -743,6 +770,11 @@ class TransferAdmission:
input_fingerprint: Optional[str] = None
planning_input: Optional[TransferPlanningInput] = None
checkpoint: Optional[TransferPlanCheckpoint] = None
lease_owner: Optional[str] = None
lease_token: Optional[str] = None
lease_expires_at: Optional[str] = None
heartbeat_at: Optional[str] = None
attempt_count: int = 0
class TransferAdmissionRepository(Protocol):
@@ -758,10 +790,6 @@ class TransferAdmissionRepository(Protocol):
"""按规划输入幂等登记源文件并返回稳定任务身份。"""
...
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
"""按登记顺序返回等待恢复或执行的任务。"""
...
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
"""记录内存队列接收失败,保留任务供后续恢复。"""
...
@@ -770,22 +798,61 @@ class TransferAdmissionRepository(Protocol):
self,
*,
task_id: str,
lease_token: str,
input_fingerprint: str,
checkpoint: TransferPlanCheckpoint,
) -> TransferAdmission:
"""原子保存完整计划并匹配输入的任务推进到已规划"""
"""按有效租约原子保存完整计划并推进匹配输入的任务。"""
...
def record_planning_failure(self, *, task_id: str, error: str) -> None:
"""记录规划失败但保留接纳状态供下次恢复重试。"""
def record_planning_failure(
self, *, task_id: str, lease_token: str, error: str
) -> None:
"""按有效租约记录规划失败,保留业务状态供恢复重试。"""
...
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
"""按登记顺序返回接纳或已规划的可恢复任务。"""
def claim_task(
self,
*,
task_id: str,
owner_id: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""为指定任务取得唯一执行租约;已有有效租约时返回 None。"""
...
def discard_task(self, *, task_id: str) -> int:
"""按稳定任务身份删除已经到达终态的登记。"""
def claim_recoverable(
self,
*,
owner_id: str,
limit: int,
lease_seconds: int,
) -> list[TransferAdmission]:
"""按登记顺序原子取得可恢复任务的唯一执行租约。"""
...
def heartbeat(
self,
*,
task_id: str,
lease_token: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""仅为当前且未过期的 token 延长租约;过期 token 不得复活。"""
...
def release_claim(
self,
*,
task_id: str,
lease_token: str,
error: Optional[str] = None,
) -> bool:
"""按 token 释放当前 claim,并可记录可恢复错误。"""
...
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅允许当前 lease owner 删除已经到达终态的登记。"""
...
+644 -42
View File
@@ -46,6 +46,7 @@ from app.application.transfer import (
TransferAdmission,
TransferFailureNotification,
TransferFailureNotificationAggregator,
TransferLeaseLostError,
TransferPlanCheckpoint,
TransferPlanningInput,
TransferPlanningStateError,
@@ -146,6 +147,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
_WORKER_RESTART_TIMEOUT_SECONDS = 30.0
_WORKER_CLOSE_TIMEOUT_SECONDS = 30.0
_QUEUE_STOP_SENTINEL = object()
_WORKER_LEASE_SECONDS = 120
_LEASE_HEARTBEAT_INTERVAL_SECONDS = 30.0
_RECOVERY_POLL_INTERVAL_SECONDS = 15.0
_RECOVERY_CLAIM_LIMIT = 100
@staticmethod
def _transfer_result_payload(
@@ -254,9 +259,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._worker_lifecycle_lock = threading.RLock()
self._worker_state_lock = threading.RLock()
self._closing = False
# pending 回放同样由整理链持有,关闭时可阻止继续处理下一条登记
# 恢复调度与租约续期均由整理链持有,关闭时与 worker 一并收敛。
self._worker_owner_id = uuid.uuid4().hex
self._owned_leases: Dict[str, Tuple[str, float]] = {}
self._queued_lease_tokens: set[Tuple[str, str]] = set()
self._replay_thread: Optional[threading.Thread] = None
self._replay_stop_event = threading.Event()
self._recovery_wakeup_event = threading.Event()
self._lease_heartbeat_thread: Optional[threading.Thread] = None
self._lease_heartbeat_stop_event = threading.Event()
self._lease_release_thread: Optional[threading.Thread] = None
self._active_tasks = 0
self._processed_num = 0
self._fail_num = 0
@@ -366,6 +378,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
:param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数
:return: 全部后台线程均已收敛时返回 True,否则返回 False
"""
self.__ensure_lease_runtime_state()
deadline = time.monotonic() + max(0.0, timeout_seconds)
if not self.__acquire_worker_lifecycle_lock(deadline):
logger.error(
@@ -378,7 +391,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._closing = True
worker_threads = self.__request_worker_stop()
replay_thread = self._replay_thread
heartbeat_thread = self._lease_heartbeat_thread
self._replay_stop_event.set()
self._recovery_wakeup_event.set()
alive_workers = self.__join_threads(worker_threads, deadline)
alive_replays = self.__join_threads(
@@ -387,7 +402,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
with self._worker_state_lock:
self._threads = []
self._retiring_threads = alive_workers
if replay_thread and not alive_replays and self._replay_thread is replay_thread:
if (
replay_thread
and replay_thread not in alive_replays
and self._replay_thread is replay_thread
):
self._replay_thread = None
alive_threads = [*alive_workers, *alive_replays]
@@ -398,6 +417,38 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
", ".join(thread.name for thread in alive_threads),
)
return False
with self._worker_state_lock:
release_thread = self.__start_lease_release_owner_locked(
error="整理宿主关闭,释放未结算任务租约"
)
alive_releases = self.__join_threads(
[release_thread] if release_thread else [], deadline
)
if alive_releases:
logger.error(
"整理租约释放线程未在 %.1f 秒内收敛,heartbeat 保持运行:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_releases),
)
return False
with self._worker_state_lock:
if self._lease_release_thread is release_thread:
self._lease_release_thread = None
self._lease_heartbeat_stop_event.set()
alive_heartbeats = self.__join_threads(
[heartbeat_thread] if heartbeat_thread else [], deadline
)
if heartbeat_thread and not alive_heartbeats:
with self._worker_state_lock:
if self._lease_heartbeat_thread is heartbeat_thread:
self._lease_heartbeat_thread = None
if alive_heartbeats:
logger.error(
"整理租约续期线程未在 %.1f 秒内收敛:%s",
max(0.0, timeout_seconds),
", ".join(thread.name for thread in alive_heartbeats),
)
return False
logger.info("文件整理 worker 与待处理回放线程已关闭")
return True
finally:
@@ -885,6 +936,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
if self._closing:
logger.warning("文件整理链已关闭,拒绝新的队列任务")
return False
if isinstance(task.lease_token, str) and task.lease_token:
return self.__enqueue_claimed_task(task)
return self._transfer_queue_service().put(task, self.__default_callback)
def _transfer_queue_service(self) -> TransferQueueService:
@@ -902,39 +955,369 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
def replay_pending(self) -> None:
"""
回放上次进程退出时仍未整理完的文件
启动唯一恢复调度 owner,并唤醒一次即时恢复扫描
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
挂死状态,同步执行会把整个启动流程堵住
启动回放、同进程入队补偿和租约过期接管都经由这个入口。调度线程只负责
claim 和重新入队;实际业务仍由普通整理 worker 执行
"""
self.__ensure_recovery_scheduler(immediate=True)
def __ensure_recovery_scheduler(self, *, immediate: bool) -> None:
"""确保唯一恢复调度 owner 存在,并只为显式请求执行即时扫描。"""
self.__ensure_lease_runtime_state()
with self._worker_state_lock:
if self._closing:
logger.info("文件整理链正在关闭,跳过待处理文件回放")
return
self.__start_lease_heartbeat_owner_locked()
if self._replay_thread and self._replay_thread.is_alive():
logger.info("待处理文件回放已在运行,跳过重复启动")
if immediate:
self._recovery_wakeup_event.set()
return
stop_event = threading.Event()
thread = threading.Thread(
target=self.__run_replay_pending,
args=(stop_event,),
args=(self._replay_stop_event, immediate),
name="MoviePilot-TransferReplay",
daemon=True,
)
self._replay_stop_event = stop_event
self._replay_thread = thread
if immediate:
self._recovery_wakeup_event.set()
# 在状态锁内启动,避免 close_workers 看到尚未 start 的线程后错误 join。
thread.start()
def __run_replay_pending(self, stop_event: threading.Event) -> None:
"""执行一次受控回放,并在自然结束后释放当前线程句柄。"""
def __run_replay_pending(
self,
stop_event: threading.Event,
initial_immediate: bool = True,
) -> None:
"""按初次扫描意图持续恢复,失败兜底新建 owner 时先等待固定轮询。"""
try:
self.__replay_pending(stop_event)
if not initial_immediate:
self._recovery_wakeup_event.wait(
timeout=self._RECOVERY_POLL_INTERVAL_SECONDS
)
while not stop_event.is_set():
self._recovery_wakeup_event.clear()
self.__replay_pending(stop_event)
self._recovery_wakeup_event.wait(
timeout=self._RECOVERY_POLL_INTERVAL_SECONDS
)
finally:
with self._worker_state_lock:
if self._replay_thread is threading.current_thread():
self._replay_thread = None
def __ensure_lease_runtime_state(self) -> None:
"""为绕过构造器的兼容调用补齐进程 owner 与租约线程状态。"""
if not hasattr(self, "_worker_owner_id"):
self._worker_owner_id = uuid.uuid4().hex
if not hasattr(self, "_owned_leases"):
self._owned_leases = {}
if not hasattr(self, "_queued_lease_tokens"):
self._queued_lease_tokens = set()
if not hasattr(self, "_recovery_wakeup_event"):
self._recovery_wakeup_event = threading.Event()
if not hasattr(self, "_lease_heartbeat_thread"):
self._lease_heartbeat_thread = None
if not hasattr(self, "_lease_heartbeat_stop_event"):
self._lease_heartbeat_stop_event = threading.Event()
if not hasattr(self, "_lease_release_thread"):
self._lease_release_thread = None
if not hasattr(self, "_replay_thread"):
self._replay_thread = None
if not hasattr(self, "_replay_stop_event"):
self._replay_stop_event = threading.Event()
if not hasattr(self, "_worker_state_lock"):
self._worker_state_lock = threading.RLock()
if not hasattr(self, "_closing"):
self._closing = False
def __start_lease_heartbeat_owner_locked(self) -> None:
"""在状态锁内确保当前进程只有一个租约续期线程。"""
heartbeat_thread = self._lease_heartbeat_thread
if heartbeat_thread and heartbeat_thread.is_alive():
return
heartbeat_thread = threading.Thread(
target=self.__run_lease_heartbeat,
args=(self._lease_heartbeat_stop_event,),
name="MoviePilot-TransferLeaseHeartbeat",
daemon=True,
)
self._lease_heartbeat_thread = heartbeat_thread
heartbeat_thread.start()
def __ensure_lease_heartbeat_owner(self) -> None:
"""按需启动进程级租约续期 owner,供启动前到达的普通任务使用。"""
self.__ensure_lease_runtime_state()
with self._worker_state_lock:
if not self._closing:
self.__start_lease_heartbeat_owner_locked()
def __run_lease_heartbeat(self, stop_event: threading.Event) -> None:
"""按固定周期续期本进程已 claim 且尚未结算的任务。"""
try:
while not stop_event.wait(self._LEASE_HEARTBEAT_INTERVAL_SECONDS):
self.__heartbeat_owned_leases()
finally:
with self._worker_state_lock:
if self._lease_heartbeat_thread is threading.current_thread():
self._lease_heartbeat_thread = None
def __heartbeat_owned_leases(self) -> None:
"""经 Application Port 续期所有排队中或执行中的任务租约。"""
self.__ensure_lease_runtime_state()
with self._worker_state_lock:
owned_leases = list(self._owned_leases.items())
for task_id, (lease_token, deadline) in owned_leases:
try:
admission = self._transfer_admissions.heartbeat(
task_id=task_id,
lease_token=lease_token,
lease_seconds=self._WORKER_LEASE_SECONDS,
)
except Exception as err:
if time.monotonic() >= deadline:
self.__forget_owned_lease(task_id, lease_token)
logger.error(
"整理任务租约续期持续失败并已超过本地期限:%s - %s",
task_id,
err,
)
else:
logger.error(f"整理任务租约续期失败:{task_id} - {err}")
continue
if (
admission is None
or admission.lease_owner != self._worker_owner_id
or admission.lease_token != lease_token
):
self.__forget_owned_lease(task_id, lease_token)
logger.error(f"整理任务租约已失效或被接管:{task_id}")
continue
with self._worker_state_lock:
current = self._owned_leases.get(task_id)
if current and current[0] == lease_token:
self._owned_leases[task_id] = (
lease_token,
time.monotonic() + self._WORKER_LEASE_SECONDS,
)
def __bind_claimed_admission(
self,
task: TransferTask,
admission: TransferAdmission,
) -> None:
"""校验仓储 claim 投影并把 token 私有绑定到执行任务。"""
if (
admission.task_id != task.admission_task_id
):
raise TransferLeaseLostError(
f"整理任务 claim 投影无效:{admission.task_id}"
)
self.__register_claimed_admission(admission)
assert admission.lease_owner is not None
assert admission.lease_token is not None
task.bind_execution_lease(
owner_id=admission.lease_owner,
lease_token=admission.lease_token,
)
def __register_claimed_admission(
self,
admission: TransferAdmission,
) -> None:
"""把仓储 claim 加入续期集合,覆盖恢复构造阶段可能发生的同步 I/O。"""
if (
admission.lease_owner != self._worker_owner_id
or not admission.lease_token
):
raise TransferLeaseLostError(
f"整理任务 claim 投影无效:{admission.task_id}"
)
with self._worker_state_lock:
self._owned_leases[admission.task_id] = (
admission.lease_token,
time.monotonic() + self._WORKER_LEASE_SECONDS,
)
self.__ensure_lease_heartbeat_owner()
def __forget_owned_lease(self, task_id: str, lease_token: str) -> None:
"""仅在 token 仍匹配时移除本进程租约镜像,避免删掉新接管记录。"""
with self._worker_state_lock:
current = self._owned_leases.get(task_id)
if current and current[0] == lease_token:
self._owned_leases.pop(task_id, None)
self._queued_lease_tokens.discard((task_id, lease_token))
def __is_claimed_task_enqueued(self, task_id: str, lease_token: str) -> bool:
"""返回指定 claim 是否已经成功进入普通 worker 队列。"""
with self._worker_state_lock:
return (task_id, lease_token) in self._queued_lease_tokens
def __owns_lease(self, task_id: str, lease_token: Optional[str]) -> bool:
"""返回本地续期镜像是否仍持有指定 token。"""
if not lease_token:
return False
with self._worker_state_lock:
current = self._owned_leases.get(task_id)
return bool(current and current[0] == lease_token)
def __assert_owned_lease(self, task: TransferTask) -> None:
"""拒绝无 token、已过本地期限或已被 heartbeat 判失效的任务推进。"""
if task.preview:
return
task_id = task.admission_task_id
lease_token = task.lease_token
if (
not task_id
or not lease_token
or task.lease_owner != self._worker_owner_id
):
raise TransferLeaseLostError("整理任务缺少当前进程的有效执行租约")
with self._worker_state_lock:
current = self._owned_leases.get(task_id)
if (
current is None
or current[0] != lease_token
or current[1] <= time.monotonic()
):
if current and current[0] == lease_token:
self._owned_leases.pop(task_id, None)
raise TransferLeaseLostError(f"整理任务租约已经失效:{task_id}")
def __claim_task_for_execution(self, task: TransferTask) -> None:
"""让普通队列任务在业务执行前取得唯一租约,恢复任务复用既有 token。"""
if task.preview:
return
self.__ensure_lease_runtime_state()
if task.lease_token:
self.__assert_owned_lease(task)
return
if not task.admission_task_id:
admitted = self.__admit_transfer(task)
task.bind_admission_task_id(admitted.task_id)
task_id = task.admission_task_id
if task_id is None:
raise TransferLeaseLostError("整理任务准入后仍缺少 durable 身份")
claimed = self._transfer_admissions.claim_task(
task_id=task_id,
owner_id=self._worker_owner_id,
lease_seconds=self._WORKER_LEASE_SECONDS,
)
if claimed is None:
raise TransferLeaseLostError(
f"整理任务已由其他 worker claim{task_id}"
)
try:
self.__bind_claimed_admission(task, claimed)
except Exception as err:
self.__release_admission_claim(claimed, error=str(err))
raise
def __release_task_claim(
self,
task: TransferTask,
*,
error: Optional[str] = None,
) -> bool:
"""按 task token 释放未到终态的 claim,并按固定轮询等待恢复。"""
task_id = task.admission_task_id if task else None
lease_token = task.lease_token if task else None
if not task_id or not lease_token:
return False
try:
return self._transfer_admissions.release_claim(
task_id=task_id,
lease_token=lease_token,
error=error,
)
except Exception as err:
logger.error(f"释放整理任务租约失败:{task_id} - {err}")
return False
finally:
self.__forget_owned_lease(task_id, lease_token)
self.__ensure_recovery_scheduler(immediate=False)
def __release_admission_claim(
self,
admission: TransferAdmission,
*,
error: Optional[str] = None,
) -> None:
"""释放尚未绑定或成功入队的恢复 claim。"""
if not admission.lease_token:
return
try:
self._transfer_admissions.release_claim(
task_id=admission.task_id,
lease_token=admission.lease_token,
error=error,
)
except Exception as err:
logger.error(f"释放恢复任务租约失败:{admission.task_id} - {err}")
finally:
self.__forget_owned_lease(admission.task_id, admission.lease_token)
def __release_all_owned_leases(self, *, error: str) -> None:
"""在执行 owner 全部收敛后释放剩余 claim,避免关停后等待租约自然过期。"""
with self._worker_state_lock:
owned_leases = list(self._owned_leases.items())
for task_id, (lease_token, _deadline) in owned_leases:
try:
self._transfer_admissions.release_claim(
task_id=task_id,
lease_token=lease_token,
error=error,
)
except Exception as err:
logger.error(f"关闭时释放整理任务租约失败:{task_id} - {err}")
finally:
self.__forget_owned_lease(task_id, lease_token)
def __start_lease_release_owner_locked(
self, *, error: str
) -> Optional[threading.Thread]:
"""启动并持有唯一租约释放线程,使同步数据库阻塞不突破关闭预算。"""
release_thread = self._lease_release_thread
if release_thread is not None:
return release_thread
if not self._owned_leases:
return None
release_thread = threading.Thread(
target=self.__release_all_owned_leases,
kwargs={"error": error},
name="MoviePilot-TransferLeaseRelease",
daemon=True,
)
self._lease_release_thread = release_thread
release_thread.start()
return release_thread
def __enqueue_claimed_task(self, task: TransferTask) -> bool:
"""把已 claim 的恢复任务送入普通队列,禁止再次准入或二次 claim。"""
self.__assert_owned_lease(task)
if not self.__put_to_jobview(task):
return False
try:
self._register_scrape_batch_task(task)
assert task.admission_task_id is not None
assert task.lease_token is not None
with self._worker_state_lock:
self._queued_lease_tokens.add(
(task.admission_task_id, task.lease_token)
)
self._queue.put(
TransferQueue(task=task, callback=self.__default_callback)
)
except Exception as err:
try:
self.__record_enqueue_failure(task, err)
finally:
self.jobview.remove_task(task.fileitem)
raise
return True
def __replay_pending(
self, stop_event: Optional[threading.Event] = None
) -> None:
@@ -948,17 +1331,35 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
stop_event = stop_event or threading.Event()
if stop_event.is_set():
return
self.__ensure_lease_runtime_state()
try:
pendings = self._transfer_admissions.list_recoverable()
pendings = self._transfer_admissions.claim_recoverable(
owner_id=self._worker_owner_id,
limit=self._RECOVERY_CLAIM_LIMIT,
lease_seconds=self._WORKER_LEASE_SECONDS,
)
except Exception as err:
logger.error(f"读取待整理文件登记失败:{err}")
return
if not pendings:
return
try:
for admission in pendings:
self.__register_claimed_admission(admission)
except Exception as err:
logger.error(f"登记恢复任务租约失败:{err}")
for claimed in pendings:
self.__release_admission_claim(claimed, error=str(err))
return
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
replayed = 0
for admission in pendings:
for index, admission in enumerate(pendings):
if stop_event.is_set():
for unprocessed in pendings[index:]:
self.__release_admission_claim(
unprocessed,
error="整理宿主关闭,恢复任务尚未入队",
)
break
storage = admission.storage
src_path = admission.src_path
@@ -970,17 +1371,50 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
)
# stat 等同步 I/O 返回后重新检查,关闭期间不得注销尚未完成的登记。
if stop_event.is_set():
self.__release_admission_claim(
admission,
error="整理宿主关闭,恢复任务尚未入队",
)
for unprocessed in pendings[index + 1:]:
self.__release_admission_claim(
unprocessed,
error="整理宿主关闭,恢复任务尚未入队",
)
break
if not fileitem:
if should_discard:
# 源文件确认已消失,注销登记避免每次启动重复回放
self._transfer_admissions.discard_task(
task_id=admission.task_id
lease_token = admission.lease_token
if lease_token is None:
raise TransferLeaseLostError(
f"恢复任务缺少 lease token{admission.task_id}"
)
discarded = self._transfer_admissions.discard_claimed(
task_id=admission.task_id,
lease_token=lease_token,
)
if not discarded:
logger.warning(
f"恢复任务终态注销被 CAS 拒绝:{admission.task_id}"
)
self.__forget_owned_lease(
admission.task_id,
lease_token,
)
else:
self.__release_admission_claim(
admission,
error="恢复源文件暂时不可读取",
)
continue
if admission.checkpoint:
if self.__queue_planned_replay(fileitem, admission):
replayed += 1
else:
self.__release_admission_claim(
admission,
error="恢复任务未进入内存队列",
)
continue
planning_input = admission.planning_input
if planning_input and (
@@ -990,6 +1424,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
):
if self.__queue_accepted_replay(fileitem, admission):
replayed += 1
else:
self.__release_admission_claim(
admission,
error="恢复任务未进入内存队列",
)
continue
replay_kwargs = self.__build_replay_kwargs(planning_input)
self._execute_transfer(
@@ -997,9 +1436,21 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
recovery_admission=admission,
**replay_kwargs,
)
replayed += 1
assert admission.lease_token is not None
if self.__is_claimed_task_enqueued(
admission.task_id,
admission.lease_token,
):
replayed += 1
else:
self.__release_admission_claim(
admission,
error="旧恢复入口未产生可执行队列任务",
)
except Exception as err:
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
if self.__owns_lease(admission.task_id, admission.lease_token):
self.__release_admission_claim(admission, error=str(err))
if stop_event.is_set():
logger.info(
"待整理文件回放收到关闭请求,已送入 %s 个文件,其余登记保持待处理",
@@ -1116,6 +1567,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
preview=False,
)
task.bind_admission_task_id(admission.task_id)
self.__bind_claimed_admission(task, admission)
task.bind_planning_input(planning_input)
if planning_input.mediainfo:
task.mark_planning_context_restored()
@@ -1168,6 +1620,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
preview=False,
)
task.bind_admission_task_id(admission.task_id)
self.__bind_claimed_admission(task, admission)
task.bind_planning_input(checkpoint.planning_input)
task.bind_plan_checkpoint(checkpoint)
self.__restore_planned_task(task)
@@ -1442,11 +1895,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
error: object,
) -> None:
"""记录 checkpoint 前失败,保留 accepted 任务供后续重新规划。"""
if task.preview or task.plan_checkpoint or not task.admission_task_id:
if (
task.preview
or task.plan_checkpoint
or not task.admission_task_id
or not task.lease_token
):
return
try:
self._transfer_admissions.record_planning_failure(
task_id=task.admission_task_id,
lease_token=task.lease_token,
error=str(error),
)
except Exception as record_error:
@@ -1488,9 +1947,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
"""先提交冻结 provider 调用,空结果时再提交并执行宿主计划。"""
planning_input = task.planning_input or self.__build_planning_input(task)
task.bind_planning_input(planning_input)
if not task.preview and not task.admission_task_id:
admission = self.__admit_transfer(task)
task.bind_admission_task_id(admission.task_id)
if not task.preview:
self.__claim_task_for_execution(task)
self.__assert_owned_lease(task)
checkpoint = task.plan_checkpoint
if checkpoint is None:
@@ -1540,6 +1999,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
raise
self.__restore_planned_task(task)
self.__assert_owned_lease(task)
legacy_result = self.__execute_legacy_transfer_providers(
task,
@@ -1571,6 +2031,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self.__record_checkpoint_failure(task, error)
raise
self.__assert_owned_lease(task)
result = self.execute_transfer_plan(
checkpoint,
meta=task.meta,
@@ -1623,8 +2084,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
return checkpoint
if not task.admission_task_id:
raise RuntimeError("整理计划提交前缺少持久任务身份")
self.__assert_owned_lease(task)
assert task.lease_token is not None
persisted = self._transfer_admissions.checkpoint_plan(
task_id=task.admission_task_id,
lease_token=task.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
@@ -1638,11 +2102,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
error: Exception,
) -> None:
"""记录当前 checkpoint 阶段失败并保留原状态供重启恢复。"""
if task.preview or not task.admission_task_id:
if task.preview or not task.admission_task_id or not task.lease_token:
return
try:
self._transfer_admissions.record_planning_failure(
task_id=task.admission_task_id,
lease_token=task.lease_token,
error=str(error),
)
setattr(error, "_transfer_planning_failure_recorded", True)
@@ -1790,6 +2255,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
)
except Exception as error:
logger.error(f"旧整理兼容命令执行失败:{error}")
self.__release_task_claim(task, error=str(error))
return TransferInfo(
success=False,
message=str(error),
@@ -1797,7 +2263,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
fail_list=[fileitem.path],
transfer_type=transfer_type,
)
self.__discard_pending(task)
if not self.__discard_pending(task):
message = "旧整理兼容命令已执行,但 durable 终态结算失去租约"
logger.error(message)
return TransferInfo(
success=False,
message=message,
fileitem=fileitem,
fail_list=[fileitem.path],
transfer_type=transfer_type,
)
return result
def __record_enqueue_failure(
@@ -1805,9 +2280,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
task: TransferTask,
error: Exception,
) -> None:
"""记录内存入队失败并撤销该任务的批次占位。"""
"""按是否已经 claim 选择 fencing 失败记录,并撤销批次占位。"""
try:
if task.admission_task_id:
if task.lease_token:
self.__release_task_claim(task, error=str(error))
elif task.admission_task_id:
self._transfer_admissions.record_enqueue_failure(
task_id=task.admission_task_id,
error=str(error),
@@ -1819,8 +2296,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
)
finally:
self._finish_scrape_batch_task(task)
self.__ensure_recovery_scheduler(immediate=False)
def __discard_pending(self, task: TransferTask):
def __discard_pending(self, task: TransferTask) -> bool:
"""
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
@@ -1829,16 +2307,34 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
:param task: 任务信息
"""
if not task or not task.admission_task_id:
return
try:
self._transfer_admissions.discard_task(
task_id=task.admission_task_id
return True
if not task.lease_token:
logger.error(
f"整理任务缺少 lease token,拒绝伪装 durable 终态:"
f"{task.admission_task_id}"
)
return False
discarded = 0
try:
discarded = self._transfer_admissions.discard_claimed(
task_id=task.admission_task_id,
lease_token=task.lease_token,
)
if not discarded:
logger.error(
f"整理任务终态注销被 CAS 拒绝:{task.admission_task_id}"
)
except Exception as err:
logger.error(
"注销整理任务 durable admission 失败: "
f"{task.admission_task_id} - {err}"
)
finally:
self.__forget_owned_lease(
task.admission_task_id,
task.lease_token,
)
return bool(discarded)
def __put_to_jobview(self, task: TransferTask) -> bool:
"""
@@ -1905,13 +2401,23 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
if marker:
marker(task)
def __finish_job_execution(self, task: TransferTask, *, terminal: bool = True):
"""结束内存执行;仅确定到达业务终态时注销 durable 记录。"""
def __finish_job_execution(
self,
task: TransferTask,
*,
terminal: bool = True,
terminal_settlement: Optional[bool] = None,
) -> bool:
"""结束内存执行,并复用成功回调前已经提交的 durable 终态结果。"""
marker = getattr(self.jobview, "finish_execution", None)
if marker:
marker(task)
if terminal:
self.__discard_pending(task)
if terminal_settlement is not None:
return terminal_settlement
return self.__discard_pending(task)
self.__release_task_claim(task)
return True
def __expire_stale_transfer_tasks(self):
"""清理外部接管后失去状态心跳的运行中整理任务。"""
@@ -2001,6 +2507,32 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self.__settle_transfer_progress_if_idle()
continue
if task.admission_task_id and task.lease_token:
with self._worker_state_lock:
self._queued_lease_tokens.discard(
(task.admission_task_id, task.lease_token)
)
try:
self.__claim_task_for_execution(task)
except TransferLeaseLostError as err:
logger.info(f"跳过未取得执行租约的整理任务:{err}")
self.__release_task_claim(task, error=str(err))
self.jobview.try_remove_job(task)
self._finish_scrape_batch_task(task)
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
continue
except Exception as err:
logger.error(
f"整理任务 claim 失败,保留 durable admission{err}"
)
self.jobview.try_remove_job(task)
self._finish_scrape_batch_task(task)
self._queue.task_done()
self.__settle_transfer_progress_if_idle()
self.__ensure_recovery_scheduler(immediate=False)
continue
# 文件信息
fileitem = task.fileitem
@@ -2030,6 +2562,29 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._active_tasks += 1
terminal = False
terminal_settlement: Optional[bool] = None
state = False
err_msg = ""
def callback_after_terminal_settlement(
callback_task: TransferTask,
transferinfo: TransferInfo,
) -> Tuple[bool, str]:
"""成功结果先提交 durable 终态,再执行历史、事件和通知回调。"""
nonlocal terminal_settlement
if transferinfo.success and not callback_task.preview:
terminal_settlement = self.__discard_pending(callback_task)
if not terminal_settlement:
self.__fail_transfer_task(callback_task)
return False, "整理任务 durable 终态结算失去租约"
if item.callback:
callback_result: Tuple[bool, str] = item.callback(
callback_task,
transferinfo,
)
return callback_result
return transferinfo.success, transferinfo.message or ""
try:
self.__start_job_execution(task)
# 更新进度
@@ -2044,7 +2599,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
)
# 整理
state, err_msg = self.__handle_transfer(
task=task, callback=item.callback
task=task,
callback=callback_after_terminal_settlement,
)
terminal = task.plan_checkpoint is not None
@@ -2063,6 +2619,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
text=__process_msg,
)
except Exception as e:
if terminal_settlement is not None:
terminal = True
logger.error(
f"{fileitem.name} 整理任务处理出现错误:{e} - {traceback.format_exc()}"
)
@@ -2071,12 +2629,25 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self._processed_num += 1
self._fail_num += 1
finally:
self.__finish_job_execution(task, terminal=terminal)
self._queue.task_done()
with task_lock:
# 减少运行中的任务数
self._active_tasks -= 1
self.__settle_transfer_progress_if_idle()
durable_settled = False
try:
durable_settled = self.__finish_job_execution(
task,
terminal=terminal,
terminal_settlement=terminal_settlement,
)
except Exception as err:
logger.error(
f"整理任务终态结算异常:{task.admission_task_id} - {err}"
)
finally:
self._queue.task_done()
with task_lock:
# 减少运行中的任务数
self._active_tasks -= 1
if terminal and state and not durable_settled:
self._fail_num += 1
self.__settle_transfer_progress_if_idle()
except queue.Empty:
# 即使队列空了,如果还有任务在运行,也不应该结束进度
@@ -3548,6 +4119,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
and file_item.path == recovery_admission.src_path
):
transfer_task.bind_admission_task_id(recovery_admission.task_id)
self.__bind_claimed_admission(
transfer_task,
recovery_admission,
)
if recovery_admission.planning_input:
transfer_task.bind_planning_input(
recovery_admission.planning_input
@@ -3651,14 +4226,37 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
},
)
terminal = False
terminal_settlement: Optional[bool] = None
def callback_after_terminal_settlement(
callback_task: TransferTask,
transferinfo: TransferInfo,
) -> Tuple[bool, str]:
"""同步成功结果先结算 durable 行,再执行兼容回调副作用。"""
nonlocal terminal_settlement
if transferinfo.success and not callback_task.preview:
terminal_settlement = self.__discard_pending(callback_task)
if not terminal_settlement:
self.__fail_transfer_task(callback_task)
return False, "整理任务 durable 终态结算失去租约"
callback = (
_preview_callback
if preview
else self.__default_callback
)
return callback(callback_task, transferinfo)
try:
self.__claim_task_for_execution(transfer_task)
self.__start_job_execution(transfer_task)
state, err_msg = self.__handle_transfer(
task=transfer_task,
callback=_preview_callback if preview else self.__default_callback,
callback=callback_after_terminal_settlement,
)
terminal = bool(preview or transfer_task.plan_checkpoint is not None)
except Exception as e:
if terminal_settlement is not None:
terminal = True
logger.error(
f"{transfer_task.fileitem.name} 整理任务处理出现错误:"
f"{e} - {traceback.format_exc()}"
@@ -3667,10 +4265,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
self.__fail_transfer_task(transfer_task)
state, err_msg = False, str(e)
finally:
self.__finish_job_execution(
durable_settled = self.__finish_job_execution(
transfer_task,
terminal=terminal,
terminal_settlement=terminal_settlement,
)
if terminal and not durable_settled:
state = False
err_msg = "整理任务 durable 终态结算失去租约"
if not state:
all_success = False
logger.warn(f"{transfer_task.fileitem.name} {err_msg}")
+290 -26
View File
@@ -1,7 +1,9 @@
"""整理任务持久准入端口的 SQLAlchemy 适配器。"""
import logging
from collections.abc import Callable
from datetime import datetime
from datetime import datetime, timedelta, timezone
from json import JSONDecodeError
from typing import Optional
from uuid import uuid4
@@ -14,6 +16,8 @@ from app.application.transfer import (
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferAdmission,
TransferAdmissionConflictError,
TransferAdmissionProjectionError,
TransferLeaseLostError,
TransferPlanCheckpoint,
TransferPlanningInput,
TransferPlanningStateError,
@@ -22,10 +26,14 @@ from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
from app.db.uow import SqlAlchemyUnitOfWork
_diagnostic_logger = logging.getLogger(__name__)
class TransactionalTransferAdmissionRepository:
"""以短生命周期 Session 实现整理任务持久准入端口。"""
_MAX_RECOVERY_SCAN_TASKS = 5000
def __init__(self, session_factory: Callable[[], Session]) -> None:
"""保存由组合根提供的同步会话工厂。"""
self._session_factory = session_factory
@@ -35,6 +43,33 @@ class TransactionalTransferAdmissionRepository:
"""生成与历史登记时间可按字典序比较的当前时间。"""
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@staticmethod
def _lease_now() -> datetime:
"""生成不受宿主时区影响的当前 UTC 租约时间。"""
return datetime.now(timezone.utc)
@staticmethod
def _format_lease_time(value: datetime) -> str:
"""把 UTC 时间编码为可稳定排序的固定宽度字符串。"""
return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")
@staticmethod
def _validate_claim_arguments(*, owner_id: str, lease_seconds: int) -> None:
"""拒绝无法建立有效租约身份或正向期限的调用。"""
if not owner_id:
raise ValueError("整理任务 claim 缺少 owner_id")
if lease_seconds <= 0:
raise ValueError("整理任务 lease_seconds 必须大于零")
@staticmethod
def _recoverable_states() -> tuple[str, ...]:
"""返回允许被 worker claim 的稳定业务状态。"""
return (
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
)
@staticmethod
def _project(pending: TransferPending) -> TransferAdmission:
"""在 Session 有效期内把 ORM 行冻结为应用层 DTO。"""
@@ -84,6 +119,11 @@ class TransactionalTransferAdmissionRepository:
input_fingerprint=pending.input_fingerprint,
planning_input=planning_input,
checkpoint=checkpoint,
lease_owner=pending.lease_owner,
lease_token=pending.lease_token,
lease_expires_at=pending.lease_expires_at,
heartbeat_at=pending.heartbeat_at,
attempt_count=pending.attempt_count,
)
@staticmethod
@@ -151,27 +191,225 @@ class TransactionalTransferAdmissionRepository:
self._assert_input_match(pending, effective_input)
return self._project(pending)
def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]:
"""在独立只读会话中投影等待恢复或执行的准入记录。"""
def claim_task(
self,
*,
task_id: str,
owner_id: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""原子 claim 指定任务,任何已存在的有效租约都拒绝重复领取。"""
self._validate_claim_arguments(
owner_id=owner_id,
lease_seconds=lease_seconds,
)
if not task_id:
raise ValueError("整理任务 claim 缺少 task_id")
now = self._lease_now()
now_time = self._format_lease_time(now)
lease_expires_at = self._format_lease_time(
now + timedelta(seconds=lease_seconds)
)
with self._session_factory() as session:
pending_items = TransferPendingOper(db=session).list_by_state(
state=TRANSFER_ADMISSION_ACCEPTED,
limit=limit,
)
return [self._project(pending) for pending in pending_items]
transaction = SqlAlchemyUnitOfWork(session)
try:
oper = TransferPendingOper(db=session)
lease_token = uuid4().hex
updated = oper.stage_claim_task(
task_id=task_id,
states=self._recoverable_states(),
owner_id=owner_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
updated_at=self._now(),
)
session.flush()
session.expire_all()
try:
pending = oper.get_by_task_id(task_id=task_id)
except JSONDecodeError as error:
raise TransferAdmissionProjectionError(
f"整理任务持久 JSON 无法解码: {task_id} - {error}"
) from error
if updated:
if pending is None:
raise TransferPlanningStateError(
f"claim 后未找到整理任务: {task_id}"
)
try:
admission = self._project(pending)
except (
TransferAdmissionConflictError,
TransferPlanningStateError,
TypeError,
ValueError,
) as error:
raise TransferAdmissionProjectionError(
f"整理任务持久投影损坏: {task_id} - {error}"
) from error
transaction.commit()
return admission
transaction.commit()
return None
except Exception:
transaction.rollback()
raise
def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]:
"""投影接纳、provider 待执行或已规划的全部可恢复任务。"""
with self._session_factory() as session:
pending_items = TransferPendingOper(db=session).list_by_states(
states=(
TRANSFER_ADMISSION_ACCEPTED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TRANSFER_ADMISSION_PLANNED,
),
limit=limit,
def claim_recoverable(
self,
*,
owner_id: str,
limit: int,
lease_seconds: int,
) -> list[TransferAdmission]:
"""按登记顺序逐条 CAS claim 未租用或租约已过期的恢复任务。"""
self._validate_claim_arguments(
owner_id=owner_id,
lease_seconds=lease_seconds,
)
if limit <= 0:
return []
claimed: list[TransferAdmission] = []
after_cursor: Optional[tuple[str, int]] = None
scanned_count = 0
scan_limit = self._MAX_RECOVERY_SCAN_TASKS
while len(claimed) < limit and scanned_count < scan_limit:
candidate_limit = min(
limit - len(claimed),
scan_limit - scanned_count,
)
return [self._project(pending) for pending in pending_items]
now_time = self._format_lease_time(self._lease_now())
with self._session_factory() as session:
candidates = TransferPendingOper(db=session).list_claimable_candidates(
states=self._recoverable_states(),
now_time=now_time,
limit=candidate_limit,
after_cursor=after_cursor,
)
if not candidates:
break
scanned_count += len(candidates)
_, cursor_created_at, cursor_id = candidates[-1]
after_cursor = (cursor_created_at, cursor_id)
for task_id, _, _ in candidates:
try:
admission = self.claim_task(
task_id=task_id,
owner_id=owner_id,
lease_seconds=lease_seconds,
)
except TransferAdmissionProjectionError as error:
self._record_projection_failure(
task_id=task_id,
error=error,
)
continue
if admission is not None:
claimed.append(admission)
if len(claimed) >= limit:
break
return claimed
def _record_projection_failure(
self,
*,
task_id: str,
error: TransferAdmissionProjectionError,
) -> bool:
"""以独立 CAS 留存变化后的投影错误,并仅为新诊断记一次运行日志。"""
diagnostic = f"恢复投影失败: {error}"
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
recorded = TransferPendingOper(
db=session
).stage_record_projection_failure(
task_id=task_id,
states=self._recoverable_states(),
error=diagnostic,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
transaction.commit()
except Exception:
transaction.rollback()
raise
if recorded:
_diagnostic_logger.error(
f"整理恢复任务投影损坏:task_id={task_id}, error={error}"
)
return bool(recorded)
def heartbeat(
self,
*,
task_id: str,
lease_token: str,
lease_seconds: int,
) -> Optional[TransferAdmission]:
"""仅以当前且未过期的 token 续租,禁止陈旧 worker 复活租约。"""
if not task_id or not lease_token:
raise ValueError("整理任务 heartbeat 缺少任务或租约身份")
if lease_seconds <= 0:
raise ValueError("整理任务 lease_seconds 必须大于零")
now = self._lease_now()
now_time = self._format_lease_time(now)
lease_expires_at = self._format_lease_time(
now + timedelta(seconds=lease_seconds)
)
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
oper = TransferPendingOper(db=session)
updated = oper.stage_heartbeat(
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
)
if not updated:
transaction.commit()
return None
session.flush()
session.expire_all()
pending = oper.get_by_task_id(task_id=task_id)
if pending is None:
raise TransferPlanningStateError(
f"heartbeat 后未找到整理任务: {task_id}"
)
admission = self._project(pending)
transaction.commit()
return admission
except Exception:
transaction.rollback()
raise
def release_claim(
self,
*,
task_id: str,
lease_token: str,
error: Optional[str] = None,
) -> bool:
"""仅以当前未过期 token 释放租约,陈旧 worker 不得改变任务。"""
if not task_id or not lease_token:
return False
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
released = TransferPendingOper(db=session).stage_release_claim(
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
transaction.commit()
return bool(released)
except Exception:
transaction.rollback()
raise
def record_enqueue_failure(self, *, task_id: str, error: str) -> None:
"""独立提交最近一次入队失败,保留准入记录供后续恢复。"""
@@ -192,6 +430,7 @@ class TransactionalTransferAdmissionRepository:
self,
*,
task_id: str,
lease_token: str,
input_fingerprint: str,
checkpoint: TransferPlanCheckpoint,
) -> TransferAdmission:
@@ -223,7 +462,9 @@ class TransactionalTransferAdmissionRepository:
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
now_time=self._now(),
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
session.flush()
session.expire_all()
@@ -232,6 +473,13 @@ class TransactionalTransferAdmissionRepository:
raise TransferPlanningStateError(f"未找到整理任务: {task_id}")
if pending.input_fingerprint != input_fingerprint:
raise TransferAdmissionConflictError("整理任务输入指纹已经改变")
now_time = self._format_lease_time(self._lease_now())
if (
pending.lease_token != lease_token
or not pending.lease_expires_at
or pending.lease_expires_at <= now_time
):
raise TransferLeaseLostError("整理任务租约已过期或已被其他 worker 接管")
if not updated and not (
pending.state == target_state
and pending.checkpoint_payload == checkpoint_payload
@@ -246,28 +494,44 @@ class TransactionalTransferAdmissionRepository:
transaction.rollback()
raise
def record_planning_failure(self, *, task_id: str, error: str) -> None:
def record_planning_failure(
self,
*,
task_id: str,
lease_token: str,
error: str,
) -> None:
"""独立提交规划错误并保持任务处于接纳态供恢复重试。"""
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
TransferPendingOper(db=session).stage_record_planning_failure(
updated = TransferPendingOper(db=session).stage_record_planning_failure(
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=self._now(),
now_time=self._format_lease_time(self._lease_now()),
updated_at=self._now(),
)
if not updated:
raise TransferLeaseLostError(
"整理任务租约已过期或已被其他 worker 接管"
)
transaction.commit()
except Exception:
transaction.rollback()
raise
def discard_task(self, *, task_id: str) -> int:
"""在独立事务中按稳定任务标识删除已到终态的准入记录"""
def discard_claimed(self, *, task_id: str, lease_token: str) -> int:
"""仅以当前未过期 token 删除终态任务,拒绝陈旧 worker 变更"""
if not task_id or not lease_token:
return 0
with self._session_factory() as session:
transaction = SqlAlchemyUnitOfWork(session)
try:
deleted = TransferPendingOper(db=session).stage_discard_task(
deleted = TransferPendingOper(db=session).stage_discard_claimed(
task_id=task_id,
lease_token=lease_token,
now_time=self._format_lease_time(self._lease_now()),
)
transaction.commit()
return deleted
+338 -148
View File
@@ -4,7 +4,20 @@ from datetime import datetime
from typing import Any, List, Optional, cast
from uuid import uuid4
from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update
from sqlalchemy import (
JSON,
Index,
Integer,
String,
Text,
UniqueConstraint,
and_,
delete,
func,
or_,
select,
update,
)
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import Base, execute_dml, get_id_column
@@ -72,7 +85,7 @@ class TransferPending(Base):
准入时保存版本化规划输入和指纹;纯规划完成后以同一行原子保存完整有序计划并
推进到 planned。重启恢复可直接消费已规划路径,避免再次触发 rename 等插件事件。
旧路径登记接口仍生成最小 legacy_replan 输入,供插件兼容调用方继续使用
所有执行期 mutation 都以稳定任务身份和租约 token 进行 CAS fencing
"""
id = get_id_column()
@@ -111,6 +124,16 @@ class TransferPending(Base):
checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON)
# 规划完成时间
planned_at: Mapped[Optional[str]] = mapped_column(String(40))
# 当前租约拥有者
lease_owner: Mapped[Optional[str]] = mapped_column(String(128))
# 当前租约的唯一防陈旧令牌
lease_token: Mapped[Optional[str]] = mapped_column(String(64))
# 当前租约的 UTC 到期时间
lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40))
# 最近一次成功 claim 或 heartbeat 的 UTC 时间
heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40))
# 真正取得新 token 的累计次数
attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
__table_args__ = (
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
@@ -122,41 +145,17 @@ class TransferPending(Base):
"created_at",
"id",
),
# 恢复调度按业务状态和租约到期时间筛选可接管任务
Index(
"ix_transferpending_recovery_lease",
"state",
"lease_expires_at",
"created_at",
"id",
),
UniqueConstraint("task_id", name="uq_transferpending_task_id"),
)
@classmethod
def register(cls, db: Session, storage: str, src_path: str,
now_time: str) -> Optional["TransferPending"]:
"""
登记一个待整理文件,已存在时保持原登记时间不变。
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:param now_time: 当前时间
:return: 登记记录
"""
if not storage or not src_path:
return None
pending = db.execute(
select(cls).where(cls.storage == storage, cls.src_path == src_path)
).scalars().first()
if pending:
return cast("TransferPending", pending)
planning_input = _legacy_planning_payload(storage, src_path)
pending = cls(
storage=storage,
src_path=src_path,
state="accepted",
created_at=now_time,
updated_at=now_time,
input_version=1,
planning_input=planning_input,
input_fingerprint=_planning_fingerprint(planning_input),
)
db.add(pending)
return pending
@classmethod
def stage_admit(cls, db: Session, *, task_id: str, storage: str,
src_path: str, state: str,
@@ -201,44 +200,6 @@ class TransferPending(Base):
db.add(pending)
return pending
@classmethod
def list_by_state(cls, db: Session, *, state: str,
limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
按登记顺序列出指定持久状态的接纳记录。
:param db: 数据库会话
:param state: 持久状态
:param limit: 单次读取上限
:return: 接纳记录列表
"""
if not state:
return []
return list(db.execute(
select(cls)
.where(cls.state == state)
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def list_by_states(cls, db: Session, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
按登记顺序列出多个可恢复持久状态的记录。
:param db: 数据库会话
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: 接纳记录列表
"""
if not states:
return []
return list(db.execute(
select(cls)
.where(cls.state.in_(states))
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def get_by_identity(cls, db: Session, *, storage: str,
src_path: str) -> Optional["TransferPending"]:
@@ -276,12 +237,283 @@ class TransferPending(Base):
db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(),
)
@classmethod
def list_claimable_candidates(
cls,
db: Session,
*,
states: tuple[str, ...],
now_time: str,
limit: int,
after_cursor: Optional[tuple[str, int]] = None,
) -> List[tuple[str, str, int]]:
"""
按稳定游标列出未租用或租约已过期的候选任务。
返回候选不等于取得租约;调用方必须继续执行带相同过期条件的 claim CAS,
并以受影响行数决定竞争结果。
:param db: 数据库会话
:param states: 可恢复业务状态
:param now_time: 当前 UTC 时间
:param limit: 候选数量上限
:param after_cursor: 上一页最后一条的规范登记时间与主键
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
"""
if not states or not now_time or limit <= 0:
return []
cursor_created_at = func.coalesce(cls.created_at, "")
statement = select(cls.task_id, cursor_created_at, cls.id).where(
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
)
if after_cursor is not None:
after_created_at, after_id = after_cursor
statement = statement.where(or_(
cursor_created_at > after_created_at,
and_(
cursor_created_at == after_created_at,
cls.id > after_id,
),
))
rows = db.execute(
statement
.order_by(cursor_created_at.asc(), cls.id.asc())
.limit(limit)
).all()
return [
(task_id, created_at or "", int(row_id))
for task_id, created_at, row_id in rows
]
@classmethod
def claim_task(
cls,
db: Session,
*,
task_id: str,
states: tuple[str, ...],
owner_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
updated_at: str,
) -> int:
"""
以未租用或租约已过期为条件原子取得任务租约。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param states: 允许 claim 的业务状态
:param owner_id: 新租约拥有者
:param lease_token: 新租约唯一令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数,1 表示赢得竞争
"""
if not all((
task_id,
states,
owner_id,
lease_token,
now_time,
lease_expires_at,
updated_at,
)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
)
.values(
lease_owner=owner_id,
lease_token=lease_token,
lease_expires_at=lease_expires_at,
heartbeat_at=now_time,
attempt_count=cls.attempt_count + 1,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def record_projection_failure(
cls,
db: Session,
*,
task_id: str,
states: tuple[str, ...],
error: str,
now_time: str,
updated_at: str,
) -> int:
"""
在没有有效租约且诊断发生变化时原子记录恢复投影损坏。
claim 的投影失败会先回滚,因此这里不得重新占用租约。CAS 同时保护
已被其他 worker 领取的任务,并避免周期恢复反复刷新相同错误。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param error: 可持久化的稳定诊断文本
:param now_time: 当前 UTC 租约时间
:param updated_at: 宿主本地业务审计时间
:return: 更新的记录数,1 表示首次或变化后的诊断被记录
"""
if not all((task_id, states, error, now_time, updated_at)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.state.in_(states),
or_(
cls.lease_token.is_(None),
cls.lease_expires_at.is_(None),
cls.lease_expires_at <= now_time,
),
cls.last_error.is_distinct_from(error),
)
.values(
last_error=error,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def heartbeat(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
) -> int:
"""
仅以当前且未过期的 token 原子延长任务租约。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:return: 更新的记录数
"""
if not all((task_id, lease_token, now_time, lease_expires_at)):
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
lease_expires_at=lease_expires_at,
heartbeat_at=now_time,
),
execution_options={"synchronize_session": False},
)
@classmethod
def release_claim(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
error: Optional[str],
now_time: str,
updated_at: str,
) -> int:
"""
仅以当前且未过期的 token 释放租约并保存本次执行错误。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param error: 本次执行错误,成功释放时为空
:param now_time: 当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if not task_id or not lease_token or not now_time or not updated_at:
return 0
return execute_dml(
db,
update(cls)
.where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
lease_owner=None,
lease_token=None,
lease_expires_at=None,
heartbeat_at=None,
last_error=error,
updated_at=updated_at,
),
execution_options={"synchronize_session": False},
)
@classmethod
def discard_claimed(
cls,
db: Session,
*,
task_id: str,
lease_token: str,
now_time: str,
) -> int:
"""
仅以当前且未过期的 token 删除已经到达终态的租约任务。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:return: 删除的记录数
"""
if not task_id or not lease_token or not now_time:
return 0
return execute_dml(
db,
delete(cls).where(
cls.task_id == task_id,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
),
execution_options={"synchronize_session": False},
)
@classmethod
def checkpoint_plan(cls, db: Session, *, task_id: str,
input_fingerprint: str, checkpoint_version: int,
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...], target_state: str,
now_time: str) -> int:
lease_token: str, now_time: str,
updated_at: str) -> int:
"""
以输入指纹为 CAS 条件原子保存计划并推进到已规划。
:param db: 数据库会话
@@ -291,7 +523,9 @@ class TransferPending(Base):
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许推进检查点的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:param lease_token: 当前且未过期的租约令牌
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if (
@@ -300,8 +534,19 @@ class TransferPending(Base):
or not checkpoint_payload
or not source_states
or not target_state
or not lease_token
or not updated_at
):
return 0
values: dict[str, Any] = {
"state": target_state,
"checkpoint_version": checkpoint_version,
"checkpoint_payload": checkpoint_payload,
"last_error": None,
"updated_at": updated_at,
}
if target_state == "planned":
values["planned_at"] = updated_at
return execute_dml(
db,
update(cls)
@@ -309,30 +554,29 @@ class TransferPending(Base):
cls.task_id == task_id,
cls.state.in_(source_states),
cls.input_fingerprint == input_fingerprint,
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(
state=target_state,
checkpoint_version=checkpoint_version,
checkpoint_payload=checkpoint_payload,
planned_at=now_time,
last_error=None,
updated_at=now_time,
),
.values(**values),
execution_options={"synchronize_session": False},
)
@classmethod
def record_planning_failure(cls, db: Session, *, task_id: str,
error: str, now_time: str) -> int:
lease_token: str, error: str,
now_time: str, updated_at: str) -> int:
"""
为接纳态或 provider 待执行任务记录规划失败,不改变其恢复状态。
:param db: 数据库会话
:param task_id: 稳定任务标识
:param lease_token: 当前且未过期的租约令牌
:param error: 失败原因
:param now_time: 当前时间
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
if not task_id:
if not task_id or not lease_token or not now_time or not updated_at:
return 0
return execute_dml(
db,
@@ -340,8 +584,11 @@ class TransferPending(Base):
.where(
cls.task_id == task_id,
cls.state.in_(("accepted", "provider_pending")),
cls.lease_token == lease_token,
cls.lease_expires_at.is_not(None),
cls.lease_expires_at > now_time,
)
.values(last_error=error, updated_at=now_time),
.values(last_error=error, updated_at=updated_at),
execution_options={"synchronize_session": False},
)
@@ -361,67 +608,10 @@ class TransferPending(Base):
return execute_dml(
db,
update(cls)
.where(cls.task_id == task_id)
.where(
cls.task_id == task_id,
cls.lease_token.is_(None),
)
.values(last_error=error, updated_at=now_time),
execution_options={"synchronize_session": False},
)
@classmethod
def discard_task(cls, db: Session, *, task_id: str) -> int:
"""
在调用方会话中按任务标识删除接纳记录。
:param db: 数据库会话
:param task_id: 任务标识
:return: 删除的记录数
"""
if not task_id:
return 0
return execute_dml(
db, delete(cls).where(cls.task_id == task_id),
execution_options={"synchronize_session": False},
)
@classmethod
def discard(cls, db: Session, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
if not storage or not src_path:
return 0
return execute_dml(
db, delete(cls).where(cls.storage == storage, cls.src_path == src_path),
execution_options={"synchronize_session": False},
)
@classmethod
def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
列出全部待整理登记,供启动回放使用。
按登记时间升序回放,保持与原入队顺序一致;上限避免异常积压时
一次性把整理链压垮。
:param db: 数据库会话
:param limit: 单次回放上限
:return: 待整理登记列表
"""
return list(db.execute(
select(cls)
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
).scalars().all())
@classmethod
def clear(cls, db: Session) -> int:
"""
清空全部待整理登记。
:param db: 数据库会话
:return: 删除的记录数
"""
return execute_dml(
db, delete(cls),
execution_options={"synchronize_session": False},
execution_options={"synchronize_session": "fetch"},
)
+191 -110
View File
@@ -1,5 +1,4 @@
from datetime import datetime
from typing import Any, List, Optional, Tuple
from typing import Any, Optional
from app.db.base import DbOper
from app.db.models.transferpending import TransferPending
@@ -10,27 +9,9 @@ class TransferPendingOper(DbOper):
待整理文件登记管理。
保存稳定任务身份、存储、源文件路径和准入状态,用于在进程重启后把没走完
整理链的文件重新送回去,避免挂载故障重启后永久漏件。旧版路径登记接口继续
保留,供插件和兼容调用方使用。
整理链的文件重新送回去,避免挂载故障重启后永久漏件。
"""
def register(self, storage: str, src_path: str) -> Optional[TransferPending]:
"""
登记一个待整理文件。
:param storage: 存储
:param src_path: 源文件路径
:return: 登记记录
"""
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return self._execute_sync_write(
lambda session: TransferPending.register(
session,
storage=storage,
src_path=src_path,
now_time=now_time,
)
)
def stage_admit(self, *, task_id: str, storage: str, src_path: str,
state: str, now_time: str, input_version: int = 1,
planning_input: Optional[dict[str, Any]] = None,
@@ -63,38 +44,6 @@ class TransferPendingOper(DbOper):
)
)
def list_by_state(self, *, state: str,
limit: Optional[int] = 5000) -> List[TransferPending]:
"""
使用当前会话列出指定状态记录。
:param state: 持久状态
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_by_state(
session,
state=state,
limit=limit,
)
) or []
def list_by_states(self, *, states: tuple[str, ...],
limit: Optional[int] = 5000) -> List[TransferPending]:
"""
使用当前会话列出多个可恢复状态的记录。
:param states: 允许恢复的状态集合
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_by_states(
session,
states=states,
limit=limit,
)
) or []
def get_by_identity(self, *, storage: str,
src_path: str) -> Optional[TransferPending]:
"""
@@ -133,7 +82,9 @@ class TransferPendingOper(DbOper):
checkpoint_payload: dict[str, Any],
source_states: tuple[str, ...],
target_state: str,
lease_token: str,
now_time: str,
updated_at: str,
) -> int:
"""
在当前会话中以输入指纹为条件暂存完整计划检查点。
@@ -143,7 +94,9 @@ class TransferPendingOper(DbOper):
:param checkpoint_payload: 完整有序计划 JSON
:param source_states: 允许执行 CAS 的起始状态
:param target_state: 检查点提交后的目标状态
:param now_time: 当前时间
:param lease_token: 当前且未过期的租约令牌
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
@@ -155,25 +108,206 @@ class TransferPendingOper(DbOper):
checkpoint_payload=checkpoint_payload,
source_states=source_states,
target_state=target_state,
lease_token=lease_token,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_record_planning_failure(self, *, task_id: str, error: str,
now_time: str) -> int:
def stage_record_planning_failure(self, *, task_id: str, lease_token: str,
error: str, now_time: str,
updated_at: str) -> int:
"""
在当前会话中记录规划失败并保持任务处于接纳态。
:param task_id: 稳定任务标识
:param lease_token: 当前且未过期的租约令牌
:param error: 失败原因
:param now_time: 当前时间
:param now_time: 用于租约 fencing 的当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.record_planning_failure(
session,
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def list_claimable_candidates(
self,
*,
states: tuple[str, ...],
now_time: str,
limit: int,
after_cursor: Optional[tuple[str, int]] = None,
) -> list[tuple[str, str, int]]:
"""
使用当前会话按稳定游标读取未租用或已过期的恢复候选。
:param states: 可恢复业务状态
:param now_time: 当前 UTC 时间
:param limit: 候选数量上限
:param after_cursor: 上一页最后一条的规范登记时间与主键
:return: 任务标识、规范登记时间与主键组成的稳定游标列表
"""
return self._execute_sync_query(
lambda session: TransferPending.list_claimable_candidates(
session,
states=states,
now_time=now_time,
limit=limit,
after_cursor=after_cursor,
)
) or []
def stage_claim_task(
self,
*,
task_id: str,
states: tuple[str, ...],
owner_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
updated_at: str,
) -> int:
"""
使用当前会话以未租或过期条件竞争一个新租约。
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param owner_id: 新租约拥有者
:param lease_token: 新租约唯一令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.claim_task(
session,
task_id=task_id,
states=states,
owner_id=owner_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
updated_at=updated_at,
)
)
def stage_record_projection_failure(
self,
*,
task_id: str,
states: tuple[str, ...],
error: str,
now_time: str,
updated_at: str,
) -> int:
"""
使用当前会话按无有效租约和诊断变化条件记录投影损坏。
:param task_id: 稳定任务标识
:param states: 可恢复业务状态
:param error: 可持久化的稳定诊断文本
:param now_time: 当前 UTC 租约时间
:param updated_at: 宿主本地业务审计时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.record_projection_failure(
session,
task_id=task_id,
states=states,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_heartbeat(
self,
*,
task_id: str,
lease_token: str,
now_time: str,
lease_expires_at: str,
) -> int:
"""
使用当前会话以当前未过期 token 延长租约。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:param lease_expires_at: 新租约到期时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.heartbeat(
session,
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
lease_expires_at=lease_expires_at,
)
)
def stage_release_claim(
self,
*,
task_id: str,
lease_token: str,
error: Optional[str],
now_time: str,
updated_at: str,
) -> int:
"""
使用当前会话按未过期 token 释放租约并记录本次错误。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param error: 本次执行错误,成功释放时为空
:param now_time: 当前 UTC 时间
:param updated_at: 与既有业务审计字段一致的宿主本地时间
:return: 更新的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.release_claim(
session,
task_id=task_id,
lease_token=lease_token,
error=error,
now_time=now_time,
updated_at=updated_at,
)
)
def stage_discard_claimed(
self,
*,
task_id: str,
lease_token: str,
now_time: str,
) -> int:
"""
使用当前会话按当前未过期 token 删除终态任务。
:param task_id: 稳定任务标识
:param lease_token: 当前租约令牌
:param now_time: 当前 UTC 时间
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_claimed(
session,
task_id=task_id,
lease_token=lease_token,
now_time=now_time,
)
)
@@ -194,56 +328,3 @@ class TransferPendingOper(DbOper):
now_time=now_time,
)
)
def stage_discard_task(self, *, task_id: str) -> int:
"""
在当前会话中暂存按任务标识删除接纳记录。
:param task_id: 任务标识
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard_task(
session,
task_id=task_id,
)
)
def discard(self, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记。
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: TransferPending.discard(
session,
storage=storage,
src_path=src_path,
)
)
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
"""
列出全部待整理登记,供启动回放使用。
返回纯元组而不是 ORM 实例:回放发生在会话之外,ORM 实例脱离 session
后访问属性会触发 DetachedInstanceError。
:param limit: 单次回放上限
:return: (存储, 源文件路径) 列表
"""
items = self._execute_sync_query(
lambda session: TransferPending.list_all(session, limit=limit)
)
return [
(item.storage, item.src_path)
for item in items or []
if item and item.storage and item.src_path
]
def clear(self) -> int:
"""
清空全部待整理登记。
:return: 删除的记录数
"""
return self._execute_sync_write(TransferPending.clear)
+3 -3
View File
@@ -135,10 +135,10 @@ MODULE_ALIASES: Dict[str, ModuleAlias] = {
owner="sdk",
),
"app.db.transferpending_oper": ModuleAlias(
target="app.db.oper.transferpending",
replacement="app.db.oper.transferpending",
target="app.sdk._legacy.transferpending",
replacement="app.application.transfer",
introduced="v3.0.0",
owner="db",
owner="sdk",
),
"app.db.user_oper": ModuleAlias(
target="app.sdk._legacy.user",
+185
View File
@@ -0,0 +1,185 @@
"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。"""
from datetime import datetime
from typing import List, Optional, Tuple
from uuid import uuid4
from sqlalchemy import delete, select
from app.db.base import DbOper, execute_dml
from app.db.models.transferpending import TransferPending as _TransferPending
class TransferPendingOper(DbOper):
"""
保留旧待整理登记 ABI,并对执行租约实施兼容写 fencing。
本类只由精确旧导入映射加载。宿主整理链仍使用 Application Port 和显式
Session 的 canonical Oper;旧删除入口只能处理从未取得租约的记录。
"""
def register(self, storage: str, src_path: str) -> Optional[_TransferPending]:
"""
按旧签名登记待整理文件,重复登记保持原任务和租约不变。
:param storage: 存储
:param src_path: 源文件路径
:return: 登记记录
"""
now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
return self._execute_sync_write(
lambda session: _TransferPending.stage_admit(
session,
task_id=uuid4().hex,
storage=storage,
src_path=src_path,
state="accepted",
now_time=now_time,
)
)
def list_by_state(
self,
*,
state: str,
limit: Optional[int] = 5000,
) -> List[_TransferPending]:
"""
按旧签名和登记顺序列出指定状态记录。
:param state: 持久状态
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
if not state:
return []
return self._execute_sync_query(
lambda session: list(session.execute(
select(_TransferPending)
.where(_TransferPending.state == state)
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
.limit(limit)
).scalars().all())
)
def list_by_states(
self,
*,
states: tuple[str, ...],
limit: Optional[int] = 5000,
) -> List[_TransferPending]:
"""
按旧签名和登记顺序列出多个状态记录。
:param states: 持久状态集合
:param limit: 单次读取上限
:return: ORM 接纳记录列表
"""
if not states:
return []
return self._execute_sync_query(
lambda session: list(session.execute(
select(_TransferPending)
.where(_TransferPending.state.in_(states))
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
.limit(limit)
).scalars().all())
)
def get_by_identity(
self,
*,
storage: str,
src_path: str,
) -> Optional[_TransferPending]:
"""
按旧签名查询指定存储与源路径的登记。
:param storage: 存储
:param src_path: 源文件路径
:return: 登记记录
"""
return self._execute_sync_query(
lambda session: _TransferPending.get_by_identity(
session,
storage=storage,
src_path=src_path,
)
)
def get_by_task_id(self, *, task_id: str) -> Optional[_TransferPending]:
"""
按旧签名查询稳定任务标识对应的登记。
:param task_id: 稳定任务标识
:return: 登记记录
"""
return self._execute_sync_query(
lambda session: _TransferPending.get_by_task_id(
session,
task_id=task_id,
)
)
def discard(self, storage: str, src_path: str) -> int:
"""
按旧签名删除未 claim 的指定登记。
任何带 token 的记录都由当前租约拥有者通过 fenced canonical API 收口;
即使租约已经过期,旧插件也不得越权代替恢复调度器删除。
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
if not storage or not src_path:
return 0
return self._execute_sync_write(
lambda session: execute_dml(
session,
delete(_TransferPending).where(
_TransferPending.storage == storage,
_TransferPending.src_path == src_path,
_TransferPending.lease_token.is_(None),
),
execution_options={"synchronize_session": False},
)
)
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
"""
按旧返回形态列出全部待整理路径。
:param limit: 单次读取上限
:return: ``(存储, 源文件路径)`` 列表
"""
rows = self._execute_sync_query(
lambda session: session.execute(
select(_TransferPending.storage, _TransferPending.src_path)
.order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc())
.limit(limit)
).all()
)
return [
(storage, src_path)
for storage, src_path in rows
if storage and src_path
]
def clear(self) -> int:
"""
清空全部未 claim 登记,保留任何带租约 token 的任务。
:return: 删除的记录数
"""
return self._execute_sync_write(
lambda session: execute_dml(
session,
delete(_TransferPending).where(
_TransferPending.lease_token.is_(None),
),
execution_options={"synchronize_session": False},
)
)
__all__ = ["TransferPendingOper"]
+2 -2
View File
@@ -3,12 +3,12 @@ from app.chain.transfer import TransferChain
def replay_pending_transfers():
"""
回放上次进程退出时仍未整理完的文件
启动唯一整理恢复调度器
整理队列是纯内存的,挂载挂死后的人工重启、版本升级、OOM、宿主重启都会让
队列连同「这些文件还没整理」这个事实一起蒸发;而已稳定落地的文件不会再产生
任何监控事件,也不会有新的补偿扫描起点,结果就是永久漏件。
回放本身在后台线程执行,不阻塞启动流程。
启动回放、同进程补偿和过期租约接管共用同一个后台调度入口,不阻塞启动流程。
"""
TransferChain().replay_pending()
+124
View File
@@ -0,0 +1,124 @@
"""3.0.15 为整理恢复任务增加强 CAS 租约字段。
Revision ID: d3a9e5f7b2c4
Revises: c2f8a4d6e1b3
Create Date: 2026-08-27
"""
import sqlalchemy as sa
from alembic import op
revision = "d3a9e5f7b2c4"
down_revision = "c2f8a4d6e1b3"
branch_labels = None
depends_on = None
_TABLE_NAME = "transferpending"
_LEASE_INDEX = "ix_transferpending_recovery_lease"
_LEASE_COLUMNS = {
"lease_owner",
"lease_token",
"lease_expires_at",
"heartbeat_at",
"attempt_count",
}
def _column_names() -> set[str]:
"""返回当前待整理登记表的字段集合。"""
inspector = sa.inspect(op.get_bind())
if _TABLE_NAME not in inspector.get_table_names():
return set()
return {
column["name"]
for column in inspector.get_columns(_TABLE_NAME)
}
def _index_names() -> set[str]:
"""返回当前待整理登记表的索引名称集合。"""
inspector = sa.inspect(op.get_bind())
if _TABLE_NAME not in inspector.get_table_names():
return set()
return {
index["name"]
for index in inspector.get_indexes(_TABLE_NAME)
if index.get("name")
}
def upgrade() -> None:
"""增加租约身份、到期、心跳和接管次数,并支持中断后重跑。"""
columns = _column_names()
if not columns:
return
additions = (
(
"lease_owner",
sa.Column("lease_owner", sa.String(length=128), nullable=True),
),
(
"lease_token",
sa.Column("lease_token", sa.String(length=64), nullable=True),
),
(
"lease_expires_at",
sa.Column("lease_expires_at", sa.String(length=40), nullable=True),
),
(
"heartbeat_at",
sa.Column("heartbeat_at", sa.String(length=40), nullable=True),
),
(
"attempt_count",
sa.Column("attempt_count", sa.Integer(), nullable=True),
),
)
for column_name, column in additions:
if column_name not in columns:
op.add_column(_TABLE_NAME, column)
columns = _column_names()
if "attempt_count" in columns:
pending = sa.table(
_TABLE_NAME,
sa.column("attempt_count", sa.Integer()),
)
op.get_bind().execute(
pending.update()
.where(pending.c.attempt_count.is_(None))
.values(attempt_count=0)
)
with op.batch_alter_table(_TABLE_NAME) as batch_op:
batch_op.alter_column(
"attempt_count",
existing_type=sa.Integer(),
nullable=False,
)
if _LEASE_INDEX not in _index_names():
op.create_index(
_LEASE_INDEX,
_TABLE_NAME,
["state", "lease_expires_at", "created_at", "id"],
unique=False,
)
def downgrade() -> None:
"""移除租约索引和字段,保留原业务状态及规划检查点。"""
columns = _column_names()
if not columns or not (_LEASE_COLUMNS & columns):
return
if _LEASE_INDEX in _index_names():
op.drop_index(_LEASE_INDEX, table_name=_TABLE_NAME)
with op.batch_alter_table(_TABLE_NAME) as batch_op:
for column_name in (
"attempt_count",
"heartbeat_at",
"lease_expires_at",
"lease_token",
"lease_owner",
):
if column_name in columns:
batch_op.drop_column(column_name)
+11 -6
View File
@@ -69,7 +69,7 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
| 指标 | 当前值 | 解释 |
|---|---:|---|
| 宿主 Python 模块 / 内部依赖边 | 836 / 6,832 | `dependency-baseline.json` 当前快照 |
| 宿主 Python 模块 / 内部依赖边 | 837 / 6,836 | `dependency-baseline.json` 当前快照 |
| 非平凡 SCC | 2 | 新增 Chain 包根环;另一个是隔离的 29 模块 TMDB 移植包环 |
| 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 |
| Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 |
@@ -192,7 +192,8 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
- `S1-L1.2 Planning checkpoint``VERIFIED`。版本化请求与指纹先准入;无 legacy provider 时通过
`accepted -> planned` CAS 提交完整目标和有序操作,有 provider 时先提交 `provider_pending`,全部
返回空后再以第二次 CAS 提交 `planned`;planned 重放只消费冻结上下文和目标。
- `S1-L1.3 Lease 与恢复调度``PLANNED`。交付 claim/lease/heartbeat/attempt、过期接管与唯一恢复入口。
- `S1-L1.3 Lease 与恢复调度``VERIFIED`交付 token fencing 的
claim/lease/heartbeat/attempt、过期接管、固定退避的唯一恢复入口和有界关闭 owner。
- `S1-L1.4 幂等执行与终态结算``PLANNED`。交付文件/历史幂等、唯一 retry owner 和
`manual_review` 语义。
- `S1-L1.5 E3 全链收口``PLANNED`。完成崩溃矩阵、兼容验收与旧路径删除。此叶交付前,
@@ -210,9 +211,13 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
解析并严格执行,缺失或异常不 fallback。全部返回空后才生成宿主计划,并以第二次 CAS 提升为
`planned` 后执行。旧 caller 只经 `ChainBase.transfer` 注入式兼容门面进入同一 durable command
宿主 FileManager/TransHandler 的旧执行入口已删除。
- `TransferPending` 现在可区分 `accepted/provider_pending/planned`,但尚无
claim/lease/heartbeat/attempt、逐步骤执行结果和 `manual_review`,仍无法判定“文件已移动、历史未提交”
等后续中间态。
- `S1-L1.3` 已把执行所有权与 planning phase 正交:恢复任务入队前原子 claim,普通任务在任何业务
副作用前 claimheartbeat、checkpoint、失败留痕、release 和终态删除均受当前未过期 token
fencing。启动和同进程恢复共享唯一 scheduler,确定性失败按固定轮询退避,关闭时 worker、replay、
lease release 和 heartbeat 都由有界生命周期 owner 持有。损坏投影以无有效租约 CAS 留痕,同错不
重复刷写,且不会阻塞后续健康任务。
- `TransferPending` 仍缺少逐步骤执行结果和 `manual_review`,因此还不能判定“文件已移动、历史未提交”
等外部结果未知的后续中间态。
- 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、
lease/heartbeat 和人工恢复要求不一致。
@@ -222,7 +227,7 @@ MoviePilot V3 已经形成较清晰的模块化单体:`foundation`、`domain`
原子提交,入队失败时必须保留 pending 供重放。
- [x] 初始登记保存稳定源身份、版本化请求和状态;目标与有序操作在纯规划完成后以 planning
checkpoint 原子更新,任何文件副作用不得早于该提交。
- [ ] 增加 claim/lease/heartbeat/attempt 与过期接管,同一任务同时只能有一个 worker owner。
- [x] 增加 claim/lease/heartbeat/attempt 与过期接管,同一任务同时只能有一个 worker owner。
- [ ] 设计幂等文件操作和历史提交;只有所有必要步骤达到持久终态后才能删除记录。
- [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner,定义旧记录迁移和兼容规则。
- [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界,不直接套用 E2
+2 -2
View File
@@ -704,8 +704,8 @@ flowchart LR
| 指标 | 当前值 |
|---|---:|
| Python 模块 | 836 |
| 内部导入边 | 6,832 |
| Python 模块 | 837 |
| 内部导入边 | 6,836 |
| 非平凡 SCC | 2`ARCH-107` 临时 Chain 包根环;精确 containment 的 TMDB 移植包环) |
| Direct egress | 6612 条待迁移债务,54 条精确 containment |
| Module Contract V2 spec | 217(其中 215 个进入 `run_module` 观察面) |
+81 -1
View File
@@ -94,7 +94,7 @@ G-ARCH 只有在以下条件全部满足后才可完成:
|---|---|---|---|
| S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地;先持久 commit 再入队,入队失败保留可恢复记录;宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission |
| S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化;无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划,有 provider 时先提交 `provider_pending`,全部返回空后再以第二次 CAS 提交 `planned`;重放只执行冻结目标,所有文件副作用晚于对应 checkpoint commit |
| S1-L1.3 Lease 与恢复调度 | `PLANNED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地;启动回放和同进程恢复共用唯一调度入口,同一任务同时只有一个 worker owner |
| S1-L1.3 Lease 与恢复调度 | `VERIFIED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地;启动回放和同进程恢复共用唯一调度入口,同一任务同时只有一个 worker owner |
| S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放;唯一 retry owner 生效,未知外部结果进入 `manual_review`,仅完整终态删除 pending |
| S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整;旧 fail-open、重复状态与兼容层外旧入口删除,ARCH-102 债务归零 |
| S1-L2 Workflow typed query | `PLANNED` | S0 | Workflow Application Port 不返回 `Any`/ORMSession 内投影 DTO,正式调用方全部切换 |
@@ -274,3 +274,83 @@ git diff --check
- failure injection 覆盖 commit 前零文件副作用、commit 后崩溃重放、离线 resolved context 恢复、
配置漂移仍使用冻结 target storage、规划失败留痕、旧 provider 提交后短路、严格异常、空结果两阶段
fallback、缺失引用零 cleanup,以及 cleanup 顺序/幂等/瞬时失败。
### S1-L1.3 Lease 与恢复调度
**Status:** `VERIFIED`
**Outcome**
为 durable transfer 增加可过期、可接管且带 fencing token 的执行租约。`owner` 只用于标识执行者,
每次有效 claim 生成的新 token 才是后续 heartbeat、checkpoint、失败留痕和终态删除的授权;所有写入
必须同时匹配当前且未过期的 token,过期 worker 即使稍后恢复也不能修改新 owner 的记录。
`accepted``provider_pending``planned` 仍是业务 planning phaselease 与其正交:claim、heartbeat
和 takeover 不改变 phase,恢复 worker 继续按冻结 checkpoint 决定执行路径。启动回放与同进程恢复共用
唯一调度入口:恢复任务在入队前完成原子 claim 并绑定 token,新 admission 由 worker 在任何副作用前
claim。已经 claim 的恢复任务不在 worker 内二次 claim。
**State machine**
| 当前租约 | 操作 | 结果与 fencing 约束 |
|---|---|---|
| 无租约 | `claim` | 生成新 token、设置 owner/到期时间并递增 attemptphase 不变 |
| 当前租约未过期 | 同 token `heartbeat` | 只延长当前租约;owner、token、attempt 和 phase 不变 |
| 当前租约未过期 | 任意再次 `claim`,包括同 owner | 原子拒绝,不递增 attempt、不入队、不执行副作用 |
| 当前租约已过期 | `heartbeat` 或旧 token 写入 | 原子拒绝;旧租约不可复活 |
| 当前租约已过期 | 新 worker `claim` | 生成新 token 并递增 attempt;旧 token 永久失效,phase 不变 |
| 当前 token 有效 | checkpoint、失败留痕或终态删除 | 仅精确 token CAS 成功;状态提交后不得由旧 worker 覆盖 |
任意时刻一条 pending 记录最多只有一个数据库认可的有效 fencing owner。该保证不等同于外部文件、
插件或历史副作用 exactly-once;租约在不可中断调用期间过期时,旧调用的外部结果仍可能未知。
**Scheduling and shutdown**
- 启动回放和同进程恢复只调用一个 claim-and-schedule 入口;禁止另建 list-then-enqueue 回放路径或
第二个 retry owner。批量恢复必须先原子 claim,再把已绑定 token 的任务交给唯一 worker 队列。
- heartbeat 由 Transfer worker 生命周期拥有,不创建游离后台 owner;停止时先封口新 claim 和调度,
再通知并有限等待 workerheartbeat 在 worker 存活期间继续维持其租约。worker 收敛后才停止
heartbeat;超时时保留仍存活的 worker/heartbeat owner 并返回失败。
- checkpoint、失败留痕和 pending 删除在同次持久化写入中执行 token CAS;租约丢失后 worker 停止
继续提交状态,不以预查询替代 fencing,也不把 stale mutation 伪装成成功。
- 确定性失败只确保唯一 scheduler 存在,不即时唤醒;新建的失败恢复 owner 首次扫描先等待固定轮询
周期。损坏持久投影按单任务事务回滚,再以无有效租约 CAS 留下去重诊断,不能饿死后续健康记录。
- 精确旧 `app.db.transferpending_oper` 导入只解析到 `app/sdk/_legacy` 门面;canonical Model、Oper、
Application Port 不保留旧 list-then-act 或无 fencing mutation。兼容 `discard/clear` 也不得删除任何
带 token 的有效或过期 claim。
**Excluded**
- 本叶不承诺文件、legacy provider、历史写入或其他外部副作用 exactly-once,也不以延长 lease
掩盖不可判定结果。
- 文件步骤幂等键、逐步结果 checkpoint、崩溃后未知结果判定、唯一 retry owner 和
`manual_review` 终态由 `S1-L1.4` 完整交付。
- 不增加 `processing` 等与 planning phase 重复的业务状态;不修改插件公开 Transfer ABI,不扫描或
修改 `app/plugins/**`,不恢复宿主旧 pending 入口或重复导出。
**Acceptance matrix**
| 场景 | 必须证明 |
|---|---|
| 新 admission 与恢复记录竞争 | 只有 claim 成功者入队并执行;失败者零文件副作用 |
| 两进程同时 claim 同一记录 | 仅一个新 token 成功,attempt 只按真实新 claim 增长 |
| heartbeat 与 takeover 竞争 | 未过期 heartbeat 可续租;过期租约不可复活,接管 token 唯一有效 |
| 旧 worker 延迟提交 | checkpoint、失败留痕和删除均被 token CAS 拒绝,不覆盖新 owner |
| 三种 planning phase 恢复 | claim/续租/接管保持 phase,并消费各自冻结输入或 checkpoint |
| 启动与同进程恢复同时触发 | 只经过唯一 scheduler 入口,同一任务只有一个 worker owner |
| 正常与超时关闭 | 先封口后有限等待;存活 scheduler/heartbeat/worker 不丢失 owner、不报告成功 |
**Failure injection matrix**
| 注入点 | 预期持久结果 |
|---|---|
| claim commit 前崩溃 | 无新 token、attempt 不变,可由后续 worker claim |
| claim commit 后、入队前崩溃 | 租约到期后可接管,新 token fencing 旧 worker |
| worker 执行前 lease 丢失 | 不执行文件副作用,不提交失败或终态状态 |
| heartbeat commit 前后崩溃 | 仅已提交到期时间生效;过期后不可用旧 token 续租 |
| checkpoint/失败留痕提交时被接管 | stale CAS 失败,新 owner 的 phase、错误和 token 不被覆盖 |
| 终态删除提交时被接管 | stale delete 为零行,pending 保留给当前 owner |
| shutdown 时 scheduler、worker 或 lease release 阻塞 | 有限等待返回失败并保留 owner/heartbeat,禁止清句柄后重建重复 owner |
本叶验收还必须运行 lease persistence、replay/worker、startup lifecycle、migration、架构与兼容聚焦
测试,以及锁定全量测试和 scoped Pylint;插件 ABI 只经统一 Compat/SDK 验证。
+24 -2
View File
@@ -484,8 +484,30 @@ result permits host planning and a second CAS to `planned`. Host-only `plan_tran
is the sole legacy caller facade and delegates the startup-injected durable
command; `FileManagerModule.transfer` and `TransHandler.transfer_media` must not
be recreated.
Canonical host chains never obtain `TransferPendingOper`; its no-Session API
remains only for the exact legacy plugin import contract.
Transfer execution ownership is orthogonal to those planning phases.
`app/application/transfer.py` defines the claim, heartbeat, release and fenced
mutation Port; `app/db/adapters/transfer.py` implements each operation in a
short UoW with a unique lease token. Any active lease rejects another claim,
including one from the same process owner. Expired leases may be taken over with
a new token and incremented attempt count, while the stale token cannot renew,
checkpoint, record failure, release or delete the task. Startup replay and
same-process recovery use the single scheduler owned by `TransferChain`; the
scheduler claims before enqueueing, and queued or executing claims are renewed
by one lifecycle-managed heartbeat owner. Lease ownership guarantees one
database-authorized worker, not physical exactly-once behavior for an already
issued file or legacy-plugin side effect; step idempotency and unknown outcomes
remain explicit execution concerns.
Canonical host chains never obtain `TransferPendingOper`. The canonical Model,
Oper and Application Port do not retain the historical `register`, `list_all`,
`discard`, `clear` or `list_by_*` surface. The exact
`app.db.transferpending_oper` mapping resolves instead to the private
`app/sdk/_legacy/transferpending.py` facade, which preserves the old no-Session
query ABI without becoming a host implementation. Its `register` delegates to
canonical durable admission without overwriting an existing row, while
`discard` and `clear` delete only rows whose `lease_token` is null; an active or
expired claimed task remains exclusively owned by fenced recovery APIs.
## Composition and Compatibility Boundaries
+8 -3
View File
@@ -1441,8 +1441,8 @@
"runtime_only": true
}
},
"edge_count": 6832,
"edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39",
"edge_count": 6836,
"edge_sha256": "3709e09a49257075f48a44aba353be1f25db610471ec756e09cbbfd7070acb9a",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -7625,6 +7625,10 @@
"app.sdk._legacy.subscribe -> app.domain.context",
"app.sdk._legacy.transfer -> app.application",
"app.sdk._legacy.transfer -> app.application.transfer",
"app.sdk._legacy.transferpending -> app.db",
"app.sdk._legacy.transferpending -> app.db.base",
"app.sdk._legacy.transferpending -> app.db.models",
"app.sdk._legacy.transferpending -> app.db.models.transferpending",
"app.sdk._legacy.user -> app.api",
"app.sdk._legacy.user -> app.api.deps",
"app.sdk._legacy.user -> app.db",
@@ -8277,7 +8281,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 836,
"module_count": 837,
"modules": [
"app",
"app.adapters",
@@ -9058,6 +9062,7 @@
"app.sdk._legacy.history",
"app.sdk._legacy.subscribe",
"app.sdk._legacy.transfer",
"app.sdk._legacy.transferpending",
"app.sdk._legacy.user",
"app.sdk.browser",
"app.sdk.cache",
+1 -1
View File
@@ -1594,7 +1594,7 @@
"misc": 2,
"no-any-return": 3,
"no-untyped-call": 8,
"no-untyped-def": 14,
"no-untyped-def": 12,
"operator": 3,
"return-value": 2,
"truthy-function": 5,
+3 -3
View File
@@ -249,9 +249,9 @@
"app.db.transferpending_oper": {
"introduced": "v3.0.0",
"is_package": false,
"owner": "db",
"replacement": "app.db.oper.transferpending",
"target": "app.db.oper.transferpending"
"owner": "sdk",
"replacement": "app.application.transfer",
"target": "app.sdk._legacy.transferpending"
},
"app.db.user_oper": {
"introduced": "v3.0.0",
+6 -2
View File
@@ -828,15 +828,19 @@ def test_models_and_base_require_explicit_database_sessions():
def test_plugin_sdk_does_not_import_or_export_host_models():
"""插件 SDK 只能暴露 Oper,不得把宿主 ORM Model 作为插件接口"""
"""插件 SDK 不得暴露 ORM Model,只有精确旧 ABI 门面可在内部访问"""
internal_compat_imports = {
("app/sdk/_legacy/transferpending.py", "app.db.models.transferpending"),
}
violations: list[str] = []
for path in (APP_ROOT / "sdk").rglob("*.py"):
relative = path.relative_to(PROJECT_ROOT).as_posix()
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
for node in ast.walk(tree):
if isinstance(node, ast.ImportFrom) and node.module and (
node.module == "app.db.models"
or node.module.startswith("app.db.models.")
):
) and (relative, node.module) not in internal_compat_imports:
violations.append(
f"{path.relative_to(PROJECT_ROOT)}:{node.lineno}:{node.module}"
)
+14 -4
View File
@@ -158,8 +158,18 @@ def test_tables_without_identity_columns_are_untouched(db):
不带身份列的表不受影响事件挂在 Mapper 上覆盖全部映射必须靠列名检查收窄
否则会去动一张根本没有这两列的表
"""
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-14 10:00:00")
TransferPending.stage_admit(
db.session,
task_id="identity-free-table",
storage="local",
src_path="/mnt/a.mkv",
state="accepted",
now_time="2026-08-14 10:00:00",
)
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/mnt/a.mkv"]
assert len(rows) == 1
row = TransferPending.get_by_identity(
db.session,
storage="local",
src_path="/mnt/a.mkv",
)
assert row.task_id == "identity-free-table"
+57 -198
View File
@@ -5,8 +5,10 @@
任何一件出偏差都直接表现为文件被漏整理或被重复整理而不是一个可见的报错
因此这里对着真实数据库断言查回的内容而不是断言调用了什么
"""
import inspect
import pytest
from sqlalchemy import create_engine
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.db import base as db_base
@@ -21,185 +23,6 @@ def _track(db):
db.watermark(TransferPending)
def test_register_is_idempotent_and_keeps_first_time(db):
"""
同一文件重复登记只保留一条且登记时间保持首次的值
监控在挂载抖动时会对同一个文件反复触发事件若每次都新增一条回放时同一个
文件会被送进整理链多次保留首次时间则保证回放顺序仍是最早发现的顺序
"""
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-13 10:00:00")
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-13 12:00:00")
rows = TransferPending.list_all(db.session)
same_path = [r for r in rows if r.src_path == "/mnt/a.mkv"]
assert len(same_path) == 1
assert same_path[0].created_at == "2026-08-13 10:00:00"
def test_register_scopes_by_storage(db):
"""
存储不同即为不同文件路径相同但分属不同存储时不能互相去重
"""
TransferPending.register(db.session, storage="local", src_path="/data/x.mkv",
now_time="2026-08-13 10:00:00")
TransferPending.register(db.session, storage="alist", src_path="/data/x.mkv",
now_time="2026-08-13 10:00:01")
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/data/x.mkv"]
assert {r.storage for r in rows} == {"local", "alist"}
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", ""), ("", "")])
def test_register_rejects_incomplete_identity(db, storage, src_path):
"""
缺少存储或路径的登记必须直接丢弃不能写入半条记录
半条记录回放时既定位不到文件也无法被 discard 匹配会永久留在表里
"""
assert TransferPending.register(db.session, storage=storage, src_path=src_path,
now_time="2026-08-13 10:00:00") is None
def test_list_all_replays_in_registration_order(db):
"""
回放顺序必须是登记时间升序同时间按主键升序
乱序回放会让后发现的文件先进整理链与原入队顺序不一致
"""
for path, moment in [("/mnt/c.mkv", "2026-08-13 12:00:00"),
("/mnt/a.mkv", "2026-08-13 10:00:00"),
("/mnt/b.mkv", "2026-08-13 11:00:00")]:
TransferPending.register(db.session, storage="local", src_path=path, now_time=moment)
ordered = [r.src_path for r in TransferPending.list_all(db.session)
if r.src_path.startswith("/mnt/")]
assert ordered == ["/mnt/a.mkv", "/mnt/b.mkv", "/mnt/c.mkv"]
def test_list_all_honours_limit(db):
"""
回放上限必须生效异常积压时一次性全放会把整理链直接压垮
"""
for index in range(5):
TransferPending.register(db.session, storage="local", src_path=f"/mnt/{index}.mkv",
now_time=f"2026-08-13 10:00:0{index}")
assert len(TransferPending.list_all(db.session, limit=3)) == 3
def test_discard_removes_only_the_matching_row(db):
"""
注销只应删除匹配的那一条并返回删除条数
整理到达终态时按存储 + 路径注销误删其他登记等于把别的文件也判成已完成
"""
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-13 10:00:00")
TransferPending.register(db.session, storage="local", src_path="/mnt/b.mkv",
now_time="2026-08-13 10:00:01")
assert TransferPending.discard(db.session, storage="local", src_path="/mnt/a.mkv") == 1
remaining = [r.src_path for r in TransferPending.list_all(db.session)
if r.src_path.startswith("/mnt/")]
assert remaining == ["/mnt/b.mkv"]
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", "")])
def test_discard_rejects_incomplete_identity(db, storage, src_path):
"""
身份不全时必须直接返回 0不能退化成条件为空的全表删除
"""
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-13 10:00:00")
assert TransferPending.discard(db.session, storage=storage, src_path=src_path) == 0
assert TransferPending.list_all(db.session)
def test_discard_returns_zero_when_absent(db):
"""
注销不存在的登记返回 0不抛异常整理链的终态回调不应因此中断
"""
assert TransferPending.discard(db.session, storage="local", src_path="/nope.mkv") == 0
def test_clear_empties_the_table(db):
"""
清空返回删除条数且表内不再有登记
"""
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
now_time="2026-08-13 10:00:00")
assert TransferPending.clear(db.session) >= 1
assert TransferPending.list_all(db.session) == []
def test_oper_returns_plain_tuples_not_orm_instances(db):
"""
回放接口必须返回纯元组
回放发生在会话之外ORM 实例脱离 session 后访问属性会抛
DetachedInstanceError那时启动流程已经在跑报错等于整批漏件
"""
oper = TransferPendingOper(db=db.session)
oper.register(storage="local", src_path="/mnt/a.mkv")
listed = oper.list_all()
assert ("local", "/mnt/a.mkv") in listed
assert all(isinstance(item, tuple) for item in listed)
def test_oper_reuses_explicit_query_session(db, monkeypatch):
"""TransferPendingOper 查询必须复用调用方会话。"""
db.add(TransferPending(
storage="local",
src_path="/mnt/explicit.mkv",
created_at="2026-08-13 10:00:00",
))
monkeypatch.setattr(
db_base,
"run_sync_transaction",
lambda _operation: (_ for _ in ()).throw(
AssertionError("不应创建额外同步事务")
),
)
assert ("local", "/mnt/explicit.mkv") in TransferPendingOper(db.session).list_all()
def test_oper_drops_rows_with_missing_fields(db):
"""
回放时必须跳过字段残缺的历史遗留行不能把空存储送进整理链
列上有 NOT NULL 约束残缺只可能表现为空串直接绕过 register 写入
模拟历史数据或外部写库留下的半条记录
"""
db.add(TransferPending(storage="", src_path="/mnt/broken.mkv",
created_at="2026-08-13 10:00:00"))
oper = TransferPendingOper(db=db.session)
oper.register(storage="local", src_path="/mnt/ok.mkv")
assert oper.list_all() == [("local", "/mnt/ok.mkv")]
def test_oper_discard_and_clear_report_counts(db):
"""
注销与清空都要如实返回条数调用方据此判断是否真的清理掉了
"""
oper = TransferPendingOper(db=db.session)
oper.register(storage="local", src_path="/mnt/a.mkv")
oper.register(storage="local", src_path="/mnt/b.mkv")
assert oper.discard(storage="local", src_path="/mnt/a.mkv") == 1
assert oper.clear() >= 1
assert oper.list_all() == []
def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
"""显式准入重复执行时必须复用首个稳定任务标识。"""
first = TransferPending.stage_admit(
@@ -224,8 +47,32 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
assert second.updated_at == "2026-08-27 10:00:00"
def test_state_queries_failure_record_and_task_discard(db):
"""状态查询、失败留痕和按任务删除应共享同一稳定身份"""
def test_unfenced_legacy_mutation_apis_are_absent() -> None:
"""持久整理表不得重新暴露绕过稳定任务身份和租约的旧写入口"""
for owner in (TransferPending, TransferPendingOper):
for method_name in (
"register",
"discard",
"list_all",
"list_by_state",
"list_by_states",
"clear",
):
assert not hasattr(owner, method_name)
enqueue_failure_source = inspect.getsource(
TransferPending.record_enqueue_failure
)
assert "cls.lease_token.is_(None)" in enqueue_failure_source
claimable_source = inspect.getsource(
TransferPending.list_claimable_candidates
)
assert "after_cursor" in claimable_source
assert ".not_in(" not in claimable_source
def test_state_queries_and_failure_record_share_stable_identity(db):
"""状态查询与失败留痕应共享同一稳定身份。"""
TransferPending.stage_admit(
db.session,
task_id="task-accepted",
@@ -244,11 +91,11 @@ def test_state_queries_failure_record_and_task_discard(db):
))
db.session.flush()
accepted = TransferPending.list_by_state(
accepted = TransferPending.get_by_task_id(
db.session,
state="accepted",
task_id="task-accepted",
)
assert [item.task_id for item in accepted] == ["task-accepted"]
assert accepted.state == "accepted"
assert TransferPending.record_enqueue_failure(
db.session,
task_id="task-accepted",
@@ -263,10 +110,6 @@ def test_state_queries_failure_record_and_task_discard(db):
)
assert failed.last_error == "queue full"
assert failed.updated_at == "2026-08-27 10:01:00"
assert TransferPending.discard_task(
db.session,
task_id="task-accepted",
) == 1
def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
@@ -288,15 +131,13 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
now_time="2026-08-27 10:00:00",
)
assert pending.task_id == "task-explicit"
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
"task-explicit"
]
assert oper.get_by_task_id(task_id="task-explicit").state == "accepted"
assert oper.stage_record_enqueue_failure(
task_id="task-explicit",
error="queue full",
now_time="2026-08-27 10:01:00",
) == 1
assert oper.stage_discard_task(task_id="task-explicit") == 1
assert oper.get_by_task_id(task_id="task-explicit").last_error == "queue full"
def test_transactional_repository_commits_frozen_projections(tmp_path):
@@ -317,16 +158,34 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
assert repeated == admitted
assert admitted.task_id
assert admitted.state == "accepted"
assert repository.list_accepted() == [admitted]
repository.record_enqueue_failure(
task_id=admitted.task_id,
error="queue full",
)
failed = repository.list_accepted()[0]
assert failed.last_error == "queue full"
assert repository.discard_task(task_id=admitted.task_id) == 1
assert repository.list_accepted() == []
with factory() as session:
failed = session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one()
assert failed.last_error == "queue full"
claimed = repository.claim_task(
task_id=admitted.task_id,
owner_id="repository-test-worker",
lease_seconds=60,
)
assert claimed is not None
assert repository.discard_claimed(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
) == 1
with factory() as session:
assert session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one_or_none() is None
engine.dispose()
+6
View File
@@ -305,10 +305,16 @@ def test_legacy_transfer_task_hides_internal_admission_identity():
public_fields = set(task.to_dict())
task.bind_admission_task_id("internal-task-id")
task.bind_execution_lease(
owner_id="internal-worker",
lease_token="internal-lease-token",
)
assert set(task.to_dict()) == public_fields
assert "task_id" not in task.to_dict()
assert "admission_task_id" not in task.to_dict()
assert "lease_owner" not in task.to_dict()
assert "lease_token" not in task.to_dict()
def test_chain_media_legacy_scraping_symbols_resolve_to_scraping_chain():
+67 -1
View File
@@ -1,4 +1,5 @@
import unittest
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
@@ -8,7 +9,11 @@ from app.application.history import (
failed_retry_count,
record_transfer_failure,
)
from app.application.transfer import TransferPlanningInput, TransferTask
from app.application.transfer import (
TransferAdmission,
TransferPlanningInput,
TransferTask,
)
from app.chain.transfer import JobManager, TransferChain
from app.domain.context import MediaInfo
from app.domain.meta.metabase import MetaBase
@@ -151,6 +156,7 @@ def make_task(episode: int, season: int = 1) -> TransferTask:
def make_transfer_chain() -> TransferChain:
"""构造带内存 durable admission 契约的整理链测试骨架。"""
chain = object.__new__(TransferChain)
chain.jobview = JobManager()
chain._media_exts = settings.RMT_MEDIAEXT
@@ -161,6 +167,66 @@ def make_transfer_chain() -> TransferChain:
)
chain._success_target_files = {}
chain._scrape_batches = {}
admissions = MagicMock()
admissions_by_identity = {}
admissions_by_id = {}
def admit(*, storage, src_path, planning_input=None):
"""按源身份幂等返回测试用 durable admission。"""
identity = storage, src_path
existing = admissions_by_identity.get(identity)
if existing is not None:
return existing
admission = TransferAdmission(
task_id=f"test-task-{len(admissions_by_id) + 1}",
storage=storage,
src_path=src_path,
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=planning_input,
)
admissions_by_identity[identity] = admission
admissions_by_id[admission.task_id] = admission
return admission
def claim_task(*, task_id, owner_id, lease_seconds):
"""为测试任务返回唯一 token,并保留正式 claim 的参数约束。"""
assert lease_seconds > 0
admission = admissions_by_id[task_id]
claimed = replace(
admission,
lease_owner=owner_id,
lease_token=f"lease-{task_id}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=admission.attempt_count + 1,
)
admissions_by_identity[(claimed.storage, claimed.src_path)] = claimed
admissions_by_id[task_id] = claimed
return claimed
def checkpoint_plan(*, task_id, lease_token, input_fingerprint, checkpoint):
"""回读带检查点的持久投影,供同步整理测试执行真实编排。"""
del input_fingerprint
admission = admissions_by_id[task_id]
assert admission.lease_token == lease_token
planned = replace(
admission,
state="provider_pending" if checkpoint.is_provider_pending else "planned",
checkpoint=checkpoint,
)
admissions_by_identity[(planned.storage, planned.src_path)] = planned
admissions_by_id[task_id] = planned
return planned
admissions.admit.side_effect = admit
admissions.claim_task.side_effect = claim_task
admissions.checkpoint_plan.side_effect = checkpoint_plan
admissions.discard_claimed.return_value = 1
admissions.release_claim.return_value = True
chain._transfer_admissions = admissions
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
return chain
+197
View File
@@ -0,0 +1,197 @@
"""整理恢复租约字段的 Alembic 可逆迁移测试。"""
import importlib
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from app.db.models.transferpending import TransferPending
MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
LEASE_COLUMNS = {
"lease_owner",
"lease_token",
"lease_expires_at",
"heartbeat_at",
"attempt_count",
}
def _bind_migration(monkeypatch, connection):
"""把 3.0.15 迁移绑定到隔离数据库连接。"""
migration = importlib.import_module(MIGRATION)
monkeypatch.setattr(
migration,
"op",
Operations(MigrationContext.configure(connection)),
)
return migration
def _create_planning_table(connection) -> None:
"""创建 3.0.14 时代包含完整规划检查点的登记表。"""
metadata = sa.MetaData()
table = sa.Table(
"transferpending",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("task_id", sa.String(64), nullable=False),
sa.Column("storage", sa.String(), nullable=False),
sa.Column("src_path", sa.String(), nullable=False),
sa.Column("created_at", sa.String(), nullable=True),
sa.Column("state", sa.String(32), nullable=False),
sa.Column("updated_at", sa.String(40), nullable=False),
sa.Column("last_error", sa.Text(), nullable=True),
sa.Column("input_version", sa.Integer(), nullable=False),
sa.Column("planning_input", sa.JSON(), nullable=False),
sa.Column("input_fingerprint", sa.String(64), nullable=False),
sa.Column("checkpoint_version", sa.Integer(), nullable=True),
sa.Column("checkpoint_payload", sa.JSON(), nullable=True),
sa.Column("planned_at", sa.String(40), nullable=True),
sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"),
)
sa.Index(
"ux_transferpending_storage_path",
table.c.storage,
table.c.src_path,
unique=True,
)
sa.Index(
"ix_transferpending_state_created",
table.c.state,
table.c.created_at,
table.c.id,
)
metadata.create_all(connection)
connection.execute(
table.insert(),
{
"id": 1,
"task_id": "stable-task",
"storage": "local",
"src_path": "/downloads/Movie.mkv",
"created_at": "2026-08-27 10:00:00",
"state": "planned",
"updated_at": "2026-08-27 10:00:00",
"last_error": None,
"input_version": 1,
"planning_input": {
"schema_version": 1,
"source_fileitem": {
"storage": "local",
"path": "/downloads/Movie.mkv",
},
},
"input_fingerprint": "0" * 64,
"checkpoint_version": 1,
"checkpoint_payload": {"schema_version": 1},
"planned_at": "2026-08-27 10:00:00",
},
)
def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None:
"""SQLite 应支持租约字段重复升级、降级和再次升级。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_planning_table(connection)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
migration.upgrade()
inspector = sa.inspect(connection)
assert {
column["name"]
for column in inspector.get_columns("transferpending")
} == {column.name for column in TransferPending.__table__.columns}
assert "ix_transferpending_recovery_lease" in {
index["name"]
for index in inspector.get_indexes("transferpending")
}
row = connection.execute(
sa.text(
"SELECT lease_owner, lease_token, lease_expires_at, "
"heartbeat_at, attempt_count FROM transferpending WHERE id = 1"
)
).mappings().one()
assert dict(row) == {
"lease_owner": None,
"lease_token": None,
"lease_expires_at": None,
"heartbeat_at": None,
"attempt_count": 0,
}
connection.execute(
sa.text(
"UPDATE transferpending SET lease_owner = 'worker', "
"lease_token = 'token', "
"lease_expires_at = '2026-08-27 10:05:00.000000', "
"heartbeat_at = '2026-08-27 10:00:00.000000', "
"attempt_count = 3 WHERE id = 1"
)
)
migration.downgrade()
downgraded = sa.inspect(connection)
assert LEASE_COLUMNS.isdisjoint({
column["name"]
for column in downgraded.get_columns("transferpending")
})
assert "ix_transferpending_recovery_lease" not in {
index["name"]
for index in downgraded.get_indexes("transferpending")
}
assert connection.execute(
sa.text("SELECT state FROM transferpending WHERE id = 1")
).scalar_one() == "planned"
migration.upgrade()
reupgraded = connection.execute(
sa.text(
"SELECT lease_owner, lease_token, attempt_count "
"FROM transferpending WHERE id = 1"
)
).mappings().one()
assert dict(reupgraded) == {
"lease_owner": None,
"lease_token": None,
"attempt_count": 0,
}
engine.dispose()
def test_partial_transfer_lease_upgrade_preserves_existing_owner(
monkeypatch,
) -> None:
"""迁移中断后重跑应补齐字段且不得覆盖已经写入的租约拥有者。"""
engine = sa.create_engine("sqlite://")
with engine.begin() as connection:
_create_planning_table(connection)
connection.execute(
sa.text("ALTER TABLE transferpending ADD COLUMN lease_owner VARCHAR(128)")
)
connection.execute(
sa.text(
"UPDATE transferpending SET lease_owner = 'preserved-worker' "
"WHERE id = 1"
)
)
migration = _bind_migration(monkeypatch, connection)
migration.upgrade()
row = connection.execute(
sa.text(
"SELECT lease_owner, lease_token, attempt_count "
"FROM transferpending WHERE id = 1"
)
).mappings().one()
assert dict(row) == {
"lease_owner": "preserved-worker",
"lease_token": None,
"attempt_count": 0,
}
engine.dispose()
+801
View File
@@ -0,0 +1,801 @@
"""整理恢复租约的原子 claim、续租和陈旧 token 防护测试。"""
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta, timezone
from threading import Barrier, Lock
from typing import Any
import pytest
from sqlalchemy import create_engine, select, text
from sqlalchemy.orm import sessionmaker
from app.application.transfer import (
TRANSFER_ADMISSION_PLANNED,
TransferAdmission,
TransferAdmissionProjectionError,
TransferLeaseLostError,
TransferPlanCheckpoint,
TransferPlanningInput,
TransferProviderInvocationSnapshot,
TransferProviderReference,
)
from app.db.adapters.transfer import TransactionalTransferAdmissionRepository
from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper
def _planning_input(path: str) -> TransferPlanningInput:
"""构造与测试源路径绑定的最小规划输入。"""
return TransferPlanningInput(
source_fileitem={
"storage": "local",
"path": path,
"type": "file",
"name": path.rsplit("/", maxsplit=1)[-1],
},
meta={"name": "Movie"},
mediainfo={"title": "Movie"},
)
def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
"""构造无需文件副作用的合法宿主跳过检查点。"""
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="local",
root_target_path="/library",
final_target_path="/library",
resolved_transfer_type="copy",
items=(),
skip_reason="测试跳过计划",
)
def _provider_checkpoint(
planning_input: TransferPlanningInput,
) -> TransferPlanCheckpoint:
"""构造只冻结 provider ABI、尚未完成宿主规划的检查点。"""
invocation = TransferProviderInvocationSnapshot(
fileitem=planning_input.source_fileitem,
meta=planning_input.meta,
meta_kind="MetaVideo",
mediainfo=planning_input.mediainfo,
mediainfo_kind="MediaInfo",
)
return TransferPlanCheckpoint(
planning_input=planning_input,
target_storage="",
root_target_path="",
final_target_path="",
resolved_transfer_type="",
items=(),
resolved_meta=invocation.meta,
resolved_meta_kind=invocation.meta_kind,
resolved_mediainfo=invocation.mediainfo,
resolved_mediainfo_kind=invocation.mediainfo_kind,
legacy_transfer_providers=(
TransferProviderReference(
plugin_id="provider-a",
plugin_name="Provider A",
),
),
provider_invocation=invocation,
)
@pytest.fixture
def repository_factory(tmp_path):
"""创建允许多线程独立 Session 竞争的 SQLite 租约仓储工厂。"""
engine = create_engine(
f"sqlite:///{tmp_path / 'transfer-lease.db'}",
connect_args={"check_same_thread": False, "timeout": 10},
)
TransferPending.__table__.create(engine)
factory = sessionmaker(bind=engine)
yield lambda: TransactionalTransferAdmissionRepository(factory)
engine.dispose()
@pytest.fixture
def lease_clock(monkeypatch):
"""为所有仓储实例提供可推进的固定 UTC 时钟。"""
clock = {"now": datetime(2026, 8, 27, 10, 0, tzinfo=timezone.utc)}
monkeypatch.setattr(
TransactionalTransferAdmissionRepository,
"_lease_now",
staticmethod(lambda: clock["now"]),
)
return clock
@pytest.fixture
def business_clock(monkeypatch):
"""为业务审计字段提供与 UTC 租约时钟明确分离的宿主本地时间。"""
clock = {"now": "2026-08-27 18:00:00"}
monkeypatch.setattr(
TransactionalTransferAdmissionRepository,
"_now",
staticmethod(lambda: clock["now"]),
)
return clock
def _admit(
repository: TransactionalTransferAdmissionRepository,
path: str,
) -> TransferAdmission:
"""登记一个带完整版本化输入的测试任务。"""
return repository.admit(
storage="local",
src_path=path,
planning_input=_planning_input(path),
)
def _pending_snapshot(
repository: TransactionalTransferAdmissionRepository,
task_id: str,
) -> dict[str, Any]:
"""在 Session 关闭前冻结测试需要检查的持久登记字段。"""
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(TransferPending.task_id == task_id)
).scalar_one()
return {
"lease_owner": pending.lease_owner,
"lease_token": pending.lease_token,
"lease_expires_at": pending.lease_expires_at,
"heartbeat_at": pending.heartbeat_at,
"last_error": pending.last_error,
"attempt_count": pending.attempt_count,
"updated_at": pending.updated_at,
"planned_at": pending.planned_at,
}
def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
repository_factory,
lease_clock,
business_clock,
) -> None:
"""新 token 才增加 attempt,过期 token 不能续租、释放或删除接管者。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/movie.mkv")
assert admitted.created_at == "2026-08-27 18:00:00"
business_clock["now"] = "2026-08-27 18:01:00"
first = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-a",
lease_seconds=60,
)
assert first is not None
assert first.lease_owner == "worker-a"
assert first.lease_token
assert first.attempt_count == 1
assert first.updated_at == "2026-08-27 18:01:00"
assert repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-a",
lease_seconds=60,
) is None
assert repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-b",
lease_seconds=60,
) is None
lease_clock["now"] += timedelta(seconds=30)
business_clock["now"] = "2026-08-27 18:02:00"
renewed = repository.heartbeat(
task_id=admitted.task_id,
lease_token=first.lease_token,
lease_seconds=60,
)
assert renewed is not None
assert renewed.attempt_count == 1
assert renewed.heartbeat_at == "2026-08-27 10:00:30.000000"
assert renewed.updated_at == first.updated_at
lease_clock["now"] += timedelta(seconds=61)
business_clock["now"] = "2026-08-27 18:03:00"
assert repository.release_claim(
task_id=admitted.task_id,
lease_token=first.lease_token,
error="expired worker",
) is False
assert repository.discard_claimed(
task_id=admitted.task_id,
lease_token=first.lease_token,
) == 0
takeover = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-b",
lease_seconds=60,
)
assert takeover is not None
assert takeover.lease_token != first.lease_token
assert takeover.attempt_count == 2
assert takeover.updated_at == "2026-08-27 18:03:00"
assert repository.heartbeat(
task_id=admitted.task_id,
lease_token=first.lease_token,
lease_seconds=60,
) is None
business_clock["now"] = "2026-08-27 18:04:00"
assert repository.release_claim(
task_id=admitted.task_id,
lease_token=first.lease_token,
error="stale worker",
) is False
assert repository.discard_claimed(
task_id=admitted.task_id,
lease_token=first.lease_token,
) == 0
assert repository.release_claim(
task_id=admitted.task_id,
lease_token=takeover.lease_token,
error="retry later",
) is True
released = _pending_snapshot(repository, admitted.task_id)
assert released["lease_owner"] is None
assert released["lease_token"] is None
assert released["lease_expires_at"] is None
assert released["heartbeat_at"] is None
assert released["last_error"] == "retry later"
assert released["attempt_count"] == 2
assert released["updated_at"] == "2026-08-27 18:04:00"
third = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-c",
lease_seconds=60,
)
assert third is not None
assert third.attempt_count == 3
assert repository.discard_claimed(
task_id=admitted.task_id,
lease_token=third.lease_token,
) == 1
with repository._session_factory() as session: # noqa: SLF001
assert session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one_or_none() is None
def test_claim_recoverable_respects_order_limit_and_active_lease(
repository_factory,
lease_clock,
) -> None:
"""批量恢复跳过有效租约,并按登记顺序逐条 CAS 到请求上限。"""
repository = repository_factory()
first = _admit(repository, "/downloads/a.mkv")
second = _admit(repository, "/downloads/b.mkv")
third = _admit(repository, "/downloads/c.mkv")
active = repository.claim_task(
task_id=first.task_id,
owner_id="active-worker",
lease_seconds=60,
)
assert active is not None
claimed = repository.claim_recoverable(
owner_id="recovery-worker",
limit=2,
lease_seconds=60,
)
assert [item.task_id for item in claimed] == [second.task_id, third.task_id]
assert all(item.lease_owner == "recovery-worker" for item in claimed)
assert all(item.attempt_count == 1 for item in claimed)
assert repository.claim_recoverable(
owner_id="other-worker",
limit=10,
lease_seconds=60,
) == []
lease_clock["now"] += timedelta(seconds=61)
reclaimed = repository.claim_recoverable(
owner_id="takeover-worker",
limit=2,
lease_seconds=60,
)
assert [item.task_id for item in reclaimed] == [first.task_id, second.task_id]
assert reclaimed[0].attempt_count == 2
assert reclaimed[1].attempt_count == 2
def test_claim_recoverable_skips_corrupt_projection_and_claims_later_tasks(
repository_factory,
lease_clock,
business_clock,
monkeypatch,
) -> None:
"""毒行应留下单次诊断但不持有租约或饿死后续健康任务。"""
repository = repository_factory()
messages: list[str] = []
monkeypatch.setattr(
"app.db.adapters.transfer._diagnostic_logger.error",
messages.append,
)
corrupt = _admit(repository, "/downloads/a-corrupt.mkv")
healthy = [
_admit(repository, "/downloads/b-healthy.mkv"),
_admit(repository, "/downloads/c-healthy.mkv"),
]
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == corrupt.task_id
)
).scalar_one()
pending.input_fingerprint = "corrupt"
session.commit()
claimed = repository.claim_recoverable(
owner_id="recovery-worker",
limit=2,
lease_seconds=60,
)
assert [item.task_id for item in claimed] == [item.task_id for item in healthy]
corrupt_snapshot = _pending_snapshot(repository, corrupt.task_id)
assert corrupt_snapshot["lease_token"] is None
assert corrupt_snapshot["attempt_count"] == 0
assert corrupt_snapshot["last_error"].startswith("恢复投影失败:")
assert corrupt_snapshot["updated_at"] == "2026-08-27 18:00:00"
assert len(messages) == 1
business_clock["now"] = "2026-08-27 18:01:00"
assert repository.claim_recoverable(
owner_id="second-recovery-worker",
limit=1,
lease_seconds=60,
) == []
assert _pending_snapshot(repository, corrupt.task_id) == corrupt_snapshot
assert len(messages) == 1
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == corrupt.task_id
)
).scalar_one()
pending.input_fingerprint = _planning_input(
"/downloads/a-corrupt.mkv"
).fingerprint
session.commit()
repaired = repository.claim_recoverable(
owner_id="repaired-worker",
limit=1,
lease_seconds=60,
)
assert [item.task_id for item in repaired] == [corrupt.task_id]
def test_projection_diagnostic_changes_are_recorded_once_each(
repository_factory,
business_clock,
monkeypatch,
) -> None:
"""相同投影错误不重复写库,错误类型变化时才更新诊断并再次告警。"""
repository = repository_factory()
messages: list[str] = []
monkeypatch.setattr(
"app.db.adapters.transfer._diagnostic_logger.error",
messages.append,
)
admitted = _admit(repository, "/downloads/changing-corrupt.mkv")
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one()
pending.input_fingerprint = "corrupt"
session.commit()
assert repository.claim_recoverable(
owner_id="recovery-a",
limit=1,
lease_seconds=60,
) == []
first = _pending_snapshot(repository, admitted.task_id)
assert len(messages) == 1
business_clock["now"] = "2026-08-27 18:01:00"
assert repository.claim_recoverable(
owner_id="recovery-b",
limit=1,
lease_seconds=60,
) == []
assert _pending_snapshot(repository, admitted.task_id) == first
assert len(messages) == 1
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one()
pending.input_fingerprint = _planning_input(
"/downloads/changing-corrupt.mkv"
).fingerprint
pending.input_version = 999
session.commit()
business_clock["now"] = "2026-08-27 18:02:00"
assert repository.claim_recoverable(
owner_id="recovery-c",
limit=1,
lease_seconds=60,
) == []
changed = _pending_snapshot(repository, admitted.task_id)
assert changed["last_error"] != first["last_error"]
assert changed["updated_at"] == "2026-08-27 18:02:00"
assert len(messages) == 2
def test_projection_diagnostic_cas_is_concurrency_safe(
repository_factory,
monkeypatch,
) -> None:
"""并发恢复观察到同一损坏时只允许一个诊断写入者和一条运行日志。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/concurrent-corrupt.mkv")
messages: list[str] = []
message_lock = Lock()
def capture(message: str) -> None:
"""并发安全收集错误日志。"""
with message_lock:
messages.append(message)
monkeypatch.setattr(
"app.db.adapters.transfer._diagnostic_logger.error",
capture,
)
barrier = Barrier(2)
projection_error = TransferAdmissionProjectionError("same corruption")
def record(_: int) -> bool:
"""让两个独立 Session 同时竞争同一诊断 CAS。"""
barrier.wait(timeout=5)
return repository_factory()._record_projection_failure( # noqa: SLF001
task_id=admitted.task_id,
error=projection_error,
)
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(record, range(2)))
assert sorted(results) == [False, True]
assert len(messages) == 1
snapshot = _pending_snapshot(repository, admitted.task_id)
assert snapshot["last_error"] == "恢复投影失败: same corruption"
def test_projection_diagnostic_does_not_overwrite_active_lease(
repository_factory,
monkeypatch,
) -> None:
"""诊断 CAS 不得覆盖已经由健康 worker 取得有效租约的任务。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/active-lease.mkv")
claimed = repository.claim_task(
task_id=admitted.task_id,
owner_id="active-worker",
lease_seconds=60,
)
assert claimed is not None
before = _pending_snapshot(repository, admitted.task_id)
messages: list[str] = []
monkeypatch.setattr(
"app.db.adapters.transfer._diagnostic_logger.error",
messages.append,
)
recorded = repository._record_projection_failure( # noqa: SLF001
task_id=admitted.task_id,
error=TransferAdmissionProjectionError("stale observation"),
)
assert recorded is False
assert _pending_snapshot(repository, admitted.task_id) == before
assert messages == []
def test_projection_diagnostic_database_failure_propagates(
repository_factory,
monkeypatch,
) -> None:
"""诊断留痕的数据库基础设施异常必须向上游传播而非静默跳过。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/db-error-corrupt.mkv")
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == admitted.task_id
)
).scalar_one()
pending.input_fingerprint = "corrupt"
session.commit()
def fail_diagnostic(*_args, **_kwargs):
"""模拟诊断短事务的底层数据库写入失败。"""
raise RuntimeError("database unavailable")
monkeypatch.setattr(
TransferPendingOper,
"stage_record_projection_failure",
fail_diagnostic,
)
with pytest.raises(RuntimeError, match="database unavailable"):
repository.claim_recoverable(
owner_id="recovery-worker",
limit=1,
lease_seconds=60,
)
def test_claim_task_wraps_persisted_json_decode_failure(
repository_factory,
) -> None:
"""持久 JSON 解码错误应归类为投影损坏,而不是数据库基础设施故障。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/invalid-json.mkv")
with repository._session_factory() as session: # noqa: SLF001
session.execute(
text(
"UPDATE transferpending SET planning_input = 'not-json' "
"WHERE task_id = :task_id"
),
{"task_id": admitted.task_id},
)
session.commit()
with pytest.raises(TransferAdmissionProjectionError, match="JSON"):
repository.claim_task(
task_id=admitted.task_id,
owner_id="recovery-worker",
lease_seconds=60,
)
with repository._session_factory() as session: # noqa: SLF001
snapshot = session.execute(
text(
"SELECT lease_token, attempt_count FROM transferpending "
"WHERE task_id = :task_id"
),
{"task_id": admitted.task_id},
).mappings().one()
assert snapshot["lease_token"] is None
assert snapshot["attempt_count"] == 0
def test_concurrent_recovery_callers_scan_past_lost_candidates(
repository_factory,
lease_clock,
monkeypatch,
) -> None:
"""并发 caller 竞争同一首批后应继续向后扫描并各自填满限额。"""
setup_repository = repository_factory()
admitted = [
_admit(setup_repository, f"/downloads/concurrent-{index}.mkv")
for index in range(4)
]
barrier = Barrier(2)
barrier_lock = Lock()
initial_scans = 0
original = TransferPending.list_claimable_candidates.__func__
def synchronized_candidates(cls, db, **kwargs):
"""强制两个 caller 在取得相同首批候选后再进入逐任务 CAS。"""
nonlocal initial_scans
candidates = original(cls, db, **kwargs)
if kwargs.get("after_cursor") is None:
with barrier_lock:
initial_scans += 1
barrier.wait(timeout=5)
return candidates
monkeypatch.setattr(
TransferPending,
"list_claimable_candidates",
classmethod(synchronized_candidates),
)
def recover(owner_id: str) -> list[TransferAdmission]:
"""使用独立仓储与 Session 执行一次有界恢复扫描。"""
return repository_factory().claim_recoverable(
owner_id=owner_id,
limit=2,
lease_seconds=60,
)
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(recover, ("worker-a", "worker-b")))
assert initial_scans == 2
assert [len(result) for result in results] == [2, 2]
claimed_task_ids = [item.task_id for result in results for item in result]
assert len(set(claimed_task_ids)) == 4
assert set(claimed_task_ids) == {item.task_id for item in admitted}
def test_unclaimed_enqueue_failure_cannot_overwrite_claimed_task(
repository_factory,
business_clock,
) -> None:
"""task-id-only 入队失败入口不得改写已经由 worker claim 的登记。"""
repository = repository_factory()
admitted = _admit(repository, "/downloads/claimed.mkv")
business_clock["now"] = "2026-08-27 18:01:00"
claimed = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-a",
lease_seconds=60,
)
assert claimed is not None
before = _pending_snapshot(repository, admitted.task_id)
business_clock["now"] = "2026-08-27 18:02:00"
repository.record_enqueue_failure(
task_id=admitted.task_id,
error="stale queue failure",
)
after = _pending_snapshot(repository, admitted.task_id)
assert after == before
def test_planning_writes_require_current_unexpired_lease(
repository_factory,
lease_clock,
) -> None:
"""checkpoint 和规划错误均不得由已过期或已被接管的 worker 写入。"""
repository = repository_factory()
path = "/downloads/planning.mkv"
planning_input = _planning_input(path)
admitted = repository.admit(
storage="local",
src_path=path,
planning_input=planning_input,
)
first = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-a",
lease_seconds=30,
)
assert first is not None
lease_clock["now"] += timedelta(seconds=31)
takeover = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-b",
lease_seconds=60,
)
assert takeover is not None
with pytest.raises(TransferLeaseLostError, match="租约"):
repository.record_planning_failure(
task_id=admitted.task_id,
lease_token=first.lease_token,
error="stale planning failure",
)
with pytest.raises(TransferLeaseLostError, match="租约"):
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=first.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
repository.record_planning_failure(
task_id=admitted.task_id,
lease_token=takeover.lease_token,
error="retryable planning failure",
)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=takeover.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
repeated = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=takeover.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
assert planned.state == TRANSFER_ADMISSION_PLANNED
assert planned.last_error is None
assert repeated == planned
assert planned.attempt_count == 2
def test_provider_checkpoint_sets_planned_time_only_after_host_plan(
repository_factory,
lease_clock,
business_clock,
) -> None:
"""provider 快照不是规划完成,planned_at 只记录首次宿主完整计划。"""
repository = repository_factory()
path = "/downloads/provider-plan.mkv"
planning_input = _planning_input(path)
admitted = repository.admit(
storage="local",
src_path=path,
planning_input=planning_input,
)
business_clock["now"] = "2026-08-27 18:01:00"
claimed = repository.claim_task(
task_id=admitted.task_id,
owner_id="worker-a",
lease_seconds=60,
)
assert claimed is not None
business_clock["now"] = "2026-08-27 18:02:00"
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_provider_checkpoint(planning_input),
)
provider_snapshot = _pending_snapshot(repository, admitted.task_id)
assert provider_snapshot["planned_at"] is None
assert provider_snapshot["updated_at"] == "2026-08-27 18:02:00"
business_clock["now"] = "2026-08-27 18:03:00"
checkpoint = _checkpoint(planning_input)
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
planned_snapshot = _pending_snapshot(repository, admitted.task_id)
assert planned_snapshot["planned_at"] == "2026-08-27 18:03:00"
business_clock["now"] = "2026-08-27 18:04:00"
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
assert _pending_snapshot(repository, admitted.task_id) == planned_snapshot
def test_concurrent_claim_uses_rowcount_as_single_winner(
repository_factory,
lease_clock,
) -> None:
"""并发 worker 即使读取同一任务,也只能有一个 CAS 更新获胜。"""
setup_repository = repository_factory()
admitted = _admit(setup_repository, "/downloads/concurrent.mkv")
barrier = Barrier(2)
def claim(owner_id: str):
"""等待竞争者就绪后使用独立 Session claim 同一任务。"""
repository = repository_factory()
barrier.wait()
return repository.claim_task(
task_id=admitted.task_id,
owner_id=owner_id,
lease_seconds=60,
)
with ThreadPoolExecutor(max_workers=2) as executor:
results = list(executor.map(claim, ("worker-a", "worker-b")))
winners = [result for result in results if result is not None]
assert len(winners) == 1
assert winners[0].attempt_count == 1
@@ -0,0 +1,158 @@
"""旧待整理 Oper 的精确插件兼容与租约 fencing 测试。"""
import importlib
from collections.abc import Callable
from typing import Any
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, sessionmaker
from app.db import base as db_base
from app.db.models.transferpending import TransferPending
from app.db.oper.transferpending import TransferPendingOper as CanonicalTransferPendingOper
from app.runtime.compat.manifest import MODULE_ALIASES
@pytest.fixture
def legacy_session_factory(tmp_path, monkeypatch):
"""为无 Session 兼容 Oper 提供独占事务,并在提交后保留返回快照。"""
engine = create_engine(f"sqlite:///{tmp_path / 'legacy-transferpending.db'}")
TransferPending.__table__.create(engine)
factory = sessionmaker(bind=engine)
def run_transaction(operation: Callable[[Session], Any]) -> Any:
"""按生产组合根语义执行一次同步兼容事务。"""
with factory() as session:
session.expire_on_commit = False
try:
result = operation(session)
session.commit()
return result
except Exception:
session.rollback()
raise
monkeypatch.setattr(db_base, "run_sync_transaction", run_transaction)
yield factory
engine.dispose()
def test_legacy_import_targets_private_sdk_facade() -> None:
"""旧模块路径只能解析到私有 SDK 门面,不能回退到 canonical Oper。"""
alias = MODULE_ALIASES["app.db.transferpending_oper"]
legacy = importlib.import_module("app.db.transferpending_oper")
assert alias.target == "app.sdk._legacy.transferpending"
assert alias.owner == "sdk"
assert alias.replacement == "app.application.transfer"
assert legacy is importlib.import_module(alias.target)
assert legacy.__all__ == ["TransferPendingOper"]
assert not hasattr(legacy, "TransferPending")
assert legacy.TransferPendingOper is not CanonicalTransferPendingOper
for internal_method in (
"stage_admit",
"stage_claim_task",
"stage_discard_claimed",
"stage_checkpoint_plan",
):
assert not hasattr(legacy.TransferPendingOper, internal_method)
def test_legacy_no_session_queries_preserve_historical_shapes(
legacy_session_factory,
) -> None:
"""旧插件无需 Session 即可登记和查询,返回形态与原 ABI 一致。"""
legacy = importlib.import_module("app.db.transferpending_oper")
oper = legacy.TransferPendingOper()
first = oper.register("local", "/downloads/first.mkv")
repeated = oper.register("local", "/downloads/first.mkv")
second = oper.register("remote", "/downloads/second.mkv")
assert first is not None
assert repeated is not None
assert second is not None
assert repeated.task_id == first.task_id
assert oper.list_all() == [
("local", "/downloads/first.mkv"),
("remote", "/downloads/second.mkv"),
]
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
first.task_id,
second.task_id,
]
assert [
item.task_id
for item in oper.list_by_states(states=("accepted", "planned"), limit=1)
] == [first.task_id]
assert oper.get_by_identity(
storage="local",
src_path="/downloads/first.mkv",
).task_id == first.task_id
assert oper.get_by_task_id(task_id=second.task_id).src_path == second.src_path
assert oper.register("", "/downloads/invalid.mkv") is None
assert oper.list_by_state(state="") == []
assert oper.list_by_states(states=()) == []
def test_legacy_mutations_never_delete_claimed_rows(
legacy_session_factory,
) -> None:
"""旧 discard/clear 只处理未 claim 行,有效或过期 token 均受保护。"""
legacy = importlib.import_module("app.db.transferpending_oper")
oper = legacy.TransferPendingOper()
active = oper.register("local", "/downloads/active.mkv")
expired = oper.register("local", "/downloads/expired.mkv")
free = oper.register("local", "/downloads/free.mkv")
assert active is not None
assert expired is not None
assert free is not None
with legacy_session_factory() as session:
assert TransferPending.claim_task(
session,
task_id=active.task_id,
states=("accepted",),
owner_id="active-worker",
lease_token="active-token",
now_time="2026-08-27 10:00:00.000000",
lease_expires_at="2026-08-27 10:01:00.000000",
updated_at="2026-08-27 18:00:00",
) == 1
assert TransferPending.claim_task(
session,
task_id=expired.task_id,
states=("accepted",),
owner_id="expired-worker",
lease_token="expired-token",
now_time="2026-08-27 10:00:00.000000",
lease_expires_at="2026-08-27 09:59:00.000000",
updated_at="2026-08-27 18:00:00",
) == 1
session.commit()
repeated = oper.register("local", "/downloads/active.mkv")
assert repeated is not None
assert repeated.task_id == active.task_id
assert repeated.lease_token == "active-token"
assert oper.discard("local", "/downloads/active.mkv") == 0
assert oper.discard("local", "/downloads/expired.mkv") == 0
assert oper.discard("local", "/downloads/free.mkv") == 1
removable = oper.register("local", "/downloads/removable.mkv")
assert removable is not None
assert oper.clear() == 1
assert oper.list_all() == [
("local", "/downloads/active.mkv"),
("local", "/downloads/expired.mkv"),
]
with legacy_session_factory() as session:
rows = session.execute(
select(TransferPending).order_by(TransferPending.id.asc())
).scalars().all()
assert [(row.task_id, row.lease_token) for row in rows] == [
(active.task_id, "active-token"),
(expired.task_id, "expired-token"),
]
+140 -13
View File
@@ -8,6 +8,8 @@
这些测试固定三项不变量入队即落盘登记终态即注销重启能回放
"""
import threading
import time
from dataclasses import replace
from pathlib import Path
from unittest.mock import MagicMock
@@ -24,6 +26,17 @@ def _build_chain(admissions) -> TransferChain:
"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = admissions
chain._worker_owner_id = "test-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
chain._worker_state_lock = threading.RLock()
chain._closing = False
chain._recovery_wakeup_event = threading.Event()
chain._replay_stop_event = threading.Event()
chain._lease_heartbeat_stop_event = threading.Event()
chain._lease_heartbeat_thread = None
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
return chain
@@ -36,6 +49,11 @@ def _admission(path: str, task_id: str = "task-1") -> TransferAdmission:
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
lease_owner="test-owner",
lease_token=f"lease-{task_id}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
@@ -87,10 +105,19 @@ def test_discard_pending_on_terminal_state():
chain = _build_chain(admissions)
task = _task("/mnt/cd2/downloads/Movie.2024.mkv")
task.bind_admission_task_id("task-1")
task.bind_execution_lease(owner_id="test-owner", lease_token="lease-task-1")
chain._worker_owner_id = "test-owner"
chain._owned_leases = {
"task-1": ("lease-task-1", time.monotonic() + 120)
}
admissions.discard_claimed.return_value = 1
chain._TransferChain__discard_pending(task)
assert chain._TransferChain__discard_pending(task) is True
admissions.discard_task.assert_called_once_with(task_id="task-1")
admissions.discard_claimed.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
)
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
@@ -101,7 +128,7 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
media.write_bytes(b"x" * 10)
admissions = MagicMock()
admissions.list_recoverable.return_value = [_admission(str(media))]
admissions.claim_recoverable.return_value = [_admission(str(media))]
chain = _build_chain(admissions)
transferred = []
@@ -128,14 +155,18 @@ def test_replay_discards_vanished_files(tmp_path):
"""
admissions = MagicMock()
missing = tmp_path / "gone.mkv"
admissions.list_recoverable.return_value = [_admission(str(missing))]
admissions.claim_recoverable.return_value = [_admission(str(missing))]
admissions.discard_claimed.return_value = 1
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain._execute_transfer.assert_not_called()
admissions.discard_task.assert_called_once_with(task_id="task-1")
admissions.discard_claimed.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
)
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
@@ -148,7 +179,8 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
media.write_bytes(b"x")
admissions = MagicMock()
admissions.list_recoverable.return_value = [_admission(str(media))]
admissions.claim_recoverable.return_value = [_admission(str(media))]
admissions.release_claim.return_value = True
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
@@ -163,7 +195,12 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
chain._TransferChain__replay_pending()
chain._execute_transfer.assert_not_called()
admissions.discard_task.assert_not_called()
admissions.discard_claimed.assert_not_called()
admissions.release_claim.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
error="恢复源文件暂时不可读取",
)
def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
@@ -175,7 +212,7 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
src_path = f"{bluray.as_posix()}/"
admissions = MagicMock()
admissions.list_recoverable.return_value = [_admission(src_path)]
admissions.claim_recoverable.return_value = [_admission(src_path)]
chain = _build_chain(admissions)
transferred = []
@@ -197,7 +234,7 @@ def test_replay_is_noop_without_registrations():
没有登记时回放不应触碰整理链
"""
admissions = MagicMock()
admissions.list_recoverable.return_value = []
admissions.claim_recoverable.return_value = []
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
@@ -211,7 +248,7 @@ def test_replay_survives_db_failure():
读取登记失败不能让启动流程报错
"""
admissions = MagicMock()
admissions.list_recoverable.side_effect = RuntimeError("db gone")
admissions.claim_recoverable.side_effect = RuntimeError("db gone")
chain = _build_chain(admissions)
chain._execute_transfer = MagicMock()
@@ -230,7 +267,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
item.write_bytes(b"x")
admissions = MagicMock()
admissions.list_recoverable.return_value = [
admissions.claim_recoverable.return_value = [
_admission(str(first), "task-1"),
_admission(str(second), "task-2"),
]
@@ -261,7 +298,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
first.write_bytes(b"x")
missing_second = tmp_path / "gone.mkv"
admissions = MagicMock()
admissions.list_recoverable.return_value = [
admissions.claim_recoverable.return_value = [
_admission(str(first), "task-1"),
_admission(str(missing_second), "task-2"),
]
@@ -279,4 +316,94 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
chain._TransferChain__replay_pending(stop_event)
assert transferred == [first.as_posix()]
admissions.discard_task.assert_not_called()
admissions.discard_claimed.assert_not_called()
assert admissions.release_claim.call_count == 2
def test_replay_registers_entire_claimed_batch_before_first_source_stat(
tmp_path,
monkeypatch,
):
"""批量 claim 返回后必须先把全部 token 交给 heartbeat,再做逐条同步 I/O。"""
first = _admission(str(tmp_path / "A.mkv"), "task-1")
second = _admission(str(tmp_path / "B.mkv"), "task-2")
admissions = MagicMock()
admissions.claim_recoverable.return_value = [first, second]
admissions.release_claim.return_value = True
chain = _build_chain(admissions)
def observe_owned_batch(*_args, **_kwargs):
"""首个 stat 前观察两个 token 已同时进入续期集合。"""
assert set(chain._owned_leases) == {"task-1", "task-2"}
return None, False
monkeypatch.setattr(
chain,
"_TransferChain__build_replay_fileitem",
observe_owned_batch,
)
chain._TransferChain__replay_pending()
assert admissions.release_claim.call_count == 2
assert chain._owned_leases == {}
def test_replay_releases_claim_when_jobview_rejects_recovered_task(
tmp_path,
monkeypatch,
):
"""恢复任务未进入队列时必须立即 release,不能靠租约自然过期。"""
media = tmp_path / "Movie.2024.mkv"
media.write_bytes(b"x")
admission = _admission(str(media))
admission = replace(admission, checkpoint=MagicMock())
admissions = MagicMock()
admissions.claim_recoverable.return_value = [admission]
admissions.release_claim.return_value = True
chain = _build_chain(admissions)
monkeypatch.setattr(
chain,
"_TransferChain__queue_planned_replay",
MagicMock(return_value=False),
)
chain._TransferChain__replay_pending()
admissions.release_claim.assert_called_once_with(
task_id="task-1",
lease_token="lease-task-1",
error="恢复任务未进入内存队列",
)
assert chain._owned_leases == {}
def test_claimed_enqueue_failure_never_uses_unfenced_error_writer() -> None:
"""陈旧 token 入队失败只能 release_claim,不能覆盖新 owner 的 last_error。"""
admissions = MagicMock()
admissions.release_claim.return_value = False
chain = _build_chain(admissions)
chain._finish_scrape_batch_task = MagicMock()
chain.replay_pending = MagicMock()
task = _task("/downloads/stale-enqueue.mkv")
task.bind_admission_task_id("stale-task")
task.bind_execution_lease(
owner_id="test-owner",
lease_token="stale-token",
)
chain._owned_leases = {
"stale-task": ("stale-token", time.monotonic() + 120)
}
chain._TransferChain__record_enqueue_failure(
task,
RuntimeError("queue closed"),
)
admissions.record_enqueue_failure.assert_not_called()
admissions.release_claim.assert_called_once_with(
task_id="stale-task",
lease_token="stale-token",
error="queue closed",
)
assert chain._owned_leases == {}
+136 -21
View File
@@ -2,13 +2,14 @@
import ast
import inspect
import threading
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application import transfer as transfer_application
@@ -202,6 +203,31 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
"""构造只保留规划编排依赖的 TransferChain 骨架。"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = repository or Mock()
chain._worker_owner_id = "planning-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
chain._worker_state_lock = threading.RLock()
chain._closing = False
chain._recovery_wakeup_event = threading.Event()
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
def claim_task(**kwargs):
"""为规划测试返回与进程 owner 匹配的稳定 claim。"""
return transfer_application.TransferAdmission(
task_id=kwargs["task_id"],
storage="local",
src_path="/downloads/Movie.2026.mkv",
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
lease_owner=kwargs["owner_id"],
lease_token=f"lease-{kwargs['task_id']}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
chain._transfer_admissions.claim_task.side_effect = claim_task
chain._module_dispatcher = Mock()
chain._module_dispatcher.freeze_plugin_providers.return_value = ()
chain.eventmanager = Mock()
@@ -217,6 +243,20 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
return chain
def _replay_chain(repository) -> TransferChain:
"""构造绑定固定恢复 owner 且不启动真实 heartbeat 线程的测试链。"""
chain = object.__new__(TransferChain)
chain._transfer_admissions = repository
chain._worker_owner_id = "replay-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
chain._worker_state_lock = threading.RLock()
chain._closing = False
chain._recovery_wakeup_event = threading.Event()
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
return chain
def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher:
"""构造使用真实冻结解析与执行内核的内存插件调度器。"""
plugin_catalog = Mock()
@@ -725,6 +765,7 @@ def test_provider_empty_fallback_cas_failure_blocks_host_execution():
chain.execute_transfer_plan.assert_not_called()
repository.record_planning_failure.assert_called_once_with(
task_id="task-provider-cas-failure",
lease_token="lease-task-provider-cas-failure",
error="CAS failed",
)
@@ -804,7 +845,7 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
return SimpleNamespace(checkpoint=kwargs["checkpoint"])
repository.checkpoint_plan.side_effect = checkpoint_plan
repository.discard_task.side_effect = (
repository.discard_claimed.side_effect = (
lambda **_kwargs: calls.append("discard") or 1
)
result = TransferInfo(
@@ -830,8 +871,9 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
assert returned is result
assert calls == ["admit", "checkpoint", "execute", "discard"]
repository.discard_task.assert_called_once_with(
task_id="task-legacy-command"
repository.discard_claimed.assert_called_once_with(
task_id="task-legacy-command",
lease_token="lease-task-legacy-command",
)
@@ -898,16 +940,32 @@ def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_
src_path="/downloads/Movie.2026.mkv",
planning_input=accepted_input,
)
claimed = repository.claim_task(
task_id=admission.task_id,
owner_id="fingerprint-test",
lease_seconds=120,
)
assert claimed is not None
assert claimed.lease_token is not None
mismatched = _checkpoint(target_path="/library/B/Movie.mkv")
with pytest.raises(ValueError, match="指纹|fingerprint|规划输入"):
repository.checkpoint_plan(
task_id=admission.task_id,
lease_token=claimed.lease_token,
input_fingerprint=accepted_input.fingerprint,
checkpoint=mismatched,
)
recovered = repository.list_recoverable()
assert repository.release_claim(
task_id=admission.task_id,
lease_token=claimed.lease_token,
) is True
recovered = repository.claim_recoverable(
owner_id="fingerprint-recovery",
limit=10,
lease_seconds=120,
)
assert len(recovered) == 1
assert recovered[0].state == "accepted"
assert recovered[0].checkpoint is None
@@ -926,23 +984,35 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
planning_input=planning_input,
)
accepted = repository.list_recoverable()
assert len(accepted) == 1
assert accepted[0].state == "accepted"
assert accepted[0].planning_input == planning_input
assert accepted[0].checkpoint is None
claimed = repository.claim_task(
task_id=admission.task_id,
owner_id="roundtrip-owner",
lease_seconds=120,
)
assert claimed is not None
assert claimed.lease_token is not None
assert claimed.state == "accepted"
assert claimed.planning_input == planning_input
assert claimed.checkpoint is None
repository.record_planning_failure(
task_id=admission.task_id,
lease_token=claimed.lease_token,
error="rename unavailable",
)
retryable = repository.list_recoverable()[0]
assert retryable.state == "accepted"
assert retryable.last_error == "rename unavailable"
with sessionmaker(bind=engine)() as session:
retryable = session.execute(
select(TransferPending).where(
TransferPending.task_id == admission.task_id
)
).scalar_one()
assert retryable.state == "accepted"
assert retryable.last_error == "rename unavailable"
checkpoint = _checkpoint()
planned = repository.checkpoint_plan(
task_id=admission.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
@@ -950,7 +1020,18 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
assert planned.state == "planned"
assert planned.checkpoint == checkpoint
assert planned.last_error is None
assert repository.list_recoverable() == [planned]
assert repository.release_claim(
task_id=admission.task_id,
lease_token=claimed.lease_token,
) is True
recovered = repository.claim_recoverable(
owner_id="roundtrip-recovery",
limit=10,
lease_seconds=120,
)
assert len(recovered) == 1
assert recovered[0].state == "planned"
assert recovered[0].checkpoint == checkpoint
engine.dispose()
@@ -982,6 +1063,10 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
repository.admit.side_effect = admit
repository.checkpoint_plan.side_effect = checkpoint_plan
chain = _chain(repository=repository, checkpoint=checkpoint, result=result)
claim_task = repository.claim_task.side_effect
repository.claim_task.side_effect = lambda **kwargs: (
order.append("claim") or claim_task(**kwargs)
)
chain.plan_transfer.side_effect = lambda *_args, **_kwargs: (
order.append("plan") or checkpoint
)
@@ -992,12 +1077,13 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
returned = chain._plan_checkpoint_and_execute(task)
assert returned is result
assert order == ["admit", "plan", "commit-checkpoint", "execute"]
assert order == ["admit", "claim", "plan", "commit-checkpoint", "execute"]
assert task.admission_task_id == "task-sync"
repository.admit.assert_called_once()
assert repository.admit.call_args.kwargs["planning_input"] is planning_input
repository.checkpoint_plan.assert_called_once_with(
task_id="task-sync",
lease_token="lease-task-sync",
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
@@ -1192,11 +1278,15 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
updated_at="2026-08-27 10:00:00",
input_fingerprint=planning_input.fingerprint,
planning_input=planning_input,
lease_owner="replay-owner",
lease_token="lease-task-accepted-offline",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
replay_repository = Mock()
replay_repository.list_recoverable.return_value = [admission]
replay_chain = object.__new__(TransferChain)
replay_chain._transfer_admissions = replay_repository
replay_repository.claim_recoverable.return_value = [admission]
replay_chain = _replay_chain(replay_repository)
replay_chain._execute_transfer = Mock()
queued_tasks = []
replay_chain.put_to_queue = Mock(
@@ -1230,6 +1320,10 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
repository=execution_repository,
checkpoint=checkpoint,
)
execution_chain._worker_owner_id = "replay-owner"
execution_chain._owned_leases = {
admission.task_id: (str(admission.lease_token), float("inf"))
}
execution_chain.jobview = Mock()
execution_chain.eventmanager = Mock()
execution_chain.eventmanager.send_event.return_value = None
@@ -1292,11 +1386,15 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
planning_input=planning_input,
lease_owner="replay-owner",
lease_token="lease-task-accepted-empty-episodes",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
replay_repository = Mock()
replay_repository.list_recoverable.return_value = [admission]
replay_chain = object.__new__(TransferChain)
replay_chain._transfer_admissions = replay_repository
replay_repository.claim_recoverable.return_value = [admission]
replay_chain = _replay_chain(replay_repository)
queued_tasks = []
replay_chain.put_to_queue = Mock(
side_effect=lambda task: queued_tasks.append(task) or True
@@ -1319,6 +1417,10 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
repository=execution_repository,
checkpoint=checkpoint,
)
execution_chain._worker_owner_id = "replay-owner"
execution_chain._owned_leases = {
admission.task_id: (str(admission.lease_token), float("inf"))
}
execution_chain.jobview = Mock()
execution_chain.eventmanager = Mock()
execution_chain.eventmanager.send_event.return_value = None
@@ -1425,8 +1527,20 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
task = _task()
task.meta = MetaBase("Unrecognized.Movie.2026.mkv")
task.bind_admission_task_id("task-before-checkpoint")
task.bind_execution_lease(
owner_id="recognition-owner",
lease_token="lease-task-before-checkpoint",
)
chain = object.__new__(TransferChain)
chain._transfer_admissions = Mock()
chain._worker_owner_id = "recognition-owner"
chain._owned_leases = {
"task-before-checkpoint": (
"lease-task-before-checkpoint",
float("inf"),
)
}
chain._worker_state_lock = threading.RLock()
chain.jobview = Mock()
chain.queue_failed_transfer_notification = Mock()
chain.runtime_config = SimpleNamespace(
@@ -1449,6 +1563,7 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
assert result == (False, "未识别到媒体信息")
chain._transfer_admissions.record_planning_failure.assert_called_once_with(
task_id="task-before-checkpoint",
lease_token="lease-task-before-checkpoint",
error="未识别到媒体信息",
)
+5
View File
@@ -57,6 +57,8 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
task.bind_admission_task_id("task-stable")
values = task.to_dict()
task.bind_execution_lease(owner_id="worker-owner", lease_token="lease-token")
leased_values = task.to_dict()
assert set(values) == {
"fileitem",
@@ -84,9 +86,12 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
}
assert values["fileitem"] == task.fileitem.model_dump()
assert values["target_path"] == Path("/library/Movie (2026)")
assert leased_values == values
assert "admission_task_id" not in values
assert "planning_input" not in values
assert "plan_checkpoint" not in values
assert "lease_owner" not in values
assert "lease_token" not in values
def test_transfer_chain_do_transfer_keeps_legacy_signature():
+19 -8
View File
@@ -24,12 +24,13 @@ except ModuleNotFoundError:
POSTGRESQL_DIALECT = "postgresql+psycopg"
MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
def _bind_migration(monkeypatch, connection):
def _bind_migration(monkeypatch, connection, module_name=PLANNING_MIGRATION):
"""把迁移绑定到隔离数据库连接。"""
migration = importlib.import_module(MIGRATION)
migration = importlib.import_module(module_name)
monkeypatch.setattr(
migration,
"op",
@@ -91,10 +92,13 @@ def _planning_row(connection) -> dict[str, object]:
def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
"""断言规划迁移在当前隔离连接上的完整可逆生命周期。"""
_create_admission_table(connection)
migration = _bind_migration(monkeypatch, connection)
planning_migration = _bind_migration(monkeypatch, connection)
lease_migration = _bind_migration(monkeypatch, connection, LEASE_MIGRATION)
migration.upgrade()
migration.upgrade()
planning_migration.upgrade()
planning_migration.upgrade()
lease_migration.upgrade()
lease_migration.upgrade()
inspector = sa.inspect(connection)
assert {
@@ -133,7 +137,9 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
planned_at="2026-08-27 11:00:00",
)
)
migration.downgrade()
lease_migration.downgrade()
lease_migration.downgrade()
planning_migration.downgrade()
downgraded = sa.inspect(connection)
assert {
@@ -152,12 +158,17 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
for index in downgraded.get_indexes("transferpending")
} == {"ix_transferpending_state_created", "ux_transferpending_storage_path"}
migration.upgrade()
planning_migration.upgrade()
lease_migration.upgrade()
reupgraded = _planning_row(connection)
assert reupgraded["task_id"] == "stable-task"
assert reupgraded["state"] == "accepted"
assert reupgraded["checkpoint_payload"] is None
assert reupgraded["input_fingerprint"] == planning_input.fingerprint
assert {
column["name"]
for column in sa.inspect(connection).get_columns("transferpending")
} == {column.name for column in TransferPending.__table__.columns}
def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None:
+105 -22
View File
@@ -11,6 +11,7 @@ from app.application.transfer import (
TRANSFER_ADMISSION_PLANNED,
TRANSFER_ADMISSION_PROVIDER_PENDING,
TransferAdmissionConflictError,
TransferAdmissionProjectionError,
TransferPlanCheckpoint,
TransferPlanItem,
TransferPlanningInput,
@@ -30,6 +31,31 @@ def repository(tmp_path):
return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
def _claim(repository, task_id: str):
"""为需要变更规划状态的测试取得独占租约。"""
claimed = repository.claim_task(
task_id=task_id,
owner_id="planning-test-worker",
lease_seconds=3600,
)
assert claimed is not None
assert claimed.lease_token
return claimed
def _pending_snapshot(repository, task_id: str) -> dict[str, object]:
"""使用隔离 Session 冻结测试所需的持久状态字段。"""
with repository._session_factory() as session: # noqa: SLF001
pending = session.execute(
select(TransferPending).where(TransferPending.task_id == task_id)
).scalar_one()
return {
"state": pending.state,
"last_error": pending.last_error,
"checkpoint_payload": pending.checkpoint_payload,
}
def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput:
"""构造包含恢复所需媒体上下文的完整规划输入。"""
return TransferPlanningInput(
@@ -264,6 +290,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
checkpoint = replace(
_checkpoint(planning_input),
resolved_meta={"name": "Resolved Movie", "year": 2026},
@@ -275,6 +302,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
@@ -287,6 +315,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=replace(
checkpoint,
@@ -326,15 +355,18 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
checkpoint = _checkpoint(planning_input)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
repeated = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=checkpoint,
)
@@ -350,8 +382,9 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
"plugin-provider-a",
)
assert repeated == planned
assert repository.list_accepted() == []
assert repository.list_recoverable() == [planned]
assert _pending_snapshot(repository, admitted.task_id)["state"] == (
TRANSFER_ADMISSION_PLANNED
)
def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None:
@@ -362,23 +395,34 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
provider_checkpoint = _provider_checkpoint(planning_input)
provider_pending = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=provider_checkpoint,
)
assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING
assert provider_pending.checkpoint == provider_checkpoint
assert repository.list_recoverable() == [provider_pending]
repository.record_planning_failure(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
error="host planning unavailable",
)
failed = repository.list_recoverable()[0]
assert repository.release_claim(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
error="host planning unavailable",
)
failed = repository.claim_recoverable(
owner_id="planning-recovery-worker",
limit=1,
lease_seconds=3600,
)[0]
assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING
assert failed.checkpoint == provider_checkpoint
assert failed.last_error == "host planning unavailable"
@@ -386,11 +430,13 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
host_checkpoint = _checkpoint(planning_input)
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=failed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=host_checkpoint,
)
repeated = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=failed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=host_checkpoint,
)
@@ -402,6 +448,7 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=failed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=provider_checkpoint,
)
@@ -415,18 +462,19 @@ def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> Non
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
with pytest.raises(TransferAdmissionConflictError):
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint="0" * 64,
checkpoint=_checkpoint(planning_input),
)
recovered = repository.list_recoverable()
assert len(recovered) == 1
assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED
assert recovered[0].checkpoint is None
recovered = _pending_snapshot(repository, admitted.task_id)
assert recovered["state"] == TRANSFER_ADMISSION_ACCEPTED
assert recovered["checkpoint_payload"] is None
def test_planning_failure_stays_accepted_until_success(repository) -> None:
@@ -437,15 +485,21 @@ def test_planning_failure_stays_accepted_until_success(repository) -> None:
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
repository.record_planning_failure(task_id=admitted.task_id, error="rename failed")
failed = repository.list_recoverable()[0]
assert failed.state == TRANSFER_ADMISSION_ACCEPTED
assert failed.last_error == "rename failed"
assert failed.checkpoint is None
repository.record_planning_failure(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
error="rename failed",
)
failed = _pending_snapshot(repository, admitted.task_id)
assert failed["state"] == TRANSFER_ADMISSION_ACCEPTED
assert failed["last_error"] == "rename failed"
assert failed["checkpoint_payload"] is None
planned = repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
@@ -459,6 +513,7 @@ def test_checkpoint_rejects_missing_task(repository) -> None:
with pytest.raises(TransferPlanningStateError):
repository.checkpoint_plan(
task_id="missing",
lease_token="missing-token",
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
@@ -470,17 +525,25 @@ def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
factory = sessionmaker(bind=engine)
TransferPending.__table__.create(engine)
with factory() as session:
session.add(TransferPending(
pending = TransferPending(
storage="local",
src_path="/downloads/legacy.mkv",
state=TRANSFER_ADMISSION_ACCEPTED,
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
))
)
session.add(pending)
session.commit()
task_id = pending.task_id
admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0]
repository = TransactionalTransferAdmissionRepository(factory)
admitted = repository.claim_task(
task_id=task_id,
owner_id="legacy-projection-worker",
lease_seconds=3600,
)
assert admitted is not None
assert admitted.planning_input == TransferPlanningInput.legacy(
storage="local",
src_path="/downloads/legacy.mkv",
@@ -508,8 +571,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
row.input_version = 2
session.commit()
with pytest.raises(TransferPlanningStateError, match="版本"):
repository.list_accepted()
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
repository.claim_task(
task_id=admitted.task_id,
owner_id="corruption-worker",
lease_seconds=3600,
)
with factory() as session:
row = session.execute(
@@ -521,8 +588,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
row.planning_input = corrupted
session.commit()
with pytest.raises(TransferAdmissionConflictError, match="指纹"):
repository.list_accepted()
with pytest.raises(TransferAdmissionProjectionError, match="指纹"):
repository.claim_task(
task_id=admitted.task_id,
owner_id="corruption-worker",
lease_seconds=3600,
)
engine.dispose()
@@ -538,8 +609,10 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
src_path="/downloads/Movie.2026.mkv",
planning_input=planning_input,
)
claimed = _claim(repository, admitted.task_id)
repository.checkpoint_plan(
task_id=admitted.task_id,
lease_token=claimed.lease_token,
input_fingerprint=planning_input.fingerprint,
checkpoint=_checkpoint(planning_input),
)
@@ -548,8 +621,18 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
).scalar_one()
row.checkpoint_version = 2
row.lease_expires_at = "2000-01-01 00:00:00.000000"
session.commit()
with pytest.raises(TransferPlanningStateError, match="版本"):
repository.list_recoverable()
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
repository.claim_task(
task_id=admitted.task_id,
owner_id="direct-corruption-test-worker",
lease_seconds=3600,
)
assert repository.claim_recoverable(
owner_id="batch-corruption-test-worker",
limit=1,
lease_seconds=3600,
) == []
engine.dispose()
+10 -6
View File
@@ -2,7 +2,7 @@ from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from sqlalchemy import create_engine
from sqlalchemy import create_engine, select
from sqlalchemy.orm import sessionmaker
from app.application.transfer import TransferAdmission, TransferQueueService
@@ -118,7 +118,8 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
"""真实仓储已提交后即使内存入队失败,任务也必须带原因留待恢复。"""
engine = create_engine(f"sqlite:///{tmp_path / 'durable-admission.db'}")
TransferPending.__table__.create(engine)
repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
factory = sessionmaker(bind=engine)
repository = TransactionalTransferAdmissionRepository(factory)
task = make_task(1)
service, _ = _service(
admit_task=lambda item: repository.admit(
@@ -135,10 +136,13 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
with pytest.raises(RuntimeError, match="queue closed"):
service.put(task, Mock())
admissions = repository.list_accepted()
assert len(admissions) == 1
assert admissions[0].task_id == task.admission_task_id
assert admissions[0].last_error == "queue closed"
with factory() as session:
pending = session.execute(
select(TransferPending).where(
TransferPending.task_id == task.admission_task_id
)
).scalar_one()
assert pending.last_error == "queue closed"
engine.dispose()
+7 -12
View File
@@ -1,12 +1,15 @@
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
from app.chain.transfer import JobManager, TransferChain
from app.chain.transfer import TransferChain
from app.domain.meta.metabase import MetaBase
from app.runtime.config import settings
from app.schemas.file import FileItem
from app.schemas.transfer import EpisodeFormat
from app.schemas.types import MediaType
from tests.test_transfer_job_manager import (
make_transfer_chain as make_base_transfer_chain,
)
class FakeMeta(MetaBase):
@@ -46,16 +49,8 @@ def make_transfer_chain() -> TransferChain:
"""
构造不启动后台线程的整理链实例
"""
chain = object.__new__(TransferChain)
chain.jobview = JobManager()
chain._media_exts = settings.RMT_MEDIAEXT
chain._subtitle_exts = settings.RMT_SUBEXT
chain._audio_exts = settings.RMT_AUDIOEXT
chain._allowed_exts = (
chain._media_exts + chain._audio_exts + chain._subtitle_exts
)
chain._success_target_files = {}
chain._scrape_batches = {}
chain = make_base_transfer_chain()
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
return chain
+422 -3
View File
@@ -15,13 +15,17 @@ from app.chain.transfer import TransferChain
from app.foundation.singleton import Singleton
from app.runtime.config import global_vars
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
from app.startup.initializers import transfer as transfer_initializer
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
chain = object.__new__(TransferChain)
chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads)
chain.runtime_config = SimpleNamespace(
transfer_threads=transfer_threads,
transfer_task_timeout=0,
)
chain._queue = queue.Queue()
chain._transfer_interval = 0.1
chain._threads = []
@@ -33,9 +37,59 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
chain._closing = False
chain._replay_thread = None
chain._replay_stop_event = threading.Event()
chain._recovery_wakeup_event = threading.Event()
chain._lease_heartbeat_thread = None
chain._lease_heartbeat_stop_event = threading.Event()
chain._worker_owner_id = "worker-owner"
chain._owned_leases = {}
chain._queued_lease_tokens = set()
admissions = MagicMock()
admissions.admit.side_effect = lambda **kwargs: TransferAdmission(
task_id="admitted-task",
storage=kwargs["storage"],
src_path=kwargs["src_path"],
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
)
admissions.claim_task.side_effect = lambda **kwargs: TransferAdmission(
task_id=kwargs["task_id"],
storage="local",
src_path="/downloads/test.mkv",
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
lease_owner=kwargs["owner_id"],
lease_token=f"lease-{kwargs['task_id']}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
admissions.discard_claimed.return_value = 1
admissions.release_claim.return_value = True
chain._transfer_admissions = admissions
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
return chain
def _claimed_admission(task: TransferTask, task_id: str) -> TransferAdmission:
"""构造属于测试进程 owner 的有效 claim 投影。"""
return TransferAdmission(
task_id=task_id,
storage=task.fileitem.storage,
src_path=task.fileitem.path,
state="accepted",
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
lease_owner="worker-owner",
lease_token=f"lease-{task_id}",
lease_expires_at="2026-08-27 10:02:00.000000",
heartbeat_at="2026-08-27 10:00:00.000000",
attempt_count=1,
)
def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None:
"""热更新应等待旧 worker 收敛,再启动使用独立停止信号的新一代。"""
chain = _build_chain(transfer_threads=1)
@@ -433,7 +487,11 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
created_at="2026-08-27 10:00:00",
updated_at="2026-08-27 10:00:00",
)
admissions.discard_task.side_effect = (
admissions.claim_task.return_value = _claimed_admission(
task,
"durable-task-id",
)
admissions.discard_claimed.side_effect = (
lambda **_kwargs: discarded.set() or 1
)
chain._transfer_admissions = admissions
@@ -471,7 +529,15 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
assert worker.is_alive() is False
assert task.admission_task_id == "durable-task-id"
admissions.discard_task.assert_called_once_with(task_id="durable-task-id")
admissions.claim_task.assert_called_once_with(
task_id="durable-task-id",
owner_id="worker-owner",
lease_seconds=120,
)
admissions.discard_claimed.assert_called_once_with(
task_id="durable-task-id",
lease_token="lease-durable-task-id",
)
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
@@ -527,6 +593,7 @@ def test_claimed_task_prevents_progress_settlement_before_active_registration()
def test_replay_has_single_owner_and_close_waits_for_it() -> None:
"""重复回放只保留一个线程,关闭会通知并等待该线程退出。"""
chain = _build_chain()
del chain._TransferChain__ensure_recovery_scheduler
replay_started = threading.Event()
replay_calls = []
@@ -548,3 +615,355 @@ def test_replay_has_single_owner_and_close_waits_for_it() -> None:
assert chain.close_workers(timeout_seconds=1) is True
assert replay_thread.is_alive() is False
assert chain._replay_thread is None
def test_recovered_worker_reuses_claimed_token_without_second_claim(
monkeypatch,
) -> None:
"""恢复任务携带 token 入队后,普通 worker 必须直接执行而非二次 claim。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/recovered.mkv",
type="file",
name="recovered.mkv",
basename="recovered",
extension="mkv",
))
task.bind_admission_task_id("recovered-task")
task.bind_execution_lease(
owner_id="worker-owner",
lease_token="lease-recovered-task",
)
chain._owned_leases = {
"recovered-task": ("lease-recovered-task", time.monotonic() + 120)
}
chain.jobview = MagicMock()
chain.jobview.pending_total.return_value = 1
chain._finish_scrape_batch_task = MagicMock()
chain._progress = MagicMock()
chain._active_tasks = 0
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 1
stop_event = threading.Event()
def complete_recovery(*, task, callback):
"""模拟恢复任务成功提交检查点并让 worker 在本项后退出。"""
del callback
task.bind_plan_checkpoint(MagicMock())
stop_event.set()
return True, ""
chain._TransferChain__handle_transfer = complete_recovery
chain._queue.put(TransferQueue(task=task))
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
worker = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
worker.start()
worker.join(timeout=1)
assert worker.is_alive() is False
chain._transfer_admissions.claim_task.assert_not_called()
chain._transfer_admissions.discard_claimed.assert_called_once_with(
task_id="recovered-task",
lease_token="lease-recovered-task",
)
def test_heartbeat_refreshes_current_token_and_forgets_lost_lease() -> None:
"""heartbeat 成功应刷新本地期限,CAS 拒绝后必须立即停止本地推进资格。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/heartbeat.mkv",
type="file",
))
current = _claimed_admission(task, "heartbeat-task")
initial_deadline = time.monotonic() + 1
chain._owned_leases = {
"heartbeat-task": ("lease-heartbeat-task", initial_deadline)
}
chain._transfer_admissions.heartbeat.return_value = current
chain._TransferChain__heartbeat_owned_leases()
assert chain._owned_leases["heartbeat-task"][1] > initial_deadline
chain._transfer_admissions.heartbeat.return_value = None
chain._TransferChain__heartbeat_owned_leases()
assert "heartbeat-task" not in chain._owned_leases
def test_close_timeout_keeps_heartbeat_alive_until_blocked_worker_converges() -> None:
"""阻塞 worker 未退出时关闭不得停止 heartbeat 或允许租约过期接管。"""
chain = _build_chain()
worker_release = threading.Event()
worker = threading.Thread(
target=worker_release.wait,
name="transfer-blocked-owner",
daemon=True,
)
heartbeat = threading.Thread(
target=chain._lease_heartbeat_stop_event.wait,
name="transfer-heartbeat-owner",
daemon=True,
)
chain._threads = [worker]
chain._lease_heartbeat_thread = heartbeat
chain._owned_leases = {
"blocked-task": ("blocked-token", time.monotonic() + 120)
}
worker.start()
heartbeat.start()
assert chain.close_workers(timeout_seconds=0.01) is False
assert heartbeat.is_alive() is True
assert chain._lease_heartbeat_stop_event.is_set() is False
worker_release.set()
assert chain.close_workers(timeout_seconds=1) is True
assert worker.is_alive() is False
assert heartbeat.is_alive() is False
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="blocked-task",
lease_token="blocked-token",
error="整理宿主关闭,释放未结算任务租约",
)
def test_worker_reports_failed_settlement_without_skipping_queue_bookkeeping(
monkeypatch,
) -> None:
"""终态 CAS=0 必须计为失败,同时仍完成 task_done 与 active 归零。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/stale.mkv",
type="file",
name="stale.mkv",
basename="stale",
extension="mkv",
))
chain.jobview = MagicMock()
chain.jobview.add_task.return_value = True
chain.jobview.pending_total.return_value = 1
chain._register_scrape_batch_task = MagicMock()
chain._finish_scrape_batch_task = MagicMock()
chain._progress = MagicMock()
chain._active_tasks = 0
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 0
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
stop_event = threading.Event()
def complete_with_stale_lease(*, task, callback):
"""模拟文件副作用完成后终态 token 已被新 owner 接管。"""
del callback
task.bind_plan_checkpoint(MagicMock())
stop_event.set()
return True, ""
chain._TransferChain__handle_transfer = complete_with_stale_lease
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
assert chain.put_to_queue(task) is True
worker = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
worker.start()
worker.join(timeout=1)
assert worker.is_alive() is False
assert chain._active_tasks == 0
assert chain._fail_num == 1
assert chain._queue.unfinished_tasks == 0
def test_failed_claim_release_waits_for_fixed_recovery_poll() -> None:
"""失败释放不得即时唤醒恢复线程,避免确定性错误形成热重试。"""
chain = _build_chain()
del chain._TransferChain__ensure_recovery_scheduler
chain._RECOVERY_POLL_INTERVAL_SECONDS = 0.05
chain._TransferChain__replay_pending = MagicMock()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/retry-later.mkv",
type="file",
))
task.bind_admission_task_id("retry-later")
task.bind_execution_lease(
owner_id="worker-owner",
lease_token="retry-token",
)
chain._owned_leases = {
"retry-later": ("retry-token", time.monotonic() + 120)
}
assert chain._TransferChain__release_task_claim(
task,
error="planning failed",
) is True
assert chain._recovery_wakeup_event.is_set() is False
assert chain._replay_thread is not None
time.sleep(0.01)
chain._TransferChain__replay_pending.assert_not_called()
deadline = time.monotonic() + 0.5
while (
not chain._TransferChain__replay_pending.called
and time.monotonic() < deadline
):
time.sleep(0.01)
chain._TransferChain__replay_pending.assert_called()
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="retry-later",
lease_token="retry-token",
error="planning failed",
)
assert chain.close_workers(timeout_seconds=1) is True
def test_worker_fenced_releases_lost_lease_and_completes_queue_bookkeeping(
monkeypatch,
) -> None:
"""本地租约失效时仍尝试 token CAS release,并完整结算内存队列。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/lost-lease.mkv",
type="file",
))
task.bind_admission_task_id("lost-lease")
task.bind_execution_lease(
owner_id="worker-owner",
lease_token="lost-token",
)
chain.jobview = MagicMock()
chain._finish_scrape_batch_task = MagicMock()
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
stop_event = threading.Event()
chain._transfer_admissions.release_claim.side_effect = (
lambda **_kwargs: stop_event.set() or True
)
chain._queue.put(TransferQueue(task=task))
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
worker = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
worker.start()
worker.join(timeout=1)
assert worker.is_alive() is False
chain._transfer_admissions.release_claim.assert_called_once_with(
task_id="lost-lease",
lease_token="lost-token",
error="整理任务租约已经失效:lost-lease",
)
assert chain._queue.unfinished_tasks == 0
assert chain._recovery_wakeup_event.is_set() is False
def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) -> None:
"""终态 CAS 被拒绝时不得写成功历史、事件或通知。"""
chain = _build_chain()
task = TransferTask(fileitem=FileItem(
storage="local",
path="/downloads/fenced-success.mkv",
type="file",
name="fenced-success.mkv",
))
chain.jobview = MagicMock()
chain.jobview.add_task.return_value = True
chain.jobview.pending_total.return_value = 1
chain._register_scrape_batch_task = MagicMock()
chain._finish_scrape_batch_task = MagicMock()
chain._progress = MagicMock()
chain._active_tasks = 0
chain._processed_num = 0
chain._fail_num = 0
chain._total_num = 0
chain._transfer_admissions.discard_claimed.return_value = 0
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
success_callback = MagicMock(return_value=(True, ""))
chain._TransferChain__default_callback = success_callback
stop_event = threading.Event()
def complete_with_success(*, task, callback):
"""模拟文件成功后进入受 durable 终态保护的回调。"""
task.bind_plan_checkpoint(MagicMock())
stop_event.set()
return callback(
task,
TransferInfo(success=True, fileitem=task.fileitem),
)
chain._TransferChain__handle_transfer = complete_with_success
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
assert chain.put_to_queue(task) is True
worker = threading.Thread(
target=chain._TransferChain__start_transfer,
args=(stop_event,),
daemon=True,
)
worker.start()
worker.join(timeout=1)
assert worker.is_alive() is False
success_callback.assert_not_called()
chain.jobview.fail_unfinished_task.assert_called_once_with(task)
assert chain._fail_num == 1
assert chain._queue.unfinished_tasks == 0
def test_close_release_db_block_respects_deadline_and_keeps_heartbeat() -> None:
"""关闭租约释放被数据库阻塞时应按预算返回,并继续 heartbeat。"""
chain = _build_chain()
release_started = threading.Event()
release_db = threading.Event()
heartbeat = threading.Thread(
target=chain._lease_heartbeat_stop_event.wait,
name="transfer-heartbeat-release-test",
daemon=True,
)
chain._lease_heartbeat_thread = heartbeat
chain._owned_leases = {
"blocked-release": ("blocked-token", time.monotonic() + 120)
}
def block_release(**_kwargs):
"""模拟数据库锁住 release_claim,直到测试显式放行。"""
release_started.set()
release_db.wait()
return True
chain._transfer_admissions.release_claim.side_effect = block_release
heartbeat.start()
started_at = time.monotonic()
assert chain.close_workers(timeout_seconds=0.01) is False
assert time.monotonic() - started_at < 0.5
assert release_started.is_set()
assert heartbeat.is_alive() is True
assert chain._lease_heartbeat_stop_event.is_set() is False
assert chain._lease_release_thread is not None
assert chain._lease_release_thread.is_alive() is True
release_db.set()
assert chain.close_workers(timeout_seconds=1) is True
assert heartbeat.is_alive() is False
assert chain._lease_release_thread is None