diff --git a/app/application/transfer.py b/app/application/transfer.py index 01550cd89..5d01871ca 100644 --- a/app/application/transfer.py +++ b/app/application/transfer.py @@ -627,6 +627,14 @@ class TransferPlanningStateError(RuntimeError): """计划检查点无法从当前持久状态推进时抛出的状态错误。""" +class TransferAdmissionProjectionError(RuntimeError): + """持久登记无法安全恢复为应用层整理准入投影时抛出的错误。""" + + +class TransferLeaseLostError(TransferPlanningStateError): + """整理 worker 已失去持久租约、不得继续推进任务时抛出的错误。""" + + class TransferTask(OptionalMediaIdentityMixin, BaseModel): """ 文件整理任务。 @@ -661,6 +669,8 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): _planning_input: Optional[TransferPlanningInput] = PrivateAttr(default=None) _plan_checkpoint: Optional[TransferPlanCheckpoint] = PrivateAttr(default=None) _planning_context_restored: bool = PrivateAttr(default=False) + _lease_owner: Optional[str] = PrivateAttr(default=None) + _lease_token: Optional[str] = PrivateAttr(default=None) @property def admission_task_id(self) -> Optional[str]: @@ -698,6 +708,23 @@ class TransferTask(OptionalMediaIdentityMixin, BaseModel): """标记领域上下文已离线恢复，禁止旧流程再次在线补充。""" self._planning_context_restored = True + @property + def lease_owner(self) -> Optional[str]: + """返回当前任务绑定的进程级 worker owner。""" + return self._lease_owner + + @property + def lease_token(self) -> Optional[str]: + """返回当前任务绑定的持久租约令牌。""" + return self._lease_token + + def bind_execution_lease(self, *, owner_id: str, lease_token: str) -> None: + """绑定持久 claim 结果，且不改变插件可见的旧任务序列化字段。""" + if not owner_id or not lease_token: + raise ValueError("整理执行租约缺少 owner 或 token") + self._lease_owner = owner_id + self._lease_token = lease_token + def to_dict(self): """ 返回字典。 @@ -743,6 +770,11 @@ class TransferAdmission: input_fingerprint: Optional[str] = None planning_input: Optional[TransferPlanningInput] = None checkpoint: Optional[TransferPlanCheckpoint] = None + lease_owner: Optional[str] = None + lease_token: Optional[str] = None + lease_expires_at: Optional[str] = None + heartbeat_at: Optional[str] = None + attempt_count: int = 0 class TransferAdmissionRepository(Protocol): @@ -758,10 +790,6 @@ class TransferAdmissionRepository(Protocol): """按规划输入幂等登记源文件并返回稳定任务身份。""" ... - def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]: - """按登记顺序返回等待恢复或执行的任务。""" - ... - def record_enqueue_failure(self, *, task_id: str, error: str) -> None: """记录内存队列接收失败，保留任务供后续恢复。""" ... @@ -770,22 +798,61 @@ class TransferAdmissionRepository(Protocol): self, *, task_id: str, + lease_token: str, input_fingerprint: str, checkpoint: TransferPlanCheckpoint, ) -> TransferAdmission: - """原子保存完整计划并将匹配输入的任务推进到已规划。""" + """按有效租约原子保存完整计划并推进匹配输入的任务。""" ... - def record_planning_failure(self, *, task_id: str, error: str) -> None: - """记录规划失败但保留接纳状态供下次恢复重试。""" + def record_planning_failure( + self, *, task_id: str, lease_token: str, error: str + ) -> None: + """按有效租约记录规划失败，保留业务状态供恢复重试。""" ... - def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]: - """按登记顺序返回接纳或已规划的可恢复任务。""" + def claim_task( + self, + *, + task_id: str, + owner_id: str, + lease_seconds: int, + ) -> Optional[TransferAdmission]: + """为指定任务取得唯一执行租约；已有有效租约时返回 None。""" ... - def discard_task(self, *, task_id: str) -> int: - """按稳定任务身份删除已经到达终态的登记。""" + def claim_recoverable( + self, + *, + owner_id: str, + limit: int, + lease_seconds: int, + ) -> list[TransferAdmission]: + """按登记顺序原子取得可恢复任务的唯一执行租约。""" + ... + + def heartbeat( + self, + *, + task_id: str, + lease_token: str, + lease_seconds: int, + ) -> Optional[TransferAdmission]: + """仅为当前且未过期的 token 延长租约；过期 token 不得复活。""" + ... + + def release_claim( + self, + *, + task_id: str, + lease_token: str, + error: Optional[str] = None, + ) -> bool: + """按 token 释放当前 claim，并可记录可恢复错误。""" + ... + + def discard_claimed(self, *, task_id: str, lease_token: str) -> int: + """仅允许当前 lease owner 删除已经到达终态的登记。""" ... diff --git a/app/chain/transfer.py b/app/chain/transfer.py index df64c1630..0bad6bead 100755 --- a/app/chain/transfer.py +++ b/app/chain/transfer.py @@ -46,6 +46,7 @@ from app.application.transfer import ( TransferAdmission, TransferFailureNotification, TransferFailureNotificationAggregator, + TransferLeaseLostError, TransferPlanCheckpoint, TransferPlanningInput, TransferPlanningStateError, @@ -146,6 +147,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo _WORKER_RESTART_TIMEOUT_SECONDS = 30.0 _WORKER_CLOSE_TIMEOUT_SECONDS = 30.0 _QUEUE_STOP_SENTINEL = object() + _WORKER_LEASE_SECONDS = 120 + _LEASE_HEARTBEAT_INTERVAL_SECONDS = 30.0 + _RECOVERY_POLL_INTERVAL_SECONDS = 15.0 + _RECOVERY_CLAIM_LIMIT = 100 @staticmethod def _transfer_result_payload( @@ -254,9 +259,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._worker_lifecycle_lock = threading.RLock() self._worker_state_lock = threading.RLock() self._closing = False - # pending 回放同样由整理链持有，关闭时可阻止继续处理下一条登记 + # 恢复调度与租约续期均由整理链持有，关闭时与 worker 一并收敛。 + self._worker_owner_id = uuid.uuid4().hex + self._owned_leases: Dict[str, Tuple[str, float]] = {} + self._queued_lease_tokens: set[Tuple[str, str]] = set() self._replay_thread: Optional[threading.Thread] = None self._replay_stop_event = threading.Event() + self._recovery_wakeup_event = threading.Event() + self._lease_heartbeat_thread: Optional[threading.Thread] = None + self._lease_heartbeat_stop_event = threading.Event() + self._lease_release_thread: Optional[threading.Thread] = None self._active_tasks = 0 self._processed_num = 0 self._fail_num = 0 @@ -366,6 +378,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo :param timeout_seconds: 生命周期锁、worker 与回放线程共享的最大等待秒数 :return: 全部后台线程均已收敛时返回 True，否则返回 False """ + self.__ensure_lease_runtime_state() deadline = time.monotonic() + max(0.0, timeout_seconds) if not self.__acquire_worker_lifecycle_lock(deadline): logger.error( @@ -378,7 +391,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._closing = True worker_threads = self.__request_worker_stop() replay_thread = self._replay_thread + heartbeat_thread = self._lease_heartbeat_thread self._replay_stop_event.set() + self._recovery_wakeup_event.set() alive_workers = self.__join_threads(worker_threads, deadline) alive_replays = self.__join_threads( @@ -387,7 +402,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo with self._worker_state_lock: self._threads = [] self._retiring_threads = alive_workers - if replay_thread and not alive_replays and self._replay_thread is replay_thread: + if ( + replay_thread + and replay_thread not in alive_replays + and self._replay_thread is replay_thread + ): self._replay_thread = None alive_threads = [*alive_workers, *alive_replays] @@ -398,6 +417,38 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ", ".join(thread.name for thread in alive_threads), ) return False + with self._worker_state_lock: + release_thread = self.__start_lease_release_owner_locked( + error="整理宿主关闭，释放未结算任务租约" + ) + alive_releases = self.__join_threads( + [release_thread] if release_thread else [], deadline + ) + if alive_releases: + logger.error( + "整理租约释放线程未在 %.1f 秒内收敛，heartbeat 保持运行：%s", + max(0.0, timeout_seconds), + ", ".join(thread.name for thread in alive_releases), + ) + return False + with self._worker_state_lock: + if self._lease_release_thread is release_thread: + self._lease_release_thread = None + self._lease_heartbeat_stop_event.set() + alive_heartbeats = self.__join_threads( + [heartbeat_thread] if heartbeat_thread else [], deadline + ) + if heartbeat_thread and not alive_heartbeats: + with self._worker_state_lock: + if self._lease_heartbeat_thread is heartbeat_thread: + self._lease_heartbeat_thread = None + if alive_heartbeats: + logger.error( + "整理租约续期线程未在 %.1f 秒内收敛：%s", + max(0.0, timeout_seconds), + ", ".join(thread.name for thread in alive_heartbeats), + ) + return False logger.info("文件整理 worker 与待处理回放线程已关闭") return True finally: @@ -885,6 +936,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo if self._closing: logger.warning("文件整理链已关闭，拒绝新的队列任务") return False + if isinstance(task.lease_token, str) and task.lease_token: + return self.__enqueue_claimed_task(task) return self._transfer_queue_service().put(task, self.__default_callback) def _transfer_queue_service(self) -> TransferQueueService: @@ -902,39 +955,369 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo def replay_pending(self) -> None: """ - 回放上次进程退出时仍未整理完的文件。 + 启动唯一恢复调度 owner，并唤醒一次即时恢复扫描。 - 在后台线程执行：回放要 stat 源文件，而启动期挂载可能尚未就绪甚至处于 - 挂死状态，同步执行会把整个启动流程堵住。 + 启动回放、同进程入队补偿和租约过期接管都经由这个入口。调度线程只负责 + claim 和重新入队；实际业务仍由普通整理 worker 执行。 """ + self.__ensure_recovery_scheduler(immediate=True) + + def __ensure_recovery_scheduler(self, *, immediate: bool) -> None: + """确保唯一恢复调度 owner 存在，并只为显式请求执行即时扫描。""" + self.__ensure_lease_runtime_state() with self._worker_state_lock: if self._closing: logger.info("文件整理链正在关闭，跳过待处理文件回放") return + self.__start_lease_heartbeat_owner_locked() if self._replay_thread and self._replay_thread.is_alive(): - logger.info("待处理文件回放已在运行，跳过重复启动") + if immediate: + self._recovery_wakeup_event.set() return - stop_event = threading.Event() thread = threading.Thread( target=self.__run_replay_pending, - args=(stop_event,), + args=(self._replay_stop_event, immediate), name="MoviePilot-TransferReplay", daemon=True, ) - self._replay_stop_event = stop_event self._replay_thread = thread + if immediate: + self._recovery_wakeup_event.set() # 在状态锁内启动，避免 close_workers 看到尚未 start 的线程后错误 join。 thread.start() - def __run_replay_pending(self, stop_event: threading.Event) -> None: - """执行一次受控回放，并在自然结束后释放当前线程句柄。""" + def __run_replay_pending( + self, + stop_event: threading.Event, + initial_immediate: bool = True, + ) -> None: + """按初次扫描意图持续恢复，失败兜底新建 owner 时先等待固定轮询。""" try: - self.__replay_pending(stop_event) + if not initial_immediate: + self._recovery_wakeup_event.wait( + timeout=self._RECOVERY_POLL_INTERVAL_SECONDS + ) + while not stop_event.is_set(): + self._recovery_wakeup_event.clear() + self.__replay_pending(stop_event) + self._recovery_wakeup_event.wait( + timeout=self._RECOVERY_POLL_INTERVAL_SECONDS + ) finally: with self._worker_state_lock: if self._replay_thread is threading.current_thread(): self._replay_thread = None + def __ensure_lease_runtime_state(self) -> None: + """为绕过构造器的兼容调用补齐进程 owner 与租约线程状态。""" + if not hasattr(self, "_worker_owner_id"): + self._worker_owner_id = uuid.uuid4().hex + if not hasattr(self, "_owned_leases"): + self._owned_leases = {} + if not hasattr(self, "_queued_lease_tokens"): + self._queued_lease_tokens = set() + if not hasattr(self, "_recovery_wakeup_event"): + self._recovery_wakeup_event = threading.Event() + if not hasattr(self, "_lease_heartbeat_thread"): + self._lease_heartbeat_thread = None + if not hasattr(self, "_lease_heartbeat_stop_event"): + self._lease_heartbeat_stop_event = threading.Event() + if not hasattr(self, "_lease_release_thread"): + self._lease_release_thread = None + if not hasattr(self, "_replay_thread"): + self._replay_thread = None + if not hasattr(self, "_replay_stop_event"): + self._replay_stop_event = threading.Event() + if not hasattr(self, "_worker_state_lock"): + self._worker_state_lock = threading.RLock() + if not hasattr(self, "_closing"): + self._closing = False + + def __start_lease_heartbeat_owner_locked(self) -> None: + """在状态锁内确保当前进程只有一个租约续期线程。""" + heartbeat_thread = self._lease_heartbeat_thread + if heartbeat_thread and heartbeat_thread.is_alive(): + return + heartbeat_thread = threading.Thread( + target=self.__run_lease_heartbeat, + args=(self._lease_heartbeat_stop_event,), + name="MoviePilot-TransferLeaseHeartbeat", + daemon=True, + ) + self._lease_heartbeat_thread = heartbeat_thread + heartbeat_thread.start() + + def __ensure_lease_heartbeat_owner(self) -> None: + """按需启动进程级租约续期 owner，供启动前到达的普通任务使用。""" + self.__ensure_lease_runtime_state() + with self._worker_state_lock: + if not self._closing: + self.__start_lease_heartbeat_owner_locked() + + def __run_lease_heartbeat(self, stop_event: threading.Event) -> None: + """按固定周期续期本进程已 claim 且尚未结算的任务。""" + try: + while not stop_event.wait(self._LEASE_HEARTBEAT_INTERVAL_SECONDS): + self.__heartbeat_owned_leases() + finally: + with self._worker_state_lock: + if self._lease_heartbeat_thread is threading.current_thread(): + self._lease_heartbeat_thread = None + + def __heartbeat_owned_leases(self) -> None: + """经 Application Port 续期所有排队中或执行中的任务租约。""" + self.__ensure_lease_runtime_state() + with self._worker_state_lock: + owned_leases = list(self._owned_leases.items()) + for task_id, (lease_token, deadline) in owned_leases: + try: + admission = self._transfer_admissions.heartbeat( + task_id=task_id, + lease_token=lease_token, + lease_seconds=self._WORKER_LEASE_SECONDS, + ) + except Exception as err: + if time.monotonic() >= deadline: + self.__forget_owned_lease(task_id, lease_token) + logger.error( + "整理任务租约续期持续失败并已超过本地期限：%s - %s", + task_id, + err, + ) + else: + logger.error(f"整理任务租约续期失败：{task_id} - {err}") + continue + if ( + admission is None + or admission.lease_owner != self._worker_owner_id + or admission.lease_token != lease_token + ): + self.__forget_owned_lease(task_id, lease_token) + logger.error(f"整理任务租约已失效或被接管：{task_id}") + continue + with self._worker_state_lock: + current = self._owned_leases.get(task_id) + if current and current[0] == lease_token: + self._owned_leases[task_id] = ( + lease_token, + time.monotonic() + self._WORKER_LEASE_SECONDS, + ) + + def __bind_claimed_admission( + self, + task: TransferTask, + admission: TransferAdmission, + ) -> None: + """校验仓储 claim 投影并把 token 私有绑定到执行任务。""" + if ( + admission.task_id != task.admission_task_id + ): + raise TransferLeaseLostError( + f"整理任务 claim 投影无效：{admission.task_id}" + ) + self.__register_claimed_admission(admission) + assert admission.lease_owner is not None + assert admission.lease_token is not None + task.bind_execution_lease( + owner_id=admission.lease_owner, + lease_token=admission.lease_token, + ) + + def __register_claimed_admission( + self, + admission: TransferAdmission, + ) -> None: + """把仓储 claim 加入续期集合，覆盖恢复构造阶段可能发生的同步 I/O。""" + if ( + admission.lease_owner != self._worker_owner_id + or not admission.lease_token + ): + raise TransferLeaseLostError( + f"整理任务 claim 投影无效：{admission.task_id}" + ) + with self._worker_state_lock: + self._owned_leases[admission.task_id] = ( + admission.lease_token, + time.monotonic() + self._WORKER_LEASE_SECONDS, + ) + self.__ensure_lease_heartbeat_owner() + + def __forget_owned_lease(self, task_id: str, lease_token: str) -> None: + """仅在 token 仍匹配时移除本进程租约镜像，避免删掉新接管记录。""" + with self._worker_state_lock: + current = self._owned_leases.get(task_id) + if current and current[0] == lease_token: + self._owned_leases.pop(task_id, None) + self._queued_lease_tokens.discard((task_id, lease_token)) + + def __is_claimed_task_enqueued(self, task_id: str, lease_token: str) -> bool: + """返回指定 claim 是否已经成功进入普通 worker 队列。""" + with self._worker_state_lock: + return (task_id, lease_token) in self._queued_lease_tokens + + def __owns_lease(self, task_id: str, lease_token: Optional[str]) -> bool: + """返回本地续期镜像是否仍持有指定 token。""" + if not lease_token: + return False + with self._worker_state_lock: + current = self._owned_leases.get(task_id) + return bool(current and current[0] == lease_token) + + def __assert_owned_lease(self, task: TransferTask) -> None: + """拒绝无 token、已过本地期限或已被 heartbeat 判失效的任务推进。""" + if task.preview: + return + task_id = task.admission_task_id + lease_token = task.lease_token + if ( + not task_id + or not lease_token + or task.lease_owner != self._worker_owner_id + ): + raise TransferLeaseLostError("整理任务缺少当前进程的有效执行租约") + with self._worker_state_lock: + current = self._owned_leases.get(task_id) + if ( + current is None + or current[0] != lease_token + or current[1] <= time.monotonic() + ): + if current and current[0] == lease_token: + self._owned_leases.pop(task_id, None) + raise TransferLeaseLostError(f"整理任务租约已经失效：{task_id}") + + def __claim_task_for_execution(self, task: TransferTask) -> None: + """让普通队列任务在业务执行前取得唯一租约，恢复任务复用既有 token。""" + if task.preview: + return + self.__ensure_lease_runtime_state() + if task.lease_token: + self.__assert_owned_lease(task) + return + if not task.admission_task_id: + admitted = self.__admit_transfer(task) + task.bind_admission_task_id(admitted.task_id) + task_id = task.admission_task_id + if task_id is None: + raise TransferLeaseLostError("整理任务准入后仍缺少 durable 身份") + claimed = self._transfer_admissions.claim_task( + task_id=task_id, + owner_id=self._worker_owner_id, + lease_seconds=self._WORKER_LEASE_SECONDS, + ) + if claimed is None: + raise TransferLeaseLostError( + f"整理任务已由其他 worker claim：{task_id}" + ) + try: + self.__bind_claimed_admission(task, claimed) + except Exception as err: + self.__release_admission_claim(claimed, error=str(err)) + raise + + def __release_task_claim( + self, + task: TransferTask, + *, + error: Optional[str] = None, + ) -> bool: + """按 task token 释放未到终态的 claim，并按固定轮询等待恢复。""" + task_id = task.admission_task_id if task else None + lease_token = task.lease_token if task else None + if not task_id or not lease_token: + return False + try: + return self._transfer_admissions.release_claim( + task_id=task_id, + lease_token=lease_token, + error=error, + ) + except Exception as err: + logger.error(f"释放整理任务租约失败：{task_id} - {err}") + return False + finally: + self.__forget_owned_lease(task_id, lease_token) + self.__ensure_recovery_scheduler(immediate=False) + + def __release_admission_claim( + self, + admission: TransferAdmission, + *, + error: Optional[str] = None, + ) -> None: + """释放尚未绑定或成功入队的恢复 claim。""" + if not admission.lease_token: + return + try: + self._transfer_admissions.release_claim( + task_id=admission.task_id, + lease_token=admission.lease_token, + error=error, + ) + except Exception as err: + logger.error(f"释放恢复任务租约失败：{admission.task_id} - {err}") + finally: + self.__forget_owned_lease(admission.task_id, admission.lease_token) + + def __release_all_owned_leases(self, *, error: str) -> None: + """在执行 owner 全部收敛后释放剩余 claim，避免关停后等待租约自然过期。""" + with self._worker_state_lock: + owned_leases = list(self._owned_leases.items()) + for task_id, (lease_token, _deadline) in owned_leases: + try: + self._transfer_admissions.release_claim( + task_id=task_id, + lease_token=lease_token, + error=error, + ) + except Exception as err: + logger.error(f"关闭时释放整理任务租约失败：{task_id} - {err}") + finally: + self.__forget_owned_lease(task_id, lease_token) + + def __start_lease_release_owner_locked( + self, *, error: str + ) -> Optional[threading.Thread]: + """启动并持有唯一租约释放线程，使同步数据库阻塞不突破关闭预算。""" + release_thread = self._lease_release_thread + if release_thread is not None: + return release_thread + if not self._owned_leases: + return None + release_thread = threading.Thread( + target=self.__release_all_owned_leases, + kwargs={"error": error}, + name="MoviePilot-TransferLeaseRelease", + daemon=True, + ) + self._lease_release_thread = release_thread + release_thread.start() + return release_thread + + def __enqueue_claimed_task(self, task: TransferTask) -> bool: + """把已 claim 的恢复任务送入普通队列，禁止再次准入或二次 claim。""" + self.__assert_owned_lease(task) + if not self.__put_to_jobview(task): + return False + try: + self._register_scrape_batch_task(task) + assert task.admission_task_id is not None + assert task.lease_token is not None + with self._worker_state_lock: + self._queued_lease_tokens.add( + (task.admission_task_id, task.lease_token) + ) + self._queue.put( + TransferQueue(task=task, callback=self.__default_callback) + ) + except Exception as err: + try: + self.__record_enqueue_failure(task, err) + finally: + self.jobview.remove_task(task.fileitem) + raise + return True + def __replay_pending( self, stop_event: Optional[threading.Event] = None ) -> None: @@ -948,17 +1331,35 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo stop_event = stop_event or threading.Event() if stop_event.is_set(): return + self.__ensure_lease_runtime_state() try: - pendings = self._transfer_admissions.list_recoverable() + pendings = self._transfer_admissions.claim_recoverable( + owner_id=self._worker_owner_id, + limit=self._RECOVERY_CLAIM_LIMIT, + lease_seconds=self._WORKER_LEASE_SECONDS, + ) except Exception as err: logger.error(f"读取待整理文件登记失败：{err}") return if not pendings: return + try: + for admission in pendings: + self.__register_claimed_admission(admission) + except Exception as err: + logger.error(f"登记恢复任务租约失败：{err}") + for claimed in pendings: + self.__release_admission_claim(claimed, error=str(err)) + return logger.info(f"发现 {len(pendings)} 个上次未整理完的文件，正在重新送入整理链 ...") replayed = 0 - for admission in pendings: + for index, admission in enumerate(pendings): if stop_event.is_set(): + for unprocessed in pendings[index:]: + self.__release_admission_claim( + unprocessed, + error="整理宿主关闭，恢复任务尚未入队", + ) break storage = admission.storage src_path = admission.src_path @@ -970,17 +1371,50 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) # stat 等同步 I/O 返回后重新检查，关闭期间不得注销尚未完成的登记。 if stop_event.is_set(): + self.__release_admission_claim( + admission, + error="整理宿主关闭，恢复任务尚未入队", + ) + for unprocessed in pendings[index + 1:]: + self.__release_admission_claim( + unprocessed, + error="整理宿主关闭，恢复任务尚未入队", + ) break if not fileitem: if should_discard: # 源文件确认已消失，注销登记避免每次启动重复回放 - self._transfer_admissions.discard_task( - task_id=admission.task_id + lease_token = admission.lease_token + if lease_token is None: + raise TransferLeaseLostError( + f"恢复任务缺少 lease token：{admission.task_id}" + ) + discarded = self._transfer_admissions.discard_claimed( + task_id=admission.task_id, + lease_token=lease_token, + ) + if not discarded: + logger.warning( + f"恢复任务终态注销被 CAS 拒绝：{admission.task_id}" + ) + self.__forget_owned_lease( + admission.task_id, + lease_token, + ) + else: + self.__release_admission_claim( + admission, + error="恢复源文件暂时不可读取", ) continue if admission.checkpoint: if self.__queue_planned_replay(fileitem, admission): replayed += 1 + else: + self.__release_admission_claim( + admission, + error="恢复任务未进入内存队列", + ) continue planning_input = admission.planning_input if planning_input and ( @@ -990,6 +1424,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ): if self.__queue_accepted_replay(fileitem, admission): replayed += 1 + else: + self.__release_admission_claim( + admission, + error="恢复任务未进入内存队列", + ) continue replay_kwargs = self.__build_replay_kwargs(planning_input) self._execute_transfer( @@ -997,9 +1436,21 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo recovery_admission=admission, **replay_kwargs, ) - replayed += 1 + assert admission.lease_token is not None + if self.__is_claimed_task_enqueued( + admission.task_id, + admission.lease_token, + ): + replayed += 1 + else: + self.__release_admission_claim( + admission, + error="旧恢复入口未产生可执行队列任务", + ) except Exception as err: logger.error(f"回放待整理文件失败：{storage}:{src_path} - {err}") + if self.__owns_lease(admission.task_id, admission.lease_token): + self.__release_admission_claim(admission, error=str(err)) if stop_event.is_set(): logger.info( "待整理文件回放收到关闭请求，已送入 %s 个文件，其余登记保持待处理", @@ -1116,6 +1567,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo preview=False, ) task.bind_admission_task_id(admission.task_id) + self.__bind_claimed_admission(task, admission) task.bind_planning_input(planning_input) if planning_input.mediainfo: task.mark_planning_context_restored() @@ -1168,6 +1620,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo preview=False, ) task.bind_admission_task_id(admission.task_id) + self.__bind_claimed_admission(task, admission) task.bind_planning_input(checkpoint.planning_input) task.bind_plan_checkpoint(checkpoint) self.__restore_planned_task(task) @@ -1442,11 +1895,17 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo error: object, ) -> None: """记录 checkpoint 前失败，保留 accepted 任务供后续重新规划。""" - if task.preview or task.plan_checkpoint or not task.admission_task_id: + if ( + task.preview + or task.plan_checkpoint + or not task.admission_task_id + or not task.lease_token + ): return try: self._transfer_admissions.record_planning_failure( task_id=task.admission_task_id, + lease_token=task.lease_token, error=str(error), ) except Exception as record_error: @@ -1488,9 +1947,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo """先提交冻结 provider 调用，空结果时再提交并执行宿主计划。""" planning_input = task.planning_input or self.__build_planning_input(task) task.bind_planning_input(planning_input) - if not task.preview and not task.admission_task_id: - admission = self.__admit_transfer(task) - task.bind_admission_task_id(admission.task_id) + if not task.preview: + self.__claim_task_for_execution(task) + self.__assert_owned_lease(task) checkpoint = task.plan_checkpoint if checkpoint is None: @@ -1540,6 +1999,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo raise self.__restore_planned_task(task) + self.__assert_owned_lease(task) legacy_result = self.__execute_legacy_transfer_providers( task, @@ -1571,6 +2031,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__record_checkpoint_failure(task, error) raise + self.__assert_owned_lease(task) result = self.execute_transfer_plan( checkpoint, meta=task.meta, @@ -1623,8 +2084,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo return checkpoint if not task.admission_task_id: raise RuntimeError("整理计划提交前缺少持久任务身份") + self.__assert_owned_lease(task) + assert task.lease_token is not None persisted = self._transfer_admissions.checkpoint_plan( task_id=task.admission_task_id, + lease_token=task.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) @@ -1638,11 +2102,12 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo error: Exception, ) -> None: """记录当前 checkpoint 阶段失败并保留原状态供重启恢复。""" - if task.preview or not task.admission_task_id: + if task.preview or not task.admission_task_id or not task.lease_token: return try: self._transfer_admissions.record_planning_failure( task_id=task.admission_task_id, + lease_token=task.lease_token, error=str(error), ) setattr(error, "_transfer_planning_failure_recorded", True) @@ -1790,6 +2255,7 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) except Exception as error: logger.error(f"旧整理兼容命令执行失败：{error}") + self.__release_task_claim(task, error=str(error)) return TransferInfo( success=False, message=str(error), @@ -1797,7 +2263,16 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo fail_list=[fileitem.path], transfer_type=transfer_type, ) - self.__discard_pending(task) + if not self.__discard_pending(task): + message = "旧整理兼容命令已执行，但 durable 终态结算失去租约" + logger.error(message) + return TransferInfo( + success=False, + message=message, + fileitem=fileitem, + fail_list=[fileitem.path], + transfer_type=transfer_type, + ) return result def __record_enqueue_failure( @@ -1805,9 +2280,11 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo task: TransferTask, error: Exception, ) -> None: - """记录内存入队失败并撤销该任务的批次占位。""" + """按是否已经 claim 选择 fencing 失败记录，并撤销批次占位。""" try: - if task.admission_task_id: + if task.lease_token: + self.__release_task_claim(task, error=str(error)) + elif task.admission_task_id: self._transfer_admissions.record_enqueue_failure( task_id=task.admission_task_id, error=str(error), @@ -1819,8 +2296,9 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) finally: self._finish_scrape_batch_task(task) + self.__ensure_recovery_scheduler(immediate=False) - def __discard_pending(self, task: TransferTask): + def __discard_pending(self, task: TransferTask) -> bool: """ 注销一个待整理文件登记，整理到达终态（成功或失败）时调用。 @@ -1829,16 +2307,34 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo :param task: 任务信息 """ if not task or not task.admission_task_id: - return - try: - self._transfer_admissions.discard_task( - task_id=task.admission_task_id + return True + if not task.lease_token: + logger.error( + f"整理任务缺少 lease token，拒绝伪装 durable 终态：" + f"{task.admission_task_id}" ) + return False + discarded = 0 + try: + discarded = self._transfer_admissions.discard_claimed( + task_id=task.admission_task_id, + lease_token=task.lease_token, + ) + if not discarded: + logger.error( + f"整理任务终态注销被 CAS 拒绝：{task.admission_task_id}" + ) except Exception as err: logger.error( "注销整理任务 durable admission 失败: " f"{task.admission_task_id} - {err}" ) + finally: + self.__forget_owned_lease( + task.admission_task_id, + task.lease_token, + ) + return bool(discarded) def __put_to_jobview(self, task: TransferTask) -> bool: """ @@ -1905,13 +2401,23 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo if marker: marker(task) - def __finish_job_execution(self, task: TransferTask, *, terminal: bool = True): - """结束内存执行；仅确定到达业务终态时注销 durable 记录。""" + def __finish_job_execution( + self, + task: TransferTask, + *, + terminal: bool = True, + terminal_settlement: Optional[bool] = None, + ) -> bool: + """结束内存执行，并复用成功回调前已经提交的 durable 终态结果。""" marker = getattr(self.jobview, "finish_execution", None) if marker: marker(task) if terminal: - self.__discard_pending(task) + if terminal_settlement is not None: + return terminal_settlement + return self.__discard_pending(task) + self.__release_task_claim(task) + return True def __expire_stale_transfer_tasks(self): """清理外部接管后失去状态心跳的运行中整理任务。""" @@ -2001,6 +2507,32 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__settle_transfer_progress_if_idle() continue + if task.admission_task_id and task.lease_token: + with self._worker_state_lock: + self._queued_lease_tokens.discard( + (task.admission_task_id, task.lease_token) + ) + try: + self.__claim_task_for_execution(task) + except TransferLeaseLostError as err: + logger.info(f"跳过未取得执行租约的整理任务：{err}") + self.__release_task_claim(task, error=str(err)) + self.jobview.try_remove_job(task) + self._finish_scrape_batch_task(task) + self._queue.task_done() + self.__settle_transfer_progress_if_idle() + continue + except Exception as err: + logger.error( + f"整理任务 claim 失败，保留 durable admission：{err}" + ) + self.jobview.try_remove_job(task) + self._finish_scrape_batch_task(task) + self._queue.task_done() + self.__settle_transfer_progress_if_idle() + self.__ensure_recovery_scheduler(immediate=False) + continue + # 文件信息 fileitem = task.fileitem @@ -2030,6 +2562,29 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._active_tasks += 1 terminal = False + terminal_settlement: Optional[bool] = None + state = False + err_msg = "" + + def callback_after_terminal_settlement( + callback_task: TransferTask, + transferinfo: TransferInfo, + ) -> Tuple[bool, str]: + """成功结果先提交 durable 终态，再执行历史、事件和通知回调。""" + nonlocal terminal_settlement + if transferinfo.success and not callback_task.preview: + terminal_settlement = self.__discard_pending(callback_task) + if not terminal_settlement: + self.__fail_transfer_task(callback_task) + return False, "整理任务 durable 终态结算失去租约" + if item.callback: + callback_result: Tuple[bool, str] = item.callback( + callback_task, + transferinfo, + ) + return callback_result + return transferinfo.success, transferinfo.message or "" + try: self.__start_job_execution(task) # 更新进度 @@ -2044,7 +2599,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo ) # 整理 state, err_msg = self.__handle_transfer( - task=task, callback=item.callback + task=task, + callback=callback_after_terminal_settlement, ) terminal = task.plan_checkpoint is not None @@ -2063,6 +2619,8 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo text=__process_msg, ) except Exception as e: + if terminal_settlement is not None: + terminal = True logger.error( f"{fileitem.name} 整理任务处理出现错误：{e} - {traceback.format_exc()}" ) @@ -2071,12 +2629,25 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self._processed_num += 1 self._fail_num += 1 finally: - self.__finish_job_execution(task, terminal=terminal) - self._queue.task_done() - with task_lock: - # 减少运行中的任务数 - self._active_tasks -= 1 - self.__settle_transfer_progress_if_idle() + durable_settled = False + try: + durable_settled = self.__finish_job_execution( + task, + terminal=terminal, + terminal_settlement=terminal_settlement, + ) + except Exception as err: + logger.error( + f"整理任务终态结算异常：{task.admission_task_id} - {err}" + ) + finally: + self._queue.task_done() + with task_lock: + # 减少运行中的任务数 + self._active_tasks -= 1 + if terminal and state and not durable_settled: + self._fail_num += 1 + self.__settle_transfer_progress_if_idle() except queue.Empty: # 即使队列空了，如果还有任务在运行，也不应该结束进度 @@ -3548,6 +4119,10 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo and file_item.path == recovery_admission.src_path ): transfer_task.bind_admission_task_id(recovery_admission.task_id) + self.__bind_claimed_admission( + transfer_task, + recovery_admission, + ) if recovery_admission.planning_input: transfer_task.bind_planning_input( recovery_admission.planning_input @@ -3651,14 +4226,37 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo }, ) terminal = False + terminal_settlement: Optional[bool] = None + + def callback_after_terminal_settlement( + callback_task: TransferTask, + transferinfo: TransferInfo, + ) -> Tuple[bool, str]: + """同步成功结果先结算 durable 行，再执行兼容回调副作用。""" + nonlocal terminal_settlement + if transferinfo.success and not callback_task.preview: + terminal_settlement = self.__discard_pending(callback_task) + if not terminal_settlement: + self.__fail_transfer_task(callback_task) + return False, "整理任务 durable 终态结算失去租约" + callback = ( + _preview_callback + if preview + else self.__default_callback + ) + return callback(callback_task, transferinfo) + try: + self.__claim_task_for_execution(transfer_task) self.__start_job_execution(transfer_task) state, err_msg = self.__handle_transfer( task=transfer_task, - callback=_preview_callback if preview else self.__default_callback, + callback=callback_after_terminal_settlement, ) terminal = bool(preview or transfer_task.plan_checkpoint is not None) except Exception as e: + if terminal_settlement is not None: + terminal = True logger.error( f"{transfer_task.fileitem.name} 整理任务处理出现错误：" f"{e} - {traceback.format_exc()}" @@ -3667,10 +4265,14 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo self.__fail_transfer_task(transfer_task) state, err_msg = False, str(e) finally: - self.__finish_job_execution( + durable_settled = self.__finish_job_execution( transfer_task, terminal=terminal, + terminal_settlement=terminal_settlement, ) + if terminal and not durable_settled: + state = False + err_msg = "整理任务 durable 终态结算失去租约" if not state: all_success = False logger.warn(f"{transfer_task.fileitem.name} {err_msg}") diff --git a/app/db/adapters/transfer.py b/app/db/adapters/transfer.py index fe29fd8af..51d48ff2c 100644 --- a/app/db/adapters/transfer.py +++ b/app/db/adapters/transfer.py @@ -1,7 +1,9 @@ """整理任务持久准入端口的 SQLAlchemy 适配器。""" +import logging from collections.abc import Callable -from datetime import datetime +from datetime import datetime, timedelta, timezone +from json import JSONDecodeError from typing import Optional from uuid import uuid4 @@ -14,6 +16,8 @@ from app.application.transfer import ( TRANSFER_ADMISSION_PROVIDER_PENDING, TransferAdmission, TransferAdmissionConflictError, + TransferAdmissionProjectionError, + TransferLeaseLostError, TransferPlanCheckpoint, TransferPlanningInput, TransferPlanningStateError, @@ -22,10 +26,14 @@ from app.db.models.transferpending import TransferPending from app.db.oper.transferpending import TransferPendingOper from app.db.uow import SqlAlchemyUnitOfWork +_diagnostic_logger = logging.getLogger(__name__) + class TransactionalTransferAdmissionRepository: """以短生命周期 Session 实现整理任务持久准入端口。""" + _MAX_RECOVERY_SCAN_TASKS = 5000 + def __init__(self, session_factory: Callable[[], Session]) -> None: """保存由组合根提供的同步会话工厂。""" self._session_factory = session_factory @@ -35,6 +43,33 @@ class TransactionalTransferAdmissionRepository: """生成与历史登记时间可按字典序比较的当前时间。""" return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + @staticmethod + def _lease_now() -> datetime: + """生成不受宿主时区影响的当前 UTC 租约时间。""" + return datetime.now(timezone.utc) + + @staticmethod + def _format_lease_time(value: datetime) -> str: + """把 UTC 时间编码为可稳定排序的固定宽度字符串。""" + return value.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f") + + @staticmethod + def _validate_claim_arguments(*, owner_id: str, lease_seconds: int) -> None: + """拒绝无法建立有效租约身份或正向期限的调用。""" + if not owner_id: + raise ValueError("整理任务 claim 缺少 owner_id") + if lease_seconds <= 0: + raise ValueError("整理任务 lease_seconds 必须大于零") + + @staticmethod + def _recoverable_states() -> tuple[str, ...]: + """返回允许被 worker claim 的稳定业务状态。""" + return ( + TRANSFER_ADMISSION_ACCEPTED, + TRANSFER_ADMISSION_PROVIDER_PENDING, + TRANSFER_ADMISSION_PLANNED, + ) + @staticmethod def _project(pending: TransferPending) -> TransferAdmission: """在 Session 有效期内把 ORM 行冻结为应用层 DTO。""" @@ -84,6 +119,11 @@ class TransactionalTransferAdmissionRepository: input_fingerprint=pending.input_fingerprint, planning_input=planning_input, checkpoint=checkpoint, + lease_owner=pending.lease_owner, + lease_token=pending.lease_token, + lease_expires_at=pending.lease_expires_at, + heartbeat_at=pending.heartbeat_at, + attempt_count=pending.attempt_count, ) @staticmethod @@ -151,27 +191,225 @@ class TransactionalTransferAdmissionRepository: self._assert_input_match(pending, effective_input) return self._project(pending) - def list_accepted(self, limit: int = 5000) -> list[TransferAdmission]: - """在独立只读会话中投影等待恢复或执行的准入记录。""" + def claim_task( + self, + *, + task_id: str, + owner_id: str, + lease_seconds: int, + ) -> Optional[TransferAdmission]: + """原子 claim 指定任务，任何已存在的有效租约都拒绝重复领取。""" + self._validate_claim_arguments( + owner_id=owner_id, + lease_seconds=lease_seconds, + ) + if not task_id: + raise ValueError("整理任务 claim 缺少 task_id") + now = self._lease_now() + now_time = self._format_lease_time(now) + lease_expires_at = self._format_lease_time( + now + timedelta(seconds=lease_seconds) + ) with self._session_factory() as session: - pending_items = TransferPendingOper(db=session).list_by_state( - state=TRANSFER_ADMISSION_ACCEPTED, - limit=limit, - ) - return [self._project(pending) for pending in pending_items] + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferPendingOper(db=session) + lease_token = uuid4().hex + updated = oper.stage_claim_task( + task_id=task_id, + states=self._recoverable_states(), + owner_id=owner_id, + lease_token=lease_token, + now_time=now_time, + lease_expires_at=lease_expires_at, + updated_at=self._now(), + ) + session.flush() + session.expire_all() + try: + pending = oper.get_by_task_id(task_id=task_id) + except JSONDecodeError as error: + raise TransferAdmissionProjectionError( + f"整理任务持久 JSON 无法解码: {task_id} - {error}" + ) from error + if updated: + if pending is None: + raise TransferPlanningStateError( + f"claim 后未找到整理任务: {task_id}" + ) + try: + admission = self._project(pending) + except ( + TransferAdmissionConflictError, + TransferPlanningStateError, + TypeError, + ValueError, + ) as error: + raise TransferAdmissionProjectionError( + f"整理任务持久投影损坏: {task_id} - {error}" + ) from error + transaction.commit() + return admission + transaction.commit() + return None + except Exception: + transaction.rollback() + raise - def list_recoverable(self, limit: int = 5000) -> list[TransferAdmission]: - """投影接纳、provider 待执行或已规划的全部可恢复任务。""" - with self._session_factory() as session: - pending_items = TransferPendingOper(db=session).list_by_states( - states=( - TRANSFER_ADMISSION_ACCEPTED, - TRANSFER_ADMISSION_PROVIDER_PENDING, - TRANSFER_ADMISSION_PLANNED, - ), - limit=limit, + def claim_recoverable( + self, + *, + owner_id: str, + limit: int, + lease_seconds: int, + ) -> list[TransferAdmission]: + """按登记顺序逐条 CAS claim 未租用或租约已过期的恢复任务。""" + self._validate_claim_arguments( + owner_id=owner_id, + lease_seconds=lease_seconds, + ) + if limit <= 0: + return [] + claimed: list[TransferAdmission] = [] + after_cursor: Optional[tuple[str, int]] = None + scanned_count = 0 + scan_limit = self._MAX_RECOVERY_SCAN_TASKS + while len(claimed) < limit and scanned_count < scan_limit: + candidate_limit = min( + limit - len(claimed), + scan_limit - scanned_count, ) - return [self._project(pending) for pending in pending_items] + now_time = self._format_lease_time(self._lease_now()) + with self._session_factory() as session: + candidates = TransferPendingOper(db=session).list_claimable_candidates( + states=self._recoverable_states(), + now_time=now_time, + limit=candidate_limit, + after_cursor=after_cursor, + ) + if not candidates: + break + scanned_count += len(candidates) + _, cursor_created_at, cursor_id = candidates[-1] + after_cursor = (cursor_created_at, cursor_id) + for task_id, _, _ in candidates: + try: + admission = self.claim_task( + task_id=task_id, + owner_id=owner_id, + lease_seconds=lease_seconds, + ) + except TransferAdmissionProjectionError as error: + self._record_projection_failure( + task_id=task_id, + error=error, + ) + continue + if admission is not None: + claimed.append(admission) + if len(claimed) >= limit: + break + return claimed + + def _record_projection_failure( + self, + *, + task_id: str, + error: TransferAdmissionProjectionError, + ) -> bool: + """以独立 CAS 留存变化后的投影错误，并仅为新诊断记一次运行日志。""" + diagnostic = f"恢复投影失败: {error}" + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + recorded = TransferPendingOper( + db=session + ).stage_record_projection_failure( + task_id=task_id, + states=self._recoverable_states(), + error=diagnostic, + now_time=self._format_lease_time(self._lease_now()), + updated_at=self._now(), + ) + transaction.commit() + except Exception: + transaction.rollback() + raise + if recorded: + _diagnostic_logger.error( + f"整理恢复任务投影损坏：task_id={task_id}, error={error}" + ) + return bool(recorded) + + def heartbeat( + self, + *, + task_id: str, + lease_token: str, + lease_seconds: int, + ) -> Optional[TransferAdmission]: + """仅以当前且未过期的 token 续租，禁止陈旧 worker 复活租约。""" + if not task_id or not lease_token: + raise ValueError("整理任务 heartbeat 缺少任务或租约身份") + if lease_seconds <= 0: + raise ValueError("整理任务 lease_seconds 必须大于零") + now = self._lease_now() + now_time = self._format_lease_time(now) + lease_expires_at = self._format_lease_time( + now + timedelta(seconds=lease_seconds) + ) + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + oper = TransferPendingOper(db=session) + updated = oper.stage_heartbeat( + task_id=task_id, + lease_token=lease_token, + now_time=now_time, + lease_expires_at=lease_expires_at, + ) + if not updated: + transaction.commit() + return None + session.flush() + session.expire_all() + pending = oper.get_by_task_id(task_id=task_id) + if pending is None: + raise TransferPlanningStateError( + f"heartbeat 后未找到整理任务: {task_id}" + ) + admission = self._project(pending) + transaction.commit() + return admission + except Exception: + transaction.rollback() + raise + + def release_claim( + self, + *, + task_id: str, + lease_token: str, + error: Optional[str] = None, + ) -> bool: + """仅以当前未过期 token 释放租约，陈旧 worker 不得改变任务。""" + if not task_id or not lease_token: + return False + with self._session_factory() as session: + transaction = SqlAlchemyUnitOfWork(session) + try: + released = TransferPendingOper(db=session).stage_release_claim( + task_id=task_id, + lease_token=lease_token, + error=error, + now_time=self._format_lease_time(self._lease_now()), + updated_at=self._now(), + ) + transaction.commit() + return bool(released) + except Exception: + transaction.rollback() + raise def record_enqueue_failure(self, *, task_id: str, error: str) -> None: """独立提交最近一次入队失败，保留准入记录供后续恢复。""" @@ -192,6 +430,7 @@ class TransactionalTransferAdmissionRepository: self, *, task_id: str, + lease_token: str, input_fingerprint: str, checkpoint: TransferPlanCheckpoint, ) -> TransferAdmission: @@ -223,7 +462,9 @@ class TransactionalTransferAdmissionRepository: checkpoint_payload=checkpoint_payload, source_states=source_states, target_state=target_state, - now_time=self._now(), + lease_token=lease_token, + now_time=self._format_lease_time(self._lease_now()), + updated_at=self._now(), ) session.flush() session.expire_all() @@ -232,6 +473,13 @@ class TransactionalTransferAdmissionRepository: raise TransferPlanningStateError(f"未找到整理任务: {task_id}") if pending.input_fingerprint != input_fingerprint: raise TransferAdmissionConflictError("整理任务输入指纹已经改变") + now_time = self._format_lease_time(self._lease_now()) + if ( + pending.lease_token != lease_token + or not pending.lease_expires_at + or pending.lease_expires_at <= now_time + ): + raise TransferLeaseLostError("整理任务租约已过期或已被其他 worker 接管") if not updated and not ( pending.state == target_state and pending.checkpoint_payload == checkpoint_payload @@ -246,28 +494,44 @@ class TransactionalTransferAdmissionRepository: transaction.rollback() raise - def record_planning_failure(self, *, task_id: str, error: str) -> None: + def record_planning_failure( + self, + *, + task_id: str, + lease_token: str, + error: str, + ) -> None: """独立提交规划错误并保持任务处于接纳态供恢复重试。""" with self._session_factory() as session: transaction = SqlAlchemyUnitOfWork(session) try: - TransferPendingOper(db=session).stage_record_planning_failure( + updated = TransferPendingOper(db=session).stage_record_planning_failure( task_id=task_id, + lease_token=lease_token, error=error, - now_time=self._now(), + now_time=self._format_lease_time(self._lease_now()), + updated_at=self._now(), ) + if not updated: + raise TransferLeaseLostError( + "整理任务租约已过期或已被其他 worker 接管" + ) transaction.commit() except Exception: transaction.rollback() raise - def discard_task(self, *, task_id: str) -> int: - """在独立事务中按稳定任务标识删除已到终态的准入记录。""" + def discard_claimed(self, *, task_id: str, lease_token: str) -> int: + """仅以当前未过期 token 删除终态任务，拒绝陈旧 worker 变更。""" + if not task_id or not lease_token: + return 0 with self._session_factory() as session: transaction = SqlAlchemyUnitOfWork(session) try: - deleted = TransferPendingOper(db=session).stage_discard_task( + deleted = TransferPendingOper(db=session).stage_discard_claimed( task_id=task_id, + lease_token=lease_token, + now_time=self._format_lease_time(self._lease_now()), ) transaction.commit() return deleted diff --git a/app/db/models/transferpending.py b/app/db/models/transferpending.py index dc87e5f1c..3964a4385 100644 --- a/app/db/models/transferpending.py +++ b/app/db/models/transferpending.py @@ -4,7 +4,20 @@ from datetime import datetime from typing import Any, List, Optional, cast from uuid import uuid4 -from sqlalchemy import JSON, Index, Integer, String, Text, UniqueConstraint, delete, select, update +from sqlalchemy import ( + JSON, + Index, + Integer, + String, + Text, + UniqueConstraint, + and_, + delete, + func, + or_, + select, + update, +) from sqlalchemy.orm import Mapped, Session, mapped_column from app.db.base import Base, execute_dml, get_id_column @@ -72,7 +85,7 @@ class TransferPending(Base): 准入时保存版本化规划输入和指纹；纯规划完成后以同一行原子保存完整有序计划并 推进到 planned。重启恢复可直接消费已规划路径，避免再次触发 rename 等插件事件。 - 旧路径登记接口仍生成最小 legacy_replan 输入，供插件兼容调用方继续使用。 + 所有执行期 mutation 都以稳定任务身份和租约 token 进行 CAS fencing。 """ id = get_id_column() @@ -111,6 +124,16 @@ class TransferPending(Base): checkpoint_payload: Mapped[Optional[dict[str, Any]]] = mapped_column(JSON) # 规划完成时间 planned_at: Mapped[Optional[str]] = mapped_column(String(40)) + # 当前租约拥有者 + lease_owner: Mapped[Optional[str]] = mapped_column(String(128)) + # 当前租约的唯一防陈旧令牌 + lease_token: Mapped[Optional[str]] = mapped_column(String(64)) + # 当前租约的 UTC 到期时间 + lease_expires_at: Mapped[Optional[str]] = mapped_column(String(40)) + # 最近一次成功 claim 或 heartbeat 的 UTC 时间 + heartbeat_at: Mapped[Optional[str]] = mapped_column(String(40)) + # 真正取得新 token 的累计次数 + attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) __table_args__ = ( # 同一个文件重复入队只保留一条，回放时不会重复送入整理链 @@ -122,41 +145,17 @@ class TransferPending(Base): "created_at", "id", ), + # 恢复调度按业务状态和租约到期时间筛选可接管任务 + Index( + "ix_transferpending_recovery_lease", + "state", + "lease_expires_at", + "created_at", + "id", + ), UniqueConstraint("task_id", name="uq_transferpending_task_id"), ) - @classmethod - def register(cls, db: Session, storage: str, src_path: str, - now_time: str) -> Optional["TransferPending"]: - """ - 登记一个待整理文件，已存在时保持原登记时间不变。 - :param db: 数据库会话 - :param storage: 存储 - :param src_path: 源文件路径 - :param now_time: 当前时间 - :return: 登记记录 - """ - if not storage or not src_path: - return None - pending = db.execute( - select(cls).where(cls.storage == storage, cls.src_path == src_path) - ).scalars().first() - if pending: - return cast("TransferPending", pending) - planning_input = _legacy_planning_payload(storage, src_path) - pending = cls( - storage=storage, - src_path=src_path, - state="accepted", - created_at=now_time, - updated_at=now_time, - input_version=1, - planning_input=planning_input, - input_fingerprint=_planning_fingerprint(planning_input), - ) - db.add(pending) - return pending - @classmethod def stage_admit(cls, db: Session, *, task_id: str, storage: str, src_path: str, state: str, @@ -201,44 +200,6 @@ class TransferPending(Base): db.add(pending) return pending - @classmethod - def list_by_state(cls, db: Session, *, state: str, - limit: Optional[int] = 5000) -> List["TransferPending"]: - """ - 按登记顺序列出指定持久状态的接纳记录。 - :param db: 数据库会话 - :param state: 持久状态 - :param limit: 单次读取上限 - :return: 接纳记录列表 - """ - if not state: - return [] - return list(db.execute( - select(cls) - .where(cls.state == state) - .order_by(cls.created_at.asc(), cls.id.asc()) - .limit(limit) - ).scalars().all()) - - @classmethod - def list_by_states(cls, db: Session, *, states: tuple[str, ...], - limit: Optional[int] = 5000) -> List["TransferPending"]: - """ - 按登记顺序列出多个可恢复持久状态的记录。 - :param db: 数据库会话 - :param states: 允许恢复的状态集合 - :param limit: 单次读取上限 - :return: 接纳记录列表 - """ - if not states: - return [] - return list(db.execute( - select(cls) - .where(cls.state.in_(states)) - .order_by(cls.created_at.asc(), cls.id.asc()) - .limit(limit) - ).scalars().all()) - @classmethod def get_by_identity(cls, db: Session, *, storage: str, src_path: str) -> Optional["TransferPending"]: @@ -276,12 +237,283 @@ class TransferPending(Base): db.execute(select(cls).where(cls.task_id == task_id)).scalars().first(), ) + @classmethod + def list_claimable_candidates( + cls, + db: Session, + *, + states: tuple[str, ...], + now_time: str, + limit: int, + after_cursor: Optional[tuple[str, int]] = None, + ) -> List[tuple[str, str, int]]: + """ + 按稳定游标列出未租用或租约已过期的候选任务。 + + 返回候选不等于取得租约；调用方必须继续执行带相同过期条件的 claim CAS， + 并以受影响行数决定竞争结果。 + :param db: 数据库会话 + :param states: 可恢复业务状态 + :param now_time: 当前 UTC 时间 + :param limit: 候选数量上限 + :param after_cursor: 上一页最后一条的规范登记时间与主键 + :return: 任务标识、规范登记时间与主键组成的稳定游标列表 + """ + if not states or not now_time or limit <= 0: + return [] + cursor_created_at = func.coalesce(cls.created_at, "") + statement = select(cls.task_id, cursor_created_at, cls.id).where( + cls.state.in_(states), + or_( + cls.lease_token.is_(None), + cls.lease_expires_at.is_(None), + cls.lease_expires_at <= now_time, + ), + ) + if after_cursor is not None: + after_created_at, after_id = after_cursor + statement = statement.where(or_( + cursor_created_at > after_created_at, + and_( + cursor_created_at == after_created_at, + cls.id > after_id, + ), + )) + rows = db.execute( + statement + .order_by(cursor_created_at.asc(), cls.id.asc()) + .limit(limit) + ).all() + return [ + (task_id, created_at or "", int(row_id)) + for task_id, created_at, row_id in rows + ] + + @classmethod + def claim_task( + cls, + db: Session, + *, + task_id: str, + states: tuple[str, ...], + owner_id: str, + lease_token: str, + now_time: str, + lease_expires_at: str, + updated_at: str, + ) -> int: + """ + 以未租用或租约已过期为条件原子取得任务租约。 + + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param states: 允许 claim 的业务状态 + :param owner_id: 新租约拥有者 + :param lease_token: 新租约唯一令牌 + :param now_time: 当前 UTC 时间 + :param lease_expires_at: 新租约到期时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 + :return: 更新的记录数，1 表示赢得竞争 + """ + if not all(( + task_id, + states, + owner_id, + lease_token, + now_time, + lease_expires_at, + updated_at, + )): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state.in_(states), + or_( + cls.lease_token.is_(None), + cls.lease_expires_at.is_(None), + cls.lease_expires_at <= now_time, + ), + ) + .values( + lease_owner=owner_id, + lease_token=lease_token, + lease_expires_at=lease_expires_at, + heartbeat_at=now_time, + attempt_count=cls.attempt_count + 1, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def record_projection_failure( + cls, + db: Session, + *, + task_id: str, + states: tuple[str, ...], + error: str, + now_time: str, + updated_at: str, + ) -> int: + """ + 在没有有效租约且诊断发生变化时原子记录恢复投影损坏。 + + claim 的投影失败会先回滚，因此这里不得重新占用租约。CAS 同时保护 + 已被其他 worker 领取的任务，并避免周期恢复反复刷新相同错误。 + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param states: 可恢复业务状态 + :param error: 可持久化的稳定诊断文本 + :param now_time: 当前 UTC 租约时间 + :param updated_at: 宿主本地业务审计时间 + :return: 更新的记录数，1 表示首次或变化后的诊断被记录 + """ + if not all((task_id, states, error, now_time, updated_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.state.in_(states), + or_( + cls.lease_token.is_(None), + cls.lease_expires_at.is_(None), + cls.lease_expires_at <= now_time, + ), + cls.last_error.is_distinct_from(error), + ) + .values( + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def heartbeat( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + now_time: str, + lease_expires_at: str, + ) -> int: + """ + 仅以当前且未过期的 token 原子延长任务租约。 + + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param now_time: 当前 UTC 时间 + :param lease_expires_at: 新租约到期时间 + :return: 更新的记录数 + """ + if not all((task_id, lease_token, now_time, lease_expires_at)): + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_time, + ) + .values( + lease_expires_at=lease_expires_at, + heartbeat_at=now_time, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def release_claim( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + error: Optional[str], + now_time: str, + updated_at: str, + ) -> int: + """ + 仅以当前且未过期的 token 释放租约并保存本次执行错误。 + + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param error: 本次执行错误，成功释放时为空 + :param now_time: 当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 + :return: 更新的记录数 + """ + if not task_id or not lease_token or not now_time or not updated_at: + return 0 + return execute_dml( + db, + update(cls) + .where( + cls.task_id == task_id, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_time, + ) + .values( + lease_owner=None, + lease_token=None, + lease_expires_at=None, + heartbeat_at=None, + last_error=error, + updated_at=updated_at, + ), + execution_options={"synchronize_session": False}, + ) + + @classmethod + def discard_claimed( + cls, + db: Session, + *, + task_id: str, + lease_token: str, + now_time: str, + ) -> int: + """ + 仅以当前且未过期的 token 删除已经到达终态的租约任务。 + + :param db: 数据库会话 + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param now_time: 当前 UTC 时间 + :return: 删除的记录数 + """ + if not task_id or not lease_token or not now_time: + return 0 + return execute_dml( + db, + delete(cls).where( + cls.task_id == task_id, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_time, + ), + execution_options={"synchronize_session": False}, + ) + @classmethod def checkpoint_plan(cls, db: Session, *, task_id: str, input_fingerprint: str, checkpoint_version: int, checkpoint_payload: dict[str, Any], source_states: tuple[str, ...], target_state: str, - now_time: str) -> int: + lease_token: str, now_time: str, + updated_at: str) -> int: """ 以输入指纹为 CAS 条件原子保存计划并推进到已规划。 :param db: 数据库会话 @@ -291,7 +523,9 @@ class TransferPending(Base): :param checkpoint_payload: 完整有序计划 JSON :param source_states: 允许推进检查点的起始状态 :param target_state: 检查点提交后的目标状态 - :param now_time: 当前时间 + :param lease_token: 当前且未过期的租约令牌 + :param now_time: 用于租约 fencing 的当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 :return: 更新的记录数 """ if ( @@ -300,8 +534,19 @@ class TransferPending(Base): or not checkpoint_payload or not source_states or not target_state + or not lease_token + or not updated_at ): return 0 + values: dict[str, Any] = { + "state": target_state, + "checkpoint_version": checkpoint_version, + "checkpoint_payload": checkpoint_payload, + "last_error": None, + "updated_at": updated_at, + } + if target_state == "planned": + values["planned_at"] = updated_at return execute_dml( db, update(cls) @@ -309,30 +554,29 @@ class TransferPending(Base): cls.task_id == task_id, cls.state.in_(source_states), cls.input_fingerprint == input_fingerprint, + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_time, ) - .values( - state=target_state, - checkpoint_version=checkpoint_version, - checkpoint_payload=checkpoint_payload, - planned_at=now_time, - last_error=None, - updated_at=now_time, - ), + .values(**values), execution_options={"synchronize_session": False}, ) @classmethod def record_planning_failure(cls, db: Session, *, task_id: str, - error: str, now_time: str) -> int: + lease_token: str, error: str, + now_time: str, updated_at: str) -> int: """ 为接纳态或 provider 待执行任务记录规划失败，不改变其恢复状态。 :param db: 数据库会话 :param task_id: 稳定任务标识 + :param lease_token: 当前且未过期的租约令牌 :param error: 失败原因 - :param now_time: 当前时间 + :param now_time: 用于租约 fencing 的当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 :return: 更新的记录数 """ - if not task_id: + if not task_id or not lease_token or not now_time or not updated_at: return 0 return execute_dml( db, @@ -340,8 +584,11 @@ class TransferPending(Base): .where( cls.task_id == task_id, cls.state.in_(("accepted", "provider_pending")), + cls.lease_token == lease_token, + cls.lease_expires_at.is_not(None), + cls.lease_expires_at > now_time, ) - .values(last_error=error, updated_at=now_time), + .values(last_error=error, updated_at=updated_at), execution_options={"synchronize_session": False}, ) @@ -361,67 +608,10 @@ class TransferPending(Base): return execute_dml( db, update(cls) - .where(cls.task_id == task_id) + .where( + cls.task_id == task_id, + cls.lease_token.is_(None), + ) .values(last_error=error, updated_at=now_time), - execution_options={"synchronize_session": False}, - ) - - @classmethod - def discard_task(cls, db: Session, *, task_id: str) -> int: - """ - 在调用方会话中按任务标识删除接纳记录。 - :param db: 数据库会话 - :param task_id: 任务标识 - :return: 删除的记录数 - """ - if not task_id: - return 0 - return execute_dml( - db, delete(cls).where(cls.task_id == task_id), - execution_options={"synchronize_session": False}, - ) - - @classmethod - def discard(cls, db: Session, storage: str, src_path: str) -> int: - """ - 注销一个待整理文件登记，整理到达终态（成功或失败）时调用。 - :param db: 数据库会话 - :param storage: 存储 - :param src_path: 源文件路径 - :return: 删除的记录数 - """ - if not storage or not src_path: - return 0 - return execute_dml( - db, delete(cls).where(cls.storage == storage, cls.src_path == src_path), - execution_options={"synchronize_session": False}, - ) - - @classmethod - def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]: - """ - 列出全部待整理登记，供启动回放使用。 - - 按登记时间升序回放，保持与原入队顺序一致；上限避免异常积压时 - 一次性把整理链压垮。 - :param db: 数据库会话 - :param limit: 单次回放上限 - :return: 待整理登记列表 - """ - return list(db.execute( - select(cls) - .order_by(cls.created_at.asc(), cls.id.asc()) - .limit(limit) - ).scalars().all()) - - @classmethod - def clear(cls, db: Session) -> int: - """ - 清空全部待整理登记。 - :param db: 数据库会话 - :return: 删除的记录数 - """ - return execute_dml( - db, delete(cls), - execution_options={"synchronize_session": False}, + execution_options={"synchronize_session": "fetch"}, ) diff --git a/app/db/oper/transferpending.py b/app/db/oper/transferpending.py index 531b6ad41..4bc34fbae 100644 --- a/app/db/oper/transferpending.py +++ b/app/db/oper/transferpending.py @@ -1,5 +1,4 @@ -from datetime import datetime -from typing import Any, List, Optional, Tuple +from typing import Any, Optional from app.db.base import DbOper from app.db.models.transferpending import TransferPending @@ -10,27 +9,9 @@ class TransferPendingOper(DbOper): 待整理文件登记管理。 保存稳定任务身份、存储、源文件路径和准入状态，用于在进程重启后把没走完 - 整理链的文件重新送回去，避免挂载故障重启后永久漏件。旧版路径登记接口继续 - 保留，供插件和兼容调用方使用。 + 整理链的文件重新送回去，避免挂载故障重启后永久漏件。 """ - def register(self, storage: str, src_path: str) -> Optional[TransferPending]: - """ - 登记一个待整理文件。 - :param storage: 存储 - :param src_path: 源文件路径 - :return: 登记记录 - """ - now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - return self._execute_sync_write( - lambda session: TransferPending.register( - session, - storage=storage, - src_path=src_path, - now_time=now_time, - ) - ) - def stage_admit(self, *, task_id: str, storage: str, src_path: str, state: str, now_time: str, input_version: int = 1, planning_input: Optional[dict[str, Any]] = None, @@ -63,38 +44,6 @@ class TransferPendingOper(DbOper): ) ) - def list_by_state(self, *, state: str, - limit: Optional[int] = 5000) -> List[TransferPending]: - """ - 使用当前会话列出指定状态记录。 - :param state: 持久状态 - :param limit: 单次读取上限 - :return: ORM 接纳记录列表 - """ - return self._execute_sync_query( - lambda session: TransferPending.list_by_state( - session, - state=state, - limit=limit, - ) - ) or [] - - def list_by_states(self, *, states: tuple[str, ...], - limit: Optional[int] = 5000) -> List[TransferPending]: - """ - 使用当前会话列出多个可恢复状态的记录。 - :param states: 允许恢复的状态集合 - :param limit: 单次读取上限 - :return: ORM 接纳记录列表 - """ - return self._execute_sync_query( - lambda session: TransferPending.list_by_states( - session, - states=states, - limit=limit, - ) - ) or [] - def get_by_identity(self, *, storage: str, src_path: str) -> Optional[TransferPending]: """ @@ -133,7 +82,9 @@ class TransferPendingOper(DbOper): checkpoint_payload: dict[str, Any], source_states: tuple[str, ...], target_state: str, + lease_token: str, now_time: str, + updated_at: str, ) -> int: """ 在当前会话中以输入指纹为条件暂存完整计划检查点。 @@ -143,7 +94,9 @@ class TransferPendingOper(DbOper): :param checkpoint_payload: 完整有序计划 JSON :param source_states: 允许执行 CAS 的起始状态 :param target_state: 检查点提交后的目标状态 - :param now_time: 当前时间 + :param lease_token: 当前且未过期的租约令牌 + :param now_time: 用于租约 fencing 的当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 :return: 更新的记录数 """ return self._execute_sync_write( @@ -155,25 +108,206 @@ class TransferPendingOper(DbOper): checkpoint_payload=checkpoint_payload, source_states=source_states, target_state=target_state, + lease_token=lease_token, now_time=now_time, + updated_at=updated_at, ) ) - def stage_record_planning_failure(self, *, task_id: str, error: str, - now_time: str) -> int: + def stage_record_planning_failure(self, *, task_id: str, lease_token: str, + error: str, now_time: str, + updated_at: str) -> int: """ 在当前会话中记录规划失败并保持任务处于接纳态。 :param task_id: 稳定任务标识 + :param lease_token: 当前且未过期的租约令牌 :param error: 失败原因 - :param now_time: 当前时间 + :param now_time: 用于租约 fencing 的当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 :return: 更新的记录数 """ return self._execute_sync_write( lambda session: TransferPending.record_planning_failure( session, task_id=task_id, + lease_token=lease_token, error=error, now_time=now_time, + updated_at=updated_at, + ) + ) + + def list_claimable_candidates( + self, + *, + states: tuple[str, ...], + now_time: str, + limit: int, + after_cursor: Optional[tuple[str, int]] = None, + ) -> list[tuple[str, str, int]]: + """ + 使用当前会话按稳定游标读取未租用或已过期的恢复候选。 + + :param states: 可恢复业务状态 + :param now_time: 当前 UTC 时间 + :param limit: 候选数量上限 + :param after_cursor: 上一页最后一条的规范登记时间与主键 + :return: 任务标识、规范登记时间与主键组成的稳定游标列表 + """ + return self._execute_sync_query( + lambda session: TransferPending.list_claimable_candidates( + session, + states=states, + now_time=now_time, + limit=limit, + after_cursor=after_cursor, + ) + ) or [] + + def stage_claim_task( + self, + *, + task_id: str, + states: tuple[str, ...], + owner_id: str, + lease_token: str, + now_time: str, + lease_expires_at: str, + updated_at: str, + ) -> int: + """ + 使用当前会话以未租或过期条件竞争一个新租约。 + + :param task_id: 稳定任务标识 + :param states: 可恢复业务状态 + :param owner_id: 新租约拥有者 + :param lease_token: 新租约唯一令牌 + :param now_time: 当前 UTC 时间 + :param lease_expires_at: 新租约到期时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.claim_task( + session, + task_id=task_id, + states=states, + owner_id=owner_id, + lease_token=lease_token, + now_time=now_time, + lease_expires_at=lease_expires_at, + updated_at=updated_at, + ) + ) + + def stage_record_projection_failure( + self, + *, + task_id: str, + states: tuple[str, ...], + error: str, + now_time: str, + updated_at: str, + ) -> int: + """ + 使用当前会话按无有效租约和诊断变化条件记录投影损坏。 + + :param task_id: 稳定任务标识 + :param states: 可恢复业务状态 + :param error: 可持久化的稳定诊断文本 + :param now_time: 当前 UTC 租约时间 + :param updated_at: 宿主本地业务审计时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.record_projection_failure( + session, + task_id=task_id, + states=states, + error=error, + now_time=now_time, + updated_at=updated_at, + ) + ) + + def stage_heartbeat( + self, + *, + task_id: str, + lease_token: str, + now_time: str, + lease_expires_at: str, + ) -> int: + """ + 使用当前会话以当前未过期 token 延长租约。 + + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param now_time: 当前 UTC 时间 + :param lease_expires_at: 新租约到期时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.heartbeat( + session, + task_id=task_id, + lease_token=lease_token, + now_time=now_time, + lease_expires_at=lease_expires_at, + ) + ) + + def stage_release_claim( + self, + *, + task_id: str, + lease_token: str, + error: Optional[str], + now_time: str, + updated_at: str, + ) -> int: + """ + 使用当前会话按未过期 token 释放租约并记录本次错误。 + + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param error: 本次执行错误，成功释放时为空 + :param now_time: 当前 UTC 时间 + :param updated_at: 与既有业务审计字段一致的宿主本地时间 + :return: 更新的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.release_claim( + session, + task_id=task_id, + lease_token=lease_token, + error=error, + now_time=now_time, + updated_at=updated_at, + ) + ) + + def stage_discard_claimed( + self, + *, + task_id: str, + lease_token: str, + now_time: str, + ) -> int: + """ + 使用当前会话按当前未过期 token 删除终态任务。 + + :param task_id: 稳定任务标识 + :param lease_token: 当前租约令牌 + :param now_time: 当前 UTC 时间 + :return: 删除的记录数 + """ + return self._execute_sync_write( + lambda session: TransferPending.discard_claimed( + session, + task_id=task_id, + lease_token=lease_token, + now_time=now_time, ) ) @@ -194,56 +328,3 @@ class TransferPendingOper(DbOper): now_time=now_time, ) ) - - def stage_discard_task(self, *, task_id: str) -> int: - """ - 在当前会话中暂存按任务标识删除接纳记录。 - :param task_id: 任务标识 - :return: 删除的记录数 - """ - return self._execute_sync_write( - lambda session: TransferPending.discard_task( - session, - task_id=task_id, - ) - ) - - def discard(self, storage: str, src_path: str) -> int: - """ - 注销一个待整理文件登记。 - :param storage: 存储 - :param src_path: 源文件路径 - :return: 删除的记录数 - """ - return self._execute_sync_write( - lambda session: TransferPending.discard( - session, - storage=storage, - src_path=src_path, - ) - ) - - def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]: - """ - 列出全部待整理登记，供启动回放使用。 - - 返回纯元组而不是 ORM 实例：回放发生在会话之外，ORM 实例脱离 session - 后访问属性会触发 DetachedInstanceError。 - :param limit: 单次回放上限 - :return: (存储, 源文件路径) 列表 - """ - items = self._execute_sync_query( - lambda session: TransferPending.list_all(session, limit=limit) - ) - return [ - (item.storage, item.src_path) - for item in items or [] - if item and item.storage and item.src_path - ] - - def clear(self) -> int: - """ - 清空全部待整理登记。 - :return: 删除的记录数 - """ - return self._execute_sync_write(TransferPending.clear) diff --git a/app/runtime/compat/manifest.py b/app/runtime/compat/manifest.py index 76149e127..535439b51 100644 --- a/app/runtime/compat/manifest.py +++ b/app/runtime/compat/manifest.py @@ -135,10 +135,10 @@ MODULE_ALIASES: Dict[str, ModuleAlias] = { owner="sdk", ), "app.db.transferpending_oper": ModuleAlias( - target="app.db.oper.transferpending", - replacement="app.db.oper.transferpending", + target="app.sdk._legacy.transferpending", + replacement="app.application.transfer", introduced="v3.0.0", - owner="db", + owner="sdk", ), "app.db.user_oper": ModuleAlias( target="app.sdk._legacy.user", diff --git a/app/sdk/_legacy/transferpending.py b/app/sdk/_legacy/transferpending.py new file mode 100644 index 000000000..791e0a22e --- /dev/null +++ b/app/sdk/_legacy/transferpending.py @@ -0,0 +1,185 @@ +"""兼容旧 ``app.db.transferpending_oper`` 的无 Session 数据访问接口。""" + +from datetime import datetime +from typing import List, Optional, Tuple +from uuid import uuid4 + +from sqlalchemy import delete, select + +from app.db.base import DbOper, execute_dml +from app.db.models.transferpending import TransferPending as _TransferPending + + +class TransferPendingOper(DbOper): + """ + 保留旧待整理登记 ABI，并对执行租约实施兼容写 fencing。 + + 本类只由精确旧导入映射加载。宿主整理链仍使用 Application Port 和显式 + Session 的 canonical Oper；旧删除入口只能处理从未取得租约的记录。 + """ + + def register(self, storage: str, src_path: str) -> Optional[_TransferPending]: + """ + 按旧签名登记待整理文件，重复登记保持原任务和租约不变。 + + :param storage: 存储 + :param src_path: 源文件路径 + :return: 登记记录 + """ + now_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return self._execute_sync_write( + lambda session: _TransferPending.stage_admit( + session, + task_id=uuid4().hex, + storage=storage, + src_path=src_path, + state="accepted", + now_time=now_time, + ) + ) + + def list_by_state( + self, + *, + state: str, + limit: Optional[int] = 5000, + ) -> List[_TransferPending]: + """ + 按旧签名和登记顺序列出指定状态记录。 + + :param state: 持久状态 + :param limit: 单次读取上限 + :return: ORM 接纳记录列表 + """ + if not state: + return [] + return self._execute_sync_query( + lambda session: list(session.execute( + select(_TransferPending) + .where(_TransferPending.state == state) + .order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc()) + .limit(limit) + ).scalars().all()) + ) + + def list_by_states( + self, + *, + states: tuple[str, ...], + limit: Optional[int] = 5000, + ) -> List[_TransferPending]: + """ + 按旧签名和登记顺序列出多个状态记录。 + + :param states: 持久状态集合 + :param limit: 单次读取上限 + :return: ORM 接纳记录列表 + """ + if not states: + return [] + return self._execute_sync_query( + lambda session: list(session.execute( + select(_TransferPending) + .where(_TransferPending.state.in_(states)) + .order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc()) + .limit(limit) + ).scalars().all()) + ) + + def get_by_identity( + self, + *, + storage: str, + src_path: str, + ) -> Optional[_TransferPending]: + """ + 按旧签名查询指定存储与源路径的登记。 + + :param storage: 存储 + :param src_path: 源文件路径 + :return: 登记记录 + """ + return self._execute_sync_query( + lambda session: _TransferPending.get_by_identity( + session, + storage=storage, + src_path=src_path, + ) + ) + + def get_by_task_id(self, *, task_id: str) -> Optional[_TransferPending]: + """ + 按旧签名查询稳定任务标识对应的登记。 + + :param task_id: 稳定任务标识 + :return: 登记记录 + """ + return self._execute_sync_query( + lambda session: _TransferPending.get_by_task_id( + session, + task_id=task_id, + ) + ) + + def discard(self, storage: str, src_path: str) -> int: + """ + 按旧签名删除未 claim 的指定登记。 + + 任何带 token 的记录都由当前租约拥有者通过 fenced canonical API 收口； + 即使租约已经过期，旧插件也不得越权代替恢复调度器删除。 + :param storage: 存储 + :param src_path: 源文件路径 + :return: 删除的记录数 + """ + if not storage or not src_path: + return 0 + return self._execute_sync_write( + lambda session: execute_dml( + session, + delete(_TransferPending).where( + _TransferPending.storage == storage, + _TransferPending.src_path == src_path, + _TransferPending.lease_token.is_(None), + ), + execution_options={"synchronize_session": False}, + ) + ) + + def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]: + """ + 按旧返回形态列出全部待整理路径。 + + :param limit: 单次读取上限 + :return: ``(存储, 源文件路径)`` 列表 + """ + rows = self._execute_sync_query( + lambda session: session.execute( + select(_TransferPending.storage, _TransferPending.src_path) + .order_by(_TransferPending.created_at.asc(), _TransferPending.id.asc()) + .limit(limit) + ).all() + ) + return [ + (storage, src_path) + for storage, src_path in rows + if storage and src_path + ] + + def clear(self) -> int: + """ + 清空全部未 claim 登记，保留任何带租约 token 的任务。 + + :return: 删除的记录数 + """ + return self._execute_sync_write( + lambda session: execute_dml( + session, + delete(_TransferPending).where( + _TransferPending.lease_token.is_(None), + ), + execution_options={"synchronize_session": False}, + ) + ) + + +__all__ = ["TransferPendingOper"] diff --git a/app/startup/initializers/transfer.py b/app/startup/initializers/transfer.py index 6a242d6fe..87bde74d3 100644 --- a/app/startup/initializers/transfer.py +++ b/app/startup/initializers/transfer.py @@ -3,12 +3,12 @@ from app.chain.transfer import TransferChain def replay_pending_transfers(): """ - 回放上次进程退出时仍未整理完的文件。 + 启动唯一整理恢复调度器。 整理队列是纯内存的，挂载挂死后的人工重启、版本升级、OOM、宿主重启都会让 队列连同「这些文件还没整理」这个事实一起蒸发；而已稳定落地的文件不会再产生 任何监控事件，也不会有新的补偿扫描起点，结果就是永久漏件。 - 回放本身在后台线程执行，不阻塞启动流程。 + 启动回放、同进程补偿和过期租约接管共用同一个后台调度入口，不阻塞启动流程。 """ TransferChain().replay_pending() diff --git a/database/versions/d3a9e5f7b2c4_3_0_15.py b/database/versions/d3a9e5f7b2c4_3_0_15.py new file mode 100644 index 000000000..213a5b4dc --- /dev/null +++ b/database/versions/d3a9e5f7b2c4_3_0_15.py @@ -0,0 +1,124 @@ +"""3.0.15 为整理恢复任务增加强 CAS 租约字段。 + +Revision ID: d3a9e5f7b2c4 +Revises: c2f8a4d6e1b3 +Create Date: 2026-08-27 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "d3a9e5f7b2c4" +down_revision = "c2f8a4d6e1b3" +branch_labels = None +depends_on = None + +_TABLE_NAME = "transferpending" +_LEASE_INDEX = "ix_transferpending_recovery_lease" +_LEASE_COLUMNS = { + "lease_owner", + "lease_token", + "lease_expires_at", + "heartbeat_at", + "attempt_count", +} + + +def _column_names() -> set[str]: + """返回当前待整理登记表的字段集合。""" + inspector = sa.inspect(op.get_bind()) + if _TABLE_NAME not in inspector.get_table_names(): + return set() + return { + column["name"] + for column in inspector.get_columns(_TABLE_NAME) + } + + +def _index_names() -> set[str]: + """返回当前待整理登记表的索引名称集合。""" + inspector = sa.inspect(op.get_bind()) + if _TABLE_NAME not in inspector.get_table_names(): + return set() + return { + index["name"] + for index in inspector.get_indexes(_TABLE_NAME) + if index.get("name") + } + + +def upgrade() -> None: + """增加租约身份、到期、心跳和接管次数，并支持中断后重跑。""" + columns = _column_names() + if not columns: + return + additions = ( + ( + "lease_owner", + sa.Column("lease_owner", sa.String(length=128), nullable=True), + ), + ( + "lease_token", + sa.Column("lease_token", sa.String(length=64), nullable=True), + ), + ( + "lease_expires_at", + sa.Column("lease_expires_at", sa.String(length=40), nullable=True), + ), + ( + "heartbeat_at", + sa.Column("heartbeat_at", sa.String(length=40), nullable=True), + ), + ( + "attempt_count", + sa.Column("attempt_count", sa.Integer(), nullable=True), + ), + ) + for column_name, column in additions: + if column_name not in columns: + op.add_column(_TABLE_NAME, column) + + columns = _column_names() + if "attempt_count" in columns: + pending = sa.table( + _TABLE_NAME, + sa.column("attempt_count", sa.Integer()), + ) + op.get_bind().execute( + pending.update() + .where(pending.c.attempt_count.is_(None)) + .values(attempt_count=0) + ) + with op.batch_alter_table(_TABLE_NAME) as batch_op: + batch_op.alter_column( + "attempt_count", + existing_type=sa.Integer(), + nullable=False, + ) + + if _LEASE_INDEX not in _index_names(): + op.create_index( + _LEASE_INDEX, + _TABLE_NAME, + ["state", "lease_expires_at", "created_at", "id"], + unique=False, + ) + + +def downgrade() -> None: + """移除租约索引和字段，保留原业务状态及规划检查点。""" + columns = _column_names() + if not columns or not (_LEASE_COLUMNS & columns): + return + if _LEASE_INDEX in _index_names(): + op.drop_index(_LEASE_INDEX, table_name=_TABLE_NAME) + with op.batch_alter_table(_TABLE_NAME) as batch_op: + for column_name in ( + "attempt_count", + "heartbeat_at", + "lease_expires_at", + "lease_token", + "lease_owner", + ): + if column_name in columns: + batch_op.drop_column(column_name) diff --git a/docs/architecture-optimization-checklist.md b/docs/architecture-optimization-checklist.md index 040681927..8b77c93fd 100644 --- a/docs/architecture-optimization-checklist.md +++ b/docs/architecture-optimization-checklist.md @@ -69,7 +69,7 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` | 指标 | 当前值 | 解释 | |---|---:|---| -| 宿主 Python 模块 / 内部依赖边 | 836 / 6,832 | `dependency-baseline.json` 当前快照 | +| 宿主 Python 模块 / 内部依赖边 | 837 / 6,836 | `dependency-baseline.json` 当前快照 | | 非平凡 SCC | 2 | 新增 Chain 包根环；另一个是隔离的 29 模块 TMDB 移植包环 | | 跨层 DB 边界债务 | 0 | Application、Chain、API、Agent、Runtime、Workflow 到 DB 的受控债务均为零 | | Model/Oper 事务债务 | 0 | 自建 Session、自动事务装饰器、直接 commit/rollback 等基线均为零 | @@ -192,7 +192,8 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` - `S1-L1.2 Planning checkpoint`：`VERIFIED`。版本化请求与指纹先准入；无 legacy provider 时通过 `accepted -> planned` CAS 提交完整目标和有序操作，有 provider 时先提交 `provider_pending`，全部 返回空后再以第二次 CAS 提交 `planned`；planned 重放只消费冻结上下文和目标。 -- `S1-L1.3 Lease 与恢复调度`：`PLANNED`。交付 claim/lease/heartbeat/attempt、过期接管与唯一恢复入口。 +- `S1-L1.3 Lease 与恢复调度`：`VERIFIED`。已交付 token fencing 的 + claim/lease/heartbeat/attempt、过期接管、固定退避的唯一恢复入口和有界关闭 owner。 - `S1-L1.4 幂等执行与终态结算`：`PLANNED`。交付文件/历史幂等、唯一 retry owner 和 `manual_review` 语义。 - `S1-L1.5 E3 全链收口`：`PLANNED`。完成崩溃矩阵、兼容验收与旧路径删除。此叶交付前， @@ -210,9 +211,13 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` 解析并严格执行，缺失或异常不 fallback。全部返回空后才生成宿主计划，并以第二次 CAS 提升为 `planned` 后执行。旧 caller 只经 `ChainBase.transfer` 注入式兼容门面进入同一 durable command， 宿主 FileManager/TransHandler 的旧执行入口已删除。 -- `TransferPending` 现在可区分 `accepted/provider_pending/planned`，但尚无 - claim/lease/heartbeat/attempt、逐步骤执行结果和 `manual_review`，仍无法判定“文件已移动、历史未提交” - 等后续中间态。 +- `S1-L1.3` 已把执行所有权与 planning phase 正交：恢复任务入队前原子 claim，普通任务在任何业务 + 副作用前 claim；heartbeat、checkpoint、失败留痕、release 和终态删除均受当前未过期 token + fencing。启动和同进程恢复共享唯一 scheduler，确定性失败按固定轮询退避，关闭时 worker、replay、 + lease release 和 heartbeat 都由有界生命周期 owner 持有。损坏投影以无有效租约 CAS 留痕，同错不 + 重复刷写，且不会阻塞后续健康任务。 +- `TransferPending` 仍缺少逐步骤执行结果和 `manual_review`，因此还不能判定“文件已移动、历史未提交” + 等外部结果未知的后续中间态。 - 这与 `docs/adr/0007-background-action-reliability.md:123-139` 对 E3 的稳定身份、步骤状态、 lease/heartbeat 和人工恢复要求不一致。 @@ -222,7 +227,7 @@ MoviePilot V3 已经形成较清晰的模块化单体：`foundation`、`domain` 原子提交，入队失败时必须保留 pending 供重放。 - [x] 初始登记保存稳定源身份、版本化请求和状态；目标与有序操作在纯规划完成后以 planning checkpoint 原子更新，任何文件副作用不得早于该提交。 -- [ ] 增加 claim/lease/heartbeat/attempt 与过期接管，同一任务同时只能有一个 worker owner。 +- [x] 增加 claim/lease/heartbeat/attempt 与过期接管，同一任务同时只能有一个 worker owner。 - [ ] 设计幂等文件操作和历史提交；只有所有必要步骤达到持久终态后才能删除记录。 - [ ] 在持久状态机与现有失败历史/AI retry 之间指定唯一 retry owner，定义旧记录迁移和兼容规则。 - [ ] E3 失败使用持久 `failed/manual_review`、最后稳定 checkpoint 和补偿边界，不直接套用 E2 diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md index ba9a57eee..c683caa35 100644 --- a/docs/architecture-overview.md +++ b/docs/architecture-overview.md @@ -704,8 +704,8 @@ flowchart LR | 指标 | 当前值 | |---|---:| -| Python 模块 | 836 | -| 内部导入边 | 6,832 | +| Python 模块 | 837 | +| 内部导入边 | 6,836 | | 非平凡 SCC | 2（`ARCH-107` 临时 Chain 包根环；精确 containment 的 TMDB 移植包环） | | Direct egress | 66（12 条待迁移债务，54 条精确 containment） | | Module Contract V2 spec | 217（其中 215 个进入 `run_module` 观察面） | diff --git a/docs/architecture-refactor-roadmap.md b/docs/architecture-refactor-roadmap.md index d1fbe227c..816d9105e 100644 --- a/docs/architecture-refactor-roadmap.md +++ b/docs/architecture-refactor-roadmap.md @@ -94,7 +94,7 @@ G-ARCH 只有在以下条件全部满足后才可完成： |---|---|---|---| | S1-L1.1 Durable admission | `VERIFIED` | S0 | Application-owned typed Port + DB adapter + migration 落地；先持久 commit 再入队，入队失败保留可恢复记录；宿主不再通过 raw/`Any` `TransferPendingOper` 处理 admission | | S1-L1.2 Planning checkpoint | `VERIFIED` | S1-L1.1 | 版本化输入与指纹先持久化；无 legacy provider 时以 `accepted -> planned` CAS 提交完整计划，有 provider 时先提交 `provider_pending`，全部返回空后再以第二次 CAS 提交 `planned`；重放只执行冻结目标，所有文件副作用晚于对应 checkpoint commit | -| S1-L1.3 Lease 与恢复调度 | `PLANNED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地；启动回放和同进程恢复共用唯一调度入口，同一任务同时只有一个 worker owner | +| S1-L1.3 Lease 与恢复调度 | `VERIFIED` | S1-L1.2 | claim/lease/heartbeat/attempt 与过期接管规则落地；启动回放和同进程恢复共用唯一调度入口，同一任务同时只有一个 worker owner | | S1-L1.4 幂等执行与终态结算 | `PLANNED` | S1-L1.3 | 文件操作、历史提交和 checkpoint 可重放；唯一 retry owner 生效，未知外部结果进入 `manual_review`，仅完整终态删除 pending | | S1-L1.5 E3 全链收口 | `PLANNED` | S1-L1.4 | 崩溃矩阵、升级/降级、重复回放和插件 ABI 验收完整；旧 fail-open、重复状态与兼容层外旧入口删除，ARCH-102 债务归零 | | S1-L2 Workflow typed query | `PLANNED` | S0 | Workflow Application Port 不返回 `Any`/ORM，Session 内投影 DTO，正式调用方全部切换 | @@ -274,3 +274,83 @@ git diff --check - failure injection 覆盖 commit 前零文件副作用、commit 后崩溃重放、离线 resolved context 恢复、 配置漂移仍使用冻结 target storage、规划失败留痕、旧 provider 提交后短路、严格异常、空结果两阶段 fallback、缺失引用零 cleanup，以及 cleanup 顺序/幂等/瞬时失败。 + +### S1-L1.3 Lease 与恢复调度 + +**Status:** `VERIFIED` + +**Outcome** + +为 durable transfer 增加可过期、可接管且带 fencing token 的执行租约。`owner` 只用于标识执行者， +每次有效 claim 生成的新 token 才是后续 heartbeat、checkpoint、失败留痕和终态删除的授权；所有写入 +必须同时匹配当前且未过期的 token，过期 worker 即使稍后恢复也不能修改新 owner 的记录。 + +`accepted`、`provider_pending`、`planned` 仍是业务 planning phase，lease 与其正交：claim、heartbeat +和 takeover 不改变 phase，恢复 worker 继续按冻结 checkpoint 决定执行路径。启动回放与同进程恢复共用 +唯一调度入口：恢复任务在入队前完成原子 claim 并绑定 token，新 admission 由 worker 在任何副作用前 +claim。已经 claim 的恢复任务不在 worker 内二次 claim。 + +**State machine** + +| 当前租约 | 操作 | 结果与 fencing 约束 | +|---|---|---| +| 无租约 | `claim` | 生成新 token、设置 owner/到期时间并递增 attempt；phase 不变 | +| 当前租约未过期 | 同 token `heartbeat` | 只延长当前租约；owner、token、attempt 和 phase 不变 | +| 当前租约未过期 | 任意再次 `claim`，包括同 owner | 原子拒绝，不递增 attempt、不入队、不执行副作用 | +| 当前租约已过期 | `heartbeat` 或旧 token 写入 | 原子拒绝；旧租约不可复活 | +| 当前租约已过期 | 新 worker `claim` | 生成新 token 并递增 attempt；旧 token 永久失效，phase 不变 | +| 当前 token 有效 | checkpoint、失败留痕或终态删除 | 仅精确 token CAS 成功；状态提交后不得由旧 worker 覆盖 | + +任意时刻一条 pending 记录最多只有一个数据库认可的有效 fencing owner。该保证不等同于外部文件、 +插件或历史副作用 exactly-once；租约在不可中断调用期间过期时，旧调用的外部结果仍可能未知。 + +**Scheduling and shutdown** + +- 启动回放和同进程恢复只调用一个 claim-and-schedule 入口；禁止另建 list-then-enqueue 回放路径或 + 第二个 retry owner。批量恢复必须先原子 claim，再把已绑定 token 的任务交给唯一 worker 队列。 +- heartbeat 由 Transfer worker 生命周期拥有，不创建游离后台 owner；停止时先封口新 claim 和调度， + 再通知并有限等待 worker，heartbeat 在 worker 存活期间继续维持其租约。worker 收敛后才停止 + heartbeat；超时时保留仍存活的 worker/heartbeat owner 并返回失败。 +- checkpoint、失败留痕和 pending 删除在同次持久化写入中执行 token CAS；租约丢失后 worker 停止 + 继续提交状态，不以预查询替代 fencing，也不把 stale mutation 伪装成成功。 +- 确定性失败只确保唯一 scheduler 存在，不即时唤醒；新建的失败恢复 owner 首次扫描先等待固定轮询 + 周期。损坏持久投影按单任务事务回滚，再以无有效租约 CAS 留下去重诊断，不能饿死后续健康记录。 +- 精确旧 `app.db.transferpending_oper` 导入只解析到 `app/sdk/_legacy` 门面；canonical Model、Oper、 + Application Port 不保留旧 list-then-act 或无 fencing mutation。兼容 `discard/clear` 也不得删除任何 + 带 token 的有效或过期 claim。 + +**Excluded** + +- 本叶不承诺文件、legacy provider、历史写入或其他外部副作用 exactly-once，也不以延长 lease + 掩盖不可判定结果。 +- 文件步骤幂等键、逐步结果 checkpoint、崩溃后未知结果判定、唯一 retry owner 和 + `manual_review` 终态由 `S1-L1.4` 完整交付。 +- 不增加 `processing` 等与 planning phase 重复的业务状态；不修改插件公开 Transfer ABI，不扫描或 + 修改 `app/plugins/**`，不恢复宿主旧 pending 入口或重复导出。 + +**Acceptance matrix** + +| 场景 | 必须证明 | +|---|---| +| 新 admission 与恢复记录竞争 | 只有 claim 成功者入队并执行；失败者零文件副作用 | +| 两进程同时 claim 同一记录 | 仅一个新 token 成功，attempt 只按真实新 claim 增长 | +| heartbeat 与 takeover 竞争 | 未过期 heartbeat 可续租；过期租约不可复活，接管 token 唯一有效 | +| 旧 worker 延迟提交 | checkpoint、失败留痕和删除均被 token CAS 拒绝，不覆盖新 owner | +| 三种 planning phase 恢复 | claim/续租/接管保持 phase，并消费各自冻结输入或 checkpoint | +| 启动与同进程恢复同时触发 | 只经过唯一 scheduler 入口，同一任务只有一个 worker owner | +| 正常与超时关闭 | 先封口后有限等待；存活 scheduler/heartbeat/worker 不丢失 owner、不报告成功 | + +**Failure injection matrix** + +| 注入点 | 预期持久结果 | +|---|---| +| claim commit 前崩溃 | 无新 token、attempt 不变，可由后续 worker claim | +| claim commit 后、入队前崩溃 | 租约到期后可接管，新 token fencing 旧 worker | +| worker 执行前 lease 丢失 | 不执行文件副作用，不提交失败或终态状态 | +| heartbeat commit 前后崩溃 | 仅已提交到期时间生效；过期后不可用旧 token 续租 | +| checkpoint/失败留痕提交时被接管 | stale CAS 失败，新 owner 的 phase、错误和 token 不被覆盖 | +| 终态删除提交时被接管 | stale delete 为零行，pending 保留给当前 owner | +| shutdown 时 scheduler、worker 或 lease release 阻塞 | 有限等待返回失败并保留 owner/heartbeat，禁止清句柄后重建重复 owner | + +本叶验收还必须运行 lease persistence、replay/worker、startup lifecycle、migration、架构与兼容聚焦 +测试，以及锁定全量测试和 scoped Pylint；插件 ABI 只经统一 Compat/SDK 验证。 diff --git a/docs/rules/05-architecture.md b/docs/rules/05-architecture.md index c0dfc7bcd..ab2f25521 100644 --- a/docs/rules/05-architecture.md +++ b/docs/rules/05-architecture.md @@ -484,8 +484,30 @@ result permits host planning and a second CAS to `planned`. Host-only `plan_tran is the sole legacy caller facade and delegates the startup-injected durable command; `FileManagerModule.transfer` and `TransHandler.transfer_media` must not be recreated. -Canonical host chains never obtain `TransferPendingOper`; its no-Session API -remains only for the exact legacy plugin import contract. + +Transfer execution ownership is orthogonal to those planning phases. +`app/application/transfer.py` defines the claim, heartbeat, release and fenced +mutation Port; `app/db/adapters/transfer.py` implements each operation in a +short UoW with a unique lease token. Any active lease rejects another claim, +including one from the same process owner. Expired leases may be taken over with +a new token and incremented attempt count, while the stale token cannot renew, +checkpoint, record failure, release or delete the task. Startup replay and +same-process recovery use the single scheduler owned by `TransferChain`; the +scheduler claims before enqueueing, and queued or executing claims are renewed +by one lifecycle-managed heartbeat owner. Lease ownership guarantees one +database-authorized worker, not physical exactly-once behavior for an already +issued file or legacy-plugin side effect; step idempotency and unknown outcomes +remain explicit execution concerns. + +Canonical host chains never obtain `TransferPendingOper`. The canonical Model, +Oper and Application Port do not retain the historical `register`, `list_all`, +`discard`, `clear` or `list_by_*` surface. The exact +`app.db.transferpending_oper` mapping resolves instead to the private +`app/sdk/_legacy/transferpending.py` facade, which preserves the old no-Session +query ABI without becoming a host implementation. Its `register` delegates to +canonical durable admission without overwriting an existing row, while +`discard` and `clear` delete only rows whose `lease_token` is null; an active or +expired claimed task remains exclusively owned by fenced recovery APIs. ## Composition and Compatibility Boundaries diff --git a/tests/fixtures/architecture/dependency-baseline.json b/tests/fixtures/architecture/dependency-baseline.json index 9fcfdfee5..25c7df622 100644 --- a/tests/fixtures/architecture/dependency-baseline.json +++ b/tests/fixtures/architecture/dependency-baseline.json @@ -1441,8 +1441,8 @@ "runtime_only": true } }, - "edge_count": 6832, - "edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39", + "edge_count": 6836, + "edge_sha256": "3709e09a49257075f48a44aba353be1f25db610471ec756e09cbbfd7070acb9a", "edges": [ "app -> app.runtime", "app -> app.runtime.compat", @@ -7625,6 +7625,10 @@ "app.sdk._legacy.subscribe -> app.domain.context", "app.sdk._legacy.transfer -> app.application", "app.sdk._legacy.transfer -> app.application.transfer", + "app.sdk._legacy.transferpending -> app.db", + "app.sdk._legacy.transferpending -> app.db.base", + "app.sdk._legacy.transferpending -> app.db.models", + "app.sdk._legacy.transferpending -> app.db.models.transferpending", "app.sdk._legacy.user -> app.api", "app.sdk._legacy.user -> app.api.deps", "app.sdk._legacy.user -> app.db", @@ -8277,7 +8281,7 @@ "app.workflow.actions.transfer_file -> app.workflow", "app.workflow.actions.transfer_file -> app.workflow.actions" ], - "module_count": 836, + "module_count": 837, "modules": [ "app", "app.adapters", @@ -9058,6 +9062,7 @@ "app.sdk._legacy.history", "app.sdk._legacy.subscribe", "app.sdk._legacy.transfer", + "app.sdk._legacy.transferpending", "app.sdk._legacy.user", "app.sdk.browser", "app.sdk.cache", diff --git a/tests/fixtures/architecture/mypy-baseline.json b/tests/fixtures/architecture/mypy-baseline.json index 02ae0dd14..338f84127 100644 --- a/tests/fixtures/architecture/mypy-baseline.json +++ b/tests/fixtures/architecture/mypy-baseline.json @@ -1594,7 +1594,7 @@ "misc": 2, "no-any-return": 3, "no-untyped-call": 8, - "no-untyped-def": 14, + "no-untyped-def": 12, "operator": 3, "return-value": 2, "truthy-function": 5, diff --git a/tests/fixtures/architecture/runtime-contract-baseline.json b/tests/fixtures/architecture/runtime-contract-baseline.json index 377f06328..956c04fa7 100644 --- a/tests/fixtures/architecture/runtime-contract-baseline.json +++ b/tests/fixtures/architecture/runtime-contract-baseline.json @@ -249,9 +249,9 @@ "app.db.transferpending_oper": { "introduced": "v3.0.0", "is_package": false, - "owner": "db", - "replacement": "app.db.oper.transferpending", - "target": "app.db.oper.transferpending" + "owner": "sdk", + "replacement": "app.application.transfer", + "target": "app.sdk._legacy.transferpending" }, "app.db.user_oper": { "introduced": "v3.0.0", diff --git a/tests/test_architecture_dependencies.py b/tests/test_architecture_dependencies.py index 211953dff..eff18dd46 100644 --- a/tests/test_architecture_dependencies.py +++ b/tests/test_architecture_dependencies.py @@ -828,15 +828,19 @@ def test_models_and_base_require_explicit_database_sessions(): def test_plugin_sdk_does_not_import_or_export_host_models(): - """插件 SDK 只能暴露 Oper，不得把宿主 ORM Model 作为插件接口。""" + """插件 SDK 不得暴露 ORM Model，只有精确旧 ABI 门面可在内部访问。""" + internal_compat_imports = { + ("app/sdk/_legacy/transferpending.py", "app.db.models.transferpending"), + } violations: list[str] = [] for path in (APP_ROOT / "sdk").rglob("*.py"): + relative = path.relative_to(PROJECT_ROOT).as_posix() tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path)) for node in ast.walk(tree): if isinstance(node, ast.ImportFrom) and node.module and ( node.module == "app.db.models" or node.module.startswith("app.db.models.") - ): + ) and (relative, node.module) not in internal_compat_imports: violations.append( f"{path.relative_to(PROJECT_ROOT)}:{node.lineno}:{node.module}" ) diff --git a/tests/test_db_media_identity_normalizer.py b/tests/test_db_media_identity_normalizer.py index 59fc47833..5e52f0916 100644 --- a/tests/test_db_media_identity_normalizer.py +++ b/tests/test_db_media_identity_normalizer.py @@ -158,8 +158,18 @@ def test_tables_without_identity_columns_are_untouched(db): 不带身份列的表不受影响——事件挂在 Mapper 上覆盖全部映射，必须靠列名检查收窄， 否则会去动一张根本没有这两列的表。 """ - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-14 10:00:00") + TransferPending.stage_admit( + db.session, + task_id="identity-free-table", + storage="local", + src_path="/mnt/a.mkv", + state="accepted", + now_time="2026-08-14 10:00:00", + ) - rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/mnt/a.mkv"] - assert len(rows) == 1 + row = TransferPending.get_by_identity( + db.session, + storage="local", + src_path="/mnt/a.mkv", + ) + assert row.task_id == "identity-free-table" diff --git a/tests/test_db_transferpending_queries.py b/tests/test_db_transferpending_queries.py index a902937cf..39a7367a6 100644 --- a/tests/test_db_transferpending_queries.py +++ b/tests/test_db_transferpending_queries.py @@ -5,8 +5,10 @@ 任何一件出偏差，都直接表现为文件被漏整理或被重复整理，而不是一个可见的报错。 因此这里对着真实数据库断言查回的内容，而不是断言调用了什么。 """ +import inspect + import pytest -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker from app.db import base as db_base @@ -21,185 +23,6 @@ def _track(db): db.watermark(TransferPending) -def test_register_is_idempotent_and_keeps_first_time(db): - """ - 同一文件重复登记只保留一条，且登记时间保持首次的值。 - - 监控在挂载抖动时会对同一个文件反复触发事件，若每次都新增一条，回放时同一个 - 文件会被送进整理链多次。保留首次时间则保证回放顺序仍是「最早发现」的顺序。 - """ - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-13 10:00:00") - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-13 12:00:00") - - rows = TransferPending.list_all(db.session) - same_path = [r for r in rows if r.src_path == "/mnt/a.mkv"] - assert len(same_path) == 1 - assert same_path[0].created_at == "2026-08-13 10:00:00" - - -def test_register_scopes_by_storage(db): - """ - 存储不同即为不同文件——路径相同但分属不同存储时不能互相去重。 - """ - TransferPending.register(db.session, storage="local", src_path="/data/x.mkv", - now_time="2026-08-13 10:00:00") - TransferPending.register(db.session, storage="alist", src_path="/data/x.mkv", - now_time="2026-08-13 10:00:01") - - rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/data/x.mkv"] - assert {r.storage for r in rows} == {"local", "alist"} - - -@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", ""), ("", "")]) -def test_register_rejects_incomplete_identity(db, storage, src_path): - """ - 缺少存储或路径的登记必须直接丢弃，不能写入半条记录。 - - 半条记录回放时既定位不到文件、也无法被 discard 匹配，会永久留在表里。 - """ - assert TransferPending.register(db.session, storage=storage, src_path=src_path, - now_time="2026-08-13 10:00:00") is None - - -def test_list_all_replays_in_registration_order(db): - """ - 回放顺序必须是登记时间升序、同时间按主键升序。 - - 乱序回放会让后发现的文件先进整理链，与原入队顺序不一致。 - """ - for path, moment in [("/mnt/c.mkv", "2026-08-13 12:00:00"), - ("/mnt/a.mkv", "2026-08-13 10:00:00"), - ("/mnt/b.mkv", "2026-08-13 11:00:00")]: - TransferPending.register(db.session, storage="local", src_path=path, now_time=moment) - - ordered = [r.src_path for r in TransferPending.list_all(db.session) - if r.src_path.startswith("/mnt/")] - assert ordered == ["/mnt/a.mkv", "/mnt/b.mkv", "/mnt/c.mkv"] - - -def test_list_all_honours_limit(db): - """ - 回放上限必须生效——异常积压时一次性全放会把整理链直接压垮。 - """ - for index in range(5): - TransferPending.register(db.session, storage="local", src_path=f"/mnt/{index}.mkv", - now_time=f"2026-08-13 10:00:0{index}") - - assert len(TransferPending.list_all(db.session, limit=3)) == 3 - - -def test_discard_removes_only_the_matching_row(db): - """ - 注销只应删除匹配的那一条，并返回删除条数。 - - 整理到达终态时按「存储 + 路径」注销，误删其他登记等于把别的文件也判成已完成。 - """ - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-13 10:00:00") - TransferPending.register(db.session, storage="local", src_path="/mnt/b.mkv", - now_time="2026-08-13 10:00:01") - - assert TransferPending.discard(db.session, storage="local", src_path="/mnt/a.mkv") == 1 - - remaining = [r.src_path for r in TransferPending.list_all(db.session) - if r.src_path.startswith("/mnt/")] - assert remaining == ["/mnt/b.mkv"] - - -@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", "")]) -def test_discard_rejects_incomplete_identity(db, storage, src_path): - """ - 身份不全时必须直接返回 0，不能退化成「条件为空」的全表删除。 - """ - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-13 10:00:00") - - assert TransferPending.discard(db.session, storage=storage, src_path=src_path) == 0 - assert TransferPending.list_all(db.session) - - -def test_discard_returns_zero_when_absent(db): - """ - 注销不存在的登记返回 0，不抛异常——整理链的终态回调不应因此中断。 - """ - assert TransferPending.discard(db.session, storage="local", src_path="/nope.mkv") == 0 - - -def test_clear_empties_the_table(db): - """ - 清空返回删除条数且表内不再有登记。 - """ - TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv", - now_time="2026-08-13 10:00:00") - - assert TransferPending.clear(db.session) >= 1 - assert TransferPending.list_all(db.session) == [] - - -def test_oper_returns_plain_tuples_not_orm_instances(db): - """ - 回放接口必须返回纯元组。 - - 回放发生在会话之外，ORM 实例脱离 session 后访问属性会抛 - DetachedInstanceError——那时启动流程已经在跑，报错等于整批漏件。 - """ - oper = TransferPendingOper(db=db.session) - oper.register(storage="local", src_path="/mnt/a.mkv") - - listed = oper.list_all() - - assert ("local", "/mnt/a.mkv") in listed - assert all(isinstance(item, tuple) for item in listed) - - -def test_oper_reuses_explicit_query_session(db, monkeypatch): - """TransferPendingOper 查询必须复用调用方会话。""" - db.add(TransferPending( - storage="local", - src_path="/mnt/explicit.mkv", - created_at="2026-08-13 10:00:00", - )) - monkeypatch.setattr( - db_base, - "run_sync_transaction", - lambda _operation: (_ for _ in ()).throw( - AssertionError("不应创建额外同步事务") - ), - ) - - assert ("local", "/mnt/explicit.mkv") in TransferPendingOper(db.session).list_all() - - -def test_oper_drops_rows_with_missing_fields(db): - """ - 回放时必须跳过字段残缺的历史遗留行，不能把空存储送进整理链。 - - 列上有 NOT NULL 约束，残缺只可能表现为空串；直接绕过 register 写入， - 模拟历史数据或外部写库留下的半条记录。 - """ - db.add(TransferPending(storage="", src_path="/mnt/broken.mkv", - created_at="2026-08-13 10:00:00")) - oper = TransferPendingOper(db=db.session) - oper.register(storage="local", src_path="/mnt/ok.mkv") - - assert oper.list_all() == [("local", "/mnt/ok.mkv")] - - -def test_oper_discard_and_clear_report_counts(db): - """ - 注销与清空都要如实返回条数，调用方据此判断是否真的清理掉了。 - """ - oper = TransferPendingOper(db=db.session) - oper.register(storage="local", src_path="/mnt/a.mkv") - oper.register(storage="local", src_path="/mnt/b.mkv") - - assert oper.discard(storage="local", src_path="/mnt/a.mkv") == 1 - assert oper.clear() >= 1 - assert oper.list_all() == [] - - def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db): """显式准入重复执行时必须复用首个稳定任务标识。""" first = TransferPending.stage_admit( @@ -224,8 +47,32 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db): assert second.updated_at == "2026-08-27 10:00:00" -def test_state_queries_failure_record_and_task_discard(db): - """状态查询、失败留痕和按任务删除应共享同一稳定身份。""" +def test_unfenced_legacy_mutation_apis_are_absent() -> None: + """持久整理表不得重新暴露绕过稳定任务身份和租约的旧写入口。""" + for owner in (TransferPending, TransferPendingOper): + for method_name in ( + "register", + "discard", + "list_all", + "list_by_state", + "list_by_states", + "clear", + ): + assert not hasattr(owner, method_name) + + enqueue_failure_source = inspect.getsource( + TransferPending.record_enqueue_failure + ) + assert "cls.lease_token.is_(None)" in enqueue_failure_source + claimable_source = inspect.getsource( + TransferPending.list_claimable_candidates + ) + assert "after_cursor" in claimable_source + assert ".not_in(" not in claimable_source + + +def test_state_queries_and_failure_record_share_stable_identity(db): + """状态查询与失败留痕应共享同一稳定身份。""" TransferPending.stage_admit( db.session, task_id="task-accepted", @@ -244,11 +91,11 @@ def test_state_queries_failure_record_and_task_discard(db): )) db.session.flush() - accepted = TransferPending.list_by_state( + accepted = TransferPending.get_by_task_id( db.session, - state="accepted", + task_id="task-accepted", ) - assert [item.task_id for item in accepted] == ["task-accepted"] + assert accepted.state == "accepted" assert TransferPending.record_enqueue_failure( db.session, task_id="task-accepted", @@ -263,10 +110,6 @@ def test_state_queries_failure_record_and_task_discard(db): ) assert failed.last_error == "queue full" assert failed.updated_at == "2026-08-27 10:01:00" - assert TransferPending.discard_task( - db.session, - task_id="task-accepted", - ) == 1 def test_oper_staging_reuses_explicit_write_session(db, monkeypatch): @@ -288,15 +131,13 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch): now_time="2026-08-27 10:00:00", ) assert pending.task_id == "task-explicit" - assert [item.task_id for item in oper.list_by_state(state="accepted")] == [ - "task-explicit" - ] + assert oper.get_by_task_id(task_id="task-explicit").state == "accepted" assert oper.stage_record_enqueue_failure( task_id="task-explicit", error="queue full", now_time="2026-08-27 10:01:00", ) == 1 - assert oper.stage_discard_task(task_id="task-explicit") == 1 + assert oper.get_by_task_id(task_id="task-explicit").last_error == "queue full" def test_transactional_repository_commits_frozen_projections(tmp_path): @@ -317,16 +158,34 @@ def test_transactional_repository_commits_frozen_projections(tmp_path): assert repeated == admitted assert admitted.task_id assert admitted.state == "accepted" - assert repository.list_accepted() == [admitted] repository.record_enqueue_failure( task_id=admitted.task_id, error="queue full", ) - failed = repository.list_accepted()[0] - assert failed.last_error == "queue full" - assert repository.discard_task(task_id=admitted.task_id) == 1 - assert repository.list_accepted() == [] + with factory() as session: + failed = session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one() + assert failed.last_error == "queue full" + claimed = repository.claim_task( + task_id=admitted.task_id, + owner_id="repository-test-worker", + lease_seconds=60, + ) + assert claimed is not None + assert repository.discard_claimed( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + ) == 1 + with factory() as session: + assert session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one_or_none() is None engine.dispose() diff --git a/tests/test_legacy_import_compat.py b/tests/test_legacy_import_compat.py index ca138ba6f..2dc7eee88 100644 --- a/tests/test_legacy_import_compat.py +++ b/tests/test_legacy_import_compat.py @@ -305,10 +305,16 @@ def test_legacy_transfer_task_hides_internal_admission_identity(): public_fields = set(task.to_dict()) task.bind_admission_task_id("internal-task-id") + task.bind_execution_lease( + owner_id="internal-worker", + lease_token="internal-lease-token", + ) assert set(task.to_dict()) == public_fields assert "task_id" not in task.to_dict() assert "admission_task_id" not in task.to_dict() + assert "lease_owner" not in task.to_dict() + assert "lease_token" not in task.to_dict() def test_chain_media_legacy_scraping_symbols_resolve_to_scraping_chain(): diff --git a/tests/test_transfer_job_manager.py b/tests/test_transfer_job_manager.py index 044acf8b0..9c973ad0e 100644 --- a/tests/test_transfer_job_manager.py +++ b/tests/test_transfer_job_manager.py @@ -1,4 +1,5 @@ import unittest +from dataclasses import replace from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch @@ -8,7 +9,11 @@ from app.application.history import ( failed_retry_count, record_transfer_failure, ) -from app.application.transfer import TransferPlanningInput, TransferTask +from app.application.transfer import ( + TransferAdmission, + TransferPlanningInput, + TransferTask, +) from app.chain.transfer import JobManager, TransferChain from app.domain.context import MediaInfo from app.domain.meta.metabase import MetaBase @@ -151,6 +156,7 @@ def make_task(episode: int, season: int = 1) -> TransferTask: def make_transfer_chain() -> TransferChain: + """构造带内存 durable admission 契约的整理链测试骨架。""" chain = object.__new__(TransferChain) chain.jobview = JobManager() chain._media_exts = settings.RMT_MEDIAEXT @@ -161,6 +167,66 @@ def make_transfer_chain() -> TransferChain: ) chain._success_target_files = {} chain._scrape_batches = {} + admissions = MagicMock() + admissions_by_identity = {} + admissions_by_id = {} + + def admit(*, storage, src_path, planning_input=None): + """按源身份幂等返回测试用 durable admission。""" + identity = storage, src_path + existing = admissions_by_identity.get(identity) + if existing is not None: + return existing + admission = TransferAdmission( + task_id=f"test-task-{len(admissions_by_id) + 1}", + storage=storage, + src_path=src_path, + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + planning_input=planning_input, + ) + admissions_by_identity[identity] = admission + admissions_by_id[admission.task_id] = admission + return admission + + def claim_task(*, task_id, owner_id, lease_seconds): + """为测试任务返回唯一 token，并保留正式 claim 的参数约束。""" + assert lease_seconds > 0 + admission = admissions_by_id[task_id] + claimed = replace( + admission, + lease_owner=owner_id, + lease_token=f"lease-{task_id}", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=admission.attempt_count + 1, + ) + admissions_by_identity[(claimed.storage, claimed.src_path)] = claimed + admissions_by_id[task_id] = claimed + return claimed + + def checkpoint_plan(*, task_id, lease_token, input_fingerprint, checkpoint): + """回读带检查点的持久投影，供同步整理测试执行真实编排。""" + del input_fingerprint + admission = admissions_by_id[task_id] + assert admission.lease_token == lease_token + planned = replace( + admission, + state="provider_pending" if checkpoint.is_provider_pending else "planned", + checkpoint=checkpoint, + ) + admissions_by_identity[(planned.storage, planned.src_path)] = planned + admissions_by_id[task_id] = planned + return planned + + admissions.admit.side_effect = admit + admissions.claim_task.side_effect = claim_task + admissions.checkpoint_plan.side_effect = checkpoint_plan + admissions.discard_claimed.return_value = 1 + admissions.release_claim.return_value = True + chain._transfer_admissions = admissions + chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock() return chain diff --git a/tests/test_transfer_lease_migration.py b/tests/test_transfer_lease_migration.py new file mode 100644 index 000000000..ac8f2deb2 --- /dev/null +++ b/tests/test_transfer_lease_migration.py @@ -0,0 +1,197 @@ +"""整理恢复租约字段的 Alembic 可逆迁移测试。""" + +import importlib + +import sqlalchemy as sa +from alembic.migration import MigrationContext +from alembic.operations import Operations + +from app.db.models.transferpending import TransferPending + +MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15" +LEASE_COLUMNS = { + "lease_owner", + "lease_token", + "lease_expires_at", + "heartbeat_at", + "attempt_count", +} + + +def _bind_migration(monkeypatch, connection): + """把 3.0.15 迁移绑定到隔离数据库连接。""" + migration = importlib.import_module(MIGRATION) + monkeypatch.setattr( + migration, + "op", + Operations(MigrationContext.configure(connection)), + ) + return migration + + +def _create_planning_table(connection) -> None: + """创建 3.0.14 时代包含完整规划检查点的登记表。""" + metadata = sa.MetaData() + table = sa.Table( + "transferpending", + metadata, + sa.Column("id", sa.Integer(), primary_key=True), + sa.Column("task_id", sa.String(64), nullable=False), + sa.Column("storage", sa.String(), nullable=False), + sa.Column("src_path", sa.String(), nullable=False), + sa.Column("created_at", sa.String(), nullable=True), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("updated_at", sa.String(40), nullable=False), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("input_version", sa.Integer(), nullable=False), + sa.Column("planning_input", sa.JSON(), nullable=False), + sa.Column("input_fingerprint", sa.String(64), nullable=False), + sa.Column("checkpoint_version", sa.Integer(), nullable=True), + sa.Column("checkpoint_payload", sa.JSON(), nullable=True), + sa.Column("planned_at", sa.String(40), nullable=True), + sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"), + ) + sa.Index( + "ux_transferpending_storage_path", + table.c.storage, + table.c.src_path, + unique=True, + ) + sa.Index( + "ix_transferpending_state_created", + table.c.state, + table.c.created_at, + table.c.id, + ) + metadata.create_all(connection) + connection.execute( + table.insert(), + { + "id": 1, + "task_id": "stable-task", + "storage": "local", + "src_path": "/downloads/Movie.mkv", + "created_at": "2026-08-27 10:00:00", + "state": "planned", + "updated_at": "2026-08-27 10:00:00", + "last_error": None, + "input_version": 1, + "planning_input": { + "schema_version": 1, + "source_fileitem": { + "storage": "local", + "path": "/downloads/Movie.mkv", + }, + }, + "input_fingerprint": "0" * 64, + "checkpoint_version": 1, + "checkpoint_payload": {"schema_version": 1}, + "planned_at": "2026-08-27 10:00:00", + }, + ) + + +def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None: + """SQLite 应支持租约字段重复升级、降级和再次升级。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + _create_planning_table(connection) + migration = _bind_migration(monkeypatch, connection) + + migration.upgrade() + migration.upgrade() + + inspector = sa.inspect(connection) + assert { + column["name"] + for column in inspector.get_columns("transferpending") + } == {column.name for column in TransferPending.__table__.columns} + assert "ix_transferpending_recovery_lease" in { + index["name"] + for index in inspector.get_indexes("transferpending") + } + row = connection.execute( + sa.text( + "SELECT lease_owner, lease_token, lease_expires_at, " + "heartbeat_at, attempt_count FROM transferpending WHERE id = 1" + ) + ).mappings().one() + assert dict(row) == { + "lease_owner": None, + "lease_token": None, + "lease_expires_at": None, + "heartbeat_at": None, + "attempt_count": 0, + } + + connection.execute( + sa.text( + "UPDATE transferpending SET lease_owner = 'worker', " + "lease_token = 'token', " + "lease_expires_at = '2026-08-27 10:05:00.000000', " + "heartbeat_at = '2026-08-27 10:00:00.000000', " + "attempt_count = 3 WHERE id = 1" + ) + ) + migration.downgrade() + + downgraded = sa.inspect(connection) + assert LEASE_COLUMNS.isdisjoint({ + column["name"] + for column in downgraded.get_columns("transferpending") + }) + assert "ix_transferpending_recovery_lease" not in { + index["name"] + for index in downgraded.get_indexes("transferpending") + } + assert connection.execute( + sa.text("SELECT state FROM transferpending WHERE id = 1") + ).scalar_one() == "planned" + + migration.upgrade() + reupgraded = connection.execute( + sa.text( + "SELECT lease_owner, lease_token, attempt_count " + "FROM transferpending WHERE id = 1" + ) + ).mappings().one() + assert dict(reupgraded) == { + "lease_owner": None, + "lease_token": None, + "attempt_count": 0, + } + engine.dispose() + + +def test_partial_transfer_lease_upgrade_preserves_existing_owner( + monkeypatch, +) -> None: + """迁移中断后重跑应补齐字段且不得覆盖已经写入的租约拥有者。""" + engine = sa.create_engine("sqlite://") + with engine.begin() as connection: + _create_planning_table(connection) + connection.execute( + sa.text("ALTER TABLE transferpending ADD COLUMN lease_owner VARCHAR(128)") + ) + connection.execute( + sa.text( + "UPDATE transferpending SET lease_owner = 'preserved-worker' " + "WHERE id = 1" + ) + ) + migration = _bind_migration(monkeypatch, connection) + + migration.upgrade() + + row = connection.execute( + sa.text( + "SELECT lease_owner, lease_token, attempt_count " + "FROM transferpending WHERE id = 1" + ) + ).mappings().one() + assert dict(row) == { + "lease_owner": "preserved-worker", + "lease_token": None, + "attempt_count": 0, + } + engine.dispose() diff --git a/tests/test_transfer_lease_persistence.py b/tests/test_transfer_lease_persistence.py new file mode 100644 index 000000000..b8355923d --- /dev/null +++ b/tests/test_transfer_lease_persistence.py @@ -0,0 +1,801 @@ +"""整理恢复租约的原子 claim、续租和陈旧 token 防护测试。""" + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from threading import Barrier, Lock +from typing import Any + +import pytest +from sqlalchemy import create_engine, select, text +from sqlalchemy.orm import sessionmaker + +from app.application.transfer import ( + TRANSFER_ADMISSION_PLANNED, + TransferAdmission, + TransferAdmissionProjectionError, + TransferLeaseLostError, + TransferPlanCheckpoint, + TransferPlanningInput, + TransferProviderInvocationSnapshot, + TransferProviderReference, +) +from app.db.adapters.transfer import TransactionalTransferAdmissionRepository +from app.db.models.transferpending import TransferPending +from app.db.oper.transferpending import TransferPendingOper + + +def _planning_input(path: str) -> TransferPlanningInput: + """构造与测试源路径绑定的最小规划输入。""" + return TransferPlanningInput( + source_fileitem={ + "storage": "local", + "path": path, + "type": "file", + "name": path.rsplit("/", maxsplit=1)[-1], + }, + meta={"name": "Movie"}, + mediainfo={"title": "Movie"}, + ) + + +def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint: + """构造无需文件副作用的合法宿主跳过检查点。""" + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="local", + root_target_path="/library", + final_target_path="/library", + resolved_transfer_type="copy", + items=(), + skip_reason="测试跳过计划", + ) + + +def _provider_checkpoint( + planning_input: TransferPlanningInput, +) -> TransferPlanCheckpoint: + """构造只冻结 provider ABI、尚未完成宿主规划的检查点。""" + invocation = TransferProviderInvocationSnapshot( + fileitem=planning_input.source_fileitem, + meta=planning_input.meta, + meta_kind="MetaVideo", + mediainfo=planning_input.mediainfo, + mediainfo_kind="MediaInfo", + ) + return TransferPlanCheckpoint( + planning_input=planning_input, + target_storage="", + root_target_path="", + final_target_path="", + resolved_transfer_type="", + items=(), + resolved_meta=invocation.meta, + resolved_meta_kind=invocation.meta_kind, + resolved_mediainfo=invocation.mediainfo, + resolved_mediainfo_kind=invocation.mediainfo_kind, + legacy_transfer_providers=( + TransferProviderReference( + plugin_id="provider-a", + plugin_name="Provider A", + ), + ), + provider_invocation=invocation, + ) + + +@pytest.fixture +def repository_factory(tmp_path): + """创建允许多线程独立 Session 竞争的 SQLite 租约仓储工厂。""" + engine = create_engine( + f"sqlite:///{tmp_path / 'transfer-lease.db'}", + connect_args={"check_same_thread": False, "timeout": 10}, + ) + TransferPending.__table__.create(engine) + factory = sessionmaker(bind=engine) + yield lambda: TransactionalTransferAdmissionRepository(factory) + engine.dispose() + + +@pytest.fixture +def lease_clock(monkeypatch): + """为所有仓储实例提供可推进的固定 UTC 时钟。""" + clock = {"now": datetime(2026, 8, 27, 10, 0, tzinfo=timezone.utc)} + monkeypatch.setattr( + TransactionalTransferAdmissionRepository, + "_lease_now", + staticmethod(lambda: clock["now"]), + ) + return clock + + +@pytest.fixture +def business_clock(monkeypatch): + """为业务审计字段提供与 UTC 租约时钟明确分离的宿主本地时间。""" + clock = {"now": "2026-08-27 18:00:00"} + monkeypatch.setattr( + TransactionalTransferAdmissionRepository, + "_now", + staticmethod(lambda: clock["now"]), + ) + return clock + + +def _admit( + repository: TransactionalTransferAdmissionRepository, + path: str, +) -> TransferAdmission: + """登记一个带完整版本化输入的测试任务。""" + return repository.admit( + storage="local", + src_path=path, + planning_input=_planning_input(path), + ) + + +def _pending_snapshot( + repository: TransactionalTransferAdmissionRepository, + task_id: str, +) -> dict[str, Any]: + """在 Session 关闭前冻结测试需要检查的持久登记字段。""" + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where(TransferPending.task_id == task_id) + ).scalar_one() + return { + "lease_owner": pending.lease_owner, + "lease_token": pending.lease_token, + "lease_expires_at": pending.lease_expires_at, + "heartbeat_at": pending.heartbeat_at, + "last_error": pending.last_error, + "attempt_count": pending.attempt_count, + "updated_at": pending.updated_at, + "planned_at": pending.planned_at, + } + + +def test_claim_heartbeat_expired_takeover_and_stale_token_guards( + repository_factory, + lease_clock, + business_clock, +) -> None: + """新 token 才增加 attempt，过期 token 不能续租、释放或删除接管者。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/movie.mkv") + assert admitted.created_at == "2026-08-27 18:00:00" + business_clock["now"] = "2026-08-27 18:01:00" + + first = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-a", + lease_seconds=60, + ) + assert first is not None + assert first.lease_owner == "worker-a" + assert first.lease_token + assert first.attempt_count == 1 + assert first.updated_at == "2026-08-27 18:01:00" + + assert repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-a", + lease_seconds=60, + ) is None + assert repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-b", + lease_seconds=60, + ) is None + + lease_clock["now"] += timedelta(seconds=30) + business_clock["now"] = "2026-08-27 18:02:00" + renewed = repository.heartbeat( + task_id=admitted.task_id, + lease_token=first.lease_token, + lease_seconds=60, + ) + assert renewed is not None + assert renewed.attempt_count == 1 + assert renewed.heartbeat_at == "2026-08-27 10:00:30.000000" + assert renewed.updated_at == first.updated_at + + lease_clock["now"] += timedelta(seconds=61) + business_clock["now"] = "2026-08-27 18:03:00" + assert repository.release_claim( + task_id=admitted.task_id, + lease_token=first.lease_token, + error="expired worker", + ) is False + assert repository.discard_claimed( + task_id=admitted.task_id, + lease_token=first.lease_token, + ) == 0 + takeover = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-b", + lease_seconds=60, + ) + assert takeover is not None + assert takeover.lease_token != first.lease_token + assert takeover.attempt_count == 2 + assert takeover.updated_at == "2026-08-27 18:03:00" + assert repository.heartbeat( + task_id=admitted.task_id, + lease_token=first.lease_token, + lease_seconds=60, + ) is None + business_clock["now"] = "2026-08-27 18:04:00" + assert repository.release_claim( + task_id=admitted.task_id, + lease_token=first.lease_token, + error="stale worker", + ) is False + assert repository.discard_claimed( + task_id=admitted.task_id, + lease_token=first.lease_token, + ) == 0 + + assert repository.release_claim( + task_id=admitted.task_id, + lease_token=takeover.lease_token, + error="retry later", + ) is True + released = _pending_snapshot(repository, admitted.task_id) + assert released["lease_owner"] is None + assert released["lease_token"] is None + assert released["lease_expires_at"] is None + assert released["heartbeat_at"] is None + assert released["last_error"] == "retry later" + assert released["attempt_count"] == 2 + assert released["updated_at"] == "2026-08-27 18:04:00" + + third = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-c", + lease_seconds=60, + ) + assert third is not None + assert third.attempt_count == 3 + assert repository.discard_claimed( + task_id=admitted.task_id, + lease_token=third.lease_token, + ) == 1 + with repository._session_factory() as session: # noqa: SLF001 + assert session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one_or_none() is None + + +def test_claim_recoverable_respects_order_limit_and_active_lease( + repository_factory, + lease_clock, +) -> None: + """批量恢复跳过有效租约，并按登记顺序逐条 CAS 到请求上限。""" + repository = repository_factory() + first = _admit(repository, "/downloads/a.mkv") + second = _admit(repository, "/downloads/b.mkv") + third = _admit(repository, "/downloads/c.mkv") + active = repository.claim_task( + task_id=first.task_id, + owner_id="active-worker", + lease_seconds=60, + ) + assert active is not None + + claimed = repository.claim_recoverable( + owner_id="recovery-worker", + limit=2, + lease_seconds=60, + ) + + assert [item.task_id for item in claimed] == [second.task_id, third.task_id] + assert all(item.lease_owner == "recovery-worker" for item in claimed) + assert all(item.attempt_count == 1 for item in claimed) + assert repository.claim_recoverable( + owner_id="other-worker", + limit=10, + lease_seconds=60, + ) == [] + + lease_clock["now"] += timedelta(seconds=61) + reclaimed = repository.claim_recoverable( + owner_id="takeover-worker", + limit=2, + lease_seconds=60, + ) + assert [item.task_id for item in reclaimed] == [first.task_id, second.task_id] + assert reclaimed[0].attempt_count == 2 + assert reclaimed[1].attempt_count == 2 + + +def test_claim_recoverable_skips_corrupt_projection_and_claims_later_tasks( + repository_factory, + lease_clock, + business_clock, + monkeypatch, +) -> None: + """毒行应留下单次诊断但不持有租约或饿死后续健康任务。""" + repository = repository_factory() + messages: list[str] = [] + monkeypatch.setattr( + "app.db.adapters.transfer._diagnostic_logger.error", + messages.append, + ) + corrupt = _admit(repository, "/downloads/a-corrupt.mkv") + healthy = [ + _admit(repository, "/downloads/b-healthy.mkv"), + _admit(repository, "/downloads/c-healthy.mkv"), + ] + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == corrupt.task_id + ) + ).scalar_one() + pending.input_fingerprint = "corrupt" + session.commit() + + claimed = repository.claim_recoverable( + owner_id="recovery-worker", + limit=2, + lease_seconds=60, + ) + + assert [item.task_id for item in claimed] == [item.task_id for item in healthy] + corrupt_snapshot = _pending_snapshot(repository, corrupt.task_id) + assert corrupt_snapshot["lease_token"] is None + assert corrupt_snapshot["attempt_count"] == 0 + assert corrupt_snapshot["last_error"].startswith("恢复投影失败:") + assert corrupt_snapshot["updated_at"] == "2026-08-27 18:00:00" + assert len(messages) == 1 + + business_clock["now"] = "2026-08-27 18:01:00" + assert repository.claim_recoverable( + owner_id="second-recovery-worker", + limit=1, + lease_seconds=60, + ) == [] + assert _pending_snapshot(repository, corrupt.task_id) == corrupt_snapshot + assert len(messages) == 1 + + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == corrupt.task_id + ) + ).scalar_one() + pending.input_fingerprint = _planning_input( + "/downloads/a-corrupt.mkv" + ).fingerprint + session.commit() + + repaired = repository.claim_recoverable( + owner_id="repaired-worker", + limit=1, + lease_seconds=60, + ) + assert [item.task_id for item in repaired] == [corrupt.task_id] + + +def test_projection_diagnostic_changes_are_recorded_once_each( + repository_factory, + business_clock, + monkeypatch, +) -> None: + """相同投影错误不重复写库，错误类型变化时才更新诊断并再次告警。""" + repository = repository_factory() + messages: list[str] = [] + monkeypatch.setattr( + "app.db.adapters.transfer._diagnostic_logger.error", + messages.append, + ) + admitted = _admit(repository, "/downloads/changing-corrupt.mkv") + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one() + pending.input_fingerprint = "corrupt" + session.commit() + + assert repository.claim_recoverable( + owner_id="recovery-a", + limit=1, + lease_seconds=60, + ) == [] + first = _pending_snapshot(repository, admitted.task_id) + assert len(messages) == 1 + + business_clock["now"] = "2026-08-27 18:01:00" + assert repository.claim_recoverable( + owner_id="recovery-b", + limit=1, + lease_seconds=60, + ) == [] + assert _pending_snapshot(repository, admitted.task_id) == first + assert len(messages) == 1 + + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one() + pending.input_fingerprint = _planning_input( + "/downloads/changing-corrupt.mkv" + ).fingerprint + pending.input_version = 999 + session.commit() + + business_clock["now"] = "2026-08-27 18:02:00" + assert repository.claim_recoverable( + owner_id="recovery-c", + limit=1, + lease_seconds=60, + ) == [] + changed = _pending_snapshot(repository, admitted.task_id) + assert changed["last_error"] != first["last_error"] + assert changed["updated_at"] == "2026-08-27 18:02:00" + assert len(messages) == 2 + + +def test_projection_diagnostic_cas_is_concurrency_safe( + repository_factory, + monkeypatch, +) -> None: + """并发恢复观察到同一损坏时只允许一个诊断写入者和一条运行日志。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/concurrent-corrupt.mkv") + messages: list[str] = [] + message_lock = Lock() + + def capture(message: str) -> None: + """并发安全收集错误日志。""" + with message_lock: + messages.append(message) + + monkeypatch.setattr( + "app.db.adapters.transfer._diagnostic_logger.error", + capture, + ) + barrier = Barrier(2) + projection_error = TransferAdmissionProjectionError("same corruption") + + def record(_: int) -> bool: + """让两个独立 Session 同时竞争同一诊断 CAS。""" + barrier.wait(timeout=5) + return repository_factory()._record_projection_failure( # noqa: SLF001 + task_id=admitted.task_id, + error=projection_error, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(record, range(2))) + + assert sorted(results) == [False, True] + assert len(messages) == 1 + snapshot = _pending_snapshot(repository, admitted.task_id) + assert snapshot["last_error"] == "恢复投影失败: same corruption" + + +def test_projection_diagnostic_does_not_overwrite_active_lease( + repository_factory, + monkeypatch, +) -> None: + """诊断 CAS 不得覆盖已经由健康 worker 取得有效租约的任务。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/active-lease.mkv") + claimed = repository.claim_task( + task_id=admitted.task_id, + owner_id="active-worker", + lease_seconds=60, + ) + assert claimed is not None + before = _pending_snapshot(repository, admitted.task_id) + messages: list[str] = [] + monkeypatch.setattr( + "app.db.adapters.transfer._diagnostic_logger.error", + messages.append, + ) + + recorded = repository._record_projection_failure( # noqa: SLF001 + task_id=admitted.task_id, + error=TransferAdmissionProjectionError("stale observation"), + ) + + assert recorded is False + assert _pending_snapshot(repository, admitted.task_id) == before + assert messages == [] + + +def test_projection_diagnostic_database_failure_propagates( + repository_factory, + monkeypatch, +) -> None: + """诊断留痕的数据库基础设施异常必须向上游传播而非静默跳过。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/db-error-corrupt.mkv") + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == admitted.task_id + ) + ).scalar_one() + pending.input_fingerprint = "corrupt" + session.commit() + + def fail_diagnostic(*_args, **_kwargs): + """模拟诊断短事务的底层数据库写入失败。""" + raise RuntimeError("database unavailable") + + monkeypatch.setattr( + TransferPendingOper, + "stage_record_projection_failure", + fail_diagnostic, + ) + + with pytest.raises(RuntimeError, match="database unavailable"): + repository.claim_recoverable( + owner_id="recovery-worker", + limit=1, + lease_seconds=60, + ) + + +def test_claim_task_wraps_persisted_json_decode_failure( + repository_factory, +) -> None: + """持久 JSON 解码错误应归类为投影损坏，而不是数据库基础设施故障。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/invalid-json.mkv") + with repository._session_factory() as session: # noqa: SLF001 + session.execute( + text( + "UPDATE transferpending SET planning_input = 'not-json' " + "WHERE task_id = :task_id" + ), + {"task_id": admitted.task_id}, + ) + session.commit() + + with pytest.raises(TransferAdmissionProjectionError, match="JSON"): + repository.claim_task( + task_id=admitted.task_id, + owner_id="recovery-worker", + lease_seconds=60, + ) + with repository._session_factory() as session: # noqa: SLF001 + snapshot = session.execute( + text( + "SELECT lease_token, attempt_count FROM transferpending " + "WHERE task_id = :task_id" + ), + {"task_id": admitted.task_id}, + ).mappings().one() + assert snapshot["lease_token"] is None + assert snapshot["attempt_count"] == 0 + + +def test_concurrent_recovery_callers_scan_past_lost_candidates( + repository_factory, + lease_clock, + monkeypatch, +) -> None: + """并发 caller 竞争同一首批后应继续向后扫描并各自填满限额。""" + setup_repository = repository_factory() + admitted = [ + _admit(setup_repository, f"/downloads/concurrent-{index}.mkv") + for index in range(4) + ] + barrier = Barrier(2) + barrier_lock = Lock() + initial_scans = 0 + original = TransferPending.list_claimable_candidates.__func__ + + def synchronized_candidates(cls, db, **kwargs): + """强制两个 caller 在取得相同首批候选后再进入逐任务 CAS。""" + nonlocal initial_scans + candidates = original(cls, db, **kwargs) + if kwargs.get("after_cursor") is None: + with barrier_lock: + initial_scans += 1 + barrier.wait(timeout=5) + return candidates + + monkeypatch.setattr( + TransferPending, + "list_claimable_candidates", + classmethod(synchronized_candidates), + ) + + def recover(owner_id: str) -> list[TransferAdmission]: + """使用独立仓储与 Session 执行一次有界恢复扫描。""" + return repository_factory().claim_recoverable( + owner_id=owner_id, + limit=2, + lease_seconds=60, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(recover, ("worker-a", "worker-b"))) + + assert initial_scans == 2 + assert [len(result) for result in results] == [2, 2] + claimed_task_ids = [item.task_id for result in results for item in result] + assert len(set(claimed_task_ids)) == 4 + assert set(claimed_task_ids) == {item.task_id for item in admitted} + + +def test_unclaimed_enqueue_failure_cannot_overwrite_claimed_task( + repository_factory, + business_clock, +) -> None: + """task-id-only 入队失败入口不得改写已经由 worker claim 的登记。""" + repository = repository_factory() + admitted = _admit(repository, "/downloads/claimed.mkv") + business_clock["now"] = "2026-08-27 18:01:00" + claimed = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-a", + lease_seconds=60, + ) + assert claimed is not None + before = _pending_snapshot(repository, admitted.task_id) + + business_clock["now"] = "2026-08-27 18:02:00" + repository.record_enqueue_failure( + task_id=admitted.task_id, + error="stale queue failure", + ) + + after = _pending_snapshot(repository, admitted.task_id) + assert after == before + + +def test_planning_writes_require_current_unexpired_lease( + repository_factory, + lease_clock, +) -> None: + """checkpoint 和规划错误均不得由已过期或已被接管的 worker 写入。""" + repository = repository_factory() + path = "/downloads/planning.mkv" + planning_input = _planning_input(path) + admitted = repository.admit( + storage="local", + src_path=path, + planning_input=planning_input, + ) + first = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-a", + lease_seconds=30, + ) + assert first is not None + + lease_clock["now"] += timedelta(seconds=31) + takeover = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-b", + lease_seconds=60, + ) + assert takeover is not None + + with pytest.raises(TransferLeaseLostError, match="租约"): + repository.record_planning_failure( + task_id=admitted.task_id, + lease_token=first.lease_token, + error="stale planning failure", + ) + with pytest.raises(TransferLeaseLostError, match="租约"): + repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=first.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + + repository.record_planning_failure( + task_id=admitted.task_id, + lease_token=takeover.lease_token, + error="retryable planning failure", + ) + planned = repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=takeover.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + repeated = repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=takeover.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=_checkpoint(planning_input), + ) + + assert planned.state == TRANSFER_ADMISSION_PLANNED + assert planned.last_error is None + assert repeated == planned + assert planned.attempt_count == 2 + + +def test_provider_checkpoint_sets_planned_time_only_after_host_plan( + repository_factory, + lease_clock, + business_clock, +) -> None: + """provider 快照不是规划完成，planned_at 只记录首次宿主完整计划。""" + repository = repository_factory() + path = "/downloads/provider-plan.mkv" + planning_input = _planning_input(path) + admitted = repository.admit( + storage="local", + src_path=path, + planning_input=planning_input, + ) + business_clock["now"] = "2026-08-27 18:01:00" + claimed = repository.claim_task( + task_id=admitted.task_id, + owner_id="worker-a", + lease_seconds=60, + ) + assert claimed is not None + + business_clock["now"] = "2026-08-27 18:02:00" + repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=_provider_checkpoint(planning_input), + ) + provider_snapshot = _pending_snapshot(repository, admitted.task_id) + assert provider_snapshot["planned_at"] is None + assert provider_snapshot["updated_at"] == "2026-08-27 18:02:00" + + business_clock["now"] = "2026-08-27 18:03:00" + checkpoint = _checkpoint(planning_input) + repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + planned_snapshot = _pending_snapshot(repository, admitted.task_id) + assert planned_snapshot["planned_at"] == "2026-08-27 18:03:00" + + business_clock["now"] = "2026-08-27 18:04:00" + repository.checkpoint_plan( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + input_fingerprint=planning_input.fingerprint, + checkpoint=checkpoint, + ) + assert _pending_snapshot(repository, admitted.task_id) == planned_snapshot + + +def test_concurrent_claim_uses_rowcount_as_single_winner( + repository_factory, + lease_clock, +) -> None: + """并发 worker 即使读取同一任务，也只能有一个 CAS 更新获胜。""" + setup_repository = repository_factory() + admitted = _admit(setup_repository, "/downloads/concurrent.mkv") + barrier = Barrier(2) + + def claim(owner_id: str): + """等待竞争者就绪后使用独立 Session claim 同一任务。""" + repository = repository_factory() + barrier.wait() + return repository.claim_task( + task_id=admitted.task_id, + owner_id=owner_id, + lease_seconds=60, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(claim, ("worker-a", "worker-b"))) + + winners = [result for result in results if result is not None] + assert len(winners) == 1 + assert winners[0].attempt_count == 1 diff --git a/tests/test_transfer_pending_legacy_compat.py b/tests/test_transfer_pending_legacy_compat.py new file mode 100644 index 000000000..34061608b --- /dev/null +++ b/tests/test_transfer_pending_legacy_compat.py @@ -0,0 +1,158 @@ +"""旧待整理 Oper 的精确插件兼容与租约 fencing 测试。""" + +import importlib +from collections.abc import Callable +from typing import Any + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker + +from app.db import base as db_base +from app.db.models.transferpending import TransferPending +from app.db.oper.transferpending import TransferPendingOper as CanonicalTransferPendingOper +from app.runtime.compat.manifest import MODULE_ALIASES + + +@pytest.fixture +def legacy_session_factory(tmp_path, monkeypatch): + """为无 Session 兼容 Oper 提供独占事务，并在提交后保留返回快照。""" + engine = create_engine(f"sqlite:///{tmp_path / 'legacy-transferpending.db'}") + TransferPending.__table__.create(engine) + factory = sessionmaker(bind=engine) + + def run_transaction(operation: Callable[[Session], Any]) -> Any: + """按生产组合根语义执行一次同步兼容事务。""" + with factory() as session: + session.expire_on_commit = False + try: + result = operation(session) + session.commit() + return result + except Exception: + session.rollback() + raise + + monkeypatch.setattr(db_base, "run_sync_transaction", run_transaction) + yield factory + engine.dispose() + + +def test_legacy_import_targets_private_sdk_facade() -> None: + """旧模块路径只能解析到私有 SDK 门面，不能回退到 canonical Oper。""" + alias = MODULE_ALIASES["app.db.transferpending_oper"] + legacy = importlib.import_module("app.db.transferpending_oper") + + assert alias.target == "app.sdk._legacy.transferpending" + assert alias.owner == "sdk" + assert alias.replacement == "app.application.transfer" + assert legacy is importlib.import_module(alias.target) + assert legacy.__all__ == ["TransferPendingOper"] + assert not hasattr(legacy, "TransferPending") + assert legacy.TransferPendingOper is not CanonicalTransferPendingOper + for internal_method in ( + "stage_admit", + "stage_claim_task", + "stage_discard_claimed", + "stage_checkpoint_plan", + ): + assert not hasattr(legacy.TransferPendingOper, internal_method) + + +def test_legacy_no_session_queries_preserve_historical_shapes( + legacy_session_factory, +) -> None: + """旧插件无需 Session 即可登记和查询，返回形态与原 ABI 一致。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper = legacy.TransferPendingOper() + + first = oper.register("local", "/downloads/first.mkv") + repeated = oper.register("local", "/downloads/first.mkv") + second = oper.register("remote", "/downloads/second.mkv") + + assert first is not None + assert repeated is not None + assert second is not None + assert repeated.task_id == first.task_id + assert oper.list_all() == [ + ("local", "/downloads/first.mkv"), + ("remote", "/downloads/second.mkv"), + ] + assert [item.task_id for item in oper.list_by_state(state="accepted")] == [ + first.task_id, + second.task_id, + ] + assert [ + item.task_id + for item in oper.list_by_states(states=("accepted", "planned"), limit=1) + ] == [first.task_id] + assert oper.get_by_identity( + storage="local", + src_path="/downloads/first.mkv", + ).task_id == first.task_id + assert oper.get_by_task_id(task_id=second.task_id).src_path == second.src_path + assert oper.register("", "/downloads/invalid.mkv") is None + assert oper.list_by_state(state="") == [] + assert oper.list_by_states(states=()) == [] + + +def test_legacy_mutations_never_delete_claimed_rows( + legacy_session_factory, +) -> None: + """旧 discard/clear 只处理未 claim 行，有效或过期 token 均受保护。""" + legacy = importlib.import_module("app.db.transferpending_oper") + oper = legacy.TransferPendingOper() + active = oper.register("local", "/downloads/active.mkv") + expired = oper.register("local", "/downloads/expired.mkv") + free = oper.register("local", "/downloads/free.mkv") + assert active is not None + assert expired is not None + assert free is not None + + with legacy_session_factory() as session: + assert TransferPending.claim_task( + session, + task_id=active.task_id, + states=("accepted",), + owner_id="active-worker", + lease_token="active-token", + now_time="2026-08-27 10:00:00.000000", + lease_expires_at="2026-08-27 10:01:00.000000", + updated_at="2026-08-27 18:00:00", + ) == 1 + assert TransferPending.claim_task( + session, + task_id=expired.task_id, + states=("accepted",), + owner_id="expired-worker", + lease_token="expired-token", + now_time="2026-08-27 10:00:00.000000", + lease_expires_at="2026-08-27 09:59:00.000000", + updated_at="2026-08-27 18:00:00", + ) == 1 + session.commit() + + repeated = oper.register("local", "/downloads/active.mkv") + assert repeated is not None + assert repeated.task_id == active.task_id + assert repeated.lease_token == "active-token" + assert oper.discard("local", "/downloads/active.mkv") == 0 + assert oper.discard("local", "/downloads/expired.mkv") == 0 + assert oper.discard("local", "/downloads/free.mkv") == 1 + + removable = oper.register("local", "/downloads/removable.mkv") + assert removable is not None + assert oper.clear() == 1 + assert oper.list_all() == [ + ("local", "/downloads/active.mkv"), + ("local", "/downloads/expired.mkv"), + ] + + with legacy_session_factory() as session: + rows = session.execute( + select(TransferPending).order_by(TransferPending.id.asc()) + ).scalars().all() + assert [(row.task_id, row.lease_token) for row in rows] == [ + (active.task_id, "active-token"), + (expired.task_id, "expired-token"), + ] diff --git a/tests/test_transfer_pending_replay.py b/tests/test_transfer_pending_replay.py index 5664429ca..6108e5dde 100644 --- a/tests/test_transfer_pending_replay.py +++ b/tests/test_transfer_pending_replay.py @@ -8,6 +8,8 @@ 这些测试固定三项不变量：入队即落盘登记、终态即注销、重启能回放。 """ import threading +import time +from dataclasses import replace from pathlib import Path from unittest.mock import MagicMock @@ -24,6 +26,17 @@ def _build_chain(admissions) -> TransferChain: """ chain = object.__new__(TransferChain) chain._transfer_admissions = admissions + chain._worker_owner_id = "test-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + chain._worker_state_lock = threading.RLock() + chain._closing = False + chain._recovery_wakeup_event = threading.Event() + chain._replay_stop_event = threading.Event() + chain._lease_heartbeat_stop_event = threading.Event() + chain._lease_heartbeat_thread = None + chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock() + chain._TransferChain__ensure_recovery_scheduler = MagicMock() return chain @@ -36,6 +49,11 @@ def _admission(path: str, task_id: str = "task-1") -> TransferAdmission: state="accepted", created_at="2026-08-27 10:00:00", updated_at="2026-08-27 10:00:00", + lease_owner="test-owner", + lease_token=f"lease-{task_id}", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, ) @@ -87,10 +105,19 @@ def test_discard_pending_on_terminal_state(): chain = _build_chain(admissions) task = _task("/mnt/cd2/downloads/Movie.2024.mkv") task.bind_admission_task_id("task-1") + task.bind_execution_lease(owner_id="test-owner", lease_token="lease-task-1") + chain._worker_owner_id = "test-owner" + chain._owned_leases = { + "task-1": ("lease-task-1", time.monotonic() + 120) + } + admissions.discard_claimed.return_value = 1 - chain._TransferChain__discard_pending(task) + assert chain._TransferChain__discard_pending(task) is True - admissions.discard_task.assert_called_once_with(task_id="task-1") + admissions.discard_claimed.assert_called_once_with( + task_id="task-1", + lease_token="lease-task-1", + ) def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch): @@ -101,7 +128,7 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch): media.write_bytes(b"x" * 10) admissions = MagicMock() - admissions.list_recoverable.return_value = [_admission(str(media))] + admissions.claim_recoverable.return_value = [_admission(str(media))] chain = _build_chain(admissions) transferred = [] @@ -128,14 +155,18 @@ def test_replay_discards_vanished_files(tmp_path): """ admissions = MagicMock() missing = tmp_path / "gone.mkv" - admissions.list_recoverable.return_value = [_admission(str(missing))] + admissions.claim_recoverable.return_value = [_admission(str(missing))] + admissions.discard_claimed.return_value = 1 chain = _build_chain(admissions) chain._execute_transfer = MagicMock() chain._TransferChain__replay_pending() chain._execute_transfer.assert_not_called() - admissions.discard_task.assert_called_once_with(task_id="task-1") + admissions.discard_claimed.assert_called_once_with( + task_id="task-1", + lease_token="lease-task-1", + ) def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch): @@ -148,7 +179,8 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch): media.write_bytes(b"x") admissions = MagicMock() - admissions.list_recoverable.return_value = [_admission(str(media))] + admissions.claim_recoverable.return_value = [_admission(str(media))] + admissions.release_claim.return_value = True chain = _build_chain(admissions) chain._execute_transfer = MagicMock() @@ -163,7 +195,12 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch): chain._TransferChain__replay_pending() chain._execute_transfer.assert_not_called() - admissions.discard_task.assert_not_called() + admissions.discard_claimed.assert_not_called() + admissions.release_claim.assert_called_once_with( + task_id="task-1", + lease_token="lease-task-1", + error="恢复源文件暂时不可读取", + ) def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch): @@ -175,7 +212,7 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch): src_path = f"{bluray.as_posix()}/" admissions = MagicMock() - admissions.list_recoverable.return_value = [_admission(src_path)] + admissions.claim_recoverable.return_value = [_admission(src_path)] chain = _build_chain(admissions) transferred = [] @@ -197,7 +234,7 @@ def test_replay_is_noop_without_registrations(): 没有登记时回放不应触碰整理链。 """ admissions = MagicMock() - admissions.list_recoverable.return_value = [] + admissions.claim_recoverable.return_value = [] chain = _build_chain(admissions) chain._execute_transfer = MagicMock() @@ -211,7 +248,7 @@ def test_replay_survives_db_failure(): 读取登记失败不能让启动流程报错。 """ admissions = MagicMock() - admissions.list_recoverable.side_effect = RuntimeError("db gone") + admissions.claim_recoverable.side_effect = RuntimeError("db gone") chain = _build_chain(admissions) chain._execute_transfer = MagicMock() @@ -230,7 +267,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch): item.write_bytes(b"x") admissions = MagicMock() - admissions.list_recoverable.return_value = [ + admissions.claim_recoverable.return_value = [ _admission(str(first), "task-1"), _admission(str(second), "task-2"), ] @@ -261,7 +298,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch): first.write_bytes(b"x") missing_second = tmp_path / "gone.mkv" admissions = MagicMock() - admissions.list_recoverable.return_value = [ + admissions.claim_recoverable.return_value = [ _admission(str(first), "task-1"), _admission(str(missing_second), "task-2"), ] @@ -279,4 +316,94 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch): chain._TransferChain__replay_pending(stop_event) assert transferred == [first.as_posix()] - admissions.discard_task.assert_not_called() + admissions.discard_claimed.assert_not_called() + assert admissions.release_claim.call_count == 2 + + +def test_replay_registers_entire_claimed_batch_before_first_source_stat( + tmp_path, + monkeypatch, +): + """批量 claim 返回后必须先把全部 token 交给 heartbeat，再做逐条同步 I/O。""" + first = _admission(str(tmp_path / "A.mkv"), "task-1") + second = _admission(str(tmp_path / "B.mkv"), "task-2") + admissions = MagicMock() + admissions.claim_recoverable.return_value = [first, second] + admissions.release_claim.return_value = True + chain = _build_chain(admissions) + + def observe_owned_batch(*_args, **_kwargs): + """首个 stat 前观察两个 token 已同时进入续期集合。""" + assert set(chain._owned_leases) == {"task-1", "task-2"} + return None, False + + monkeypatch.setattr( + chain, + "_TransferChain__build_replay_fileitem", + observe_owned_batch, + ) + + chain._TransferChain__replay_pending() + + assert admissions.release_claim.call_count == 2 + assert chain._owned_leases == {} + + +def test_replay_releases_claim_when_jobview_rejects_recovered_task( + tmp_path, + monkeypatch, +): + """恢复任务未进入队列时必须立即 release，不能靠租约自然过期。""" + media = tmp_path / "Movie.2024.mkv" + media.write_bytes(b"x") + admission = _admission(str(media)) + admission = replace(admission, checkpoint=MagicMock()) + admissions = MagicMock() + admissions.claim_recoverable.return_value = [admission] + admissions.release_claim.return_value = True + chain = _build_chain(admissions) + monkeypatch.setattr( + chain, + "_TransferChain__queue_planned_replay", + MagicMock(return_value=False), + ) + + chain._TransferChain__replay_pending() + + admissions.release_claim.assert_called_once_with( + task_id="task-1", + lease_token="lease-task-1", + error="恢复任务未进入内存队列", + ) + assert chain._owned_leases == {} + + +def test_claimed_enqueue_failure_never_uses_unfenced_error_writer() -> None: + """陈旧 token 入队失败只能 release_claim，不能覆盖新 owner 的 last_error。""" + admissions = MagicMock() + admissions.release_claim.return_value = False + chain = _build_chain(admissions) + chain._finish_scrape_batch_task = MagicMock() + chain.replay_pending = MagicMock() + task = _task("/downloads/stale-enqueue.mkv") + task.bind_admission_task_id("stale-task") + task.bind_execution_lease( + owner_id="test-owner", + lease_token="stale-token", + ) + chain._owned_leases = { + "stale-task": ("stale-token", time.monotonic() + 120) + } + + chain._TransferChain__record_enqueue_failure( + task, + RuntimeError("queue closed"), + ) + + admissions.record_enqueue_failure.assert_not_called() + admissions.release_claim.assert_called_once_with( + task_id="stale-task", + lease_token="stale-token", + error="queue closed", + ) + assert chain._owned_leases == {} diff --git a/tests/test_transfer_planning_checkpoint.py b/tests/test_transfer_planning_checkpoint.py index d89eba76a..c72e3c9f0 100644 --- a/tests/test_transfer_planning_checkpoint.py +++ b/tests/test_transfer_planning_checkpoint.py @@ -2,13 +2,14 @@ import ast import inspect +import threading from dataclasses import replace from pathlib import Path from types import SimpleNamespace from unittest.mock import Mock, patch import pytest -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker from app.application import transfer as transfer_application @@ -202,6 +203,31 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain: """构造只保留规划编排依赖的 TransferChain 骨架。""" chain = object.__new__(TransferChain) chain._transfer_admissions = repository or Mock() + chain._worker_owner_id = "planning-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + chain._worker_state_lock = threading.RLock() + chain._closing = False + chain._recovery_wakeup_event = threading.Event() + chain._TransferChain__ensure_lease_heartbeat_owner = Mock() + + def claim_task(**kwargs): + """为规划测试返回与进程 owner 匹配的稳定 claim。""" + return transfer_application.TransferAdmission( + task_id=kwargs["task_id"], + storage="local", + src_path="/downloads/Movie.2026.mkv", + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + lease_owner=kwargs["owner_id"], + lease_token=f"lease-{kwargs['task_id']}", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, + ) + + chain._transfer_admissions.claim_task.side_effect = claim_task chain._module_dispatcher = Mock() chain._module_dispatcher.freeze_plugin_providers.return_value = () chain.eventmanager = Mock() @@ -217,6 +243,20 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain: return chain +def _replay_chain(repository) -> TransferChain: + """构造绑定固定恢复 owner 且不启动真实 heartbeat 线程的测试链。""" + chain = object.__new__(TransferChain) + chain._transfer_admissions = repository + chain._worker_owner_id = "replay-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + chain._worker_state_lock = threading.RLock() + chain._closing = False + chain._recovery_wakeup_event = threading.Event() + chain._TransferChain__ensure_lease_heartbeat_owner = Mock() + return chain + + def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher: """构造使用真实冻结解析与执行内核的内存插件调度器。""" plugin_catalog = Mock() @@ -725,6 +765,7 @@ def test_provider_empty_fallback_cas_failure_blocks_host_execution(): chain.execute_transfer_plan.assert_not_called() repository.record_planning_failure.assert_called_once_with( task_id="task-provider-cas-failure", + lease_token="lease-task-provider-cas-failure", error="CAS failed", ) @@ -804,7 +845,7 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): return SimpleNamespace(checkpoint=kwargs["checkpoint"]) repository.checkpoint_plan.side_effect = checkpoint_plan - repository.discard_task.side_effect = ( + repository.discard_claimed.side_effect = ( lambda **_kwargs: calls.append("discard") or 1 ) result = TransferInfo( @@ -830,8 +871,9 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending(): assert returned is result assert calls == ["admit", "checkpoint", "execute", "discard"] - repository.discard_task.assert_called_once_with( - task_id="task-legacy-command" + repository.discard_claimed.assert_called_once_with( + task_id="task-legacy-command", + lease_token="lease-task-legacy-command", ) @@ -898,16 +940,32 @@ def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_ src_path="/downloads/Movie.2026.mkv", planning_input=accepted_input, ) + claimed = repository.claim_task( + task_id=admission.task_id, + owner_id="fingerprint-test", + lease_seconds=120, + ) + assert claimed is not None + assert claimed.lease_token is not None mismatched = _checkpoint(target_path="/library/B/Movie.mkv") with pytest.raises(ValueError, match="指纹|fingerprint|规划输入"): repository.checkpoint_plan( task_id=admission.task_id, + lease_token=claimed.lease_token, input_fingerprint=accepted_input.fingerprint, checkpoint=mismatched, ) - recovered = repository.list_recoverable() + assert repository.release_claim( + task_id=admission.task_id, + lease_token=claimed.lease_token, + ) is True + recovered = repository.claim_recoverable( + owner_id="fingerprint-recovery", + limit=10, + lease_seconds=120, + ) assert len(recovered) == 1 assert recovered[0].state == "accepted" assert recovered[0].checkpoint is None @@ -926,23 +984,35 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path): planning_input=planning_input, ) - accepted = repository.list_recoverable() - assert len(accepted) == 1 - assert accepted[0].state == "accepted" - assert accepted[0].planning_input == planning_input - assert accepted[0].checkpoint is None + claimed = repository.claim_task( + task_id=admission.task_id, + owner_id="roundtrip-owner", + lease_seconds=120, + ) + assert claimed is not None + assert claimed.lease_token is not None + assert claimed.state == "accepted" + assert claimed.planning_input == planning_input + assert claimed.checkpoint is None repository.record_planning_failure( task_id=admission.task_id, + lease_token=claimed.lease_token, error="rename unavailable", ) - retryable = repository.list_recoverable()[0] - assert retryable.state == "accepted" - assert retryable.last_error == "rename unavailable" + with sessionmaker(bind=engine)() as session: + retryable = session.execute( + select(TransferPending).where( + TransferPending.task_id == admission.task_id + ) + ).scalar_one() + assert retryable.state == "accepted" + assert retryable.last_error == "rename unavailable" checkpoint = _checkpoint() planned = repository.checkpoint_plan( task_id=admission.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) @@ -950,7 +1020,18 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path): assert planned.state == "planned" assert planned.checkpoint == checkpoint assert planned.last_error is None - assert repository.list_recoverable() == [planned] + assert repository.release_claim( + task_id=admission.task_id, + lease_token=claimed.lease_token, + ) is True + recovered = repository.claim_recoverable( + owner_id="roundtrip-recovery", + limit=10, + lease_seconds=120, + ) + assert len(recovered) == 1 + assert recovered[0].state == "planned" + assert recovered[0].checkpoint == checkpoint engine.dispose() @@ -982,6 +1063,10 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted(): repository.admit.side_effect = admit repository.checkpoint_plan.side_effect = checkpoint_plan chain = _chain(repository=repository, checkpoint=checkpoint, result=result) + claim_task = repository.claim_task.side_effect + repository.claim_task.side_effect = lambda **kwargs: ( + order.append("claim") or claim_task(**kwargs) + ) chain.plan_transfer.side_effect = lambda *_args, **_kwargs: ( order.append("plan") or checkpoint ) @@ -992,12 +1077,13 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted(): returned = chain._plan_checkpoint_and_execute(task) assert returned is result - assert order == ["admit", "plan", "commit-checkpoint", "execute"] + assert order == ["admit", "claim", "plan", "commit-checkpoint", "execute"] assert task.admission_task_id == "task-sync" repository.admit.assert_called_once() assert repository.admit.call_args.kwargs["planning_input"] is planning_input repository.checkpoint_plan.assert_called_once_with( task_id="task-sync", + lease_token="lease-task-sync", input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) @@ -1192,11 +1278,15 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup( updated_at="2026-08-27 10:00:00", input_fingerprint=planning_input.fingerprint, planning_input=planning_input, + lease_owner="replay-owner", + lease_token="lease-task-accepted-offline", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, ) replay_repository = Mock() - replay_repository.list_recoverable.return_value = [admission] - replay_chain = object.__new__(TransferChain) - replay_chain._transfer_admissions = replay_repository + replay_repository.claim_recoverable.return_value = [admission] + replay_chain = _replay_chain(replay_repository) replay_chain._execute_transfer = Mock() queued_tasks = [] replay_chain.put_to_queue = Mock( @@ -1230,6 +1320,10 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup( repository=execution_repository, checkpoint=checkpoint, ) + execution_chain._worker_owner_id = "replay-owner" + execution_chain._owned_leases = { + admission.task_id: (str(admission.lease_token), float("inf")) + } execution_chain.jobview = Mock() execution_chain.eventmanager = Mock() execution_chain.eventmanager.send_event.return_value = None @@ -1292,11 +1386,15 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline( created_at="2026-08-27 10:00:00", updated_at="2026-08-27 10:00:00", planning_input=planning_input, + lease_owner="replay-owner", + lease_token="lease-task-accepted-empty-episodes", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, ) replay_repository = Mock() - replay_repository.list_recoverable.return_value = [admission] - replay_chain = object.__new__(TransferChain) - replay_chain._transfer_admissions = replay_repository + replay_repository.claim_recoverable.return_value = [admission] + replay_chain = _replay_chain(replay_repository) queued_tasks = [] replay_chain.put_to_queue = Mock( side_effect=lambda task: queued_tasks.append(task) or True @@ -1319,6 +1417,10 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline( repository=execution_repository, checkpoint=checkpoint, ) + execution_chain._worker_owner_id = "replay-owner" + execution_chain._owned_leases = { + admission.task_id: (str(admission.lease_token), float("inf")) + } execution_chain.jobview = Mock() execution_chain.eventmanager = Mock() execution_chain.eventmanager.send_event.return_value = None @@ -1425,8 +1527,20 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch) task = _task() task.meta = MetaBase("Unrecognized.Movie.2026.mkv") task.bind_admission_task_id("task-before-checkpoint") + task.bind_execution_lease( + owner_id="recognition-owner", + lease_token="lease-task-before-checkpoint", + ) chain = object.__new__(TransferChain) chain._transfer_admissions = Mock() + chain._worker_owner_id = "recognition-owner" + chain._owned_leases = { + "task-before-checkpoint": ( + "lease-task-before-checkpoint", + float("inf"), + ) + } + chain._worker_state_lock = threading.RLock() chain.jobview = Mock() chain.queue_failed_transfer_notification = Mock() chain.runtime_config = SimpleNamespace( @@ -1449,6 +1563,7 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch) assert result == (False, "未识别到媒体信息") chain._transfer_admissions.record_planning_failure.assert_called_once_with( task_id="task-before-checkpoint", + lease_token="lease-task-before-checkpoint", error="未识别到媒体信息", ) diff --git a/tests/test_transfer_planning_compat.py b/tests/test_transfer_planning_compat.py index 87ef64cbe..7b653dcee 100644 --- a/tests/test_transfer_planning_compat.py +++ b/tests/test_transfer_planning_compat.py @@ -57,6 +57,8 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields(): task.bind_admission_task_id("task-stable") values = task.to_dict() + task.bind_execution_lease(owner_id="worker-owner", lease_token="lease-token") + leased_values = task.to_dict() assert set(values) == { "fileitem", @@ -84,9 +86,12 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields(): } assert values["fileitem"] == task.fileitem.model_dump() assert values["target_path"] == Path("/library/Movie (2026)") + assert leased_values == values assert "admission_task_id" not in values assert "planning_input" not in values assert "plan_checkpoint" not in values + assert "lease_owner" not in values + assert "lease_token" not in values def test_transfer_chain_do_transfer_keeps_legacy_signature(): diff --git a/tests/test_transfer_planning_migration.py b/tests/test_transfer_planning_migration.py index 413dedde9..20cb877f2 100644 --- a/tests/test_transfer_planning_migration.py +++ b/tests/test_transfer_planning_migration.py @@ -24,12 +24,13 @@ except ModuleNotFoundError: POSTGRESQL_DIALECT = "postgresql+psycopg" -MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14" +PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14" +LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15" -def _bind_migration(monkeypatch, connection): +def _bind_migration(monkeypatch, connection, module_name=PLANNING_MIGRATION): """把迁移绑定到隔离数据库连接。""" - migration = importlib.import_module(MIGRATION) + migration = importlib.import_module(module_name) monkeypatch.setattr( migration, "op", @@ -91,10 +92,13 @@ def _planning_row(connection) -> dict[str, object]: def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: """断言规划迁移在当前隔离连接上的完整可逆生命周期。""" _create_admission_table(connection) - migration = _bind_migration(monkeypatch, connection) + planning_migration = _bind_migration(monkeypatch, connection) + lease_migration = _bind_migration(monkeypatch, connection, LEASE_MIGRATION) - migration.upgrade() - migration.upgrade() + planning_migration.upgrade() + planning_migration.upgrade() + lease_migration.upgrade() + lease_migration.upgrade() inspector = sa.inspect(connection) assert { @@ -133,7 +137,9 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: planned_at="2026-08-27 11:00:00", ) ) - migration.downgrade() + lease_migration.downgrade() + lease_migration.downgrade() + planning_migration.downgrade() downgraded = sa.inspect(connection) assert { @@ -152,12 +158,17 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None: for index in downgraded.get_indexes("transferpending") } == {"ix_transferpending_state_created", "ux_transferpending_storage_path"} - migration.upgrade() + planning_migration.upgrade() + lease_migration.upgrade() reupgraded = _planning_row(connection) assert reupgraded["task_id"] == "stable-task" assert reupgraded["state"] == "accepted" assert reupgraded["checkpoint_payload"] is None assert reupgraded["input_fingerprint"] == planning_input.fingerprint + assert { + column["name"] + for column in sa.inspect(connection).get_columns("transferpending") + } == {column.name for column in TransferPending.__table__.columns} def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None: diff --git a/tests/test_transfer_planning_persistence.py b/tests/test_transfer_planning_persistence.py index 8a391fa49..043d5e2e9 100644 --- a/tests/test_transfer_planning_persistence.py +++ b/tests/test_transfer_planning_persistence.py @@ -11,6 +11,7 @@ from app.application.transfer import ( TRANSFER_ADMISSION_PLANNED, TRANSFER_ADMISSION_PROVIDER_PENDING, TransferAdmissionConflictError, + TransferAdmissionProjectionError, TransferPlanCheckpoint, TransferPlanItem, TransferPlanningInput, @@ -30,6 +31,31 @@ def repository(tmp_path): return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) +def _claim(repository, task_id: str): + """为需要变更规划状态的测试取得独占租约。""" + claimed = repository.claim_task( + task_id=task_id, + owner_id="planning-test-worker", + lease_seconds=3600, + ) + assert claimed is not None + assert claimed.lease_token + return claimed + + +def _pending_snapshot(repository, task_id: str) -> dict[str, object]: + """使用隔离 Session 冻结测试所需的持久状态字段。""" + with repository._session_factory() as session: # noqa: SLF001 + pending = session.execute( + select(TransferPending).where(TransferPending.task_id == task_id) + ).scalar_one() + return { + "state": pending.state, + "last_error": pending.last_error, + "checkpoint_payload": pending.checkpoint_payload, + } + + def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput: """构造包含恢复所需媒体上下文的完整规划输入。""" return TransferPlanningInput( @@ -264,6 +290,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) checkpoint = replace( _checkpoint(planning_input), resolved_meta={"name": "Resolved Movie", "year": 2026}, @@ -275,6 +302,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N planned = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) @@ -287,6 +315,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N with pytest.raises(TransferPlanningStateError): repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=replace( checkpoint, @@ -326,15 +355,18 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None: src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) checkpoint = _checkpoint(planning_input) planned = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) repeated = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=checkpoint, ) @@ -350,8 +382,9 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None: "plugin-provider-a", ) assert repeated == planned - assert repository.list_accepted() == [] - assert repository.list_recoverable() == [planned] + assert _pending_snapshot(repository, admitted.task_id)["state"] == ( + TRANSFER_ADMISSION_PLANNED + ) def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None: @@ -362,23 +395,34 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) provider_checkpoint = _provider_checkpoint(planning_input) provider_pending = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=provider_checkpoint, ) assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING assert provider_pending.checkpoint == provider_checkpoint - assert repository.list_recoverable() == [provider_pending] repository.record_planning_failure( task_id=admitted.task_id, + lease_token=claimed.lease_token, error="host planning unavailable", ) - failed = repository.list_recoverable()[0] + assert repository.release_claim( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + error="host planning unavailable", + ) + failed = repository.claim_recoverable( + owner_id="planning-recovery-worker", + limit=1, + lease_seconds=3600, + )[0] assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING assert failed.checkpoint == provider_checkpoint assert failed.last_error == "host planning unavailable" @@ -386,11 +430,13 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository host_checkpoint = _checkpoint(planning_input) planned = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=failed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=host_checkpoint, ) repeated = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=failed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=host_checkpoint, ) @@ -402,6 +448,7 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository with pytest.raises(TransferPlanningStateError): repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=failed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=provider_checkpoint, ) @@ -415,18 +462,19 @@ def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> Non src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) with pytest.raises(TransferAdmissionConflictError): repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint="0" * 64, checkpoint=_checkpoint(planning_input), ) - recovered = repository.list_recoverable() - assert len(recovered) == 1 - assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED - assert recovered[0].checkpoint is None + recovered = _pending_snapshot(repository, admitted.task_id) + assert recovered["state"] == TRANSFER_ADMISSION_ACCEPTED + assert recovered["checkpoint_payload"] is None def test_planning_failure_stays_accepted_until_success(repository) -> None: @@ -437,15 +485,21 @@ def test_planning_failure_stays_accepted_until_success(repository) -> None: src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) - repository.record_planning_failure(task_id=admitted.task_id, error="rename failed") - failed = repository.list_recoverable()[0] - assert failed.state == TRANSFER_ADMISSION_ACCEPTED - assert failed.last_error == "rename failed" - assert failed.checkpoint is None + repository.record_planning_failure( + task_id=admitted.task_id, + lease_token=claimed.lease_token, + error="rename failed", + ) + failed = _pending_snapshot(repository, admitted.task_id) + assert failed["state"] == TRANSFER_ADMISSION_ACCEPTED + assert failed["last_error"] == "rename failed" + assert failed["checkpoint_payload"] is None planned = repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=_checkpoint(planning_input), ) @@ -459,6 +513,7 @@ def test_checkpoint_rejects_missing_task(repository) -> None: with pytest.raises(TransferPlanningStateError): repository.checkpoint_plan( task_id="missing", + lease_token="missing-token", input_fingerprint=planning_input.fingerprint, checkpoint=_checkpoint(planning_input), ) @@ -470,17 +525,25 @@ def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None: factory = sessionmaker(bind=engine) TransferPending.__table__.create(engine) with factory() as session: - session.add(TransferPending( + pending = TransferPending( storage="local", src_path="/downloads/legacy.mkv", state=TRANSFER_ADMISSION_ACCEPTED, created_at="2026-08-27 10:00:00", updated_at="2026-08-27 10:00:00", - )) + ) + session.add(pending) session.commit() + task_id = pending.task_id - admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0] + repository = TransactionalTransferAdmissionRepository(factory) + admitted = repository.claim_task( + task_id=task_id, + owner_id="legacy-projection-worker", + lease_seconds=3600, + ) + assert admitted is not None assert admitted.planning_input == TransferPlanningInput.legacy( storage="local", src_path="/downloads/legacy.mkv", @@ -508,8 +571,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) - row.input_version = 2 session.commit() - with pytest.raises(TransferPlanningStateError, match="版本"): - repository.list_accepted() + with pytest.raises(TransferAdmissionProjectionError, match="版本"): + repository.claim_task( + task_id=admitted.task_id, + owner_id="corruption-worker", + lease_seconds=3600, + ) with factory() as session: row = session.execute( @@ -521,8 +588,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) - row.planning_input = corrupted session.commit() - with pytest.raises(TransferAdmissionConflictError, match="指纹"): - repository.list_accepted() + with pytest.raises(TransferAdmissionProjectionError, match="指纹"): + repository.claim_task( + task_id=admitted.task_id, + owner_id="corruption-worker", + lease_seconds=3600, + ) engine.dispose() @@ -538,8 +609,10 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None: src_path="/downloads/Movie.2026.mkv", planning_input=planning_input, ) + claimed = _claim(repository, admitted.task_id) repository.checkpoint_plan( task_id=admitted.task_id, + lease_token=claimed.lease_token, input_fingerprint=planning_input.fingerprint, checkpoint=_checkpoint(planning_input), ) @@ -548,8 +621,18 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None: select(TransferPending).where(TransferPending.task_id == admitted.task_id) ).scalar_one() row.checkpoint_version = 2 + row.lease_expires_at = "2000-01-01 00:00:00.000000" session.commit() - with pytest.raises(TransferPlanningStateError, match="版本"): - repository.list_recoverable() + with pytest.raises(TransferAdmissionProjectionError, match="版本"): + repository.claim_task( + task_id=admitted.task_id, + owner_id="direct-corruption-test-worker", + lease_seconds=3600, + ) + assert repository.claim_recoverable( + owner_id="batch-corruption-test-worker", + limit=1, + lease_seconds=3600, + ) == [] engine.dispose() diff --git a/tests/test_transfer_queue_service.py b/tests/test_transfer_queue_service.py index d80a0c4f0..36bfd6f14 100644 --- a/tests/test_transfer_queue_service.py +++ b/tests/test_transfer_queue_service.py @@ -2,7 +2,7 @@ from types import SimpleNamespace from unittest.mock import Mock, patch import pytest -from sqlalchemy import create_engine +from sqlalchemy import create_engine, select from sqlalchemy.orm import sessionmaker from app.application.transfer import TransferAdmission, TransferQueueService @@ -118,7 +118,8 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path """真实仓储已提交后即使内存入队失败，任务也必须带原因留待恢复。""" engine = create_engine(f"sqlite:///{tmp_path / 'durable-admission.db'}") TransferPending.__table__.create(engine) - repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine)) + factory = sessionmaker(bind=engine) + repository = TransactionalTransferAdmissionRepository(factory) task = make_task(1) service, _ = _service( admit_task=lambda item: repository.admit( @@ -135,10 +136,13 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path with pytest.raises(RuntimeError, match="queue closed"): service.put(task, Mock()) - admissions = repository.list_accepted() - assert len(admissions) == 1 - assert admissions[0].task_id == task.admission_task_id - assert admissions[0].last_error == "queue closed" + with factory() as session: + pending = session.execute( + select(TransferPending).where( + TransferPending.task_id == task.admission_task_id + ) + ).scalar_one() + assert pending.last_error == "queue closed" engine.dispose() diff --git a/tests/test_transfer_sync_extra_files.py b/tests/test_transfer_sync_extra_files.py index 0814232a3..3f1de01c5 100644 --- a/tests/test_transfer_sync_extra_files.py +++ b/tests/test_transfer_sync_extra_files.py @@ -1,12 +1,15 @@ from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock -from app.chain.transfer import JobManager, TransferChain +from app.chain.transfer import TransferChain from app.domain.meta.metabase import MetaBase -from app.runtime.config import settings from app.schemas.file import FileItem from app.schemas.transfer import EpisodeFormat from app.schemas.types import MediaType +from tests.test_transfer_job_manager import ( + make_transfer_chain as make_base_transfer_chain, +) class FakeMeta(MetaBase): @@ -46,16 +49,8 @@ def make_transfer_chain() -> TransferChain: """ 构造不启动后台线程的整理链实例。 """ - chain = object.__new__(TransferChain) - chain.jobview = JobManager() - chain._media_exts = settings.RMT_MEDIAEXT - chain._subtitle_exts = settings.RMT_SUBEXT - chain._audio_exts = settings.RMT_AUDIOEXT - chain._allowed_exts = ( - chain._media_exts + chain._audio_exts + chain._subtitle_exts - ) - chain._success_target_files = {} - chain._scrape_batches = {} + chain = make_base_transfer_chain() + chain._TransferChain__ensure_recovery_scheduler = MagicMock() return chain diff --git a/tests/test_transfer_worker_lifecycle.py b/tests/test_transfer_worker_lifecycle.py index a4188c575..8ddc6af22 100644 --- a/tests/test_transfer_worker_lifecycle.py +++ b/tests/test_transfer_worker_lifecycle.py @@ -15,13 +15,17 @@ from app.chain.transfer import TransferChain from app.foundation.singleton import Singleton from app.runtime.config import global_vars from app.schemas.file import FileItem +from app.schemas.transfer import TransferInfo from app.startup.initializers import transfer as transfer_initializer def _build_chain(*, transfer_threads: int = 0) -> TransferChain: """构造只包含后台线程生命周期字段的 TransferChain 测试骨架。""" chain = object.__new__(TransferChain) - chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads) + chain.runtime_config = SimpleNamespace( + transfer_threads=transfer_threads, + transfer_task_timeout=0, + ) chain._queue = queue.Queue() chain._transfer_interval = 0.1 chain._threads = [] @@ -33,9 +37,59 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain: chain._closing = False chain._replay_thread = None chain._replay_stop_event = threading.Event() + chain._recovery_wakeup_event = threading.Event() + chain._lease_heartbeat_thread = None + chain._lease_heartbeat_stop_event = threading.Event() + chain._worker_owner_id = "worker-owner" + chain._owned_leases = {} + chain._queued_lease_tokens = set() + admissions = MagicMock() + admissions.admit.side_effect = lambda **kwargs: TransferAdmission( + task_id="admitted-task", + storage=kwargs["storage"], + src_path=kwargs["src_path"], + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + ) + admissions.claim_task.side_effect = lambda **kwargs: TransferAdmission( + task_id=kwargs["task_id"], + storage="local", + src_path="/downloads/test.mkv", + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + lease_owner=kwargs["owner_id"], + lease_token=f"lease-{kwargs['task_id']}", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, + ) + admissions.discard_claimed.return_value = 1 + admissions.release_claim.return_value = True + chain._transfer_admissions = admissions + chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock() + chain._TransferChain__ensure_recovery_scheduler = MagicMock() return chain +def _claimed_admission(task: TransferTask, task_id: str) -> TransferAdmission: + """构造属于测试进程 owner 的有效 claim 投影。""" + return TransferAdmission( + task_id=task_id, + storage=task.fileitem.storage, + src_path=task.fileitem.path, + state="accepted", + created_at="2026-08-27 10:00:00", + updated_at="2026-08-27 10:00:00", + lease_owner="worker-owner", + lease_token=f"lease-{task_id}", + lease_expires_at="2026-08-27 10:02:00.000000", + heartbeat_at="2026-08-27 10:00:00.000000", + attempt_count=1, + ) + + def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None: """热更新应等待旧 worker 收敛，再启动使用独立停止信号的新一代。""" chain = _build_chain(transfer_threads=1) @@ -433,7 +487,11 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch) created_at="2026-08-27 10:00:00", updated_at="2026-08-27 10:00:00", ) - admissions.discard_task.side_effect = ( + admissions.claim_task.return_value = _claimed_admission( + task, + "durable-task-id", + ) + admissions.discard_claimed.side_effect = ( lambda **_kwargs: discarded.set() or 1 ) chain._transfer_admissions = admissions @@ -471,7 +529,15 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch) assert worker.is_alive() is False assert task.admission_task_id == "durable-task-id" - admissions.discard_task.assert_called_once_with(task_id="durable-task-id") + admissions.claim_task.assert_called_once_with( + task_id="durable-task-id", + owner_id="worker-owner", + lease_seconds=120, + ) + admissions.discard_claimed.assert_called_once_with( + task_id="durable-task-id", + lease_token="lease-durable-task-id", + ) def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None: @@ -527,6 +593,7 @@ def test_claimed_task_prevents_progress_settlement_before_active_registration() def test_replay_has_single_owner_and_close_waits_for_it() -> None: """重复回放只保留一个线程，关闭会通知并等待该线程退出。""" chain = _build_chain() + del chain._TransferChain__ensure_recovery_scheduler replay_started = threading.Event() replay_calls = [] @@ -548,3 +615,355 @@ def test_replay_has_single_owner_and_close_waits_for_it() -> None: assert chain.close_workers(timeout_seconds=1) is True assert replay_thread.is_alive() is False assert chain._replay_thread is None + + +def test_recovered_worker_reuses_claimed_token_without_second_claim( + monkeypatch, +) -> None: + """恢复任务携带 token 入队后，普通 worker 必须直接执行而非二次 claim。""" + chain = _build_chain() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/recovered.mkv", + type="file", + name="recovered.mkv", + basename="recovered", + extension="mkv", + )) + task.bind_admission_task_id("recovered-task") + task.bind_execution_lease( + owner_id="worker-owner", + lease_token="lease-recovered-task", + ) + chain._owned_leases = { + "recovered-task": ("lease-recovered-task", time.monotonic() + 120) + } + chain.jobview = MagicMock() + chain.jobview.pending_total.return_value = 1 + chain._finish_scrape_batch_task = MagicMock() + chain._progress = MagicMock() + chain._active_tasks = 0 + chain._processed_num = 0 + chain._fail_num = 0 + chain._total_num = 0 + chain._transfer_admissions.discard_claimed.return_value = 1 + stop_event = threading.Event() + + def complete_recovery(*, task, callback): + """模拟恢复任务成功提交检查点并让 worker 在本项后退出。""" + del callback + task.bind_plan_checkpoint(MagicMock()) + stop_event.set() + return True, "" + + chain._TransferChain__handle_transfer = complete_recovery + chain._queue.put(TransferQueue(task=task)) + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + + worker = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + worker.start() + worker.join(timeout=1) + + assert worker.is_alive() is False + chain._transfer_admissions.claim_task.assert_not_called() + chain._transfer_admissions.discard_claimed.assert_called_once_with( + task_id="recovered-task", + lease_token="lease-recovered-task", + ) + + +def test_heartbeat_refreshes_current_token_and_forgets_lost_lease() -> None: + """heartbeat 成功应刷新本地期限，CAS 拒绝后必须立即停止本地推进资格。""" + chain = _build_chain() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/heartbeat.mkv", + type="file", + )) + current = _claimed_admission(task, "heartbeat-task") + initial_deadline = time.monotonic() + 1 + chain._owned_leases = { + "heartbeat-task": ("lease-heartbeat-task", initial_deadline) + } + chain._transfer_admissions.heartbeat.return_value = current + + chain._TransferChain__heartbeat_owned_leases() + + assert chain._owned_leases["heartbeat-task"][1] > initial_deadline + chain._transfer_admissions.heartbeat.return_value = None + + chain._TransferChain__heartbeat_owned_leases() + + assert "heartbeat-task" not in chain._owned_leases + + +def test_close_timeout_keeps_heartbeat_alive_until_blocked_worker_converges() -> None: + """阻塞 worker 未退出时关闭不得停止 heartbeat 或允许租约过期接管。""" + chain = _build_chain() + worker_release = threading.Event() + worker = threading.Thread( + target=worker_release.wait, + name="transfer-blocked-owner", + daemon=True, + ) + heartbeat = threading.Thread( + target=chain._lease_heartbeat_stop_event.wait, + name="transfer-heartbeat-owner", + daemon=True, + ) + chain._threads = [worker] + chain._lease_heartbeat_thread = heartbeat + chain._owned_leases = { + "blocked-task": ("blocked-token", time.monotonic() + 120) + } + worker.start() + heartbeat.start() + + assert chain.close_workers(timeout_seconds=0.01) is False + assert heartbeat.is_alive() is True + assert chain._lease_heartbeat_stop_event.is_set() is False + + worker_release.set() + assert chain.close_workers(timeout_seconds=1) is True + assert worker.is_alive() is False + assert heartbeat.is_alive() is False + chain._transfer_admissions.release_claim.assert_called_once_with( + task_id="blocked-task", + lease_token="blocked-token", + error="整理宿主关闭，释放未结算任务租约", + ) + + +def test_worker_reports_failed_settlement_without_skipping_queue_bookkeeping( + monkeypatch, +) -> None: + """终态 CAS=0 必须计为失败，同时仍完成 task_done 与 active 归零。""" + chain = _build_chain() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/stale.mkv", + type="file", + name="stale.mkv", + basename="stale", + extension="mkv", + )) + chain.jobview = MagicMock() + chain.jobview.add_task.return_value = True + chain.jobview.pending_total.return_value = 1 + chain._register_scrape_batch_task = MagicMock() + chain._finish_scrape_batch_task = MagicMock() + chain._progress = MagicMock() + chain._active_tasks = 0 + chain._processed_num = 0 + chain._fail_num = 0 + chain._total_num = 0 + chain._transfer_admissions.discard_claimed.return_value = 0 + chain._TransferChain__settle_transfer_progress_if_idle = MagicMock() + stop_event = threading.Event() + + def complete_with_stale_lease(*, task, callback): + """模拟文件副作用完成后终态 token 已被新 owner 接管。""" + del callback + task.bind_plan_checkpoint(MagicMock()) + stop_event.set() + return True, "" + + chain._TransferChain__handle_transfer = complete_with_stale_lease + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + assert chain.put_to_queue(task) is True + + worker = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + worker.start() + worker.join(timeout=1) + + assert worker.is_alive() is False + assert chain._active_tasks == 0 + assert chain._fail_num == 1 + assert chain._queue.unfinished_tasks == 0 + + +def test_failed_claim_release_waits_for_fixed_recovery_poll() -> None: + """失败释放不得即时唤醒恢复线程，避免确定性错误形成热重试。""" + chain = _build_chain() + del chain._TransferChain__ensure_recovery_scheduler + chain._RECOVERY_POLL_INTERVAL_SECONDS = 0.05 + chain._TransferChain__replay_pending = MagicMock() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/retry-later.mkv", + type="file", + )) + task.bind_admission_task_id("retry-later") + task.bind_execution_lease( + owner_id="worker-owner", + lease_token="retry-token", + ) + chain._owned_leases = { + "retry-later": ("retry-token", time.monotonic() + 120) + } + + assert chain._TransferChain__release_task_claim( + task, + error="planning failed", + ) is True + + assert chain._recovery_wakeup_event.is_set() is False + assert chain._replay_thread is not None + time.sleep(0.01) + chain._TransferChain__replay_pending.assert_not_called() + deadline = time.monotonic() + 0.5 + while ( + not chain._TransferChain__replay_pending.called + and time.monotonic() < deadline + ): + time.sleep(0.01) + chain._TransferChain__replay_pending.assert_called() + chain._transfer_admissions.release_claim.assert_called_once_with( + task_id="retry-later", + lease_token="retry-token", + error="planning failed", + ) + assert chain.close_workers(timeout_seconds=1) is True + + +def test_worker_fenced_releases_lost_lease_and_completes_queue_bookkeeping( + monkeypatch, +) -> None: + """本地租约失效时仍尝试 token CAS release，并完整结算内存队列。""" + chain = _build_chain() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/lost-lease.mkv", + type="file", + )) + task.bind_admission_task_id("lost-lease") + task.bind_execution_lease( + owner_id="worker-owner", + lease_token="lost-token", + ) + chain.jobview = MagicMock() + chain._finish_scrape_batch_task = MagicMock() + chain._TransferChain__settle_transfer_progress_if_idle = MagicMock() + stop_event = threading.Event() + chain._transfer_admissions.release_claim.side_effect = ( + lambda **_kwargs: stop_event.set() or True + ) + chain._queue.put(TransferQueue(task=task)) + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + + worker = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + worker.start() + worker.join(timeout=1) + + assert worker.is_alive() is False + chain._transfer_admissions.release_claim.assert_called_once_with( + task_id="lost-lease", + lease_token="lost-token", + error="整理任务租约已经失效：lost-lease", + ) + assert chain._queue.unfinished_tasks == 0 + assert chain._recovery_wakeup_event.is_set() is False + + +def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) -> None: + """终态 CAS 被拒绝时不得写成功历史、事件或通知。""" + chain = _build_chain() + task = TransferTask(fileitem=FileItem( + storage="local", + path="/downloads/fenced-success.mkv", + type="file", + name="fenced-success.mkv", + )) + chain.jobview = MagicMock() + chain.jobview.add_task.return_value = True + chain.jobview.pending_total.return_value = 1 + chain._register_scrape_batch_task = MagicMock() + chain._finish_scrape_batch_task = MagicMock() + chain._progress = MagicMock() + chain._active_tasks = 0 + chain._processed_num = 0 + chain._fail_num = 0 + chain._total_num = 0 + chain._transfer_admissions.discard_claimed.return_value = 0 + chain._TransferChain__settle_transfer_progress_if_idle = MagicMock() + success_callback = MagicMock(return_value=(True, "")) + chain._TransferChain__default_callback = success_callback + stop_event = threading.Event() + + def complete_with_success(*, task, callback): + """模拟文件成功后进入受 durable 终态保护的回调。""" + task.bind_plan_checkpoint(MagicMock()) + stop_event.set() + return callback( + task, + TransferInfo(success=True, fileitem=task.fileitem), + ) + + chain._TransferChain__handle_transfer = complete_with_success + monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event()) + assert chain.put_to_queue(task) is True + + worker = threading.Thread( + target=chain._TransferChain__start_transfer, + args=(stop_event,), + daemon=True, + ) + worker.start() + worker.join(timeout=1) + + assert worker.is_alive() is False + success_callback.assert_not_called() + chain.jobview.fail_unfinished_task.assert_called_once_with(task) + assert chain._fail_num == 1 + assert chain._queue.unfinished_tasks == 0 + + +def test_close_release_db_block_respects_deadline_and_keeps_heartbeat() -> None: + """关闭租约释放被数据库阻塞时应按预算返回，并继续 heartbeat。""" + chain = _build_chain() + release_started = threading.Event() + release_db = threading.Event() + heartbeat = threading.Thread( + target=chain._lease_heartbeat_stop_event.wait, + name="transfer-heartbeat-release-test", + daemon=True, + ) + chain._lease_heartbeat_thread = heartbeat + chain._owned_leases = { + "blocked-release": ("blocked-token", time.monotonic() + 120) + } + + def block_release(**_kwargs): + """模拟数据库锁住 release_claim，直到测试显式放行。""" + release_started.set() + release_db.wait() + return True + + chain._transfer_admissions.release_claim.side_effect = block_release + heartbeat.start() + + started_at = time.monotonic() + assert chain.close_workers(timeout_seconds=0.01) is False + assert time.monotonic() - started_at < 0.5 + assert release_started.is_set() + assert heartbeat.is_alive() is True + assert chain._lease_heartbeat_stop_event.is_set() is False + assert chain._lease_release_thread is not None + assert chain._lease_release_thread.is_alive() is True + + release_db.set() + assert chain.close_workers(timeout_seconds=1) is True + assert heartbeat.is_alive() is False + assert chain._lease_release_thread is None