mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-07 16:36:53 +08:00
refactor: add fenced transfer recovery leases
This commit is contained in:
jxxghp
+8
-3
@@ -1441,8 +1441,8 @@
|
||||
"runtime_only": true
|
||||
}
|
||||
},
|
||||
"edge_count": 6832,
|
||||
"edge_sha256": "eb4b2f9b9689496a7821aeb151c64430cc8654b186aba50f2b7873ae09b71b39",
|
||||
"edge_count": 6836,
|
||||
"edge_sha256": "3709e09a49257075f48a44aba353be1f25db610471ec756e09cbbfd7070acb9a",
|
||||
"edges": [
|
||||
"app -> app.runtime",
|
||||
"app -> app.runtime.compat",
|
||||
@@ -7625,6 +7625,10 @@
|
||||
"app.sdk._legacy.subscribe -> app.domain.context",
|
||||
"app.sdk._legacy.transfer -> app.application",
|
||||
"app.sdk._legacy.transfer -> app.application.transfer",
|
||||
"app.sdk._legacy.transferpending -> app.db",
|
||||
"app.sdk._legacy.transferpending -> app.db.base",
|
||||
"app.sdk._legacy.transferpending -> app.db.models",
|
||||
"app.sdk._legacy.transferpending -> app.db.models.transferpending",
|
||||
"app.sdk._legacy.user -> app.api",
|
||||
"app.sdk._legacy.user -> app.api.deps",
|
||||
"app.sdk._legacy.user -> app.db",
|
||||
@@ -8277,7 +8281,7 @@
|
||||
"app.workflow.actions.transfer_file -> app.workflow",
|
||||
"app.workflow.actions.transfer_file -> app.workflow.actions"
|
||||
],
|
||||
"module_count": 836,
|
||||
"module_count": 837,
|
||||
"modules": [
|
||||
"app",
|
||||
"app.adapters",
|
||||
@@ -9058,6 +9062,7 @@
|
||||
"app.sdk._legacy.history",
|
||||
"app.sdk._legacy.subscribe",
|
||||
"app.sdk._legacy.transfer",
|
||||
"app.sdk._legacy.transferpending",
|
||||
"app.sdk._legacy.user",
|
||||
"app.sdk.browser",
|
||||
"app.sdk.cache",
|
||||
|
||||
+1
-1
@@ -1594,7 +1594,7 @@
|
||||
"misc": 2,
|
||||
"no-any-return": 3,
|
||||
"no-untyped-call": 8,
|
||||
"no-untyped-def": 14,
|
||||
"no-untyped-def": 12,
|
||||
"operator": 3,
|
||||
"return-value": 2,
|
||||
"truthy-function": 5,
|
||||
|
||||
@@ -249,9 +249,9 @@
|
||||
"app.db.transferpending_oper": {
|
||||
"introduced": "v3.0.0",
|
||||
"is_package": false,
|
||||
"owner": "db",
|
||||
"replacement": "app.db.oper.transferpending",
|
||||
"target": "app.db.oper.transferpending"
|
||||
"owner": "sdk",
|
||||
"replacement": "app.application.transfer",
|
||||
"target": "app.sdk._legacy.transferpending"
|
||||
},
|
||||
"app.db.user_oper": {
|
||||
"introduced": "v3.0.0",
|
||||
|
||||
@@ -828,15 +828,19 @@ def test_models_and_base_require_explicit_database_sessions():
|
||||
|
||||
|
||||
def test_plugin_sdk_does_not_import_or_export_host_models():
|
||||
"""插件 SDK 只能暴露 Oper，不得把宿主 ORM Model 作为插件接口。"""
|
||||
"""插件 SDK 不得暴露 ORM Model，只有精确旧 ABI 门面可在内部访问。"""
|
||||
internal_compat_imports = {
|
||||
("app/sdk/_legacy/transferpending.py", "app.db.models.transferpending"),
|
||||
}
|
||||
violations: list[str] = []
|
||||
for path in (APP_ROOT / "sdk").rglob("*.py"):
|
||||
relative = path.relative_to(PROJECT_ROOT).as_posix()
|
||||
tree = ast.parse(path.read_text(encoding="utf-8-sig"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.ImportFrom) and node.module and (
|
||||
node.module == "app.db.models"
|
||||
or node.module.startswith("app.db.models.")
|
||||
):
|
||||
) and (relative, node.module) not in internal_compat_imports:
|
||||
violations.append(
|
||||
f"{path.relative_to(PROJECT_ROOT)}:{node.lineno}:{node.module}"
|
||||
)
|
||||
|
||||
@@ -158,8 +158,18 @@ def test_tables_without_identity_columns_are_untouched(db):
|
||||
不带身份列的表不受影响——事件挂在 Mapper 上覆盖全部映射，必须靠列名检查收窄，
|
||||
否则会去动一张根本没有这两列的表。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-14 10:00:00")
|
||||
TransferPending.stage_admit(
|
||||
db.session,
|
||||
task_id="identity-free-table",
|
||||
storage="local",
|
||||
src_path="/mnt/a.mkv",
|
||||
state="accepted",
|
||||
now_time="2026-08-14 10:00:00",
|
||||
)
|
||||
|
||||
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/mnt/a.mkv"]
|
||||
assert len(rows) == 1
|
||||
row = TransferPending.get_by_identity(
|
||||
db.session,
|
||||
storage="local",
|
||||
src_path="/mnt/a.mkv",
|
||||
)
|
||||
assert row.task_id == "identity-free-table"
|
||||
|
||||
@@ -5,8 +5,10 @@
|
||||
任何一件出偏差，都直接表现为文件被漏整理或被重复整理，而不是一个可见的报错。
|
||||
因此这里对着真实数据库断言查回的内容，而不是断言调用了什么。
|
||||
"""
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.db import base as db_base
|
||||
@@ -21,185 +23,6 @@ def _track(db):
|
||||
db.watermark(TransferPending)
|
||||
|
||||
|
||||
def test_register_is_idempotent_and_keeps_first_time(db):
|
||||
"""
|
||||
同一文件重复登记只保留一条，且登记时间保持首次的值。
|
||||
|
||||
监控在挂载抖动时会对同一个文件反复触发事件，若每次都新增一条，回放时同一个
|
||||
文件会被送进整理链多次。保留首次时间则保证回放顺序仍是「最早发现」的顺序。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 12:00:00")
|
||||
|
||||
rows = TransferPending.list_all(db.session)
|
||||
same_path = [r for r in rows if r.src_path == "/mnt/a.mkv"]
|
||||
assert len(same_path) == 1
|
||||
assert same_path[0].created_at == "2026-08-13 10:00:00"
|
||||
|
||||
|
||||
def test_register_scopes_by_storage(db):
|
||||
"""
|
||||
存储不同即为不同文件——路径相同但分属不同存储时不能互相去重。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/data/x.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="alist", src_path="/data/x.mkv",
|
||||
now_time="2026-08-13 10:00:01")
|
||||
|
||||
rows = [r for r in TransferPending.list_all(db.session) if r.src_path == "/data/x.mkv"]
|
||||
assert {r.storage for r in rows} == {"local", "alist"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", ""), ("", "")])
|
||||
def test_register_rejects_incomplete_identity(db, storage, src_path):
|
||||
"""
|
||||
缺少存储或路径的登记必须直接丢弃，不能写入半条记录。
|
||||
|
||||
半条记录回放时既定位不到文件、也无法被 discard 匹配，会永久留在表里。
|
||||
"""
|
||||
assert TransferPending.register(db.session, storage=storage, src_path=src_path,
|
||||
now_time="2026-08-13 10:00:00") is None
|
||||
|
||||
|
||||
def test_list_all_replays_in_registration_order(db):
|
||||
"""
|
||||
回放顺序必须是登记时间升序、同时间按主键升序。
|
||||
|
||||
乱序回放会让后发现的文件先进整理链，与原入队顺序不一致。
|
||||
"""
|
||||
for path, moment in [("/mnt/c.mkv", "2026-08-13 12:00:00"),
|
||||
("/mnt/a.mkv", "2026-08-13 10:00:00"),
|
||||
("/mnt/b.mkv", "2026-08-13 11:00:00")]:
|
||||
TransferPending.register(db.session, storage="local", src_path=path, now_time=moment)
|
||||
|
||||
ordered = [r.src_path for r in TransferPending.list_all(db.session)
|
||||
if r.src_path.startswith("/mnt/")]
|
||||
assert ordered == ["/mnt/a.mkv", "/mnt/b.mkv", "/mnt/c.mkv"]
|
||||
|
||||
|
||||
def test_list_all_honours_limit(db):
|
||||
"""
|
||||
回放上限必须生效——异常积压时一次性全放会把整理链直接压垮。
|
||||
"""
|
||||
for index in range(5):
|
||||
TransferPending.register(db.session, storage="local", src_path=f"/mnt/{index}.mkv",
|
||||
now_time=f"2026-08-13 10:00:0{index}")
|
||||
|
||||
assert len(TransferPending.list_all(db.session, limit=3)) == 3
|
||||
|
||||
|
||||
def test_discard_removes_only_the_matching_row(db):
|
||||
"""
|
||||
注销只应删除匹配的那一条，并返回删除条数。
|
||||
|
||||
整理到达终态时按「存储 + 路径」注销，误删其他登记等于把别的文件也判成已完成。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/b.mkv",
|
||||
now_time="2026-08-13 10:00:01")
|
||||
|
||||
assert TransferPending.discard(db.session, storage="local", src_path="/mnt/a.mkv") == 1
|
||||
|
||||
remaining = [r.src_path for r in TransferPending.list_all(db.session)
|
||||
if r.src_path.startswith("/mnt/")]
|
||||
assert remaining == ["/mnt/b.mkv"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("storage,src_path", [("", "/mnt/a.mkv"), ("local", "")])
|
||||
def test_discard_rejects_incomplete_identity(db, storage, src_path):
|
||||
"""
|
||||
身份不全时必须直接返回 0，不能退化成「条件为空」的全表删除。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
|
||||
assert TransferPending.discard(db.session, storage=storage, src_path=src_path) == 0
|
||||
assert TransferPending.list_all(db.session)
|
||||
|
||||
|
||||
def test_discard_returns_zero_when_absent(db):
|
||||
"""
|
||||
注销不存在的登记返回 0，不抛异常——整理链的终态回调不应因此中断。
|
||||
"""
|
||||
assert TransferPending.discard(db.session, storage="local", src_path="/nope.mkv") == 0
|
||||
|
||||
|
||||
def test_clear_empties_the_table(db):
|
||||
"""
|
||||
清空返回删除条数且表内不再有登记。
|
||||
"""
|
||||
TransferPending.register(db.session, storage="local", src_path="/mnt/a.mkv",
|
||||
now_time="2026-08-13 10:00:00")
|
||||
|
||||
assert TransferPending.clear(db.session) >= 1
|
||||
assert TransferPending.list_all(db.session) == []
|
||||
|
||||
|
||||
def test_oper_returns_plain_tuples_not_orm_instances(db):
|
||||
"""
|
||||
回放接口必须返回纯元组。
|
||||
|
||||
回放发生在会话之外，ORM 实例脱离 session 后访问属性会抛
|
||||
DetachedInstanceError——那时启动流程已经在跑，报错等于整批漏件。
|
||||
"""
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/a.mkv")
|
||||
|
||||
listed = oper.list_all()
|
||||
|
||||
assert ("local", "/mnt/a.mkv") in listed
|
||||
assert all(isinstance(item, tuple) for item in listed)
|
||||
|
||||
|
||||
def test_oper_reuses_explicit_query_session(db, monkeypatch):
|
||||
"""TransferPendingOper 查询必须复用调用方会话。"""
|
||||
db.add(TransferPending(
|
||||
storage="local",
|
||||
src_path="/mnt/explicit.mkv",
|
||||
created_at="2026-08-13 10:00:00",
|
||||
))
|
||||
monkeypatch.setattr(
|
||||
db_base,
|
||||
"run_sync_transaction",
|
||||
lambda _operation: (_ for _ in ()).throw(
|
||||
AssertionError("不应创建额外同步事务")
|
||||
),
|
||||
)
|
||||
|
||||
assert ("local", "/mnt/explicit.mkv") in TransferPendingOper(db.session).list_all()
|
||||
|
||||
|
||||
def test_oper_drops_rows_with_missing_fields(db):
|
||||
"""
|
||||
回放时必须跳过字段残缺的历史遗留行，不能把空存储送进整理链。
|
||||
|
||||
列上有 NOT NULL 约束，残缺只可能表现为空串；直接绕过 register 写入，
|
||||
模拟历史数据或外部写库留下的半条记录。
|
||||
"""
|
||||
db.add(TransferPending(storage="", src_path="/mnt/broken.mkv",
|
||||
created_at="2026-08-13 10:00:00"))
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/ok.mkv")
|
||||
|
||||
assert oper.list_all() == [("local", "/mnt/ok.mkv")]
|
||||
|
||||
|
||||
def test_oper_discard_and_clear_report_counts(db):
|
||||
"""
|
||||
注销与清空都要如实返回条数，调用方据此判断是否真的清理掉了。
|
||||
"""
|
||||
oper = TransferPendingOper(db=db.session)
|
||||
oper.register(storage="local", src_path="/mnt/a.mkv")
|
||||
oper.register(storage="local", src_path="/mnt/b.mkv")
|
||||
|
||||
assert oper.discard(storage="local", src_path="/mnt/a.mkv") == 1
|
||||
assert oper.clear() >= 1
|
||||
assert oper.list_all() == []
|
||||
|
||||
|
||||
def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
|
||||
"""显式准入重复执行时必须复用首个稳定任务标识。"""
|
||||
first = TransferPending.stage_admit(
|
||||
@@ -224,8 +47,32 @@ def test_stage_admit_is_idempotent_and_keeps_stable_task_id(db):
|
||||
assert second.updated_at == "2026-08-27 10:00:00"
|
||||
|
||||
|
||||
def test_state_queries_failure_record_and_task_discard(db):
|
||||
"""状态查询、失败留痕和按任务删除应共享同一稳定身份。"""
|
||||
def test_unfenced_legacy_mutation_apis_are_absent() -> None:
|
||||
"""持久整理表不得重新暴露绕过稳定任务身份和租约的旧写入口。"""
|
||||
for owner in (TransferPending, TransferPendingOper):
|
||||
for method_name in (
|
||||
"register",
|
||||
"discard",
|
||||
"list_all",
|
||||
"list_by_state",
|
||||
"list_by_states",
|
||||
"clear",
|
||||
):
|
||||
assert not hasattr(owner, method_name)
|
||||
|
||||
enqueue_failure_source = inspect.getsource(
|
||||
TransferPending.record_enqueue_failure
|
||||
)
|
||||
assert "cls.lease_token.is_(None)" in enqueue_failure_source
|
||||
claimable_source = inspect.getsource(
|
||||
TransferPending.list_claimable_candidates
|
||||
)
|
||||
assert "after_cursor" in claimable_source
|
||||
assert ".not_in(" not in claimable_source
|
||||
|
||||
|
||||
def test_state_queries_and_failure_record_share_stable_identity(db):
|
||||
"""状态查询与失败留痕应共享同一稳定身份。"""
|
||||
TransferPending.stage_admit(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
@@ -244,11 +91,11 @@ def test_state_queries_failure_record_and_task_discard(db):
|
||||
))
|
||||
db.session.flush()
|
||||
|
||||
accepted = TransferPending.list_by_state(
|
||||
accepted = TransferPending.get_by_task_id(
|
||||
db.session,
|
||||
state="accepted",
|
||||
task_id="task-accepted",
|
||||
)
|
||||
assert [item.task_id for item in accepted] == ["task-accepted"]
|
||||
assert accepted.state == "accepted"
|
||||
assert TransferPending.record_enqueue_failure(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
@@ -263,10 +110,6 @@ def test_state_queries_failure_record_and_task_discard(db):
|
||||
)
|
||||
assert failed.last_error == "queue full"
|
||||
assert failed.updated_at == "2026-08-27 10:01:00"
|
||||
assert TransferPending.discard_task(
|
||||
db.session,
|
||||
task_id="task-accepted",
|
||||
) == 1
|
||||
|
||||
|
||||
def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
|
||||
@@ -288,15 +131,13 @@ def test_oper_staging_reuses_explicit_write_session(db, monkeypatch):
|
||||
now_time="2026-08-27 10:00:00",
|
||||
)
|
||||
assert pending.task_id == "task-explicit"
|
||||
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
|
||||
"task-explicit"
|
||||
]
|
||||
assert oper.get_by_task_id(task_id="task-explicit").state == "accepted"
|
||||
assert oper.stage_record_enqueue_failure(
|
||||
task_id="task-explicit",
|
||||
error="queue full",
|
||||
now_time="2026-08-27 10:01:00",
|
||||
) == 1
|
||||
assert oper.stage_discard_task(task_id="task-explicit") == 1
|
||||
assert oper.get_by_task_id(task_id="task-explicit").last_error == "queue full"
|
||||
|
||||
|
||||
def test_transactional_repository_commits_frozen_projections(tmp_path):
|
||||
@@ -317,16 +158,34 @@ def test_transactional_repository_commits_frozen_projections(tmp_path):
|
||||
assert repeated == admitted
|
||||
assert admitted.task_id
|
||||
assert admitted.state == "accepted"
|
||||
assert repository.list_accepted() == [admitted]
|
||||
|
||||
repository.record_enqueue_failure(
|
||||
task_id=admitted.task_id,
|
||||
error="queue full",
|
||||
)
|
||||
failed = repository.list_accepted()[0]
|
||||
assert failed.last_error == "queue full"
|
||||
assert repository.discard_task(task_id=admitted.task_id) == 1
|
||||
assert repository.list_accepted() == []
|
||||
with factory() as session:
|
||||
failed = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert failed.last_error == "queue full"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="repository-test-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) == 1
|
||||
with factory() as session:
|
||||
assert session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one_or_none() is None
|
||||
engine.dispose()
|
||||
|
||||
|
||||
|
||||
@@ -305,10 +305,16 @@ def test_legacy_transfer_task_hides_internal_admission_identity():
|
||||
public_fields = set(task.to_dict())
|
||||
|
||||
task.bind_admission_task_id("internal-task-id")
|
||||
task.bind_execution_lease(
|
||||
owner_id="internal-worker",
|
||||
lease_token="internal-lease-token",
|
||||
)
|
||||
|
||||
assert set(task.to_dict()) == public_fields
|
||||
assert "task_id" not in task.to_dict()
|
||||
assert "admission_task_id" not in task.to_dict()
|
||||
assert "lease_owner" not in task.to_dict()
|
||||
assert "lease_token" not in task.to_dict()
|
||||
|
||||
|
||||
def test_chain_media_legacy_scraping_symbols_resolve_to_scraping_chain():
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import unittest
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
@@ -8,7 +9,11 @@ from app.application.history import (
|
||||
failed_retry_count,
|
||||
record_transfer_failure,
|
||||
)
|
||||
from app.application.transfer import TransferPlanningInput, TransferTask
|
||||
from app.application.transfer import (
|
||||
TransferAdmission,
|
||||
TransferPlanningInput,
|
||||
TransferTask,
|
||||
)
|
||||
from app.chain.transfer import JobManager, TransferChain
|
||||
from app.domain.context import MediaInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
@@ -151,6 +156,7 @@ def make_task(episode: int, season: int = 1) -> TransferTask:
|
||||
|
||||
|
||||
def make_transfer_chain() -> TransferChain:
|
||||
"""构造带内存 durable admission 契约的整理链测试骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.jobview = JobManager()
|
||||
chain._media_exts = settings.RMT_MEDIAEXT
|
||||
@@ -161,6 +167,66 @@ def make_transfer_chain() -> TransferChain:
|
||||
)
|
||||
chain._success_target_files = {}
|
||||
chain._scrape_batches = {}
|
||||
admissions = MagicMock()
|
||||
admissions_by_identity = {}
|
||||
admissions_by_id = {}
|
||||
|
||||
def admit(*, storage, src_path, planning_input=None):
|
||||
"""按源身份幂等返回测试用 durable admission。"""
|
||||
identity = storage, src_path
|
||||
existing = admissions_by_identity.get(identity)
|
||||
if existing is not None:
|
||||
return existing
|
||||
admission = TransferAdmission(
|
||||
task_id=f"test-task-{len(admissions_by_id) + 1}",
|
||||
storage=storage,
|
||||
src_path=src_path,
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
admissions_by_identity[identity] = admission
|
||||
admissions_by_id[admission.task_id] = admission
|
||||
return admission
|
||||
|
||||
def claim_task(*, task_id, owner_id, lease_seconds):
|
||||
"""为测试任务返回唯一 token，并保留正式 claim 的参数约束。"""
|
||||
assert lease_seconds > 0
|
||||
admission = admissions_by_id[task_id]
|
||||
claimed = replace(
|
||||
admission,
|
||||
lease_owner=owner_id,
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=admission.attempt_count + 1,
|
||||
)
|
||||
admissions_by_identity[(claimed.storage, claimed.src_path)] = claimed
|
||||
admissions_by_id[task_id] = claimed
|
||||
return claimed
|
||||
|
||||
def checkpoint_plan(*, task_id, lease_token, input_fingerprint, checkpoint):
|
||||
"""回读带检查点的持久投影，供同步整理测试执行真实编排。"""
|
||||
del input_fingerprint
|
||||
admission = admissions_by_id[task_id]
|
||||
assert admission.lease_token == lease_token
|
||||
planned = replace(
|
||||
admission,
|
||||
state="provider_pending" if checkpoint.is_provider_pending else "planned",
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
admissions_by_identity[(planned.storage, planned.src_path)] = planned
|
||||
admissions_by_id[task_id] = planned
|
||||
return planned
|
||||
|
||||
admissions.admit.side_effect = admit
|
||||
admissions.claim_task.side_effect = claim_task
|
||||
admissions.checkpoint_plan.side_effect = checkpoint_plan
|
||||
admissions.discard_claimed.return_value = 1
|
||||
admissions.release_claim.return_value = True
|
||||
chain._transfer_admissions = admissions
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""整理恢复租约字段的 Alembic 可逆迁移测试。"""
|
||||
|
||||
import importlib
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic.migration import MigrationContext
|
||||
from alembic.operations import Operations
|
||||
|
||||
from app.db.models.transferpending import TransferPending
|
||||
|
||||
MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
|
||||
LEASE_COLUMNS = {
|
||||
"lease_owner",
|
||||
"lease_token",
|
||||
"lease_expires_at",
|
||||
"heartbeat_at",
|
||||
"attempt_count",
|
||||
}
|
||||
|
||||
|
||||
def _bind_migration(monkeypatch, connection):
|
||||
"""把 3.0.15 迁移绑定到隔离数据库连接。"""
|
||||
migration = importlib.import_module(MIGRATION)
|
||||
monkeypatch.setattr(
|
||||
migration,
|
||||
"op",
|
||||
Operations(MigrationContext.configure(connection)),
|
||||
)
|
||||
return migration
|
||||
|
||||
|
||||
def _create_planning_table(connection) -> None:
|
||||
"""创建 3.0.14 时代包含完整规划检查点的登记表。"""
|
||||
metadata = sa.MetaData()
|
||||
table = sa.Table(
|
||||
"transferpending",
|
||||
metadata,
|
||||
sa.Column("id", sa.Integer(), primary_key=True),
|
||||
sa.Column("task_id", sa.String(64), nullable=False),
|
||||
sa.Column("storage", sa.String(), nullable=False),
|
||||
sa.Column("src_path", sa.String(), nullable=False),
|
||||
sa.Column("created_at", sa.String(), nullable=True),
|
||||
sa.Column("state", sa.String(32), nullable=False),
|
||||
sa.Column("updated_at", sa.String(40), nullable=False),
|
||||
sa.Column("last_error", sa.Text(), nullable=True),
|
||||
sa.Column("input_version", sa.Integer(), nullable=False),
|
||||
sa.Column("planning_input", sa.JSON(), nullable=False),
|
||||
sa.Column("input_fingerprint", sa.String(64), nullable=False),
|
||||
sa.Column("checkpoint_version", sa.Integer(), nullable=True),
|
||||
sa.Column("checkpoint_payload", sa.JSON(), nullable=True),
|
||||
sa.Column("planned_at", sa.String(40), nullable=True),
|
||||
sa.UniqueConstraint("task_id", name="uq_transferpending_task_id"),
|
||||
)
|
||||
sa.Index(
|
||||
"ux_transferpending_storage_path",
|
||||
table.c.storage,
|
||||
table.c.src_path,
|
||||
unique=True,
|
||||
)
|
||||
sa.Index(
|
||||
"ix_transferpending_state_created",
|
||||
table.c.state,
|
||||
table.c.created_at,
|
||||
table.c.id,
|
||||
)
|
||||
metadata.create_all(connection)
|
||||
connection.execute(
|
||||
table.insert(),
|
||||
{
|
||||
"id": 1,
|
||||
"task_id": "stable-task",
|
||||
"storage": "local",
|
||||
"src_path": "/downloads/Movie.mkv",
|
||||
"created_at": "2026-08-27 10:00:00",
|
||||
"state": "planned",
|
||||
"updated_at": "2026-08-27 10:00:00",
|
||||
"last_error": None,
|
||||
"input_version": 1,
|
||||
"planning_input": {
|
||||
"schema_version": 1,
|
||||
"source_fileitem": {
|
||||
"storage": "local",
|
||||
"path": "/downloads/Movie.mkv",
|
||||
},
|
||||
},
|
||||
"input_fingerprint": "0" * 64,
|
||||
"checkpoint_version": 1,
|
||||
"checkpoint_payload": {"schema_version": 1},
|
||||
"planned_at": "2026-08-27 10:00:00",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_transfer_lease_upgrade_downgrade_reupgrade(monkeypatch) -> None:
|
||||
"""SQLite 应支持租约字段重复升级、降级和再次升级。"""
|
||||
engine = sa.create_engine("sqlite://")
|
||||
with engine.begin() as connection:
|
||||
_create_planning_table(connection)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
|
||||
migration.upgrade()
|
||||
migration.upgrade()
|
||||
|
||||
inspector = sa.inspect(connection)
|
||||
assert {
|
||||
column["name"]
|
||||
for column in inspector.get_columns("transferpending")
|
||||
} == {column.name for column in TransferPending.__table__.columns}
|
||||
assert "ix_transferpending_recovery_lease" in {
|
||||
index["name"]
|
||||
for index in inspector.get_indexes("transferpending")
|
||||
}
|
||||
row = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, lease_expires_at, "
|
||||
"heartbeat_at, attempt_count FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(row) == {
|
||||
"lease_owner": None,
|
||||
"lease_token": None,
|
||||
"lease_expires_at": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"UPDATE transferpending SET lease_owner = 'worker', "
|
||||
"lease_token = 'token', "
|
||||
"lease_expires_at = '2026-08-27 10:05:00.000000', "
|
||||
"heartbeat_at = '2026-08-27 10:00:00.000000', "
|
||||
"attempt_count = 3 WHERE id = 1"
|
||||
)
|
||||
)
|
||||
migration.downgrade()
|
||||
|
||||
downgraded = sa.inspect(connection)
|
||||
assert LEASE_COLUMNS.isdisjoint({
|
||||
column["name"]
|
||||
for column in downgraded.get_columns("transferpending")
|
||||
})
|
||||
assert "ix_transferpending_recovery_lease" not in {
|
||||
index["name"]
|
||||
for index in downgraded.get_indexes("transferpending")
|
||||
}
|
||||
assert connection.execute(
|
||||
sa.text("SELECT state FROM transferpending WHERE id = 1")
|
||||
).scalar_one() == "planned"
|
||||
|
||||
migration.upgrade()
|
||||
reupgraded = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, attempt_count "
|
||||
"FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(reupgraded) == {
|
||||
"lease_owner": None,
|
||||
"lease_token": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_partial_transfer_lease_upgrade_preserves_existing_owner(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""迁移中断后重跑应补齐字段且不得覆盖已经写入的租约拥有者。"""
|
||||
engine = sa.create_engine("sqlite://")
|
||||
with engine.begin() as connection:
|
||||
_create_planning_table(connection)
|
||||
connection.execute(
|
||||
sa.text("ALTER TABLE transferpending ADD COLUMN lease_owner VARCHAR(128)")
|
||||
)
|
||||
connection.execute(
|
||||
sa.text(
|
||||
"UPDATE transferpending SET lease_owner = 'preserved-worker' "
|
||||
"WHERE id = 1"
|
||||
)
|
||||
)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
|
||||
migration.upgrade()
|
||||
|
||||
row = connection.execute(
|
||||
sa.text(
|
||||
"SELECT lease_owner, lease_token, attempt_count "
|
||||
"FROM transferpending WHERE id = 1"
|
||||
)
|
||||
).mappings().one()
|
||||
assert dict(row) == {
|
||||
"lease_owner": "preserved-worker",
|
||||
"lease_token": None,
|
||||
"attempt_count": 0,
|
||||
}
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,801 @@
|
||||
"""整理恢复租约的原子 claim、续租和陈旧 token 防护测试。"""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from threading import Barrier, Lock
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select, text
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application.transfer import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TransferAdmission,
|
||||
TransferAdmissionProjectionError,
|
||||
TransferLeaseLostError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanningInput,
|
||||
TransferProviderInvocationSnapshot,
|
||||
TransferProviderReference,
|
||||
)
|
||||
from app.db.adapters.transfer import TransactionalTransferAdmissionRepository
|
||||
from app.db.models.transferpending import TransferPending
|
||||
from app.db.oper.transferpending import TransferPendingOper
|
||||
|
||||
|
||||
def _planning_input(path: str) -> TransferPlanningInput:
|
||||
"""构造与测试源路径绑定的最小规划输入。"""
|
||||
return TransferPlanningInput(
|
||||
source_fileitem={
|
||||
"storage": "local",
|
||||
"path": path,
|
||||
"type": "file",
|
||||
"name": path.rsplit("/", maxsplit=1)[-1],
|
||||
},
|
||||
meta={"name": "Movie"},
|
||||
mediainfo={"title": "Movie"},
|
||||
)
|
||||
|
||||
|
||||
def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
|
||||
"""构造无需文件副作用的合法宿主跳过检查点。"""
|
||||
return TransferPlanCheckpoint(
|
||||
planning_input=planning_input,
|
||||
target_storage="local",
|
||||
root_target_path="/library",
|
||||
final_target_path="/library",
|
||||
resolved_transfer_type="copy",
|
||||
items=(),
|
||||
skip_reason="测试跳过计划",
|
||||
)
|
||||
|
||||
|
||||
def _provider_checkpoint(
|
||||
planning_input: TransferPlanningInput,
|
||||
) -> TransferPlanCheckpoint:
|
||||
"""构造只冻结 provider ABI、尚未完成宿主规划的检查点。"""
|
||||
invocation = TransferProviderInvocationSnapshot(
|
||||
fileitem=planning_input.source_fileitem,
|
||||
meta=planning_input.meta,
|
||||
meta_kind="MetaVideo",
|
||||
mediainfo=planning_input.mediainfo,
|
||||
mediainfo_kind="MediaInfo",
|
||||
)
|
||||
return TransferPlanCheckpoint(
|
||||
planning_input=planning_input,
|
||||
target_storage="",
|
||||
root_target_path="",
|
||||
final_target_path="",
|
||||
resolved_transfer_type="",
|
||||
items=(),
|
||||
resolved_meta=invocation.meta,
|
||||
resolved_meta_kind=invocation.meta_kind,
|
||||
resolved_mediainfo=invocation.mediainfo,
|
||||
resolved_mediainfo_kind=invocation.mediainfo_kind,
|
||||
legacy_transfer_providers=(
|
||||
TransferProviderReference(
|
||||
plugin_id="provider-a",
|
||||
plugin_name="Provider A",
|
||||
),
|
||||
),
|
||||
provider_invocation=invocation,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repository_factory(tmp_path):
|
||||
"""创建允许多线程独立 Session 竞争的 SQLite 租约仓储工厂。"""
|
||||
engine = create_engine(
|
||||
f"sqlite:///{tmp_path / 'transfer-lease.db'}",
|
||||
connect_args={"check_same_thread": False, "timeout": 10},
|
||||
)
|
||||
TransferPending.__table__.create(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
yield lambda: TransactionalTransferAdmissionRepository(factory)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def lease_clock(monkeypatch):
|
||||
"""为所有仓储实例提供可推进的固定 UTC 时钟。"""
|
||||
clock = {"now": datetime(2026, 8, 27, 10, 0, tzinfo=timezone.utc)}
|
||||
monkeypatch.setattr(
|
||||
TransactionalTransferAdmissionRepository,
|
||||
"_lease_now",
|
||||
staticmethod(lambda: clock["now"]),
|
||||
)
|
||||
return clock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def business_clock(monkeypatch):
|
||||
"""为业务审计字段提供与 UTC 租约时钟明确分离的宿主本地时间。"""
|
||||
clock = {"now": "2026-08-27 18:00:00"}
|
||||
monkeypatch.setattr(
|
||||
TransactionalTransferAdmissionRepository,
|
||||
"_now",
|
||||
staticmethod(lambda: clock["now"]),
|
||||
)
|
||||
return clock
|
||||
|
||||
|
||||
def _admit(
|
||||
repository: TransactionalTransferAdmissionRepository,
|
||||
path: str,
|
||||
) -> TransferAdmission:
|
||||
"""登记一个带完整版本化输入的测试任务。"""
|
||||
return repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=_planning_input(path),
|
||||
)
|
||||
|
||||
|
||||
def _pending_snapshot(
|
||||
repository: TransactionalTransferAdmissionRepository,
|
||||
task_id: str,
|
||||
) -> dict[str, Any]:
|
||||
"""在 Session 关闭前冻结测试需要检查的持久登记字段。"""
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(TransferPending.task_id == task_id)
|
||||
).scalar_one()
|
||||
return {
|
||||
"lease_owner": pending.lease_owner,
|
||||
"lease_token": pending.lease_token,
|
||||
"lease_expires_at": pending.lease_expires_at,
|
||||
"heartbeat_at": pending.heartbeat_at,
|
||||
"last_error": pending.last_error,
|
||||
"attempt_count": pending.attempt_count,
|
||||
"updated_at": pending.updated_at,
|
||||
"planned_at": pending.planned_at,
|
||||
}
|
||||
|
||||
|
||||
def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""新 token 才增加 attempt，过期 token 不能续租、释放或删除接管者。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/movie.mkv")
|
||||
assert admitted.created_at == "2026-08-27 18:00:00"
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
|
||||
first = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert first is not None
|
||||
assert first.lease_owner == "worker-a"
|
||||
assert first.lease_token
|
||||
assert first.attempt_count == 1
|
||||
assert first.updated_at == "2026-08-27 18:01:00"
|
||||
|
||||
assert repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
assert repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=30)
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
renewed = repository.heartbeat(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert renewed is not None
|
||||
assert renewed.attempt_count == 1
|
||||
assert renewed.heartbeat_at == "2026-08-27 10:00:30.000000"
|
||||
assert renewed.updated_at == first.updated_at
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=61)
|
||||
business_clock["now"] = "2026-08-27 18:03:00"
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="expired worker",
|
||||
) is False
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
) == 0
|
||||
takeover = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert takeover is not None
|
||||
assert takeover.lease_token != first.lease_token
|
||||
assert takeover.attempt_count == 2
|
||||
assert takeover.updated_at == "2026-08-27 18:03:00"
|
||||
assert repository.heartbeat(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
lease_seconds=60,
|
||||
) is None
|
||||
business_clock["now"] = "2026-08-27 18:04:00"
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="stale worker",
|
||||
) is False
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
) == 0
|
||||
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
error="retry later",
|
||||
) is True
|
||||
released = _pending_snapshot(repository, admitted.task_id)
|
||||
assert released["lease_owner"] is None
|
||||
assert released["lease_token"] is None
|
||||
assert released["lease_expires_at"] is None
|
||||
assert released["heartbeat_at"] is None
|
||||
assert released["last_error"] == "retry later"
|
||||
assert released["attempt_count"] == 2
|
||||
assert released["updated_at"] == "2026-08-27 18:04:00"
|
||||
|
||||
third = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-c",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert third is not None
|
||||
assert third.attempt_count == 3
|
||||
assert repository.discard_claimed(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=third.lease_token,
|
||||
) == 1
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
assert session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one_or_none() is None
|
||||
|
||||
|
||||
def test_claim_recoverable_respects_order_limit_and_active_lease(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""批量恢复跳过有效租约，并按登记顺序逐条 CAS 到请求上限。"""
|
||||
repository = repository_factory()
|
||||
first = _admit(repository, "/downloads/a.mkv")
|
||||
second = _admit(repository, "/downloads/b.mkv")
|
||||
third = _admit(repository, "/downloads/c.mkv")
|
||||
active = repository.claim_task(
|
||||
task_id=first.task_id,
|
||||
owner_id="active-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert active is not None
|
||||
|
||||
claimed = repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
assert [item.task_id for item in claimed] == [second.task_id, third.task_id]
|
||||
assert all(item.lease_owner == "recovery-worker" for item in claimed)
|
||||
assert all(item.attempt_count == 1 for item in claimed)
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="other-worker",
|
||||
limit=10,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=61)
|
||||
reclaimed = repository.claim_recoverable(
|
||||
owner_id="takeover-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert [item.task_id for item in reclaimed] == [first.task_id, second.task_id]
|
||||
assert reclaimed[0].attempt_count == 2
|
||||
assert reclaimed[1].attempt_count == 2
|
||||
|
||||
|
||||
def test_claim_recoverable_skips_corrupt_projection_and_claims_later_tasks(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""毒行应留下单次诊断但不持有租约或饿死后续健康任务。"""
|
||||
repository = repository_factory()
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
corrupt = _admit(repository, "/downloads/a-corrupt.mkv")
|
||||
healthy = [
|
||||
_admit(repository, "/downloads/b-healthy.mkv"),
|
||||
_admit(repository, "/downloads/c-healthy.mkv"),
|
||||
]
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == corrupt.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
claimed = repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
assert [item.task_id for item in claimed] == [item.task_id for item in healthy]
|
||||
corrupt_snapshot = _pending_snapshot(repository, corrupt.task_id)
|
||||
assert corrupt_snapshot["lease_token"] is None
|
||||
assert corrupt_snapshot["attempt_count"] == 0
|
||||
assert corrupt_snapshot["last_error"].startswith("恢复投影失败:")
|
||||
assert corrupt_snapshot["updated_at"] == "2026-08-27 18:00:00"
|
||||
assert len(messages) == 1
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="second-recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
assert _pending_snapshot(repository, corrupt.task_id) == corrupt_snapshot
|
||||
assert len(messages) == 1
|
||||
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == corrupt.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = _planning_input(
|
||||
"/downloads/a-corrupt.mkv"
|
||||
).fingerprint
|
||||
session.commit()
|
||||
|
||||
repaired = repository.claim_recoverable(
|
||||
owner_id="repaired-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert [item.task_id for item in repaired] == [corrupt.task_id]
|
||||
|
||||
|
||||
def test_projection_diagnostic_changes_are_recorded_once_each(
|
||||
repository_factory,
|
||||
business_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""相同投影错误不重复写库，错误类型变化时才更新诊断并再次告警。"""
|
||||
repository = repository_factory()
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
admitted = _admit(repository, "/downloads/changing-corrupt.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-a",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
first = _pending_snapshot(repository, admitted.task_id)
|
||||
assert len(messages) == 1
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-b",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
assert _pending_snapshot(repository, admitted.task_id) == first
|
||||
assert len(messages) == 1
|
||||
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = _planning_input(
|
||||
"/downloads/changing-corrupt.mkv"
|
||||
).fingerprint
|
||||
pending.input_version = 999
|
||||
session.commit()
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="recovery-c",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
) == []
|
||||
changed = _pending_snapshot(repository, admitted.task_id)
|
||||
assert changed["last_error"] != first["last_error"]
|
||||
assert changed["updated_at"] == "2026-08-27 18:02:00"
|
||||
assert len(messages) == 2
|
||||
|
||||
|
||||
def test_projection_diagnostic_cas_is_concurrency_safe(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""并发恢复观察到同一损坏时只允许一个诊断写入者和一条运行日志。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/concurrent-corrupt.mkv")
|
||||
messages: list[str] = []
|
||||
message_lock = Lock()
|
||||
|
||||
def capture(message: str) -> None:
|
||||
"""并发安全收集错误日志。"""
|
||||
with message_lock:
|
||||
messages.append(message)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
capture,
|
||||
)
|
||||
barrier = Barrier(2)
|
||||
projection_error = TransferAdmissionProjectionError("same corruption")
|
||||
|
||||
def record(_: int) -> bool:
|
||||
"""让两个独立 Session 同时竞争同一诊断 CAS。"""
|
||||
barrier.wait(timeout=5)
|
||||
return repository_factory()._record_projection_failure( # noqa: SLF001
|
||||
task_id=admitted.task_id,
|
||||
error=projection_error,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(record, range(2)))
|
||||
|
||||
assert sorted(results) == [False, True]
|
||||
assert len(messages) == 1
|
||||
snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert snapshot["last_error"] == "恢复投影失败: same corruption"
|
||||
|
||||
|
||||
def test_projection_diagnostic_does_not_overwrite_active_lease(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""诊断 CAS 不得覆盖已经由健康 worker 取得有效租约的任务。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/active-lease.mkv")
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="active-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
before = _pending_snapshot(repository, admitted.task_id)
|
||||
messages: list[str] = []
|
||||
monkeypatch.setattr(
|
||||
"app.db.adapters.transfer._diagnostic_logger.error",
|
||||
messages.append,
|
||||
)
|
||||
|
||||
recorded = repository._record_projection_failure( # noqa: SLF001
|
||||
task_id=admitted.task_id,
|
||||
error=TransferAdmissionProjectionError("stale observation"),
|
||||
)
|
||||
|
||||
assert recorded is False
|
||||
assert _pending_snapshot(repository, admitted.task_id) == before
|
||||
assert messages == []
|
||||
|
||||
|
||||
def test_projection_diagnostic_database_failure_propagates(
|
||||
repository_factory,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""诊断留痕的数据库基础设施异常必须向上游传播而非静默跳过。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/db-error-corrupt.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admitted.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
pending.input_fingerprint = "corrupt"
|
||||
session.commit()
|
||||
|
||||
def fail_diagnostic(*_args, **_kwargs):
|
||||
"""模拟诊断短事务的底层数据库写入失败。"""
|
||||
raise RuntimeError("database unavailable")
|
||||
|
||||
monkeypatch.setattr(
|
||||
TransferPendingOper,
|
||||
"stage_record_projection_failure",
|
||||
fail_diagnostic,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="database unavailable"):
|
||||
repository.claim_recoverable(
|
||||
owner_id="recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
|
||||
def test_claim_task_wraps_persisted_json_decode_failure(
|
||||
repository_factory,
|
||||
) -> None:
|
||||
"""持久 JSON 解码错误应归类为投影损坏，而不是数据库基础设施故障。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/invalid-json.mkv")
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
session.execute(
|
||||
text(
|
||||
"UPDATE transferpending SET planning_input = 'not-json' "
|
||||
"WHERE task_id = :task_id"
|
||||
),
|
||||
{"task_id": admitted.task_id},
|
||||
)
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="JSON"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="recovery-worker",
|
||||
lease_seconds=60,
|
||||
)
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
snapshot = session.execute(
|
||||
text(
|
||||
"SELECT lease_token, attempt_count FROM transferpending "
|
||||
"WHERE task_id = :task_id"
|
||||
),
|
||||
{"task_id": admitted.task_id},
|
||||
).mappings().one()
|
||||
assert snapshot["lease_token"] is None
|
||||
assert snapshot["attempt_count"] == 0
|
||||
|
||||
|
||||
def test_concurrent_recovery_callers_scan_past_lost_candidates(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""并发 caller 竞争同一首批后应继续向后扫描并各自填满限额。"""
|
||||
setup_repository = repository_factory()
|
||||
admitted = [
|
||||
_admit(setup_repository, f"/downloads/concurrent-{index}.mkv")
|
||||
for index in range(4)
|
||||
]
|
||||
barrier = Barrier(2)
|
||||
barrier_lock = Lock()
|
||||
initial_scans = 0
|
||||
original = TransferPending.list_claimable_candidates.__func__
|
||||
|
||||
def synchronized_candidates(cls, db, **kwargs):
|
||||
"""强制两个 caller 在取得相同首批候选后再进入逐任务 CAS。"""
|
||||
nonlocal initial_scans
|
||||
candidates = original(cls, db, **kwargs)
|
||||
if kwargs.get("after_cursor") is None:
|
||||
with barrier_lock:
|
||||
initial_scans += 1
|
||||
barrier.wait(timeout=5)
|
||||
return candidates
|
||||
|
||||
monkeypatch.setattr(
|
||||
TransferPending,
|
||||
"list_claimable_candidates",
|
||||
classmethod(synchronized_candidates),
|
||||
)
|
||||
|
||||
def recover(owner_id: str) -> list[TransferAdmission]:
|
||||
"""使用独立仓储与 Session 执行一次有界恢复扫描。"""
|
||||
return repository_factory().claim_recoverable(
|
||||
owner_id=owner_id,
|
||||
limit=2,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(recover, ("worker-a", "worker-b")))
|
||||
|
||||
assert initial_scans == 2
|
||||
assert [len(result) for result in results] == [2, 2]
|
||||
claimed_task_ids = [item.task_id for result in results for item in result]
|
||||
assert len(set(claimed_task_ids)) == 4
|
||||
assert set(claimed_task_ids) == {item.task_id for item in admitted}
|
||||
|
||||
|
||||
def test_unclaimed_enqueue_failure_cannot_overwrite_claimed_task(
|
||||
repository_factory,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""task-id-only 入队失败入口不得改写已经由 worker claim 的登记。"""
|
||||
repository = repository_factory()
|
||||
admitted = _admit(repository, "/downloads/claimed.mkv")
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
before = _pending_snapshot(repository, admitted.task_id)
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
repository.record_enqueue_failure(
|
||||
task_id=admitted.task_id,
|
||||
error="stale queue failure",
|
||||
)
|
||||
|
||||
after = _pending_snapshot(repository, admitted.task_id)
|
||||
assert after == before
|
||||
|
||||
|
||||
def test_planning_writes_require_current_unexpired_lease(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""checkpoint 和规划错误均不得由已过期或已被接管的 worker 写入。"""
|
||||
repository = repository_factory()
|
||||
path = "/downloads/planning.mkv"
|
||||
planning_input = _planning_input(path)
|
||||
admitted = repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=planning_input,
|
||||
)
|
||||
first = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=30,
|
||||
)
|
||||
assert first is not None
|
||||
|
||||
lease_clock["now"] += timedelta(seconds=31)
|
||||
takeover = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-b",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert takeover is not None
|
||||
|
||||
with pytest.raises(TransferLeaseLostError, match="租约"):
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
error="stale planning failure",
|
||||
)
|
||||
with pytest.raises(TransferLeaseLostError, match="租约"):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=first.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
error="retryable planning failure",
|
||||
)
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=takeover.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
assert planned.state == TRANSFER_ADMISSION_PLANNED
|
||||
assert planned.last_error is None
|
||||
assert repeated == planned
|
||||
assert planned.attempt_count == 2
|
||||
|
||||
|
||||
def test_provider_checkpoint_sets_planned_time_only_after_host_plan(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
business_clock,
|
||||
) -> None:
|
||||
"""provider 快照不是规划完成，planned_at 只记录首次宿主完整计划。"""
|
||||
repository = repository_factory()
|
||||
path = "/downloads/provider-plan.mkv"
|
||||
planning_input = _planning_input(path)
|
||||
admitted = repository.admit(
|
||||
storage="local",
|
||||
src_path=path,
|
||||
planning_input=planning_input,
|
||||
)
|
||||
business_clock["now"] = "2026-08-27 18:01:00"
|
||||
claimed = repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="worker-a",
|
||||
lease_seconds=60,
|
||||
)
|
||||
assert claimed is not None
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:02:00"
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_provider_checkpoint(planning_input),
|
||||
)
|
||||
provider_snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert provider_snapshot["planned_at"] is None
|
||||
assert provider_snapshot["updated_at"] == "2026-08-27 18:02:00"
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:03:00"
|
||||
checkpoint = _checkpoint(planning_input)
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
planned_snapshot = _pending_snapshot(repository, admitted.task_id)
|
||||
assert planned_snapshot["planned_at"] == "2026-08-27 18:03:00"
|
||||
|
||||
business_clock["now"] = "2026-08-27 18:04:00"
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
assert _pending_snapshot(repository, admitted.task_id) == planned_snapshot
|
||||
|
||||
|
||||
def test_concurrent_claim_uses_rowcount_as_single_winner(
|
||||
repository_factory,
|
||||
lease_clock,
|
||||
) -> None:
|
||||
"""并发 worker 即使读取同一任务，也只能有一个 CAS 更新获胜。"""
|
||||
setup_repository = repository_factory()
|
||||
admitted = _admit(setup_repository, "/downloads/concurrent.mkv")
|
||||
barrier = Barrier(2)
|
||||
|
||||
def claim(owner_id: str):
|
||||
"""等待竞争者就绪后使用独立 Session claim 同一任务。"""
|
||||
repository = repository_factory()
|
||||
barrier.wait()
|
||||
return repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id=owner_id,
|
||||
lease_seconds=60,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as executor:
|
||||
results = list(executor.map(claim, ("worker-a", "worker-b")))
|
||||
|
||||
winners = [result for result in results if result is not None]
|
||||
assert len(winners) == 1
|
||||
assert winners[0].attempt_count == 1
|
||||
@@ -0,0 +1,158 @@
|
||||
"""旧待整理 Oper 的精确插件兼容与租约 fencing 测试。"""
|
||||
|
||||
import importlib
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from app.db import base as db_base
|
||||
from app.db.models.transferpending import TransferPending
|
||||
from app.db.oper.transferpending import TransferPendingOper as CanonicalTransferPendingOper
|
||||
from app.runtime.compat.manifest import MODULE_ALIASES
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def legacy_session_factory(tmp_path, monkeypatch):
|
||||
"""为无 Session 兼容 Oper 提供独占事务，并在提交后保留返回快照。"""
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'legacy-transferpending.db'}")
|
||||
TransferPending.__table__.create(engine)
|
||||
factory = sessionmaker(bind=engine)
|
||||
|
||||
def run_transaction(operation: Callable[[Session], Any]) -> Any:
|
||||
"""按生产组合根语义执行一次同步兼容事务。"""
|
||||
with factory() as session:
|
||||
session.expire_on_commit = False
|
||||
try:
|
||||
result = operation(session)
|
||||
session.commit()
|
||||
return result
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(db_base, "run_sync_transaction", run_transaction)
|
||||
yield factory
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_legacy_import_targets_private_sdk_facade() -> None:
|
||||
"""旧模块路径只能解析到私有 SDK 门面，不能回退到 canonical Oper。"""
|
||||
alias = MODULE_ALIASES["app.db.transferpending_oper"]
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
|
||||
assert alias.target == "app.sdk._legacy.transferpending"
|
||||
assert alias.owner == "sdk"
|
||||
assert alias.replacement == "app.application.transfer"
|
||||
assert legacy is importlib.import_module(alias.target)
|
||||
assert legacy.__all__ == ["TransferPendingOper"]
|
||||
assert not hasattr(legacy, "TransferPending")
|
||||
assert legacy.TransferPendingOper is not CanonicalTransferPendingOper
|
||||
for internal_method in (
|
||||
"stage_admit",
|
||||
"stage_claim_task",
|
||||
"stage_discard_claimed",
|
||||
"stage_checkpoint_plan",
|
||||
):
|
||||
assert not hasattr(legacy.TransferPendingOper, internal_method)
|
||||
|
||||
|
||||
def test_legacy_no_session_queries_preserve_historical_shapes(
|
||||
legacy_session_factory,
|
||||
) -> None:
|
||||
"""旧插件无需 Session 即可登记和查询，返回形态与原 ABI 一致。"""
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
oper = legacy.TransferPendingOper()
|
||||
|
||||
first = oper.register("local", "/downloads/first.mkv")
|
||||
repeated = oper.register("local", "/downloads/first.mkv")
|
||||
second = oper.register("remote", "/downloads/second.mkv")
|
||||
|
||||
assert first is not None
|
||||
assert repeated is not None
|
||||
assert second is not None
|
||||
assert repeated.task_id == first.task_id
|
||||
assert oper.list_all() == [
|
||||
("local", "/downloads/first.mkv"),
|
||||
("remote", "/downloads/second.mkv"),
|
||||
]
|
||||
assert [item.task_id for item in oper.list_by_state(state="accepted")] == [
|
||||
first.task_id,
|
||||
second.task_id,
|
||||
]
|
||||
assert [
|
||||
item.task_id
|
||||
for item in oper.list_by_states(states=("accepted", "planned"), limit=1)
|
||||
] == [first.task_id]
|
||||
assert oper.get_by_identity(
|
||||
storage="local",
|
||||
src_path="/downloads/first.mkv",
|
||||
).task_id == first.task_id
|
||||
assert oper.get_by_task_id(task_id=second.task_id).src_path == second.src_path
|
||||
assert oper.register("", "/downloads/invalid.mkv") is None
|
||||
assert oper.list_by_state(state="") == []
|
||||
assert oper.list_by_states(states=()) == []
|
||||
|
||||
|
||||
def test_legacy_mutations_never_delete_claimed_rows(
|
||||
legacy_session_factory,
|
||||
) -> None:
|
||||
"""旧 discard/clear 只处理未 claim 行，有效或过期 token 均受保护。"""
|
||||
legacy = importlib.import_module("app.db.transferpending_oper")
|
||||
oper = legacy.TransferPendingOper()
|
||||
active = oper.register("local", "/downloads/active.mkv")
|
||||
expired = oper.register("local", "/downloads/expired.mkv")
|
||||
free = oper.register("local", "/downloads/free.mkv")
|
||||
assert active is not None
|
||||
assert expired is not None
|
||||
assert free is not None
|
||||
|
||||
with legacy_session_factory() as session:
|
||||
assert TransferPending.claim_task(
|
||||
session,
|
||||
task_id=active.task_id,
|
||||
states=("accepted",),
|
||||
owner_id="active-worker",
|
||||
lease_token="active-token",
|
||||
now_time="2026-08-27 10:00:00.000000",
|
||||
lease_expires_at="2026-08-27 10:01:00.000000",
|
||||
updated_at="2026-08-27 18:00:00",
|
||||
) == 1
|
||||
assert TransferPending.claim_task(
|
||||
session,
|
||||
task_id=expired.task_id,
|
||||
states=("accepted",),
|
||||
owner_id="expired-worker",
|
||||
lease_token="expired-token",
|
||||
now_time="2026-08-27 10:00:00.000000",
|
||||
lease_expires_at="2026-08-27 09:59:00.000000",
|
||||
updated_at="2026-08-27 18:00:00",
|
||||
) == 1
|
||||
session.commit()
|
||||
|
||||
repeated = oper.register("local", "/downloads/active.mkv")
|
||||
assert repeated is not None
|
||||
assert repeated.task_id == active.task_id
|
||||
assert repeated.lease_token == "active-token"
|
||||
assert oper.discard("local", "/downloads/active.mkv") == 0
|
||||
assert oper.discard("local", "/downloads/expired.mkv") == 0
|
||||
assert oper.discard("local", "/downloads/free.mkv") == 1
|
||||
|
||||
removable = oper.register("local", "/downloads/removable.mkv")
|
||||
assert removable is not None
|
||||
assert oper.clear() == 1
|
||||
assert oper.list_all() == [
|
||||
("local", "/downloads/active.mkv"),
|
||||
("local", "/downloads/expired.mkv"),
|
||||
]
|
||||
|
||||
with legacy_session_factory() as session:
|
||||
rows = session.execute(
|
||||
select(TransferPending).order_by(TransferPending.id.asc())
|
||||
).scalars().all()
|
||||
assert [(row.task_id, row.lease_token) for row in rows] == [
|
||||
(active.task_id, "active-token"),
|
||||
(expired.task_id, "expired-token"),
|
||||
]
|
||||
@@ -8,6 +8,8 @@
|
||||
这些测试固定三项不变量：入队即落盘登记、终态即注销、重启能回放。
|
||||
"""
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -24,6 +26,17 @@ def _build_chain(admissions) -> TransferChain:
|
||||
"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = admissions
|
||||
chain._worker_owner_id = "test-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._replay_stop_event = threading.Event()
|
||||
chain._lease_heartbeat_stop_event = threading.Event()
|
||||
chain._lease_heartbeat_thread = None
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
@@ -36,6 +49,11 @@ def _admission(path: str, task_id: str = "task-1") -> TransferAdmission:
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner="test-owner",
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
|
||||
@@ -87,10 +105,19 @@ def test_discard_pending_on_terminal_state():
|
||||
chain = _build_chain(admissions)
|
||||
task = _task("/mnt/cd2/downloads/Movie.2024.mkv")
|
||||
task.bind_admission_task_id("task-1")
|
||||
task.bind_execution_lease(owner_id="test-owner", lease_token="lease-task-1")
|
||||
chain._worker_owner_id = "test-owner"
|
||||
chain._owned_leases = {
|
||||
"task-1": ("lease-task-1", time.monotonic() + 120)
|
||||
}
|
||||
admissions.discard_claimed.return_value = 1
|
||||
|
||||
chain._TransferChain__discard_pending(task)
|
||||
assert chain._TransferChain__discard_pending(task) is True
|
||||
|
||||
admissions.discard_task.assert_called_once_with(task_id="task-1")
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
|
||||
@@ -101,7 +128,7 @@ def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
|
||||
media.write_bytes(b"x" * 10)
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(media))]
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
transferred = []
|
||||
@@ -128,14 +155,18 @@ def test_replay_discards_vanished_files(tmp_path):
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
missing = tmp_path / "gone.mkv"
|
||||
admissions.list_recoverable.return_value = [_admission(str(missing))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(missing))]
|
||||
admissions.discard_claimed.return_value = 1
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
chain._execute_transfer.assert_not_called()
|
||||
admissions.discard_task.assert_called_once_with(task_id="task-1")
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
@@ -148,7 +179,8 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
media.write_bytes(b"x")
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.claim_recoverable.return_value = [_admission(str(media))]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -163,7 +195,12 @@ def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
chain._execute_transfer.assert_not_called()
|
||||
admissions.discard_task.assert_not_called()
|
||||
admissions.discard_claimed.assert_not_called()
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
error="恢复源文件暂时不可读取",
|
||||
)
|
||||
|
||||
|
||||
def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
|
||||
@@ -175,7 +212,7 @@ def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
|
||||
src_path = f"{bluray.as_posix()}/"
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [_admission(src_path)]
|
||||
admissions.claim_recoverable.return_value = [_admission(src_path)]
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
transferred = []
|
||||
@@ -197,7 +234,7 @@ def test_replay_is_noop_without_registrations():
|
||||
没有登记时回放不应触碰整理链。
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = []
|
||||
admissions.claim_recoverable.return_value = []
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -211,7 +248,7 @@ def test_replay_survives_db_failure():
|
||||
读取登记失败不能让启动流程报错。
|
||||
"""
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.side_effect = RuntimeError("db gone")
|
||||
admissions.claim_recoverable.side_effect = RuntimeError("db gone")
|
||||
chain = _build_chain(admissions)
|
||||
chain._execute_transfer = MagicMock()
|
||||
|
||||
@@ -230,7 +267,7 @@ def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
|
||||
item.write_bytes(b"x")
|
||||
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [
|
||||
admissions.claim_recoverable.return_value = [
|
||||
_admission(str(first), "task-1"),
|
||||
_admission(str(second), "task-2"),
|
||||
]
|
||||
@@ -261,7 +298,7 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
|
||||
first.write_bytes(b"x")
|
||||
missing_second = tmp_path / "gone.mkv"
|
||||
admissions = MagicMock()
|
||||
admissions.list_recoverable.return_value = [
|
||||
admissions.claim_recoverable.return_value = [
|
||||
_admission(str(first), "task-1"),
|
||||
_admission(str(missing_second), "task-2"),
|
||||
]
|
||||
@@ -279,4 +316,94 @@ def test_replay_stop_keeps_unprocessed_registrations(tmp_path, monkeypatch):
|
||||
chain._TransferChain__replay_pending(stop_event)
|
||||
|
||||
assert transferred == [first.as_posix()]
|
||||
admissions.discard_task.assert_not_called()
|
||||
admissions.discard_claimed.assert_not_called()
|
||||
assert admissions.release_claim.call_count == 2
|
||||
|
||||
|
||||
def test_replay_registers_entire_claimed_batch_before_first_source_stat(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""批量 claim 返回后必须先把全部 token 交给 heartbeat，再做逐条同步 I/O。"""
|
||||
first = _admission(str(tmp_path / "A.mkv"), "task-1")
|
||||
second = _admission(str(tmp_path / "B.mkv"), "task-2")
|
||||
admissions = MagicMock()
|
||||
admissions.claim_recoverable.return_value = [first, second]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
|
||||
def observe_owned_batch(*_args, **_kwargs):
|
||||
"""首个 stat 前观察两个 token 已同时进入续期集合。"""
|
||||
assert set(chain._owned_leases) == {"task-1", "task-2"}
|
||||
return None, False
|
||||
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"_TransferChain__build_replay_fileitem",
|
||||
observe_owned_batch,
|
||||
)
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
assert admissions.release_claim.call_count == 2
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
|
||||
def test_replay_releases_claim_when_jobview_rejects_recovered_task(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
"""恢复任务未进入队列时必须立即 release，不能靠租约自然过期。"""
|
||||
media = tmp_path / "Movie.2024.mkv"
|
||||
media.write_bytes(b"x")
|
||||
admission = _admission(str(media))
|
||||
admission = replace(admission, checkpoint=MagicMock())
|
||||
admissions = MagicMock()
|
||||
admissions.claim_recoverable.return_value = [admission]
|
||||
admissions.release_claim.return_value = True
|
||||
chain = _build_chain(admissions)
|
||||
monkeypatch.setattr(
|
||||
chain,
|
||||
"_TransferChain__queue_planned_replay",
|
||||
MagicMock(return_value=False),
|
||||
)
|
||||
|
||||
chain._TransferChain__replay_pending()
|
||||
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="task-1",
|
||||
lease_token="lease-task-1",
|
||||
error="恢复任务未进入内存队列",
|
||||
)
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
|
||||
def test_claimed_enqueue_failure_never_uses_unfenced_error_writer() -> None:
|
||||
"""陈旧 token 入队失败只能 release_claim，不能覆盖新 owner 的 last_error。"""
|
||||
admissions = MagicMock()
|
||||
admissions.release_claim.return_value = False
|
||||
chain = _build_chain(admissions)
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain.replay_pending = MagicMock()
|
||||
task = _task("/downloads/stale-enqueue.mkv")
|
||||
task.bind_admission_task_id("stale-task")
|
||||
task.bind_execution_lease(
|
||||
owner_id="test-owner",
|
||||
lease_token="stale-token",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"stale-task": ("stale-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
chain._TransferChain__record_enqueue_failure(
|
||||
task,
|
||||
RuntimeError("queue closed"),
|
||||
)
|
||||
|
||||
admissions.record_enqueue_failure.assert_not_called()
|
||||
admissions.release_claim.assert_called_once_with(
|
||||
task_id="stale-task",
|
||||
lease_token="stale-token",
|
||||
error="queue closed",
|
||||
)
|
||||
assert chain._owned_leases == {}
|
||||
|
||||
@@ -2,13 +2,14 @@
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import threading
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application import transfer as transfer_application
|
||||
@@ -202,6 +203,31 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
|
||||
"""构造只保留规划编排依赖的 TransferChain 骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = repository or Mock()
|
||||
chain._worker_owner_id = "planning-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
|
||||
|
||||
def claim_task(**kwargs):
|
||||
"""为规划测试返回与进程 owner 匹配的稳定 claim。"""
|
||||
return transfer_application.TransferAdmission(
|
||||
task_id=kwargs["task_id"],
|
||||
storage="local",
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner=kwargs["owner_id"],
|
||||
lease_token=f"lease-{kwargs['task_id']}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
chain._transfer_admissions.claim_task.side_effect = claim_task
|
||||
chain._module_dispatcher = Mock()
|
||||
chain._module_dispatcher.freeze_plugin_providers.return_value = ()
|
||||
chain.eventmanager = Mock()
|
||||
@@ -217,6 +243,20 @@ def _chain(*, repository=None, checkpoint=None, result=None) -> TransferChain:
|
||||
return chain
|
||||
|
||||
|
||||
def _replay_chain(repository) -> TransferChain:
|
||||
"""构造绑定固定恢复 owner 且不启动真实 heartbeat 线程的测试链。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = repository
|
||||
chain._worker_owner_id = "replay-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain._closing = False
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = Mock()
|
||||
return chain
|
||||
|
||||
|
||||
def _real_dispatcher(plugins: dict) -> ModuleInvocationDispatcher:
|
||||
"""构造使用真实冻结解析与执行内核的内存插件调度器。"""
|
||||
plugin_catalog = Mock()
|
||||
@@ -725,6 +765,7 @@ def test_provider_empty_fallback_cas_failure_blocks_host_execution():
|
||||
chain.execute_transfer_plan.assert_not_called()
|
||||
repository.record_planning_failure.assert_called_once_with(
|
||||
task_id="task-provider-cas-failure",
|
||||
lease_token="lease-task-provider-cas-failure",
|
||||
error="CAS failed",
|
||||
)
|
||||
|
||||
@@ -804,7 +845,7 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
|
||||
return SimpleNamespace(checkpoint=kwargs["checkpoint"])
|
||||
|
||||
repository.checkpoint_plan.side_effect = checkpoint_plan
|
||||
repository.discard_task.side_effect = (
|
||||
repository.discard_claimed.side_effect = (
|
||||
lambda **_kwargs: calls.append("discard") or 1
|
||||
)
|
||||
result = TransferInfo(
|
||||
@@ -830,8 +871,9 @@ def test_legacy_transfer_command_uses_durable_pipeline_and_settles_pending():
|
||||
|
||||
assert returned is result
|
||||
assert calls == ["admit", "checkpoint", "execute", "discard"]
|
||||
repository.discard_task.assert_called_once_with(
|
||||
task_id="task-legacy-command"
|
||||
repository.discard_claimed.assert_called_once_with(
|
||||
task_id="task-legacy-command",
|
||||
lease_token="lease-task-legacy-command",
|
||||
)
|
||||
|
||||
|
||||
@@ -898,16 +940,32 @@ def test_repository_rejects_checkpoint_with_mismatched_planning_fingerprint(tmp_
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=accepted_input,
|
||||
)
|
||||
claimed = repository.claim_task(
|
||||
task_id=admission.task_id,
|
||||
owner_id="fingerprint-test",
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token is not None
|
||||
mismatched = _checkpoint(target_path="/library/B/Movie.mkv")
|
||||
|
||||
with pytest.raises(ValueError, match="指纹|fingerprint|规划输入"):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=accepted_input.fingerprint,
|
||||
checkpoint=mismatched,
|
||||
)
|
||||
|
||||
recovered = repository.list_recoverable()
|
||||
assert repository.release_claim(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) is True
|
||||
recovered = repository.claim_recoverable(
|
||||
owner_id="fingerprint-recovery",
|
||||
limit=10,
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == "accepted"
|
||||
assert recovered[0].checkpoint is None
|
||||
@@ -926,23 +984,35 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
|
||||
planning_input=planning_input,
|
||||
)
|
||||
|
||||
accepted = repository.list_recoverable()
|
||||
assert len(accepted) == 1
|
||||
assert accepted[0].state == "accepted"
|
||||
assert accepted[0].planning_input == planning_input
|
||||
assert accepted[0].checkpoint is None
|
||||
claimed = repository.claim_task(
|
||||
task_id=admission.task_id,
|
||||
owner_id="roundtrip-owner",
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token is not None
|
||||
assert claimed.state == "accepted"
|
||||
assert claimed.planning_input == planning_input
|
||||
assert claimed.checkpoint is None
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="rename unavailable",
|
||||
)
|
||||
retryable = repository.list_recoverable()[0]
|
||||
assert retryable.state == "accepted"
|
||||
assert retryable.last_error == "rename unavailable"
|
||||
with sessionmaker(bind=engine)() as session:
|
||||
retryable = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == admission.task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert retryable.state == "accepted"
|
||||
assert retryable.last_error == "rename unavailable"
|
||||
|
||||
checkpoint = _checkpoint()
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -950,7 +1020,18 @@ def test_repository_round_trips_accepted_and_planned_recovery_states(tmp_path):
|
||||
assert planned.state == "planned"
|
||||
assert planned.checkpoint == checkpoint
|
||||
assert planned.last_error is None
|
||||
assert repository.list_recoverable() == [planned]
|
||||
assert repository.release_claim(
|
||||
task_id=admission.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
) is True
|
||||
recovered = repository.claim_recoverable(
|
||||
owner_id="roundtrip-recovery",
|
||||
limit=10,
|
||||
lease_seconds=120,
|
||||
)
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == "planned"
|
||||
assert recovered[0].checkpoint == checkpoint
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -982,6 +1063,10 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
|
||||
repository.admit.side_effect = admit
|
||||
repository.checkpoint_plan.side_effect = checkpoint_plan
|
||||
chain = _chain(repository=repository, checkpoint=checkpoint, result=result)
|
||||
claim_task = repository.claim_task.side_effect
|
||||
repository.claim_task.side_effect = lambda **kwargs: (
|
||||
order.append("claim") or claim_task(**kwargs)
|
||||
)
|
||||
chain.plan_transfer.side_effect = lambda *_args, **_kwargs: (
|
||||
order.append("plan") or checkpoint
|
||||
)
|
||||
@@ -992,12 +1077,13 @@ def test_checkpoint_commit_precedes_executor_and_sync_task_is_admitted():
|
||||
returned = chain._plan_checkpoint_and_execute(task)
|
||||
|
||||
assert returned is result
|
||||
assert order == ["admit", "plan", "commit-checkpoint", "execute"]
|
||||
assert order == ["admit", "claim", "plan", "commit-checkpoint", "execute"]
|
||||
assert task.admission_task_id == "task-sync"
|
||||
repository.admit.assert_called_once()
|
||||
assert repository.admit.call_args.kwargs["planning_input"] is planning_input
|
||||
repository.checkpoint_plan.assert_called_once_with(
|
||||
task_id="task-sync",
|
||||
lease_token="lease-task-sync",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -1192,11 +1278,15 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
planning_input=planning_input,
|
||||
lease_owner="replay-owner",
|
||||
lease_token="lease-task-accepted-offline",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
replay_repository = Mock()
|
||||
replay_repository.list_recoverable.return_value = [admission]
|
||||
replay_chain = object.__new__(TransferChain)
|
||||
replay_chain._transfer_admissions = replay_repository
|
||||
replay_repository.claim_recoverable.return_value = [admission]
|
||||
replay_chain = _replay_chain(replay_repository)
|
||||
replay_chain._execute_transfer = Mock()
|
||||
queued_tasks = []
|
||||
replay_chain.put_to_queue = Mock(
|
||||
@@ -1230,6 +1320,10 @@ def test_accepted_replay_restores_explicit_context_without_online_lookup(
|
||||
repository=execution_repository,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
execution_chain._worker_owner_id = "replay-owner"
|
||||
execution_chain._owned_leases = {
|
||||
admission.task_id: (str(admission.lease_token), float("inf"))
|
||||
}
|
||||
execution_chain.jobview = Mock()
|
||||
execution_chain.eventmanager = Mock()
|
||||
execution_chain.eventmanager.send_event.return_value = None
|
||||
@@ -1292,11 +1386,15 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
planning_input=planning_input,
|
||||
lease_owner="replay-owner",
|
||||
lease_token="lease-task-accepted-empty-episodes",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
replay_repository = Mock()
|
||||
replay_repository.list_recoverable.return_value = [admission]
|
||||
replay_chain = object.__new__(TransferChain)
|
||||
replay_chain._transfer_admissions = replay_repository
|
||||
replay_repository.claim_recoverable.return_value = [admission]
|
||||
replay_chain = _replay_chain(replay_repository)
|
||||
queued_tasks = []
|
||||
replay_chain.put_to_queue = Mock(
|
||||
side_effect=lambda task: queued_tasks.append(task) or True
|
||||
@@ -1319,6 +1417,10 @@ def test_accepted_replay_with_explicit_empty_episodes_stays_offline(
|
||||
repository=execution_repository,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
execution_chain._worker_owner_id = "replay-owner"
|
||||
execution_chain._owned_leases = {
|
||||
admission.task_id: (str(admission.lease_token), float("inf"))
|
||||
}
|
||||
execution_chain.jobview = Mock()
|
||||
execution_chain.eventmanager = Mock()
|
||||
execution_chain.eventmanager.send_event.return_value = None
|
||||
@@ -1425,8 +1527,20 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
|
||||
task = _task()
|
||||
task.meta = MetaBase("Unrecognized.Movie.2026.mkv")
|
||||
task.bind_admission_task_id("task-before-checkpoint")
|
||||
task.bind_execution_lease(
|
||||
owner_id="recognition-owner",
|
||||
lease_token="lease-task-before-checkpoint",
|
||||
)
|
||||
chain = object.__new__(TransferChain)
|
||||
chain._transfer_admissions = Mock()
|
||||
chain._worker_owner_id = "recognition-owner"
|
||||
chain._owned_leases = {
|
||||
"task-before-checkpoint": (
|
||||
"lease-task-before-checkpoint",
|
||||
float("inf"),
|
||||
)
|
||||
}
|
||||
chain._worker_state_lock = threading.RLock()
|
||||
chain.jobview = Mock()
|
||||
chain.queue_failed_transfer_notification = Mock()
|
||||
chain.runtime_config = SimpleNamespace(
|
||||
@@ -1449,6 +1563,7 @@ def test_pre_checkpoint_recognition_failure_records_retryable_error(monkeypatch)
|
||||
assert result == (False, "未识别到媒体信息")
|
||||
chain._transfer_admissions.record_planning_failure.assert_called_once_with(
|
||||
task_id="task-before-checkpoint",
|
||||
lease_token="lease-task-before-checkpoint",
|
||||
error="未识别到媒体信息",
|
||||
)
|
||||
|
||||
|
||||
@@ -57,6 +57,8 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
|
||||
task.bind_admission_task_id("task-stable")
|
||||
|
||||
values = task.to_dict()
|
||||
task.bind_execution_lease(owner_id="worker-owner", lease_token="lease-token")
|
||||
leased_values = task.to_dict()
|
||||
|
||||
assert set(values) == {
|
||||
"fileitem",
|
||||
@@ -84,9 +86,12 @@ def test_transfer_task_to_dict_keeps_exact_legacy_fields():
|
||||
}
|
||||
assert values["fileitem"] == task.fileitem.model_dump()
|
||||
assert values["target_path"] == Path("/library/Movie (2026)")
|
||||
assert leased_values == values
|
||||
assert "admission_task_id" not in values
|
||||
assert "planning_input" not in values
|
||||
assert "plan_checkpoint" not in values
|
||||
assert "lease_owner" not in values
|
||||
assert "lease_token" not in values
|
||||
|
||||
|
||||
def test_transfer_chain_do_transfer_keeps_legacy_signature():
|
||||
|
||||
@@ -24,12 +24,13 @@ except ModuleNotFoundError:
|
||||
|
||||
POSTGRESQL_DIALECT = "postgresql+psycopg"
|
||||
|
||||
MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
|
||||
PLANNING_MIGRATION = "database.versions.c2f8a4d6e1b3_3_0_14"
|
||||
LEASE_MIGRATION = "database.versions.d3a9e5f7b2c4_3_0_15"
|
||||
|
||||
|
||||
def _bind_migration(monkeypatch, connection):
|
||||
def _bind_migration(monkeypatch, connection, module_name=PLANNING_MIGRATION):
|
||||
"""把迁移绑定到隔离数据库连接。"""
|
||||
migration = importlib.import_module(MIGRATION)
|
||||
migration = importlib.import_module(module_name)
|
||||
monkeypatch.setattr(
|
||||
migration,
|
||||
"op",
|
||||
@@ -91,10 +92,13 @@ def _planning_row(connection) -> dict[str, object]:
|
||||
def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
"""断言规划迁移在当前隔离连接上的完整可逆生命周期。"""
|
||||
_create_admission_table(connection)
|
||||
migration = _bind_migration(monkeypatch, connection)
|
||||
planning_migration = _bind_migration(monkeypatch, connection)
|
||||
lease_migration = _bind_migration(monkeypatch, connection, LEASE_MIGRATION)
|
||||
|
||||
migration.upgrade()
|
||||
migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
|
||||
inspector = sa.inspect(connection)
|
||||
assert {
|
||||
@@ -133,7 +137,9 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
planned_at="2026-08-27 11:00:00",
|
||||
)
|
||||
)
|
||||
migration.downgrade()
|
||||
lease_migration.downgrade()
|
||||
lease_migration.downgrade()
|
||||
planning_migration.downgrade()
|
||||
|
||||
downgraded = sa.inspect(connection)
|
||||
assert {
|
||||
@@ -152,12 +158,17 @@ def _assert_upgrade_downgrade_reupgrade(connection, monkeypatch) -> None:
|
||||
for index in downgraded.get_indexes("transferpending")
|
||||
} == {"ix_transferpending_state_created", "ux_transferpending_storage_path"}
|
||||
|
||||
migration.upgrade()
|
||||
planning_migration.upgrade()
|
||||
lease_migration.upgrade()
|
||||
reupgraded = _planning_row(connection)
|
||||
assert reupgraded["task_id"] == "stable-task"
|
||||
assert reupgraded["state"] == "accepted"
|
||||
assert reupgraded["checkpoint_payload"] is None
|
||||
assert reupgraded["input_fingerprint"] == planning_input.fingerprint
|
||||
assert {
|
||||
column["name"]
|
||||
for column in sa.inspect(connection).get_columns("transferpending")
|
||||
} == {column.name for column in TransferPending.__table__.columns}
|
||||
|
||||
|
||||
def test_transfer_planning_upgrade_downgrade_reupgrade(monkeypatch) -> None:
|
||||
|
||||
@@ -11,6 +11,7 @@ from app.application.transfer import (
|
||||
TRANSFER_ADMISSION_PLANNED,
|
||||
TRANSFER_ADMISSION_PROVIDER_PENDING,
|
||||
TransferAdmissionConflictError,
|
||||
TransferAdmissionProjectionError,
|
||||
TransferPlanCheckpoint,
|
||||
TransferPlanItem,
|
||||
TransferPlanningInput,
|
||||
@@ -30,6 +31,31 @@ def repository(tmp_path):
|
||||
return TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
|
||||
|
||||
|
||||
def _claim(repository, task_id: str):
|
||||
"""为需要变更规划状态的测试取得独占租约。"""
|
||||
claimed = repository.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id="planning-test-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
assert claimed is not None
|
||||
assert claimed.lease_token
|
||||
return claimed
|
||||
|
||||
|
||||
def _pending_snapshot(repository, task_id: str) -> dict[str, object]:
|
||||
"""使用隔离 Session 冻结测试所需的持久状态字段。"""
|
||||
with repository._session_factory() as session: # noqa: SLF001
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(TransferPending.task_id == task_id)
|
||||
).scalar_one()
|
||||
return {
|
||||
"state": pending.state,
|
||||
"last_error": pending.last_error,
|
||||
"checkpoint_payload": pending.checkpoint_payload,
|
||||
}
|
||||
|
||||
|
||||
def _planning_input(*, target_path: str = "/library/Movies") -> TransferPlanningInput:
|
||||
"""构造包含恢复所需媒体上下文的完整规划输入。"""
|
||||
return TransferPlanningInput(
|
||||
@@ -264,6 +290,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
checkpoint = replace(
|
||||
_checkpoint(planning_input),
|
||||
resolved_meta={"name": "Resolved Movie", "year": 2026},
|
||||
@@ -275,6 +302,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -287,6 +315,7 @@ def test_resolved_context_does_not_change_admission_fingerprint(repository) -> N
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=replace(
|
||||
checkpoint,
|
||||
@@ -326,15 +355,18 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
checkpoint = _checkpoint(planning_input)
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
@@ -350,8 +382,9 @@ def test_checkpoint_atomically_advances_and_is_idempotent(repository) -> None:
|
||||
"plugin-provider-a",
|
||||
)
|
||||
assert repeated == planned
|
||||
assert repository.list_accepted() == []
|
||||
assert repository.list_recoverable() == [planned]
|
||||
assert _pending_snapshot(repository, admitted.task_id)["state"] == (
|
||||
TRANSFER_ADMISSION_PLANNED
|
||||
)
|
||||
|
||||
|
||||
def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository) -> None:
|
||||
@@ -362,23 +395,34 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
provider_checkpoint = _provider_checkpoint(planning_input)
|
||||
|
||||
provider_pending = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=provider_checkpoint,
|
||||
)
|
||||
|
||||
assert provider_pending.state == TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
assert provider_pending.checkpoint == provider_checkpoint
|
||||
assert repository.list_recoverable() == [provider_pending]
|
||||
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="host planning unavailable",
|
||||
)
|
||||
failed = repository.list_recoverable()[0]
|
||||
assert repository.release_claim(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="host planning unavailable",
|
||||
)
|
||||
failed = repository.claim_recoverable(
|
||||
owner_id="planning-recovery-worker",
|
||||
limit=1,
|
||||
lease_seconds=3600,
|
||||
)[0]
|
||||
assert failed.state == TRANSFER_ADMISSION_PROVIDER_PENDING
|
||||
assert failed.checkpoint == provider_checkpoint
|
||||
assert failed.last_error == "host planning unavailable"
|
||||
@@ -386,11 +430,13 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
host_checkpoint = _checkpoint(planning_input)
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=host_checkpoint,
|
||||
)
|
||||
repeated = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=host_checkpoint,
|
||||
)
|
||||
@@ -402,6 +448,7 @@ def test_provider_pending_checkpoint_atomically_upgrades_to_host_plan(repository
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=failed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=provider_checkpoint,
|
||||
)
|
||||
@@ -415,18 +462,19 @@ def test_checkpoint_rejects_fingerprint_without_partial_state(repository) -> Non
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
|
||||
with pytest.raises(TransferAdmissionConflictError):
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint="0" * 64,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
|
||||
recovered = repository.list_recoverable()
|
||||
assert len(recovered) == 1
|
||||
assert recovered[0].state == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert recovered[0].checkpoint is None
|
||||
recovered = _pending_snapshot(repository, admitted.task_id)
|
||||
assert recovered["state"] == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert recovered["checkpoint_payload"] is None
|
||||
|
||||
|
||||
def test_planning_failure_stays_accepted_until_success(repository) -> None:
|
||||
@@ -437,15 +485,21 @@ def test_planning_failure_stays_accepted_until_success(repository) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
|
||||
repository.record_planning_failure(task_id=admitted.task_id, error="rename failed")
|
||||
failed = repository.list_recoverable()[0]
|
||||
assert failed.state == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert failed.last_error == "rename failed"
|
||||
assert failed.checkpoint is None
|
||||
repository.record_planning_failure(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
error="rename failed",
|
||||
)
|
||||
failed = _pending_snapshot(repository, admitted.task_id)
|
||||
assert failed["state"] == TRANSFER_ADMISSION_ACCEPTED
|
||||
assert failed["last_error"] == "rename failed"
|
||||
assert failed["checkpoint_payload"] is None
|
||||
|
||||
planned = repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -459,6 +513,7 @@ def test_checkpoint_rejects_missing_task(repository) -> None:
|
||||
with pytest.raises(TransferPlanningStateError):
|
||||
repository.checkpoint_plan(
|
||||
task_id="missing",
|
||||
lease_token="missing-token",
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -470,17 +525,25 @@ def test_direct_orm_defaults_create_valid_legacy_projection(tmp_path) -> None:
|
||||
factory = sessionmaker(bind=engine)
|
||||
TransferPending.__table__.create(engine)
|
||||
with factory() as session:
|
||||
session.add(TransferPending(
|
||||
pending = TransferPending(
|
||||
storage="local",
|
||||
src_path="/downloads/legacy.mkv",
|
||||
state=TRANSFER_ADMISSION_ACCEPTED,
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
))
|
||||
)
|
||||
session.add(pending)
|
||||
session.commit()
|
||||
task_id = pending.task_id
|
||||
|
||||
admitted = TransactionalTransferAdmissionRepository(factory).list_accepted()[0]
|
||||
repository = TransactionalTransferAdmissionRepository(factory)
|
||||
admitted = repository.claim_task(
|
||||
task_id=task_id,
|
||||
owner_id="legacy-projection-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
|
||||
assert admitted is not None
|
||||
assert admitted.planning_input == TransferPlanningInput.legacy(
|
||||
storage="local",
|
||||
src_path="/downloads/legacy.mkv",
|
||||
@@ -508,8 +571,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
|
||||
row.input_version = 2
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferPlanningStateError, match="版本"):
|
||||
repository.list_accepted()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="corruption-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
|
||||
with factory() as session:
|
||||
row = session.execute(
|
||||
@@ -521,8 +588,12 @@ def test_projection_rejects_input_version_and_fingerprint_corruption(tmp_path) -
|
||||
row.planning_input = corrupted
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferAdmissionConflictError, match="指纹"):
|
||||
repository.list_accepted()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="指纹"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="corruption-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -538,8 +609,10 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
|
||||
src_path="/downloads/Movie.2026.mkv",
|
||||
planning_input=planning_input,
|
||||
)
|
||||
claimed = _claim(repository, admitted.task_id)
|
||||
repository.checkpoint_plan(
|
||||
task_id=admitted.task_id,
|
||||
lease_token=claimed.lease_token,
|
||||
input_fingerprint=planning_input.fingerprint,
|
||||
checkpoint=_checkpoint(planning_input),
|
||||
)
|
||||
@@ -548,8 +621,18 @@ def test_projection_rejects_checkpoint_version_corruption(tmp_path) -> None:
|
||||
select(TransferPending).where(TransferPending.task_id == admitted.task_id)
|
||||
).scalar_one()
|
||||
row.checkpoint_version = 2
|
||||
row.lease_expires_at = "2000-01-01 00:00:00.000000"
|
||||
session.commit()
|
||||
|
||||
with pytest.raises(TransferPlanningStateError, match="版本"):
|
||||
repository.list_recoverable()
|
||||
with pytest.raises(TransferAdmissionProjectionError, match="版本"):
|
||||
repository.claim_task(
|
||||
task_id=admitted.task_id,
|
||||
owner_id="direct-corruption-test-worker",
|
||||
lease_seconds=3600,
|
||||
)
|
||||
assert repository.claim_recoverable(
|
||||
owner_id="batch-corruption-test-worker",
|
||||
limit=1,
|
||||
lease_seconds=3600,
|
||||
) == []
|
||||
engine.dispose()
|
||||
|
||||
@@ -2,7 +2,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import create_engine, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from app.application.transfer import TransferAdmission, TransferQueueService
|
||||
@@ -118,7 +118,8 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
|
||||
"""真实仓储已提交后即使内存入队失败，任务也必须带原因留待恢复。"""
|
||||
engine = create_engine(f"sqlite:///{tmp_path / 'durable-admission.db'}")
|
||||
TransferPending.__table__.create(engine)
|
||||
repository = TransactionalTransferAdmissionRepository(sessionmaker(bind=engine))
|
||||
factory = sessionmaker(bind=engine)
|
||||
repository = TransactionalTransferAdmissionRepository(factory)
|
||||
task = make_task(1)
|
||||
service, _ = _service(
|
||||
admit_task=lambda item: repository.admit(
|
||||
@@ -135,10 +136,13 @@ def test_transfer_queue_service_commits_admission_before_failed_enqueue(tmp_path
|
||||
with pytest.raises(RuntimeError, match="queue closed"):
|
||||
service.put(task, Mock())
|
||||
|
||||
admissions = repository.list_accepted()
|
||||
assert len(admissions) == 1
|
||||
assert admissions[0].task_id == task.admission_task_id
|
||||
assert admissions[0].last_error == "queue closed"
|
||||
with factory() as session:
|
||||
pending = session.execute(
|
||||
select(TransferPending).where(
|
||||
TransferPending.task_id == task.admission_task_id
|
||||
)
|
||||
).scalar_one()
|
||||
assert pending.last_error == "queue closed"
|
||||
engine.dispose()
|
||||
|
||||
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from app.chain.transfer import JobManager, TransferChain
|
||||
from app.chain.transfer import TransferChain
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.runtime.config import settings
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.transfer import EpisodeFormat
|
||||
from app.schemas.types import MediaType
|
||||
from tests.test_transfer_job_manager import (
|
||||
make_transfer_chain as make_base_transfer_chain,
|
||||
)
|
||||
|
||||
|
||||
class FakeMeta(MetaBase):
|
||||
@@ -46,16 +49,8 @@ def make_transfer_chain() -> TransferChain:
|
||||
"""
|
||||
构造不启动后台线程的整理链实例。
|
||||
"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.jobview = JobManager()
|
||||
chain._media_exts = settings.RMT_MEDIAEXT
|
||||
chain._subtitle_exts = settings.RMT_SUBEXT
|
||||
chain._audio_exts = settings.RMT_AUDIOEXT
|
||||
chain._allowed_exts = (
|
||||
chain._media_exts + chain._audio_exts + chain._subtitle_exts
|
||||
)
|
||||
chain._success_target_files = {}
|
||||
chain._scrape_batches = {}
|
||||
chain = make_base_transfer_chain()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
|
||||
@@ -15,13 +15,17 @@ from app.chain.transfer import TransferChain
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.config import global_vars
|
||||
from app.schemas.file import FileItem
|
||||
from app.schemas.transfer import TransferInfo
|
||||
from app.startup.initializers import transfer as transfer_initializer
|
||||
|
||||
|
||||
def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
|
||||
"""构造只包含后台线程生命周期字段的 TransferChain 测试骨架。"""
|
||||
chain = object.__new__(TransferChain)
|
||||
chain.runtime_config = SimpleNamespace(transfer_threads=transfer_threads)
|
||||
chain.runtime_config = SimpleNamespace(
|
||||
transfer_threads=transfer_threads,
|
||||
transfer_task_timeout=0,
|
||||
)
|
||||
chain._queue = queue.Queue()
|
||||
chain._transfer_interval = 0.1
|
||||
chain._threads = []
|
||||
@@ -33,9 +37,59 @@ def _build_chain(*, transfer_threads: int = 0) -> TransferChain:
|
||||
chain._closing = False
|
||||
chain._replay_thread = None
|
||||
chain._replay_stop_event = threading.Event()
|
||||
chain._recovery_wakeup_event = threading.Event()
|
||||
chain._lease_heartbeat_thread = None
|
||||
chain._lease_heartbeat_stop_event = threading.Event()
|
||||
chain._worker_owner_id = "worker-owner"
|
||||
chain._owned_leases = {}
|
||||
chain._queued_lease_tokens = set()
|
||||
admissions = MagicMock()
|
||||
admissions.admit.side_effect = lambda **kwargs: TransferAdmission(
|
||||
task_id="admitted-task",
|
||||
storage=kwargs["storage"],
|
||||
src_path=kwargs["src_path"],
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
)
|
||||
admissions.claim_task.side_effect = lambda **kwargs: TransferAdmission(
|
||||
task_id=kwargs["task_id"],
|
||||
storage="local",
|
||||
src_path="/downloads/test.mkv",
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner=kwargs["owner_id"],
|
||||
lease_token=f"lease-{kwargs['task_id']}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
admissions.discard_claimed.return_value = 1
|
||||
admissions.release_claim.return_value = True
|
||||
chain._transfer_admissions = admissions
|
||||
chain._TransferChain__ensure_lease_heartbeat_owner = MagicMock()
|
||||
chain._TransferChain__ensure_recovery_scheduler = MagicMock()
|
||||
return chain
|
||||
|
||||
|
||||
def _claimed_admission(task: TransferTask, task_id: str) -> TransferAdmission:
|
||||
"""构造属于测试进程 owner 的有效 claim 投影。"""
|
||||
return TransferAdmission(
|
||||
task_id=task_id,
|
||||
storage=task.fileitem.storage,
|
||||
src_path=task.fileitem.path,
|
||||
state="accepted",
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
lease_owner="worker-owner",
|
||||
lease_token=f"lease-{task_id}",
|
||||
lease_expires_at="2026-08-27 10:02:00.000000",
|
||||
heartbeat_at="2026-08-27 10:00:00.000000",
|
||||
attempt_count=1,
|
||||
)
|
||||
|
||||
|
||||
def test_config_reload_replaces_worker_generation_and_keeps_accepting() -> None:
|
||||
"""热更新应等待旧 worker 收敛，再启动使用独立停止信号的新一代。"""
|
||||
chain = _build_chain(transfer_threads=1)
|
||||
@@ -433,7 +487,11 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
|
||||
created_at="2026-08-27 10:00:00",
|
||||
updated_at="2026-08-27 10:00:00",
|
||||
)
|
||||
admissions.discard_task.side_effect = (
|
||||
admissions.claim_task.return_value = _claimed_admission(
|
||||
task,
|
||||
"durable-task-id",
|
||||
)
|
||||
admissions.discard_claimed.side_effect = (
|
||||
lambda **_kwargs: discarded.set() or 1
|
||||
)
|
||||
chain._transfer_admissions = admissions
|
||||
@@ -471,7 +529,15 @@ def test_durable_task_identity_flows_from_queue_to_terminal_discard(monkeypatch)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
assert task.admission_task_id == "durable-task-id"
|
||||
admissions.discard_task.assert_called_once_with(task_id="durable-task-id")
|
||||
admissions.claim_task.assert_called_once_with(
|
||||
task_id="durable-task-id",
|
||||
owner_id="worker-owner",
|
||||
lease_seconds=120,
|
||||
)
|
||||
admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="durable-task-id",
|
||||
lease_token="lease-durable-task-id",
|
||||
)
|
||||
|
||||
|
||||
def test_claimed_task_prevents_progress_settlement_before_active_registration() -> None:
|
||||
@@ -527,6 +593,7 @@ def test_claimed_task_prevents_progress_settlement_before_active_registration()
|
||||
def test_replay_has_single_owner_and_close_waits_for_it() -> None:
|
||||
"""重复回放只保留一个线程，关闭会通知并等待该线程退出。"""
|
||||
chain = _build_chain()
|
||||
del chain._TransferChain__ensure_recovery_scheduler
|
||||
replay_started = threading.Event()
|
||||
replay_calls = []
|
||||
|
||||
@@ -548,3 +615,355 @@ def test_replay_has_single_owner_and_close_waits_for_it() -> None:
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert replay_thread.is_alive() is False
|
||||
assert chain._replay_thread is None
|
||||
|
||||
|
||||
def test_recovered_worker_reuses_claimed_token_without_second_claim(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""恢复任务携带 token 入队后，普通 worker 必须直接执行而非二次 claim。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/recovered.mkv",
|
||||
type="file",
|
||||
name="recovered.mkv",
|
||||
basename="recovered",
|
||||
extension="mkv",
|
||||
))
|
||||
task.bind_admission_task_id("recovered-task")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="lease-recovered-task",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"recovered-task": ("lease-recovered-task", time.monotonic() + 120)
|
||||
}
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 1
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_recovery(*, task, callback):
|
||||
"""模拟恢复任务成功提交检查点并让 worker 在本项后退出。"""
|
||||
del callback
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_recovery
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._transfer_admissions.claim_task.assert_not_called()
|
||||
chain._transfer_admissions.discard_claimed.assert_called_once_with(
|
||||
task_id="recovered-task",
|
||||
lease_token="lease-recovered-task",
|
||||
)
|
||||
|
||||
|
||||
def test_heartbeat_refreshes_current_token_and_forgets_lost_lease() -> None:
|
||||
"""heartbeat 成功应刷新本地期限，CAS 拒绝后必须立即停止本地推进资格。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/heartbeat.mkv",
|
||||
type="file",
|
||||
))
|
||||
current = _claimed_admission(task, "heartbeat-task")
|
||||
initial_deadline = time.monotonic() + 1
|
||||
chain._owned_leases = {
|
||||
"heartbeat-task": ("lease-heartbeat-task", initial_deadline)
|
||||
}
|
||||
chain._transfer_admissions.heartbeat.return_value = current
|
||||
|
||||
chain._TransferChain__heartbeat_owned_leases()
|
||||
|
||||
assert chain._owned_leases["heartbeat-task"][1] > initial_deadline
|
||||
chain._transfer_admissions.heartbeat.return_value = None
|
||||
|
||||
chain._TransferChain__heartbeat_owned_leases()
|
||||
|
||||
assert "heartbeat-task" not in chain._owned_leases
|
||||
|
||||
|
||||
def test_close_timeout_keeps_heartbeat_alive_until_blocked_worker_converges() -> None:
|
||||
"""阻塞 worker 未退出时关闭不得停止 heartbeat 或允许租约过期接管。"""
|
||||
chain = _build_chain()
|
||||
worker_release = threading.Event()
|
||||
worker = threading.Thread(
|
||||
target=worker_release.wait,
|
||||
name="transfer-blocked-owner",
|
||||
daemon=True,
|
||||
)
|
||||
heartbeat = threading.Thread(
|
||||
target=chain._lease_heartbeat_stop_event.wait,
|
||||
name="transfer-heartbeat-owner",
|
||||
daemon=True,
|
||||
)
|
||||
chain._threads = [worker]
|
||||
chain._lease_heartbeat_thread = heartbeat
|
||||
chain._owned_leases = {
|
||||
"blocked-task": ("blocked-token", time.monotonic() + 120)
|
||||
}
|
||||
worker.start()
|
||||
heartbeat.start()
|
||||
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert heartbeat.is_alive() is True
|
||||
assert chain._lease_heartbeat_stop_event.is_set() is False
|
||||
|
||||
worker_release.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert worker.is_alive() is False
|
||||
assert heartbeat.is_alive() is False
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="blocked-task",
|
||||
lease_token="blocked-token",
|
||||
error="整理宿主关闭，释放未结算任务租约",
|
||||
)
|
||||
|
||||
|
||||
def test_worker_reports_failed_settlement_without_skipping_queue_bookkeeping(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""终态 CAS=0 必须计为失败，同时仍完成 task_done 与 active 归零。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/stale.mkv",
|
||||
type="file",
|
||||
name="stale.mkv",
|
||||
basename="stale",
|
||||
extension="mkv",
|
||||
))
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.add_task.return_value = True
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._register_scrape_batch_task = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 0
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_with_stale_lease(*, task, callback):
|
||||
"""模拟文件副作用完成后终态 token 已被新 owner 接管。"""
|
||||
del callback
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return True, ""
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_with_stale_lease
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
assert chain.put_to_queue(task) is True
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
assert chain._active_tasks == 0
|
||||
assert chain._fail_num == 1
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
|
||||
|
||||
def test_failed_claim_release_waits_for_fixed_recovery_poll() -> None:
|
||||
"""失败释放不得即时唤醒恢复线程，避免确定性错误形成热重试。"""
|
||||
chain = _build_chain()
|
||||
del chain._TransferChain__ensure_recovery_scheduler
|
||||
chain._RECOVERY_POLL_INTERVAL_SECONDS = 0.05
|
||||
chain._TransferChain__replay_pending = MagicMock()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/retry-later.mkv",
|
||||
type="file",
|
||||
))
|
||||
task.bind_admission_task_id("retry-later")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="retry-token",
|
||||
)
|
||||
chain._owned_leases = {
|
||||
"retry-later": ("retry-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
assert chain._TransferChain__release_task_claim(
|
||||
task,
|
||||
error="planning failed",
|
||||
) is True
|
||||
|
||||
assert chain._recovery_wakeup_event.is_set() is False
|
||||
assert chain._replay_thread is not None
|
||||
time.sleep(0.01)
|
||||
chain._TransferChain__replay_pending.assert_not_called()
|
||||
deadline = time.monotonic() + 0.5
|
||||
while (
|
||||
not chain._TransferChain__replay_pending.called
|
||||
and time.monotonic() < deadline
|
||||
):
|
||||
time.sleep(0.01)
|
||||
chain._TransferChain__replay_pending.assert_called()
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="retry-later",
|
||||
lease_token="retry-token",
|
||||
error="planning failed",
|
||||
)
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
|
||||
|
||||
def test_worker_fenced_releases_lost_lease_and_completes_queue_bookkeeping(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
"""本地租约失效时仍尝试 token CAS release，并完整结算内存队列。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/lost-lease.mkv",
|
||||
type="file",
|
||||
))
|
||||
task.bind_admission_task_id("lost-lease")
|
||||
task.bind_execution_lease(
|
||||
owner_id="worker-owner",
|
||||
lease_token="lost-token",
|
||||
)
|
||||
chain.jobview = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
stop_event = threading.Event()
|
||||
chain._transfer_admissions.release_claim.side_effect = (
|
||||
lambda **_kwargs: stop_event.set() or True
|
||||
)
|
||||
chain._queue.put(TransferQueue(task=task))
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
chain._transfer_admissions.release_claim.assert_called_once_with(
|
||||
task_id="lost-lease",
|
||||
lease_token="lost-token",
|
||||
error="整理任务租约已经失效：lost-lease",
|
||||
)
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
assert chain._recovery_wakeup_event.is_set() is False
|
||||
|
||||
|
||||
def test_success_callback_runs_only_after_terminal_cas_succeeds(monkeypatch) -> None:
|
||||
"""终态 CAS 被拒绝时不得写成功历史、事件或通知。"""
|
||||
chain = _build_chain()
|
||||
task = TransferTask(fileitem=FileItem(
|
||||
storage="local",
|
||||
path="/downloads/fenced-success.mkv",
|
||||
type="file",
|
||||
name="fenced-success.mkv",
|
||||
))
|
||||
chain.jobview = MagicMock()
|
||||
chain.jobview.add_task.return_value = True
|
||||
chain.jobview.pending_total.return_value = 1
|
||||
chain._register_scrape_batch_task = MagicMock()
|
||||
chain._finish_scrape_batch_task = MagicMock()
|
||||
chain._progress = MagicMock()
|
||||
chain._active_tasks = 0
|
||||
chain._processed_num = 0
|
||||
chain._fail_num = 0
|
||||
chain._total_num = 0
|
||||
chain._transfer_admissions.discard_claimed.return_value = 0
|
||||
chain._TransferChain__settle_transfer_progress_if_idle = MagicMock()
|
||||
success_callback = MagicMock(return_value=(True, ""))
|
||||
chain._TransferChain__default_callback = success_callback
|
||||
stop_event = threading.Event()
|
||||
|
||||
def complete_with_success(*, task, callback):
|
||||
"""模拟文件成功后进入受 durable 终态保护的回调。"""
|
||||
task.bind_plan_checkpoint(MagicMock())
|
||||
stop_event.set()
|
||||
return callback(
|
||||
task,
|
||||
TransferInfo(success=True, fileitem=task.fileitem),
|
||||
)
|
||||
|
||||
chain._TransferChain__handle_transfer = complete_with_success
|
||||
monkeypatch.setattr(global_vars, "STOP_EVENT", threading.Event())
|
||||
assert chain.put_to_queue(task) is True
|
||||
|
||||
worker = threading.Thread(
|
||||
target=chain._TransferChain__start_transfer,
|
||||
args=(stop_event,),
|
||||
daemon=True,
|
||||
)
|
||||
worker.start()
|
||||
worker.join(timeout=1)
|
||||
|
||||
assert worker.is_alive() is False
|
||||
success_callback.assert_not_called()
|
||||
chain.jobview.fail_unfinished_task.assert_called_once_with(task)
|
||||
assert chain._fail_num == 1
|
||||
assert chain._queue.unfinished_tasks == 0
|
||||
|
||||
|
||||
def test_close_release_db_block_respects_deadline_and_keeps_heartbeat() -> None:
|
||||
"""关闭租约释放被数据库阻塞时应按预算返回，并继续 heartbeat。"""
|
||||
chain = _build_chain()
|
||||
release_started = threading.Event()
|
||||
release_db = threading.Event()
|
||||
heartbeat = threading.Thread(
|
||||
target=chain._lease_heartbeat_stop_event.wait,
|
||||
name="transfer-heartbeat-release-test",
|
||||
daemon=True,
|
||||
)
|
||||
chain._lease_heartbeat_thread = heartbeat
|
||||
chain._owned_leases = {
|
||||
"blocked-release": ("blocked-token", time.monotonic() + 120)
|
||||
}
|
||||
|
||||
def block_release(**_kwargs):
|
||||
"""模拟数据库锁住 release_claim，直到测试显式放行。"""
|
||||
release_started.set()
|
||||
release_db.wait()
|
||||
return True
|
||||
|
||||
chain._transfer_admissions.release_claim.side_effect = block_release
|
||||
heartbeat.start()
|
||||
|
||||
started_at = time.monotonic()
|
||||
assert chain.close_workers(timeout_seconds=0.01) is False
|
||||
assert time.monotonic() - started_at < 0.5
|
||||
assert release_started.is_set()
|
||||
assert heartbeat.is_alive() is True
|
||||
assert chain._lease_heartbeat_stop_event.is_set() is False
|
||||
assert chain._lease_release_thread is not None
|
||||
assert chain._lease_release_thread.is_alive() is True
|
||||
|
||||
release_db.set()
|
||||
assert chain.close_workers(timeout_seconds=1) is True
|
||||
assert heartbeat.is_alive() is False
|
||||
assert chain._lease_release_thread is None
|
||||
|
||||
Reference in New Issue
Block a user