mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-31 13:07:56 +08:00
806 lines
27 KiB
Python
806 lines
27 KiB
Python
"""整理恢复租约的原子 claim、续租和陈旧 token 防护测试。"""
|
|
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timedelta, timezone
|
|
from threading import Barrier, Lock
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine, select, text
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from app.application.transfer.workflow import (
|
|
TRANSFER_ADMISSION_PLANNED,
|
|
TransferAdmission,
|
|
TransferAdmissionProjectionError,
|
|
TransferLeaseLostError,
|
|
TransferPlanCheckpoint,
|
|
TransferPlanningInput,
|
|
TransferProviderInvocationSnapshot,
|
|
TransferProviderReference,
|
|
)
|
|
from app.db.adapters.transfer.admission import TransactionalTransferAdmissionRepository
|
|
from app.db.models.transferexecutionstep import TransferExecutionStep
|
|
from app.db.models.transferhistory import TransferHistory
|
|
from app.db.models.transferpending import TransferPending
|
|
from app.db.oper.transferpending import TransferPendingOper
|
|
|
|
|
|
def _planning_input(path: str) -> TransferPlanningInput:
|
|
"""构造与测试源路径绑定的最小规划输入。"""
|
|
return TransferPlanningInput(
|
|
source_fileitem={
|
|
"storage": "local",
|
|
"path": path,
|
|
"type": "file",
|
|
"name": path.rsplit("/", maxsplit=1)[-1],
|
|
},
|
|
meta={"name": "Movie"},
|
|
mediainfo={"title": "Movie"},
|
|
)
|
|
|
|
|
|
def _checkpoint(planning_input: TransferPlanningInput) -> TransferPlanCheckpoint:
|
|
"""构造无需文件副作用的合法宿主跳过检查点。"""
|
|
return TransferPlanCheckpoint(
|
|
planning_input=planning_input,
|
|
target_storage="local",
|
|
root_target_path="/library",
|
|
final_target_path="/library",
|
|
resolved_transfer_type="copy",
|
|
items=(),
|
|
skip_reason="测试跳过计划",
|
|
)
|
|
|
|
|
|
def _provider_checkpoint(
|
|
planning_input: TransferPlanningInput,
|
|
) -> TransferPlanCheckpoint:
|
|
"""构造只冻结 provider ABI、尚未完成宿主规划的检查点。"""
|
|
invocation = TransferProviderInvocationSnapshot(
|
|
fileitem=planning_input.source_fileitem,
|
|
meta=planning_input.meta,
|
|
meta_kind="MetaVideo",
|
|
mediainfo=planning_input.mediainfo,
|
|
mediainfo_kind="MediaInfo",
|
|
)
|
|
return TransferPlanCheckpoint(
|
|
planning_input=planning_input,
|
|
target_storage="",
|
|
root_target_path="",
|
|
final_target_path="",
|
|
resolved_transfer_type="",
|
|
items=(),
|
|
resolved_meta=invocation.meta,
|
|
resolved_meta_kind=invocation.meta_kind,
|
|
resolved_mediainfo=invocation.mediainfo,
|
|
resolved_mediainfo_kind=invocation.mediainfo_kind,
|
|
legacy_transfer_providers=(
|
|
TransferProviderReference(
|
|
plugin_id="provider-a",
|
|
plugin_name="Provider A",
|
|
),
|
|
),
|
|
provider_invocation=invocation,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def repository_factory(tmp_path):
|
|
"""创建允许多线程独立 Session 竞争的 SQLite 租约仓储工厂。"""
|
|
engine = create_engine(
|
|
f"sqlite:///{tmp_path / 'transfer-lease.db'}",
|
|
connect_args={"check_same_thread": False, "timeout": 10},
|
|
)
|
|
TransferHistory.__table__.create(engine)
|
|
TransferPending.__table__.create(engine)
|
|
TransferExecutionStep.__table__.create(engine)
|
|
factory = sessionmaker(bind=engine)
|
|
yield lambda: TransactionalTransferAdmissionRepository(factory)
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
def lease_clock(monkeypatch):
|
|
"""为所有仓储实例提供可推进的固定 UTC 时钟。"""
|
|
clock = {"now": datetime(2026, 8, 27, 10, 0, tzinfo=timezone.utc)}
|
|
monkeypatch.setattr(
|
|
TransactionalTransferAdmissionRepository,
|
|
"_lease_now",
|
|
staticmethod(lambda: clock["now"]),
|
|
)
|
|
return clock
|
|
|
|
|
|
@pytest.fixture
|
|
def business_clock(monkeypatch):
|
|
"""为业务审计字段提供与 UTC 租约时钟明确分离的宿主本地时间。"""
|
|
clock = {"now": "2026-08-27 18:00:00"}
|
|
monkeypatch.setattr(
|
|
TransactionalTransferAdmissionRepository,
|
|
"_now",
|
|
staticmethod(lambda: clock["now"]),
|
|
)
|
|
return clock
|
|
|
|
|
|
def _admit(
|
|
repository: TransactionalTransferAdmissionRepository,
|
|
path: str,
|
|
) -> TransferAdmission:
|
|
"""登记一个带完整版本化输入的测试任务。"""
|
|
return repository.admit(
|
|
storage="local",
|
|
src_path=path,
|
|
planning_input=_planning_input(path),
|
|
)
|
|
|
|
|
|
def _pending_snapshot(
|
|
repository: TransactionalTransferAdmissionRepository,
|
|
task_id: str,
|
|
) -> dict[str, Any]:
|
|
"""在 Session 关闭前冻结测试需要检查的持久登记字段。"""
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(TransferPending.task_id == task_id)
|
|
).scalar_one()
|
|
return {
|
|
"lease_owner": pending.lease_owner,
|
|
"lease_token": pending.lease_token,
|
|
"lease_expires_at": pending.lease_expires_at,
|
|
"heartbeat_at": pending.heartbeat_at,
|
|
"last_error": pending.last_error,
|
|
"attempt_count": pending.attempt_count,
|
|
"updated_at": pending.updated_at,
|
|
"planned_at": pending.planned_at,
|
|
}
|
|
|
|
|
|
def test_claim_heartbeat_expired_takeover_and_stale_token_guards(
|
|
repository_factory,
|
|
lease_clock,
|
|
business_clock,
|
|
) -> None:
|
|
"""新 token 才增加 attempt，过期 token 不能续租、释放或删除接管者。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/movie.mkv")
|
|
assert admitted.created_at == "2026-08-27 18:00:00"
|
|
business_clock["now"] = "2026-08-27 18:01:00"
|
|
|
|
first = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-a",
|
|
lease_seconds=60,
|
|
)
|
|
assert first is not None
|
|
assert first.lease_owner == "worker-a"
|
|
assert first.lease_token
|
|
assert first.attempt_count == 1
|
|
assert first.updated_at == "2026-08-27 18:01:00"
|
|
|
|
assert repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-a",
|
|
lease_seconds=60,
|
|
) is None
|
|
assert repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-b",
|
|
lease_seconds=60,
|
|
) is None
|
|
|
|
lease_clock["now"] += timedelta(seconds=30)
|
|
business_clock["now"] = "2026-08-27 18:02:00"
|
|
renewed = repository.heartbeat(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
lease_seconds=60,
|
|
)
|
|
assert renewed is not None
|
|
assert renewed.attempt_count == 1
|
|
assert renewed.heartbeat_at == "2026-08-27 10:00:30.000000"
|
|
assert renewed.updated_at == first.updated_at
|
|
|
|
lease_clock["now"] += timedelta(seconds=61)
|
|
business_clock["now"] = "2026-08-27 18:03:00"
|
|
assert repository.release_claim(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
error="expired worker",
|
|
) is False
|
|
assert repository.abandon_unstarted(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
) == 0
|
|
takeover = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-b",
|
|
lease_seconds=60,
|
|
)
|
|
assert takeover is not None
|
|
assert takeover.lease_token != first.lease_token
|
|
assert takeover.attempt_count == 2
|
|
assert takeover.updated_at == "2026-08-27 18:03:00"
|
|
assert repository.heartbeat(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
lease_seconds=60,
|
|
) is None
|
|
business_clock["now"] = "2026-08-27 18:04:00"
|
|
assert repository.release_claim(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
error="stale worker",
|
|
) is False
|
|
assert repository.abandon_unstarted(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
) == 0
|
|
|
|
assert repository.release_claim(
|
|
task_id=admitted.task_id,
|
|
lease_token=takeover.lease_token,
|
|
error="retry later",
|
|
) is True
|
|
released = _pending_snapshot(repository, admitted.task_id)
|
|
assert released["lease_owner"] is None
|
|
assert released["lease_token"] is None
|
|
assert released["lease_expires_at"] is None
|
|
assert released["heartbeat_at"] is None
|
|
assert released["last_error"] == "retry later"
|
|
assert released["attempt_count"] == 2
|
|
assert released["updated_at"] == "2026-08-27 18:04:00"
|
|
|
|
third = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-c",
|
|
lease_seconds=60,
|
|
)
|
|
assert third is not None
|
|
assert third.attempt_count == 3
|
|
assert repository.abandon_unstarted(
|
|
task_id=admitted.task_id,
|
|
lease_token=third.lease_token,
|
|
) == 1
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
assert session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == admitted.task_id
|
|
)
|
|
).scalar_one_or_none() is None
|
|
|
|
|
|
def test_claim_recoverable_respects_order_limit_and_active_lease(
|
|
repository_factory,
|
|
lease_clock,
|
|
) -> None:
|
|
"""批量恢复跳过有效租约，并按登记顺序逐条 CAS 到请求上限。"""
|
|
repository = repository_factory()
|
|
first = _admit(repository, "/downloads/a.mkv")
|
|
second = _admit(repository, "/downloads/b.mkv")
|
|
third = _admit(repository, "/downloads/c.mkv")
|
|
active = repository.claim_task(
|
|
task_id=first.task_id,
|
|
owner_id="active-worker",
|
|
lease_seconds=60,
|
|
)
|
|
assert active is not None
|
|
|
|
claimed = repository.claim_recoverable(
|
|
owner_id="recovery-worker",
|
|
limit=2,
|
|
lease_seconds=60,
|
|
)
|
|
|
|
assert [item.task_id for item in claimed] == [second.task_id, third.task_id]
|
|
assert all(item.lease_owner == "recovery-worker" for item in claimed)
|
|
assert all(item.attempt_count == 1 for item in claimed)
|
|
assert repository.claim_recoverable(
|
|
owner_id="other-worker",
|
|
limit=10,
|
|
lease_seconds=60,
|
|
) == []
|
|
|
|
lease_clock["now"] += timedelta(seconds=61)
|
|
reclaimed = repository.claim_recoverable(
|
|
owner_id="takeover-worker",
|
|
limit=2,
|
|
lease_seconds=60,
|
|
)
|
|
assert [item.task_id for item in reclaimed] == [first.task_id, second.task_id]
|
|
assert reclaimed[0].attempt_count == 2
|
|
assert reclaimed[1].attempt_count == 2
|
|
|
|
|
|
def test_claim_recoverable_skips_corrupt_projection_and_claims_later_tasks(
|
|
repository_factory,
|
|
lease_clock,
|
|
business_clock,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""毒行应留下单次诊断但不持有租约或饿死后续健康任务。"""
|
|
repository = repository_factory()
|
|
messages: list[str] = []
|
|
monkeypatch.setattr(
|
|
"app.db.adapters.transfer.admission._diagnostic_logger.error",
|
|
messages.append,
|
|
)
|
|
corrupt = _admit(repository, "/downloads/a-corrupt.mkv")
|
|
healthy = [
|
|
_admit(repository, "/downloads/b-healthy.mkv"),
|
|
_admit(repository, "/downloads/c-healthy.mkv"),
|
|
]
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == corrupt.task_id
|
|
)
|
|
).scalar_one()
|
|
pending.input_fingerprint = "corrupt"
|
|
session.commit()
|
|
|
|
claimed = repository.claim_recoverable(
|
|
owner_id="recovery-worker",
|
|
limit=2,
|
|
lease_seconds=60,
|
|
)
|
|
|
|
assert [item.task_id for item in claimed] == [item.task_id for item in healthy]
|
|
corrupt_snapshot = _pending_snapshot(repository, corrupt.task_id)
|
|
assert corrupt_snapshot["lease_token"] is None
|
|
assert corrupt_snapshot["attempt_count"] == 0
|
|
assert corrupt_snapshot["last_error"].startswith("恢复投影失败:")
|
|
assert corrupt_snapshot["updated_at"] == "2026-08-27 18:00:00"
|
|
assert len(messages) == 1
|
|
|
|
business_clock["now"] = "2026-08-27 18:01:00"
|
|
assert repository.claim_recoverable(
|
|
owner_id="second-recovery-worker",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
) == []
|
|
assert _pending_snapshot(repository, corrupt.task_id) == corrupt_snapshot
|
|
assert len(messages) == 1
|
|
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == corrupt.task_id
|
|
)
|
|
).scalar_one()
|
|
pending.input_fingerprint = _planning_input(
|
|
"/downloads/a-corrupt.mkv"
|
|
).fingerprint
|
|
session.commit()
|
|
|
|
repaired = repository.claim_recoverable(
|
|
owner_id="repaired-worker",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
)
|
|
assert [item.task_id for item in repaired] == [corrupt.task_id]
|
|
|
|
|
|
def test_projection_diagnostic_changes_are_recorded_once_each(
|
|
repository_factory,
|
|
business_clock,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""相同投影错误不重复写库，错误类型变化时才更新诊断并再次告警。"""
|
|
repository = repository_factory()
|
|
messages: list[str] = []
|
|
monkeypatch.setattr(
|
|
"app.db.adapters.transfer.admission._diagnostic_logger.error",
|
|
messages.append,
|
|
)
|
|
admitted = _admit(repository, "/downloads/changing-corrupt.mkv")
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == admitted.task_id
|
|
)
|
|
).scalar_one()
|
|
pending.input_fingerprint = "corrupt"
|
|
session.commit()
|
|
|
|
assert repository.claim_recoverable(
|
|
owner_id="recovery-a",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
) == []
|
|
first = _pending_snapshot(repository, admitted.task_id)
|
|
assert len(messages) == 1
|
|
|
|
business_clock["now"] = "2026-08-27 18:01:00"
|
|
assert repository.claim_recoverable(
|
|
owner_id="recovery-b",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
) == []
|
|
assert _pending_snapshot(repository, admitted.task_id) == first
|
|
assert len(messages) == 1
|
|
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == admitted.task_id
|
|
)
|
|
).scalar_one()
|
|
pending.input_fingerprint = _planning_input(
|
|
"/downloads/changing-corrupt.mkv"
|
|
).fingerprint
|
|
pending.input_version = 999
|
|
session.commit()
|
|
|
|
business_clock["now"] = "2026-08-27 18:02:00"
|
|
assert repository.claim_recoverable(
|
|
owner_id="recovery-c",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
) == []
|
|
changed = _pending_snapshot(repository, admitted.task_id)
|
|
assert changed["last_error"] != first["last_error"]
|
|
assert changed["updated_at"] == "2026-08-27 18:02:00"
|
|
assert len(messages) == 2
|
|
|
|
|
|
def test_projection_diagnostic_cas_is_concurrency_safe(
|
|
repository_factory,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""并发恢复观察到同一损坏时只允许一个诊断写入者和一条运行日志。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/concurrent-corrupt.mkv")
|
|
messages: list[str] = []
|
|
message_lock = Lock()
|
|
|
|
def capture(message: str) -> None:
|
|
"""并发安全收集错误日志。"""
|
|
with message_lock:
|
|
messages.append(message)
|
|
|
|
monkeypatch.setattr(
|
|
"app.db.adapters.transfer.admission._diagnostic_logger.error",
|
|
capture,
|
|
)
|
|
barrier = Barrier(2)
|
|
projection_error = TransferAdmissionProjectionError("same corruption")
|
|
|
|
def record(_: int) -> bool:
|
|
"""让两个独立 Session 同时竞争同一诊断 CAS。"""
|
|
barrier.wait(timeout=5)
|
|
return repository_factory()._record_projection_failure( # noqa: SLF001
|
|
task_id=admitted.task_id,
|
|
error=projection_error,
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(executor.map(record, range(2)))
|
|
|
|
assert sorted(results) == [False, True]
|
|
assert len(messages) == 1
|
|
snapshot = _pending_snapshot(repository, admitted.task_id)
|
|
assert snapshot["last_error"] == "恢复投影失败: same corruption"
|
|
|
|
|
|
def test_projection_diagnostic_does_not_overwrite_active_lease(
|
|
repository_factory,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""诊断 CAS 不得覆盖已经由健康 worker 取得有效租约的任务。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/active-lease.mkv")
|
|
claimed = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="active-worker",
|
|
lease_seconds=60,
|
|
)
|
|
assert claimed is not None
|
|
before = _pending_snapshot(repository, admitted.task_id)
|
|
messages: list[str] = []
|
|
monkeypatch.setattr(
|
|
"app.db.adapters.transfer.admission._diagnostic_logger.error",
|
|
messages.append,
|
|
)
|
|
|
|
recorded = repository._record_projection_failure( # noqa: SLF001
|
|
task_id=admitted.task_id,
|
|
error=TransferAdmissionProjectionError("stale observation"),
|
|
)
|
|
|
|
assert recorded is False
|
|
assert _pending_snapshot(repository, admitted.task_id) == before
|
|
assert messages == []
|
|
|
|
|
|
def test_projection_diagnostic_database_failure_propagates(
|
|
repository_factory,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""诊断留痕的数据库基础设施异常必须向上游传播而非静默跳过。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/db-error-corrupt.mkv")
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
pending = session.execute(
|
|
select(TransferPending).where(
|
|
TransferPending.task_id == admitted.task_id
|
|
)
|
|
).scalar_one()
|
|
pending.input_fingerprint = "corrupt"
|
|
session.commit()
|
|
|
|
def fail_diagnostic(*_args, **_kwargs):
|
|
"""模拟诊断短事务的底层数据库写入失败。"""
|
|
raise RuntimeError("database unavailable")
|
|
|
|
monkeypatch.setattr(
|
|
TransferPendingOper,
|
|
"stage_record_projection_failure",
|
|
fail_diagnostic,
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="database unavailable"):
|
|
repository.claim_recoverable(
|
|
owner_id="recovery-worker",
|
|
limit=1,
|
|
lease_seconds=60,
|
|
)
|
|
|
|
|
|
def test_claim_task_wraps_persisted_json_decode_failure(
|
|
repository_factory,
|
|
) -> None:
|
|
"""持久 JSON 解码错误应归类为投影损坏，而不是数据库基础设施故障。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/invalid-json.mkv")
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
session.execute(
|
|
text(
|
|
"UPDATE transferpending SET planning_input = 'not-json' "
|
|
"WHERE task_id = :task_id"
|
|
),
|
|
{"task_id": admitted.task_id},
|
|
)
|
|
session.commit()
|
|
|
|
with pytest.raises(TransferAdmissionProjectionError, match="JSON"):
|
|
repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="recovery-worker",
|
|
lease_seconds=60,
|
|
)
|
|
with repository._session_factory() as session: # noqa: SLF001
|
|
snapshot = session.execute(
|
|
text(
|
|
"SELECT lease_token, attempt_count FROM transferpending "
|
|
"WHERE task_id = :task_id"
|
|
),
|
|
{"task_id": admitted.task_id},
|
|
).mappings().one()
|
|
assert snapshot["lease_token"] is None
|
|
assert snapshot["attempt_count"] == 0
|
|
|
|
|
|
def test_concurrent_recovery_callers_scan_past_lost_candidates(
|
|
repository_factory,
|
|
lease_clock,
|
|
monkeypatch,
|
|
) -> None:
|
|
"""并发 caller 竞争同一首批后应继续向后扫描并各自填满限额。"""
|
|
setup_repository = repository_factory()
|
|
admitted = [
|
|
_admit(setup_repository, f"/downloads/concurrent-{index}.mkv")
|
|
for index in range(4)
|
|
]
|
|
barrier = Barrier(2)
|
|
barrier_lock = Lock()
|
|
initial_scans = 0
|
|
original = TransferPending.list_claimable_candidates.__func__
|
|
|
|
def synchronized_candidates(cls, db, **kwargs):
|
|
"""强制两个 caller 在取得相同首批候选后再进入逐任务 CAS。"""
|
|
nonlocal initial_scans
|
|
candidates = original(cls, db, **kwargs)
|
|
if kwargs.get("after_cursor") is None:
|
|
with barrier_lock:
|
|
initial_scans += 1
|
|
barrier.wait(timeout=5)
|
|
return candidates
|
|
|
|
monkeypatch.setattr(
|
|
TransferPending,
|
|
"list_claimable_candidates",
|
|
classmethod(synchronized_candidates),
|
|
)
|
|
|
|
def recover(owner_id: str) -> list[TransferAdmission]:
|
|
"""使用独立仓储与 Session 执行一次有界恢复扫描。"""
|
|
return repository_factory().claim_recoverable(
|
|
owner_id=owner_id,
|
|
limit=2,
|
|
lease_seconds=60,
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(executor.map(recover, ("worker-a", "worker-b")))
|
|
|
|
assert initial_scans == 2
|
|
assert [len(result) for result in results] == [2, 2]
|
|
claimed_task_ids = [item.task_id for result in results for item in result]
|
|
assert len(set(claimed_task_ids)) == 4
|
|
assert set(claimed_task_ids) == {item.task_id for item in admitted}
|
|
|
|
|
|
def test_unclaimed_enqueue_failure_cannot_overwrite_claimed_task(
|
|
repository_factory,
|
|
business_clock,
|
|
) -> None:
|
|
"""task-id-only 入队失败入口不得改写已经由 worker claim 的登记。"""
|
|
repository = repository_factory()
|
|
admitted = _admit(repository, "/downloads/claimed.mkv")
|
|
business_clock["now"] = "2026-08-27 18:01:00"
|
|
claimed = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-a",
|
|
lease_seconds=60,
|
|
)
|
|
assert claimed is not None
|
|
before = _pending_snapshot(repository, admitted.task_id)
|
|
|
|
business_clock["now"] = "2026-08-27 18:02:00"
|
|
repository.record_enqueue_failure(
|
|
task_id=admitted.task_id,
|
|
error="stale queue failure",
|
|
)
|
|
|
|
after = _pending_snapshot(repository, admitted.task_id)
|
|
assert after == before
|
|
|
|
|
|
def test_planning_writes_require_current_unexpired_lease(
|
|
repository_factory,
|
|
lease_clock,
|
|
) -> None:
|
|
"""checkpoint 和规划错误均不得由已过期或已被接管的 worker 写入。"""
|
|
repository = repository_factory()
|
|
path = "/downloads/planning.mkv"
|
|
planning_input = _planning_input(path)
|
|
admitted = repository.admit(
|
|
storage="local",
|
|
src_path=path,
|
|
planning_input=planning_input,
|
|
)
|
|
first = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-a",
|
|
lease_seconds=30,
|
|
)
|
|
assert first is not None
|
|
|
|
lease_clock["now"] += timedelta(seconds=31)
|
|
takeover = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-b",
|
|
lease_seconds=60,
|
|
)
|
|
assert takeover is not None
|
|
|
|
with pytest.raises(TransferLeaseLostError, match="租约"):
|
|
repository.record_planning_failure(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
error="stale planning failure",
|
|
)
|
|
with pytest.raises(TransferLeaseLostError, match="租约"):
|
|
repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=first.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=_checkpoint(planning_input),
|
|
)
|
|
|
|
repository.record_planning_failure(
|
|
task_id=admitted.task_id,
|
|
lease_token=takeover.lease_token,
|
|
error="retryable planning failure",
|
|
)
|
|
planned = repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=takeover.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=_checkpoint(planning_input),
|
|
)
|
|
repeated = repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=takeover.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=_checkpoint(planning_input),
|
|
)
|
|
|
|
assert planned.state == TRANSFER_ADMISSION_PLANNED
|
|
assert planned.last_error is None
|
|
assert repeated == planned
|
|
assert planned.attempt_count == 2
|
|
|
|
|
|
def test_provider_checkpoint_sets_planned_time_only_after_host_plan(
|
|
repository_factory,
|
|
lease_clock,
|
|
business_clock,
|
|
) -> None:
|
|
"""provider 快照不是规划完成，planned_at 只记录首次宿主完整计划。"""
|
|
repository = repository_factory()
|
|
path = "/downloads/provider-plan.mkv"
|
|
planning_input = _planning_input(path)
|
|
admitted = repository.admit(
|
|
storage="local",
|
|
src_path=path,
|
|
planning_input=planning_input,
|
|
)
|
|
business_clock["now"] = "2026-08-27 18:01:00"
|
|
claimed = repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id="worker-a",
|
|
lease_seconds=60,
|
|
)
|
|
assert claimed is not None
|
|
|
|
business_clock["now"] = "2026-08-27 18:02:00"
|
|
repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=claimed.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=_provider_checkpoint(planning_input),
|
|
)
|
|
provider_snapshot = _pending_snapshot(repository, admitted.task_id)
|
|
assert provider_snapshot["planned_at"] is None
|
|
assert provider_snapshot["updated_at"] == "2026-08-27 18:02:00"
|
|
|
|
business_clock["now"] = "2026-08-27 18:03:00"
|
|
checkpoint = _checkpoint(planning_input)
|
|
repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=claimed.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=checkpoint,
|
|
)
|
|
planned_snapshot = _pending_snapshot(repository, admitted.task_id)
|
|
assert planned_snapshot["planned_at"] == "2026-08-27 18:03:00"
|
|
|
|
business_clock["now"] = "2026-08-27 18:04:00"
|
|
repository.checkpoint_plan(
|
|
task_id=admitted.task_id,
|
|
lease_token=claimed.lease_token,
|
|
input_fingerprint=planning_input.fingerprint,
|
|
checkpoint=checkpoint,
|
|
)
|
|
assert _pending_snapshot(repository, admitted.task_id) == planned_snapshot
|
|
|
|
|
|
def test_concurrent_claim_uses_rowcount_as_single_winner(
|
|
repository_factory,
|
|
lease_clock,
|
|
) -> None:
|
|
"""并发 worker 即使读取同一任务，也只能有一个 CAS 更新获胜。"""
|
|
setup_repository = repository_factory()
|
|
admitted = _admit(setup_repository, "/downloads/concurrent.mkv")
|
|
barrier = Barrier(2)
|
|
|
|
def claim(owner_id: str):
|
|
"""等待竞争者就绪后使用独立 Session claim 同一任务。"""
|
|
repository = repository_factory()
|
|
barrier.wait()
|
|
return repository.claim_task(
|
|
task_id=admitted.task_id,
|
|
owner_id=owner_id,
|
|
lease_seconds=60,
|
|
)
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as executor:
|
|
results = list(executor.map(claim, ("worker-a", "worker-b")))
|
|
|
|
winners = [result for result in results if result is not None]
|
|
assert len(winners) == 1
|
|
assert winners[0].attempt_count == 1
|