refactor: make download and transfer events durable

This commit is contained in:
jxxghp
2026-08-22 08:03:00 +08:00
parent c5de1c7b1b
commit 820f7e8850
26 changed files with 1134 additions and 184 deletions
+33 -3
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6271,
"edge_sha256": "0119519add8e4499684044fc69f3003c54cfca20bf04f4614a419374692d5a94",
"edge_count": 6299,
"edge_sha256": "f98ab7d05c884a07a881eff1ab34b1ba5023751f1e95b956debcbad54c385810",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -2438,6 +2438,19 @@
"app.application.chain.context -> app.application",
"app.application.chain.context -> app.application.chain",
"app.application.chain.context -> app.application.chain.data",
"app.application.chain.context -> app.application.chain.durable_events",
"app.application.chain.durable_events -> app.application",
"app.application.chain.durable_events -> app.application.history",
"app.application.chain.durable_events -> app.domain",
"app.application.chain.durable_events -> app.domain.context",
"app.application.chain.durable_events -> app.domain.meta",
"app.application.chain.durable_events -> app.domain.meta.metabase",
"app.application.chain.durable_events -> app.domain.meta.metamusic",
"app.application.chain.durable_events -> app.domain.metainfo",
"app.application.chain.durable_events -> app.schemas",
"app.application.chain.durable_events -> app.schemas.file",
"app.application.chain.durable_events -> app.schemas.transfer",
"app.application.chain.durable_events -> app.schemas.types",
"app.application.dashboard -> app.schemas",
"app.application.dashboard -> app.schemas.dashboard",
"app.application.database -> app.application",
@@ -5623,6 +5636,7 @@
"app.schemas.event -> app.schemas.common",
"app.schemas.event -> app.schemas.file",
"app.schemas.event -> app.schemas.media",
"app.schemas.event -> app.schemas.transfer",
"app.schemas.event -> app.schemas.types",
"app.schemas.file -> app.schemas",
"app.schemas.file -> app.schemas.types",
@@ -5841,6 +5855,18 @@
"app.startup.cache_initializer -> app.adapters",
"app.startup.cache_initializer -> app.adapters.cache",
"app.startup.cache_initializer -> app.adapters.cache.backends",
"app.startup.chain_events -> app.application",
"app.startup.chain_events -> app.application.chain",
"app.startup.chain_events -> app.application.chain.durable_events",
"app.startup.chain_events -> app.application.history",
"app.startup.chain_events -> app.application.outbox",
"app.startup.chain_events -> app.db",
"app.startup.chain_events -> app.db.oper",
"app.startup.chain_events -> app.db.oper.downloadhistory",
"app.startup.chain_events -> app.db.oper.transferhistory",
"app.startup.chain_events -> app.db.uow",
"app.startup.chain_events -> app.startup",
"app.startup.chain_events -> app.startup.outbox",
"app.startup.command_initializer -> app.application",
"app.startup.command_initializer -> app.application.commands",
"app.startup.command_initializer -> app.command",
@@ -5950,6 +5976,7 @@
"app.startup.modules_initializer -> app.application.chain",
"app.startup.modules_initializer -> app.application.chain.context",
"app.startup.modules_initializer -> app.application.chain.data",
"app.startup.modules_initializer -> app.application.chain.durable_events",
"app.startup.modules_initializer -> app.application.configuration",
"app.startup.modules_initializer -> app.application.database",
"app.startup.modules_initializer -> app.application.history",
@@ -6026,6 +6053,7 @@
"app.startup.modules_initializer -> app.schemas.types",
"app.startup.modules_initializer -> app.startup",
"app.startup.modules_initializer -> app.startup.agent_initializer",
"app.startup.modules_initializer -> app.startup.chain_events",
"app.startup.modules_initializer -> app.startup.context",
"app.startup.modules_initializer -> app.startup.database",
"app.startup.modules_initializer -> app.startup.managed_resources_initializer",
@@ -6288,7 +6316,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 782,
"module_count": 784,
"modules": [
"app",
"app.adapters",
@@ -6542,6 +6570,7 @@
"app.application.chain",
"app.application.chain.context",
"app.application.chain.data",
"app.application.chain.durable_events",
"app.application.commands",
"app.application.configuration",
"app.application.dashboard",
@@ -7034,6 +7063,7 @@
"app.startup",
"app.startup.agent_initializer",
"app.startup.cache_initializer",
"app.startup.chain_events",
"app.startup.command_initializer",
"app.startup.context",
"app.startup.database",
+19 -7
View File
@@ -1441,10 +1441,10 @@
"EventType.DownloadAdded": {
"delivery": "durable_required",
"error_behavior": "notify",
"legacy_reason": "现有插件 payload 尚未收敛为稳定 model,保留原始 dict ABI",
"legacy_reason": null,
"mode": "broadcast",
"ordering": "priority_queue",
"payload_contract": "legacy_dict",
"payload_contract": "DownloadAddedEventData",
"sensitive_fields": [],
"visibility": "plugin_public"
},
@@ -1651,20 +1651,20 @@
"EventType.TransferComplete": {
"delivery": "durable_required",
"error_behavior": "notify",
"legacy_reason": "现有插件 payload 尚未收敛为稳定 model,保留原始 dict ABI",
"legacy_reason": null,
"mode": "broadcast",
"ordering": "priority_queue",
"payload_contract": "legacy_dict",
"payload_contract": "TransferResultEventData",
"sensitive_fields": [],
"visibility": "plugin_public"
},
"EventType.TransferFailed": {
"delivery": "durable_required",
"error_behavior": "notify",
"legacy_reason": "现有插件 payload 尚未收敛为稳定 model,保留原始 dict ABI",
"legacy_reason": null,
"mode": "broadcast",
"ordering": "priority_queue",
"payload_contract": "legacy_dict",
"payload_contract": "TransferResultEventData",
"sensitive_fields": [],
"visibility": "plugin_public"
},
@@ -2032,6 +2032,10 @@
"producers": [
{
"caller": "app.chain.download",
"count": 2
},
{
"caller": "app.startup.modules_initializer",
"count": 1
}
]
@@ -2274,6 +2278,10 @@
"producers": [
{
"caller": "app.chain.transfer",
"count": 2
},
{
"caller": "app.startup.modules_initializer",
"count": 1
}
]
@@ -2283,6 +2291,10 @@
"producers": [
{
"caller": "app.chain.transfer",
"count": 2
},
{
"caller": "app.startup.modules_initializer",
"count": 1
}
]
@@ -2320,7 +2332,7 @@
]
}
},
"producer_count": 70
"producer_count": 76
},
"module_method_specs": {
"download_file": {
+3 -13
View File
@@ -2,11 +2,11 @@
"model_decorators": {
"by_kind": {
"async_db_query": 49,
"async_db_update": 13,
"async_db_update": 12,
"db_query": 75,
"db_update": 41
"db_update": 40
},
"count": 178,
"count": 176,
"methods": [
{
"decorator": "async_db_query",
@@ -463,11 +463,6 @@
"file": "app/db/models/siteuserdata.py",
"method": "SiteUserData.get_latest"
},
{
"decorator": "async_db_update",
"file": "app/db/models/subscribe.py",
"method": "Subscribe.async_delete_by_media_identity"
},
{
"decorator": "async_db_query",
"file": "app/db/models/subscribe.py",
@@ -513,11 +508,6 @@
"file": "app/db/models/subscribe.py",
"method": "Subscribe.async_list_by_username"
},
{
"decorator": "db_update",
"file": "app/db/models/subscribe.py",
"method": "Subscribe.delete_by_media_identity"
},
{
"decorator": "db_query",
"file": "app/db/models/subscribe.py",
+2 -2
View File
@@ -125,8 +125,8 @@ def test_transaction_debt_baseline_is_a_model_and_oper_ratchet() -> None:
baseline = json.loads(baseline_path.read_text(encoding="utf-8"))
assert baseline["schema_version"] == 1
assert baseline["model_decorators"]["count"] == 178
assert sum(baseline["model_decorators"]["by_kind"].values()) == 178
assert baseline["model_decorators"]["count"] == 176
assert sum(baseline["model_decorators"]["by_kind"].values()) == 176
assert baseline["model_transaction_calls"] == {"count": 0, "calls": []}
assert baseline["model_session_factories"] == {"count": 0, "calls": []}
assert baseline["oper_transaction_calls"] == {"count": 0, "calls": []}
+219
View File
@@ -0,0 +1,219 @@
"""下载与整理 durable 事件的原子写入和对象恢复测试。"""
import json
import pytest
from sqlalchemy import create_engine, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import sessionmaker
from app.application.chain.durable_events import (
restore_download_added,
restore_transfer_result,
snapshot_download_added,
snapshot_transfer_result,
)
from app.db.base import Base
from app.db.models.downloadhistory import DownloadFiles, DownloadHistory
from app.db.models.outbox import OutboxMessage
from app.db.models.transferhistory import TransferHistory
from app.domain.context import Context, MediaInfo, TorrentInfo
from app.domain.metainfo import MetaInfo
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
from app.schemas.types import MediaSource, MediaType
from app.startup.chain_events import TransactionalChainDurableEventWriter
def _session_factory():
"""创建只服务当前测试的内存数据库和同步 Session 工厂。"""
engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(engine)
return sessionmaker(bind=engine)
def _objects():
"""构造下载和整理事件共用的最小真实领域对象。"""
meta = MetaInfo("Demo.2026.1080p.mkv")
media = MediaInfo(
type=MediaType.MOVIE,
title="Demo",
year="2026",
media_source=MediaSource.TMDB,
media_id="42",
)
torrent = TorrentInfo(title="Demo torrent", site_name="test")
context = Context(meta_info=meta, media_info=media, torrent_info=torrent)
fileitem = FileItem(
storage="local",
path="/downloads/Demo.mkv",
name="Demo.mkv",
type="file",
)
transferinfo = TransferInfo(
success=True,
fileitem=fileitem,
target_item=FileItem(
storage="local",
path="/library/Demo (2026)/Demo.mkv",
name="Demo.mkv",
type="file",
),
transfer_type="copy",
)
return meta, media, context, fileitem, transferinfo
def test_durable_snapshots_are_json_and_restore_plugin_runtime_objects():
"""outbox 只存 JSON 快照,重放时恢复插件一直收到的对象类型。"""
meta, media, context, fileitem, transferinfo = _objects()
download_payload = {
"hash": "hash-1",
"context": context,
"username": "alice",
"downloader": "qb",
"episodes": [1, 2],
"source": "manual",
"idempotency_key": "download.added:qb:hash-1:v1",
}
transfer_payload = {
"fileitem": fileitem,
"meta": meta,
"mediainfo": media,
"transferinfo": transferinfo,
"downloader": "qb",
"download_hash": "hash-1",
"transfer_history_id": 9,
"idempotency_key": "transfer.completed:9:v1",
}
download_snapshot = snapshot_download_added(download_payload)
transfer_snapshot = snapshot_transfer_result(transfer_payload)
json.dumps(download_snapshot)
json.dumps(transfer_snapshot)
restored_download = restore_download_added(download_snapshot)
restored_transfer = restore_transfer_result(transfer_snapshot)
assert isinstance(restored_download["context"], Context)
assert isinstance(restored_download["context"].media_info, MediaInfo)
assert isinstance(restored_transfer["fileitem"], FileItem)
assert type(restored_transfer["meta"]) is type(meta)
assert isinstance(restored_transfer["mediainfo"], MediaInfo)
assert isinstance(restored_transfer["transferinfo"], TransferInfo)
def test_download_history_and_event_intent_share_one_transaction():
"""下载历史、文件清单和 intent 提交后才执行通知与即时事件。"""
factory = _session_factory()
writer = TransactionalChainDurableEventWriter(factory)
_, _, context, _, _ = _objects()
calls = []
writer.download_added(
history_payload={
"path": "/downloads/Demo.mkv",
"type": MediaType.MOVIE.value,
"title": "Demo",
"download_hash": "hash-2",
},
file_payloads=[
{
"download_hash": "hash-2",
"downloader": "qb",
"fullpath": "/downloads/Demo.mkv",
"savepath": "/downloads",
"filepath": "Demo.mkv",
"torrentname": "Demo torrent",
}
],
event_payload={
"hash": "hash-2",
"context": context,
"username": "alice",
"downloader": "qb",
"episodes": [],
"source": "manual",
},
after_commit=lambda: calls.append("after_commit"),
publish=lambda payload: calls.append(("event", payload)),
)
with factory() as session:
history = session.execute(select(DownloadHistory)).scalar_one()
download_file = session.execute(select(DownloadFiles)).scalar_one()
outbox = session.execute(select(OutboxMessage)).scalar_one()
assert history.download_hash == "hash-2"
assert download_file.fullpath == "/downloads/Demo.mkv"
assert outbox.status == "completed"
assert outbox.event_key == "download.added:qb:hash-2:v1"
assert [call if isinstance(call, str) else call[0] for call in calls] == [
"after_commit",
"event",
]
with pytest.raises(IntegrityError):
writer.download_added(
history_payload={
"path": "/downloads/duplicate.mkv",
"type": MediaType.MOVIE.value,
"title": "Duplicate",
"download_hash": "hash-2",
},
file_payloads=[],
event_payload={
"hash": "hash-2",
"context": context,
"downloader": "qb",
"episodes": [],
},
after_commit=lambda: None,
publish=lambda _payload: None,
)
with factory() as session:
assert len(session.execute(select(DownloadHistory)).scalars().all()) == 1
def test_transfer_event_failure_leaves_committed_intent_pending():
"""整理历史提交后即时广播失败不得误删可供恢复的 pending intent。"""
factory = _session_factory()
writer = TransactionalChainDurableEventWriter(factory)
meta, media, _, fileitem, transferinfo = _objects()
def stage_history(repository):
"""通过应用历史端口名暂存一条成功整理记录。"""
return repository.add_force(
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(mode="json"),
dest=transferinfo.target_item.path,
dest_storage=transferinfo.target_item.storage,
dest_fileitem=transferinfo.target_item.model_dump(mode="json"),
status=1,
)
def fail_publish(_payload):
"""模拟插件事件总线在业务提交后失败。"""
raise RuntimeError("event failed")
with pytest.raises(RuntimeError, match="event failed"):
writer.transfer_result(
topic="transfer.completed",
stage_history=stage_history,
event_payload={
"fileitem": fileitem,
"meta": meta,
"mediainfo": media,
"transferinfo": transferinfo,
"downloader": "qb",
"download_hash": "hash-3",
"transfer_history_id": None,
},
publish=fail_publish,
)
with factory() as session:
history = session.execute(select(TransferHistory)).scalar_one()
outbox = session.execute(select(OutboxMessage)).scalar_one()
assert history.status is True
assert outbox.status == "pending"
assert outbox.event_key == f"transfer.completed:{history.id}:v1"
+2
View File
@@ -20,6 +20,7 @@ def _context() -> ChainRuntimeContext:
async_file_cache=Mock(),
message_queue_factory=Mock(return_value=Mock()),
module_dispatcher_factory=ModuleInvocationDispatcher,
durable_event_writer=Mock(),
)
@@ -33,6 +34,7 @@ def test_chain_accepts_explicit_runtime_context() -> None:
assert chain.pluginmanager is context.plugin_manager
assert chain.eventmanager is context.event_manager
assert chain.messagehelper is context.message_helper
assert chain.durable_event_writer is context.durable_event_writer
context.message_queue_factory.assert_called_once_with(chain.run_module)
-15
View File
@@ -250,21 +250,6 @@ def test_list_by_type_includes_the_window_start_boundary(db, frozen_now):
assert "窗口起点上" in async_names and "窗口起点前一秒" not in async_names
def test_delete_by_media_identity_removes_matching_seasons_only(db):
"""
按媒体身份删除时,给出季号只删该季,不给则删全部季。
"""
db.add(_sub("第一季", season=1), _sub("第二季", season=2))
Subscribe().delete_by_media_identity(db.session, TMDB, "9001", season=1)
remaining = Subscribe.list_by_media_identity(db.session, MediaSource.TMDB, "9001")
assert [s.season for s in remaining] == [2]
Subscribe().delete_by_media_identity(db.session, TMDB, "9001")
assert Subscribe.list_by_media_identity(db.session, MediaSource.TMDB, "9001") == []
# --------------------------------------------------------------------------- #
# SubscribeHistory
# --------------------------------------------------------------------------- #
+35 -1
View File
@@ -9,8 +9,12 @@ from app.runtime.event.contracts import (
validate_event_payload,
)
from app.runtime.events import Event
from app.domain.context import Context, MediaInfo
from app.domain.metainfo import MetaInfo
from app.schemas.event import ConfigChangeEventData
from app.schemas.types import ChainEventType, EventType
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
from app.schemas.types import ChainEventType, EventType, MediaType
def test_every_event_enum_has_complete_contract() -> None:
@@ -61,6 +65,36 @@ def test_selected_user_side_effects_are_marked_durable_required() -> None:
assert get_event_contract(event_type).delivery is EventDelivery.DURABLE_REQUIRED
def test_download_and_transfer_typed_contracts_accept_legacy_runtime_objects() -> None:
"""新增 typed contract 只做诊断,不把插件收到的领域对象替换成 dict。"""
meta = MetaInfo("Demo.2026.mkv")
media = MediaInfo(type=MediaType.MOVIE, title="Demo", year="2026")
context = Context(meta_info=meta, media_info=media)
fileitem = FileItem(storage="local", path="/downloads/Demo.mkv", type="file")
transferinfo = TransferInfo(success=True, fileitem=fileitem)
assert validate_event_payload(
EventType.DownloadAdded,
{
"hash": "hash-1",
"context": context,
"downloader": "qb",
"episodes": [],
},
) == ()
for event_type in (EventType.TransferComplete, EventType.TransferFailed):
assert validate_event_payload(
event_type,
{
"fileitem": fileitem,
"meta": meta,
"mediainfo": media,
"transferinfo": transferinfo,
"transfer_history_id": 1,
},
) == ()
def test_model_instance_remains_mutable_chain_payload() -> None:
"""链式处理器继续接收原 model 实例,确保输出字段可原地接力。"""
payload = ConfigChangeEventData(key={"PROXY_HOST"})
+50
View File
@@ -221,3 +221,53 @@ def test_default_callback_keeps_original_failure_semantics_without_success_histo
if call.args[0] == EventType.TransferFailed
]
assert len(transfer_failed_events) == 1
def test_default_callback_delegates_primary_failure_to_durable_writer():
"""正式上下文存在 writer 时,主要媒体失败历史和事件必须走同一事务端口。"""
chain = make_transfer_chain()
chain.eventmanager = MagicMock()
chain.post_message = MagicMock()
chain.durable_event_writer = MagicMock()
task = _make_failed_task()
add_fail_calls = []
transfer_history_oper = make_history_oper(history=None)
transferinfo = TransferInfo(
success=False,
fileitem=task.fileitem,
message="copy failed",
transfer_type="copy",
need_notify=False,
)
def durable_transfer_result(**kwargs):
"""执行 writer 收到的历史暂存与提交后发布回调。"""
history = kwargs["stage_history"](SimpleNamespace())
payload = dict(kwargs["event_payload"])
payload["transfer_history_id"] = history.id
payload["idempotency_key"] = f"transfer.failed:{history.id}:v1"
kwargs["publish"](payload)
return history
chain.durable_event_writer.transfer_result.side_effect = durable_transfer_result
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=transfer_history_oper,
), patch(
"app.chain.transfer.add_transfer_fail",
make_fail_recorder(add_fail_calls),
), patch(
"app.chain.transfer.settings.AI_AGENT_ENABLE",
False,
):
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
assert state is False
assert errmsg == "copy failed"
assert len(add_fail_calls) == 1
chain.durable_event_writer.transfer_result.assert_called_once()
topic = chain.durable_event_writer.transfer_result.call_args.kwargs["topic"]
assert topic == "transfer.failed"
event_type, event_payload = chain.eventmanager.send_event.call_args.args
assert event_type == EventType.TransferFailed
assert event_payload["idempotency_key"] == "transfer.failed:1:v1"