refactor: make download and transfer events durable

This commit is contained in:
jxxghp
2026-08-22 08:03:00 +08:00
parent c5de1c7b1b
commit 820f7e8850
26 changed files with 1134 additions and 184 deletions
+2
View File
@@ -7,6 +7,7 @@ from dataclasses import dataclass
from typing import Any, Optional
from app.application.chain.data import ChainDataPorts
from app.application.chain.durable_events import ChainDurableEventWriter
MessageQueueFactory = Callable[[Callable[..., Any]], Any]
@@ -28,6 +29,7 @@ class ChainRuntimeContext:
message_queue_factory: MessageQueueFactory
module_dispatcher_factory: ModuleDispatcherFactory
data_ports: Optional[ChainDataPorts] = None
durable_event_writer: Optional[ChainDurableEventWriter] = None
def _unconfigured_chain_runtime_context() -> ChainRuntimeContext:
+230
View File
@@ -0,0 +1,230 @@
"""Chain durable 事件的事务写端口与可重放 payload 转换。"""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass, fields
from datetime import date, datetime
from enum import Enum
from pathlib import Path
from typing import Any, Protocol, cast
from app.application.history import TransferHistoryRecord, TransferHistoryWriter
from app.domain.context import Context, MediaInfo, MusicInfo, TorrentInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo
from app.schemas.file import FileItem
from app.schemas.transfer import TransferInfo
from app.schemas.types import MediaType
class ChainDurableEventWriter(Protocol):
"""下载与整理 Chain 原子写业务记录和 outbox 的宿主端口。"""
def download_added(
self,
*,
history_payload: dict[str, Any],
file_payloads: list[dict[str, Any]],
event_payload: dict[str, Any],
after_commit: Callable[[], None],
publish: Callable[[dict[str, Any]], None],
) -> None:
"""提交下载历史与 DownloadAdded intent,再执行原有提交后编排。"""
def transfer_result(
self,
*,
topic: str,
stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None],
event_payload: dict[str, Any],
publish: Callable[[dict[str, Any]], None],
) -> TransferHistoryRecord | None:
"""提交整理历史与结果 intent,并在提交后广播兼容事件。"""
@dataclass(frozen=True, slots=True)
class TransferHistoryRef:
"""事务关闭后仍可安全读取的最小整理历史投影。"""
id: int
status: bool
src: str | None
src_storage: str | None
src_fileitem: dict[str, Any] | None
def download_added_event_key(payload: dict[str, Any]) -> str:
"""由下载器与任务 hash 构造重试期间稳定的 DownloadAdded 幂等键。"""
return (
f"download.added:{payload.get('downloader') or 'unknown'}:"
f"{payload.get('hash') or 'unknown'}:v1"
)
def transfer_result_event_key(topic: str, history_id: int) -> str:
"""由结果 topic 与整理历史 ID 构造稳定幂等键。"""
return f"{topic}:{history_id}:v1"
def snapshot_download_added(payload: dict[str, Any]) -> dict[str, Any]:
"""把插件运行时 Context 转为 outbox 可 JSON 序列化的稳定快照。"""
context = payload.get("context")
return cast(dict[str, Any], _json_snapshot({
"hash": payload.get("hash"),
"context": context.to_dict() if isinstance(context, Context) else context,
"username": payload.get("username"),
"downloader": payload.get("downloader"),
"episodes": list(payload.get("episodes") or []),
"source": payload.get("source"),
"idempotency_key": payload.get("idempotency_key"),
}))
def restore_download_added(payload: dict[str, Any]) -> dict[str, Any]:
"""从 outbox 快照恢复插件既有 DownloadAdded 运行时对象形状。"""
restored = dict(payload)
context = payload.get("context")
if isinstance(context, dict):
restored["context"] = _restore_context(context)
return restored
def snapshot_transfer_result(payload: dict[str, Any]) -> dict[str, Any]:
"""把整理事件中的领域对象转换为可恢复 JSON 快照。"""
return cast(dict[str, Any], _json_snapshot({
"fileitem": _model_snapshot(payload.get("fileitem")),
"meta": _object_snapshot(payload.get("meta")),
"mediainfo": _object_snapshot(payload.get("mediainfo")),
"transferinfo": _model_snapshot(payload.get("transferinfo")),
"downloader": payload.get("downloader"),
"download_hash": payload.get("download_hash"),
"transfer_history_id": payload.get("transfer_history_id"),
"idempotency_key": payload.get("idempotency_key"),
}))
def restore_transfer_result(payload: dict[str, Any]) -> dict[str, Any]:
"""从 outbox 快照恢复 TransferComplete/Failed 的旧对象 payload。"""
restored = dict(payload)
fileitem = payload.get("fileitem")
meta = payload.get("meta")
mediainfo = payload.get("mediainfo")
transferinfo = payload.get("transferinfo")
restored["fileitem"] = (
FileItem.model_validate(fileitem) if isinstance(fileitem, dict) else fileitem
)
restored["meta"] = _restore_meta(meta) if isinstance(meta, dict) else meta
restored["mediainfo"] = (
_restore_media(mediainfo) if isinstance(mediainfo, dict) else mediainfo
)
restored["transferinfo"] = (
TransferInfo.model_validate(transferinfo)
if isinstance(transferinfo, dict)
else transferinfo
)
return restored
def _model_snapshot(value: Any) -> Any:
"""序列化 Pydantic 风格对象,空值和已有 JSON 值原样返回。"""
if hasattr(value, "model_dump"):
return value.model_dump(mode="json")
return value
def _object_snapshot(value: Any) -> Any:
"""序列化领域对象,避免把不可持久化实例写入 JSON 列。"""
if hasattr(value, "to_dict"):
return value.to_dict()
return _model_snapshot(value)
def _json_snapshot(value: Any) -> Any:
"""递归归一化快照,确保 SQLAlchemy JSON 不接收运行时专用对象。"""
if value is None or isinstance(value, (str, int, float, bool)):
return value
if isinstance(value, Enum):
return _json_snapshot(value.value)
if isinstance(value, (Path, date, datetime)):
return value.isoformat() if hasattr(value, "isoformat") else str(value)
if isinstance(value, dict):
return {
str(key): _json_snapshot(item)
for key, item in value.items()
}
if isinstance(value, (list, tuple, set, frozenset)):
return [_json_snapshot(item) for item in value]
if hasattr(value, "model_dump"):
return _json_snapshot(value.model_dump(mode="json"))
if hasattr(value, "to_dict"):
return _json_snapshot(value.to_dict())
return str(value)
def _restore_context(payload: dict[str, Any]) -> Context:
"""恢复 DownloadAdded 插件依赖的 Context 聚合对象。"""
meta_payload = payload.get("meta_info")
media_payload = payload.get("media_info")
torrent_payload = payload.get("torrent_info")
allowed_episodes = payload.get("allowed_episodes")
return Context(
meta_info=(
_restore_meta(meta_payload) if isinstance(meta_payload, dict) else None
),
media_info=(
_restore_media(media_payload) if isinstance(media_payload, dict) else None
),
torrent_info=(
_restore_torrent(torrent_payload)
if isinstance(torrent_payload, dict)
else None
),
media_recognize_fail_count=int(
payload.get("media_recognize_fail_count") or 0
),
resource_source=str(payload.get("resource_source") or "unknown"),
match_source=str(payload.get("match_source") or "unknown"),
candidate_recognized=bool(payload.get("candidate_recognized")),
media_info_is_target=bool(payload.get("media_info_is_target")),
allowed_episodes=(
set(allowed_episodes) if allowed_episodes is not None else None
),
confirmed_full_coverage=bool(payload.get("confirmed_full_coverage")),
)
def _restore_meta(payload: dict[str, Any]) -> MetaBase:
"""恢复影视或音乐文件名解析对象,并保留快照中的解析字段。"""
if payload.get("type") in {MediaType.MUSIC, MediaType.MUSIC.value, "music"}:
return MetaMusic.from_dict(payload)
title = str(
payload.get("org_string")
or payload.get("title")
or payload.get("name")
or ""
)
meta = MetaInfo(title)
for key, value in payload.items():
if key in {"season_episode", "edition", "name", "episode_list"}:
continue
if key == "type" and value:
value = MediaType(value)
setattr(meta, key, value)
return meta
def _restore_media(payload: dict[str, Any]) -> MediaInfo | MusicInfo:
"""恢复影视或音乐媒体对象,不触发任何远端识别。"""
if payload.get("type") in {MediaType.MUSIC, MediaType.MUSIC.value, "music"}:
return MusicInfo.from_dict(payload)
media = MediaInfo()
media.from_dict(payload)
return media
def _restore_torrent(payload: dict[str, Any]) -> TorrentInfo:
"""按 dataclass 构造字段恢复种子对象,忽略快照中的计算属性。"""
allowed = {item.name for item in fields(TorrentInfo) if item.init}
return TorrentInfo(**{key: value for key, value in payload.items() if key in allowed})
+68 -1
View File
@@ -5,7 +5,10 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import Any, Protocol
from typing import Any, Protocol, TypeVar
T = TypeVar("T")
@dataclass(frozen=True, slots=True)
@@ -67,6 +70,70 @@ class AsyncOutboxTransaction(Protocol):
"""即时投递成功后按稳定幂等键标记 intent 完成。"""
class SyncUnitOfWork(Protocol):
"""同步 durable 业务切片的最小事务端口。"""
def commit(self) -> None:
"""提交业务写入与 outbox intent。"""
def rollback(self) -> None:
"""回滚业务写入与 outbox intent。"""
class SyncOutboxTransaction(Protocol):
"""同步业务事务暂存并收口 durable intent 的最小端口。"""
def stage(self, intent: OutboxIntent, now: datetime) -> None:
"""把 intent 加入调用方事务,但不自行提交。"""
def complete_by_event_key(
self,
event_key: str,
completed_at: datetime,
) -> None:
"""即时投递成功后按幂等键标记 intent 完成。"""
class DurableEventCommand:
"""把一次同步业务写入与可恢复事件 intent 原子提交。"""
def __init__(
self,
unit_of_work: SyncUnitOfWork,
outbox: SyncOutboxTransaction,
) -> None:
"""注入共享同一 Session 的事务与 outbox 端口。"""
self._unit_of_work = unit_of_work
self._outbox = outbox
def execute(
self,
*,
intent: OutboxIntent | Callable[[T], OutboxIntent],
stage_business: Callable[[], T],
publish: Callable[[], None],
after_commit: Callable[[], None] | None = None,
) -> T:
"""先原子提交业务与 intent,再保持原顺序执行提交后动作和即时广播。"""
try:
result = stage_business()
resolved_intent = intent(result) if callable(intent) else intent
self._outbox.stage(resolved_intent, datetime.now(timezone.utc))
self._unit_of_work.commit()
except Exception:
self._unit_of_work.rollback()
raise
if after_commit:
after_commit()
publish()
self._outbox.complete_by_event_key(
resolved_intent.event_key,
datetime.now(timezone.utc),
)
return result
class OutboxDispatcher:
"""认领并派发 outbox,按 event key 依赖 handler 幂等。"""
+1
View File
@@ -53,6 +53,7 @@ class ChainBase(RecognitionMixin, MessageProcessingMixin, NotificationMixin,
self.filecache = context.file_cache
self.async_filecache = context.async_file_cache
self.data_ports = context.data_ports or get_chain_data_ports()
self.durable_event_writer = context.durable_event_writer
self._module_dispatcher = context.module_dispatcher_factory(
module_catalog=self.modulemanager,
plugin_catalog=self.pluginmanager,
+105 -59
View File
@@ -105,6 +105,45 @@ class DownloadChain(ChainBase):
}
return note
def _after_download_history_commit(
self,
*,
context: Context,
media: MediaInfo | MusicInfo,
meta: MetaBase,
torrent: TorrentInfo,
channel: NotificationChannel | None,
source: str | None,
userid: str | None,
username: str | None,
download_episodes: list[int] | None,
download_dir: Path,
torrent_content: bytes,
) -> None:
"""保持下载历史提交后的通知和后处理顺序。"""
self.post_message(
Message(
channel=channel,
source=source if channel else None,
mtype=MessageType.Download,
ctype=ContentType.DownloadAdded,
image=media.get_message_image(),
link=settings.MP_DOMAIN('/#/downloading'),
userid=userid,
username=username,
),
meta=meta,
mediainfo=media,
torrentinfo=torrent,
download_episodes=download_episodes,
username=username,
)
self._submit_download_added_task(
context=context,
download_dir=download_dir,
torrent_content=torrent_content,
)
@staticmethod
def _validate_music_album_resource(
context: Context,
@@ -1161,35 +1200,33 @@ class DownloadChain(ChainBase):
# 文件保存路径
_save_path = download_dir if _layout == "NoSubfolder" or not _folder_name else download_path
# 登记下载记录
downloadhis = DownloadHistoryOper()
media_source, media_id = resolve_media_identity(media=_media)
downloadhis.add(
path=download_path.as_posix(),
type=_media.type.value,
title=_media.title,
year=_media.year,
media_source=media_source,
media_id=media_id,
music_type=getattr(_media, "music_type", None),
seasons=_meta.season,
episodes=download_episodes or _meta.episode,
image=_media.get_backdrop_image(),
poster=_media.get_poster_image(),
downloader=_downloader,
download_hash=_hash,
torrent_name=_torrent.title,
torrent_description=_torrent.description,
torrent_site=_torrent.site_name,
userid=userid,
username=username,
channel=channel.value if channel else None,
date=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
media_category=_media.category,
episode_group=_media.episode_group,
note=self._build_download_note(source, _media, _meta),
custom_words=custom_words
)
history_payload = {
"path": download_path.as_posix(),
"type": _media.type.value,
"title": _media.title,
"year": _media.year,
"media_source": media_source,
"media_id": media_id,
"music_type": getattr(_media, "music_type", None),
"seasons": _meta.season,
"episodes": download_episodes or _meta.episode,
"image": _media.get_backdrop_image(),
"poster": _media.get_poster_image(),
"downloader": _downloader,
"download_hash": _hash,
"torrent_name": _torrent.title,
"torrent_description": _torrent.description,
"torrent_site": _torrent.site_name,
"userid": userid,
"username": username,
"channel": channel.value if channel else None,
"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
"media_category": _media.category,
"episode_group": _media.episode_group,
"note": self._build_download_note(source, _media, _meta),
"custom_words": custom_words,
}
# 登记下载文件
files_to_add = []
@@ -1213,42 +1250,51 @@ class DownloadChain(ChainBase):
"filepath": file,
"torrentname": _meta.org_string,
})
if files_to_add:
downloadhis.add_files(files_to_add)
# 下载成功发送消息
self.post_message(
Message(
channel=channel,
source=source if channel else None,
mtype=MessageType.Download,
ctype=ContentType.DownloadAdded,
image=_media.get_message_image(),
link=settings.MP_DOMAIN('/#/downloading'),
userid=userid,
username=username
),
meta=_meta,
mediainfo=_media,
torrentinfo=_torrent,
download_episodes=download_episodes,
username=username,
)
# 下载成功后处理
self._submit_download_added_task(
context=context,
download_dir=download_dir,
torrent_content=torrent_content,
)
# 广播事件
self.eventmanager.send_event(EventType.DownloadAdded, {
event_payload = {
"hash": _hash,
"context": context,
"username": username,
"downloader": _downloader,
"episodes": episodes or _meta.episode_list,
"source": source
})
"source": source,
}
def after_commit() -> None:
"""在历史与 intent 提交后保持原有通知和任务编排。"""
self._after_download_history_commit(
context=context,
media=_media,
meta=_meta,
torrent=_torrent,
channel=channel,
source=source,
userid=userid,
username=username,
download_episodes=download_episodes,
download_dir=download_dir,
torrent_content=torrent_content,
)
durable_event_writer = getattr(self, "durable_event_writer", None)
if durable_event_writer:
durable_event_writer.download_added(
history_payload=history_payload,
file_payloads=files_to_add,
event_payload=event_payload,
after_commit=after_commit,
publish=lambda payload: self.eventmanager.send_event(
EventType.DownloadAdded,
payload,
),
)
else:
# 显式注入旧测试上下文时保持兼容;正式启动上下文总会提供 durable writer。
downloadhis = DownloadHistoryOper()
downloadhis.add(**history_payload)
if files_to_add:
downloadhis.add_files(files_to_add)
after_commit()
self.eventmanager.send_event(EventType.DownloadAdded, event_payload)
else:
# 下载失败
logger.error(f"{_media.title_year} 添加下载任务失败:"
+105 -46
View File
@@ -87,6 +87,23 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
"TRANSFER_THREADS",
}
@staticmethod
def _transfer_result_payload(
task: TransferTask,
transferinfo: TransferInfo,
history_id: int | None = None,
) -> dict[str, Any]:
"""构造保持插件旧对象字段不变的整理结果事件 payload。"""
return {
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history_id,
}
def __init__(self):
"""初始化文件整理处理链。"""
super().__init__()
@@ -229,33 +246,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
fileid=task.fileitem.fileid if task.fileitem else None,
)
# 新增转移失败历史记录
history = add_transfer_fail(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
durable_transfer_failed = bool(
getattr(self, "durable_event_writer", None)
and self._is_media_file(task.fileitem)
)
if durable_transfer_failed:
event_payload = self._transfer_result_payload(task, transferinfo)
history = self.durable_event_writer.transfer_result(
topic="transfer.failed",
stage_history=lambda writer: add_transfer_fail(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=writer,
),
event_payload=event_payload,
publish=lambda payload: self.eventmanager.send_event(
EventType.TransferFailed,
payload,
),
)
else:
history = add_transfer_fail(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
)
# 整理失败事件
if self._is_media_file(task.fileitem):
# 主要媒体文件整理失败事件
self.eventmanager.send_event(
EventType.TransferFailed,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
if not durable_transfer_failed:
# 显式旧测试上下文仍走原始发送;正式上下文由 outbox writer 发布。
self.eventmanager.send_event(
EventType.TransferFailed,
self._transfer_result_payload(
task,
transferinfo,
history.id if history else None,
),
)
elif self._is_subtitle_file(task.fileitem):
# 字幕整理失败事件
self.eventmanager.send_event(
@@ -350,33 +388,54 @@ class TransferChain(FileFilterMixin, ScrapeBatchMixin, EpisodeFormatMixin, Histo
task.fileitem.storage if task.fileitem else None,
)
# 新增task转移成功历史记录
history = add_transfer_success(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
durable_transfer_complete = bool(
getattr(self, "durable_event_writer", None)
and self._is_primary_media_file(task.fileitem, task.mediainfo)
)
if durable_transfer_complete:
event_payload = self._transfer_result_payload(task, transferinfo)
history = self.durable_event_writer.transfer_result(
topic="transfer.completed",
stage_history=lambda writer: add_transfer_success(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=writer,
),
event_payload=event_payload,
publish=lambda payload: self.eventmanager.send_event(
EventType.TransferComplete,
payload,
),
)
else:
history = add_transfer_success(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
)
# task整理完成事件
if self._is_primary_media_file(task.fileitem, task.mediainfo):
# 主要媒体文件整理完成事件
self.eventmanager.send_event(
EventType.TransferComplete,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
if not durable_transfer_complete:
# 显式旧测试上下文仍走原始发送;正式上下文由 outbox writer 发布。
self.eventmanager.send_event(
EventType.TransferComplete,
self._transfer_result_payload(
task,
transferinfo,
history.id if history else None,
),
)
elif self._is_subtitle_file(task.fileitem):
# 字幕整理完成事件
self.eventmanager.send_event(
+2 -32
View File
@@ -1,12 +1,12 @@
import time
from typing import Any, Optional
from sqlalchemy import Integer, String, Float, JSON, Index, delete, or_, select
from sqlalchemy import Integer, String, Float, JSON, Index, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db.base import get_id_column, Base
from app.db.decorators import db_query, db_update, async_db_query, async_db_update
from app.db.decorators import async_db_query, db_query
from app.db.models._constraints import media_identity_constraint
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource
@@ -363,36 +363,6 @@ class Subscribe(Base):
result = await db.execute(query)
return result.scalars().first()
@db_update
def delete_by_media_identity(
self, db: Session, media_source: MediaSource, media_id: str,
season: Optional[int] = None,
) -> bool:
"""按规范媒体身份删除订阅。"""
model = type(self)
statement = delete(model).where(
model.media_source == media_source,
model.media_id == str(media_id),
)
if season is not None:
statement = statement.where(model.season == season)
db.execute(statement, execution_options={"synchronize_session": False})
return True
@async_db_update
async def async_delete_by_media_identity(
self, db: AsyncSession, media_source: MediaSource, media_id: str,
season: Optional[int] = None,
) -> bool:
"""异步按规范媒体身份删除订阅。"""
rows = await self.async_list_by_media_identity(
db, media_source=media_source, media_id=media_id
)
for row in rows:
if season is None or row.season == season:
await row.async_delete(db, row.id)
return True
@classmethod
@db_query
def list_by_username(cls, db: Session, username: str, state: Optional[str] = None, mtype: Optional[str] = None):
+17
View File
@@ -1,6 +1,7 @@
from typing import Dict, List, Optional, cast
from sqlalchemy import delete as sqlalchemy_delete, update as sqlalchemy_update
from sqlalchemy.orm import Session
from app.db.base import DbOper
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
@@ -60,6 +61,15 @@ class DownloadHistoryOper(DbOper):
"""
DownloadHistory(**kwargs).create(self._db)
def stage_add(self, payload: dict) -> DownloadHistory:
"""在调用方同步 Session 中暂存下载历史并返回已分配 ID 的记录。"""
if not isinstance(self._db, Session):
raise RuntimeError("下载历史事务写入需要调用方提供同步 Session")
history = DownloadHistory(**payload)
self._db.add(history)
self._db.flush()
return history
def add_files(self, file_items: List[dict]):
"""
新增下载历史文件
@@ -68,6 +78,13 @@ class DownloadHistoryOper(DbOper):
downloadfile = DownloadFiles(**file_item)
downloadfile.create(self._db)
def stage_add_files(self, file_items: List[dict]) -> None:
"""在调用方事务内批量暂存下载文件,不逐条提交。"""
if not isinstance(self._db, Session):
raise RuntimeError("下载文件事务写入需要调用方提供同步 Session")
self._db.add_all(DownloadFiles(**item) for item in file_items)
self._db.flush()
def truncate_files(self):
"""
清空下载历史文件记录
+19
View File
@@ -2,6 +2,7 @@ import time
from typing import Any, List, Optional
from sqlalchemy import delete as sqlalchemy_delete
from sqlalchemy.orm import Session
from app.db.base import DbOper
from app.db.models.transferhistory import TransferHistory
@@ -272,6 +273,24 @@ class TransferHistoryOper(DbOper):
kwargs["src_storage"],
)
def stage_replace_by_src(self, **kwargs) -> TransferHistory:
"""在调用方事务内按源路径替换整理历史并返回已分配 ID 的新记录。"""
if not isinstance(self._db, Session):
raise RuntimeError("整理历史事务写入需要调用方提供同步 Session")
kwargs["src_storage"] = kwargs.get("src_storage") or "local"
kwargs["date"] = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
self._db.execute(
sqlalchemy_delete(TransferHistory).where(
TransferHistory.src == kwargs.get("src"),
TransferHistory.src_storage == kwargs["src_storage"],
)
)
self._db.flush()
history = TransferHistory(**kwargs)
self._db.add(history)
self._db.flush()
return history
def update_download_hash(self, historyid, download_hash):
"""
补充转移记录download_hash
+3
View File
@@ -57,6 +57,9 @@ _PAYLOAD_MODELS: dict[EventType | ChainEventType, type[BaseModel]] = {
EventType.SubscribeAdded: event_schemas.SubscribeAddedEventData,
EventType.SubscribeDeleted: event_schemas.SubscribeDeletedEventData,
EventType.SubscribeModified: event_schemas.SubscribeModifiedEventData,
EventType.DownloadAdded: event_schemas.DownloadAddedEventData,
EventType.TransferComplete: event_schemas.TransferResultEventData,
EventType.TransferFailed: event_schemas.TransferResultEventData,
ChainEventType.PluginDataReset: event_schemas.PluginDataResetEventData,
ChainEventType.AuthVerification: event_schemas.AuthCredentials,
ChainEventType.AuthIntercept: event_schemas.AuthInterceptCredentials,
+26
View File
@@ -7,6 +7,7 @@ from app.schemas.common import JsonData
from app.schemas.types import MediaType, NotificationChannel
from app.schemas.file import FileItem
from app.schemas.media import OptionalMediaIdentityMixin, RequiredMediaIdentityMixin
from app.schemas.transfer import TransferInfo
from app.schemas.types import MediaSource
@@ -730,6 +731,31 @@ class SubscribeDeletedEventData(BaseEventData):
idempotency_key: Optional[str] = Field(default=None, description="宿主生成的幂等键")
class DownloadAddedEventData(BaseEventData):
"""DownloadAdded 广播事件的插件兼容 payload。"""
hash: str = Field(description="下载任务 hash")
context: Any = Field(description="下载上下文对象")
username: Optional[str] = Field(default=None, description="发起下载的用户")
downloader: Optional[str] = Field(default=None, description="下载器名称")
episodes: List[int] = Field(default_factory=list, description="下载剧集列表")
source: Optional[str] = Field(default=None, description="下载来源")
idempotency_key: Optional[str] = Field(default=None, description="宿主生成的幂等键")
class TransferResultEventData(BaseEventData):
"""TransferComplete/Failed 共用的插件兼容 payload。"""
fileitem: Optional[FileItem] = Field(default=None, description="源文件项")
meta: Any = Field(default=None, description="文件名解析对象")
mediainfo: Any = Field(default=None, description="媒体信息对象")
transferinfo: Optional[TransferInfo] = Field(default=None, description="整理结果")
downloader: Optional[str] = Field(default=None, description="下载器名称")
download_hash: Optional[str] = Field(default=None, description="下载任务 hash")
transfer_history_id: Optional[int] = Field(default=None, description="整理历史 ID")
idempotency_key: Optional[str] = Field(default=None, description="宿主生成的幂等键")
class SubscribeCompletionCheckEventData(ChainEventData):
"""
SubscribeCompletionCheck 事件的数据模型
+3
View File
@@ -85,6 +85,7 @@ SCHEMA_EXPORTS = {
'DiscoverSourceEventData': ('app.schemas.event', 'DiscoverSourceEventData'),
'Discriminator': ('app.schemas.context', 'Discriminator'),
'DownloadAddedData': ('app.schemas.download', 'DownloadAddedData'),
'DownloadAddedEventData': ('app.schemas.event', 'DownloadAddedEventData'),
'DownloadDirectory': ('app.schemas.download', 'DownloadDirectory'),
'DownloadHistory': ('app.schemas.history', 'DownloadHistory'),
'DownloadTask': ('app.schemas.workflow', 'DownloadTask'),
@@ -369,6 +370,7 @@ SCHEMA_EXPORTS = {
'TransferOverwriteCheckEventData': ('app.schemas.event', 'TransferOverwriteCheckEventData'),
'TransferRenameBuildEventData': ('app.schemas.event', 'TransferRenameBuildEventData'),
'TransferRenameEventData': ('app.schemas.event', 'TransferRenameEventData'),
'TransferResultEventData': ('app.schemas.event', 'TransferResultEventData'),
'TransferTorrent': ('app.schemas.transfer', 'TransferTorrent'),
'TypeAdapter': ('app.schemas.mcp', 'TypeAdapter'),
'TypeAlias': ('app.schemas.mcp', 'TypeAlias'),
@@ -441,6 +443,7 @@ SCHEMA_CONFLICTS = {
'Subscribe': ['app.schemas.subscribe', 'app.schemas.workflow'],
'SubtitleInfo': ['app.schemas.context', 'app.schemas.search'],
'TorrentInfo': ['app.schemas.context', 'app.schemas.search', 'app.schemas.system'],
'TransferInfo': ['app.schemas.event', 'app.schemas.transfer'],
'Union': ['app.schemas.agent', 'app.schemas.common', 'app.schemas.context', 'app.schemas.mediaserver', 'app.schemas.message', 'app.schemas.plugin', 'app.schemas.search', 'app.schemas.site', 'app.schemas.transfer', 'app.schemas.mcp'],
'UserPermissions': ['app.schemas.token', 'app.schemas.user'],
'dataclass': ['app.schemas.notification', 'app.schemas.system'],
+152
View File
@@ -0,0 +1,152 @@
"""Chain durable 事件写入端口的 SQLAlchemy 启动适配器。"""
from __future__ import annotations
from collections.abc import Callable
from typing import Any
from sqlalchemy.orm import Session
from app.application.chain.durable_events import (
ChainDurableEventWriter,
TransferHistoryRef,
download_added_event_key,
snapshot_download_added,
snapshot_transfer_result,
transfer_result_event_key,
)
from app.application.history import TransferHistoryRecord, TransferHistoryWriter
from app.application.outbox import DurableEventCommand, OutboxIntent
from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.oper.transferhistory import TransferHistoryOper
from app.db.uow import SqlAlchemyUnitOfWork
from app.startup.outbox import SqlAlchemyOutboxRepository
class _StagingTransferHistoryWriter:
"""让既有历史字段映射复用无提交的 replace 适配器。"""
def __init__(self, repository: TransferHistoryOper) -> None:
"""保存绑定调用方 Session 的整理历史仓储。"""
self._repository = repository
def get_by_src(
self,
src: str,
storage: str | None = None,
) -> TransferHistoryRecord | None:
"""转发按源路径读取。"""
return self._repository.get_by_src(src, storage)
def get_success_by_src(
self,
src: str,
storage: str | None = None,
) -> TransferHistoryRecord | None:
"""转发按源路径读取成功记录。"""
return self._repository.get_success_by_src(src, storage)
def add_force(self, **payload: Any) -> TransferHistoryRecord:
"""保持应用层旧端口名,但只暂存替换而不自行提交。"""
return self._repository.stage_replace_by_src(**payload)
class TransactionalChainDurableEventWriter(ChainDurableEventWriter):
"""为每次 Chain 结果事件创建独占同步 Session 和 UoW。"""
def __init__(self, session_factory: Callable[[], Session]) -> None:
"""注入惰性同步 Session 工厂。"""
self._session_factory = session_factory
def download_added(
self,
*,
history_payload: dict[str, Any],
file_payloads: list[dict[str, Any]],
event_payload: dict[str, Any],
after_commit: Callable[[], None],
publish: Callable[[dict[str, Any]], None],
) -> None:
"""原子写下载历史、文件清单和 DownloadAdded intent。"""
session = self._session_factory()
try:
repository = DownloadHistoryOper(session)
outbox = SqlAlchemyOutboxRepository(session)
command = DurableEventCommand(
unit_of_work=SqlAlchemyUnitOfWork(session),
outbox=outbox,
)
event_key = download_added_event_key(event_payload)
event_payload["idempotency_key"] = event_key
def stage_business() -> None:
"""在同一事务暂存下载历史和可选文件清单。"""
repository.stage_add(history_payload)
if file_payloads:
repository.stage_add_files(file_payloads)
command.execute(
intent=OutboxIntent(
event_key=event_key,
topic="download.added",
payload=snapshot_download_added(event_payload),
),
stage_business=stage_business,
after_commit=after_commit,
publish=lambda: publish(event_payload),
)
finally:
session.close()
def transfer_result(
self,
*,
topic: str,
stage_history: Callable[[TransferHistoryWriter], TransferHistoryRecord | None],
event_payload: dict[str, Any],
publish: Callable[[dict[str, Any]], None],
) -> TransferHistoryRecord | None:
"""原子写整理历史与结果 intent,并返回脱离 Session 的最小投影。"""
session = self._session_factory()
try:
staging = _StagingTransferHistoryWriter(TransferHistoryOper(session))
command = DurableEventCommand(
unit_of_work=SqlAlchemyUnitOfWork(session),
outbox=SqlAlchemyOutboxRepository(session),
)
def stage_business() -> TransferHistoryRef | None:
"""复用历史字段映射,并在 flush 后冻结安全投影。"""
history = stage_history(staging)
if history is None:
return None
return TransferHistoryRef(
id=history.id,
status=bool(history.status),
src=history.src,
src_storage=history.src_storage,
src_fileitem=history.src_fileitem,
)
def build_intent(
history: TransferHistoryRef | None,
) -> OutboxIntent:
"""历史 ID 确定后构造事件键与可恢复快照。"""
if history is None:
raise RuntimeError("整理历史暂存失败,无法登记 durable 结果事件")
event_key = transfer_result_event_key(topic, history.id)
event_payload["transfer_history_id"] = history.id
event_payload["idempotency_key"] = event_key
return OutboxIntent(
event_key=event_key,
topic=topic,
payload=snapshot_transfer_result(event_payload),
)
return command.execute(
intent=build_intent,
stage_business=stage_business,
publish=lambda: publish(event_payload),
)
finally:
session.close()
+18
View File
@@ -107,6 +107,7 @@ from app.startup.subscription import (
TransactionalSubscribeWriter,
configure_transactional_subscription_scopes,
)
from app.startup.chain_events import TransactionalChainDurableEventWriter
from app.startup.context import AgentChatRuntime, HostRuntime, SubscriptionRuntime
from app.adapters.web.security.access import set_superuser_token_payload_provider
from app.application.security.auth import build_superuser_token_payload
@@ -115,6 +116,10 @@ from app.application.chain.context import (
ChainRuntimeContext,
configure_chain_runtime_context_provider,
)
from app.application.chain.durable_events import (
restore_download_added,
restore_transfer_result,
)
from app.application.chain.data import configure_chain_data_ports, get_chain_data_ports
from app.runtime.extensions.service_config import (
ServiceConfigHelper,
@@ -147,6 +152,7 @@ def _build_chain_runtime_context() -> ChainRuntimeContext:
),
module_dispatcher_factory=ModuleInvocationDispatcher,
data_ports=get_chain_data_ports(),
durable_event_writer=TransactionalChainDurableEventWriter(SessionFactory),
)
@@ -217,6 +223,18 @@ def _build_outbox_dispatcher() -> OutboxDispatcher:
EventType.SubscribeDeleted,
message.payload,
),
"download.added": lambda message: EventManager().send_event(
EventType.DownloadAdded,
restore_download_added(message.payload),
),
"transfer.completed": lambda message: EventManager().send_event(
EventType.TransferComplete,
restore_transfer_result(message.payload),
),
"transfer.failed": lambda message: EventManager().send_event(
EventType.TransferFailed,
restore_transfer_result(message.payload),
),
},
close=session.close,
)