refactor(db): 修复异步连接池无界增长,并完成 SQLAlchemy 2.0 迁移与分层归位 (#6320)

This commit is contained in:
Aqr-K
2026-08-15 06:58:38 +08:00
committed by GitHub
parent e28de9cfe1
commit 8a11214a43
252 changed files with 11405 additions and 2889 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import List, Optional, Tuple
from app import schemas
from app.domain.context import MediaInfo
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger
from app.schemas.types import MediaType, StorageSchema, SystemConfigKey
from app.adapters.system.host import SystemUtils
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import List, Optional
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.domain.context import MediaInfo
from app.schemas import CustomRule, FilterRuleGroup
from app.schemas.types import SystemConfigKey
+163 -2
View File
@@ -1,10 +1,16 @@
from typing import Any, Dict, Optional
from typing import Any, Dict, Optional, Union
from app.domain.context import MediaInfo, MusicInfo
from app.schemas.media import resolve_media_identity
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.runtime.cache import TTLCache
from app.runtime.config import settings
from app.db.models.transferhistory import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper
from app.db.oper.transferhistory import TransferHistoryOper
from app.runtime.log import logger
from app.schemas import FileItem, TransferInfo
from app.schemas.types import MUSIC_ENTITY_RECORDING
# 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多
@@ -420,3 +426,158 @@ def describe_history_gate(history: Optional[TransferHistory],
if recorded_size is None and current_size is None:
return f"成功记录 #{history.id},大小不可比对"
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"
# --------------------------------------------------------------------------- #
# 整理历史的写入路径
#
# 这两个函数把 FileItem / MetaBase / MediaInfo / TransferInfo 四个领域对象翻译成
# 一行整理历史,是整理历史表的唯一写入口。它们此前长在 TransferHistoryOper 上,但
# 拼标题、拆季集、取海报、判音乐字段都是整理链的业务规则而非数据访问——Oper 只该
# 收敛查询,领域对象不该出现在它的入参里。搬到本模块与查重闸(读侧)作伴:同一张
# 表的读写规则放在一起,字段含义只有一处需要维护。
# --------------------------------------------------------------------------- #
def _history_title(meta: MetaBase,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None) -> Optional[str]:
"""音乐文件优先记录曲目标题,其它媒体保持识别标题。"""
if isinstance(meta, MetaMusic) and meta.title:
return meta.title
if mediainfo and mediainfo.title:
return mediainfo.title
return meta.name
def add_transfer_success(fileitem: FileItem, mode: str, meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo], transferinfo: TransferInfo,
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
新增转移成功历史记录。
:param fileitem: 源文件项
:param mode: 整理方式
:param meta: 文件名识别结果
:param mediainfo: 媒体识别结果
:param transferinfo: 整理结果
:param downloader: 下载器
:param download_hash: 种子 hash
:param transfer_history_oper: 复用的历史操作对象,未传时新建
:return: 落库后的整理记录
"""
oper = transfer_history_oper or TransferHistoryOper()
media_source, media_id = resolve_media_identity(media=mediainfo)
return oper.add_force(
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
dest=transferinfo.target_item.path if transferinfo.target_item else None,
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
mode=mode,
type=mediainfo.type.value,
category=mediainfo.category,
title=_history_title(meta, mediainfo),
year=mediainfo.year,
media_source=media_source,
media_id=media_id,
music_type=getattr(mediainfo, "music_type", None),
total_tracks=getattr(mediainfo, "total_tracks", None),
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
seasons=meta.season,
episodes=meta.episode,
image=mediainfo.get_poster_image(),
downloader=downloader,
download_hash=download_hash,
status=1,
files=transferinfo.file_list
)
def add_transfer_fail(fileitem: FileItem, mode: str, meta: MetaBase,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
transferinfo: Optional[TransferInfo] = None,
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
新增转移失败历史记录。
识别结果与整理结果齐备时按完整字段落库;缺任一项则走「未识别到媒体信息」分支,
此时只有文件名解析出的元数据可用,不写目标路径。
:param fileitem: 源文件项
:param mode: 整理方式
:param meta: 文件名识别结果
:param mediainfo: 媒体识别结果,未识别时为 None
:param transferinfo: 整理结果,未进入整理时为 None
:param downloader: 下载器
:param download_hash: 种子 hash
:param transfer_history_oper: 复用的历史操作对象,未传时新建
:return: 落库后的整理记录
"""
oper = transfer_history_oper or TransferHistoryOper()
if mediainfo and transferinfo:
media_source, media_id = resolve_media_identity(media=mediainfo)
his = oper.add_force(
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
dest=transferinfo.target_item.path if transferinfo.target_item else None,
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
mode=mode,
type=mediainfo.type.value,
category=mediainfo.category,
title=_history_title(meta, mediainfo),
year=mediainfo.year or meta.year,
media_source=media_source,
media_id=media_id,
music_type=getattr(mediainfo, "music_type", None),
total_tracks=getattr(mediainfo, "total_tracks", None),
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
seasons=meta.season,
episodes=meta.episode,
image=mediainfo.get_poster_image(),
downloader=downloader,
download_hash=download_hash,
episode_group=mediainfo.episode_group,
status=0,
errmsg=transferinfo.message or '未知错误',
files=transferinfo.file_list
)
else:
media_source, media_id = resolve_media_identity(media=meta)
his = oper.add_force(
type=meta.type.value if meta.type else None,
title=_history_title(meta),
year=meta.year,
media_source=media_source,
media_id=media_id,
music_type=MUSIC_ENTITY_RECORDING if isinstance(meta, MetaMusic) else None,
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
mode=mode,
seasons=meta.season,
episodes=meta.episode,
downloader=downloader,
download_hash=download_hash,
status=0,
errmsg="未识别到媒体信息"
)
return his
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Optional
from app import schemas
from app.domain.context import MusicInfo
from app.domain.media import normalize_media_source, resolve_media_identity
from app.schemas.media import normalize_media_source, resolve_media_identity
from app.runtime.extensions.service_registry import ServiceBaseHelper
from app.schemas import MediaServerConf, ServiceInfo
from app.schemas.types import (
+1 -1
View File
@@ -18,7 +18,7 @@ from app.runtime.config import global_vars
from app.domain.context import MediaInfo, MusicInfo, TorrentInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger
from app.schemas.message import Notification
from app.schemas.tmdb import TmdbEpisode
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import Optional
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.schemas.types import SystemConfigKey
+2 -2
View File
@@ -10,8 +10,8 @@ from app import schemas
from app.application.security import access as security
from app.runtime.config import settings
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import UserOper
from app.db.oper.systemconfig import SystemConfigOper
from app.db.oper.user import UserOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.schemas.types import SystemConfigKey
from app.foundation.singleton import Singleton
+1 -1
View File
@@ -1,7 +1,7 @@
from typing import List, Optional
from app import schemas
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.schemas.types import SystemConfigKey
+122
View File
@@ -0,0 +1,122 @@
"""
订阅的写入路径。
这两个函数把 MediaInfo / MusicInfo 翻译成一行订阅,是订阅表的唯一写入口。翻译此前
长在 SubscribeOper.add 上,但取标题、选海报尺寸、判音乐实体、决定哪几个字段构成一条
订阅的身份,都是订阅业务的规则而非数据访问——Oper 只该收敛查询,领域对象不该出现在
它的入参里。搬上来之后 SubscribeOper 收到的是纯粹的持久化字典,与
app/application/history.py 里整理历史的写入路径同构。
留在 Oper 的是列类型强转与建库时间戳:那几步是为 PostgreSQL 的严格类型检查和订阅表
自己的列类型而存在的,跟着列走比跟着调用方走更不容易漂。
字段映射错了不会报错,只会让订阅静静地记错——而搜索、洗版、完成判定、去重全都读这
张表。同步与异步是两份逐字复制的实现,改一条漏一条就是真实缺陷,故翻译与身份构造由
下方 _translate 单点承担,两条链路只在「怎么查、怎么写」上分叉。
"""
from typing import Optional, Tuple
from app.db.oper.subscribe import SubscribeOper
from app.domain.context import MediaInfo, MusicInfo
from app.schemas.media import resolve_media_identity
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaType
# 身份不完整时的固定返回。身份不全的订阅写进去就是一条永远匹配不上资源的僵尸订阅,
# 而后续按身份去重也会失效,所以必须在查询与建模之前短路
INCOMPLETE_IDENTITY = (0, "媒体身份不完整")
def _music_entity(mediainfo: MediaInfo | MusicInfo) -> Optional[str]:
"""
取音乐实体类型;非音乐媒体一律为空。
影视订阅带着 music_type 会被音乐去重逻辑当成音乐实体,造成串号。查重身份与写入
字段都取这一个值,避免两处各算一遍后悄悄分叉。
:param mediainfo: 识别结果
:return: 音乐实体类型,非音乐媒体为 None
"""
if mediainfo.type != MediaType.MUSIC:
return None
return getattr(mediainfo, "music_type", None)
def _translate(mediainfo: MediaInfo | MusicInfo,
kwargs: dict) -> Optional[Tuple[dict, dict, Optional[str]]]:
"""
把识别结果翻译成查重身份与写入字段。
:param mediainfo: 识别结果
:param kwargs: 调用方传入的订阅设置,媒体相关的同名字段会被识别结果覆盖
:return: (查重身份, 写入字段, 限定用户);媒体身份不完整时返回 None
"""
owner_scope = bool(kwargs.pop("owner_scope", False))
username = kwargs.get("username") if owner_scope else None
media_source, media_id = resolve_media_identity(
media=mediainfo,
media_source=kwargs.get("media_source"),
media_id=kwargs.get("media_id"),
)
if not media_source or not media_id:
return None
music_type = _music_entity(mediainfo)
identity = {
"media_source": str(media_source),
"media_id": media_id,
"music_type": music_type,
"season": kwargs.get("season"),
"episode_group": mediainfo.episode_group,
}
payload = dict(kwargs)
payload.update({
"name": mediainfo.title,
"year": mediainfo.year,
"type": mediainfo.type.value,
"media_source": str(media_source),
"media_id": media_id,
"episode_group": mediainfo.episode_group,
"poster": mediainfo.get_poster_image(),
"backdrop": mediainfo.get_backdrop_image(),
"vote": mediainfo.vote_average,
"description": mediainfo.overview,
"music_type": music_type,
# 整专完成判定拿 total_tracks 当分母,单曲带着专辑的曲目数会永远判不到完成
"total_tracks": getattr(mediainfo, "total_tracks", None)
if music_type == MUSIC_ENTITY_ALBUM else None,
})
return identity, payload, username
def add_subscribe(mediainfo: MediaInfo | MusicInfo,
subscribe_oper: Optional[SubscribeOper] = None,
**kwargs) -> Tuple[int, str]:
"""
新增订阅。
:param mediainfo: 识别结果
:param subscribe_oper: 复用的订阅操作对象,未传时新建
:param kwargs: 订阅设置;owner_scope 为真时按用户名限定查重范围
:return: (订阅 ID, 结果说明)ID 为 0 表示未新增
"""
translated = _translate(mediainfo, kwargs)
if translated is None:
return INCOMPLETE_IDENTITY
identity, payload, username = translated
oper = subscribe_oper or SubscribeOper()
return oper.add(identity=identity, payload=payload, username=username)
async def async_add_subscribe(mediainfo: MediaInfo | MusicInfo,
subscribe_oper: Optional[SubscribeOper] = None,
**kwargs) -> Tuple[int, str]:
"""
异步新增订阅。
:param mediainfo: 识别结果
:param subscribe_oper: 复用的订阅操作对象,未传时新建
:param kwargs: 订阅设置;owner_scope 为真时按用户名限定查重范围
:return: (订阅 ID, 结果说明)ID 为 0 表示未新增
"""
translated = _translate(mediainfo, kwargs)
if translated is None:
return INCOMPLETE_IDENTITY
identity, payload, username = translated
oper = subscribe_oper or SubscribeOper()
return await oper.async_add(identity=identity, payload=payload, username=username)
+3 -3
View File
@@ -13,12 +13,12 @@ from app.domain.context import Context, TorrentInfo, MediaInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format, parse_audio_quality
from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.site import SiteOper
from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger
from app.schemas.types import MediaType, SystemConfigKey
from app.adapters.network.http import RequestUtils
from app.domain.media import resolve_media_identity
from app.schemas.media import resolve_media_identity
from app.domain.string import StringUtils
+91
View File
@@ -0,0 +1,91 @@
"""
整理任务:整理链的进程内工作项。
TransferTask 此前住在 app/schemas/transfer.py,但它不是出网的 DTO——meta 装的是领域侧
的 MetaBase 子类,mediainfo 装的是领域侧的 MediaInfo / MusicInfo,都带行为而非纯数据。
放在 app.schemas 的代价是它没法命名自己真正装的类型:app.schemas 一旦 import 领域类型,
app.schemas -> app.schemas.transfer -> app.domain.* -> app.schemas.types -> app.schemas
就闭环,仓库自己的 test_migrated_modules_are_not_in_import_cycles 会红(已实测)。于是
两个字段只能标成 Optional[Any],把「这里到底能放什么」这件事整个交给了口头约定。
搬到应用层就没有这个约束:app.application 允许依赖 app.domain 与 app.schemas,两个
字段因此能标出真实类型。它面向前端的投影仍是 app/schemas/transfer.py 里的
TransferJob / TransferJobTask,那两个用 app.schemas 的同名 DTO——一个是工作项,一个是
视图,分开表达之后两边都不必再迁就对方。
"""
from pathlib import Path
from typing import Callable, List, Optional, Union
from pydantic import BaseModel, ConfigDict
from app.domain.context import MediaInfo, MusicInfo
from app.domain.meta.metabase import MetaBase
from app.schemas.file import FileItem
from app.schemas.history import DownloadHistory
from app.schemas.media import OptionalMediaIdentityMixin
from app.schemas.system import TransferDirectoryConf
from app.schemas.tmdb import TmdbEpisode
from app.schemas.transfer import TransferInfo
from app.schemas.types import MediaSource, MediaType
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""
文件整理任务。
"""
# MetaBase 与 MediaInfo / MusicInfo 都是普通类而非 BaseModelpydantic 需要显式放行
model_config = ConfigDict(arbitrary_types_allowed=True)
fileitem: FileItem
meta: Optional[MetaBase] = None
mediainfo: Optional[Union[MusicInfo, MediaInfo]] = None
media_source: Optional[MediaSource] = None
media_id: Optional[str] = None
mtype: Optional[MediaType] = None
target_directory: Optional[TransferDirectoryConf] = None
target_storage: Optional[str] = None
target_path: Optional[Path] = None
transfer_type: Optional[str] = None
scrape: Optional[bool] = False
library_type_folder: Optional[bool] = False
library_category_folder: Optional[bool] = False
episodes_info: Optional[List[TmdbEpisode]] = None
username: Optional[str] = None
downloader: Optional[str] = None
download_hash: Optional[str] = None
download_history: Optional[DownloadHistory] = None
transfer_batch_id: Optional[str] = None
manual: Optional[bool] = False
background: Optional[bool] = True
preview: Optional[bool] = False
def to_dict(self):
"""
返回字典。
meta 与 mediainfo 用 to_dict() 而非 model_dump():它们是领域对象,没有
model_dump。此前这里写的是 model_dump(),仓内无人调用才一直没炸——字段类型
标成 Any 时,这种错配静态检查也看不出来。
"""
dicts = vars(self).copy()
dicts["fileitem"] = self.fileitem.model_dump() if self.fileitem else None
dicts["meta"] = self.meta.to_dict() if self.meta else None
dicts["mediainfo"] = self.mediainfo.to_dict() if self.mediainfo else None
dicts["target_directory"] = self.target_directory.model_dump() if self.target_directory else None
return dicts
class TransferQueue(BaseModel):
"""
异步整理队列信息。
和 TransferTask 一起从 app/schemas 搬来:它装着一个 TransferTask 和一个回调函数,
回调根本不可序列化,因此从来就不是 DTO,只是恰好和视图模型住在同一个文件里。
"""
# 任务信息
task: Optional[TransferTask] = None
# 回调函数
callback: Optional[Callable] = None
# 整理结果
result: Optional[TransferInfo] = None