refactor(db): 修复异步连接池无界增长,并完成 SQLAlchemy 2.0 迁移与分层归位 (#6320)

This commit is contained in:
Aqr-K
2026-08-15 06:58:38 +08:00
committed by GitHub
parent e28de9cfe1
commit 8a11214a43
252 changed files with 11405 additions and 2889 deletions
+6 -6
View File
@@ -20,9 +20,9 @@ from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.runtime.extensions.module_manager import ModuleManager
from app.runtime.extensions.plugin_manager import PluginManager
from app.db.message_oper import MessageOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import UserOper
from app.db.oper.message import MessageOper
from app.db.oper.systemconfig import SystemConfigOper
from app.db.oper.user import UserOper
from app.application.messaging.message import MessageHelper, MessageQueueManager, MessageTemplateHelper
from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.extensions.service_registry import ServiceConfigHelper
@@ -42,7 +42,7 @@ from app.schemas import (
MessageResponse,
)
from app.foundation.identity import normalize_internal_user_id
from app.domain.media import resolve_media_identity
from app.schemas.media import resolve_media_identity
from app.schemas.message import ChannelCapability, ChannelCapabilityManager
from app.schemas.category import CategoryConfig
from app.schemas.types import (
@@ -1773,7 +1773,7 @@ class ChainBase(metaclass=ABCMeta):
self,
message: Optional[Notification] = None,
meta: Optional[MetaBase] = None,
mediainfo: Optional[MediaInfo] = None,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
torrentinfo: Optional[TorrentInfo] = None,
transferinfo: Optional[TransferInfo] = None,
**kwargs,
@@ -1889,7 +1889,7 @@ class ChainBase(metaclass=ABCMeta):
self,
message: Optional[Notification] = None,
meta: Optional[MetaBase] = None,
mediainfo: Optional[MediaInfo] = None,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
torrentinfo: Optional[TorrentInfo] = None,
transferinfo: Optional[TransferInfo] = None,
**kwargs,
+4 -4
View File
@@ -26,9 +26,9 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo
from app.db.downloadfailure_oper import DownloadFailureOper
from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.mediaserver_oper import MediaServerOper
from app.db.oper.downloadfailure import DownloadFailureOper
from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.oper.mediaserver import MediaServerOper
from app.application.directory import DirectoryHelper, validate_download_save_path
from app.runtime.thread import ThreadHelper
from app.application.torrent import TorrentHelper
@@ -38,7 +38,7 @@ from app.schemas import ExistMediaInfo, FileURI, NotExistMediaInfo, DownloaderTo
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, TorrentStatus, EventType, MessageChannel, NotificationType, ContentType, \
ChainEventType
from app.adapters.network.http import RequestUtils
from app.domain.media import build_media_key, resolve_media_identity
from app.schemas.media import build_media_key, resolve_media_identity
from app.domain.string import StringUtils
from app.adapters.system.host import SystemUtils
+8 -8
View File
@@ -37,11 +37,8 @@ from app.schemas.types import (
MediaSourceSelection,
MediaType,
)
from app.domain.media import (
is_music_media_source,
normalize_media_source,
resolve_media_identity,
)
from app.domain.media import is_music_media_source
from app.schemas.media import normalize_media_source, resolve_media_identity
from app.foundation.singleton import Singleton
from app.foundation.text import convert as zhconv_convert
from app.domain.string import StringUtils
@@ -643,9 +640,9 @@ class MediaChain(ChainBase, metaclass=Singleton):
def supplement_tmdb_info(
self,
mediainfo: Optional[MediaInfo],
mediainfo: Optional[Union[MediaInfo, MusicInfo]],
metainfo: Optional[MetaBase] = None,
) -> Optional[MediaInfo]:
) -> Optional[Union[MediaInfo, MusicInfo]]:
"""
为任意主识别源补充 TMDB 辅助信息,同时保留原始媒体身份。
@@ -655,7 +652,10 @@ class MediaChain(ChainBase, metaclass=Singleton):
"""
if not mediainfo:
return None
if mediainfo.type == MediaType.MUSIC:
# 音乐原样返回:下面全是 TMDB 影视字段,MusicInfo 上根本没有。用 isinstance
# 而不只看 type,一来静态检查能据此收窄(.type == 的比较收窄不了类型),二来
# type 没被正确赋值的 MusicInfo 也挡得住,不至于到下一行才 AttributeError
if isinstance(mediainfo, MusicInfo) or mediainfo.type == MediaType.MUSIC:
return mediainfo
if mediainfo.tmdb_id and mediainfo.tmdb_info and mediainfo.genre_ids:
return mediainfo
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import Callable, Dict, List, Union, Optional, Generator, Any
from app.chain import ChainBase
from app.runtime.config import global_vars
from app.db.mediaserver_oper import MediaServerOper
from app.db.oper.mediaserver import MediaServerOper
from app.runtime.extensions.service_registry import ServiceConfigHelper
from app.runtime.log import logger
from app.schemas import MediaServerLibrary, MediaServerItem, MediaServerSeasonInfo, MediaServerPlayItem
+3 -3
View File
@@ -26,8 +26,8 @@ from app.runtime.config import settings, global_vars
from app.domain.context import MediaInfo, Context
from app.domain.meta.metabase import MetaBase
from app.db.models import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper
from app.db.user_oper import UserOper
from app.db.oper.transferhistory import TransferHistoryOper
from app.db.oper.user import UserOper
from app.application.directory import DirectoryHelper
from app.application.messaging.interaction import (
agent_interaction_manager,
@@ -42,7 +42,7 @@ from app.schemas.message import ChannelCapabilityManager, ChannelCapability
from app.schemas.system import TransferDirectoryConf
from app.schemas.types import EventType, MessageChannel, MediaType
from app.adapters.network.http import RequestUtils
from app.domain.media import build_media_key, resolve_media_identity
from app.schemas.media import build_media_key, resolve_media_identity
from app.domain.string import StringUtils
+1 -1
View File
@@ -19,7 +19,7 @@ from app.schemas.types import (
MediaSource,
)
from app.runtime.execution import log_execution_time
from app.domain.media import normalize_media_source
from app.schemas.media import normalize_media_source
from app.foundation.singleton import Singleton
+3 -6
View File
@@ -26,7 +26,7 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo, MetaInfoPath
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.application.audio import AudioMetadataHelper
from app.runtime.log import logger
from app.schemas import FileItem
@@ -43,11 +43,8 @@ from app.schemas.types import (
SystemConfigKey,
)
from app.adapters.network.http import RequestUtils
from app.domain.media import (
is_music_media_source,
normalize_media_source,
resolve_media_identity,
)
from app.domain.media import is_music_media_source
from app.schemas.media import normalize_media_source, resolve_media_identity
from app.runtime.reload import ConfigReloadMixin
from app.foundation.singleton import Singleton
from app.domain.string import StringUtils
+2 -6
View File
@@ -21,7 +21,7 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo
from app.domain.context import MusicInfo
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.progress import ProgressHelper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.application.torrent import TorrentHelper
@@ -35,11 +35,7 @@ from app.schemas.types import (
ProgressKey,
SystemConfigKey,
)
from app.domain.media import (
build_media_key,
parse_media_key,
resolve_media_identity,
)
from app.schemas.media import build_media_key, parse_media_key, resolve_media_identity
from app.domain.string import StringUtils
from app.foundation.text import convert as zhconv_convert
+2 -2
View File
@@ -11,8 +11,8 @@ from app.chain import ChainBase
from app.runtime.config import global_vars, settings
from app.runtime.events import Event, eventmanager
from app.db.models.site import Site
from app.db.site_oper import SiteOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.site import SiteOper
from app.db.oper.systemconfig import SystemConfigOper
from app.adapters.network.browser import PlaywrightHelper
from app.adapters.network.cloudflare import under_challenge
from app.application.security.cookie import CookieHelper
+9 -12
View File
@@ -27,11 +27,11 @@ from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.meta.words import WordsMatcher
from app.domain.metainfo import MetaInfo
from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.models.subscribe import Subscribe
from app.db.site_oper import SiteOper
from app.db.subscribe_oper import SubscribeOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.site import SiteOper
from app.db.oper.subscribe import SubscribeOper
from app.db.oper.systemconfig import SystemConfigOper
from app.application.messaging.interaction import (
SlashInteractionManager,
build_navigation_buttons,
@@ -42,6 +42,7 @@ from app.application.messaging.interaction import (
update_or_post_message,
)
from app.application.mediaserver import MediaServerHelper
from app.application.subscribe import add_subscribe, async_add_subscribe
from app.adapters.external.server import MoviePilotServerHelper
from app.application.torrent import TorrentHelper
from app.runtime.log import logger
@@ -49,12 +50,8 @@ from app.schemas import (SubscribeEpisodesRefreshEventData,
SubscribeCompletionCheckEventData)
from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaSource, MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType, ChainEventType, \
ContentType
from app.domain.media import (
MUSIC_SUBSCRIBABLE_TYPES,
build_media_key,
normalize_media_source,
resolve_media_identity,
)
from app.domain.media import MUSIC_SUBSCRIBABLE_TYPES
from app.schemas.media import build_media_key, normalize_media_source, resolve_media_identity
subscribe_interaction_manager = SlashInteractionManager()
@@ -989,7 +986,7 @@ class SubscribeChain(ChainBase):
kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs))
# 操作数据库
sid, err_msg = SubscribeOper().add(mediainfo=mediainfo, season=season, username=username, **kwargs)
sid, err_msg = add_subscribe(mediainfo=mediainfo, season=season, username=username, **kwargs)
if not sid:
logger.error(f'{mediainfo.title_year} {err_msg}')
if not exist_ok and message:
@@ -1193,7 +1190,7 @@ class SubscribeChain(ChainBase):
kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs))
# 操作数据库
sid, err_msg = await SubscribeOper().async_add(mediainfo=mediainfo, season=season, username=username, **kwargs)
sid, err_msg = await async_add_subscribe(mediainfo=mediainfo, season=season, username=username, **kwargs)
if not sid:
logger.error(f'{mediainfo.title_year} {err_msg}')
if not exist_ok and message:
+3 -3
View File
@@ -12,14 +12,14 @@ from app.domain.context import TorrentInfo, Context, MediaInfo
from app.domain.context import MusicInfo
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper
from app.db.systemconfig_oper import SystemConfigOper
from app.db.oper.site import SiteOper
from app.db.oper.systemconfig import SystemConfigOper
from app.application.rss import RssHelper
from app.application.torrent import TorrentHelper
from app.runtime.log import logger
from app.schemas import Notification
from app.schemas.types import SystemConfigKey, MessageChannel, NotificationType, MediaType
from app.domain.media import resolve_media_identity
from app.schemas.media import resolve_media_identity
from app.domain.string import StringUtils
+54 -35
View File
@@ -22,17 +22,18 @@ from app.runtime.events import eventmanager
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfoPath
from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
from app.db.models.transferhistory import TransferHistory
from app.db.systemconfig_oper import SystemConfigOper
from app.db.transferpending_oper import TransferPendingOper
from app.db.transferhistory_oper import TransferHistoryOper
from app.db.oper.systemconfig import SystemConfigOper
from app.db.oper.transferpending import TransferPendingOper
from app.db.oper.transferhistory import TransferHistoryOper
from app.application.directory import DirectoryHelper
from app.application.audio import AudioMetadataHelper
from app.application.formatting import EpisodeFormatRuleHelper, FormatParser
from app.runtime.progress import ProgressHelper
from app.application.history import (clear_transfer_failures, describe_history_gate,
from app.application.history import (add_transfer_fail, add_transfer_success,
clear_transfer_failures, describe_history_gate,
evaluate_history_gate, is_skip_action,
record_transfer_failure, resolve_history)
from app.runtime.log import logger
@@ -43,8 +44,6 @@ from app.schemas import (
EpisodeFormat,
FileItem,
TransferDirectoryConf,
TransferTask,
TransferQueue,
TransferJob,
TransferJobTask,
TmdbEpisode,
@@ -65,11 +64,9 @@ from app.schemas.types import (
MediaSource,
)
from app.runtime.reload import ConfigReloadMixin
from app.domain.media import (
normalize_media_source,
normalize_music_type,
resolve_media_identity,
)
from app.application.transfer import TransferQueue, TransferTask
from app.domain.media import normalize_music_type
from app.schemas.media import normalize_media_source, resolve_media_identity
from app.foundation.singleton import Singleton
from app.domain.string import StringUtils
from app.adapters.system.host import SystemUtils
@@ -156,7 +153,8 @@ class JobManager:
return meta.name, season
@staticmethod
def __get_media_id(media: MediaInfo = None, season: Optional[int] = None) -> Tuple:
def __get_media_id(media: Optional[Union[MediaInfo, MusicInfo]] = None,
season: Optional[int] = None) -> Tuple:
"""
获取媒体ID音乐额外区分实体类型并为无远端ID的曲目构造稳定身份
"""
@@ -225,7 +223,7 @@ class JobManager:
return self.__get_id(task)
@staticmethod
def __get_media(task: TransferTask) -> schemas.MediaInfo:
def __get_media(task: TransferTask) -> Union[schemas.MediaInfo, schemas.MusicInfo]:
"""
获取媒体信息
"""
@@ -762,7 +760,7 @@ class JobManager:
)
def success_tasks(
self, media: MediaInfo, season: Optional[int] = None
self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
) -> List[TransferJobTask]:
"""
获取作业中所有成功的任务
@@ -789,7 +787,7 @@ class JobManager:
return []
return self._job_view[__mediaid__].tasks
def count(self, media: MediaInfo, season: Optional[int] = None) -> int:
def count(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
"""
获取作业中成功总数
"""
@@ -805,7 +803,7 @@ class JobManager:
]
)
def size(self, media: MediaInfo, season: Optional[int] = None) -> int:
def size(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
"""
获取作业中所有成功文件总大小
"""
@@ -858,7 +856,7 @@ class JobManager:
return list(self._job_view.values())
def season_episodes(
self, media: MediaInfo, season: Optional[int] = None
self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
) -> List[int]:
"""
获取作业的季集清单
@@ -1276,7 +1274,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self,
history: TransferHistory,
src_path: Path,
) -> Optional[MusicInfo]:
) -> Optional[Union[MusicInfo, MediaInfo]]:
"""
重新整理重试时恢复音乐信息
@@ -1466,7 +1464,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
)
# 新增转移失败历史记录
history = transferhis.add_fail(
history = add_transfer_fail(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
@@ -1474,6 +1472,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
)
# 整理失败事件
@@ -1586,7 +1585,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
)
# 新增task转移成功历史记录
history = transferhis.add_success(
history = add_transfer_success(
fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader,
@@ -1594,6 +1593,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
transfer_history_oper=transferhis,
)
# task整理完成事件
@@ -2285,7 +2285,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
try:
# 识别
transferhis = TransferHistoryOper()
mediainfo = task.mediainfo
# 显式标注联合:下面既会赋回音乐识别结果(MusicInfo),也会赋回影视识别
# 结果(MediaInfo),不标注时会被推断成其中一种,另一种就成了假错误
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = task.mediainfo
mediainfo_changed = False
need_obtain_images = False
if not mediainfo:
@@ -2302,8 +2304,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
and download_history.media_id
and not history_year_conflict
):
# 下载记录中已存在识别信息
mediainfo: Optional[MediaInfo] = MediaChain().recognize_media(
# 下载记录中已存在识别信息。这里不再重复标注类型:函数开头
# 已把 mediainfo 声明为 MediaInfo | MusicInfo | None,重复
# 声明会遮蔽它,把音乐识别结果判成类型错误
mediainfo = MediaChain().recognize_media(
mtype=task.mtype or MediaType(download_history.type),
media_source=download_history.media_source,
media_id=download_history.media_id,
@@ -2367,24 +2371,35 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
fileid=task.fileitem.fileid if task.fileitem else None,
)
# 新增整理失败历史记录
his = transferhis.add_fail(
his = add_transfer_fail(
fileitem=task.fileitem,
mode=task.transfer_type,
meta=task.meta,
downloader=task.downloader,
download_hash=task.download_hash,
transfer_history_oper=transferhis,
)
self.post_message(
Notification(
mtype=NotificationType.Manual,
title=f"{task.fileitem.name} 未识别到媒体信息,无法入库!",
text=(
"原因:未识别到媒体信息\n"
"如果按钮不可用,可回复:\n"
f"```\n/redo {his.id}\n"
f"/redo {his.id} [media_source]|[media_id]|[类型]\n```\n"
"自动重试或手动识别整理。"
),
# 历史落库失败时 his 为 Noneadd_transfer_fail 末尾的
# get_by_src 查不到即返回 None),此时 /redo 无 ID 可用,
# 只省去这段指引而不是让整条通知连同后续的作业清理、
# 种子完成标记一起崩在 NoneType 上
text="\n".join(
[
"原因:未识别到媒体信息",
(
"如果按钮不可用,可回复:\n"
f"```\n/redo {his.id}\n"
f"/redo {his.id} [media_source]|[media_id]|[类型]\n```\n"
"自动重试或手动识别整理。"
if his
else ""
),
]
).strip(),
username=task.username,
link=settings.MP_DOMAIN("#/history"),
buttons=self.build_failed_transfer_buttons(
@@ -3166,7 +3181,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
@staticmethod
def _is_movie_year_conflict(
file_meta: MetaBase, media: Union[DownloadHistory, MediaInfo]
file_meta: MetaBase,
# 两种 DownloadHistory 都会进来:库模型(本文件按 ORM 行查历史)与
# schemas DTOTransferTask.download_history)。本函数只按 getattr 取
# year 与 type,对两者一视同仁
media: Union[DownloadHistory, schemas.DownloadHistory, MediaInfo, MusicInfo]
) -> bool:
"""
判断文件名年份是否与已识别电影年份冲突
@@ -3450,7 +3469,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self,
fileitem: FileItem,
meta: MetaBase = None,
mediainfo: MediaInfo = None,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
mtype: Optional[MediaType] = None,
media_source: Optional[MediaSource] = None,
media_id: Optional[str] = None,
@@ -4686,7 +4705,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
def send_transfer_message(
self,
meta: MetaBase,
mediainfo: MediaInfo,
mediainfo: Union[MediaInfo, MusicInfo],
transferinfo: TransferInfo,
season_episode: Optional[str] = None,
episodes_info: Optional[List[TmdbEpisode]] = None,
+1 -1
View File
@@ -6,7 +6,7 @@ from app.chain import ChainBase
from app.runtime.config import settings
from app.application.security.access import get_password_hash, verify_password
from app.db.models.user import User
from app.db.user_oper import UserOper
from app.db.oper.user import UserOper
from app.runtime.log import logger
from app.schemas import AuthCredentials, AuthInterceptCredentials
from app.schemas.types import ChainEventType
+1 -1
View File
@@ -16,7 +16,7 @@ from app.chain import ChainBase
from app.runtime.config import global_vars
from app.runtime.events import Event, eventmanager
from app.db.models import Workflow
from app.db.workflow_oper import WorkflowOper
from app.db.oper.workflow import WorkflowOper
from app.runtime.log import logger
from app.schemas import ActionContext, ActionFlow, Action, ActionExecution, ActionResult
from app.schemas.types import EventType