refactor(db): 修复异步连接池无界增长,并完成 SQLAlchemy 2.0 迁移与分层归位 (#6320)

This commit is contained in:
Aqr-K
2026-08-15 06:58:38 +08:00
committed by GitHub
parent e28de9cfe1
commit 8a11214a43
252 changed files with 11405 additions and 2889 deletions
+3
View File
@@ -11,6 +11,9 @@ nginx/
test.py test.py
safety_report.txt safety_report.txt
app/application/site/*.bin app/application/site/*.bin
# 站点数据的运行期下载产物。上游 v3 架构重构后落点从 app/application/site 移到了
# app/helper,同目录的 .so/.pyd 由上面的通配兜住,只有 .bin 漏了网
app/helper/*.bin
app/plugins/** app/plugins/**
!app/plugins/__init__.py !app/plugins/__init__.py
config/cookies/ config/cookies/
+1 -1
View File
@@ -30,7 +30,7 @@ from requests import Response
from app.runtime.cache import cached, is_fresh from app.runtime.cache import cached, is_fresh
from app.runtime.config import settings from app.runtime.config import settings
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.adapters.system.package import PackageInstallRequest, build_package_install_strategies from app.adapters.system.package import PackageInstallRequest, build_package_install_strategies
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+5 -4
View File
@@ -9,9 +9,9 @@ from app.runtime.cache import cached
from app.runtime.config import settings from app.runtime.config import settings
from app.domain.context import MediaInfo, MusicInfo from app.domain.context import MediaInfo, MusicInfo
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.workflow_oper import WorkflowOper from app.db.oper.workflow import WorkflowOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import ( from app.schemas.types import (
MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_RECORDING,
@@ -20,7 +20,8 @@ from app.schemas.types import (
media_type_to_agent, media_type_to_agent,
) )
from app.adapters.network.http import AsyncRequestUtils, RequestUtils from app.adapters.network.http import AsyncRequestUtils, RequestUtils
from app.domain.media import normalize_music_type, resolve_media_identity from app.domain.media import normalize_music_type
from app.schemas.media import resolve_media_identity
from app.adapters.system.host import SystemUtils from app.adapters.system.host import SystemUtils
from version import APP_VERSION, FRONTEND_VERSION from version import APP_VERSION, FRONTEND_VERSION
+1 -1
View File
@@ -21,7 +21,7 @@ import httpx
import jwt import jwt
from app.runtime.config import settings from app.runtime.config import settings
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
+1 -1
View File
@@ -12,7 +12,7 @@ from dataclasses import dataclass
from typing import Any, Optional from typing import Any, Optional
from urllib.parse import urljoin from urllib.parse import urljoin
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.agent import ( from app.schemas.agent import (
AgentMcpServerConfig, AgentMcpServerConfig,
+1 -1
View File
@@ -7,7 +7,7 @@ from typing import Dict, List, Optional
from langchain_core.messages import BaseMessage, messages_from_dict, messages_to_dict from langchain_core.messages import BaseMessage, messages_from_dict, messages_to_dict
from app.runtime.config import settings from app.runtime.config import settings
from app.db.agentchat_oper import AgentChatOper from app.db.oper.agentchat import AgentChatOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.agent import ConversationMemory from app.schemas.agent import ConversationMemory
+3 -3
View File
@@ -72,9 +72,9 @@ from app.chain import ChainBase
from app.runtime.config import settings from app.runtime.config import settings
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.runtime.extensions.plugin_manager import PluginManager from app.runtime.extensions.plugin_manager import PluginManager
from app.db.agentchat_oper import AgentChatOper from app.db.oper.agentchat import AgentChatOper
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import AgentLLMProviderEventData, AgentTokensUsageEventData, Notification, NotificationType from app.schemas import AgentLLMProviderEventData, AgentTokensUsageEventData, Notification, NotificationType
from app.schemas.message import ChannelCapabilityManager, ChannelCapability from app.schemas.message import ChannelCapabilityManager, ChannelCapability
+2 -2
View File
@@ -5,8 +5,8 @@ import re
from typing import Any, Dict, Iterable, Optional from typing import Any, Dict, Iterable, Optional
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.filter import RuleHelper from app.application.filter import RuleHelper
from app.modules.filter.RuleParser import RuleParser from app.modules.filter.RuleParser import RuleParser
from app.modules.filter.builtin_rules import BUILTIN_RULE_SET from app.modules.filter.builtin_rules import BUILTIN_RULE_SET
+1 -1
View File
@@ -6,7 +6,7 @@ from typing import Any, Optional
from app.runtime.config import settings from app.runtime.config import settings
from app.runtime.extensions.plugin_manager import PluginManager from app.runtime.extensions.plugin_manager import PluginManager
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.adapters.external.market import PluginHelper from app.adapters.external.market import PluginHelper
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+1 -1
View File
@@ -15,7 +15,7 @@ from app.chain.search import SearchChain
from app.runtime.config import settings from app.runtime.config import settings
from app.domain.context import Context from app.domain.context import Context
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.application.directory import DirectoryHelper, validate_download_save_path from app.application.directory import DirectoryHelper, validate_download_save_path
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import FileURI from app.schemas import FileURI
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.subscribe import SubscribeChain from app.chain.subscribe import SubscribeChain
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, MessageChannel from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, MessageChannel
from ._music_utils import normalize_music_type from ._music_utils import normalize_music_type
+2 -2
View File
@@ -8,8 +8,8 @@ from pydantic import BaseModel, Field, model_validator
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.config import settings from app.runtime.config import settings
from app.db.agentchat_oper import AgentChatOper from app.db.oper.agentchat import AgentChatOper
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
from app.runtime.scheduling import TimerUtils from app.runtime.scheduling import TimerUtils
+1 -1
View File
@@ -4,7 +4,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
class DeleteAgentTaskInput(BaseModel): class DeleteAgentTaskInput(BaseModel):
@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.oper.downloadhistory import DownloadHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import EventType from app.schemas.types import EventType
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.storage import StorageChain from app.chain.storage import StorageChain
from app.db.transferhistory_oper import TransferHistoryOper from app.db.oper.transferhistory import TransferHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import FileItem from app.schemas import FileItem
+1 -1
View File
@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.config import settings from app.runtime.config import settings
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
class QueryAgentTasksInput(BaseModel): class QueryAgentTasksInput(BaseModel):
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.download import DownloadChain from app.chain.download import DownloadChain
from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.oper.downloadhistory import DownloadHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import DownloaderTorrent from app.schemas import DownloaderTorrent
from app.schemas.types import MUSIC_ENTITY_RECORDING, TorrentQueryStatus, media_type_to_agent from app.schemas.types import MUSIC_ENTITY_RECORDING, TorrentQueryStatus, media_type_to_agent
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+1 -1
View File
@@ -12,7 +12,7 @@ from app.agent.tools.impl._plugin_tool_utils import (
build_preview_payload, build_preview_payload,
get_plugin_snapshot, get_plugin_snapshot,
) )
from app.db.plugindata_oper import PluginDataOper from app.db.oper.plugindata import PluginDataOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.runtime.log import logger from app.runtime.log import logger
SITE_USERDATA_DETAIL_PREVIEW_LIMIT = 10 SITE_USERDATA_DETAIL_PREVIEW_LIMIT = 10
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.runtime.log import logger from app.runtime.log import logger
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.subscribehistory_oper import SubscribeHistoryOper from app.db.oper.subscribehistory import SubscribeHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaType, media_type_to_agent from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaType, media_type_to_agent
from ._music_utils import normalize_music_type from ._music_utils import normalize_music_type
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.subscribe import Subscribe as SubscribeSchema from app.schemas.subscribe import Subscribe as SubscribeSchema
from app.schemas.types import ( from app.schemas.types import (
@@ -16,7 +16,7 @@ from app.agent.tools.impl._system_setting_utils import (
should_redact_setting, should_redact_setting,
) )
from app.runtime.config import settings from app.runtime.config import settings
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.transferhistory_oper import TransferHistoryOper from app.db.oper.transferhistory import TransferHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import media_type_to_agent from app.schemas.types import media_type_to_agent
from app.foundation.text import cut as jieba_cut from app.foundation.text import cut as jieba_cut
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.workflow_oper import WorkflowOper from app.db.oper.workflow import WorkflowOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -6,7 +6,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
class RunAgentTaskInput(BaseModel): class RunAgentTaskInput(BaseModel):
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.workflow import WorkflowChain from app.chain.workflow import WorkflowChain
from app.db.workflow_oper import WorkflowOper from app.db.oper.workflow import WorkflowOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -19,7 +19,7 @@ from app.schemas.types import (
MediaType, MediaType,
media_type_to_agent, media_type_to_agent,
) )
from app.domain.media import normalize_media_source from app.schemas.media import normalize_media_source
from ._music_utils import normalize_music_type, simplify_music_info from ._music_utils import normalize_music_type, simplify_music_info
+1 -1
View File
@@ -10,7 +10,7 @@ from app.agent.tools.tags import ToolTag
from app.chain.media import MediaChain from app.chain.media import MediaChain
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MediaType, media_type_to_agent from app.schemas.types import MediaType, media_type_to_agent
from app.domain.media import resolve_media_identity from app.schemas.media import resolve_media_identity
from ._music_utils import normalize_music_type, simplify_music_info from ._music_utils import normalize_music_type, simplify_music_info
@@ -11,7 +11,7 @@ from app.chain.douban import DoubanChain
from app.chain.tmdb import TmdbChain from app.chain.tmdb import TmdbChain
from app.chain.bangumi import BangumiChain from app.chain.bangumi import BangumiChain
from app.runtime.log import logger from app.runtime.log import logger
from app.domain.media import resolve_media_identity from app.schemas.media import resolve_media_identity
class SearchPersonCreditsInput(BaseModel): class SearchPersonCreditsInput(BaseModel):
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.subscribe import SubscribeChain from app.chain.subscribe import SubscribeChain
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import media_type_to_agent from app.schemas.types import media_type_to_agent
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.search import SearchChain from app.chain.search import SearchChain
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MediaSource, MediaType, SystemConfigKey from app.schemas.types import MediaSource, MediaType, SystemConfigKey
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.site import SiteChain from app.chain.site import SiteChain
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field, model_validator
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.config import settings from app.runtime.config import settings
from app.db.agenttask_oper import AgentTaskOper from app.db.oper.agenttask import AgentTaskOper
from app.runtime.scheduling import TimerUtils from app.runtime.scheduling import TimerUtils
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.domain.metainfo import clear_rust_parse_options_cache from app.domain.metainfo import clear_rust_parse_options_cache
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import EventType from app.schemas.types import EventType
from app.domain.string import StringUtils from app.domain.string import StringUtils
+1 -1
View File
@@ -7,7 +7,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.chain.site import SiteChain from app.chain.site import SiteChain
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.runtime.log import logger from app.runtime.log import logger
+1 -1
View File
@@ -8,7 +8,7 @@ from pydantic import BaseModel, Field
from app.agent.tools.base import MoviePilotTool from app.agent.tools.base import MoviePilotTool
from app.agent.tools.tags import ToolTag from app.agent.tools.tags import ToolTag
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.event import SubscribeModifiedEventData from app.schemas.event import SubscribeModifiedEventData
from app.schemas.types import EventType, media_type_to_agent from app.schemas.types import EventType, media_type_to_agent
@@ -18,7 +18,7 @@ from app.agent.tools.impl._system_setting_utils import (
) )
from app.runtime.config import settings from app.runtime.config import settings
from app.runtime.events import eventmanager from app.runtime.events import eventmanager
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.event import ConfigChangeEventData from app.schemas.event import ConfigChangeEventData
from app.schemas.types import EventType from app.schemas.types import EventType
+8 -79
View File
@@ -1,12 +1,18 @@
from typing import Optional, List """
API 层的公共依赖
这些是 FastAPI 的路由依赖从令牌解出用户校验激活状态与权限失败一律以
HTTPException 表达它们此前住在 app/db/oper/user.py 与数据访问混在一处
鉴权是 HTTP 层的关注点产出的是 403/400 而不是数据放在 db 包里既让数据层反向
依赖了 fastapi也使这部分逻辑无法与数据访问分开度量
"""
from fastapi import Depends, HTTPException from fastapi import Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from app import schemas from app import schemas
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db import DbOper, get_db, get_async_db from app.db import get_async_db, get_db
from app.db.models.user import User from app.db.models.user import User
@@ -112,80 +118,3 @@ async def get_current_active_superuser_async(
status_code=400, detail="用户权限不足" status_code=400, detail="用户权限不足"
) )
return current_user return current_user
class UserOper(DbOper):
"""
用户管理
"""
def list(self) -> List[User]:
"""
获取用户列表
"""
return User.list(self._db)
def add(self, **kwargs):
"""
新增用户
"""
user = User(**kwargs)
user.create(self._db)
def get_by_name(self, name: str) -> User:
"""
根据用户名获取用户
"""
return User.get_by_name(self._db, name)
async def async_get_by_name(self, name: str) -> User:
"""
异步根据用户名获取用户
"""
return await User.async_get_by_name(self._db, name)
async def async_get_by_id(self, user_id: int) -> User:
"""
异步根据用户 ID 获取用户
"""
return await User.async_get_by_id(self._db, user_id)
def get_permissions(self, name: str) -> dict:
"""
获取用户权限
"""
user = User.get_by_name(self._db, name)
if user:
return user.permissions or {}
return {}
def get_settings(self, name: str) -> Optional[dict]:
"""
获取用户个性化设置返回None表示用户不存在
"""
user = User.get_by_name(self._db, name)
if user:
return user.settings or {}
return None
def get_setting(self, name: str, key: str) -> Optional[str]:
"""
获取用户个性化设置
"""
settings = self.get_settings(name)
if settings:
return settings.get(key)
return None
def get_name(self, **kwargs) -> Optional[str]:
"""
根据绑定账号获取用户名称
"""
users = self.list()
for user in users:
user_setting = user.settings
if user_setting:
for k, v in kwargs.items():
if user_setting.get(k) == str(v):
return user.name
return None
+3 -2
View File
@@ -32,10 +32,11 @@ from app.command import Command
from app.runtime.config import global_vars, settings from app.runtime.config import global_vars, settings
from app.runtime.events import Event, EventManager from app.runtime.events import Event, EventManager
from app.db import get_async_db from app.db import get_async_db
from app.db.agentchat_oper import AgentChatOper from app.db.oper.agentchat import AgentChatOper
from app.db.models import User from app.db.models import User
from app.db.models.agentchat import AgentChat from app.db.models.agentchat import AgentChat
from app.db.user_oper import UserOper, get_current_active_user from app.db.oper.user import UserOper
from app.api.deps import get_current_active_user
from app.application.messaging.agent import attach_web_agent_edit_queue, detach_web_agent_edit_queue from app.application.messaging.agent import attach_web_agent_edit_queue, detach_web_agent_edit_queue
from app.application.messaging.interaction import agent_interaction_manager, media_interaction_manager from app.application.messaging.interaction import agent_interaction_manager, media_interaction_manager
from app.runtime.localization import LocaleHelper from app.runtime.localization import LocaleHelper
+1 -1
View File
@@ -12,7 +12,7 @@ from app.runtime.config import settings
from app.application.security.access import verify_apitoken from app.application.security.access import verify_apitoken
from app.db import get_db from app.db import get_db
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.user_oper import get_current_active_superuser from app.api.deps import get_current_active_superuser
from app.application.directory import DirectoryHelper from app.application.directory import DirectoryHelper
from app.scheduler import Scheduler from app.scheduler import Scheduler
from app.adapters.system.host import SystemUtils from app.adapters.system.host import SystemUtils
+3 -3
View File
@@ -11,9 +11,9 @@ from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db.models.user import User from app.db.models.user import User
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import get_current_active_user from app.api.deps import get_current_active_user
from app.application.directory import DirectoryHelper from app.application.directory import DirectoryHelper
from app.schemas.types import ( from app.schemas.types import (
MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_RECORDING,
+1 -1
View File
@@ -22,7 +22,7 @@ from app.db import get_async_db, get_db
from app.db.models import User from app.db.models import User
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.user_oper import ( from app.api.deps import (
get_current_active_manage_user, get_current_active_manage_user,
get_current_active_superuser, get_current_active_superuser,
get_current_active_superuser_async, get_current_active_superuser_async,
+1 -4
View File
@@ -15,10 +15,7 @@ from app.agent.llm import (
) )
from app.runtime.config import settings from app.runtime.config import settings
from app.db.models import User from app.db.models import User
from app.db.user_oper import ( from app.api.deps import get_current_active_superuser_async, get_current_active_user_async
get_current_active_superuser_async,
get_current_active_user_async,
)
from app.runtime.log import logger from app.runtime.log import logger
router = ResponseAPIRouter() router = ResponseAPIRouter()
+1 -1
View File
@@ -10,7 +10,7 @@ from app.api.response import RAW_RESPONSE_OPENAPI_KEY, ResponseAPIRouter
from app.chain.user import MfaRequired, UserChain from app.chain.user import MfaRequired, UserChain
from app.application.security import access as security from app.application.security import access as security
from app.runtime.config import settings from app.runtime.config import settings
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.application.image import WallpaperHelper from app.application.image import WallpaperHelper
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+3 -8
View File
@@ -17,17 +17,12 @@ from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo, MetaInfoPath from app.domain.metainfo import MetaInfo, MetaInfoPath
from app.application.security.access import verify_token, verify_apitoken from app.application.security.access import verify_token, verify_apitoken
from app.db.models import User from app.db.models import User
from app.db.user_oper import get_current_active_user, get_current_active_superuser from app.api.deps import get_current_active_user, get_current_active_superuser
from app.schemas import MediaType from app.schemas import MediaType
from app.schemas.category import CategoryConfig from app.schemas.category import CategoryConfig
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource
from app.domain.media import ( from app.domain.media import is_music_media_source, normalize_music_type, parse_media_source_selection
is_music_media_source, from app.schemas.media import normalize_media_source, resolve_media_identity
normalize_media_source,
normalize_music_type,
parse_media_source_selection,
resolve_media_identity,
)
router = ResponseAPIRouter() router = ResponseAPIRouter()
+3 -3
View File
@@ -11,13 +11,13 @@ from app.domain.context import MediaInfo
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db import get_async_db from app.db import get_async_db
from app.db.mediaserver_oper import MediaServerOper from app.db.oper.mediaserver import MediaServerOper
from app.db.models import MediaServerItem from app.db.models import MediaServerItem
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.mediaserver import MediaServerHelper from app.application.mediaserver import MediaServerHelper
from app.schemas import MediaType, NotExistMediaInfo from app.schemas import MediaType, NotExistMediaInfo
from app.schemas.types import MediaSource, SystemConfigKey from app.schemas.types import MediaSource, SystemConfigKey
from app.domain.media import build_media_key, resolve_media_identity from app.schemas.media import build_media_key, resolve_media_identity
router = ResponseAPIRouter() router = ResponseAPIRouter()
+3 -3
View File
@@ -14,9 +14,9 @@ from app.runtime.config import settings, global_vars
from app.application.security.access import verify_token, verify_apitoken from app.application.security.access import verify_token, verify_apitoken
from app.db import get_async_db from app.db import get_async_db
from app.db.models import User from app.db.models import User
from app.db.message_oper import MessageOper from app.db.oper.message import MessageOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import get_current_active_superuser from app.api.deps import get_current_active_superuser
from app.runtime.extensions.service_registry import ServiceConfigHelper from app.runtime.extensions.service_registry import ServiceConfigHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.modules.wechat.WXBizMsgCrypt3 import WXBizMsgCrypt from app.modules.wechat.WXBizMsgCrypt3 import WXBizMsgCrypt
+2 -2
View File
@@ -18,8 +18,8 @@ from app.runtime.config import settings
from app.db import get_async_db from app.db import get_async_db
from app.db.models.passkey import PassKey from app.db.models.passkey import PassKey
from app.db.models.user import User from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import get_current_active_user, get_current_active_user_async from app.api.deps import get_current_active_user, get_current_active_user_async
from app.application.security.passkey import ( from app.application.security.passkey import (
PassKeyHelper, PassKeyHelper,
PassKeyRegistrationOriginMismatchError, PassKeyRegistrationOriginMismatchError,
+1 -1
View File
@@ -10,7 +10,7 @@ from app.schemas.types import MediaSource, MediaType
from app.domain.context import MusicAlbumInfo, MusicArtistInfo, MusicInfo from app.domain.context import MusicAlbumInfo, MusicArtistInfo, MusicInfo
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db.models.user import User from app.db.models.user import User
from app.db.user_oper import get_current_active_superuser_async from app.api.deps import get_current_active_superuser_async
from app.modules.listenbrainz import ( from app.modules.listenbrainz import (
LISTENBRAINZ_CHART_RANGES, LISTENBRAINZ_CHART_RANGES,
LISTENBRAINZ_FRESH_MAX_DAYS, LISTENBRAINZ_FRESH_MAX_DAYS,
+1 -1
View File
@@ -6,7 +6,7 @@ from app import schemas
from app.api.response import ResponseAPIRouter from app.api.response import ResponseAPIRouter
from app.runtime.extensions.module_manager import ModuleManager from app.runtime.extensions.module_manager import ModuleManager
from app.db.models import User from app.db.models import User
from app.db.user_oper import get_current_active_superuser from app.api.deps import get_current_active_superuser
from app.modules.wechatclawbot.wechatclawbot import WechatClawBot from app.modules.wechatclawbot.wechatclawbot import WechatClawBot
router = ResponseAPIRouter() router = ResponseAPIRouter()
+2 -5
View File
@@ -24,11 +24,8 @@ from app.application.security.access import (
verify_token, verify_token,
) )
from app.db.models import User from app.db.models import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import ( from app.api.deps import get_current_active_superuser, get_current_active_superuser_async
get_current_active_superuser,
get_current_active_superuser_async,
)
from app.factory import app from app.factory import app
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.adapters.external.market import PluginHelper from app.adapters.external.market import PluginHelper
+2 -1
View File
@@ -14,7 +14,8 @@ from app.application.security.access import verify_resource_token, verify_token
from app.runtime.localization import LocaleHelper from app.runtime.localization import LocaleHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MediaSource, MediaType from app.schemas.types import MediaSource, MediaType
from app.domain.media import normalize_music_type, resolve_media_identity from app.domain.media import normalize_music_type
from app.schemas.media import resolve_media_identity
from app.application.security.url import SecurityUtils from app.application.security.url import SecurityUtils
router = ResponseAPIRouter() router = ResponseAPIRouter()
+3 -3
View File
@@ -20,9 +20,9 @@ from app.db.models.site import Site
from app.db.models.siteicon import SiteIcon from app.db.models.siteicon import SiteIcon
from app.db.models.sitestatistic import SiteStatistic from app.db.models.sitestatistic import SiteStatistic
from app.db.models.siteuserdata import SiteUserData from app.db.models.siteuserdata import SiteUserData
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import ( from app.api.deps import (
get_current_active_manage_user, get_current_active_manage_user,
get_current_active_manage_user_async, get_current_active_manage_user_async,
get_current_active_superuser, get_current_active_superuser,
+1 -1
View File
@@ -15,7 +15,7 @@ from app.chain.transfer import TransferChain
from app.runtime.config import settings from app.runtime.config import settings
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db.models import User from app.db.models import User
from app.db.user_oper import ( from app.api.deps import (
get_current_active_manage_user, get_current_active_manage_user,
get_current_active_superuser, get_current_active_superuser,
get_current_active_superuser_async, get_current_active_superuser_async,
+3 -3
View File
@@ -17,8 +17,8 @@ from app.db import get_async_db, get_db
from app.db.models.subscribe import Subscribe from app.db.models.subscribe import Subscribe
from app.db.models.subscribehistory import SubscribeHistory from app.db.models.subscribehistory import SubscribeHistory
from app.db.models.user import User from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import get_current_active_user, get_current_active_user_async from app.api.deps import get_current_active_user, get_current_active_user_async
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.scheduler import Scheduler from app.scheduler import Scheduler
@@ -31,7 +31,7 @@ from app.schemas.types import (
EventType, EventType,
SystemConfigKey, SystemConfigKey,
) )
from app.domain.media import normalize_media_source, resolve_media_identity from app.schemas.media import normalize_media_source, resolve_media_identity
router = ResponseAPIRouter() router = ResponseAPIRouter()
+2 -6
View File
@@ -29,12 +29,8 @@ from app.domain.metainfo import MetaInfo
from app.runtime.extensions.module_manager import ModuleManager from app.runtime.extensions.module_manager import ModuleManager
from app.application.security.access import verify_apitoken, verify_resource_token, verify_token from app.application.security.access import verify_apitoken, verify_resource_token, verify_token
from app.db.models import User from app.db.models import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import ( from app.api.deps import get_current_active_superuser, get_current_active_superuser_async, get_current_active_user_async
get_current_active_superuser,
get_current_active_superuser_async,
get_current_active_user_async,
)
from app.application.image import ImageHelper from app.application.image import ImageHelper
from app.runtime.localization import LocaleHelper from app.runtime.localization import LocaleHelper
from app.adapters.external.market import ( from app.adapters.external.market import (
+2 -2
View File
@@ -8,8 +8,8 @@ from app.chain.tmdb import TmdbChain
from app.runtime.config import settings from app.runtime.config import settings
from app.application.security.access import verify_token from app.application.security.access import verify_token
from app.db.models.user import User from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import get_current_active_superuser_async from app.api.deps import get_current_active_superuser_async
from app.modules.themoviedb.tmdb_cache import TmdbCache from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.schemas.types import MediaType, SystemConfigKey from app.schemas.types import MediaType, SystemConfigKey
+3 -9
View File
@@ -11,10 +11,7 @@ from app.domain.context import MediaInfo, MusicInfo
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.models import User from app.db.models import User
from app.db.user_oper import ( from app.api.deps import get_current_active_superuser, get_current_active_superuser_async
get_current_active_superuser,
get_current_active_superuser_async,
)
from app.schemas.types import ( from app.schemas.types import (
MUSIC_ENTITY_RECORDING, MUSIC_ENTITY_RECORDING,
MediaSource, MediaSource,
@@ -22,11 +19,8 @@ from app.schemas.types import (
MusicTargetEntityType, MusicTargetEntityType,
) )
from app.foundation.crypto import HashUtils from app.foundation.crypto import HashUtils
from app.domain.media import ( from app.domain.media import is_music_media_source, normalize_music_type
is_music_media_source, from app.schemas.media import resolve_media_identity
normalize_music_type,
resolve_media_identity,
)
router = ResponseAPIRouter() router = ResponseAPIRouter()
+1 -4
View File
@@ -13,10 +13,7 @@ from app.application.security.access import verify_token, verify_apitoken
from app.db import get_db from app.db import get_db
from app.db.models import User from app.db.models import User
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.user_oper import ( from app.api.deps import get_current_active_manage_user, get_current_active_superuser
get_current_active_manage_user,
get_current_active_superuser,
)
from app.application.directory import DirectoryHelper from app.application.directory import DirectoryHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import ( from app.schemas import (
+2 -6
View File
@@ -10,12 +10,8 @@ from app.api.response import ResponseAPIRouter
from app.application.security.access import get_password_hash from app.application.security.access import get_password_hash
from app.db import get_async_db from app.db import get_async_db
from app.db.models.user import User from app.db.models.user import User
from app.db.user_oper import ( from app.api.deps import get_current_active_superuser_async, get_current_active_user_async, get_current_active_user
get_current_active_superuser_async, from app.db.oper.userconfig import UserConfigOper
get_current_active_user_async,
get_current_active_user,
)
from app.db.userconfig_oper import UserConfigOper
router = ResponseAPIRouter() router = ResponseAPIRouter()
+3 -6
View File
@@ -14,12 +14,9 @@ from app.runtime.extensions.plugin_manager import PluginManager
from app.workflow import WorkFlowManager from app.workflow import WorkFlowManager
from app.db import get_async_db, get_db from app.db import get_async_db, get_db
from app.db.models import Workflow, User from app.db.models import Workflow, User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import ( from app.api.deps import get_current_active_manage_user, get_current_active_manage_user_async
get_current_active_manage_user, from app.db.oper.workflow import WorkflowOper
get_current_active_manage_user_async,
)
from app.db.workflow_oper import WorkflowOper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.scheduler import Scheduler from app.scheduler import Scheduler
from app.schemas.types import EventType, EVENT_TYPE_NAMES from app.schemas.types import EventType, EVENT_TYPE_NAMES
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import List, Optional, Tuple
from app import schemas from app import schemas
from app.domain.context import MediaInfo from app.domain.context import MediaInfo
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MediaType, StorageSchema, SystemConfigKey from app.schemas.types import MediaType, StorageSchema, SystemConfigKey
from app.adapters.system.host import SystemUtils from app.adapters.system.host import SystemUtils
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import List, Optional from typing import List, Optional
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.domain.context import MediaInfo from app.domain.context import MediaInfo
from app.schemas import CustomRule, FilterRuleGroup from app.schemas import CustomRule, FilterRuleGroup
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+163 -2
View File
@@ -1,10 +1,16 @@
from typing import Any, Dict, Optional from typing import Any, Dict, Optional, Union
from app.domain.context import MediaInfo, MusicInfo
from app.schemas.media import resolve_media_identity
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic
from app.runtime.cache import TTLCache from app.runtime.cache import TTLCache
from app.runtime.config import settings from app.runtime.config import settings
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper from app.db.oper.transferhistory import TransferHistoryOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import FileItem, TransferInfo
from app.schemas.types import MUSIC_ENTITY_RECORDING
# 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败) # 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多 # 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多
@@ -420,3 +426,158 @@ def describe_history_gate(history: Optional[TransferHistory],
if recorded_size is None and current_size is None: if recorded_size is None and current_size is None:
return f"成功记录 #{history.id},大小不可比对" return f"成功记录 #{history.id},大小不可比对"
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}" return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"
# --------------------------------------------------------------------------- #
# 整理历史的写入路径
#
# 这两个函数把 FileItem / MetaBase / MediaInfo / TransferInfo 四个领域对象翻译成
# 一行整理历史,是整理历史表的唯一写入口。它们此前长在 TransferHistoryOper 上,但
# 拼标题、拆季集、取海报、判音乐字段都是整理链的业务规则而非数据访问——Oper 只该
# 收敛查询,领域对象不该出现在它的入参里。搬到本模块与查重闸(读侧)作伴:同一张
# 表的读写规则放在一起,字段含义只有一处需要维护。
# --------------------------------------------------------------------------- #
def _history_title(meta: MetaBase,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None) -> Optional[str]:
"""音乐文件优先记录曲目标题,其它媒体保持识别标题。"""
if isinstance(meta, MetaMusic) and meta.title:
return meta.title
if mediainfo and mediainfo.title:
return mediainfo.title
return meta.name
def add_transfer_success(fileitem: FileItem, mode: str, meta: MetaBase,
mediainfo: Union[MediaInfo, MusicInfo], transferinfo: TransferInfo,
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
新增转移成功历史记录
:param fileitem: 源文件项
:param mode: 整理方式
:param meta: 文件名识别结果
:param mediainfo: 媒体识别结果
:param transferinfo: 整理结果
:param downloader: 下载器
:param download_hash: 种子 hash
:param transfer_history_oper: 复用的历史操作对象未传时新建
:return: 落库后的整理记录
"""
oper = transfer_history_oper or TransferHistoryOper()
media_source, media_id = resolve_media_identity(media=mediainfo)
return oper.add_force(
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
dest=transferinfo.target_item.path if transferinfo.target_item else None,
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
mode=mode,
type=mediainfo.type.value,
category=mediainfo.category,
title=_history_title(meta, mediainfo),
year=mediainfo.year,
media_source=media_source,
media_id=media_id,
music_type=getattr(mediainfo, "music_type", None),
total_tracks=getattr(mediainfo, "total_tracks", None),
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
seasons=meta.season,
episodes=meta.episode,
image=mediainfo.get_poster_image(),
downloader=downloader,
download_hash=download_hash,
status=1,
files=transferinfo.file_list
)
def add_transfer_fail(fileitem: FileItem, mode: str, meta: MetaBase,
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
transferinfo: Optional[TransferInfo] = None,
downloader: Optional[str] = None,
download_hash: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
新增转移失败历史记录
识别结果与整理结果齐备时按完整字段落库缺任一项则走未识别到媒体信息分支
此时只有文件名解析出的元数据可用不写目标路径
:param fileitem: 源文件项
:param mode: 整理方式
:param meta: 文件名识别结果
:param mediainfo: 媒体识别结果未识别时为 None
:param transferinfo: 整理结果未进入整理时为 None
:param downloader: 下载器
:param download_hash: 种子 hash
:param transfer_history_oper: 复用的历史操作对象未传时新建
:return: 落库后的整理记录
"""
oper = transfer_history_oper or TransferHistoryOper()
if mediainfo and transferinfo:
media_source, media_id = resolve_media_identity(media=mediainfo)
his = oper.add_force(
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
dest=transferinfo.target_item.path if transferinfo.target_item else None,
dest_storage=transferinfo.target_item.storage if transferinfo.target_item else None,
dest_fileitem=transferinfo.target_item.model_dump() if transferinfo.target_item else None,
mode=mode,
type=mediainfo.type.value,
category=mediainfo.category,
title=_history_title(meta, mediainfo),
year=mediainfo.year or meta.year,
media_source=media_source,
media_id=media_id,
music_type=getattr(mediainfo, "music_type", None),
total_tracks=getattr(mediainfo, "total_tracks", None),
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
seasons=meta.season,
episodes=meta.episode,
image=mediainfo.get_poster_image(),
downloader=downloader,
download_hash=download_hash,
episode_group=mediainfo.episode_group,
status=0,
errmsg=transferinfo.message or '未知错误',
files=transferinfo.file_list
)
else:
media_source, media_id = resolve_media_identity(media=meta)
his = oper.add_force(
type=meta.type.value if meta.type else None,
title=_history_title(meta),
year=meta.year,
media_source=media_source,
media_id=media_id,
music_type=MUSIC_ENTITY_RECORDING if isinstance(meta, MetaMusic) else None,
audio_format=getattr(meta, "audio_format", None),
audio_lossless=getattr(meta, "audio_lossless", None),
bit_depth=getattr(meta, "bit_depth", None),
sample_rate=getattr(meta, "sample_rate", None),
bitrate=getattr(meta, "bitrate", None),
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem=fileitem.model_dump(),
mode=mode,
seasons=meta.season,
episodes=meta.episode,
downloader=downloader,
download_hash=download_hash,
status=0,
errmsg="未识别到媒体信息"
)
return his
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import Any, Optional
from app import schemas from app import schemas
from app.domain.context import MusicInfo from app.domain.context import MusicInfo
from app.domain.media import normalize_media_source, resolve_media_identity from app.schemas.media import normalize_media_source, resolve_media_identity
from app.runtime.extensions.service_registry import ServiceBaseHelper from app.runtime.extensions.service_registry import ServiceBaseHelper
from app.schemas import MediaServerConf, ServiceInfo from app.schemas import MediaServerConf, ServiceInfo
from app.schemas.types import ( from app.schemas.types import (
+1 -1
View File
@@ -18,7 +18,7 @@ from app.runtime.config import global_vars
from app.domain.context import MediaInfo, MusicInfo, TorrentInfo from app.domain.context import MediaInfo, MusicInfo, TorrentInfo
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.message import Notification from app.schemas.message import Notification
from app.schemas.tmdb import TmdbEpisode from app.schemas.tmdb import TmdbEpisode
+1 -1
View File
@@ -1,6 +1,6 @@
from typing import Optional from typing import Optional
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+2 -2
View File
@@ -10,8 +10,8 @@ from app import schemas
from app.application.security import access as security from app.application.security import access as security
from app.runtime.config import settings from app.runtime.config import settings
from app.db.models.user import User from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
+1 -1
View File
@@ -1,7 +1,7 @@
from typing import List, Optional from typing import List, Optional
from app import schemas from app import schemas
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.schemas.types import SystemConfigKey from app.schemas.types import SystemConfigKey
+122
View File
@@ -0,0 +1,122 @@
"""
订阅的写入路径
这两个函数把 MediaInfo / MusicInfo 翻译成一行订阅是订阅表的唯一写入口翻译此前
长在 SubscribeOper.add 但取标题选海报尺寸判音乐实体决定哪几个字段构成一条
订阅的身份都是订阅业务的规则而非数据访问Oper 只该收敛查询领域对象不该出现在
它的入参里搬上来之后 SubscribeOper 收到的是纯粹的持久化字典
app/application/history.py 里整理历史的写入路径同构
留在 Oper 的是列类型强转与建库时间戳那几步是为 PostgreSQL 的严格类型检查和订阅表
自己的列类型而存在的跟着列走比跟着调用方走更不容易漂
字段映射错了不会报错只会让订阅静静地记错而搜索洗版完成判定去重全都读这
张表同步与异步是两份逐字复制的实现改一条漏一条就是真实缺陷故翻译与身份构造由
下方 _translate 单点承担两条链路只在怎么查怎么写上分叉
"""
from typing import Optional, Tuple
from app.db.oper.subscribe import SubscribeOper
from app.domain.context import MediaInfo, MusicInfo
from app.schemas.media import resolve_media_identity
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaType
# 身份不完整时的固定返回。身份不全的订阅写进去就是一条永远匹配不上资源的僵尸订阅,
# 而后续按身份去重也会失效,所以必须在查询与建模之前短路
INCOMPLETE_IDENTITY = (0, "媒体身份不完整")
def _music_entity(mediainfo: MediaInfo | MusicInfo) -> Optional[str]:
"""
取音乐实体类型非音乐媒体一律为空
影视订阅带着 music_type 会被音乐去重逻辑当成音乐实体造成串号查重身份与写入
字段都取这一个值避免两处各算一遍后悄悄分叉
:param mediainfo: 识别结果
:return: 音乐实体类型非音乐媒体为 None
"""
if mediainfo.type != MediaType.MUSIC:
return None
return getattr(mediainfo, "music_type", None)
def _translate(mediainfo: MediaInfo | MusicInfo,
kwargs: dict) -> Optional[Tuple[dict, dict, Optional[str]]]:
"""
把识别结果翻译成查重身份与写入字段
:param mediainfo: 识别结果
:param kwargs: 调用方传入的订阅设置媒体相关的同名字段会被识别结果覆盖
:return: (查重身份, 写入字段, 限定用户)媒体身份不完整时返回 None
"""
owner_scope = bool(kwargs.pop("owner_scope", False))
username = kwargs.get("username") if owner_scope else None
media_source, media_id = resolve_media_identity(
media=mediainfo,
media_source=kwargs.get("media_source"),
media_id=kwargs.get("media_id"),
)
if not media_source or not media_id:
return None
music_type = _music_entity(mediainfo)
identity = {
"media_source": str(media_source),
"media_id": media_id,
"music_type": music_type,
"season": kwargs.get("season"),
"episode_group": mediainfo.episode_group,
}
payload = dict(kwargs)
payload.update({
"name": mediainfo.title,
"year": mediainfo.year,
"type": mediainfo.type.value,
"media_source": str(media_source),
"media_id": media_id,
"episode_group": mediainfo.episode_group,
"poster": mediainfo.get_poster_image(),
"backdrop": mediainfo.get_backdrop_image(),
"vote": mediainfo.vote_average,
"description": mediainfo.overview,
"music_type": music_type,
# 整专完成判定拿 total_tracks 当分母,单曲带着专辑的曲目数会永远判不到完成
"total_tracks": getattr(mediainfo, "total_tracks", None)
if music_type == MUSIC_ENTITY_ALBUM else None,
})
return identity, payload, username
def add_subscribe(mediainfo: MediaInfo | MusicInfo,
subscribe_oper: Optional[SubscribeOper] = None,
**kwargs) -> Tuple[int, str]:
"""
新增订阅
:param mediainfo: 识别结果
:param subscribe_oper: 复用的订阅操作对象未传时新建
:param kwargs: 订阅设置owner_scope 为真时按用户名限定查重范围
:return: (订阅 ID, 结果说明)ID 0 表示未新增
"""
translated = _translate(mediainfo, kwargs)
if translated is None:
return INCOMPLETE_IDENTITY
identity, payload, username = translated
oper = subscribe_oper or SubscribeOper()
return oper.add(identity=identity, payload=payload, username=username)
async def async_add_subscribe(mediainfo: MediaInfo | MusicInfo,
subscribe_oper: Optional[SubscribeOper] = None,
**kwargs) -> Tuple[int, str]:
"""
异步新增订阅
:param mediainfo: 识别结果
:param subscribe_oper: 复用的订阅操作对象未传时新建
:param kwargs: 订阅设置owner_scope 为真时按用户名限定查重范围
:return: (订阅 ID, 结果说明)ID 0 表示未新增
"""
translated = _translate(mediainfo, kwargs)
if translated is None:
return INCOMPLETE_IDENTITY
identity, payload, username = translated
oper = subscribe_oper or SubscribeOper()
return await oper.async_add(identity=identity, payload=payload, username=username)
+3 -3
View File
@@ -13,12 +13,12 @@ from app.domain.context import Context, TorrentInfo, MediaInfo
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format, parse_audio_quality from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format, parse_audio_quality
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas.types import MediaType, SystemConfigKey from app.schemas.types import MediaType, SystemConfigKey
from app.adapters.network.http import RequestUtils from app.adapters.network.http import RequestUtils
from app.domain.media import resolve_media_identity from app.schemas.media import resolve_media_identity
from app.domain.string import StringUtils from app.domain.string import StringUtils
+91
View File
@@ -0,0 +1,91 @@
"""
整理任务整理链的进程内工作项
TransferTask 此前住在 app/schemas/transfer.py但它不是出网的 DTOmeta 装的是领域侧
MetaBase 子类mediainfo 装的是领域侧的 MediaInfo / MusicInfo都带行为而非纯数据
放在 app.schemas 的代价是它没法命名自己真正装的类型app.schemas 一旦 import 领域类型
app.schemas -> app.schemas.transfer -> app.domain.* -> app.schemas.types -> app.schemas
就闭环仓库自己的 test_migrated_modules_are_not_in_import_cycles 会红已实测于是
两个字段只能标成 Optional[Any]这里到底能放什么这件事整个交给了口头约定
搬到应用层就没有这个约束app.application 允许依赖 app.domain app.schemas两个
字段因此能标出真实类型它面向前端的投影仍是 app/schemas/transfer.py 里的
TransferJob / TransferJobTask那两个用 app.schemas 的同名 DTO一个是工作项一个是
视图分开表达之后两边都不必再迁就对方
"""
from pathlib import Path
from typing import Callable, List, Optional, Union
from pydantic import BaseModel, ConfigDict
from app.domain.context import MediaInfo, MusicInfo
from app.domain.meta.metabase import MetaBase
from app.schemas.file import FileItem
from app.schemas.history import DownloadHistory
from app.schemas.media import OptionalMediaIdentityMixin
from app.schemas.system import TransferDirectoryConf
from app.schemas.tmdb import TmdbEpisode
from app.schemas.transfer import TransferInfo
from app.schemas.types import MediaSource, MediaType
class TransferTask(OptionalMediaIdentityMixin, BaseModel):
"""
文件整理任务
"""
# MetaBase 与 MediaInfo / MusicInfo 都是普通类而非 BaseModelpydantic 需要显式放行
model_config = ConfigDict(arbitrary_types_allowed=True)
fileitem: FileItem
meta: Optional[MetaBase] = None
mediainfo: Optional[Union[MusicInfo, MediaInfo]] = None
media_source: Optional[MediaSource] = None
media_id: Optional[str] = None
mtype: Optional[MediaType] = None
target_directory: Optional[TransferDirectoryConf] = None
target_storage: Optional[str] = None
target_path: Optional[Path] = None
transfer_type: Optional[str] = None
scrape: Optional[bool] = False
library_type_folder: Optional[bool] = False
library_category_folder: Optional[bool] = False
episodes_info: Optional[List[TmdbEpisode]] = None
username: Optional[str] = None
downloader: Optional[str] = None
download_hash: Optional[str] = None
download_history: Optional[DownloadHistory] = None
transfer_batch_id: Optional[str] = None
manual: Optional[bool] = False
background: Optional[bool] = True
preview: Optional[bool] = False
def to_dict(self):
"""
返回字典
meta mediainfo to_dict() 而非 model_dump()它们是领域对象没有
model_dump此前这里写的是 model_dump()仓内无人调用才一直没炸字段类型
标成 Any 这种错配静态检查也看不出来
"""
dicts = vars(self).copy()
dicts["fileitem"] = self.fileitem.model_dump() if self.fileitem else None
dicts["meta"] = self.meta.to_dict() if self.meta else None
dicts["mediainfo"] = self.mediainfo.to_dict() if self.mediainfo else None
dicts["target_directory"] = self.target_directory.model_dump() if self.target_directory else None
return dicts
class TransferQueue(BaseModel):
"""
异步整理队列信息
TransferTask 一起从 app/schemas 搬来它装着一个 TransferTask 和一个回调函数
回调根本不可序列化因此从来就不是 DTO只是恰好和视图模型住在同一个文件里
"""
# 任务信息
task: Optional[TransferTask] = None
# 回调函数
callback: Optional[Callable] = None
# 整理结果
result: Optional[TransferInfo] = None
+6 -6
View File
@@ -20,9 +20,9 @@ from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.runtime.extensions.module_manager import ModuleManager from app.runtime.extensions.module_manager import ModuleManager
from app.runtime.extensions.plugin_manager import PluginManager from app.runtime.extensions.plugin_manager import PluginManager
from app.db.message_oper import MessageOper from app.db.oper.message import MessageOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.application.messaging.message import MessageHelper, MessageQueueManager, MessageTemplateHelper from app.application.messaging.message import MessageHelper, MessageQueueManager, MessageTemplateHelper
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.extensions.service_registry import ServiceConfigHelper from app.runtime.extensions.service_registry import ServiceConfigHelper
@@ -42,7 +42,7 @@ from app.schemas import (
MessageResponse, MessageResponse,
) )
from app.foundation.identity import normalize_internal_user_id from app.foundation.identity import normalize_internal_user_id
from app.domain.media import resolve_media_identity from app.schemas.media import resolve_media_identity
from app.schemas.message import ChannelCapability, ChannelCapabilityManager from app.schemas.message import ChannelCapability, ChannelCapabilityManager
from app.schemas.category import CategoryConfig from app.schemas.category import CategoryConfig
from app.schemas.types import ( from app.schemas.types import (
@@ -1773,7 +1773,7 @@ class ChainBase(metaclass=ABCMeta):
self, self,
message: Optional[Notification] = None, message: Optional[Notification] = None,
meta: Optional[MetaBase] = None, meta: Optional[MetaBase] = None,
mediainfo: Optional[MediaInfo] = None, mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
torrentinfo: Optional[TorrentInfo] = None, torrentinfo: Optional[TorrentInfo] = None,
transferinfo: Optional[TransferInfo] = None, transferinfo: Optional[TransferInfo] = None,
**kwargs, **kwargs,
@@ -1889,7 +1889,7 @@ class ChainBase(metaclass=ABCMeta):
self, self,
message: Optional[Notification] = None, message: Optional[Notification] = None,
meta: Optional[MetaBase] = None, meta: Optional[MetaBase] = None,
mediainfo: Optional[MediaInfo] = None, mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
torrentinfo: Optional[TorrentInfo] = None, torrentinfo: Optional[TorrentInfo] = None,
transferinfo: Optional[TransferInfo] = None, transferinfo: Optional[TransferInfo] = None,
**kwargs, **kwargs,
+4 -4
View File
@@ -26,9 +26,9 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.downloadfailure_oper import DownloadFailureOper from app.db.oper.downloadfailure import DownloadFailureOper
from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.mediaserver_oper import MediaServerOper from app.db.oper.mediaserver import MediaServerOper
from app.application.directory import DirectoryHelper, validate_download_save_path from app.application.directory import DirectoryHelper, validate_download_save_path
from app.runtime.thread import ThreadHelper from app.runtime.thread import ThreadHelper
from app.application.torrent import TorrentHelper from app.application.torrent import TorrentHelper
@@ -38,7 +38,7 @@ from app.schemas import ExistMediaInfo, FileURI, NotExistMediaInfo, DownloaderTo
from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, TorrentStatus, EventType, MessageChannel, NotificationType, ContentType, \ from app.schemas.types import MUSIC_ENTITY_ALBUM, MediaSource, MediaType, TorrentStatus, EventType, MessageChannel, NotificationType, ContentType, \
ChainEventType ChainEventType
from app.adapters.network.http import RequestUtils from app.adapters.network.http import RequestUtils
from app.domain.media import build_media_key, resolve_media_identity from app.schemas.media import build_media_key, resolve_media_identity
from app.domain.string import StringUtils from app.domain.string import StringUtils
from app.adapters.system.host import SystemUtils from app.adapters.system.host import SystemUtils
+8 -8
View File
@@ -37,11 +37,8 @@ from app.schemas.types import (
MediaSourceSelection, MediaSourceSelection,
MediaType, MediaType,
) )
from app.domain.media import ( from app.domain.media import is_music_media_source
is_music_media_source, from app.schemas.media import normalize_media_source, resolve_media_identity
normalize_media_source,
resolve_media_identity,
)
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
from app.foundation.text import convert as zhconv_convert from app.foundation.text import convert as zhconv_convert
from app.domain.string import StringUtils from app.domain.string import StringUtils
@@ -643,9 +640,9 @@ class MediaChain(ChainBase, metaclass=Singleton):
def supplement_tmdb_info( def supplement_tmdb_info(
self, self,
mediainfo: Optional[MediaInfo], mediainfo: Optional[Union[MediaInfo, MusicInfo]],
metainfo: Optional[MetaBase] = None, metainfo: Optional[MetaBase] = None,
) -> Optional[MediaInfo]: ) -> Optional[Union[MediaInfo, MusicInfo]]:
""" """
为任意主识别源补充 TMDB 辅助信息同时保留原始媒体身份 为任意主识别源补充 TMDB 辅助信息同时保留原始媒体身份
@@ -655,7 +652,10 @@ class MediaChain(ChainBase, metaclass=Singleton):
""" """
if not mediainfo: if not mediainfo:
return None return None
if mediainfo.type == MediaType.MUSIC: # 音乐原样返回:下面全是 TMDB 影视字段,MusicInfo 上根本没有。用 isinstance
# 而不只看 type,一来静态检查能据此收窄(.type == 的比较收窄不了类型),二来
# type 没被正确赋值的 MusicInfo 也挡得住,不至于到下一行才 AttributeError
if isinstance(mediainfo, MusicInfo) or mediainfo.type == MediaType.MUSIC:
return mediainfo return mediainfo
if mediainfo.tmdb_id and mediainfo.tmdb_info and mediainfo.genre_ids: if mediainfo.tmdb_id and mediainfo.tmdb_info and mediainfo.genre_ids:
return mediainfo return mediainfo
+1 -1
View File
@@ -4,7 +4,7 @@ from typing import Callable, Dict, List, Union, Optional, Generator, Any
from app.chain import ChainBase from app.chain import ChainBase
from app.runtime.config import global_vars from app.runtime.config import global_vars
from app.db.mediaserver_oper import MediaServerOper from app.db.oper.mediaserver import MediaServerOper
from app.runtime.extensions.service_registry import ServiceConfigHelper from app.runtime.extensions.service_registry import ServiceConfigHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import MediaServerLibrary, MediaServerItem, MediaServerSeasonInfo, MediaServerPlayItem from app.schemas import MediaServerLibrary, MediaServerItem, MediaServerSeasonInfo, MediaServerPlayItem
+3 -3
View File
@@ -26,8 +26,8 @@ from app.runtime.config import settings, global_vars
from app.domain.context import MediaInfo, Context from app.domain.context import MediaInfo, Context
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.db.models import TransferHistory from app.db.models import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper from app.db.oper.transferhistory import TransferHistoryOper
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.application.directory import DirectoryHelper from app.application.directory import DirectoryHelper
from app.application.messaging.interaction import ( from app.application.messaging.interaction import (
agent_interaction_manager, agent_interaction_manager,
@@ -42,7 +42,7 @@ from app.schemas.message import ChannelCapabilityManager, ChannelCapability
from app.schemas.system import TransferDirectoryConf from app.schemas.system import TransferDirectoryConf
from app.schemas.types import EventType, MessageChannel, MediaType from app.schemas.types import EventType, MessageChannel, MediaType
from app.adapters.network.http import RequestUtils from app.adapters.network.http import RequestUtils
from app.domain.media import build_media_key, resolve_media_identity from app.schemas.media import build_media_key, resolve_media_identity
from app.domain.string import StringUtils from app.domain.string import StringUtils
+1 -1
View File
@@ -19,7 +19,7 @@ from app.schemas.types import (
MediaSource, MediaSource,
) )
from app.runtime.execution import log_execution_time from app.runtime.execution import log_execution_time
from app.domain.media import normalize_media_source from app.schemas.media import normalize_media_source
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
+3 -6
View File
@@ -26,7 +26,7 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo, MetaInfoPath from app.domain.metainfo import MetaInfo, MetaInfoPath
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.audio import AudioMetadataHelper from app.application.audio import AudioMetadataHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import FileItem from app.schemas import FileItem
@@ -43,11 +43,8 @@ from app.schemas.types import (
SystemConfigKey, SystemConfigKey,
) )
from app.adapters.network.http import RequestUtils from app.adapters.network.http import RequestUtils
from app.domain.media import ( from app.domain.media import is_music_media_source
is_music_media_source, from app.schemas.media import normalize_media_source, resolve_media_identity
normalize_media_source,
resolve_media_identity,
)
from app.runtime.reload import ConfigReloadMixin from app.runtime.reload import ConfigReloadMixin
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
from app.domain.string import StringUtils from app.domain.string import StringUtils
+2 -6
View File
@@ -21,7 +21,7 @@ from app.runtime.events import eventmanager, Event
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.domain.context import MusicInfo from app.domain.context import MusicInfo
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.runtime.progress import ProgressHelper from app.runtime.progress import ProgressHelper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.application.torrent import TorrentHelper from app.application.torrent import TorrentHelper
@@ -35,11 +35,7 @@ from app.schemas.types import (
ProgressKey, ProgressKey,
SystemConfigKey, SystemConfigKey,
) )
from app.domain.media import ( from app.schemas.media import build_media_key, parse_media_key, resolve_media_identity
build_media_key,
parse_media_key,
resolve_media_identity,
)
from app.domain.string import StringUtils from app.domain.string import StringUtils
from app.foundation.text import convert as zhconv_convert from app.foundation.text import convert as zhconv_convert
+2 -2
View File
@@ -11,8 +11,8 @@ from app.chain import ChainBase
from app.runtime.config import global_vars, settings from app.runtime.config import global_vars, settings
from app.runtime.events import Event, eventmanager from app.runtime.events import Event, eventmanager
from app.db.models.site import Site from app.db.models.site import Site
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.adapters.network.browser import PlaywrightHelper from app.adapters.network.browser import PlaywrightHelper
from app.adapters.network.cloudflare import under_challenge from app.adapters.network.cloudflare import under_challenge
from app.application.security.cookie import CookieHelper from app.application.security.cookie import CookieHelper
+9 -12
View File
@@ -27,11 +27,11 @@ from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.meta.words import WordsMatcher from app.domain.meta.words import WordsMatcher
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.models.subscribe import Subscribe from app.db.models.subscribe import Subscribe
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.subscribe_oper import SubscribeOper from app.db.oper.subscribe import SubscribeOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.messaging.interaction import ( from app.application.messaging.interaction import (
SlashInteractionManager, SlashInteractionManager,
build_navigation_buttons, build_navigation_buttons,
@@ -42,6 +42,7 @@ from app.application.messaging.interaction import (
update_or_post_message, update_or_post_message,
) )
from app.application.mediaserver import MediaServerHelper from app.application.mediaserver import MediaServerHelper
from app.application.subscribe import add_subscribe, async_add_subscribe
from app.adapters.external.server import MoviePilotServerHelper from app.adapters.external.server import MoviePilotServerHelper
from app.application.torrent import TorrentHelper from app.application.torrent import TorrentHelper
from app.runtime.log import logger from app.runtime.log import logger
@@ -49,12 +50,8 @@ from app.schemas import (SubscribeEpisodesRefreshEventData,
SubscribeCompletionCheckEventData) SubscribeCompletionCheckEventData)
from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaSource, MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType, ChainEventType, \ from app.schemas.types import MUSIC_ENTITY_ALBUM, MUSIC_ENTITY_RECORDING, MediaSource, MediaType, SystemConfigKey, MessageChannel, NotificationType, EventType, ChainEventType, \
ContentType ContentType
from app.domain.media import ( from app.domain.media import MUSIC_SUBSCRIBABLE_TYPES
MUSIC_SUBSCRIBABLE_TYPES, from app.schemas.media import build_media_key, normalize_media_source, resolve_media_identity
build_media_key,
normalize_media_source,
resolve_media_identity,
)
subscribe_interaction_manager = SlashInteractionManager() subscribe_interaction_manager = SlashInteractionManager()
@@ -989,7 +986,7 @@ class SubscribeChain(ChainBase):
kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs)) kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs))
# 操作数据库 # 操作数据库
sid, err_msg = SubscribeOper().add(mediainfo=mediainfo, season=season, username=username, **kwargs) sid, err_msg = add_subscribe(mediainfo=mediainfo, season=season, username=username, **kwargs)
if not sid: if not sid:
logger.error(f'{mediainfo.title_year} {err_msg}') logger.error(f'{mediainfo.title_year} {err_msg}')
if not exist_ok and message: if not exist_ok and message:
@@ -1193,7 +1190,7 @@ class SubscribeChain(ChainBase):
kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs)) kwargs.update(self.__get_default_kwargs(mediainfo.type, **kwargs))
# 操作数据库 # 操作数据库
sid, err_msg = await SubscribeOper().async_add(mediainfo=mediainfo, season=season, username=username, **kwargs) sid, err_msg = await async_add_subscribe(mediainfo=mediainfo, season=season, username=username, **kwargs)
if not sid: if not sid:
logger.error(f'{mediainfo.title_year} {err_msg}') logger.error(f'{mediainfo.title_year} {err_msg}')
if not exist_ok and message: if not exist_ok and message:
+3 -3
View File
@@ -12,14 +12,14 @@ from app.domain.context import TorrentInfo, Context, MediaInfo
from app.domain.context import MusicInfo from app.domain.context import MusicInfo
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfo from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper from app.db.oper.site import SiteOper
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.application.rss import RssHelper from app.application.rss import RssHelper
from app.application.torrent import TorrentHelper from app.application.torrent import TorrentHelper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import Notification from app.schemas import Notification
from app.schemas.types import SystemConfigKey, MessageChannel, NotificationType, MediaType from app.schemas.types import SystemConfigKey, MessageChannel, NotificationType, MediaType
from app.domain.media import resolve_media_identity from app.schemas.media import resolve_media_identity
from app.domain.string import StringUtils from app.domain.string import StringUtils
+54 -35
View File
@@ -22,17 +22,18 @@ from app.runtime.events import eventmanager
from app.domain.meta.metabase import MetaBase from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import MetaMusic from app.domain.meta.metamusic import MetaMusic
from app.domain.metainfo import MetaInfoPath from app.domain.metainfo import MetaInfoPath
from app.db.downloadhistory_oper import DownloadHistoryOper from app.db.oper.downloadhistory import DownloadHistoryOper
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.systemconfig_oper import SystemConfigOper from app.db.oper.systemconfig import SystemConfigOper
from app.db.transferpending_oper import TransferPendingOper from app.db.oper.transferpending import TransferPendingOper
from app.db.transferhistory_oper import TransferHistoryOper from app.db.oper.transferhistory import TransferHistoryOper
from app.application.directory import DirectoryHelper from app.application.directory import DirectoryHelper
from app.application.audio import AudioMetadataHelper from app.application.audio import AudioMetadataHelper
from app.application.formatting import EpisodeFormatRuleHelper, FormatParser from app.application.formatting import EpisodeFormatRuleHelper, FormatParser
from app.runtime.progress import ProgressHelper from app.runtime.progress import ProgressHelper
from app.application.history import (clear_transfer_failures, describe_history_gate, from app.application.history import (add_transfer_fail, add_transfer_success,
clear_transfer_failures, describe_history_gate,
evaluate_history_gate, is_skip_action, evaluate_history_gate, is_skip_action,
record_transfer_failure, resolve_history) record_transfer_failure, resolve_history)
from app.runtime.log import logger from app.runtime.log import logger
@@ -43,8 +44,6 @@ from app.schemas import (
EpisodeFormat, EpisodeFormat,
FileItem, FileItem,
TransferDirectoryConf, TransferDirectoryConf,
TransferTask,
TransferQueue,
TransferJob, TransferJob,
TransferJobTask, TransferJobTask,
TmdbEpisode, TmdbEpisode,
@@ -65,11 +64,9 @@ from app.schemas.types import (
MediaSource, MediaSource,
) )
from app.runtime.reload import ConfigReloadMixin from app.runtime.reload import ConfigReloadMixin
from app.domain.media import ( from app.application.transfer import TransferQueue, TransferTask
normalize_media_source, from app.domain.media import normalize_music_type
normalize_music_type, from app.schemas.media import normalize_media_source, resolve_media_identity
resolve_media_identity,
)
from app.foundation.singleton import Singleton from app.foundation.singleton import Singleton
from app.domain.string import StringUtils from app.domain.string import StringUtils
from app.adapters.system.host import SystemUtils from app.adapters.system.host import SystemUtils
@@ -156,7 +153,8 @@ class JobManager:
return meta.name, season return meta.name, season
@staticmethod @staticmethod
def __get_media_id(media: MediaInfo = None, season: Optional[int] = None) -> Tuple: def __get_media_id(media: Optional[Union[MediaInfo, MusicInfo]] = None,
season: Optional[int] = None) -> Tuple:
""" """
获取媒体ID音乐额外区分实体类型并为无远端ID的曲目构造稳定身份 获取媒体ID音乐额外区分实体类型并为无远端ID的曲目构造稳定身份
""" """
@@ -225,7 +223,7 @@ class JobManager:
return self.__get_id(task) return self.__get_id(task)
@staticmethod @staticmethod
def __get_media(task: TransferTask) -> schemas.MediaInfo: def __get_media(task: TransferTask) -> Union[schemas.MediaInfo, schemas.MusicInfo]:
""" """
获取媒体信息 获取媒体信息
""" """
@@ -762,7 +760,7 @@ class JobManager:
) )
def success_tasks( def success_tasks(
self, media: MediaInfo, season: Optional[int] = None self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
) -> List[TransferJobTask]: ) -> List[TransferJobTask]:
""" """
获取作业中所有成功的任务 获取作业中所有成功的任务
@@ -789,7 +787,7 @@ class JobManager:
return [] return []
return self._job_view[__mediaid__].tasks return self._job_view[__mediaid__].tasks
def count(self, media: MediaInfo, season: Optional[int] = None) -> int: def count(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
""" """
获取作业中成功总数 获取作业中成功总数
""" """
@@ -805,7 +803,7 @@ class JobManager:
] ]
) )
def size(self, media: MediaInfo, season: Optional[int] = None) -> int: def size(self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None) -> int:
""" """
获取作业中所有成功文件总大小 获取作业中所有成功文件总大小
""" """
@@ -858,7 +856,7 @@ class JobManager:
return list(self._job_view.values()) return list(self._job_view.values())
def season_episodes( def season_episodes(
self, media: MediaInfo, season: Optional[int] = None self, media: Union[MediaInfo, MusicInfo], season: Optional[int] = None
) -> List[int]: ) -> List[int]:
""" """
获取作业的季集清单 获取作业的季集清单
@@ -1276,7 +1274,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self, self,
history: TransferHistory, history: TransferHistory,
src_path: Path, src_path: Path,
) -> Optional[MusicInfo]: ) -> Optional[Union[MusicInfo, MediaInfo]]:
""" """
重新整理重试时恢复音乐信息 重新整理重试时恢复音乐信息
@@ -1466,7 +1464,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
# 新增转移失败历史记录 # 新增转移失败历史记录
history = transferhis.add_fail( history = add_transfer_fail(
fileitem=task.fileitem, fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "", mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader, downloader=task.downloader,
@@ -1474,6 +1472,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
meta=task.meta, meta=task.meta,
mediainfo=task.mediainfo, mediainfo=task.mediainfo,
transferinfo=transferinfo, transferinfo=transferinfo,
transfer_history_oper=transferhis,
) )
# 整理失败事件 # 整理失败事件
@@ -1586,7 +1585,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
# 新增task转移成功历史记录 # 新增task转移成功历史记录
history = transferhis.add_success( history = add_transfer_success(
fileitem=task.fileitem, fileitem=task.fileitem,
mode=transferinfo.transfer_type if transferinfo else "", mode=transferinfo.transfer_type if transferinfo else "",
downloader=task.downloader, downloader=task.downloader,
@@ -1594,6 +1593,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
meta=task.meta, meta=task.meta,
mediainfo=task.mediainfo, mediainfo=task.mediainfo,
transferinfo=transferinfo, transferinfo=transferinfo,
transfer_history_oper=transferhis,
) )
# task整理完成事件 # task整理完成事件
@@ -2285,7 +2285,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
try: try:
# 识别 # 识别
transferhis = TransferHistoryOper() transferhis = TransferHistoryOper()
mediainfo = task.mediainfo # 显式标注联合:下面既会赋回音乐识别结果(MusicInfo),也会赋回影视识别
# 结果(MediaInfo),不标注时会被推断成其中一种,另一种就成了假错误
mediainfo: Optional[Union[MediaInfo, MusicInfo]] = task.mediainfo
mediainfo_changed = False mediainfo_changed = False
need_obtain_images = False need_obtain_images = False
if not mediainfo: if not mediainfo:
@@ -2302,8 +2304,10 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
and download_history.media_id and download_history.media_id
and not history_year_conflict and not history_year_conflict
): ):
# 下载记录中已存在识别信息 # 下载记录中已存在识别信息。这里不再重复标注类型:函数开头
mediainfo: Optional[MediaInfo] = MediaChain().recognize_media( # 已把 mediainfo 声明为 MediaInfo | MusicInfo | None,重复
# 声明会遮蔽它,把音乐识别结果判成类型错误
mediainfo = MediaChain().recognize_media(
mtype=task.mtype or MediaType(download_history.type), mtype=task.mtype or MediaType(download_history.type),
media_source=download_history.media_source, media_source=download_history.media_source,
media_id=download_history.media_id, media_id=download_history.media_id,
@@ -2367,24 +2371,35 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
fileid=task.fileitem.fileid if task.fileitem else None, fileid=task.fileitem.fileid if task.fileitem else None,
) )
# 新增整理失败历史记录 # 新增整理失败历史记录
his = transferhis.add_fail( his = add_transfer_fail(
fileitem=task.fileitem, fileitem=task.fileitem,
mode=task.transfer_type, mode=task.transfer_type,
meta=task.meta, meta=task.meta,
downloader=task.downloader, downloader=task.downloader,
download_hash=task.download_hash, download_hash=task.download_hash,
transfer_history_oper=transferhis,
) )
self.post_message( self.post_message(
Notification( Notification(
mtype=NotificationType.Manual, mtype=NotificationType.Manual,
title=f"{task.fileitem.name} 未识别到媒体信息,无法入库!", title=f"{task.fileitem.name} 未识别到媒体信息,无法入库!",
text=( # 历史落库失败时 his 为 Noneadd_transfer_fail 末尾的
"原因:未识别到媒体信息\n" # get_by_src 查不到即返回 None),此时 /redo 无 ID 可用,
"如果按钮不可用,可回复:\n" # 只省去这段指引而不是让整条通知连同后续的作业清理、
f"```\n/redo {his.id}\n" # 种子完成标记一起崩在 NoneType 上
f"/redo {his.id} [media_source]|[media_id]|[类型]\n```\n" text="\n".join(
"自动重试或手动识别整理。" [
), "原因:未识别到媒体信息",
(
"如果按钮不可用,可回复:\n"
f"```\n/redo {his.id}\n"
f"/redo {his.id} [media_source]|[media_id]|[类型]\n```\n"
"自动重试或手动识别整理。"
if his
else ""
),
]
).strip(),
username=task.username, username=task.username,
link=settings.MP_DOMAIN("#/history"), link=settings.MP_DOMAIN("#/history"),
buttons=self.build_failed_transfer_buttons( buttons=self.build_failed_transfer_buttons(
@@ -3166,7 +3181,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
@staticmethod @staticmethod
def _is_movie_year_conflict( def _is_movie_year_conflict(
file_meta: MetaBase, media: Union[DownloadHistory, MediaInfo] file_meta: MetaBase,
# 两种 DownloadHistory 都会进来:库模型(本文件按 ORM 行查历史)与
# schemas DTOTransferTask.download_history)。本函数只按 getattr 取
# year 与 type,对两者一视同仁
media: Union[DownloadHistory, schemas.DownloadHistory, MediaInfo, MusicInfo]
) -> bool: ) -> bool:
""" """
判断文件名年份是否与已识别电影年份冲突 判断文件名年份是否与已识别电影年份冲突
@@ -3450,7 +3469,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self, self,
fileitem: FileItem, fileitem: FileItem,
meta: MetaBase = None, meta: MetaBase = None,
mediainfo: MediaInfo = None, mediainfo: Optional[Union[MediaInfo, MusicInfo]] = None,
mtype: Optional[MediaType] = None, mtype: Optional[MediaType] = None,
media_source: Optional[MediaSource] = None, media_source: Optional[MediaSource] = None,
media_id: Optional[str] = None, media_id: Optional[str] = None,
@@ -4686,7 +4705,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
def send_transfer_message( def send_transfer_message(
self, self,
meta: MetaBase, meta: MetaBase,
mediainfo: MediaInfo, mediainfo: Union[MediaInfo, MusicInfo],
transferinfo: TransferInfo, transferinfo: TransferInfo,
season_episode: Optional[str] = None, season_episode: Optional[str] = None,
episodes_info: Optional[List[TmdbEpisode]] = None, episodes_info: Optional[List[TmdbEpisode]] = None,
+1 -1
View File
@@ -6,7 +6,7 @@ from app.chain import ChainBase
from app.runtime.config import settings from app.runtime.config import settings
from app.application.security.access import get_password_hash, verify_password from app.application.security.access import get_password_hash, verify_password
from app.db.models.user import User from app.db.models.user import User
from app.db.user_oper import UserOper from app.db.oper.user import UserOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import AuthCredentials, AuthInterceptCredentials from app.schemas import AuthCredentials, AuthInterceptCredentials
from app.schemas.types import ChainEventType from app.schemas.types import ChainEventType
+1 -1
View File
@@ -16,7 +16,7 @@ from app.chain import ChainBase
from app.runtime.config import global_vars from app.runtime.config import global_vars
from app.runtime.events import Event, eventmanager from app.runtime.events import Event, eventmanager
from app.db.models import Workflow from app.db.models import Workflow
from app.db.workflow_oper import WorkflowOper from app.db.oper.workflow import WorkflowOper
from app.runtime.log import logger from app.runtime.log import logger
from app.schemas import ActionContext, ActionFlow, Action, ActionExecution, ActionResult from app.schemas import ActionContext, ActionFlow, Action, ActionExecution, ActionResult
from app.schemas.types import EventType from app.schemas.types import EventType
+106 -558
View File
@@ -1,564 +1,112 @@
import asyncio """
from typing import Any, Generator, List, Optional, Self, Tuple, AsyncGenerator, Union 数据库包入口
from sqlalchemy import NullPool, QueuePool, and_, create_engine, event, inspect, text, select, delete, Column, Integer, \ 本模块只做符号再导出不承载实现具体职责分布在
Sequence, Identity
from sqlalchemy.engine import Engine as SQLAlchemyEngine, ExceptionContext
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
from sqlalchemy.orm import Session, as_declarative, declared_attr, scoped_session, sessionmaker
from app.runtime.config import settings - diagnostics 驱动错误的统一分类与日志
from app.runtime.log import logger - engine 引擎构建连接额度核算
- session 会话获取异步连接池与配额
- decorators 同步/异步事务装饰器
- base ORM 基类与数据访问基类
- models 表结构声明一实体一文件
- oper 数据访问实现 models 同名文件一一对应
历史上这些代码全部堆在本文件里782 既让包入口承担了实现职责
使依赖图难以理清也让import 即建立数据库连接这一副作用被固化下来
"""
from typing import TYPE_CHECKING, Any
from app.db.base import Base, DbOper, execute_dml, get_id_column
from app.db.decorators import async_db_query, async_db_update, db_query, db_update
from app.db.engine import (
check_connection_budget,
connection_budget,
get_engine,
get_global_async_engine,
)
from app.db.session import (
AsyncSessionFactory,
ScopedSession,
SessionFactory,
async_session_scope,
close_database,
get_async_db,
get_async_engine,
get_async_session_factory,
get_db,
get_scoped_session,
get_session_factory,
)
# ==================== 对外契约的分层 ====================
# 下方 __all__ 是本包**对外承诺**的那一层,仓库外的插件只应依赖其中的名字:
#
# - 数据访问:继承 DbOper 子类(插件基类已备好 self.plugindata / self.systemconfig),
# 或给自己的函数套 db_query / db_update / async_db_query / async_db_update 装饰器。
# 会话的获取、提交、回滚、释放全部由装饰器收口。
# - 引擎:Engine / AsyncEngine 保留在契约内。建表、Alembic 迁移、连接诊断这些用途
# 确实需要引擎对象本身,装饰器覆盖不到,仓库外拿它是正当的。
#
# SessionFactory / AsyncSessionFactory / ScopedSession 三个名字**不在**契约内,已从
# __all__ 移除,降级为内部实现细节。它们建出来的是绕过上述装饰器的裸会话——没有提交、
# 没有回滚、没有释放,谁建谁自己兜底,本身就是误用的形状。仓库内确有几处直接
# `from app.db import SessionFactory`scheduler、postgresql 模块、Alembic 迁移脚本),
# 那是包内部的既有用法,直接导入不受 __all__ 约束,照常可用。
# 若确实需要真正的工厂对象(而非 `X()` 取一个会话),用 get_session_factory() /
# get_scoped_session() / get_async_session_factory()——转发函数上没有 sessionmaker
# 与 scoped_session 的实例接口(.remove() / .configure() / .begin() 等)。
#
# 实现上,三个工厂名字本身就是转发函数(见 session 模块),直接再导出即可——导入它们
# 不会碰引擎。Engine / AsyncEngine 则不同:调用方拿到的必须是引擎**对象**而非函数,
# 所以只能靠模块级 __getattr__ 在取属性时才创建。
#
# 注意这意味着 `from app.db import Engine` 仍会在 import 期把引擎建出来——那是调用方
# 自己选的时机。本包自身及仓库内代码一律用 get_engine(),所以 `import app.db` 不连库。
if TYPE_CHECKING:
# 只为静态检查声明这两个名字:运行期由下方 __getattr__ 解析,模块 __dict__ 里并不存在,
# 类型检查器无从知道它们属于本模块(__all__ 里的它们会被报成 reportUnsupportedDunderAll)。
# 这里同时把类型钉准,比 __getattr__ 的 Any 更有用:调用方拿到的确实是这两类引擎。
from sqlalchemy.engine import Engine as _SyncEngine
from sqlalchemy.ext.asyncio import AsyncEngine as _SaAsyncEngine
Engine: _SyncEngine
AsyncEngine: _SaAsyncEngine
def _database_error_metadata(error: BaseException) -> Optional[dict[str, Any]]: def __getattr__(name: str) -> Any:
"""提取 SQLite 与 PostgreSQL 驱动提供的稳定错误分类字段。"""
metadata = {"error_type": type(error).__name__}
# DBAPI 驱动字段并不共享统一类型,动态读取可同时兼容 sqlite3、psycopg2 与 asyncpg。
sqlite_errorcode = getattr(error, "sqlite_errorcode", None)
sqlite_errorname = getattr(error, "sqlite_errorname", None)
if sqlite_errorcode is not None or sqlite_errorname:
if sqlite_errorcode is not None:
metadata["error_code"] = sqlite_errorcode
if sqlite_errorname:
metadata["error_name"] = sqlite_errorname
return metadata
sqlstate = getattr(error, "sqlstate", None) or getattr(error, "pgcode", None)
if not sqlstate:
sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None)
if sqlstate:
metadata["sqlstate"] = sqlstate
return metadata
return None
def _log_database_error(exception_context: ExceptionContext) -> None:
"""记录非敏感驱动错误码,并保持 SQLAlchemy 原有异常传播。"""
metadata = _database_error_metadata(exception_context.original_exception)
if not metadata:
return
dialect = exception_context.dialect
fields = {
"database": dialect.name,
"driver": dialect.driver,
**metadata,
}
logger.error(
"数据库驱动异常:" + ", ".join(f"{key}={value}" for key, value in fields.items())
)
def _register_database_error_logging(engine: SQLAlchemyEngine) -> None:
"""为主程序 Engine 注册统一的底层驱动错误诊断。"""
event.listen(engine, "handle_error", _log_database_error)
def get_id_column():
""" """
根据数据库类型返回合适的ID列定义 惰性解析 Engine / AsyncEngine 两个旧名字保持仓库外插件的导入路径可用
:param name: 属性名
:return: 对应的引擎
""" """
if settings.DB_TYPE.lower() == "postgresql": if name == "Engine":
# PostgreSQL使用SERIAL类型,让数据库自动处理序列 return get_engine()
return Column(Integer, Identity(start=1, cycle=True), primary_key=True) if name == "AsyncEngine":
else: return get_global_async_engine()
# SQLite使用Sequence raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
return Column(Integer, Sequence('id'), primary_key=True)
__all__ = [
def _get_database_engine(is_async: bool = False): "AsyncEngine",
""" "Base",
获取数据库连接参数并设置WAL模式 "DbOper",
:param is_async: 是否创建异步引擎True - 异步引擎, False - 同步引擎 "Engine",
:return: 返回对应的数据库引擎 "async_db_query",
""" "async_db_update",
# 根据数据库类型选择连接方式 "async_session_scope",
if settings.DB_TYPE.lower() == "postgresql": "check_connection_budget",
return _get_postgresql_engine(is_async) "close_database",
else: "connection_budget",
return _get_sqlite_engine(is_async) "db_query",
"db_update",
"execute_dml",
def _get_sqlite_engine(is_async: bool = False): "get_async_db",
""" "get_async_engine",
获取SQLite数据库引擎 "get_async_session_factory",
""" "get_db",
# 连接参数 "get_engine",
_connect_args = { "get_global_async_engine",
"timeout": settings.DB_TIMEOUT, "get_id_column",
} "get_scoped_session",
# 启用 WAL 模式时的额外配置 "get_session_factory",
if settings.DB_WAL_ENABLE: ]
_connect_args["check_same_thread"] = False
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": f"sqlite:///{settings.CONFIG_PATH}/user.db",
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_SQLITE_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_SQLITE_MAX_OVERFLOW
})
# 创建数据库引擎
engine = create_engine(**_db_kwargs)
_register_database_error_logging(engine)
# 设置WAL模式
_journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE"
with engine.connect() as connection:
current_mode = connection.execute(text(f"PRAGMA journal_mode={_journal_mode};")).scalar()
print(f"SQLite database journal mode set to: {current_mode}")
return engine
else:
# 数据库参数,只能使用 NullPool
_db_kwargs = {
"url": f"sqlite+aiosqlite:///{settings.CONFIG_PATH}/user.db",
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": NullPool,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 创建异步数据库引擎
async_engine = create_async_engine(**_db_kwargs)
_register_database_error_logging(async_engine.sync_engine)
# 设置WAL模式
_journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE"
async def set_async_wal_mode():
"""
设置异步引擎的WAL模式
"""
async with async_engine.connect() as _connection:
result = await _connection.execute(text(f"PRAGMA journal_mode={_journal_mode};"))
_current_mode = result.scalar()
print(f"Async SQLite database journal mode set to: {_current_mode}")
try:
asyncio.run(set_async_wal_mode())
except Exception as e:
print(f"Failed to set async SQLite WAL mode: {e}")
return async_engine
def _get_postgresql_engine(is_async: bool = False):
"""
获取PostgreSQL数据库引擎
"""
db_url = settings.DB_POSTGRESQL_URL()
# PostgreSQL连接参数
_connect_args = {}
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_POSTGRESQL_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_POSTGRESQL_MAX_OVERFLOW
})
# 创建数据库引擎
engine = create_engine(**_db_kwargs)
_register_database_error_logging(engine)
print(f"PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
return engine
else:
async_db_url = settings.DB_POSTGRESQL_URL("asyncpg")
# 数据库参数,只能使用 NullPool
_db_kwargs = {
"url": async_db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": NullPool,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 创建异步数据库引擎
async_engine = create_async_engine(**_db_kwargs)
_register_database_error_logging(async_engine.sync_engine)
print(f"Async PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
return async_engine
# 同步数据库引擎
Engine = _get_database_engine(is_async=False)
# 异步数据库引擎
AsyncEngine = _get_database_engine(is_async=True)
# 同步会话工厂
SessionFactory = sessionmaker(bind=Engine)
# 异步会话工厂
AsyncSessionFactory = async_sessionmaker(bind=AsyncEngine, class_=AsyncSession)
# 同步多线程全局使用的数据库会话
ScopedSession = scoped_session(SessionFactory)
def get_db() -> Generator:
"""
获取数据库会话用于WEB请求
:return: Session
"""
db = None
try:
db = SessionFactory()
yield db
finally:
if db:
db.close()
async def get_async_db() -> AsyncGenerator[AsyncSession, None]:
"""
获取异步数据库会话用于WEB请求
:return: AsyncSession
"""
async with AsyncSessionFactory() as session:
try:
yield session
finally:
await session.close()
async def close_database():
"""
关闭所有数据库连接并清理资源
"""
try:
# 释放同步连接池
Engine.dispose() # noqa
# 释放异步连接池
await AsyncEngine.dispose()
except Exception as err:
print(f"Error while disposing database connections: {err}")
def _get_args_db(args: tuple, kwargs: dict) -> Optional[Session]:
"""
从参数中获取数据库Session对象
"""
db = None
if args:
for arg in args:
if isinstance(arg, Session):
db = arg
break
if kwargs:
for key, value in kwargs.items():
if isinstance(value, Session):
db = value
break
return db
def _get_args_async_db(args: tuple, kwargs: dict) -> Optional[AsyncSession]:
"""
从参数中获取异步数据库AsyncSession对象
"""
db = None
if args:
for arg in args:
if isinstance(arg, AsyncSession):
db = arg
break
if kwargs:
for key, value in kwargs.items():
if isinstance(value, AsyncSession):
db = value
break
return db
def _update_args_db(args: tuple, kwargs: dict, db: Session) -> Tuple[tuple, dict]:
"""
更新参数中的数据库Session对象关键字传参时更新db的值否则更新第1或第2个参数
"""
if kwargs and 'db' in kwargs:
kwargs['db'] = db
elif args:
if args[0] is None:
args = (db, *args[1:])
else:
args = (args[0], db, *args[2:])
return args, kwargs
def _update_args_async_db(args: tuple, kwargs: dict, db: AsyncSession) -> Tuple[tuple, dict]:
"""
更新参数中的异步数据库AsyncSession对象关键字传参时更新db的值否则更新第1或第2个参数
"""
if kwargs and 'db' in kwargs:
kwargs['db'] = db
elif args:
if args[0] is None:
args = (db, *args[1:])
else:
args = (args[0], db, *args[2:])
return args, kwargs
def db_update(func):
"""
数据库更新类操作装饰器第一个参数必须是数据库会话或存在db参数
"""
def wrapper(*args, **kwargs):
# 是否关闭数据库会话
_close_db = False
# 从参数中获取数据库会话
db = _get_args_db(args, kwargs)
if not db:
# 如果没有获取到数据库会话,创建一个
db = ScopedSession()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的数据库会话
args, kwargs = _update_args_db(args, kwargs, db)
try:
# 执行函数
result = func(*args, **kwargs)
# 提交事务
db.commit()
except Exception as err:
# 回滚事务
db.rollback()
raise err
finally:
# 关闭数据库会话
if _close_db:
db.close()
return result
return wrapper
def async_db_update(func):
"""
异步数据库更新类操作装饰器第一个参数必须是异步数据库会话或存在db参数
"""
async def wrapper(*args, **kwargs):
# 是否关闭数据库会话
_close_db = False
# 从参数中获取异步数据库会话
db = _get_args_async_db(args, kwargs)
if not db:
# 如果没有获取到异步数据库会话,创建一个
db = AsyncSessionFactory()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的异步数据库会话
args, kwargs = _update_args_async_db(args, kwargs, db)
try:
# 执行函数
result = await func(*args, **kwargs)
# 提交事务
await db.commit()
except Exception as err:
# 回滚事务
await db.rollback()
raise err
finally:
# 关闭数据库会话
if _close_db:
await db.close()
return result
return wrapper
def db_query(func):
"""
数据库查询操作装饰器第一个参数必须是数据库会话或存在db参数
注意db.query列表数据时需要转换为list返回
"""
def wrapper(*args, **kwargs):
# 是否关闭数据库会话
_close_db = False
# 从参数中获取数据库会话
db = _get_args_db(args, kwargs)
if not db:
# 如果没有获取到数据库会话,创建一个
db = ScopedSession()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的数据库会话
args, kwargs = _update_args_db(args, kwargs, db)
try:
# 执行函数
result = func(*args, **kwargs)
except Exception as err:
raise err
finally:
# 关闭数据库会话
if _close_db:
db.close()
return result
return wrapper
def async_db_query(func):
"""
异步数据库查询操作装饰器第一个参数必须是异步数据库会话或存在db参数
注意db.query列表数据时需要转换为list返回
"""
async def wrapper(*args, **kwargs):
# 是否关闭数据库会话
_close_db = False
# 从参数中获取异步数据库会话
db = _get_args_async_db(args, kwargs)
if not db:
# 如果没有获取到异步数据库会话,创建一个
db = AsyncSessionFactory()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的异步数据库会话
args, kwargs = _update_args_async_db(args, kwargs, db)
try:
# 执行函数
result = await func(*args, **kwargs)
except Exception as err:
raise err
finally:
# 关闭数据库会话
if _close_db:
await db.close()
return result
return wrapper
@as_declarative()
class Base:
id: Any
__name__: str
@db_update
def create(self, db: Session):
db.add(self)
@async_db_update
async def async_create(self, db: AsyncSession):
db.add(self)
await db.flush()
return self
@classmethod
@db_query
def get(cls, db: Session, rid: int) -> Self:
return db.query(cls).filter(and_(cls.id == rid)).first()
@classmethod
@async_db_query
async def async_get(cls, db: AsyncSession, rid: int) -> Self:
result = await db.execute(select(cls).where(and_(cls.id == rid)))
return result.scalars().first()
@db_update
def update(self, db: Session, payload: dict):
for key, value in payload.items():
setattr(self, key, value)
if inspect(self).detached:
db.add(self)
@async_db_update
async def async_update(self, db: AsyncSession, payload: dict):
for key, value in payload.items():
setattr(self, key, value)
if inspect(self).detached:
db.add(self)
@classmethod
@db_update
def delete(cls, db: Session, rid):
db.query(cls).filter(and_(cls.id == rid)).delete()
@classmethod
@async_db_update
async def async_delete(cls, db: AsyncSession, rid):
result = await db.execute(select(cls).where(and_(cls.id == rid)))
user = result.scalars().first()
if user:
await db.delete(user)
@classmethod
@db_update
def truncate(cls, db: Session):
db.query(cls).delete()
@classmethod
@async_db_update
async def async_truncate(cls, db: AsyncSession):
await db.execute(delete(cls))
@classmethod
@db_query
def list(cls, db: Session) -> List[Self]:
return db.query(cls).all()
@classmethod
@async_db_query
async def async_list(cls, db: AsyncSession) -> Sequence[Self]:
result = await db.execute(select(cls))
return result.scalars().all()
def to_dict(self):
return {c.name: getattr(self, c.name, None) for c in self.__table__.columns} # noqa
@declared_attr
def __tablename__(self) -> str:
return self.__name__.lower()
class DbOper:
"""
数据库操作基类
"""
def __init__(self, db: Union[Session, AsyncSession] = None):
self._db = db
+150
View File
@@ -0,0 +1,150 @@
"""
ORM 基类与数据访问基类
Base 提供声明式基类与通用的行为字典转换增删改查便利方法
DbOper 是各业务 Oper 的基类持有一个可注入的会话
"""
from typing import Any, List, Optional, Self, Union, cast
from sqlalchemy import (CursorResult, Executable, Identity, Integer, Sequence,
and_, delete, inspect, select)
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, declared_attr, mapped_column
from app.runtime.config import settings
from app.db.decorators import async_db_query, async_db_update, db_query, db_update
def execute_dml(db: Session, statement: Executable,
execution_options: Optional[dict] = None) -> int:
"""
执行 DML 语句并返回影响行数
``Session.execute`` 的类型标注一律是 ``Result``只有运行期真正拿到的
``CursorResult`` 才带 ``rowcount``2.0 只为 ``Connection.execute`` 加了
``CursorResult`` 重载这里把转换收口一次免得每个模型各写一遍 cast
:param db: 数据库会话
:param statement: delete()/update() DML 语句
:param execution_options: 执行选项不传即沿用 SQLAlchemy 默认的会话同步策略
:return: 影响行数
"""
if execution_options is None:
result = db.execute(statement)
else:
result = db.execute(statement, execution_options=execution_options)
return cast(CursorResult[Any], result).rowcount
def get_id_column() -> Mapped[int]:
"""
根据数据库类型返回合适的ID列定义
"""
if settings.DB_TYPE.lower() == "postgresql":
# PostgreSQL使用SERIAL类型,让数据库自动处理序列
return mapped_column(Integer, Identity(start=1, cycle=True), primary_key=True)
else:
# SQLite使用Sequence
return mapped_column(Integer, Sequence('id'), primary_key=True)
class Base(DeclarativeBase):
"""
声明式基类
2.0 的声明式系统会解释类级 PEP 484 注解未包裹在 Mapped[] 中的注解会直接报错
仓内模型已全部迁移到 mapped_column() + Mapped[] 注解因此不设 __allow_unmapped__
该标志此前只为仓外插件可能继承本 Base 自定义 legacy 注解模型保留插件生态
确定迭代后这条理由不再成立留着它反而会让回流的 1.x 写法在 import 期悄悄通过
等到运行期才以列不存在的形式暴露
继承本类的模型一律使用 mapped_column() + Mapped[] 注解确需非映射的类级属性时
ClassVar 显式声明而不是把这个标志加回来
"""
# 由 get_id_column() 在各模型中提供实际的列定义,这里只声明类型供 IDE 使用
id: Mapped[int]
@db_update
def create(self, db: Session):
db.add(self)
@async_db_update
async def async_create(self, db: AsyncSession):
db.add(self)
await db.flush()
return self
@classmethod
@db_query
def get(cls, db: Session, rid: int) -> Optional[Self]:
return db.execute(select(cls).where(and_(cls.id == rid))).scalars().first()
@classmethod
@async_db_query
async def async_get(cls, db: AsyncSession, rid: int) -> Optional[Self]:
result = await db.execute(select(cls).where(and_(cls.id == rid)))
return result.scalars().first()
@db_update
def update(self, db: Session, payload: dict):
for key, value in payload.items():
setattr(self, key, value)
if inspect(self).detached:
db.add(self)
@async_db_update
async def async_update(self, db: AsyncSession, payload: dict):
for key, value in payload.items():
setattr(self, key, value)
if inspect(self).detached:
db.add(self)
@classmethod
@db_update
def delete(cls, db: Session, rid):
db.execute(delete(cls).where(and_(cls.id == rid)))
@classmethod
@async_db_update
async def async_delete(cls, db: AsyncSession, rid):
result = await db.execute(select(cls).where(and_(cls.id == rid)))
user = result.scalars().first()
if user:
await db.delete(user)
@classmethod
@db_update
def truncate(cls, db: Session):
db.execute(delete(cls))
@classmethod
@async_db_update
async def async_truncate(cls, db: AsyncSession):
await db.execute(delete(cls))
@classmethod
@db_query
def list(cls, db: Session) -> List[Self]:
return list(db.execute(select(cls)).scalars().all())
@classmethod
@async_db_query
async def async_list(cls, db: AsyncSession) -> List[Self]:
result = await db.execute(select(cls))
return list(result.scalars().all())
def to_dict(self):
return {c.name: getattr(self, c.name, None) for c in self.__table__.columns} # noqa
@declared_attr.directive
def __tablename__(cls) -> str: # noqa: N805 declared_attr 的第一个参数即类本身
return cls.__name__.lower()
class DbOper:
"""
数据库操作基类
"""
def __init__(self, db: Optional[Union[Session, AsyncSession]] = None):
self._db = db
+264
View File
@@ -0,0 +1,264 @@
"""
数据库事务装饰器
同步/异步各一对查询装饰器负责会话的获取与释放更新装饰器额外负责提交与回滚
未显式传入会话时自动创建并在结束时归还异步路径经 async_session_scope 收口
连接池与配额都在那里生效
收尾故障rollback / close / __aexit__ 自身抛异常一律只记日志不上抛四个装饰器
的处理一致理由与代价都要写明别当成漏写的 raise
- 连接断开事务已失效这类故障恰恰最容易发生在出错之后的收尾阶段裸写收尾语句时
它一抛错就顶替掉原始异常调用方看到的只剩connection reset业务异常连类型都被
换掉按类型分流的 except唯一约束冲突要重试参数错误要报错一并失配
- 代价是成功路径的行为随之改变func() 成功close() 失败时调用方**静默拿到返回值**
故障只进日志这是有意为之close() 失败时事务已经提交业务确实成功了
SQLAlchemy 归还连接时已在池层吞掉异常并 invalidate 坏连接再把释放故障升级成调用方
的异常只会让一次已经落库的写入看起来像失败诱发重复提交
"""
from typing import Any, Awaitable, Callable, Optional, Tuple, TypeVar
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
from app.db.session import ScopedSession, async_session_scope
from app.runtime.log import logger
_R = TypeVar("_R")
# 四个装饰器都会重写实参列表:未传会话时自行创建一个并塞回 db 位置。因此包装后的可调用
# 对象接受的实参与被包装函数的签名并不一致——用 Callable[..., _R] 如实表达「参数由装饰器
# 接管、返回值原样透传」。否则调用方传 None 或传异步会话都会被判成类型不符,而这恰恰是
# 装饰器存在的理由(各 Oper 的 self._db 常态就是 None)。
def _get_args_db(args: tuple, kwargs: dict) -> Optional[Session]:
"""
从参数中获取数据库Session对象
"""
db = None
if args:
for arg in args:
if isinstance(arg, Session):
db = arg
break
if kwargs:
for key, value in kwargs.items():
if isinstance(value, Session):
db = value
break
return db
def _get_args_async_db(args: tuple, kwargs: dict) -> Optional[AsyncSession]:
"""
从参数中获取异步数据库AsyncSession对象
"""
db = None
if args:
for arg in args:
if isinstance(arg, AsyncSession):
db = arg
break
if kwargs:
for key, value in kwargs.items():
if isinstance(value, AsyncSession):
db = value
break
return db
def _update_args_db(args: tuple, kwargs: dict, db: Session) -> Tuple[tuple, dict]:
"""
更新参数中的数据库Session对象关键字传参时更新db的值否则更新第1或第2个参数
"""
if kwargs and 'db' in kwargs:
kwargs['db'] = db
elif args:
if args[0] is None:
args = (db, *args[1:])
else:
args = (args[0], db, *args[2:])
return args, kwargs
def _update_args_async_db(args: tuple, kwargs: dict, db: AsyncSession) -> Tuple[tuple, dict]:
"""
更新参数中的异步数据库AsyncSession对象关键字传参时更新db的值否则更新第1或第2个参数
"""
if kwargs and 'db' in kwargs:
kwargs['db'] = db
elif args:
if args[0] is None:
args = (db, *args[1:])
else:
args = (args[0], db, *args[2:])
return args, kwargs
def db_update(func: Callable[..., _R]) -> Callable[..., _R]:
"""
数据库更新类操作装饰器第一个参数必须是数据库会话或存在db参数
"""
def wrapper(*args: Any, **kwargs: Any) -> _R:
# 是否关闭数据库会话
_close_db = False
# 从参数中获取数据库会话
db = _get_args_db(args, kwargs)
if not db:
# 如果没有获取到数据库会话,创建一个
db = ScopedSession()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的数据库会话
args, kwargs = _update_args_db(args, kwargs, db)
try:
# 执行函数
result = func(*args, **kwargs)
# 提交事务
db.commit()
except Exception as err:
# 回滚事务。回滚自身失败不得顶替原始异常:连接断开、事务已失效这类收尾故障
# 恰恰最容易发生在「出错之后」,裸写 db.rollback() 时它一抛错,调用方看到的
# 就只剩「connection reset」,真正的业务异常连类型都被换掉、按类型分流的
# except 一并失配。故障本身另行记录,不静默吞掉
try:
db.rollback()
except Exception as rollback_err: # noqa: BLE001 回滚失败不能掩盖原始异常
logger.error(f"事务回滚失败,原始异常将原样上抛:{rollback_err}")
raise err
finally:
# 关闭数据库会话。释放失败只记录:既不顶替上面正在传播的业务异常,
# 成功路径下也不把一次已提交的写入变成调用方眼里的失败(见模块说明)
if _close_db:
try:
db.close()
except Exception as close_err: # noqa: BLE001 释放故障不得改变调用结果
logger.error(f"释放数据库会话失败:{close_err}")
return result
return wrapper
def async_db_update(func: Callable[..., Awaitable[_R]]) -> Callable[..., Awaitable[_R]]:
"""
异步数据库更新类操作装饰器第一个参数必须是异步数据库会话或存在db参数
"""
async def wrapper(*args: Any, **kwargs: Any) -> _R:
# 是否关闭数据库会话;作用域与 _scope 同生共死,先置空以便静态检查看清
_close_db = False
_scope = None
# 从参数中获取异步数据库会话
db = _get_args_async_db(args, kwargs)
if not db:
# 如果没有获取到异步数据库会话,创建一个。经 async_session_scope
# 统一收口:常驻主循环走连接池,其余循环走 NullPool 并占用全局配额
_scope = async_session_scope()
db = await _scope.__aenter__()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的异步数据库会话
args, kwargs = _update_args_async_db(args, kwargs, db)
try:
# 执行函数
result = await func(*args, **kwargs)
# 提交事务
await db.commit()
except Exception as err:
# 回滚事务;与同步路径同理,回滚失败只记录,不顶替原始异常
try:
await db.rollback()
except Exception as rollback_err: # noqa: BLE001 回滚失败不能掩盖原始异常
logger.error(f"事务回滚失败,原始异常将原样上抛:{rollback_err}")
raise err
finally:
# 关闭数据库会话
if _close_db and _scope is not None:
# 退出会话上下文而不是只 close:配额的释放绑定在 __aexit__ 上,
# 只关会话会让回退路径的全局配额永不归还,最终把自己饿死。
# 退出失败同样只记录,不改变调用结果(见模块说明)
try:
await _scope.__aexit__(None, None, None)
except Exception as close_err: # noqa: BLE001 释放故障不得改变调用结果
logger.error(f"释放数据库会话失败:{close_err}")
return result
return wrapper
def db_query(func: Callable[..., _R]) -> Callable[..., _R]:
"""
数据库查询操作装饰器第一个参数必须是数据库会话或存在db参数
注意db.query列表数据时需要转换为list返回
"""
def wrapper(*args: Any, **kwargs: Any) -> _R:
# 是否关闭数据库会话
_close_db = False
# 从参数中获取数据库会话
db = _get_args_db(args, kwargs)
if not db:
# 如果没有获取到数据库会话,创建一个
db = ScopedSession()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的数据库会话
args, kwargs = _update_args_db(args, kwargs, db)
try:
# 执行函数
result = func(*args, **kwargs)
except Exception as err:
raise err
finally:
# 关闭数据库会话。释放失败只记录,不顶替业务异常、也不影响成功路径的返回值
# (见模块说明)
if _close_db:
try:
db.close()
except Exception as close_err: # noqa: BLE001 释放故障不得改变调用结果
logger.error(f"释放数据库会话失败:{close_err}")
return result
return wrapper
def async_db_query(func: Callable[..., Awaitable[_R]]) -> Callable[..., Awaitable[_R]]:
"""
异步数据库查询操作装饰器第一个参数必须是异步数据库会话或存在db参数
注意db.query列表数据时需要转换为list返回
"""
async def wrapper(*args: Any, **kwargs: Any) -> _R:
# 是否关闭数据库会话
_close_db = False
_scope = None
# 从参数中获取异步数据库会话
db = _get_args_async_db(args, kwargs)
if not db:
# 如果没有获取到异步数据库会话,创建一个。经 async_session_scope
# 统一收口:常驻主循环走连接池,其余循环走 NullPool 并占用全局配额
_scope = async_session_scope()
db = await _scope.__aenter__()
# 标记需要关闭数据库会话
_close_db = True
# 更新参数中的异步数据库会话
args, kwargs = _update_args_async_db(args, kwargs, db)
try:
# 执行函数
result = await func(*args, **kwargs)
except Exception as err:
raise err
finally:
# 关闭数据库会话
if _close_db and _scope is not None:
# 退出会话上下文而不是只 close:配额的释放绑定在 __aexit__ 上,
# 只关会话会让回退路径的全局配额永不归还,最终把自己饿死。
# 退出失败同样只记录,不改变调用结果(见模块说明)
try:
await _scope.__aexit__(None, None, None)
except Exception as close_err: # noqa: BLE001 释放故障不得改变调用结果
logger.error(f"释放数据库会话失败:{close_err}")
return result
return wrapper
+58
View File
@@ -0,0 +1,58 @@
"""
数据库错误诊断
把驱动层的错误分类字段sqlite3 / psycopg2 / asyncpg 各不相同提取成统一结构
并挂到引擎的异常事件上使排障不依赖于阅读原始驱动异常
"""
from typing import Any, Optional
from sqlalchemy import event
from sqlalchemy.engine import Engine as SQLAlchemyEngine, ExceptionContext
from app.runtime.log import logger
def _database_error_metadata(error: BaseException) -> Optional[dict[str, Any]]:
"""提取 SQLite 与 PostgreSQL 驱动提供的稳定错误分类字段。"""
metadata = {"error_type": type(error).__name__}
# DBAPI 驱动字段并不共享统一类型,动态读取可同时兼容 sqlite3、psycopg2 与 asyncpg。
sqlite_errorcode = getattr(error, "sqlite_errorcode", None)
sqlite_errorname = getattr(error, "sqlite_errorname", None)
if sqlite_errorcode is not None or sqlite_errorname:
if sqlite_errorcode is not None:
metadata["error_code"] = sqlite_errorcode
if sqlite_errorname:
metadata["error_name"] = sqlite_errorname
return metadata
sqlstate = getattr(error, "sqlstate", None) or getattr(error, "pgcode", None)
if not sqlstate:
sqlstate = getattr(getattr(error, "diag", None), "sqlstate", None)
if sqlstate:
metadata["sqlstate"] = sqlstate
return metadata
return None
def _log_database_error(exception_context: ExceptionContext) -> None:
"""记录非敏感驱动错误码,并保持 SQLAlchemy 原有异常传播。"""
metadata = _database_error_metadata(exception_context.original_exception)
if not metadata:
return
dialect = exception_context.dialect
fields = {
"database": dialect.name,
"driver": dialect.driver,
**metadata,
}
logger.error(
"数据库驱动异常:" + ", ".join(f"{key}={value}" for key, value in fields.items())
)
def _register_database_error_logging(engine: SQLAlchemyEngine) -> None:
"""为主程序 Engine 注册统一的底层驱动错误诊断。"""
event.listen(engine, "handle_error", _log_database_error)
+334
View File
@@ -0,0 +1,334 @@
"""
数据库引擎的构建与连接额度核算
同步引擎与未池化的全局异步引擎都在此按需创建首次访问时不在 import
按事件循环池化的异步引擎由 session 模块创建三者的构建参数在这里收口
"""
import threading
from typing import Dict, Optional, cast
from sqlalchemy import NullPool, QueuePool, create_engine, text
from sqlalchemy.engine import Engine as SyncEngine
from sqlalchemy.ext.asyncio import AsyncEngine as SaAsyncEngine, create_async_engine
from app.runtime.config import settings
from app.db.diagnostics import _register_database_error_logging
from app.runtime.log import logger
def _async_pool_kwargs(pooled: bool) -> dict:
"""
异步引擎的连接池参数
池化时不指定 poolclassSQLAlchemy 会自动选用异步适配的
AsyncAdaptedQueuePool显式传入同步的 QueuePool 反而会出错
:param pooled: 是否启用连接池
:return: 传给 create_async_engine 的池参数
"""
if not pooled:
return {"poolclass": NullPool}
return {
"pool_size": settings.DB_ASYNC_POOL_SIZE,
"max_overflow": settings.DB_ASYNC_MAX_OVERFLOW,
"pool_timeout": settings.DB_POOL_TIMEOUT,
}
def _get_database_engine(is_async: bool = False, pooled: bool = False):
"""
获取数据库连接参数并设置WAL模式
:param is_async: 是否创建异步引擎True - 异步引擎, False - 同步引擎
:param pooled: 异步引擎是否启用连接池仅对常驻事件循环使用
:return: 返回对应的数据库引擎
"""
# 根据数据库类型选择连接方式
if settings.DB_TYPE.lower() == "postgresql":
return _get_postgresql_engine(is_async, pooled=pooled)
else:
return _get_sqlite_engine(is_async, pooled=pooled)
def _get_sqlite_engine(is_async: bool = False, pooled: bool = False):
"""
获取SQLite数据库引擎
"""
# 连接参数
_connect_args = {
"timeout": settings.DB_TIMEOUT,
}
# 允许部署侧注入驱动级参数(如 PgBouncer 事务模式下的 statement_cache_size
_connect_args.update(settings.DB_CONNECT_ARGS or {})
# 启用 WAL 模式时的额外配置
if settings.DB_WAL_ENABLE:
_connect_args["check_same_thread"] = False
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": settings.DB_SQLITE_URL(),
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_SQLITE_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_SQLITE_MAX_OVERFLOW
})
# 创建数据库引擎
engine = create_engine(**_db_kwargs)
_register_database_error_logging(engine)
# 设置WAL模式。
# 这是引擎构建里唯一的阻塞 I/O,且发生在 get_engine() 的创建锁内——异步侧因此
# 移除了对称的那一段(见下方 else 分支)。同步侧保留是因为 journal_mode 必须有人
# 设置一次,而同步引擎的首次创建由 init_db() 在启动期单线程完成,不存在一群线程
# 等在锁上的场面;即便退化到运行期首次访问,阻塞的也只是本地 SQLite 的一次 PRAGMA。
_journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE"
with engine.connect() as connection:
current_mode = connection.execute(text(f"PRAGMA journal_mode={_journal_mode};")).scalar()
print(f"SQLite database journal mode set to: {current_mode}")
return engine
else:
# 数据库参数,只能使用 NullPool
_db_kwargs = {
"url": settings.DB_SQLITE_URL("aiosqlite"),
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args,
**_async_pool_kwargs(pooled),
}
# 创建异步数据库引擎
async_engine = create_async_engine(**_db_kwargs)
_register_database_error_logging(async_engine.sync_engine)
# 异步侧不再设置 WAL。journal_mode 是数据库文件级的持久属性,同步引擎已经设置过,
# 这里重复设置本就是冗余的;而它原本用 asyncio.run() 完成,是异步引擎构建里唯一的
# 阻塞 I/O。引擎改为惰性创建之后,构建可能发生在任意线程——包括在运行中的事件循环
# 内部(async_session_scope 首次取全局引擎时),那里调 asyncio.run() 会直接抛
# RuntimeError;即便不抛,它也是在持有创建锁的状态下阻塞,会把所有等锁的线程拖死。
return async_engine
def _get_postgresql_engine(is_async: bool = False, pooled: bool = False):
"""
获取PostgreSQL数据库引擎
"""
db_url = settings.DB_POSTGRESQL_URL()
# PostgreSQL连接参数。允许部署侧注入驱动级参数,
# 例如经 PgBouncer 事务模式接入时 asyncpg 需要 statement_cache_size=0
_connect_args = dict(settings.DB_CONNECT_ARGS or {})
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args
}
# 当使用 QueuePool 时,添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_POSTGRESQL_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_POSTGRESQL_MAX_OVERFLOW
})
# 创建数据库引擎
engine = create_engine(**_db_kwargs)
_register_database_error_logging(engine)
print(f"PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
return engine
else:
async_db_url = settings.DB_POSTGRESQL_URL("asyncpg")
# 数据库参数,只能使用 NullPool
_db_kwargs = {
"url": async_db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"pool_recycle": settings.DB_POOL_RECYCLE,
"connect_args": _connect_args,
**_async_pool_kwargs(pooled),
}
# 创建异步数据库引擎
async_engine = create_async_engine(**_db_kwargs)
_register_database_error_logging(async_engine.sync_engine)
print(f"Async PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
return async_engine
# 引擎按需创建,不在 import 期建立连接。
#
# 此前这两个是模块级常量,`import app.db` 就会按 settings 连库、建出 user.db、SQLite 还要
# 去设一次 WAL——仅仅把这个包 import 进来(工具脚本、子进程探测、文档生成)就有了副作用。
#
# 注意惰性化并没有让「隔离 CONFIG_DIR 必须早于 import」这条约束消失:settings 是在
# import app.runtime.config 时构造的,那一刻 CONFIG_DIR 就定型了,晚建的引擎连的仍是真实库。
# 它消掉的是「import 本身即产生副作用」,以及由此带来的「测试想换库就必须重新起进程」。
#
# 惰性化引入的唯一新风险是首次访问的并发:这个项目有上百个调度线程,创建出多个引擎
# 意味着各自持一份连接池,实际连接数是额度核算的数倍。因此用双重检查加锁收口。
_sync_engine_lock = threading.RLock()
_async_engine_lock = threading.RLock()
_sync_engine: Optional[SyncEngine] = None
_async_engine: Optional[SaAsyncEngine] = None
def get_engine() -> SyncEngine:
"""
获取同步数据库引擎首次调用时创建
:return: 同步引擎
"""
global _sync_engine
if _sync_engine is None:
with _sync_engine_lock:
# 锁内复查:等锁期间可能已被其它线程创建
if _sync_engine is None:
_sync_engine = cast(SyncEngine, _get_database_engine(is_async=False))
return _sync_engine
def get_global_async_engine() -> SaAsyncEngine:
"""
获取未池化的全局异步引擎供非常驻事件循环回退使用首次调用时创建
:return: 异步引擎
"""
global _async_engine
if _async_engine is None:
with _async_engine_lock:
if _async_engine is None:
_async_engine = cast(SaAsyncEngine, _get_database_engine(is_async=True))
return _async_engine
def peek_sync_engine() -> Optional[SyncEngine]:
"""
取已创建的同步引擎未创建时返回 None不触发创建
关停路径close_database测试引导的 atexit需要有就释放没有就算了
get_engine() 会为了 dispose 而先连一次库在从未用过数据库的进程里尤其荒谬
取锁而不是裸读槽位另一个线程可能正卡在创建里裸读会看到 None把那个引擎漏掉
取锁则会等它建完注意这只是把竞态窗口**缩小**并没有消除 peek 返回之后才
开始创建的引擎照样漏真要杜绝得让关停之后的创建直接失败那是另一层面的改动
:return: 同步引擎或 None
"""
with _sync_engine_lock:
return _sync_engine
def peek_async_engine() -> Optional[SaAsyncEngine]:
"""
取已创建的全局异步引擎未创建时返回 None不触发创建
:return: 异步引擎或 None
"""
with _async_engine_lock:
return _async_engine
def _async_pool_enabled() -> bool:
"""
是否启用异步连接池设为 NullPool 可回退到池化前的行为
"""
return str(settings.DB_ASYNC_POOL_TYPE or "").strip().lower() != "nullpool"
def connection_budget() -> Dict[str, int]:
"""
核算数据库连接的理论峰值
各连接池此前是彼此独立配置的没有任何地方核算总和异步侧从无界收敛到有界
之后真正决定安全与否的就变成了同步池 + 异步池 + 回退配额这个总数是否
还在数据库的额度之内这里把它显式算出来供启动校验与排障使用
连接池是进程级的 worker 部署时每个进程各持一份因此合计要乘上 worker
只报单进程用量会让多 worker 在启动校验里一路绿灯实际第一个 worker 还没起完
就顶穿了 max_connections
:return: 单进程各项上限worker 数与合计
"""
if settings.DB_TYPE.lower() == "postgresql":
sync_max = settings.DB_POSTGRESQL_POOL_SIZE + settings.DB_POSTGRESQL_MAX_OVERFLOW
else:
sync_max = settings.DB_SQLITE_POOL_SIZE + settings.DB_SQLITE_MAX_OVERFLOW
if settings.DB_POOL_TYPE == "NullPool":
# 同步侧也可能被配成 NullPool,此时同样无界,用线程池规模作为可观测的上限估计
sync_max = settings.CONF.threadpool
async_max = (settings.DB_ASYNC_POOL_SIZE + settings.DB_ASYNC_MAX_OVERFLOW
if _async_pool_enabled() else 0)
fallback = settings.DB_ASYNC_FALLBACK_LIMIT if _async_pool_enabled() else settings.CONF.scheduler
per_worker = sync_max + async_max + fallback
# worker 数非法时按 1 计:退化成 0 会让合计归零、反而误判「额度充足」
workers = getattr(settings, "API_WORKERS", 1) or 1
workers = workers if isinstance(workers, int) and workers > 0 else 1
return {
"sync": sync_max,
"async_pooled": async_max,
"async_fallback": fallback,
"per_worker": per_worker,
"workers": workers,
"total": per_worker * workers,
}
def check_connection_budget() -> bool:
"""
对照数据库的真实连接上限校验理论峰值超限时告警
只对 PostgreSQL 生效SQLite 没有服务端连接上限其压力体现为 WAL 写争用而非
连接耗尽用真实的 max_connections 而不是猜测值部署方可能已经调过它
:return: 是否在额度之内
"""
budget = connection_budget()
if settings.DB_TYPE.lower() != "postgresql":
logger.info(f"数据库连接理论峰值: {budget['total']} "
f"(单进程 {budget['per_worker']} = 同步 {budget['sync']} + 异步池 "
f"{budget['async_pooled']} + 回退 {budget['async_fallback']}"
f"worker {budget['workers']})")
return True
try:
with get_engine().connect() as conn:
max_conn = int(conn.execute(text("SHOW max_connections")).scalar() or 0)
reserved = int(
conn.execute(text("SHOW superuser_reserved_connections")).scalar() or 0
)
except Exception as err:
logger.warn(f"无法读取 PostgreSQL 连接上限,跳过额度校验: {err}")
return True
available = max_conn - reserved
total = budget["total"]
detail = (f"理论峰值 {total} = 单进程 {budget['per_worker']} (同步 {budget['sync']} "
f"+ 异步池 {budget['async_pooled']} + 回退 {budget['async_fallback']}) "
f"x worker {budget['workers']},数据库可用 {available} "
f"(max_connections {max_conn} - 保留 {reserved})")
if total > available:
logger.error(
f"数据库连接额度不足:{detail}"
f"突发并发时可能出现 TooManyConnectionsError。"
f"请调大 max_connections,或调小 API_WORKERS / DB_POSTGRESQL_MAX_OVERFLOW / "
f"DB_ASYNC_MAX_OVERFLOW / DB_ASYNC_FALLBACK_LIMIT"
)
return False
logger.info(f"数据库连接额度校验通过:{detail}")
return True
+7
View File
@@ -1,3 +1,10 @@
"""
ORM 模型
_identity 必须在此处导入它在 import 期把媒体身份归一挂到 mapper 事件上是六张带
身份列的表的写入不变量导入任一模型都会先初始化本包因此这一行让强制点无处可绕
"""
from . import _identity # noqa: F401 仅为注册 mapper 事件,不导出符号
from .agentchat import AgentChat from .agentchat import AgentChat
from .agenttask import AgentTask from .agenttask import AgentTask
from .agenttaskrun import AgentTaskRun from .agenttaskrun import AgentTaskRun
@@ -1,3 +1,11 @@
"""建表约束的共享片段——本模块不声明任何表,只被同包的模型模块拼进 ``__table_args__``。
以下划线开头且不进 ``models/__init__.py`` 的再导出是为了与同目录一实体一文件
模块区分开 ``media_identity.py`` 时它看着就像一张 MediaIdentity
注意alembic 迁移脚本必须自带 SQL 常量的副本而不是 import 本模块迁移是历史快照
跟着当前代码一起演进会让旧库重放出新约束
"""
from sqlalchemy import CheckConstraint from sqlalchemy import CheckConstraint
MEDIA_IDENTITY_CHECK_SQL = ( MEDIA_IDENTITY_CHECK_SQL = (
+71
View File
@@ -0,0 +1,71 @@
"""
媒体身份的持久化不变量
media_source media_id 必须成对非零去空白这条规则此前由六张表的各个 Oper
在建模前各调一次 normalize_media_identity_payload 来保证靠调用点的纪律新加一条
写入路径忘了调就会静静写进半对身份而按身份去重从此对这行失效
这里把它下沉成 flush 前的 mapper 事件凡同时具备两列的表任何 ORM 写入都会经过
忘不掉也绕不开app/db 里没有 core insert()bulk 写法已核对因此覆盖是完整的
DTO 侧的失败语义有意不同见下方 _normalize_identity 的说明
"""
from typing import Any
from sqlalchemy import event
from sqlalchemy.orm import Mapper
from app.runtime.log import logger
from app.schemas.media import resolve_media_identity
# 构成媒体身份的两列,缺一不可
IDENTITY_COLUMNS = ("media_source", "media_id")
def _normalize_identity(mapper: Mapper, connection: Any, target: Any) -> None:
"""
写库前归一媒体身份半对非法或零值身份清空两列并记一条告警
为什么是清空 + 告警而不是像 DTO 侧那样抛错这六张表都是记账性写入整理历史
下载历史失败冷却媒体服务器同步订阅历史因身份不成对就让整条记录写不进去
等于用一个次要字段的问题换掉整条记账而丢一条整理历史意味着那个文件可能被重复
整理所以持久化侧选择降级保留**不再沉默**告警让半对身份从查不出的脏数据
变成日志里可检索的事件DTO 侧仍然抛错那里是用户输入的边界该当场拒绝
:param mapper: 触发事件的映射器
:param connection: 本次 flush 使用的连接未用到
:param target: 待写入的模型实例
"""
columns = mapper.columns.keys()
if not all(name in columns for name in IDENTITY_COLUMNS):
return
raw_source = getattr(target, "media_source", None)
raw_id = getattr(target, "media_id", None)
if raw_source is None and raw_id is None:
return
media_source, media_id = resolve_media_identity(
media_source=raw_source, media_id=raw_id
)
if not (media_source and media_id):
# 用映射类名而非 local_table.name:后者的静态类型是 FromClause,没有 name
logger.warn(
f"{mapper.class_.__name__} 的媒体身份不成对,已清空:"
f"media_source={raw_source!r}, media_id={raw_id!r}"
)
target.media_source = media_source.value if media_source else None
target.media_id = media_id
def register_identity_normalizer() -> None:
"""
注册身份归一事件
监听 Mapper 类本身而非某个基类Base 自身没有表不是映射类挂不上 mapper 事件
挂在 Mapper 上则覆盖进程内全部映射包括仓外插件自建的模型开销由上面那行列名
检查兜住不具备身份列的表直接返回
"""
event.listen(Mapper, "before_insert", _normalize_identity)
event.listen(Mapper, "before_update", _normalize_identity)
register_identity_normalizer()
+32 -33
View File
@@ -1,8 +1,8 @@
from typing import Optional from typing import Any, Optional
from sqlalchemy import Column, Integer, String, JSON, Index, select from sqlalchemy import Integer, String, JSON, Index, select
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session from sqlalchemy.orm import Mapped, Session, mapped_column
from app.db import Base, async_db_query, db_query, get_id_column from app.db import Base, async_db_query, db_query, get_id_column
@@ -14,33 +14,33 @@ class AgentChat(Base):
id = get_id_column() id = get_id_column()
# Agent 内部会话 ID,用于恢复 LangGraph 对话上下文 # Agent 内部会话 ID,用于恢复 LangGraph 对话上下文
session_id = Column(String, nullable=False) session_id: Mapped[str] = mapped_column(String, nullable=False)
# 前端或渠道侧传入的原始会话标识 # 前端或渠道侧传入的原始会话标识
client_session_id = Column(String) client_session_id: Mapped[Optional[str]] = mapped_column(String)
# 用户 ID # 用户 ID
user_id = Column(String) user_id: Mapped[Optional[str]] = mapped_column(String)
# 用户名称 # 用户名称
username = Column(String) username: Mapped[Optional[str]] = mapped_column(String)
# 消息渠道 # 消息渠道
channel = Column(String) channel: Mapped[Optional[str]] = mapped_column(String)
# 渠道来源配置名 # 渠道来源配置名
source = Column(String) source: Mapped[Optional[str]] = mapped_column(String)
# 原聊天 ID,用于区分群聊、频道或私聊 # 原聊天 ID,用于区分群聊、频道或私聊
original_chat_id = Column(String) original_chat_id: Mapped[Optional[str]] = mapped_column(String)
# 会话标题 # 会话标题
title = Column(String) title: Mapped[Optional[str]] = mapped_column(String)
# 会话预览文本 # 会话预览文本
preview = Column(String) preview: Mapped[Optional[str]] = mapped_column(String)
# 原始 LangChain messages,用于继续会话 # 原始 LangChain messages,用于继续会话
agent_messages = Column(JSON) agent_messages: Mapped[Optional[Any]] = mapped_column(JSON)
# 展示给用户的消息记录,包含文字、工具提示、附件与选择卡片 # 展示给用户的消息记录,包含文字、工具提示、附件与选择卡片
display_messages = Column(JSON) display_messages: Mapped[Optional[Any]] = mapped_column(JSON)
# 展示消息数量 # 展示消息数量
message_count = Column(Integer, default=0) message_count: Mapped[Optional[int]] = mapped_column(Integer, default=0)
# 创建时间 # 创建时间
created_at = Column(String) created_at: Mapped[Optional[str]] = mapped_column(String)
# 更新时间 # 更新时间
updated_at = Column(String) updated_at: Mapped[Optional[str]] = mapped_column(String)
__table_args__ = ( __table_args__ = (
Index("ix_agentchat_session_user", "session_id", "user_id"), Index("ix_agentchat_session_user", "session_id", "user_id"),
@@ -56,10 +56,10 @@ class AgentChat(Base):
""" """
根据会话 ID 获取 Agent 会话 根据会话 ID 获取 Agent 会话
""" """
query = db.query(cls).filter(cls.session_id == session_id) statement = select(cls).where(cls.session_id == session_id)
if user_id is not None: if user_id is not None:
query = query.filter(cls.user_id == user_id) statement = statement.where(cls.user_id == user_id)
return query.order_by(cls.id.desc()).first() return db.execute(statement.order_by(cls.id.desc())).scalars().first()
@classmethod @classmethod
@async_db_query @async_db_query
@@ -80,35 +80,34 @@ class AgentChat(Base):
def list_by_page( def list_by_page(
cls, cls,
db: Session, db: Session,
page: Optional[int] = 1, page: int = 1,
count: Optional[int] = 30, count: int = 30,
user_id: Optional[str] = None, user_id: Optional[str] = None,
username: Optional[str] = None, username: Optional[str] = None,
) -> list["AgentChat"]: ) -> list["AgentChat"]:
""" """
分页获取 Agent 会话历史 分页获取 Agent 会话历史
""" """
query = db.query(cls) statement = select(cls)
if user_id is not None and username is not None: if user_id is not None and username is not None:
query = query.filter((cls.user_id == user_id) | (cls.username == username)) statement = statement.where((cls.user_id == user_id) | (cls.username == username))
elif user_id is not None: elif user_id is not None:
query = query.filter(cls.user_id == user_id) statement = statement.where(cls.user_id == user_id)
elif username is not None: elif username is not None:
query = query.filter(cls.username == username) statement = statement.where(cls.username == username)
return ( return list(db.execute(
query.order_by(cls.updated_at.desc(), cls.id.desc()) statement.order_by(cls.updated_at.desc(), cls.id.desc())
.offset((page - 1) * count) .offset((page - 1) * count)
.limit(count) .limit(count)
.all() ).scalars().all())
)
@classmethod @classmethod
@async_db_query @async_db_query
async def async_list_by_page( async def async_list_by_page(
cls, cls,
db: AsyncSession, db: AsyncSession,
page: Optional[int] = 1, page: int = 1,
count: Optional[int] = 30, count: int = 30,
user_id: Optional[str] = None, user_id: Optional[str] = None,
username: Optional[str] = None, username: Optional[str] = None,
) -> list["AgentChat"]: ) -> list["AgentChat"]:
@@ -127,4 +126,4 @@ class AgentChat(Base):
.offset((page - 1) * count) .offset((page - 1) * count)
.limit(count) .limit(count)
) )
return result.scalars().all() return list(result.scalars().all())

Some files were not shown because too many files have changed in this diff Show More