GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-06 07:56:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(config): retire RuntimeSettingsCompat host usage

Browse Source
This commit is contained in:
jxxghp
2026-08-26 15:55:21 +08:00
parent cdab54254d
commit 9dbe424c3d
162 changed files with 1966 additions and 1745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/db/adapters/site.py
+10 -11
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
from collections.abc import Awaitable, Callable
from contextlib import AbstractAsyncContextManager
from typing import Any, TypeVar
from typing import Any, List, TypeVar
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -12,7 +12,6 @@ from sqlalchemy.orm import Session
from app.db.oper.site import SiteOper
from app.db.uow import SqlAlchemyAsyncUnitOfWork, SqlAlchemyUnitOfWork
T = TypeVar("T")
@@ -80,19 +79,19 @@ class TransactionalSiteRepository:
"""按域名查询站点。"""
return self._read(lambda repository: repository.get_by_domain(domain))
def get_domains_by_ids(self, ids: list[int]) -> list[str | None]:
def get_domains_by_ids(self, ids: List[int]) -> List[str | None]:
"""查询一组站点 ID 对应的域名。"""
return self._read(lambda repository: repository.get_domains_by_ids(ids))
def list(self) -> list[Any]:
def list(self) -> List[Any]:
"""查询全部站点。"""
return self._read(lambda repository: repository.list())
def list_order_by_pri(self) -> list[Any]:
def list_order_by_pri(self) -> List[Any]:
"""同步按优先级查询站点。"""
return self._read(lambda repository: repository.list_order_by_pri())
def get_userdata_latest(self) -> list[Any]:
def get_userdata_latest(self) -> List[Any]:
"""同步查询各站点最新用户数据。"""
return self._read(lambda repository: repository.get_userdata_latest())
@@ -112,11 +111,11 @@ class TransactionalSiteRepository:
lambda repository: repository.async_get_by_name(name)
)
async def async_list(self) -> list[Any]:
async def async_list(self) -> List[Any]:
"""异步查询全部站点。"""
return await self._async_read(lambda repository: repository.async_list())
async def async_list_order_by_pri(self) -> list[Any]:
async def async_list_order_by_pri(self) -> List[Any]:
"""异步按优先级查询站点。"""
return await self._async_read(
lambda repository: repository.async_list_order_by_pri()
@@ -132,13 +131,13 @@ class TransactionalSiteRepository:
self,
domain: str,
workdate: str | None = None,
) -> list[Any]:
) -> List[Any]:
"""异步查询站点用户数据。"""
return await self._async_read(
lambda repository: repository.async_get_userdata_by_domain(domain, workdate)
)
async def async_get_userdata_latest(self) -> list[Any]:
async def async_get_userdata_latest(self) -> List[Any]:
"""异步查询各站点最新用户数据。"""
return await self._async_read(
lambda repository: repository.async_get_userdata_latest()
@@ -156,7 +155,7 @@ class TransactionalSiteRepository:
lambda repository: repository.async_get_statistic_by_domain(domain)
)
async def async_list_statistics(self) -> list[Any]:
async def async_list_statistics(self) -> List[Any]:
"""异步查询全部站点统计。"""
return await self._async_read(
lambda repository: repository.async_list_statistics()
app/db/adapters/subscription.py
+5 -4
View File
@@ -3,6 +3,7 @@
from collections.abc import Callable
from contextlib import AbstractAsyncContextManager
from datetime import datetime, timezone
from typing import Any
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import Session
@@ -41,8 +42,8 @@ class TransactionalSubscribeWriter:
def add(
self,
identity: dict,
payload: dict,
identity: dict[str, Any],
payload: dict[str, Any],
username: str | None = None,
after_commit: AfterCommitEffect | None = None,
notification: dict[str, object] | None = None,
@@ -88,8 +89,8 @@ class TransactionalSubscribeWriter:
async def async_add(
self,
identity: dict,
payload: dict,
identity: dict[str, Any],
payload: dict[str, Any],
username: str | None = None,
after_commit: AsyncAfterCommitEffect | None = None,
notification: dict[str, object] | None = None,
app/db/base.py
+2 -2
View File
@@ -13,7 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import DeclarativeBase, Mapped, Session, declared_attr, mapped_column
from app.db.uow import run_async_transaction, run_sync_transaction
from app.runtime.config import settings
from app.runtime.settings import get_runtime_setting
T = TypeVar("T")
@@ -43,7 +43,7 @@ def get_id_column() -> Mapped[int]:
"""
根据数据库类型返回合适的ID列定义
"""
if settings.DB_TYPE.lower() == "postgresql":
if get_runtime_setting('DB_TYPE').lower() == "postgresql":
# PostgreSQL使用SERIAL类型，让数据库自动处理序列
return mapped_column(Integer, Identity(start=1, cycle=True), primary_key=True)
else:
app/db/engine.py
+47 -47
View File
@@ -13,7 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncEngine as SaAsyncEngine, create_async_en
from sqlalchemy.pool import Pool
from app.foundation.environment import is_free_threaded_runtime
from app.runtime.config import settings
from app.runtime.settings import get_runtime_setting
from app.db.diagnostics import _register_database_error_logging
from app.db.worker import DATABASE_WORKER_MAX_WORKERS
from app.runtime.log import logger
@@ -22,7 +22,7 @@ from app.runtime.observability import record_metric
def _database_backend_label() -> str:
"""把数据库类型收敛为有限的观测标签。"""
return "postgresql" if settings.DB_TYPE.lower() == "postgresql" else "sqlite"
return "postgresql" if get_runtime_setting('DB_TYPE').lower() == "postgresql" else "sqlite"
def _sync_postgresql_driver() -> Optional[str]:
@@ -61,9 +61,9 @@ def _async_pool_kwargs(pooled: bool) -> dict:
if not pooled:
return {"poolclass": NullPool}
return {
"pool_size": settings.DB_ASYNC_POOL_SIZE,
"max_overflow": settings.DB_ASYNC_MAX_OVERFLOW,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"pool_size": get_runtime_setting('DB_ASYNC_POOL_SIZE'),
"max_overflow": get_runtime_setting('DB_ASYNC_MAX_OVERFLOW'),
"pool_timeout": get_runtime_setting('DB_POOL_TIMEOUT'),
}
@@ -75,7 +75,7 @@ def _get_database_engine(is_async: bool = False, pooled: bool = False):
:return: 返回对应的数据库引擎
"""
# 根据数据库类型选择连接方式
if settings.DB_TYPE.lower() == "postgresql":
if get_runtime_setting('DB_TYPE').lower() == "postgresql":
return _get_postgresql_engine(is_async, pooled=pooled)
else:
return _get_sqlite_engine(is_async, pooled=pooled)
@@ -87,35 +87,35 @@ def _get_sqlite_engine(is_async: bool = False, pooled: bool = False):
"""
# 连接参数
_connect_args = {
"timeout": settings.DB_TIMEOUT,
"timeout": get_runtime_setting('DB_TIMEOUT'),
}
# 允许部署侧注入驱动级参数（如 PgBouncer 事务模式下的 statement_cache_size）
_connect_args.update(settings.DB_CONNECT_ARGS or {})
_connect_args.update(get_runtime_setting('DB_CONNECT_ARGS') or {})
# 启用 WAL 模式时的额外配置
if settings.DB_WAL_ENABLE:
if get_runtime_setting('DB_WAL_ENABLE'):
_connect_args["check_same_thread"] = False
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
_pool_class = NullPool if get_runtime_setting('DB_POOL_TYPE') == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": settings.DB_SQLITE_URL(),
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"url": get_runtime_setting('DB_SQLITE_URL')(),
"pool_pre_ping": get_runtime_setting('DB_POOL_PRE_PING'),
"echo": get_runtime_setting('DB_ECHO'),
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"pool_recycle": get_runtime_setting('DB_POOL_RECYCLE'),
"connect_args": _connect_args
}
# 当使用 QueuePool 时，添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_SQLITE_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_SQLITE_MAX_OVERFLOW
"pool_size": get_runtime_setting('DB_SQLITE_POOL_SIZE'),
"pool_timeout": get_runtime_setting('DB_POOL_TIMEOUT'),
"max_overflow": get_runtime_setting('DB_SQLITE_MAX_OVERFLOW')
})
# 创建数据库引擎
@@ -129,7 +129,7 @@ def _get_sqlite_engine(is_async: bool = False, pooled: bool = False):
# 设置一次，而同步引擎的首次创建由 lifespan 数据库准备组件中的 init_db() 完成，
# 不存在一群线程
# 等在锁上的场面；即便退化到运行期首次访问，阻塞的也只是本地 SQLite 的一次 PRAGMA。
_journal_mode = "WAL" if settings.DB_WAL_ENABLE else "DELETE"
_journal_mode = "WAL" if get_runtime_setting('DB_WAL_ENABLE') else "DELETE"
with engine.connect() as connection:
current_mode = connection.execute(text(f"PRAGMA journal_mode={_journal_mode};")).scalar()
print(f"SQLite database journal mode set to: {current_mode}")
@@ -138,10 +138,10 @@ def _get_sqlite_engine(is_async: bool = False, pooled: bool = False):
else:
# 数据库参数，只能使用 NullPool
_db_kwargs = {
"url": settings.DB_SQLITE_URL("aiosqlite"),
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"pool_recycle": settings.DB_POOL_RECYCLE,
"url": get_runtime_setting('DB_SQLITE_URL')("aiosqlite"),
"pool_pre_ping": get_runtime_setting('DB_POOL_PRE_PING'),
"echo": get_runtime_setting('DB_ECHO'),
"pool_recycle": get_runtime_setting('DB_POOL_RECYCLE'),
"connect_args": _connect_args,
**_async_pool_kwargs(pooled),
}
@@ -162,51 +162,51 @@ def _get_postgresql_engine(is_async: bool = False, pooled: bool = False):
"""
获取PostgreSQL数据库引擎
"""
db_url = settings.DB_POSTGRESQL_URL(_sync_postgresql_driver())
db_url = get_runtime_setting('DB_POSTGRESQL_URL')(_sync_postgresql_driver())
# PostgreSQL连接参数。允许部署侧注入驱动级参数，
# 例如经 PgBouncer 事务模式接入时 asyncpg 需要 statement_cache_size=0
_connect_args = dict(settings.DB_CONNECT_ARGS or {})
_connect_args = dict(get_runtime_setting('DB_CONNECT_ARGS') or {})
# 创建同步引擎
if not is_async:
# 根据池类型设置 poolclass 和相关参数
_pool_class = NullPool if settings.DB_POOL_TYPE == "NullPool" else QueuePool
_pool_class = NullPool if get_runtime_setting('DB_POOL_TYPE') == "NullPool" else QueuePool
# 数据库参数
_db_kwargs = {
"url": db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"pool_pre_ping": get_runtime_setting('DB_POOL_PRE_PING'),
"echo": get_runtime_setting('DB_ECHO'),
"poolclass": _pool_class,
"pool_recycle": settings.DB_POOL_RECYCLE,
"pool_recycle": get_runtime_setting('DB_POOL_RECYCLE'),
"connect_args": _connect_args
}
# 当使用 QueuePool 时，添加 QueuePool 特有的参数
if _pool_class == QueuePool:
_db_kwargs.update({
"pool_size": settings.DB_POSTGRESQL_POOL_SIZE,
"pool_timeout": settings.DB_POOL_TIMEOUT,
"max_overflow": settings.DB_POSTGRESQL_MAX_OVERFLOW
"pool_size": get_runtime_setting('DB_POSTGRESQL_POOL_SIZE'),
"pool_timeout": get_runtime_setting('DB_POOL_TIMEOUT'),
"max_overflow": get_runtime_setting('DB_POSTGRESQL_MAX_OVERFLOW')
})
# 创建数据库引擎
engine = create_engine(**_db_kwargs)
_register_database_error_logging(engine)
_register_database_pool_metrics(engine)
print(f"PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
print(f"PostgreSQL database connected to {get_runtime_setting('DB_POSTGRESQL_TARGET')}/{get_runtime_setting('DB_POSTGRESQL_DATABASE')}")
return engine
else:
async_db_url = settings.DB_POSTGRESQL_URL("asyncpg")
async_db_url = get_runtime_setting('DB_POSTGRESQL_URL')("asyncpg")
# 数据库参数，只能使用 NullPool
_db_kwargs = {
"url": async_db_url,
"pool_pre_ping": settings.DB_POOL_PRE_PING,
"echo": settings.DB_ECHO,
"pool_recycle": settings.DB_POOL_RECYCLE,
"pool_pre_ping": get_runtime_setting('DB_POOL_PRE_PING'),
"echo": get_runtime_setting('DB_ECHO'),
"pool_recycle": get_runtime_setting('DB_POOL_RECYCLE'),
"connect_args": _connect_args,
**_async_pool_kwargs(pooled),
}
@@ -214,7 +214,7 @@ def _get_postgresql_engine(is_async: bool = False, pooled: bool = False):
async_engine = create_async_engine(**_db_kwargs)
_register_database_error_logging(async_engine.sync_engine)
_register_database_pool_metrics(async_engine.sync_engine)
print(f"Async PostgreSQL database connected to {settings.DB_POSTGRESQL_TARGET}/{settings.DB_POSTGRESQL_DATABASE}")
print(f"Async PostgreSQL database connected to {get_runtime_setting('DB_POSTGRESQL_TARGET')}/{get_runtime_setting('DB_POSTGRESQL_DATABASE')}")
return async_engine
@@ -291,7 +291,7 @@ def _async_pool_enabled() -> bool:
"""
是否启用异步连接池。设为 NullPool 可回退到池化前的行为。
"""
return str(settings.DB_ASYNC_POOL_TYPE or "").strip().lower() != "nullpool"
return str(get_runtime_setting('DB_ASYNC_POOL_TYPE') or "").strip().lower() != "nullpool"
def connection_budget() -> Dict[str, int]:
@@ -306,19 +306,19 @@ def connection_budget() -> Dict[str, int]:
就顶穿了 max_connections。
:return: 单进程各项上限、worker 数与合计
"""
if settings.DB_TYPE.lower() == "postgresql":
sync_max = settings.DB_POSTGRESQL_POOL_SIZE + settings.DB_POSTGRESQL_MAX_OVERFLOW
if get_runtime_setting('DB_TYPE').lower() == "postgresql":
sync_max = get_runtime_setting('DB_POSTGRESQL_POOL_SIZE') + get_runtime_setting('DB_POSTGRESQL_MAX_OVERFLOW')
else:
sync_max = settings.DB_SQLITE_POOL_SIZE + settings.DB_SQLITE_MAX_OVERFLOW
if settings.DB_POOL_TYPE == "NullPool":
sync_max = get_runtime_setting('DB_SQLITE_POOL_SIZE') + get_runtime_setting('DB_SQLITE_MAX_OVERFLOW')
if get_runtime_setting('DB_POOL_TYPE') == "NullPool":
# 未池化连接由通用线程池和专属数据库 worker 共同创建，二者都要计入上限估计。
sync_max = settings.CONF.threadpool + DATABASE_WORKER_MAX_WORKERS
async_max = (settings.DB_ASYNC_POOL_SIZE + settings.DB_ASYNC_MAX_OVERFLOW
sync_max = get_runtime_setting('CONF').threadpool + DATABASE_WORKER_MAX_WORKERS
async_max = (get_runtime_setting('DB_ASYNC_POOL_SIZE') + get_runtime_setting('DB_ASYNC_MAX_OVERFLOW')
if _async_pool_enabled() else 0)
fallback = settings.DB_ASYNC_FALLBACK_LIMIT if _async_pool_enabled() else settings.CONF.scheduler
fallback = get_runtime_setting('DB_ASYNC_FALLBACK_LIMIT') if _async_pool_enabled() else get_runtime_setting('CONF').scheduler
per_worker = sync_max + async_max + fallback
# worker 数非法时按 1 计：退化成 0 会让合计归零、反而误判「额度充足」
workers = getattr(settings, "API_WORKERS", 1) or 1
workers = get_runtime_setting("API_WORKERS", 1) or 1
workers = workers if isinstance(workers, int) and workers > 0 else 1
return {
"sync": sync_max,
@@ -339,7 +339,7 @@ def check_connection_budget() -> bool:
:return: 是否在额度之内
"""
budget = connection_budget()
if settings.DB_TYPE.lower() != "postgresql":
if get_runtime_setting('DB_TYPE').lower() != "postgresql":
logger.info(f"数据库连接理论峰值: {budget['total']} "
f"(单进程 {budget['per_worker']} = 同步 {budget['sync']} + 异步池 "
f"{budget['async_pooled']} + 回退 {budget['async_fallback']}"
app/db/oper/agenttask.py
+18 -15
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from typing import Any, List, Optional, cast
from uuid import uuid4
from sqlalchemy import select
@@ -65,13 +65,16 @@ class AgentTaskOper(DbOper):
"""
def query(session: Session) -> Optional[AgentTask]:
"""在调用方会话中读取单个任务。"""
return session.execute(
_get_for_user_statement(
AgentTask,
task_id=task_id,
user_id=user_id,
)
).scalars().first()
return cast(
Optional[AgentTask],
session.execute(
_get_for_user_statement(
AgentTask,
task_id=task_id,
user_id=user_id,
)
).scalars().first(),
)
return self._execute_sync_query(query)
@@ -90,7 +93,7 @@ class AgentTaskOper(DbOper):
user_id=user_id,
)
)
return result.scalars().first()
return cast(Optional[AgentTask], result.scalars().first())
return await self._execute_async_query(query)
@@ -98,11 +101,11 @@ class AgentTaskOper(DbOper):
self,
user_id: Optional[str] = None,
enabled: Optional[bool] = None,
) -> list[AgentTask]:
) -> List[AgentTask]:
"""
查询 Agent 定时任务列表。
"""
def query(session: Session) -> list[AgentTask]:
def query(session: Session) -> List[AgentTask]:
"""在调用方会话中读取任务列表。"""
return list(session.execute(
_list_for_user_statement(
@@ -117,7 +120,7 @@ class AgentTaskOper(DbOper):
def update(
self,
task_id: int,
payload: dict,
payload: dict[str, Any],
user_id: Optional[str] = None,
) -> bool:
"""
@@ -217,7 +220,7 @@ class AgentTaskOper(DbOper):
task_id: int,
user_id: Optional[str] = None,
limit: int = 10,
) -> list[AgentTaskRun]:
) -> List[AgentTaskRun]:
"""查询任务最近的有界运行历史。"""
return self._execute_sync_query(
lambda session: AgentTaskRun.list_for_task(
@@ -330,7 +333,7 @@ class AgentTaskOper(DbOper):
task: AgentTask,
next_run_at: Optional[str] = None,
timezone: Optional[str] = None,
) -> dict:
) -> dict[str, Any]:
"""
将 Agent 定时任务转换为工具可返回的结构。
"""
@@ -354,7 +357,7 @@ class AgentTaskOper(DbOper):
}
@staticmethod
def run_to_dict(run: AgentTaskRun) -> dict:
def run_to_dict(run: AgentTaskRun) -> dict[str, Any]:
"""将一次 Agent 任务运行转换为工具返回结构。"""
return {
"run_id": run.run_id,
app/db/session.py
+8 -7
View File
@@ -23,7 +23,8 @@ import app.db.engine as engine_module
from app.db.engine import (_async_pool_enabled, _get_database_engine,
_database_backend_label, get_engine,
get_global_async_engine)
from app.runtime.config import global_vars, settings
from app.runtime.config import global_vars
from app.runtime.settings import get_runtime_setting
from app.runtime.log import logger
from app.runtime.observability import record_metric
@@ -134,7 +135,7 @@ _pooled_async_engines: Dict[int, Any] = {}
_pooled_async_lock = threading.Lock()
# 回退路径（未池化的临时循环）共享的全局连接配额。用 threading 信号量而非
# asyncio.Semaphore：后者绑定单个事件循环，无法跨循环生效
_fallback_slots = threading.BoundedSemaphore(max(1, settings.DB_ASYNC_FALLBACK_LIMIT))
_fallback_slots = threading.BoundedSemaphore(max(1, get_runtime_setting('DB_ASYNC_FALLBACK_LIMIT')))
def _pooled_loop() -> Optional[Any]:
@@ -179,8 +180,8 @@ def _resolve_async_engine() -> Tuple[SaAsyncEngine, bool]:
if engine is None:
engine = cast(SaAsyncEngine, _get_database_engine(is_async=True, pooled=True))
_pooled_async_engines[key] = engine
logger.info(f"异步数据库连接池已启用: pool_size={settings.DB_ASYNC_POOL_SIZE}, "
f"max_overflow={settings.DB_ASYNC_MAX_OVERFLOW}")
logger.info(f"异步数据库连接池已启用: pool_size={get_runtime_setting('DB_ASYNC_POOL_SIZE')}, "
f"max_overflow={get_runtime_setting('DB_ASYNC_MAX_OVERFLOW')}")
return engine, True
@@ -201,7 +202,7 @@ async def _acquire_fallback_slot():
因此用非阻塞获取 + 异步让出。
"""
started_at = time.monotonic()
deadline = started_at + settings.DB_POOL_TIMEOUT
deadline = started_at + get_runtime_setting('DB_POOL_TIMEOUT')
outcome = "success"
try:
while not _fallback_slots.acquire(blocking=False):
@@ -212,8 +213,8 @@ async def _acquire_fallback_slot():
backend=_database_backend_label(),
)
raise TimeoutError(
f"异步数据库连接配额已耗尽（上限 {settings.DB_ASYNC_FALLBACK_LIMIT}），"
f"等待超过 {settings.DB_POOL_TIMEOUT} 秒"
f"异步数据库连接配额已耗尽（上限 {get_runtime_setting('DB_ASYNC_FALLBACK_LIMIT')}），"
f"等待超过 {get_runtime_setting('DB_POOL_TIMEOUT')} 秒"
)
await asyncio.sleep(0.01)
finally: