mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-06 16:07:01 +08:00
refactor(database): isolate configuration transactions
This commit is contained in:
InfinityPacer
+64
-69
@@ -1,4 +1,3 @@
|
||||
import asyncio
|
||||
import copy
|
||||
import threading
|
||||
from typing import Any, Optional, Union
|
||||
@@ -14,15 +13,38 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
系统配置管理
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
加载配置到内存
|
||||
"""
|
||||
"""初始化空快照，数据库加载由启动组合根显式执行。"""
|
||||
super().__init__()
|
||||
self.__SYSTEMCONF = {}
|
||||
self._rlock = threading.RLock()
|
||||
self._alock = asyncio.Lock()
|
||||
for item in SystemConfig.list(self._db):
|
||||
self.__SYSTEMCONF[item.key] = item.value
|
||||
self._snapshot_lock = threading.RLock()
|
||||
self._write_lock = threading.RLock()
|
||||
self._loaded = False
|
||||
|
||||
def load_snapshot(self) -> None:
|
||||
"""从数据库加载完整配置，并一次性发布新的内存快照。"""
|
||||
with self._write_lock:
|
||||
snapshot = {
|
||||
item.key: copy.deepcopy(item.value)
|
||||
for item in SystemConfig.list(self._db)
|
||||
}
|
||||
with self._snapshot_lock:
|
||||
self.__SYSTEMCONF = snapshot
|
||||
self._loaded = True
|
||||
|
||||
def _require_loaded(self) -> None:
|
||||
"""阻止消费者读取尚未完成启动加载的半成品快照。"""
|
||||
if not self._loaded:
|
||||
raise RuntimeError("系统配置快照尚未加载")
|
||||
|
||||
def _publish_value(self, key: str, value: Any) -> None:
|
||||
"""在事务成功后短暂持锁发布单项配置。"""
|
||||
with self._snapshot_lock:
|
||||
self.__SYSTEMCONF[key] = copy.deepcopy(value)
|
||||
|
||||
def _publish_delete(self, key: str) -> None:
|
||||
"""在事务成功后短暂持锁移除单项配置。"""
|
||||
with self._snapshot_lock:
|
||||
self.__SYSTEMCONF.pop(key, None)
|
||||
|
||||
def set(self, key: Union[str, SystemConfigKey], value: Any) -> Optional[bool]:
|
||||
"""
|
||||
@@ -33,59 +55,25 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
"""
|
||||
if isinstance(key, SystemConfigKey):
|
||||
key = key.value
|
||||
with self._rlock:
|
||||
# 旧值
|
||||
old_value = self.__SYSTEMCONF.get(key)
|
||||
# 更新内存(deepcopy避免内存共享)
|
||||
self.__SYSTEMCONF[key] = copy.deepcopy(value)
|
||||
conf = SystemConfig.get_by_key(self._db, key)
|
||||
if conf:
|
||||
if old_value != value:
|
||||
# 假值（False/0/None/空容器）同样落库而不是删除记录：
|
||||
# 读取端以「无记录」表示未配置并回落默认值，删除会使布尔开关的关闭态无法持久化
|
||||
self._stage_update(conf, {"value": value})
|
||||
return True
|
||||
return None
|
||||
else:
|
||||
conf = SystemConfig(key=key, value=value)
|
||||
self._stage_create(conf)
|
||||
self._require_loaded()
|
||||
with self._write_lock:
|
||||
|
||||
def write(db):
|
||||
"""在当前事务中创建或更新配置记录。"""
|
||||
conf = SystemConfig.get_by_key(db, key)
|
||||
if conf:
|
||||
if conf.value == value:
|
||||
return None
|
||||
# 假值同样是有效配置；删除记录会使读取端错误回落默认值。
|
||||
conf.value = copy.deepcopy(value)
|
||||
else:
|
||||
db.add(SystemConfig(key=key, value=copy.deepcopy(value)))
|
||||
return True
|
||||
|
||||
async def async_set(self, key: Union[str, SystemConfigKey], value: Any) -> Optional[bool]:
|
||||
"""
|
||||
异步设置系统设置
|
||||
:param key: 配置键
|
||||
:param value: 配置值
|
||||
:return: 是否设置成功（True 成功/False 失败/None 无需更新）
|
||||
"""
|
||||
if isinstance(key, SystemConfigKey):
|
||||
key = key.value
|
||||
async with self._alock:
|
||||
conf = await SystemConfig.async_get_by_key(self._db, key)
|
||||
# 确定是否需要更新数据库
|
||||
needs_db_update = False
|
||||
if conf:
|
||||
if conf.value != value:
|
||||
needs_db_update = True
|
||||
else: # 记录不存在，总是需要创建/更新
|
||||
needs_db_update = True
|
||||
if not needs_db_update:
|
||||
# 即使数据库值相同，也要确保缓存同步
|
||||
with self._rlock:
|
||||
self.__SYSTEMCONF[key] = copy.deepcopy(value)
|
||||
return None
|
||||
# 执行数据库更新
|
||||
if conf:
|
||||
# 假值（False/0/None/空容器）同样落库而不是删除记录：
|
||||
# 读取端以「无记录」表示未配置并回落默认值，删除会使布尔开关的关闭态无法持久化
|
||||
await self._stage_async_update(conf, {"value": value})
|
||||
else:
|
||||
conf = SystemConfig(key=key, value=value)
|
||||
await self._stage_async_create(conf)
|
||||
# 数据库更新成功后，再更新缓存
|
||||
with self._rlock:
|
||||
self.__SYSTEMCONF[key] = copy.deepcopy(value)
|
||||
return True
|
||||
result = self._execute_sync_write(write)
|
||||
# 数据库操作返回时事务已经提交，读取方不会看到尚未持久化的配置。
|
||||
self._publish_value(key, value)
|
||||
return result
|
||||
|
||||
def get(self, key: Optional[Union[str, SystemConfigKey]] = None) -> Any:
|
||||
"""
|
||||
@@ -95,7 +83,8 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
key = key.value
|
||||
if not key:
|
||||
return self.all()
|
||||
with self._rlock:
|
||||
with self._snapshot_lock:
|
||||
self._require_loaded()
|
||||
# 避免将__SYSTEMCONF内的值引用出去，会导致set时误判没有变动
|
||||
return copy.deepcopy(self.__SYSTEMCONF.get(key))
|
||||
|
||||
@@ -107,7 +96,8 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
:param step: 递增步长
|
||||
:return: 递增后的整数值
|
||||
"""
|
||||
with self._rlock:
|
||||
self._require_loaded()
|
||||
with self._write_lock:
|
||||
value = int(self.get(key) or 0) + step
|
||||
self.set(key, value)
|
||||
return value
|
||||
@@ -116,7 +106,8 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
"""
|
||||
获取所有系统设置
|
||||
"""
|
||||
with self._rlock:
|
||||
with self._snapshot_lock:
|
||||
self._require_loaded()
|
||||
# 避免将__SYSTEMCONF内的值引用出去，会导致set时误判没有变动
|
||||
return copy.deepcopy(self.__SYSTEMCONF)
|
||||
|
||||
@@ -126,11 +117,15 @@ class SystemConfigOper(DbOper, metaclass=Singleton):
|
||||
"""
|
||||
if isinstance(key, SystemConfigKey):
|
||||
key = key.value
|
||||
with self._rlock:
|
||||
# 更新内存
|
||||
self.__SYSTEMCONF.pop(key, None)
|
||||
# 写入数据库
|
||||
conf = SystemConfig.get_by_key(self._db, key)
|
||||
if conf:
|
||||
self._stage_delete(SystemConfig, conf.id)
|
||||
self._require_loaded()
|
||||
with self._write_lock:
|
||||
|
||||
def delete(db):
|
||||
"""在当前事务中删除配置记录。"""
|
||||
conf = SystemConfig.get_by_key(db, key)
|
||||
if conf:
|
||||
db.delete(conf)
|
||||
|
||||
self._execute_sync_write(delete)
|
||||
self._publish_delete(key)
|
||||
return True
|
||||
|
||||
+59
-33
@@ -1,3 +1,5 @@
|
||||
import copy
|
||||
import threading
|
||||
from typing import Any, Union, Dict, Optional
|
||||
|
||||
from app.db.base import DbOper
|
||||
@@ -11,13 +13,30 @@ class UserConfigOper(DbOper, metaclass=Singleton):
|
||||
用户配置管理
|
||||
"""
|
||||
def __init__(self):
|
||||
"""
|
||||
加载配置到内存
|
||||
"""
|
||||
"""初始化空快照，数据库加载由启动组合根显式执行。"""
|
||||
super().__init__()
|
||||
self.__USERCONF = {}
|
||||
for item in UserConfig.list(self._db):
|
||||
self.__set_config_cache(username=item.username, key=item.key, value=item.value)
|
||||
self._snapshot_lock = threading.RLock()
|
||||
self._write_lock = threading.RLock()
|
||||
self._loaded = False
|
||||
|
||||
def load_snapshot(self) -> None:
|
||||
"""从数据库加载完整用户配置，并一次性发布新的内存快照。"""
|
||||
with self._write_lock:
|
||||
snapshot: dict[str, dict[str, Any]] = {}
|
||||
for item in UserConfig.list(self._db):
|
||||
if item.username and item.key:
|
||||
snapshot.setdefault(item.username, {})[item.key] = copy.deepcopy(
|
||||
item.value
|
||||
)
|
||||
with self._snapshot_lock:
|
||||
self.__USERCONF = snapshot
|
||||
self._loaded = True
|
||||
|
||||
def _require_loaded(self) -> None:
|
||||
"""阻止消费者读取尚未完成启动加载的半成品快照。"""
|
||||
if not self._loaded:
|
||||
raise RuntimeError("用户配置快照尚未加载")
|
||||
|
||||
def set(self, username: str, key: Union[str, UserConfigKey], value: Any):
|
||||
"""
|
||||
@@ -25,30 +44,43 @@ class UserConfigOper(DbOper, metaclass=Singleton):
|
||||
"""
|
||||
if isinstance(key, UserConfigKey):
|
||||
key = key.value
|
||||
# 更新内存
|
||||
self.__set_config_cache(username=username, key=key, value=value)
|
||||
# 写入数据库
|
||||
conf = UserConfig.get_by_key(db=self._db, username=username, key=key)
|
||||
if conf:
|
||||
if value:
|
||||
self._stage_update(conf, {"value": value})
|
||||
else:
|
||||
self._stage_delete(UserConfig, conf.id)
|
||||
else:
|
||||
conf = UserConfig(username=username, key=key, value=value)
|
||||
self._stage_create(conf)
|
||||
self._require_loaded()
|
||||
with self._write_lock:
|
||||
|
||||
def write(db):
|
||||
"""在当前事务中按用户配置的假值规则写入记录。"""
|
||||
conf = UserConfig.get_by_key(db=db, username=username, key=key)
|
||||
if conf:
|
||||
if value:
|
||||
conf.value = copy.deepcopy(value)
|
||||
else:
|
||||
db.delete(conf)
|
||||
else:
|
||||
db.add(
|
||||
UserConfig(
|
||||
username=username,
|
||||
key=key,
|
||||
value=copy.deepcopy(value),
|
||||
)
|
||||
)
|
||||
|
||||
self._execute_sync_write(write)
|
||||
# 既有运行时语义会保留刚写入的假值，即使其数据库记录被删除。
|
||||
self.__set_config_cache(username=username, key=key, value=value)
|
||||
|
||||
def get(self, username: str, key: Optional[Union[str, UserConfigKey]] = None) -> Any:
|
||||
"""
|
||||
获取用户配置
|
||||
"""
|
||||
if not username:
|
||||
return self.__USERCONF
|
||||
if isinstance(key, UserConfigKey):
|
||||
key = key.value
|
||||
if not key:
|
||||
return self.__get_config_caches(username=username)
|
||||
return self.__get_config_cache(username=username, key=key)
|
||||
with self._snapshot_lock:
|
||||
self._require_loaded()
|
||||
if not username:
|
||||
return copy.deepcopy(self.__USERCONF)
|
||||
if isinstance(key, UserConfigKey):
|
||||
key = key.value
|
||||
if not key:
|
||||
return copy.deepcopy(self.__get_config_caches(username=username))
|
||||
return copy.deepcopy(self.__get_config_cache(username=username, key=key))
|
||||
|
||||
def __set_config_cache(self, username: str, key: str, value: Any):
|
||||
"""
|
||||
@@ -56,15 +88,9 @@ class UserConfigOper(DbOper, metaclass=Singleton):
|
||||
"""
|
||||
if not username or not key:
|
||||
return
|
||||
cache = self.__USERCONF
|
||||
if not cache:
|
||||
cache = {}
|
||||
user_cache = cache.get(username)
|
||||
if not user_cache:
|
||||
user_cache = {}
|
||||
cache[username] = user_cache
|
||||
user_cache[key] = value
|
||||
self.__USERCONF = cache
|
||||
with self._snapshot_lock:
|
||||
user_cache = self.__USERCONF.setdefault(username, {})
|
||||
user_cache[key] = copy.deepcopy(value)
|
||||
|
||||
def __get_config_caches(self, username: str) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
"""同步数据库短事务的异步执行器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from contextvars import copy_context
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, TypeVar
|
||||
|
||||
from app.runtime.observability import record_metric
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class DatabaseWorkerClosedError(RuntimeError):
|
||||
"""数据库执行器尚未启动或已经停止。"""
|
||||
|
||||
|
||||
class DatabaseWorkerOverloadedError(RuntimeError):
|
||||
"""数据库执行器的运行与排队容量已经用尽。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatabaseWorkerStats:
|
||||
"""数据库执行器当前的容量和任务数量。"""
|
||||
|
||||
max_workers: int
|
||||
capacity: int
|
||||
queued: int
|
||||
running: int
|
||||
rejected: int
|
||||
closing: bool
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _WorkItem:
|
||||
"""记录一个任务的排队时间和执行状态。"""
|
||||
|
||||
submitted_at: float
|
||||
started_at: float | None = None
|
||||
|
||||
|
||||
class DatabaseWorker:
|
||||
"""以有限线程和队列执行不能原生异步化的数据库短事务。"""
|
||||
|
||||
def __init__(self, *, max_workers: int = 4, capacity: int = 32) -> None:
|
||||
"""保存容量配置，线程只在显式启动后创建。"""
|
||||
if max_workers < 1:
|
||||
raise ValueError("数据库 worker 线程数必须大于 0")
|
||||
if capacity < max_workers:
|
||||
raise ValueError("数据库 worker 总容量不能小于线程数")
|
||||
self._max_workers = max_workers
|
||||
self._capacity = capacity
|
||||
self._state_lock = threading.Lock()
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._executor: ThreadPoolExecutor | None = None
|
||||
self._futures: dict[
|
||||
Future[object], tuple[asyncio.Future[object], _WorkItem]
|
||||
] = {}
|
||||
self._queued = 0
|
||||
self._running = 0
|
||||
self._rejected = 0
|
||||
self._closing = False
|
||||
|
||||
async def start(self) -> None:
|
||||
"""绑定当前事件循环并准备专属线程池。"""
|
||||
if self._executor is not None:
|
||||
if self._loop is not asyncio.get_running_loop():
|
||||
raise RuntimeError("数据库 worker 不能跨事件循环复用")
|
||||
return
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._executor = ThreadPoolExecutor(
|
||||
max_workers=self._max_workers,
|
||||
thread_name_prefix="moviepilot-db",
|
||||
)
|
||||
self._closing = False
|
||||
self._record_depth()
|
||||
|
||||
def snapshot(self) -> DatabaseWorkerStats:
|
||||
"""返回无需访问任务对象的低基数运行快照。"""
|
||||
with self._state_lock:
|
||||
return DatabaseWorkerStats(
|
||||
max_workers=self._max_workers,
|
||||
capacity=self._capacity,
|
||||
queued=self._queued,
|
||||
running=self._running,
|
||||
rejected=self._rejected,
|
||||
closing=self._closing,
|
||||
)
|
||||
|
||||
async def run(self, operation: Callable[[], T]) -> T:
|
||||
"""执行短事务，取消时仍等待已开始的事务取得最终结果。"""
|
||||
loop = asyncio.get_running_loop()
|
||||
executor = self._executor
|
||||
if executor is None or self._loop is not loop or self._closing:
|
||||
raise DatabaseWorkerClosedError("数据库 worker 当前不可接收任务")
|
||||
|
||||
with self._state_lock:
|
||||
if self._queued + self._running >= self._capacity:
|
||||
self._rejected += 1
|
||||
record_metric("db.worker.rejected")
|
||||
raise DatabaseWorkerOverloadedError(
|
||||
f"数据库 worker 容量已用尽（上限 {self._capacity}）"
|
||||
)
|
||||
self._queued += 1
|
||||
|
||||
item = _WorkItem(submitted_at=time.perf_counter())
|
||||
context = copy_context()
|
||||
try:
|
||||
future = executor.submit(self._execute, item, context.run, operation)
|
||||
except BaseException:
|
||||
with self._state_lock:
|
||||
self._queued -= 1
|
||||
self._record_depth()
|
||||
raise
|
||||
|
||||
wrapped = asyncio.wrap_future(future, loop=loop)
|
||||
with self._state_lock:
|
||||
self._futures[future] = (wrapped, item)
|
||||
future.add_done_callback(
|
||||
lambda completed: self._schedule_completion(completed, item)
|
||||
)
|
||||
self._record_depth()
|
||||
|
||||
try:
|
||||
return await asyncio.shield(wrapped)
|
||||
except asyncio.CancelledError:
|
||||
if not future.cancel():
|
||||
await self._wait_until_done(wrapped)
|
||||
if wrapped.done() and not wrapped.cancelled():
|
||||
wrapped.exception()
|
||||
raise
|
||||
|
||||
def _execute(
|
||||
self,
|
||||
item: _WorkItem,
|
||||
context_run: Callable[..., T],
|
||||
operation: Callable[[], T],
|
||||
) -> T:
|
||||
"""在线程中标记任务开始并保留提交时的上下文。"""
|
||||
item.started_at = time.perf_counter()
|
||||
loop = self._loop
|
||||
if loop is not None:
|
||||
try:
|
||||
loop.call_soon_threadsafe(self._mark_running, item)
|
||||
except RuntimeError:
|
||||
pass
|
||||
return context_run(operation)
|
||||
|
||||
def _mark_running(self, item: _WorkItem) -> None:
|
||||
"""把任务从排队状态移入运行状态。"""
|
||||
with self._state_lock:
|
||||
self._queued -= 1
|
||||
self._running += 1
|
||||
started_at = item.started_at or time.perf_counter()
|
||||
record_metric("db.worker.wait", started_at - item.submitted_at)
|
||||
self._record_depth()
|
||||
|
||||
def _schedule_completion(
|
||||
self,
|
||||
future: Future[object],
|
||||
item: _WorkItem,
|
||||
) -> None:
|
||||
"""把线程完成通知安全地回投到所属事件循环。"""
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
return
|
||||
try:
|
||||
loop.call_soon_threadsafe(self._complete, future, item)
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def _complete(self, future: Future[object], item: _WorkItem) -> None:
|
||||
"""释放 admission，并记录任务的最终结果。"""
|
||||
with self._state_lock:
|
||||
self._futures.pop(future, None)
|
||||
if future.running() or future.done() and not future.cancelled():
|
||||
self._running -= 1
|
||||
else:
|
||||
self._queued -= 1
|
||||
outcome = "cancelled" if future.cancelled() else "success"
|
||||
if not future.cancelled():
|
||||
try:
|
||||
future.result()
|
||||
except BaseException:
|
||||
outcome = "error"
|
||||
started_at = item.started_at or item.submitted_at
|
||||
record_metric(
|
||||
"db.worker.duration",
|
||||
time.perf_counter() - started_at,
|
||||
outcome=outcome,
|
||||
)
|
||||
self._record_depth()
|
||||
|
||||
async def _wait_until_done(self, future: asyncio.Future[object]) -> None:
|
||||
"""忽略后续取消请求，直到线程内事务结束。"""
|
||||
while not future.done():
|
||||
try:
|
||||
await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
except BaseException:
|
||||
break
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""拒绝新任务，取消排队任务并等待运行中事务结束。"""
|
||||
executor = self._executor
|
||||
if executor is None:
|
||||
return
|
||||
if self._loop is not asyncio.get_running_loop():
|
||||
raise RuntimeError("数据库 worker 必须在所属事件循环中停止")
|
||||
self._closing = True
|
||||
with self._state_lock:
|
||||
futures = tuple(self._futures.items())
|
||||
for future, _state in futures:
|
||||
future.cancel()
|
||||
for future, (wrapped, _item) in futures:
|
||||
if not future.cancelled():
|
||||
await self._wait_until_done(wrapped)
|
||||
executor.shutdown(wait=True, cancel_futures=True)
|
||||
while self.snapshot().queued or self.snapshot().running:
|
||||
await asyncio.sleep(0)
|
||||
self._executor = None
|
||||
self._record_depth()
|
||||
|
||||
def _record_depth(self) -> None:
|
||||
"""记录当前排队量与运行量。"""
|
||||
stats = self.snapshot()
|
||||
record_metric("db.worker.queue.depth", stats.queued)
|
||||
record_metric("db.worker.active", stats.running)
|
||||
Reference in New Issue
Block a user