mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-04 23:17:20 +08:00
feat(database): add managed backup and offline restore (#6359)
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""数据库备份的文件系统与数据库技术适配器命名空间。"""
|
||||
@@ -0,0 +1,209 @@
|
||||
"""基于活动 SQLAlchemy 引擎的 SQLite 与 PostgreSQL 备份实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Protocol, Sequence
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatabaseBackupCheck:
|
||||
"""数据库适配器返回的基础校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class ProcessResult(Protocol):
|
||||
"""数据库命令执行结果的最小合同。"""
|
||||
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
class ProcessRunner(Protocol):
|
||||
"""可替换的数据库命令执行边界。"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
env: Mapping[str, str],
|
||||
capture_output: bool,
|
||||
text: bool,
|
||||
check: bool,
|
||||
) -> ProcessResult:
|
||||
"""执行命令并返回结果。"""
|
||||
|
||||
|
||||
class SQLiteBackupBackend:
|
||||
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
|
||||
|
||||
db_type = "sqlite"
|
||||
suffix = ".db"
|
||||
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
self._engine = engine
|
||||
database = engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
raise ValueError("SQLite 内存数据库不支持文件备份")
|
||||
self._database = Path(database)
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""从活动引擎指向的 SQLite 文件创建一致快照。"""
|
||||
source = self._engine.raw_connection()
|
||||
try:
|
||||
with closing(sqlite3.connect(destination)) as target:
|
||||
source.driver_connection.backup(target)
|
||||
target.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 SQLite integrity_check 校验备份内容。"""
|
||||
method = "PRAGMA integrity_check"
|
||||
try:
|
||||
# 已发布前的临时快照不会再变化;immutable 避免 WAL 模式为只读校验创建旁路文件。
|
||||
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
|
||||
with closing(sqlite3.connect(uri, uri=True)) as connection:
|
||||
rows = connection.execute("PRAGMA integrity_check").fetchall()
|
||||
except sqlite3.Error as error:
|
||||
return DatabaseBackupCheck(False, method, str(error))
|
||||
valid = bool(rows) and all(row[0] == "ok" for row in rows)
|
||||
detail = None if valid else "; ".join(str(row[0]) for row in rows)
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
|
||||
temporary = self._database.with_name(f".{self._database.name}.restore")
|
||||
self._engine.dispose()
|
||||
try:
|
||||
shutil.copy2(artifact, temporary)
|
||||
temporary.chmod(0o600)
|
||||
self._database.with_name(f"{self._database.name}-wal").unlink(missing_ok=True)
|
||||
self._database.with_name(f"{self._database.name}-shm").unlink(missing_ok=True)
|
||||
os.replace(temporary, self._database)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class PostgreSQLBackupBackend:
|
||||
"""使用 pg_dump 与 pg_restore 管理活动 PostgreSQL 数据库。"""
|
||||
|
||||
db_type = "postgresql"
|
||||
suffix = ".dump"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine,
|
||||
*,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_dump: str = "pg_dump",
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._runner = runner
|
||||
self._tool_resolver = tool_resolver
|
||||
self._pg_dump = pg_dump
|
||||
self._pg_restore = pg_restore
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""创建 PostgreSQL custom-format 在线备份。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_dump),
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--file",
|
||||
str(destination),
|
||||
*self._connection_arguments(),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_dump 执行失败,退出码 {result.returncode}")
|
||||
if not destination.is_file() or destination.stat().st_size == 0:
|
||||
raise RuntimeError("pg_dump 未生成有效的备份文件")
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
|
||||
method = "pg_restore --list"
|
||||
result = self._run(
|
||||
[self._require_tool(self._pg_restore), "--list", str(artifact)],
|
||||
include_password=False,
|
||||
)
|
||||
valid = result.returncode == 0 and bool(result.stdout.strip())
|
||||
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_restore),
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--single-transaction",
|
||||
"--exit-on-error",
|
||||
*self._connection_arguments(),
|
||||
str(artifact),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_restore 执行失败,退出码 {result.returncode}")
|
||||
|
||||
def _connection_arguments(self) -> list[str]:
|
||||
url = self._engine.url
|
||||
host = str(url.query.get("host") or url.host or "")
|
||||
port = str(url.query.get("port") or url.port or "")
|
||||
arguments = [
|
||||
"--username",
|
||||
str(url.username or ""),
|
||||
"--dbname",
|
||||
str(url.database or ""),
|
||||
]
|
||||
if host:
|
||||
arguments.extend(["--host", host])
|
||||
if port:
|
||||
arguments.extend(["--port", port])
|
||||
return arguments
|
||||
|
||||
def _run(self, command: Sequence[str], *, include_password: bool) -> ProcessResult:
|
||||
return self._runner(
|
||||
command,
|
||||
env=self._environment(include_password=include_password),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _require_tool(self, executable: str) -> str:
|
||||
resolved = self._tool_resolver(executable)
|
||||
if resolved is None:
|
||||
raise RuntimeError(
|
||||
f"未找到 {executable},请安装与服务端同主版本或更高的 "
|
||||
"PostgreSQL client 并加入 PATH"
|
||||
)
|
||||
return resolved
|
||||
|
||||
def _environment(self, *, include_password: bool) -> dict[str, str]:
|
||||
environment = dict(os.environ)
|
||||
environment.pop("PGPASSWORD", None)
|
||||
environment.pop("PGSSLMODE", None)
|
||||
if include_password and self._engine.url.password:
|
||||
environment["PGPASSWORD"] = str(self._engine.url.password)
|
||||
sslmode = self._engine.url.query.get("sslmode")
|
||||
if sslmode:
|
||||
environment["PGSSLMODE"] = str(sslmode)
|
||||
return environment
|
||||
@@ -0,0 +1,116 @@
|
||||
"""数据库备份单文件的受限文件系统操作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_BACKUP_NAME = re.compile(
|
||||
r"^(?P<db_type>sqlite|postgresql)_"
|
||||
r"(?P<timestamp>\d{8}_\d{6})"
|
||||
r"(?:_(?P<sequence>\d+))?"
|
||||
r"(?P<suffix>\.db|\.dump)$"
|
||||
)
|
||||
|
||||
|
||||
class BackupFiles:
|
||||
"""把备份文件操作限制在一个私有根目录内。"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def create_temporary(self, suffix: str) -> Path:
|
||||
"""在最终目录内创建私有临时文件,保证发布可使用原子替换。"""
|
||||
self._ensure_root()
|
||||
descriptor, filename = tempfile.mkstemp(
|
||||
prefix=".database-",
|
||||
suffix=f"{suffix}.partial",
|
||||
dir=self.root,
|
||||
)
|
||||
os.close(descriptor)
|
||||
path = Path(filename)
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
def publish(self, temporary: Path, name: str) -> Path:
|
||||
"""把已校验临时文件发布为正式备份文件。"""
|
||||
destination = self._resolve_name(name, require_exists=False)
|
||||
os.replace(temporary, destination)
|
||||
destination.chmod(0o600)
|
||||
return destination
|
||||
|
||||
def discard(self, temporary: Path) -> None:
|
||||
"""清理本次操作拥有的未发布临时文件。"""
|
||||
path = Path(temporary)
|
||||
if path.parent == self.root and path.name.startswith(".database-"):
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
def list(self) -> list[Path]:
|
||||
"""返回当前根目录内格式合法的正式备份文件。"""
|
||||
if not self.root.is_dir():
|
||||
return []
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.is_file() and _BACKUP_NAME.fullmatch(path.name)
|
||||
]
|
||||
return sorted(paths, key=lambda path: (self.created_at(path.name), path.name), reverse=True)
|
||||
|
||||
def resolve(self, name: str) -> Path:
|
||||
"""按受限文件名解析一个必须存在的备份文件。"""
|
||||
return self._resolve_name(name, require_exists=True)
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
"""删除一个已通过名称约束的备份文件。"""
|
||||
self.resolve(name).unlink()
|
||||
|
||||
def available_name(self, *, db_type: str, created_at: datetime, suffix: str) -> str:
|
||||
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
|
||||
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
|
||||
base = f"{db_type}_{timestamp}"
|
||||
candidate = f"{base}{suffix}"
|
||||
sequence = 1
|
||||
while (self.root / candidate).exists():
|
||||
candidate = f"{base}_{sequence}{suffix}"
|
||||
sequence += 1
|
||||
if not _BACKUP_NAME.fullmatch(candidate):
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def database_type(name: str) -> str:
|
||||
"""从受管文件名读取数据库类型。"""
|
||||
return BackupFiles._match(name).group("db_type")
|
||||
|
||||
@staticmethod
|
||||
def created_at(name: str) -> datetime:
|
||||
"""从受管文件名读取本地创建时间。"""
|
||||
return datetime.strptime(
|
||||
BackupFiles._match(name).group("timestamp"),
|
||||
"%Y%m%d_%H%M%S",
|
||||
)
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.root.chmod(0o700)
|
||||
|
||||
def _resolve_name(self, name: str, *, require_exists: bool) -> Path:
|
||||
normalized = str(name).strip()
|
||||
self._match(normalized)
|
||||
if Path(normalized).name != normalized:
|
||||
raise ValueError("数据库备份文件名不能包含路径")
|
||||
path = self.root / normalized
|
||||
if require_exists and not path.is_file():
|
||||
raise FileNotFoundError(normalized)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _match(name: str) -> re.Match[str]:
|
||||
matched = _BACKUP_NAME.fullmatch(str(name))
|
||||
if matched is None:
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return matched
|
||||
@@ -53,6 +53,7 @@ from app.adapters.external.market import (
|
||||
)
|
||||
from app.application.messaging.message import MessageHelper
|
||||
from app.runtime.progress import AsyncProgressHelper
|
||||
from app.runtime.scheduling import TimerUtils
|
||||
from app.application.rules import RuleHelper
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.runtime.state import SystemHelper
|
||||
@@ -86,6 +87,13 @@ _PUBLIC_SYSTEM_CONFIG_KEYS = {
|
||||
_PUBLIC_SETTINGS_KEYS = {"PLUGIN_MARKET"}
|
||||
_LOG_DOWNLOAD_LIMIT = 10
|
||||
_LOG_DOWNLOAD_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
_DATABASE_BACKUP_SETTING_KEYS = {
|
||||
"DB_BACKUP_ENABLE",
|
||||
"DB_BACKUP_CRON",
|
||||
"DB_BACKUP_PATH",
|
||||
"DB_BACKUP_RETENTION_DAYS",
|
||||
"DB_BACKUP_MAX_COUNT",
|
||||
}
|
||||
|
||||
|
||||
def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
|
||||
@@ -129,6 +137,38 @@ def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
|
||||
)
|
||||
|
||||
|
||||
def _validate_database_backup_config(env: dict) -> Optional[str]:
|
||||
"""在批量写入前校验数据库备份策略,避免只保存部分字段。"""
|
||||
if not _DATABASE_BACKUP_SETTING_KEYS.intersection(env):
|
||||
return None
|
||||
|
||||
cron = str(env.get("DB_BACKUP_CRON", settings.DB_BACKUP_CRON) or "").strip()
|
||||
if cron:
|
||||
try:
|
||||
TimerUtils.normalize_schedule_trigger("cron", cron, settings.TZ)
|
||||
except (TypeError, ValueError):
|
||||
return "数据库备份周期格式不正确"
|
||||
|
||||
backup_path = env.get("DB_BACKUP_PATH", settings.DB_BACKUP_PATH)
|
||||
if backup_path is not None and not isinstance(backup_path, str):
|
||||
return "数据库备份目录必须是路径字符串"
|
||||
|
||||
for key, label in (
|
||||
("DB_BACKUP_RETENTION_DAYS", "数据库备份过期天数"),
|
||||
("DB_BACKUP_MAX_COUNT", "数据库备份最大保留份数"),
|
||||
):
|
||||
value = env.get(key, getattr(settings, key))
|
||||
if isinstance(value, bool):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
try:
|
||||
converted = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
if converted < 0 or str(value).strip() != str(converted):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
return None
|
||||
|
||||
|
||||
def _is_allowed_plugin_market_wiki_url(wiki_url: str) -> bool:
|
||||
"""
|
||||
校验插件市场 Wiki 地址是否属于固定文档源。
|
||||
@@ -796,6 +836,9 @@ async def set_env_setting(
|
||||
更新系统环境变量(仅管理员)
|
||||
"""
|
||||
validation_error = _validate_llm_server_tool_config(env)
|
||||
if validation_error:
|
||||
return _SchemaResponse(success=False, message=validation_error)
|
||||
validation_error = _validate_database_backup_config(env)
|
||||
if validation_error:
|
||||
return _SchemaResponse(success=False, message=validation_error)
|
||||
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""数据库备份与离线还原用例。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol
|
||||
|
||||
from app.adapters.system.backup.files import BackupFiles
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupPolicy:
|
||||
"""一次备份操作使用的目录与保留策略快照。"""
|
||||
|
||||
root: Path
|
||||
retention_days: int = 0
|
||||
max_count: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.retention_days < 0 or self.max_count < 0:
|
||||
raise ValueError("数据库备份保留策略不能使用负数")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupArtifact:
|
||||
"""一个已完成校验并发布的数据库备份文件。"""
|
||||
|
||||
name: str
|
||||
db_type: str
|
||||
created_at: datetime
|
||||
path: Path
|
||||
size: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupVerification:
|
||||
"""数据库备份文件的基础内容校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class BackupCheck(Protocol):
|
||||
"""数据库适配器校验结果的结构合同。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None
|
||||
|
||||
|
||||
class DatabaseBackupBackend(Protocol):
|
||||
"""活动数据库创建、校验和离线还原所需的最小技术端口。"""
|
||||
|
||||
db_type: str
|
||||
suffix: str
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""把活动数据库的一致快照写入目标文件。"""
|
||||
|
||||
def verify(self, artifact: Path) -> BackupCheck:
|
||||
"""在不修改数据库的前提下校验备份文件。"""
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""把已校验制品还原到离线目标数据库。"""
|
||||
|
||||
|
||||
class DatabaseBackupService:
|
||||
"""管理单文件数据库备份及明确的离线还原操作。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
backend: DatabaseBackupBackend,
|
||||
policy_reader: Callable[[], BackupPolicy],
|
||||
clock: Callable[[], datetime] = datetime.now,
|
||||
) -> None:
|
||||
self._backend = backend
|
||||
self._policy_reader = policy_reader
|
||||
self._clock = clock
|
||||
|
||||
def create(self) -> BackupArtifact:
|
||||
"""创建、校验并发布一个在线一致快照。"""
|
||||
policy = self._policy_reader()
|
||||
files = BackupFiles(policy.root)
|
||||
created_at = self._clock()
|
||||
name = files.available_name(
|
||||
db_type=self._backend.db_type,
|
||||
created_at=created_at,
|
||||
suffix=self._backend.suffix,
|
||||
)
|
||||
temporary = files.create_temporary(self._backend.suffix)
|
||||
try:
|
||||
self._backend.create(temporary)
|
||||
verification = self._backend.verify(temporary)
|
||||
if not verification.valid:
|
||||
detail = f":{verification.detail}" if verification.detail else ""
|
||||
raise RuntimeError(
|
||||
f"数据库备份校验失败({verification.method}){detail}"
|
||||
)
|
||||
path = files.publish(temporary, name)
|
||||
except Exception:
|
||||
files.discard(temporary)
|
||||
raise
|
||||
|
||||
artifact = self._artifact(path, created_at=created_at)
|
||||
self._prune(files, policy, keep=artifact.name)
|
||||
logger.info(
|
||||
f"数据库备份完成:文件={artifact.name},类型={artifact.db_type},"
|
||||
f"大小={artifact.size} bytes"
|
||||
)
|
||||
return artifact
|
||||
|
||||
def list(self) -> tuple[BackupArtifact, ...]:
|
||||
"""按创建时间倒序列出受管数据库备份文件。"""
|
||||
files = BackupFiles(self._policy_reader().root)
|
||||
return tuple(self._artifact(path) for path in files.list())
|
||||
|
||||
def verify(self, name: str) -> BackupVerification:
|
||||
"""按文件名校验一个受管数据库备份。"""
|
||||
path = BackupFiles(self._policy_reader().root).resolve(name)
|
||||
self._require_matching_type(path)
|
||||
result = self._backend.verify(path)
|
||||
return BackupVerification(result.valid, result.method, result.detail)
|
||||
|
||||
def restore(self, name: str) -> BackupArtifact:
|
||||
"""校验后将受管制品还原到当前 CLI 解析出的离线数据库目标。"""
|
||||
path = BackupFiles(self._policy_reader().root).resolve(name)
|
||||
self._require_matching_type(path)
|
||||
verification = self._backend.verify(path)
|
||||
if not verification.valid:
|
||||
detail = f":{verification.detail}" if verification.detail else ""
|
||||
raise RuntimeError(
|
||||
f"数据库备份校验失败({verification.method}){detail}"
|
||||
)
|
||||
self._backend.restore(path)
|
||||
logger.info(
|
||||
f"数据库离线还原完成:文件={path.name},类型={self._backend.db_type}"
|
||||
)
|
||||
return self._artifact(path)
|
||||
|
||||
def _prune(self, files: BackupFiles, policy: BackupPolicy, *, keep: str) -> None:
|
||||
"""新备份发布成功后按天数或份数清理同一目录中的旧文件。"""
|
||||
cutoff = (
|
||||
self._clock() - timedelta(days=policy.retention_days)
|
||||
if policy.retention_days > 0
|
||||
else None
|
||||
)
|
||||
for index, path in enumerate(files.list()):
|
||||
if path.name == keep:
|
||||
continue
|
||||
created_at = files.created_at(path.name)
|
||||
expired = cutoff is not None and created_at < cutoff
|
||||
exceeds_count = policy.max_count > 0 and index >= policy.max_count
|
||||
if expired or exceeds_count:
|
||||
files.delete(path.name)
|
||||
|
||||
def _require_matching_type(self, path: Path) -> None:
|
||||
db_type = BackupFiles.database_type(path.name)
|
||||
if db_type != self._backend.db_type:
|
||||
raise ValueError(
|
||||
f"备份类型 {db_type} 与当前数据库类型 {self._backend.db_type} 不一致"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _artifact(path: Path, *, created_at: datetime | None = None) -> BackupArtifact:
|
||||
return BackupArtifact(
|
||||
name=path.name,
|
||||
db_type=BackupFiles.database_type(path.name),
|
||||
created_at=created_at or BackupFiles.created_at(path.name),
|
||||
path=path,
|
||||
size=path.stat().st_size,
|
||||
)
|
||||
+67
-14
@@ -1,19 +1,25 @@
|
||||
"""数据库连通性应用服务。"""
|
||||
"""数据库健康、清理、备份与离线还原的统一应用门面。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.application.backup import (
|
||||
BackupArtifact,
|
||||
BackupVerification,
|
||||
DatabaseBackupService,
|
||||
)
|
||||
from app.application.maintenance import DataCleanupService
|
||||
|
||||
|
||||
DatabaseProbe = Callable[[], Optional[str]]
|
||||
|
||||
|
||||
class DatabaseHealthService:
|
||||
"""为模块和诊断入口提供不暴露会话实现的数据库探测能力。"""
|
||||
"""提供不暴露会话实现的数据库探测能力。"""
|
||||
|
||||
def __init__(self, probe: DatabaseProbe) -> None:
|
||||
"""保存由组合根提供的数据库探测端口。"""
|
||||
self._probe = probe
|
||||
|
||||
def test(self) -> Optional[str]:
|
||||
@@ -21,17 +27,64 @@ class DatabaseHealthService:
|
||||
return self._probe()
|
||||
|
||||
|
||||
_configured_database_health: DatabaseHealthService | None = None
|
||||
class DatabaseGovernance:
|
||||
"""向宿主入口提供唯一的数据库治理能力入口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
health: DatabaseHealthService,
|
||||
cleanup: DataCleanupService,
|
||||
backup: DatabaseBackupService,
|
||||
) -> None:
|
||||
self._health = health
|
||||
self._cleanup = cleanup
|
||||
self._backup = backup
|
||||
|
||||
def test(self) -> Optional[str]:
|
||||
"""探测当前活动数据库。"""
|
||||
return self._health.test()
|
||||
|
||||
def cleanup(
|
||||
self,
|
||||
*,
|
||||
batch_size: int | None = None,
|
||||
progress_callback: Callable[..., None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""按当前配置执行数据表清理。"""
|
||||
return self._cleanup.execute(
|
||||
batch_size=batch_size,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
def create_backup(self) -> BackupArtifact:
|
||||
"""创建一个当前活动数据库的一致备份。"""
|
||||
return self._backup.create()
|
||||
|
||||
def list_backups(self) -> tuple[BackupArtifact, ...]:
|
||||
"""列出受管数据库备份文件。"""
|
||||
return self._backup.list()
|
||||
|
||||
def verify_backup(self, name: str) -> BackupVerification:
|
||||
"""校验一个受管数据库备份文件。"""
|
||||
return self._backup.verify(name)
|
||||
|
||||
def restore_backup(self, name: str) -> BackupArtifact:
|
||||
"""在离线 CLI 进程中还原一个受管数据库备份。"""
|
||||
return self._backup.restore(name)
|
||||
|
||||
|
||||
def configure_database_health(service: DatabaseHealthService) -> None:
|
||||
"""由启动组合根登记数据库探测服务。"""
|
||||
global _configured_database_health
|
||||
_configured_database_health = service
|
||||
_DATABASE_GOVERNANCE: list[DatabaseGovernance] = []
|
||||
|
||||
|
||||
def get_configured_database_health() -> DatabaseHealthService:
|
||||
"""返回启动阶段登记的数据库探测服务。"""
|
||||
if _configured_database_health is None:
|
||||
raise RuntimeError("数据库探测服务尚未配置")
|
||||
return _configured_database_health
|
||||
def configure_database_governance(governance: DatabaseGovernance) -> None:
|
||||
"""由启动组合根登记宿主唯一的数据库治理门面。"""
|
||||
_DATABASE_GOVERNANCE.clear()
|
||||
_DATABASE_GOVERNANCE.append(governance)
|
||||
|
||||
|
||||
def get_database_governance() -> DatabaseGovernance:
|
||||
"""返回启动阶段登记的数据库治理门面。"""
|
||||
if not _DATABASE_GOVERNANCE:
|
||||
raise RuntimeError("数据库治理服务尚未配置")
|
||||
return _DATABASE_GOVERNANCE[0]
|
||||
|
||||
@@ -324,24 +324,6 @@ def read_cleanup_policy() -> CleanupPolicy:
|
||||
)
|
||||
|
||||
|
||||
def build_cleanup_service() -> DataCleanupService:
|
||||
"""返回启动组合根登记的清理服务。"""
|
||||
if _configured_cleanup_service_factory is None:
|
||||
raise RuntimeError("数据清理服务尚未配置")
|
||||
return _configured_cleanup_service_factory()
|
||||
|
||||
|
||||
_configured_cleanup_service_factory: Callable[[], DataCleanupService] | None = None
|
||||
|
||||
|
||||
def configure_cleanup_service_factory(
|
||||
factory: Callable[[], DataCleanupService],
|
||||
) -> None:
|
||||
"""由启动组合根登记数据清理服务工厂。"""
|
||||
global _configured_cleanup_service_factory
|
||||
_configured_cleanup_service_factory = factory
|
||||
|
||||
|
||||
def _normalize_days(retention_days: Any) -> int:
|
||||
"""把配置保留期规范为非负整数,非法值按关闭单表清理处理。"""
|
||||
try:
|
||||
|
||||
+81
-1
@@ -17,6 +17,8 @@ import psutil
|
||||
|
||||
from app.runtime.config import Settings, settings
|
||||
from app.runtime.state import SystemHelper
|
||||
from app.application.backup import BackupArtifact
|
||||
from app.startup.database import build_database_governance
|
||||
from version import APP_VERSION
|
||||
|
||||
BACKEND_RUNTIME_FILE = settings.TEMP_PATH / "moviepilot.runtime.json"
|
||||
@@ -839,6 +841,84 @@ def cli() -> None:
|
||||
"""MoviePilot 本地 CLI"""
|
||||
|
||||
|
||||
def _format_backup_artifact(artifact: BackupArtifact) -> str:
|
||||
"""将制品信息格式化为不含数据库凭据的单行 CLI 输出。"""
|
||||
return "\t".join(
|
||||
(
|
||||
artifact.name,
|
||||
artifact.db_type,
|
||||
artifact.created_at.isoformat(),
|
||||
str(artifact.size),
|
||||
str(artifact.path),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@cli.group(context_settings=CONTEXT_SETTINGS)
|
||||
def database() -> None:
|
||||
"""创建、列举、校验和离线还原本地数据库备份"""
|
||||
|
||||
|
||||
@database.command("backup", context_settings=CONTEXT_SETTINGS)
|
||||
def database_backup() -> None:
|
||||
"""创建并验证一个在线数据库备份"""
|
||||
try:
|
||||
artifact = build_database_governance().create_backup()
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份失败:{error}") from error
|
||||
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
|
||||
click.echo(_format_backup_artifact(artifact))
|
||||
|
||||
|
||||
@database.command("list", context_settings=CONTEXT_SETTINGS)
|
||||
def database_list() -> None:
|
||||
"""列出当前受管目录中的正式数据库备份文件"""
|
||||
try:
|
||||
artifacts = build_database_governance().list_backups()
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份列表读取失败:{error}") from error
|
||||
if not artifacts:
|
||||
click.echo("暂无数据库备份")
|
||||
return
|
||||
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
|
||||
for artifact in artifacts:
|
||||
click.echo(_format_backup_artifact(artifact))
|
||||
|
||||
|
||||
@database.command("verify", context_settings=CONTEXT_SETTINGS)
|
||||
@click.argument("name")
|
||||
def database_verify(name: str) -> None:
|
||||
"""按文件名重新校验一个数据库备份"""
|
||||
try:
|
||||
result = build_database_governance().verify_backup(name)
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份校验失败:{error}") from error
|
||||
if not result.valid:
|
||||
detail = f":{result.detail}" if result.detail else ""
|
||||
raise click.ClickException(f"数据库备份校验未通过({result.method}){detail}")
|
||||
click.echo(
|
||||
f"数据库备份校验通过:name={name} method={result.method}"
|
||||
)
|
||||
|
||||
|
||||
@database.command("restore", context_settings=CONTEXT_SETTINGS)
|
||||
@click.argument("name")
|
||||
@click.option(
|
||||
"--confirm",
|
||||
is_flag=True,
|
||||
help="确认 MoviePilot 已停止,并允许覆盖当前数据库",
|
||||
)
|
||||
def database_restore(name: str, confirm: bool) -> None:
|
||||
"""在 MoviePilot 停止运行时还原一个数据库备份"""
|
||||
if not confirm:
|
||||
raise click.ClickException("离线还原必须使用 --confirm 明确确认")
|
||||
try:
|
||||
artifact = build_database_governance().restore_backup(name)
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库还原失败:{error}") from error
|
||||
click.echo(f"数据库还原完成:{artifact.name}")
|
||||
|
||||
|
||||
@cli.command(context_settings=CONTEXT_SETTINGS)
|
||||
@click.option("--timeout", default=60, show_default=True, help="等待后端与前端就绪的秒数")
|
||||
@click.option("--safe", is_flag=True, help="安全模式启动,仅保留核心 API,跳过插件和后台任务")
|
||||
@@ -989,7 +1069,7 @@ def logs(lines: int, follow: bool, stdio: bool, frontend_log: bool) -> None:
|
||||
@click.option("--deep", is_flag=True, help="执行可能较慢的深度检查")
|
||||
def doctor(json_output: bool, fix: bool, deep: bool) -> None:
|
||||
"""离线诊断本地 MoviePilot 运行环境,插件日志告警不影响整体状态"""
|
||||
from app.doctor import run_doctor
|
||||
from app.doctor import run_doctor # pylint: disable=no-name-in-module
|
||||
from app.doctor.formatters import format_json_report, format_text_report
|
||||
|
||||
report = run_doctor(fix=fix, deep=deep)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import Tuple, Union
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.application.database import get_configured_database_health
|
||||
from app.application.database import get_database_governance
|
||||
from app.modules import _ModuleBase
|
||||
from app.schemas.types import ModuleType, OtherModulesType
|
||||
|
||||
@@ -51,7 +51,7 @@ class PostgreSQLModule(_ModuleBase):
|
||||
"""
|
||||
if settings.DB_TYPE != "postgresql":
|
||||
return None
|
||||
error = get_configured_database_health().test()
|
||||
error = get_database_governance().test()
|
||||
if error:
|
||||
return False, f"PostgreSQL连接失败:{error}"
|
||||
return True, ""
|
||||
|
||||
@@ -189,6 +189,17 @@ class ConfigModel(BaseModel):
|
||||
# 例如经 PgBouncer 事务模式接入时 asyncpg 需要 {"statement_cache_size": 0}
|
||||
DB_CONNECT_ARGS: dict = Field(default_factory=dict)
|
||||
|
||||
# ==================== 数据库备份配置 ====================
|
||||
# 是否启用主程序数据库自动备份
|
||||
DB_BACKUP_ENABLE: bool = False
|
||||
# 定时备份的 Cron 表达式,留空时不注册定时任务
|
||||
DB_BACKUP_CRON: str = "0 3 * * *"
|
||||
# 备份根目录;未配置时使用 CONFIG_PATH/database_backup
|
||||
DB_BACKUP_PATH: Optional[str] = None
|
||||
# 本地备份的保留天数,0 表示不按时间清理
|
||||
DB_BACKUP_RETENTION_DAYS: int = 30
|
||||
# 本地备份的最大保留份数,0 表示不按数量清理
|
||||
DB_BACKUP_MAX_COUNT: int = 30
|
||||
# ==================== 数据清理配置 ====================
|
||||
# 是否启用数据表定时清理
|
||||
DATA_CLEANUP_ENABLE: bool = False
|
||||
@@ -1074,6 +1085,15 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
"""返回插件持久化数据目录。"""
|
||||
return self.CONFIG_PATH / "plugins"
|
||||
|
||||
@property
|
||||
def DATABASE_BACKUP_PATH(self) -> Path:
|
||||
"""返回数据库备份根目录,允许相对当前配置目录进行配置。"""
|
||||
configured = str(self.DB_BACKUP_PATH or "").strip()
|
||||
if not configured:
|
||||
return self.CONFIG_PATH / "database_backup"
|
||||
path = Path(configured).expanduser()
|
||||
return path if path.is_absolute() else self.CONFIG_PATH / path
|
||||
|
||||
@property
|
||||
def LOG_PATH(self):
|
||||
"""返回应用日志目录。"""
|
||||
|
||||
+36
-2
@@ -28,7 +28,7 @@ from app.runtime.events import Event, eventmanager
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.db.oper.agenttask import AgentTaskOper
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.application.maintenance import build_cleanup_service
|
||||
from app.application.database import get_database_governance
|
||||
from app.application.image import WallpaperHelper
|
||||
from app.application.messaging.message import MessageHelper
|
||||
from app.runtime.progress import AsyncProgressHelper, ProgressHelper
|
||||
@@ -66,7 +66,7 @@ class SchedulerChain(ChainBase):
|
||||
"""
|
||||
按配置保留期执行分批清理。
|
||||
"""
|
||||
return build_cleanup_service().execute(
|
||||
return get_database_governance().cleanup(
|
||||
batch_size=batch_size,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
@@ -94,6 +94,8 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"DATA_CLEANUP_DOWNLOAD_HISTORY_DAYS",
|
||||
"DATA_CLEANUP_SITE_USERDATA_DAYS",
|
||||
"DATA_CLEANUP_TRANSFER_HISTORY_DAYS",
|
||||
"DB_BACKUP_ENABLE",
|
||||
"DB_BACKUP_CRON",
|
||||
"USAGE_STATISTIC_SHARE",
|
||||
}
|
||||
|
||||
@@ -192,6 +194,35 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
return (value or datetime.now()).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@staticmethod
|
||||
def database_backup():
|
||||
"""按当前宿主策略创建一次定时数据库备份。"""
|
||||
return get_database_governance().create_backup()
|
||||
|
||||
def _register_database_backup_job(self) -> None:
|
||||
"""在共享调度器中按当前配置维护唯一的数据库备份作业。"""
|
||||
if not settings.DB_BACKUP_ENABLE or not settings.DB_BACKUP_CRON.strip():
|
||||
return
|
||||
|
||||
job_id = "database_backup"
|
||||
self._jobs[job_id] = {
|
||||
"name": "数据库备份",
|
||||
"func": self.database_backup,
|
||||
"running": False,
|
||||
}
|
||||
self._scheduler.add_job(
|
||||
self.start,
|
||||
trigger=TimerUtils.build_schedule_trigger(
|
||||
trigger_type="cron",
|
||||
trigger_value=settings.DB_BACKUP_CRON,
|
||||
timezone_name=settings.TZ,
|
||||
),
|
||||
id=job_id,
|
||||
name="数据库备份",
|
||||
kwargs={"job_id": job_id},
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
def init(self) -> None:
|
||||
"""
|
||||
初始化定时服务
|
||||
@@ -323,6 +354,9 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
executors={"default": ThreadPoolExecutor(settings.CONF.scheduler)},
|
||||
)
|
||||
|
||||
# 数据库备份复用宿主调度器,不创建独立定时线程。
|
||||
self._register_database_backup_job()
|
||||
|
||||
# CookieCloud定时同步
|
||||
if (
|
||||
settings.COOKIECLOUD_INTERVAL
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""插件可使用的数据库备份只读门面。"""
|
||||
|
||||
from app.application.backup import BackupArtifact, BackupVerification
|
||||
from app.application.database import get_database_governance as _get_database_governance
|
||||
|
||||
|
||||
def create_backup() -> BackupArtifact:
|
||||
"""在宿主管理目录中创建当前数据库的一致备份。"""
|
||||
return _get_database_governance().create_backup()
|
||||
|
||||
|
||||
def list_backups() -> tuple[BackupArtifact, ...]:
|
||||
"""列出宿主管理目录中的数据库备份。"""
|
||||
return _get_database_governance().list_backups()
|
||||
|
||||
|
||||
def verify_backup(name: str) -> BackupVerification:
|
||||
"""按文件名校验一个宿主管理的数据库备份。"""
|
||||
return _get_database_governance().verify_backup(name)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""数据库治理能力的宿主组合根。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.adapters.system.backup.database import (
|
||||
PostgreSQLBackupBackend,
|
||||
SQLiteBackupBackend,
|
||||
)
|
||||
from app.application.backup import BackupPolicy, DatabaseBackupService
|
||||
from app.application.database import (
|
||||
DatabaseGovernance,
|
||||
DatabaseHealthService,
|
||||
configure_database_governance,
|
||||
)
|
||||
from app.application.maintenance import (
|
||||
DataCleanupService,
|
||||
read_cleanup_policy,
|
||||
)
|
||||
from app.db.engine import get_engine
|
||||
from app.db.health import probe_database
|
||||
from app.db.maintenance import DatabaseCleanupRepository
|
||||
from app.db.session import SessionFactory
|
||||
from app.runtime.config import settings
|
||||
|
||||
|
||||
def build_database_governance() -> DatabaseGovernance:
|
||||
"""以缓存同步引擎为事实源构造一个完整数据库治理门面。"""
|
||||
engine = get_engine()
|
||||
dialect = engine.dialect.name
|
||||
if dialect == "sqlite":
|
||||
backup_backend = SQLiteBackupBackend(engine)
|
||||
elif dialect == "postgresql":
|
||||
backup_backend = PostgreSQLBackupBackend(engine)
|
||||
else:
|
||||
raise RuntimeError(f"不支持的数据库类型:{dialect}")
|
||||
|
||||
return DatabaseGovernance(
|
||||
health=DatabaseHealthService(probe_database),
|
||||
cleanup=DataCleanupService(
|
||||
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
|
||||
policy_reader=read_cleanup_policy,
|
||||
),
|
||||
backup=DatabaseBackupService(
|
||||
backend=backup_backend,
|
||||
policy_reader=read_backup_policy,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def configure_database() -> None:
|
||||
"""构造并登记宿主进程唯一的数据库治理门面。"""
|
||||
configure_database_governance(build_database_governance())
|
||||
|
||||
|
||||
def read_backup_policy() -> BackupPolicy:
|
||||
"""读取一次可热更新的数据库备份目录与保留策略。"""
|
||||
return BackupPolicy(
|
||||
root=settings.DATABASE_BACKUP_PATH,
|
||||
retention_days=settings.DB_BACKUP_RETENTION_DAYS,
|
||||
max_count=settings.DB_BACKUP_MAX_COUNT,
|
||||
)
|
||||
@@ -36,7 +36,7 @@ from app.application.messaging.message import (
|
||||
stop_message,
|
||||
)
|
||||
from app.application.configuration import SystemConfigService, configure_system_config
|
||||
from app.application.database import DatabaseHealthService, configure_database_health
|
||||
from app.application.database import configure_database_governance
|
||||
from app.application.service import configure_service_directory
|
||||
from app.application.plugin.runtime import configure_plugin_runtime
|
||||
from app.application.module import configure_module_runtime
|
||||
@@ -55,19 +55,13 @@ from app.application.workflow import WorkflowQueryService, configure_workflow_qu
|
||||
from app.application.agentdata import configure_agent_data_ports
|
||||
from app.api.data import configure_api_data_ports
|
||||
from app.application.subscription.write import configure_subscribe_writer
|
||||
from app.application.maintenance import (
|
||||
DataCleanupService,
|
||||
configure_cleanup_service_factory,
|
||||
read_cleanup_policy,
|
||||
)
|
||||
from app.adapters.external.server import (
|
||||
MoviePilotServerHelper,
|
||||
configure_server_application_services,
|
||||
)
|
||||
from app.application.server.report import ServerReportService
|
||||
from app.application.server.share import ServerSharingService
|
||||
from app.db import close_database
|
||||
from app.db.session import get_async_db, get_db
|
||||
from app.db.session import close_database, get_async_db, get_db
|
||||
from app.db.uow import SqlAlchemyAsyncUnitOfWork, SqlAlchemyUnitOfWork
|
||||
from app.db.oper.subscribe import SubscribeOper
|
||||
from app.db.oper.agentchat import AgentChatOper
|
||||
@@ -84,9 +78,6 @@ from app.db.oper.site import SiteOper
|
||||
from app.db.oper.message import MessageOper
|
||||
from app.db.oper.subscribehistory import SubscribeHistoryOper
|
||||
from app.db.oper.plugindata import PluginDataOper
|
||||
from app.db.maintenance import DatabaseCleanupRepository
|
||||
from app.db.session import SessionFactory
|
||||
from app.db.health import probe_database
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.db.oper.workflow import WorkflowOper
|
||||
from app.command import CommandChain
|
||||
@@ -94,6 +85,7 @@ from app.schemas.message import Message
|
||||
from app.schemas.message import MessageType
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.startup.agent_initializer import init_agent, stop_agent
|
||||
from app.startup.database import build_database_governance
|
||||
from app.startup.managed_resources_initializer import (
|
||||
init_managed_resources,
|
||||
stop_managed_resources,
|
||||
@@ -441,7 +433,7 @@ async def init_modules():
|
||||
user=lambda: UserOper(),
|
||||
)
|
||||
configure_system_config(SystemConfigService(repository=SystemConfigOper()))
|
||||
configure_database_health(DatabaseHealthService(probe_database))
|
||||
configure_database_governance(build_database_governance())
|
||||
configure_agent_chat_service(AgentChatService(repository=AgentChatOper()))
|
||||
configure_user_lookups(
|
||||
by_id=lambda user_id: UserOper().get_by_id(user_id),
|
||||
@@ -474,12 +466,6 @@ async def init_modules():
|
||||
plugin_data=lambda: PluginDataOper(),
|
||||
)
|
||||
configure_subscribe_writer(lambda: SubscribeOper())
|
||||
configure_cleanup_service_factory(
|
||||
lambda: DataCleanupService(
|
||||
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
|
||||
policy_reader=read_cleanup_policy,
|
||||
)
|
||||
)
|
||||
# 托管资源只在这里装配声明与 adapter,具体资源仍由首个消费者显式激活。
|
||||
init_managed_resources()
|
||||
# 应用服务不反向依赖 Chain,由启动组合层注入壁纸来源。
|
||||
|
||||
Reference in New Issue
Block a user