mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-29 03:56:43 +08:00
feat(database): add managed backup and offline restore (#6359)
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""数据库备份的文件系统与数据库技术适配器命名空间。"""
|
||||
@@ -0,0 +1,209 @@
|
||||
"""基于活动 SQLAlchemy 引擎的 SQLite 与 PostgreSQL 备份实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Protocol, Sequence
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatabaseBackupCheck:
|
||||
"""数据库适配器返回的基础校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class ProcessResult(Protocol):
|
||||
"""数据库命令执行结果的最小合同。"""
|
||||
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
class ProcessRunner(Protocol):
|
||||
"""可替换的数据库命令执行边界。"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
env: Mapping[str, str],
|
||||
capture_output: bool,
|
||||
text: bool,
|
||||
check: bool,
|
||||
) -> ProcessResult:
|
||||
"""执行命令并返回结果。"""
|
||||
|
||||
|
||||
class SQLiteBackupBackend:
|
||||
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
|
||||
|
||||
db_type = "sqlite"
|
||||
suffix = ".db"
|
||||
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
self._engine = engine
|
||||
database = engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
raise ValueError("SQLite 内存数据库不支持文件备份")
|
||||
self._database = Path(database)
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""从活动引擎指向的 SQLite 文件创建一致快照。"""
|
||||
source = self._engine.raw_connection()
|
||||
try:
|
||||
with closing(sqlite3.connect(destination)) as target:
|
||||
source.driver_connection.backup(target)
|
||||
target.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 SQLite integrity_check 校验备份内容。"""
|
||||
method = "PRAGMA integrity_check"
|
||||
try:
|
||||
# 已发布前的临时快照不会再变化;immutable 避免 WAL 模式为只读校验创建旁路文件。
|
||||
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
|
||||
with closing(sqlite3.connect(uri, uri=True)) as connection:
|
||||
rows = connection.execute("PRAGMA integrity_check").fetchall()
|
||||
except sqlite3.Error as error:
|
||||
return DatabaseBackupCheck(False, method, str(error))
|
||||
valid = bool(rows) and all(row[0] == "ok" for row in rows)
|
||||
detail = None if valid else "; ".join(str(row[0]) for row in rows)
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
|
||||
temporary = self._database.with_name(f".{self._database.name}.restore")
|
||||
self._engine.dispose()
|
||||
try:
|
||||
shutil.copy2(artifact, temporary)
|
||||
temporary.chmod(0o600)
|
||||
self._database.with_name(f"{self._database.name}-wal").unlink(missing_ok=True)
|
||||
self._database.with_name(f"{self._database.name}-shm").unlink(missing_ok=True)
|
||||
os.replace(temporary, self._database)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class PostgreSQLBackupBackend:
|
||||
"""使用 pg_dump 与 pg_restore 管理活动 PostgreSQL 数据库。"""
|
||||
|
||||
db_type = "postgresql"
|
||||
suffix = ".dump"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine,
|
||||
*,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_dump: str = "pg_dump",
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._runner = runner
|
||||
self._tool_resolver = tool_resolver
|
||||
self._pg_dump = pg_dump
|
||||
self._pg_restore = pg_restore
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""创建 PostgreSQL custom-format 在线备份。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_dump),
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--file",
|
||||
str(destination),
|
||||
*self._connection_arguments(),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_dump 执行失败,退出码 {result.returncode}")
|
||||
if not destination.is_file() or destination.stat().st_size == 0:
|
||||
raise RuntimeError("pg_dump 未生成有效的备份文件")
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
|
||||
method = "pg_restore --list"
|
||||
result = self._run(
|
||||
[self._require_tool(self._pg_restore), "--list", str(artifact)],
|
||||
include_password=False,
|
||||
)
|
||||
valid = result.returncode == 0 and bool(result.stdout.strip())
|
||||
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_restore),
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--single-transaction",
|
||||
"--exit-on-error",
|
||||
*self._connection_arguments(),
|
||||
str(artifact),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_restore 执行失败,退出码 {result.returncode}")
|
||||
|
||||
def _connection_arguments(self) -> list[str]:
|
||||
url = self._engine.url
|
||||
host = str(url.query.get("host") or url.host or "")
|
||||
port = str(url.query.get("port") or url.port or "")
|
||||
arguments = [
|
||||
"--username",
|
||||
str(url.username or ""),
|
||||
"--dbname",
|
||||
str(url.database or ""),
|
||||
]
|
||||
if host:
|
||||
arguments.extend(["--host", host])
|
||||
if port:
|
||||
arguments.extend(["--port", port])
|
||||
return arguments
|
||||
|
||||
def _run(self, command: Sequence[str], *, include_password: bool) -> ProcessResult:
|
||||
return self._runner(
|
||||
command,
|
||||
env=self._environment(include_password=include_password),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _require_tool(self, executable: str) -> str:
|
||||
resolved = self._tool_resolver(executable)
|
||||
if resolved is None:
|
||||
raise RuntimeError(
|
||||
f"未找到 {executable},请安装与服务端同主版本或更高的 "
|
||||
"PostgreSQL client 并加入 PATH"
|
||||
)
|
||||
return resolved
|
||||
|
||||
def _environment(self, *, include_password: bool) -> dict[str, str]:
|
||||
environment = dict(os.environ)
|
||||
environment.pop("PGPASSWORD", None)
|
||||
environment.pop("PGSSLMODE", None)
|
||||
if include_password and self._engine.url.password:
|
||||
environment["PGPASSWORD"] = str(self._engine.url.password)
|
||||
sslmode = self._engine.url.query.get("sslmode")
|
||||
if sslmode:
|
||||
environment["PGSSLMODE"] = str(sslmode)
|
||||
return environment
|
||||
@@ -0,0 +1,116 @@
|
||||
"""数据库备份单文件的受限文件系统操作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
_BACKUP_NAME = re.compile(
|
||||
r"^(?P<db_type>sqlite|postgresql)_"
|
||||
r"(?P<timestamp>\d{8}_\d{6})"
|
||||
r"(?:_(?P<sequence>\d+))?"
|
||||
r"(?P<suffix>\.db|\.dump)$"
|
||||
)
|
||||
|
||||
|
||||
class BackupFiles:
|
||||
"""把备份文件操作限制在一个私有根目录内。"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def create_temporary(self, suffix: str) -> Path:
|
||||
"""在最终目录内创建私有临时文件,保证发布可使用原子替换。"""
|
||||
self._ensure_root()
|
||||
descriptor, filename = tempfile.mkstemp(
|
||||
prefix=".database-",
|
||||
suffix=f"{suffix}.partial",
|
||||
dir=self.root,
|
||||
)
|
||||
os.close(descriptor)
|
||||
path = Path(filename)
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
def publish(self, temporary: Path, name: str) -> Path:
|
||||
"""把已校验临时文件发布为正式备份文件。"""
|
||||
destination = self._resolve_name(name, require_exists=False)
|
||||
os.replace(temporary, destination)
|
||||
destination.chmod(0o600)
|
||||
return destination
|
||||
|
||||
def discard(self, temporary: Path) -> None:
|
||||
"""清理本次操作拥有的未发布临时文件。"""
|
||||
path = Path(temporary)
|
||||
if path.parent == self.root and path.name.startswith(".database-"):
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
def list(self) -> list[Path]:
|
||||
"""返回当前根目录内格式合法的正式备份文件。"""
|
||||
if not self.root.is_dir():
|
||||
return []
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.is_file() and _BACKUP_NAME.fullmatch(path.name)
|
||||
]
|
||||
return sorted(paths, key=lambda path: (self.created_at(path.name), path.name), reverse=True)
|
||||
|
||||
def resolve(self, name: str) -> Path:
|
||||
"""按受限文件名解析一个必须存在的备份文件。"""
|
||||
return self._resolve_name(name, require_exists=True)
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
"""删除一个已通过名称约束的备份文件。"""
|
||||
self.resolve(name).unlink()
|
||||
|
||||
def available_name(self, *, db_type: str, created_at: datetime, suffix: str) -> str:
|
||||
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
|
||||
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
|
||||
base = f"{db_type}_{timestamp}"
|
||||
candidate = f"{base}{suffix}"
|
||||
sequence = 1
|
||||
while (self.root / candidate).exists():
|
||||
candidate = f"{base}_{sequence}{suffix}"
|
||||
sequence += 1
|
||||
if not _BACKUP_NAME.fullmatch(candidate):
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def database_type(name: str) -> str:
|
||||
"""从受管文件名读取数据库类型。"""
|
||||
return BackupFiles._match(name).group("db_type")
|
||||
|
||||
@staticmethod
|
||||
def created_at(name: str) -> datetime:
|
||||
"""从受管文件名读取本地创建时间。"""
|
||||
return datetime.strptime(
|
||||
BackupFiles._match(name).group("timestamp"),
|
||||
"%Y%m%d_%H%M%S",
|
||||
)
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.root.chmod(0o700)
|
||||
|
||||
def _resolve_name(self, name: str, *, require_exists: bool) -> Path:
|
||||
normalized = str(name).strip()
|
||||
self._match(normalized)
|
||||
if Path(normalized).name != normalized:
|
||||
raise ValueError("数据库备份文件名不能包含路径")
|
||||
path = self.root / normalized
|
||||
if require_exists and not path.is_file():
|
||||
raise FileNotFoundError(normalized)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _match(name: str) -> re.Match[str]:
|
||||
matched = _BACKUP_NAME.fullmatch(str(name))
|
||||
if matched is None:
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return matched
|
||||
@@ -53,6 +53,7 @@ from app.adapters.external.market import (
|
||||
)
|
||||
from app.application.messaging.message import MessageHelper
|
||||
from app.runtime.progress import AsyncProgressHelper
|
||||
from app.runtime.scheduling import TimerUtils
|
||||
from app.application.rules import RuleHelper
|
||||
from app.adapters.external.server import MoviePilotServerHelper
|
||||
from app.runtime.state import SystemHelper
|
||||
@@ -86,6 +87,13 @@ _PUBLIC_SYSTEM_CONFIG_KEYS = {
|
||||
_PUBLIC_SETTINGS_KEYS = {"PLUGIN_MARKET"}
|
||||
_LOG_DOWNLOAD_LIMIT = 10
|
||||
_LOG_DOWNLOAD_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
|
||||
_DATABASE_BACKUP_SETTING_KEYS = {
|
||||
"DB_BACKUP_ENABLE",
|
||||
"DB_BACKUP_CRON",
|
||||
"DB_BACKUP_PATH",
|
||||
"DB_BACKUP_RETENTION_DAYS",
|
||||
"DB_BACKUP_MAX_COUNT",
|
||||
}
|
||||
|
||||
|
||||
def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
|
||||
@@ -129,6 +137,38 @@ def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
|
||||
)
|
||||
|
||||
|
||||
def _validate_database_backup_config(env: dict) -> Optional[str]:
|
||||
"""在批量写入前校验数据库备份策略,避免只保存部分字段。"""
|
||||
if not _DATABASE_BACKUP_SETTING_KEYS.intersection(env):
|
||||
return None
|
||||
|
||||
cron = str(env.get("DB_BACKUP_CRON", settings.DB_BACKUP_CRON) or "").strip()
|
||||
if cron:
|
||||
try:
|
||||
TimerUtils.normalize_schedule_trigger("cron", cron, settings.TZ)
|
||||
except (TypeError, ValueError):
|
||||
return "数据库备份周期格式不正确"
|
||||
|
||||
backup_path = env.get("DB_BACKUP_PATH", settings.DB_BACKUP_PATH)
|
||||
if backup_path is not None and not isinstance(backup_path, str):
|
||||
return "数据库备份目录必须是路径字符串"
|
||||
|
||||
for key, label in (
|
||||
("DB_BACKUP_RETENTION_DAYS", "数据库备份过期天数"),
|
||||
("DB_BACKUP_MAX_COUNT", "数据库备份最大保留份数"),
|
||||
):
|
||||
value = env.get(key, getattr(settings, key))
|
||||
if isinstance(value, bool):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
try:
|
||||
converted = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
if converted < 0 or str(value).strip() != str(converted):
|
||||
return f"{label}必须是大于等于 0 的整数"
|
||||
return None
|
||||
|
||||
|
||||
def _is_allowed_plugin_market_wiki_url(wiki_url: str) -> bool:
|
||||
"""
|
||||
校验插件市场 Wiki 地址是否属于固定文档源。
|
||||
@@ -796,6 +836,9 @@ async def set_env_setting(
|
||||
更新系统环境变量(仅管理员)
|
||||
"""
|
||||
validation_error = _validate_llm_server_tool_config(env)
|
||||
if validation_error:
|
||||
return _SchemaResponse(success=False, message=validation_error)
|
||||
validation_error = _validate_database_backup_config(env)
|
||||
if validation_error:
|
||||
return _SchemaResponse(success=False, message=validation_error)
|
||||
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""数据库备份与离线还原用例。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Callable, Protocol
|
||||
|
||||
from app.adapters.system.backup.files import BackupFiles
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupPolicy:
|
||||
"""一次备份操作使用的目录与保留策略快照。"""
|
||||
|
||||
root: Path
|
||||
retention_days: int = 0
|
||||
max_count: int = 0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.retention_days < 0 or self.max_count < 0:
|
||||
raise ValueError("数据库备份保留策略不能使用负数")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupArtifact:
|
||||
"""一个已完成校验并发布的数据库备份文件。"""
|
||||
|
||||
name: str
|
||||
db_type: str
|
||||
created_at: datetime
|
||||
path: Path
|
||||
size: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BackupVerification:
|
||||
"""数据库备份文件的基础内容校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class BackupCheck(Protocol):
|
||||
"""数据库适配器校验结果的结构合同。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None
|
||||
|
||||
|
||||
class DatabaseBackupBackend(Protocol):
|
||||
"""活动数据库创建、校验和离线还原所需的最小技术端口。"""
|
||||
|
||||
db_type: str
|
||||
suffix: str
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""把活动数据库的一致快照写入目标文件。"""
|
||||
|
||||
def verify(self, artifact: Path) -> BackupCheck:
|
||||
"""在不修改数据库的前提下校验备份文件。"""
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""把已校验制品还原到离线目标数据库。"""
|
||||
|
||||
|
||||
class DatabaseBackupService:
|
||||
"""管理单文件数据库备份及明确的离线还原操作。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
backend: DatabaseBackupBackend,
|
||||
policy_reader: Callable[[], BackupPolicy],
|
||||
clock: Callable[[], datetime] = datetime.now,
|
||||
) -> None:
|
||||
self._backend = backend
|
||||
self._policy_reader = policy_reader
|
||||
self._clock = clock
|
||||
|
||||
def create(self) -> BackupArtifact:
|
||||
"""创建、校验并发布一个在线一致快照。"""
|
||||
policy = self._policy_reader()
|
||||
files = BackupFiles(policy.root)
|
||||
created_at = self._clock()
|
||||
name = files.available_name(
|
||||
db_type=self._backend.db_type,
|
||||
created_at=created_at,
|
||||
suffix=self._backend.suffix,
|
||||
)
|
||||
temporary = files.create_temporary(self._backend.suffix)
|
||||
try:
|
||||
self._backend.create(temporary)
|
||||
verification = self._backend.verify(temporary)
|
||||
if not verification.valid:
|
||||
detail = f":{verification.detail}" if verification.detail else ""
|
||||
raise RuntimeError(
|
||||
f"数据库备份校验失败({verification.method}){detail}"
|
||||
)
|
||||
path = files.publish(temporary, name)
|
||||
except Exception:
|
||||
files.discard(temporary)
|
||||
raise
|
||||
|
||||
artifact = self._artifact(path, created_at=created_at)
|
||||
self._prune(files, policy, keep=artifact.name)
|
||||
logger.info(
|
||||
f"数据库备份完成:文件={artifact.name},类型={artifact.db_type},"
|
||||
f"大小={artifact.size} bytes"
|
||||
)
|
||||
return artifact
|
||||
|
||||
def list(self) -> tuple[BackupArtifact, ...]:
|
||||
"""按创建时间倒序列出受管数据库备份文件。"""
|
||||
files = BackupFiles(self._policy_reader().root)
|
||||
return tuple(self._artifact(path) for path in files.list())
|
||||
|
||||
def verify(self, name: str) -> BackupVerification:
|
||||
"""按文件名校验一个受管数据库备份。"""
|
||||
path = BackupFiles(self._policy_reader().root).resolve(name)
|
||||
self._require_matching_type(path)
|
||||
result = self._backend.verify(path)
|
||||
return BackupVerification(result.valid, result.method, result.detail)
|
||||
|
||||
def restore(self, name: str) -> BackupArtifact:
|
||||
"""校验后将受管制品还原到当前 CLI 解析出的离线数据库目标。"""
|
||||
path = BackupFiles(self._policy_reader().root).resolve(name)
|
||||
self._require_matching_type(path)
|
||||
verification = self._backend.verify(path)
|
||||
if not verification.valid:
|
||||
detail = f":{verification.detail}" if verification.detail else ""
|
||||
raise RuntimeError(
|
||||
f"数据库备份校验失败({verification.method}){detail}"
|
||||
)
|
||||
self._backend.restore(path)
|
||||
logger.info(
|
||||
f"数据库离线还原完成:文件={path.name},类型={self._backend.db_type}"
|
||||
)
|
||||
return self._artifact(path)
|
||||
|
||||
def _prune(self, files: BackupFiles, policy: BackupPolicy, *, keep: str) -> None:
|
||||
"""新备份发布成功后按天数或份数清理同一目录中的旧文件。"""
|
||||
cutoff = (
|
||||
self._clock() - timedelta(days=policy.retention_days)
|
||||
if policy.retention_days > 0
|
||||
else None
|
||||
)
|
||||
for index, path in enumerate(files.list()):
|
||||
if path.name == keep:
|
||||
continue
|
||||
created_at = files.created_at(path.name)
|
||||
expired = cutoff is not None and created_at < cutoff
|
||||
exceeds_count = policy.max_count > 0 and index >= policy.max_count
|
||||
if expired or exceeds_count:
|
||||
files.delete(path.name)
|
||||
|
||||
def _require_matching_type(self, path: Path) -> None:
|
||||
db_type = BackupFiles.database_type(path.name)
|
||||
if db_type != self._backend.db_type:
|
||||
raise ValueError(
|
||||
f"备份类型 {db_type} 与当前数据库类型 {self._backend.db_type} 不一致"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _artifact(path: Path, *, created_at: datetime | None = None) -> BackupArtifact:
|
||||
return BackupArtifact(
|
||||
name=path.name,
|
||||
db_type=BackupFiles.database_type(path.name),
|
||||
created_at=created_at or BackupFiles.created_at(path.name),
|
||||
path=path,
|
||||
size=path.stat().st_size,
|
||||
)
|
||||
+67
-14
@@ -1,19 +1,25 @@
|
||||
"""数据库连通性应用服务。"""
|
||||
"""数据库健康、清理、备份与离线还原的统一应用门面。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.application.backup import (
|
||||
BackupArtifact,
|
||||
BackupVerification,
|
||||
DatabaseBackupService,
|
||||
)
|
||||
from app.application.maintenance import DataCleanupService
|
||||
|
||||
|
||||
DatabaseProbe = Callable[[], Optional[str]]
|
||||
|
||||
|
||||
class DatabaseHealthService:
|
||||
"""为模块和诊断入口提供不暴露会话实现的数据库探测能力。"""
|
||||
"""提供不暴露会话实现的数据库探测能力。"""
|
||||
|
||||
def __init__(self, probe: DatabaseProbe) -> None:
|
||||
"""保存由组合根提供的数据库探测端口。"""
|
||||
self._probe = probe
|
||||
|
||||
def test(self) -> Optional[str]:
|
||||
@@ -21,17 +27,64 @@ class DatabaseHealthService:
|
||||
return self._probe()
|
||||
|
||||
|
||||
_configured_database_health: DatabaseHealthService | None = None
|
||||
class DatabaseGovernance:
|
||||
"""向宿主入口提供唯一的数据库治理能力入口。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
health: DatabaseHealthService,
|
||||
cleanup: DataCleanupService,
|
||||
backup: DatabaseBackupService,
|
||||
) -> None:
|
||||
self._health = health
|
||||
self._cleanup = cleanup
|
||||
self._backup = backup
|
||||
|
||||
def test(self) -> Optional[str]:
|
||||
"""探测当前活动数据库。"""
|
||||
return self._health.test()
|
||||
|
||||
def cleanup(
|
||||
self,
|
||||
*,
|
||||
batch_size: int | None = None,
|
||||
progress_callback: Callable[..., None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""按当前配置执行数据表清理。"""
|
||||
return self._cleanup.execute(
|
||||
batch_size=batch_size,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
def create_backup(self) -> BackupArtifact:
|
||||
"""创建一个当前活动数据库的一致备份。"""
|
||||
return self._backup.create()
|
||||
|
||||
def list_backups(self) -> tuple[BackupArtifact, ...]:
|
||||
"""列出受管数据库备份文件。"""
|
||||
return self._backup.list()
|
||||
|
||||
def verify_backup(self, name: str) -> BackupVerification:
|
||||
"""校验一个受管数据库备份文件。"""
|
||||
return self._backup.verify(name)
|
||||
|
||||
def restore_backup(self, name: str) -> BackupArtifact:
|
||||
"""在离线 CLI 进程中还原一个受管数据库备份。"""
|
||||
return self._backup.restore(name)
|
||||
|
||||
|
||||
def configure_database_health(service: DatabaseHealthService) -> None:
|
||||
"""由启动组合根登记数据库探测服务。"""
|
||||
global _configured_database_health
|
||||
_configured_database_health = service
|
||||
_DATABASE_GOVERNANCE: list[DatabaseGovernance] = []
|
||||
|
||||
|
||||
def get_configured_database_health() -> DatabaseHealthService:
|
||||
"""返回启动阶段登记的数据库探测服务。"""
|
||||
if _configured_database_health is None:
|
||||
raise RuntimeError("数据库探测服务尚未配置")
|
||||
return _configured_database_health
|
||||
def configure_database_governance(governance: DatabaseGovernance) -> None:
|
||||
"""由启动组合根登记宿主唯一的数据库治理门面。"""
|
||||
_DATABASE_GOVERNANCE.clear()
|
||||
_DATABASE_GOVERNANCE.append(governance)
|
||||
|
||||
|
||||
def get_database_governance() -> DatabaseGovernance:
|
||||
"""返回启动阶段登记的数据库治理门面。"""
|
||||
if not _DATABASE_GOVERNANCE:
|
||||
raise RuntimeError("数据库治理服务尚未配置")
|
||||
return _DATABASE_GOVERNANCE[0]
|
||||
|
||||
@@ -324,24 +324,6 @@ def read_cleanup_policy() -> CleanupPolicy:
|
||||
)
|
||||
|
||||
|
||||
def build_cleanup_service() -> DataCleanupService:
|
||||
"""返回启动组合根登记的清理服务。"""
|
||||
if _configured_cleanup_service_factory is None:
|
||||
raise RuntimeError("数据清理服务尚未配置")
|
||||
return _configured_cleanup_service_factory()
|
||||
|
||||
|
||||
_configured_cleanup_service_factory: Callable[[], DataCleanupService] | None = None
|
||||
|
||||
|
||||
def configure_cleanup_service_factory(
|
||||
factory: Callable[[], DataCleanupService],
|
||||
) -> None:
|
||||
"""由启动组合根登记数据清理服务工厂。"""
|
||||
global _configured_cleanup_service_factory
|
||||
_configured_cleanup_service_factory = factory
|
||||
|
||||
|
||||
def _normalize_days(retention_days: Any) -> int:
|
||||
"""把配置保留期规范为非负整数,非法值按关闭单表清理处理。"""
|
||||
try:
|
||||
|
||||
+81
-1
@@ -17,6 +17,8 @@ import psutil
|
||||
|
||||
from app.runtime.config import Settings, settings
|
||||
from app.runtime.state import SystemHelper
|
||||
from app.application.backup import BackupArtifact
|
||||
from app.startup.database import build_database_governance
|
||||
from version import APP_VERSION
|
||||
|
||||
BACKEND_RUNTIME_FILE = settings.TEMP_PATH / "moviepilot.runtime.json"
|
||||
@@ -839,6 +841,84 @@ def cli() -> None:
|
||||
"""MoviePilot 本地 CLI"""
|
||||
|
||||
|
||||
def _format_backup_artifact(artifact: BackupArtifact) -> str:
|
||||
"""将制品信息格式化为不含数据库凭据的单行 CLI 输出。"""
|
||||
return "\t".join(
|
||||
(
|
||||
artifact.name,
|
||||
artifact.db_type,
|
||||
artifact.created_at.isoformat(),
|
||||
str(artifact.size),
|
||||
str(artifact.path),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@cli.group(context_settings=CONTEXT_SETTINGS)
|
||||
def database() -> None:
|
||||
"""创建、列举、校验和离线还原本地数据库备份"""
|
||||
|
||||
|
||||
@database.command("backup", context_settings=CONTEXT_SETTINGS)
|
||||
def database_backup() -> None:
|
||||
"""创建并验证一个在线数据库备份"""
|
||||
try:
|
||||
artifact = build_database_governance().create_backup()
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份失败:{error}") from error
|
||||
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
|
||||
click.echo(_format_backup_artifact(artifact))
|
||||
|
||||
|
||||
@database.command("list", context_settings=CONTEXT_SETTINGS)
|
||||
def database_list() -> None:
|
||||
"""列出当前受管目录中的正式数据库备份文件"""
|
||||
try:
|
||||
artifacts = build_database_governance().list_backups()
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份列表读取失败:{error}") from error
|
||||
if not artifacts:
|
||||
click.echo("暂无数据库备份")
|
||||
return
|
||||
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
|
||||
for artifact in artifacts:
|
||||
click.echo(_format_backup_artifact(artifact))
|
||||
|
||||
|
||||
@database.command("verify", context_settings=CONTEXT_SETTINGS)
|
||||
@click.argument("name")
|
||||
def database_verify(name: str) -> None:
|
||||
"""按文件名重新校验一个数据库备份"""
|
||||
try:
|
||||
result = build_database_governance().verify_backup(name)
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库备份校验失败:{error}") from error
|
||||
if not result.valid:
|
||||
detail = f":{result.detail}" if result.detail else ""
|
||||
raise click.ClickException(f"数据库备份校验未通过({result.method}){detail}")
|
||||
click.echo(
|
||||
f"数据库备份校验通过:name={name} method={result.method}"
|
||||
)
|
||||
|
||||
|
||||
@database.command("restore", context_settings=CONTEXT_SETTINGS)
|
||||
@click.argument("name")
|
||||
@click.option(
|
||||
"--confirm",
|
||||
is_flag=True,
|
||||
help="确认 MoviePilot 已停止,并允许覆盖当前数据库",
|
||||
)
|
||||
def database_restore(name: str, confirm: bool) -> None:
|
||||
"""在 MoviePilot 停止运行时还原一个数据库备份"""
|
||||
if not confirm:
|
||||
raise click.ClickException("离线还原必须使用 --confirm 明确确认")
|
||||
try:
|
||||
artifact = build_database_governance().restore_backup(name)
|
||||
except Exception as error:
|
||||
raise click.ClickException(f"数据库还原失败:{error}") from error
|
||||
click.echo(f"数据库还原完成:{artifact.name}")
|
||||
|
||||
|
||||
@cli.command(context_settings=CONTEXT_SETTINGS)
|
||||
@click.option("--timeout", default=60, show_default=True, help="等待后端与前端就绪的秒数")
|
||||
@click.option("--safe", is_flag=True, help="安全模式启动,仅保留核心 API,跳过插件和后台任务")
|
||||
@@ -989,7 +1069,7 @@ def logs(lines: int, follow: bool, stdio: bool, frontend_log: bool) -> None:
|
||||
@click.option("--deep", is_flag=True, help="执行可能较慢的深度检查")
|
||||
def doctor(json_output: bool, fix: bool, deep: bool) -> None:
|
||||
"""离线诊断本地 MoviePilot 运行环境,插件日志告警不影响整体状态"""
|
||||
from app.doctor import run_doctor
|
||||
from app.doctor import run_doctor # pylint: disable=no-name-in-module
|
||||
from app.doctor.formatters import format_json_report, format_text_report
|
||||
|
||||
report = run_doctor(fix=fix, deep=deep)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
from typing import Tuple, Union
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.application.database import get_configured_database_health
|
||||
from app.application.database import get_database_governance
|
||||
from app.modules import _ModuleBase
|
||||
from app.schemas.types import ModuleType, OtherModulesType
|
||||
|
||||
@@ -51,7 +51,7 @@ class PostgreSQLModule(_ModuleBase):
|
||||
"""
|
||||
if settings.DB_TYPE != "postgresql":
|
||||
return None
|
||||
error = get_configured_database_health().test()
|
||||
error = get_database_governance().test()
|
||||
if error:
|
||||
return False, f"PostgreSQL连接失败:{error}"
|
||||
return True, ""
|
||||
|
||||
@@ -189,6 +189,17 @@ class ConfigModel(BaseModel):
|
||||
# 例如经 PgBouncer 事务模式接入时 asyncpg 需要 {"statement_cache_size": 0}
|
||||
DB_CONNECT_ARGS: dict = Field(default_factory=dict)
|
||||
|
||||
# ==================== 数据库备份配置 ====================
|
||||
# 是否启用主程序数据库自动备份
|
||||
DB_BACKUP_ENABLE: bool = False
|
||||
# 定时备份的 Cron 表达式,留空时不注册定时任务
|
||||
DB_BACKUP_CRON: str = "0 3 * * *"
|
||||
# 备份根目录;未配置时使用 CONFIG_PATH/database_backup
|
||||
DB_BACKUP_PATH: Optional[str] = None
|
||||
# 本地备份的保留天数,0 表示不按时间清理
|
||||
DB_BACKUP_RETENTION_DAYS: int = 30
|
||||
# 本地备份的最大保留份数,0 表示不按数量清理
|
||||
DB_BACKUP_MAX_COUNT: int = 30
|
||||
# ==================== 数据清理配置 ====================
|
||||
# 是否启用数据表定时清理
|
||||
DATA_CLEANUP_ENABLE: bool = False
|
||||
@@ -1074,6 +1085,15 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
|
||||
"""返回插件持久化数据目录。"""
|
||||
return self.CONFIG_PATH / "plugins"
|
||||
|
||||
@property
|
||||
def DATABASE_BACKUP_PATH(self) -> Path:
|
||||
"""返回数据库备份根目录,允许相对当前配置目录进行配置。"""
|
||||
configured = str(self.DB_BACKUP_PATH or "").strip()
|
||||
if not configured:
|
||||
return self.CONFIG_PATH / "database_backup"
|
||||
path = Path(configured).expanduser()
|
||||
return path if path.is_absolute() else self.CONFIG_PATH / path
|
||||
|
||||
@property
|
||||
def LOG_PATH(self):
|
||||
"""返回应用日志目录。"""
|
||||
|
||||
+36
-2
@@ -28,7 +28,7 @@ from app.runtime.events import Event, eventmanager
|
||||
from app.runtime.extensions.plugin_manager import PluginManager
|
||||
from app.db.oper.agenttask import AgentTaskOper
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.application.maintenance import build_cleanup_service
|
||||
from app.application.database import get_database_governance
|
||||
from app.application.image import WallpaperHelper
|
||||
from app.application.messaging.message import MessageHelper
|
||||
from app.runtime.progress import AsyncProgressHelper, ProgressHelper
|
||||
@@ -66,7 +66,7 @@ class SchedulerChain(ChainBase):
|
||||
"""
|
||||
按配置保留期执行分批清理。
|
||||
"""
|
||||
return build_cleanup_service().execute(
|
||||
return get_database_governance().cleanup(
|
||||
batch_size=batch_size,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
@@ -94,6 +94,8 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"DATA_CLEANUP_DOWNLOAD_HISTORY_DAYS",
|
||||
"DATA_CLEANUP_SITE_USERDATA_DAYS",
|
||||
"DATA_CLEANUP_TRANSFER_HISTORY_DAYS",
|
||||
"DB_BACKUP_ENABLE",
|
||||
"DB_BACKUP_CRON",
|
||||
"USAGE_STATISTIC_SHARE",
|
||||
}
|
||||
|
||||
@@ -192,6 +194,35 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
"""
|
||||
return (value or datetime.now()).strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
@staticmethod
|
||||
def database_backup():
|
||||
"""按当前宿主策略创建一次定时数据库备份。"""
|
||||
return get_database_governance().create_backup()
|
||||
|
||||
def _register_database_backup_job(self) -> None:
|
||||
"""在共享调度器中按当前配置维护唯一的数据库备份作业。"""
|
||||
if not settings.DB_BACKUP_ENABLE or not settings.DB_BACKUP_CRON.strip():
|
||||
return
|
||||
|
||||
job_id = "database_backup"
|
||||
self._jobs[job_id] = {
|
||||
"name": "数据库备份",
|
||||
"func": self.database_backup,
|
||||
"running": False,
|
||||
}
|
||||
self._scheduler.add_job(
|
||||
self.start,
|
||||
trigger=TimerUtils.build_schedule_trigger(
|
||||
trigger_type="cron",
|
||||
trigger_value=settings.DB_BACKUP_CRON,
|
||||
timezone_name=settings.TZ,
|
||||
),
|
||||
id=job_id,
|
||||
name="数据库备份",
|
||||
kwargs={"job_id": job_id},
|
||||
replace_existing=True,
|
||||
)
|
||||
|
||||
def init(self) -> None:
|
||||
"""
|
||||
初始化定时服务
|
||||
@@ -323,6 +354,9 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
|
||||
executors={"default": ThreadPoolExecutor(settings.CONF.scheduler)},
|
||||
)
|
||||
|
||||
# 数据库备份复用宿主调度器,不创建独立定时线程。
|
||||
self._register_database_backup_job()
|
||||
|
||||
# CookieCloud定时同步
|
||||
if (
|
||||
settings.COOKIECLOUD_INTERVAL
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""插件可使用的数据库备份只读门面。"""
|
||||
|
||||
from app.application.backup import BackupArtifact, BackupVerification
|
||||
from app.application.database import get_database_governance as _get_database_governance
|
||||
|
||||
|
||||
def create_backup() -> BackupArtifact:
|
||||
"""在宿主管理目录中创建当前数据库的一致备份。"""
|
||||
return _get_database_governance().create_backup()
|
||||
|
||||
|
||||
def list_backups() -> tuple[BackupArtifact, ...]:
|
||||
"""列出宿主管理目录中的数据库备份。"""
|
||||
return _get_database_governance().list_backups()
|
||||
|
||||
|
||||
def verify_backup(name: str) -> BackupVerification:
|
||||
"""按文件名校验一个宿主管理的数据库备份。"""
|
||||
return _get_database_governance().verify_backup(name)
|
||||
@@ -0,0 +1,61 @@
|
||||
"""数据库治理能力的宿主组合根。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from app.adapters.system.backup.database import (
|
||||
PostgreSQLBackupBackend,
|
||||
SQLiteBackupBackend,
|
||||
)
|
||||
from app.application.backup import BackupPolicy, DatabaseBackupService
|
||||
from app.application.database import (
|
||||
DatabaseGovernance,
|
||||
DatabaseHealthService,
|
||||
configure_database_governance,
|
||||
)
|
||||
from app.application.maintenance import (
|
||||
DataCleanupService,
|
||||
read_cleanup_policy,
|
||||
)
|
||||
from app.db.engine import get_engine
|
||||
from app.db.health import probe_database
|
||||
from app.db.maintenance import DatabaseCleanupRepository
|
||||
from app.db.session import SessionFactory
|
||||
from app.runtime.config import settings
|
||||
|
||||
|
||||
def build_database_governance() -> DatabaseGovernance:
|
||||
"""以缓存同步引擎为事实源构造一个完整数据库治理门面。"""
|
||||
engine = get_engine()
|
||||
dialect = engine.dialect.name
|
||||
if dialect == "sqlite":
|
||||
backup_backend = SQLiteBackupBackend(engine)
|
||||
elif dialect == "postgresql":
|
||||
backup_backend = PostgreSQLBackupBackend(engine)
|
||||
else:
|
||||
raise RuntimeError(f"不支持的数据库类型:{dialect}")
|
||||
|
||||
return DatabaseGovernance(
|
||||
health=DatabaseHealthService(probe_database),
|
||||
cleanup=DataCleanupService(
|
||||
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
|
||||
policy_reader=read_cleanup_policy,
|
||||
),
|
||||
backup=DatabaseBackupService(
|
||||
backend=backup_backend,
|
||||
policy_reader=read_backup_policy,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def configure_database() -> None:
|
||||
"""构造并登记宿主进程唯一的数据库治理门面。"""
|
||||
configure_database_governance(build_database_governance())
|
||||
|
||||
|
||||
def read_backup_policy() -> BackupPolicy:
|
||||
"""读取一次可热更新的数据库备份目录与保留策略。"""
|
||||
return BackupPolicy(
|
||||
root=settings.DATABASE_BACKUP_PATH,
|
||||
retention_days=settings.DB_BACKUP_RETENTION_DAYS,
|
||||
max_count=settings.DB_BACKUP_MAX_COUNT,
|
||||
)
|
||||
@@ -36,7 +36,7 @@ from app.application.messaging.message import (
|
||||
stop_message,
|
||||
)
|
||||
from app.application.configuration import SystemConfigService, configure_system_config
|
||||
from app.application.database import DatabaseHealthService, configure_database_health
|
||||
from app.application.database import configure_database_governance
|
||||
from app.application.service import configure_service_directory
|
||||
from app.application.plugin.runtime import configure_plugin_runtime
|
||||
from app.application.module import configure_module_runtime
|
||||
@@ -55,19 +55,13 @@ from app.application.workflow import WorkflowQueryService, configure_workflow_qu
|
||||
from app.application.agentdata import configure_agent_data_ports
|
||||
from app.api.data import configure_api_data_ports
|
||||
from app.application.subscription.write import configure_subscribe_writer
|
||||
from app.application.maintenance import (
|
||||
DataCleanupService,
|
||||
configure_cleanup_service_factory,
|
||||
read_cleanup_policy,
|
||||
)
|
||||
from app.adapters.external.server import (
|
||||
MoviePilotServerHelper,
|
||||
configure_server_application_services,
|
||||
)
|
||||
from app.application.server.report import ServerReportService
|
||||
from app.application.server.share import ServerSharingService
|
||||
from app.db import close_database
|
||||
from app.db.session import get_async_db, get_db
|
||||
from app.db.session import close_database, get_async_db, get_db
|
||||
from app.db.uow import SqlAlchemyAsyncUnitOfWork, SqlAlchemyUnitOfWork
|
||||
from app.db.oper.subscribe import SubscribeOper
|
||||
from app.db.oper.agentchat import AgentChatOper
|
||||
@@ -84,9 +78,6 @@ from app.db.oper.site import SiteOper
|
||||
from app.db.oper.message import MessageOper
|
||||
from app.db.oper.subscribehistory import SubscribeHistoryOper
|
||||
from app.db.oper.plugindata import PluginDataOper
|
||||
from app.db.maintenance import DatabaseCleanupRepository
|
||||
from app.db.session import SessionFactory
|
||||
from app.db.health import probe_database
|
||||
from app.db.oper.systemconfig import SystemConfigOper
|
||||
from app.db.oper.workflow import WorkflowOper
|
||||
from app.command import CommandChain
|
||||
@@ -94,6 +85,7 @@ from app.schemas.message import Message
|
||||
from app.schemas.message import MessageType
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.startup.agent_initializer import init_agent, stop_agent
|
||||
from app.startup.database import build_database_governance
|
||||
from app.startup.managed_resources_initializer import (
|
||||
init_managed_resources,
|
||||
stop_managed_resources,
|
||||
@@ -441,7 +433,7 @@ async def init_modules():
|
||||
user=lambda: UserOper(),
|
||||
)
|
||||
configure_system_config(SystemConfigService(repository=SystemConfigOper()))
|
||||
configure_database_health(DatabaseHealthService(probe_database))
|
||||
configure_database_governance(build_database_governance())
|
||||
configure_agent_chat_service(AgentChatService(repository=AgentChatOper()))
|
||||
configure_user_lookups(
|
||||
by_id=lambda user_id: UserOper().get_by_id(user_id),
|
||||
@@ -474,12 +466,6 @@ async def init_modules():
|
||||
plugin_data=lambda: PluginDataOper(),
|
||||
)
|
||||
configure_subscribe_writer(lambda: SubscribeOper())
|
||||
configure_cleanup_service_factory(
|
||||
lambda: DataCleanupService(
|
||||
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
|
||||
policy_reader=read_cleanup_policy,
|
||||
)
|
||||
)
|
||||
# 托管资源只在这里装配声明与 adapter,具体资源仍由首个消费者显式激活。
|
||||
init_managed_resources()
|
||||
# 应用服务不反向依赖 Chain,由启动组合层注入壁纸来源。
|
||||
|
||||
+19
-3
@@ -17,15 +17,30 @@ ENV LANG="C.UTF-8" \
|
||||
|
||||
ENV PATH="${VENV_PATH}/bin:${PATH}"
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl \
|
||||
&& install -d -m 0755 /usr/share/postgresql-common/pgdg \
|
||||
&& curl -fsSL \
|
||||
--connect-timeout 10 \
|
||||
--max-time 30 \
|
||||
--retry 3 \
|
||||
--retry-all-errors \
|
||||
--retry-delay 2 \
|
||||
--retry-max-time 90 \
|
||||
"https://www.postgresql.org/media/keys/ACCC4CF8.asc" \
|
||||
-o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
|
||||
&& chmod 0644 /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
|
||||
&& printf '%s\n' \
|
||||
'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main' \
|
||||
> /etc/apt/sources.list.d/pgdg.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
nginx \
|
||||
gettext-base \
|
||||
locales \
|
||||
procps \
|
||||
gosu \
|
||||
bash \
|
||||
ca-certificates \
|
||||
curl \
|
||||
wget \
|
||||
git \
|
||||
gh \
|
||||
@@ -45,6 +60,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
lsof \
|
||||
nano \
|
||||
unar \
|
||||
postgresql-client-18 \
|
||||
libchromaprint-tools \
|
||||
libjemalloc2 \
|
||||
&& dpkg-reconfigure --frontend noninteractive tzdata \
|
||||
|
||||
+35
@@ -88,6 +88,7 @@ moviepilot help update
|
||||
moviepilot help agent
|
||||
moviepilot help config
|
||||
moviepilot help config set
|
||||
moviepilot help database
|
||||
moviepilot help tool
|
||||
moviepilot help scheduler
|
||||
```
|
||||
@@ -136,6 +137,10 @@ moviepilot config get
|
||||
moviepilot config set
|
||||
moviepilot config keys
|
||||
moviepilot config describe
|
||||
moviepilot database backup
|
||||
moviepilot database list
|
||||
moviepilot database verify <filename>
|
||||
moviepilot database restore <filename> --confirm
|
||||
moviepilot tool list
|
||||
moviepilot tool show
|
||||
moviepilot tool run
|
||||
@@ -464,6 +469,36 @@ moviepilot config describe API_TOKEN --show-secrets
|
||||
- `MUSIC_METADATA_TO_SIMPLIFIED` 默认开启;开启后会将识别结果中的曲名、艺术家、专辑和分类等标准音乐元数据转换为简体中文,不转换歌词与来源原始响应
|
||||
- `config describe` 显示单个配置项的类型、默认值和当前值
|
||||
|
||||
## 数据库备份命令
|
||||
|
||||
创建一次在线一致备份:
|
||||
|
||||
```shell
|
||||
moviepilot database backup
|
||||
```
|
||||
|
||||
列出本地备份,并按文件名重新校验:
|
||||
|
||||
```shell
|
||||
moviepilot database list
|
||||
moviepilot database verify <filename>
|
||||
```
|
||||
|
||||
MoviePilot 停止运行后,可通过明确确认执行离线还原:
|
||||
|
||||
```shell
|
||||
moviepilot database restore <filename> --confirm
|
||||
```
|
||||
|
||||
说明:
|
||||
|
||||
- SQLite 使用在线备份 API,PostgreSQL 使用镜像内置的 `pg_dump` custom format
|
||||
- 源码部署使用 PostgreSQL 时,宿主机需安装 `pg_dump` 和 `pg_restore` 并加入 `PATH`;Docker 镜像已内置
|
||||
- 默认目录为配置目录下的 `database_backup/`,可通过 `DB_BACKUP_PATH` 调整
|
||||
- 文件名包含数据库类型和创建时间,例如 `sqlite_20260819_030000.db`
|
||||
- 备份、列举和校验可独立通过 CLI 执行
|
||||
- 还原会覆盖当前数据库,执行前必须停止 MoviePilot;运行中的 Web API 和插件 SDK 不提供还原入口
|
||||
|
||||
## Tool 命令
|
||||
|
||||
列出所有 MCP 工具:
|
||||
|
||||
+43
-7
@@ -13,8 +13,8 @@
|
||||
"runtime_to_db": [],
|
||||
"workflow_to_db": []
|
||||
},
|
||||
"edge_count": 6029,
|
||||
"edge_sha256": "a312bbaffcb68adc5ead36130d0e1e20056cab73273727a55182f2bfed5efd34",
|
||||
"edge_count": 6059,
|
||||
"edge_sha256": "85ab4dd2f01f48bae417f0401f272763f15ba7256b58ca0ab5a8256f4946c2f9",
|
||||
"edges": [
|
||||
"app -> app.runtime",
|
||||
"app -> app.runtime.compat",
|
||||
@@ -2115,6 +2115,7 @@
|
||||
"app.api.endpoints.system -> app.runtime.localization",
|
||||
"app.api.endpoints.system -> app.runtime.log",
|
||||
"app.api.endpoints.system -> app.runtime.progress",
|
||||
"app.api.endpoints.system -> app.runtime.scheduling",
|
||||
"app.api.endpoints.system -> app.runtime.state",
|
||||
"app.api.endpoints.system -> app.schemas",
|
||||
"app.api.endpoints.system -> app.schemas.common",
|
||||
@@ -2302,11 +2303,20 @@
|
||||
"app.application.audio -> app.runtime.log",
|
||||
"app.application.audio -> app.schemas",
|
||||
"app.application.audio -> app.schemas.types",
|
||||
"app.application.backup -> app.adapters",
|
||||
"app.application.backup -> app.adapters.system",
|
||||
"app.application.backup -> app.adapters.system.backup",
|
||||
"app.application.backup -> app.adapters.system.backup.files",
|
||||
"app.application.backup -> app.runtime",
|
||||
"app.application.backup -> app.runtime.log",
|
||||
"app.application.chain.context -> app.application",
|
||||
"app.application.chain.context -> app.application.chain",
|
||||
"app.application.chain.context -> app.application.chain.data",
|
||||
"app.application.dashboard -> app.schemas",
|
||||
"app.application.dashboard -> app.schemas.dashboard",
|
||||
"app.application.database -> app.application",
|
||||
"app.application.database -> app.application.backup",
|
||||
"app.application.database -> app.application.maintenance",
|
||||
"app.application.directory -> app.adapters",
|
||||
"app.application.directory -> app.adapters.system",
|
||||
"app.application.directory -> app.adapters.system.host",
|
||||
@@ -3214,11 +3224,15 @@
|
||||
"app.chain.workflow -> app.schemas.types",
|
||||
"app.chain.workflow -> app.schemas.workflow",
|
||||
"app.chain.workflow -> app.workflow",
|
||||
"app.cli -> app.application",
|
||||
"app.cli -> app.application.backup",
|
||||
"app.cli -> app.doctor",
|
||||
"app.cli -> app.doctor.formatters",
|
||||
"app.cli -> app.runtime",
|
||||
"app.cli -> app.runtime.config",
|
||||
"app.cli -> app.runtime.state",
|
||||
"app.cli -> app.startup",
|
||||
"app.cli -> app.startup.database",
|
||||
"app.command -> app.application",
|
||||
"app.command -> app.application.messaging",
|
||||
"app.command -> app.application.messaging.message",
|
||||
@@ -5395,8 +5409,8 @@
|
||||
"app.scheduler -> app.agent",
|
||||
"app.scheduler -> app.agent.runtime_loader",
|
||||
"app.scheduler -> app.application",
|
||||
"app.scheduler -> app.application.database",
|
||||
"app.scheduler -> app.application.image",
|
||||
"app.scheduler -> app.application.maintenance",
|
||||
"app.scheduler -> app.application.messaging",
|
||||
"app.scheduler -> app.application.messaging.message",
|
||||
"app.scheduler -> app.application.scheduling",
|
||||
@@ -5554,6 +5568,9 @@
|
||||
"app.sdk.cache -> app.runtime.cache",
|
||||
"app.sdk.config -> app.runtime",
|
||||
"app.sdk.config -> app.runtime.config",
|
||||
"app.sdk.database -> app.application",
|
||||
"app.sdk.database -> app.application.backup",
|
||||
"app.sdk.database -> app.application.database",
|
||||
"app.sdk.events -> app.runtime",
|
||||
"app.sdk.events -> app.runtime.events",
|
||||
"app.sdk.logging -> app.runtime",
|
||||
@@ -5667,6 +5684,21 @@
|
||||
"app.startup.command_initializer -> app.application",
|
||||
"app.startup.command_initializer -> app.application.commands",
|
||||
"app.startup.command_initializer -> app.command",
|
||||
"app.startup.database -> app.adapters",
|
||||
"app.startup.database -> app.adapters.system",
|
||||
"app.startup.database -> app.adapters.system.backup",
|
||||
"app.startup.database -> app.adapters.system.backup.database",
|
||||
"app.startup.database -> app.application",
|
||||
"app.startup.database -> app.application.backup",
|
||||
"app.startup.database -> app.application.database",
|
||||
"app.startup.database -> app.application.maintenance",
|
||||
"app.startup.database -> app.db",
|
||||
"app.startup.database -> app.db.engine",
|
||||
"app.startup.database -> app.db.health",
|
||||
"app.startup.database -> app.db.maintenance",
|
||||
"app.startup.database -> app.db.session",
|
||||
"app.startup.database -> app.runtime",
|
||||
"app.startup.database -> app.runtime.config",
|
||||
"app.startup.database_initializer -> app.db",
|
||||
"app.startup.database_initializer -> app.db.engine",
|
||||
"app.startup.database_initializer -> app.db.models",
|
||||
@@ -5744,7 +5776,6 @@
|
||||
"app.startup.modules_initializer -> app.application.database",
|
||||
"app.startup.modules_initializer -> app.application.history",
|
||||
"app.startup.modules_initializer -> app.application.image",
|
||||
"app.startup.modules_initializer -> app.application.maintenance",
|
||||
"app.startup.modules_initializer -> app.application.messaging",
|
||||
"app.startup.modules_initializer -> app.application.messaging.chat",
|
||||
"app.startup.modules_initializer -> app.application.messaging.message",
|
||||
@@ -5777,8 +5808,6 @@
|
||||
"app.startup.modules_initializer -> app.chain.workflow",
|
||||
"app.startup.modules_initializer -> app.command",
|
||||
"app.startup.modules_initializer -> app.db",
|
||||
"app.startup.modules_initializer -> app.db.health",
|
||||
"app.startup.modules_initializer -> app.db.maintenance",
|
||||
"app.startup.modules_initializer -> app.db.oper",
|
||||
"app.startup.modules_initializer -> app.db.oper.agentchat",
|
||||
"app.startup.modules_initializer -> app.db.oper.agenttask",
|
||||
@@ -5818,6 +5847,7 @@
|
||||
"app.startup.modules_initializer -> app.schemas.types",
|
||||
"app.startup.modules_initializer -> app.startup",
|
||||
"app.startup.modules_initializer -> app.startup.agent_initializer",
|
||||
"app.startup.modules_initializer -> app.startup.database",
|
||||
"app.startup.modules_initializer -> app.startup.managed_resources_initializer",
|
||||
"app.startup.monitor_initializer -> app.monitor",
|
||||
"app.startup.plugins_initializer -> app.adapters",
|
||||
@@ -6046,7 +6076,7 @@
|
||||
"app.workflow.actions.transfer_file -> app.workflow",
|
||||
"app.workflow.actions.transfer_file -> app.workflow.actions"
|
||||
],
|
||||
"module_count": 746,
|
||||
"module_count": 752,
|
||||
"modules": [
|
||||
"app",
|
||||
"app.adapters",
|
||||
@@ -6069,6 +6099,9 @@
|
||||
"app.adapters.network.http",
|
||||
"app.adapters.network.ip",
|
||||
"app.adapters.system",
|
||||
"app.adapters.system.backup",
|
||||
"app.adapters.system.backup.database",
|
||||
"app.adapters.system.backup.files",
|
||||
"app.adapters.system.display",
|
||||
"app.adapters.system.display.resource",
|
||||
"app.adapters.system.fsproxy",
|
||||
@@ -6275,6 +6308,7 @@
|
||||
"app.application.agent",
|
||||
"app.application.agentdata",
|
||||
"app.application.audio",
|
||||
"app.application.backup",
|
||||
"app.application.chain",
|
||||
"app.application.chain.context",
|
||||
"app.application.chain.data",
|
||||
@@ -6748,6 +6782,7 @@
|
||||
"app.sdk.browser",
|
||||
"app.sdk.cache",
|
||||
"app.sdk.config",
|
||||
"app.sdk.database",
|
||||
"app.sdk.events",
|
||||
"app.sdk.logging",
|
||||
"app.sdk.media",
|
||||
@@ -6761,6 +6796,7 @@
|
||||
"app.startup.agent_initializer",
|
||||
"app.startup.cache_initializer",
|
||||
"app.startup.command_initializer",
|
||||
"app.startup.database",
|
||||
"app.startup.database_initializer",
|
||||
"app.startup.domain_initializer",
|
||||
"app.startup.lifecycle",
|
||||
|
||||
+33
-6
@@ -1210,19 +1210,19 @@
|
||||
},
|
||||
{
|
||||
"caller": "app.api.endpoints.system",
|
||||
"line": 816
|
||||
"line": 859
|
||||
},
|
||||
{
|
||||
"caller": "app.api.endpoints.system",
|
||||
"line": 930
|
||||
"line": 973
|
||||
},
|
||||
{
|
||||
"caller": "app.api.endpoints.system",
|
||||
"line": 984
|
||||
"line": 1027
|
||||
},
|
||||
{
|
||||
"caller": "app.api.endpoints.system",
|
||||
"line": 998
|
||||
"line": 1041
|
||||
},
|
||||
{
|
||||
"caller": "app.chain._messaging",
|
||||
@@ -1652,7 +1652,7 @@
|
||||
"consumers": [
|
||||
{
|
||||
"caller": "app.scheduler",
|
||||
"line": 1116
|
||||
"line": 1150
|
||||
}
|
||||
],
|
||||
"producers": []
|
||||
@@ -1813,7 +1813,7 @@
|
||||
},
|
||||
{
|
||||
"caller": "app.scheduler",
|
||||
"line": 736
|
||||
"line": 770
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -3729,6 +3729,33 @@
|
||||
"target": "app.runtime.config.settings"
|
||||
}
|
||||
],
|
||||
"app.sdk.database": [
|
||||
{
|
||||
"kind": "import",
|
||||
"name": "BackupArtifact",
|
||||
"target": "app.application.backup.BackupArtifact"
|
||||
},
|
||||
{
|
||||
"kind": "import",
|
||||
"name": "BackupVerification",
|
||||
"target": "app.application.backup.BackupVerification"
|
||||
},
|
||||
{
|
||||
"kind": "FunctionDef",
|
||||
"name": "create_backup",
|
||||
"target": ""
|
||||
},
|
||||
{
|
||||
"kind": "FunctionDef",
|
||||
"name": "list_backups",
|
||||
"target": ""
|
||||
},
|
||||
{
|
||||
"kind": "FunctionDef",
|
||||
"name": "verify_backup",
|
||||
"target": ""
|
||||
}
|
||||
],
|
||||
"app.sdk.events": [
|
||||
{
|
||||
"kind": "import",
|
||||
|
||||
@@ -2,7 +2,7 @@ import tempfile
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
@@ -16,9 +16,9 @@ from app.runtime.config import settings
|
||||
from app.scheduler import SchedulerChain
|
||||
from app.application.maintenance import (
|
||||
DataCleanupService,
|
||||
configure_cleanup_service_factory,
|
||||
read_cleanup_policy,
|
||||
)
|
||||
from app.application.database import DatabaseGovernance, configure_database_governance
|
||||
from app.db.maintenance import DatabaseCleanupRepository
|
||||
|
||||
|
||||
@@ -51,13 +51,17 @@ class DataCleanupChainTest(unittest.TestCase):
|
||||
return patch.multiple(settings, **defaults)
|
||||
|
||||
def _configure_cleanup_service(self):
|
||||
"""把当前测试数据库注入清理应用服务,替代旧的 SessionFactory 打桩。"""
|
||||
configure_cleanup_service_factory(
|
||||
lambda: DataCleanupService(
|
||||
"""把当前测试数据库注入统一数据库治理门面。"""
|
||||
configure_database_governance(
|
||||
DatabaseGovernance(
|
||||
health=MagicMock(),
|
||||
backup=MagicMock(),
|
||||
cleanup=DataCleanupService(
|
||||
repository=DatabaseCleanupRepository(
|
||||
session_factory=self.SessionFactory,
|
||||
),
|
||||
policy_reader=read_cleanup_policy,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -117,15 +117,15 @@ def test_cleanup_service_finishes_other_tables_before_raising_partial_failure()
|
||||
|
||||
def test_scheduler_cleanup_is_a_compatibility_delegate() -> None:
|
||||
"""旧 SchedulerChain 入口应原样转发参数和返回值。"""
|
||||
service = MagicMock()
|
||||
service.execute.return_value = {"enabled": True}
|
||||
governance = MagicMock()
|
||||
governance.cleanup.return_value = {"enabled": True}
|
||||
progress = MagicMock()
|
||||
|
||||
with patch("app.scheduler.build_cleanup_service", return_value=service):
|
||||
with patch("app.scheduler.get_database_governance", return_value=governance):
|
||||
result = SchedulerChain().cleanup(batch_size=7, progress_callback=progress)
|
||||
|
||||
assert result == {"enabled": True}
|
||||
service.execute.assert_called_once_with(
|
||||
governance.cleanup.assert_called_once_with(
|
||||
batch_size=7,
|
||||
progress_callback=progress,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
from app.adapters.system.backup.database import (
|
||||
PostgreSQLBackupBackend,
|
||||
SQLiteBackupBackend,
|
||||
)
|
||||
|
||||
|
||||
def test_sqlite_backup_includes_committed_wal_data(tmp_path: Path) -> None:
|
||||
source = tmp_path / "user.db"
|
||||
with sqlite3.connect(source) as connection:
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
|
||||
connection.execute("INSERT INTO entries VALUES ('from-wal')")
|
||||
|
||||
engine = create_engine(f"sqlite:///{source}")
|
||||
backend = SQLiteBackupBackend(engine)
|
||||
artifact = tmp_path / "backup.db"
|
||||
|
||||
backend.create(artifact)
|
||||
|
||||
assert backend.verify(artifact).valid is True
|
||||
with sqlite3.connect(artifact) as connection:
|
||||
assert connection.execute("SELECT value FROM entries").fetchone() == ("from-wal",)
|
||||
engine.dispose()
|
||||
|
||||
|
||||
def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None:
|
||||
source = tmp_path / "user.db"
|
||||
backup = tmp_path / "backup.db"
|
||||
for path, value in ((source, "old"), (backup, "restored")):
|
||||
with sqlite3.connect(path) as connection:
|
||||
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
|
||||
connection.execute("INSERT INTO entries VALUES (?)", (value,))
|
||||
source.with_name("user.db-wal").write_bytes(b"old wal")
|
||||
source.with_name("user.db-shm").write_bytes(b"old shm")
|
||||
|
||||
backend = SQLiteBackupBackend(create_engine(f"sqlite:///{source}"))
|
||||
backend.restore(backup)
|
||||
|
||||
with sqlite3.connect(source) as connection:
|
||||
assert connection.execute("SELECT value FROM entries").fetchone() == ("restored",)
|
||||
assert not source.with_name("user.db-wal").exists()
|
||||
assert not source.with_name("user.db-shm").exists()
|
||||
|
||||
|
||||
class _Runner:
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[tuple[list[str], dict]] = []
|
||||
|
||||
def __call__(self, command, **kwargs):
|
||||
command = list(command)
|
||||
self.calls.append((command, kwargs))
|
||||
if "--file" in command:
|
||||
Path(command[command.index("--file") + 1]).write_bytes(b"PGDMP")
|
||||
stdout = "; archive listing" if command[:2] == ["pg_restore", "--list"] else ""
|
||||
return subprocess.CompletedProcess(command, 0, stdout, "")
|
||||
|
||||
|
||||
def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend:
|
||||
engine = SimpleNamespace(
|
||||
url=make_url(
|
||||
"postgresql://moviepilot:secret@database.internal:5432/moviepilot"
|
||||
"?sslmode=require"
|
||||
),
|
||||
dispose=Mock(),
|
||||
)
|
||||
return PostgreSQLBackupBackend(
|
||||
engine,
|
||||
runner=runner,
|
||||
tool_resolver=lambda executable: executable,
|
||||
)
|
||||
|
||||
|
||||
def test_postgresql_backup_keeps_password_out_of_command_and_file(tmp_path: Path) -> None:
|
||||
runner = _Runner()
|
||||
backend = _postgres_backend(runner)
|
||||
artifact = tmp_path / "backup.dump"
|
||||
|
||||
backend.create(artifact)
|
||||
|
||||
command, kwargs = runner.calls[0]
|
||||
assert "--format=custom" in command
|
||||
assert "secret" not in " ".join(command)
|
||||
assert kwargs["env"]["PGPASSWORD"] == "secret"
|
||||
assert kwargs["env"]["PGSSLMODE"] == "require"
|
||||
assert artifact.read_bytes() == b"PGDMP"
|
||||
|
||||
|
||||
def test_postgresql_verify_and_restore_use_pg_restore(tmp_path: Path) -> None:
|
||||
runner = _Runner()
|
||||
backend = _postgres_backend(runner)
|
||||
artifact = tmp_path / "backup.dump"
|
||||
artifact.write_bytes(b"PGDMP")
|
||||
|
||||
assert backend.verify(artifact).valid is True
|
||||
backend.restore(artifact)
|
||||
|
||||
verify_command, verify_kwargs = runner.calls[0]
|
||||
restore_command, restore_kwargs = runner.calls[1]
|
||||
assert verify_command == ["pg_restore", "--list", str(artifact)]
|
||||
assert "PGPASSWORD" not in verify_kwargs["env"]
|
||||
assert "--single-transaction" in restore_command
|
||||
assert "--clean" in restore_command
|
||||
assert restore_kwargs["env"]["PGPASSWORD"] == "secret"
|
||||
|
||||
|
||||
def test_postgresql_source_install_reports_missing_native_client() -> None:
|
||||
runner = _Runner()
|
||||
engine = SimpleNamespace(
|
||||
url=make_url("postgresql://moviepilot:secret@database/moviepilot"),
|
||||
dispose=Mock(),
|
||||
)
|
||||
backend = PostgreSQLBackupBackend(
|
||||
engine,
|
||||
runner=runner,
|
||||
tool_resolver=lambda _executable: None,
|
||||
)
|
||||
|
||||
try:
|
||||
backend.create(Path("unused.dump"))
|
||||
except RuntimeError as error:
|
||||
assert "安装与服务端同主版本或更高的 PostgreSQL client" in str(error)
|
||||
else:
|
||||
raise AssertionError("缺少 pg_dump 时未给出安装提示")
|
||||
@@ -0,0 +1,77 @@
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from app import cli as cli_module
|
||||
from app.application.backup import BackupArtifact, BackupVerification
|
||||
from app.cli import cli
|
||||
from app.sdk import database as database_sdk
|
||||
|
||||
|
||||
NAME = "sqlite_20260819_030000.db"
|
||||
|
||||
|
||||
def _artifact(tmp_path: Path) -> BackupArtifact:
|
||||
path = tmp_path / NAME
|
||||
path.write_bytes(b"snapshot")
|
||||
return BackupArtifact(NAME, "sqlite", datetime(2026, 8, 19, 3, 0), path, 8)
|
||||
|
||||
|
||||
def test_cli_backup_list_and_verify_use_local_governance(tmp_path: Path, monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
governance.create_backup.return_value = _artifact(tmp_path)
|
||||
governance.list_backups.return_value = (governance.create_backup.return_value,)
|
||||
governance.verify_backup.return_value = BackupVerification(True, "integrity_check")
|
||||
monkeypatch.setattr(cli_module, "build_database_governance", lambda: governance)
|
||||
|
||||
backup = CliRunner().invoke(cli, ["database", "backup"])
|
||||
listed = CliRunner().invoke(cli, ["database", "list"])
|
||||
verified = CliRunner().invoke(cli, ["database", "verify", NAME])
|
||||
|
||||
assert backup.exit_code == 0, backup.output
|
||||
assert "name\tdb_type\tcreated_at\tsize\tpath" in backup.output
|
||||
assert NAME in backup.output
|
||||
assert listed.exit_code == 0 and NAME in listed.output
|
||||
assert verified.exit_code == 0 and "校验通过" in verified.output
|
||||
|
||||
|
||||
def test_cli_restore_requires_explicit_offline_confirmation(tmp_path: Path, monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
governance.restore_backup.return_value = _artifact(tmp_path)
|
||||
monkeypatch.setattr(cli_module, "build_database_governance", lambda: governance)
|
||||
|
||||
rejected = CliRunner().invoke(cli, ["database", "restore", NAME])
|
||||
restored = CliRunner().invoke(cli, ["database", "restore", NAME, "--confirm"])
|
||||
|
||||
assert rejected.exit_code == 1
|
||||
governance.restore_backup.assert_called_once_with(NAME)
|
||||
assert restored.exit_code == 0 and "还原完成" in restored.output
|
||||
|
||||
|
||||
def test_sdk_exposes_backup_without_restore_or_policy_controls(tmp_path: Path, monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
artifact = _artifact(tmp_path)
|
||||
verification = BackupVerification(True, "integrity_check")
|
||||
governance.create_backup.return_value = artifact
|
||||
governance.list_backups.return_value = (artifact,)
|
||||
governance.verify_backup.return_value = verification
|
||||
monkeypatch.setattr(database_sdk, "_get_database_governance", lambda: governance)
|
||||
|
||||
assert database_sdk.create_backup() == artifact
|
||||
assert database_sdk.list_backups() == (artifact,)
|
||||
assert database_sdk.verify_backup(NAME) == verification
|
||||
assert not hasattr(database_sdk, "get_database_governance")
|
||||
assert not hasattr(database_sdk, "restore_backup")
|
||||
assert not hasattr(database_sdk, "delete_backup")
|
||||
|
||||
|
||||
def test_sdk_rejects_invalid_name_at_host_boundary(monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
governance.verify_backup.side_effect = ValueError("数据库备份文件名无效")
|
||||
monkeypatch.setattr(database_sdk, "_get_database_governance", lambda: governance)
|
||||
|
||||
with pytest.raises(ValueError, match="文件名"):
|
||||
database_sdk.verify_backup("../../user.db")
|
||||
@@ -0,0 +1,50 @@
|
||||
"""数据库备份所需 Docker 运行时工具合同。"""
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def test_runtime_image_installs_postgresql_18_client_from_pgdg() -> None:
|
||||
"""Bookworm 镜像必须从签名的 PGDG 源安装固定主版本客户端。"""
|
||||
dockerfile = (
|
||||
Path(__file__).resolve().parents[1] / "docker" / "Dockerfile"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert re.search(
|
||||
r"^FROM python:[^\s]+-slim-bookworm AS base$", dockerfile, re.MULTILINE
|
||||
)
|
||||
assert "https://www.postgresql.org/media/keys/ACCC4CF8.asc" in dockerfile
|
||||
for curl_option in (
|
||||
"--connect-timeout 10",
|
||||
"--max-time 30",
|
||||
"--retry 3",
|
||||
"--retry-all-errors",
|
||||
"--retry-max-time 90",
|
||||
):
|
||||
assert curl_option in dockerfile
|
||||
keyring = "/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc"
|
||||
assert f"chmod 0644 {keyring}" in dockerfile
|
||||
assert f"signed-by={keyring}" in dockerfile
|
||||
assert "https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" in dockerfile
|
||||
assert re.search(r"^\s+postgresql-client-18 \\$", dockerfile, re.MULTILINE)
|
||||
assert not re.search(r"^\s+postgresql-client \\$", dockerfile, re.MULTILINE)
|
||||
|
||||
|
||||
def test_runtime_image_keeps_pgdg_setup_architecture_neutral_and_cleans_apt_cache(
|
||||
) -> None:
|
||||
"""PGDG 原生架构解析需同时适用于 amd64/arm64,且不得遗留 APT 索引。"""
|
||||
dockerfile = (
|
||||
Path(__file__).resolve().parents[1] / "docker" / "Dockerfile"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
pgdg_sources = [
|
||||
line for line in dockerfile.splitlines() if "apt.postgresql.org/pub/repos/apt" in line
|
||||
]
|
||||
|
||||
assert len(pgdg_sources) == 1
|
||||
pgdg_source = pgdg_sources[0]
|
||||
assert "arch=" not in pgdg_source
|
||||
assert "/var/lib/apt/lists/*" in dockerfile
|
||||
assert dockerfile.index("postgresql-client-18") < dockerfile.index(
|
||||
"/var/lib/apt/lists/*"
|
||||
)
|
||||
@@ -0,0 +1,40 @@
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from app.application.database import get_database_governance
|
||||
from app.startup import database as startup_database
|
||||
|
||||
|
||||
def test_builder_uses_cached_engine_as_database_fact_source(monkeypatch) -> None:
|
||||
engine = SimpleNamespace(dialect=SimpleNamespace(name="sqlite"))
|
||||
backend = Mock(db_type="sqlite", suffix=".db")
|
||||
sqlite_backend = Mock(return_value=backend)
|
||||
monkeypatch.setattr(startup_database, "get_engine", lambda: engine)
|
||||
monkeypatch.setattr(startup_database, "SQLiteBackupBackend", sqlite_backend)
|
||||
monkeypatch.setattr(startup_database.settings, "DB_TYPE", "postgresql")
|
||||
|
||||
startup_database.build_database_governance()
|
||||
|
||||
sqlite_backend.assert_called_once_with(engine)
|
||||
|
||||
|
||||
def test_backup_policy_reads_current_path_and_retention(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_PATH", str(tmp_path))
|
||||
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_RETENTION_DAYS", 7)
|
||||
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_MAX_COUNT", 5)
|
||||
|
||||
policy = startup_database.read_backup_policy()
|
||||
|
||||
assert policy.root == tmp_path
|
||||
assert policy.retention_days == 7
|
||||
assert policy.max_count == 5
|
||||
|
||||
|
||||
def test_configure_registers_one_database_governance(monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
monkeypatch.setattr(startup_database, "build_database_governance", lambda: governance)
|
||||
|
||||
startup_database.configure_database()
|
||||
|
||||
assert get_database_governance() is governance
|
||||
@@ -0,0 +1,93 @@
|
||||
"""数据库备份与宿主调度器的接入合同。"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
from app import scheduler as scheduler_module
|
||||
from app.scheduler import Scheduler
|
||||
|
||||
|
||||
class _SchedulerStub:
|
||||
def __init__(self) -> None:
|
||||
self.jobs = {}
|
||||
|
||||
def add_job(self, func, *, trigger, id, **kwargs) -> None:
|
||||
self.jobs[id] = {"func": func, "trigger": trigger, **kwargs}
|
||||
|
||||
|
||||
def _scheduler() -> Scheduler:
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler._scheduler = _SchedulerStub()
|
||||
scheduler._jobs = {}
|
||||
return scheduler
|
||||
|
||||
|
||||
def test_database_backup_schedule_only_watches_job_shape() -> None:
|
||||
assert Scheduler.CONFIG_WATCH.intersection({
|
||||
"DB_BACKUP_ENABLE",
|
||||
"DB_BACKUP_CRON",
|
||||
"DB_BACKUP_PATH",
|
||||
"DB_BACKUP_RETENTION_DAYS",
|
||||
"DB_BACKUP_MAX_COUNT",
|
||||
}) == {"DB_BACKUP_ENABLE", "DB_BACKUP_CRON"}
|
||||
|
||||
|
||||
def test_disabled_database_backup_does_not_register_job(monkeypatch) -> None:
|
||||
scheduler = _scheduler()
|
||||
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", False)
|
||||
|
||||
scheduler._register_database_backup_job()
|
||||
|
||||
assert scheduler._scheduler.jobs == {}
|
||||
|
||||
|
||||
def test_enabled_database_backup_without_cron_does_not_register_job(monkeypatch) -> None:
|
||||
"""总开关开启但未配置周期时,不启用定时备份。"""
|
||||
scheduler = _scheduler()
|
||||
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", True)
|
||||
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_CRON", "")
|
||||
|
||||
scheduler._register_database_backup_job()
|
||||
|
||||
assert scheduler._scheduler.jobs == {}
|
||||
|
||||
|
||||
def test_enabled_database_backup_registers_single_replaceable_job(monkeypatch) -> None:
|
||||
scheduler = _scheduler()
|
||||
trigger = object()
|
||||
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", True)
|
||||
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_CRON", "0 3 * * *")
|
||||
monkeypatch.setattr(scheduler_module.TimerUtils, "build_schedule_trigger", Mock(return_value=trigger))
|
||||
|
||||
scheduler._register_database_backup_job()
|
||||
scheduler._register_database_backup_job()
|
||||
|
||||
assert list(scheduler._scheduler.jobs) == ["database_backup"]
|
||||
assert scheduler._scheduler.jobs["database_backup"]["replace_existing"] is True
|
||||
|
||||
|
||||
def test_scheduled_backup_uses_registered_database_governance(monkeypatch) -> None:
|
||||
governance = Mock()
|
||||
monkeypatch.setattr(scheduler_module, "get_database_governance", lambda: governance)
|
||||
|
||||
result = Scheduler.database_backup()
|
||||
|
||||
assert result is governance.create_backup.return_value
|
||||
governance.create_backup.assert_called_once_with()
|
||||
|
||||
|
||||
def test_scheduler_database_dependencies_are_explicit_module_imports() -> None:
|
||||
tree = ast.parse(
|
||||
(Path(__file__).parents[1] / "app" / "scheduler.py").read_text(encoding="utf-8")
|
||||
)
|
||||
function_imports = [
|
||||
node
|
||||
for function in ast.walk(tree)
|
||||
if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef))
|
||||
for node in ast.walk(function)
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom))
|
||||
and getattr(node, "module", "")
|
||||
and str(getattr(node, "module", "")).startswith("app.application.database")
|
||||
]
|
||||
assert function_imports == []
|
||||
@@ -0,0 +1,115 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import stat
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.application.backup import BackupPolicy, DatabaseBackupService
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Check:
|
||||
valid: bool
|
||||
method: str = "test-check"
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class _Backend:
|
||||
db_type = "sqlite"
|
||||
suffix = ".db"
|
||||
|
||||
def __init__(self, *, valid: bool = True) -> None:
|
||||
self.valid = valid
|
||||
self.restored: Path | None = None
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
destination.write_bytes(b"database snapshot")
|
||||
|
||||
def verify(self, artifact: Path) -> _Check:
|
||||
return _Check(self.valid and artifact.read_bytes() == b"database snapshot")
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
self.restored = artifact
|
||||
|
||||
|
||||
def _service(
|
||||
root: Path,
|
||||
*,
|
||||
backend: _Backend | None = None,
|
||||
now: datetime | None = None,
|
||||
retention_days: int = 0,
|
||||
max_count: int = 0,
|
||||
) -> DatabaseBackupService:
|
||||
return DatabaseBackupService(
|
||||
backend=backend or _Backend(),
|
||||
policy_reader=lambda: BackupPolicy(root, retention_days, max_count),
|
||||
clock=lambda: now or datetime(2026, 8, 19, 13, 45, 26),
|
||||
)
|
||||
|
||||
|
||||
def test_create_publishes_one_readable_private_file(tmp_path: Path) -> None:
|
||||
artifact = _service(tmp_path).create()
|
||||
|
||||
assert artifact.name == "sqlite_20260819_134526.db"
|
||||
assert artifact.path.read_bytes() == b"database snapshot"
|
||||
assert stat.S_IMODE(tmp_path.stat().st_mode) == 0o700
|
||||
assert stat.S_IMODE(artifact.path.stat().st_mode) == 0o600
|
||||
assert not list(tmp_path.glob("*.partial"))
|
||||
|
||||
|
||||
def test_failed_verification_does_not_publish_artifact(tmp_path: Path) -> None:
|
||||
with pytest.raises(RuntimeError, match="数据库备份校验失败"):
|
||||
_service(tmp_path, backend=_Backend(valid=False)).create()
|
||||
|
||||
assert list(tmp_path.iterdir()) == []
|
||||
|
||||
|
||||
def test_same_second_backups_receive_short_sequence_suffix(tmp_path: Path) -> None:
|
||||
service = _service(tmp_path)
|
||||
|
||||
first = service.create()
|
||||
second = service.create()
|
||||
|
||||
assert first.name == "sqlite_20260819_134526.db"
|
||||
assert second.name == "sqlite_20260819_134526_1.db"
|
||||
|
||||
|
||||
def test_retention_applies_after_new_artifact_is_available(tmp_path: Path) -> None:
|
||||
old = _service(tmp_path, now=datetime(2026, 8, 1, 3, 0, 0)).create()
|
||||
current = _service(
|
||||
tmp_path,
|
||||
now=datetime(2026, 8, 19, 3, 0, 0),
|
||||
retention_days=7,
|
||||
max_count=1,
|
||||
).create()
|
||||
|
||||
assert current.path.exists()
|
||||
assert not old.path.exists()
|
||||
|
||||
|
||||
def test_list_ignores_unmanaged_files_and_rejects_paths(tmp_path: Path) -> None:
|
||||
artifact = _service(tmp_path).create()
|
||||
(tmp_path / "notes.txt").write_text("ignore", encoding="utf-8")
|
||||
|
||||
assert [item.name for item in _service(tmp_path).list()] == [artifact.name]
|
||||
with pytest.raises(ValueError, match="文件名"):
|
||||
_service(tmp_path).verify("../user.db")
|
||||
|
||||
|
||||
def test_restore_requires_matching_database_type(tmp_path: Path) -> None:
|
||||
backend = _Backend()
|
||||
service = _service(tmp_path, backend=backend)
|
||||
artifact = service.create()
|
||||
|
||||
restored = service.restore(artifact.name)
|
||||
|
||||
assert restored.name == artifact.name
|
||||
assert backend.restored == artifact.path
|
||||
|
||||
postgres = tmp_path / "postgresql_20260819_134526.dump"
|
||||
postgres.write_bytes(b"database snapshot")
|
||||
with pytest.raises(ValueError, match="当前数据库类型"):
|
||||
service.restore(postgres.name)
|
||||
@@ -0,0 +1,89 @@
|
||||
"""系统数据库备份策略配置测试。"""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from app.api.endpoints import system as system_endpoint
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env", "message"),
|
||||
[
|
||||
({"DB_BACKUP_CRON": "0 3 * *"}, "数据库备份周期格式不正确"),
|
||||
({"DB_BACKUP_PATH": 123}, "数据库备份目录必须是路径字符串"),
|
||||
({"DB_BACKUP_RETENTION_DAYS": -1}, "数据库备份过期天数"),
|
||||
({"DB_BACKUP_RETENTION_DAYS": 1.5}, "数据库备份过期天数"),
|
||||
({"DB_BACKUP_MAX_COUNT": True}, "数据库备份最大保留份数"),
|
||||
({"DB_BACKUP_MAX_COUNT": "many"}, "数据库备份最大保留份数"),
|
||||
],
|
||||
)
|
||||
def test_database_backup_policy_rejects_invalid_values(
|
||||
env: dict,
|
||||
message: str,
|
||||
) -> None:
|
||||
"""无效策略必须在批量配置写入前被拒绝。"""
|
||||
env["DB_BACKUP_ENABLE"] = True
|
||||
assert message in str(system_endpoint._validate_database_backup_config(env))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"env",
|
||||
[
|
||||
{"DB_BACKUP_CRON": ""},
|
||||
{"DB_BACKUP_CRON": "0 3 * * *"},
|
||||
{"DB_BACKUP_PATH": None},
|
||||
{"DB_BACKUP_PATH": ""},
|
||||
{"DB_BACKUP_PATH": "database_backup"},
|
||||
{"DB_BACKUP_RETENTION_DAYS": 0},
|
||||
{"DB_BACKUP_MAX_COUNT": "0"},
|
||||
],
|
||||
)
|
||||
def test_database_backup_policy_accepts_supported_boundaries(env: dict) -> None:
|
||||
"""空目录使用默认路径,两个保留值的零均表示不限制。"""
|
||||
env["DB_BACKUP_ENABLE"] = True
|
||||
assert system_endpoint._validate_database_backup_config(env) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disabled", [False, "false", "0", "off"])
|
||||
def test_database_backup_policy_rejects_invalid_hidden_values_when_disabled(disabled) -> None:
|
||||
"""关闭总开关只暂停调度,不能把无效策略写入持久配置。"""
|
||||
error = system_endpoint._validate_database_backup_config({
|
||||
"DB_BACKUP_ENABLE": disabled,
|
||||
"DB_BACKUP_CRON": "invalid",
|
||||
"DB_BACKUP_PATH": 123,
|
||||
"DB_BACKUP_RETENTION_DAYS": -1,
|
||||
"DB_BACKUP_MAX_COUNT": 1.5,
|
||||
})
|
||||
|
||||
assert error is not None
|
||||
|
||||
|
||||
def test_set_env_rejects_invalid_database_backup_policy_without_partial_write() -> None:
|
||||
"""备份策略校验失败时不得调用 Settings 的批量写入。"""
|
||||
env = {
|
||||
"DB_BACKUP_ENABLE": True,
|
||||
"DB_BACKUP_CRON": "invalid",
|
||||
"DB_BACKUP_RETENTION_DAYS": 30,
|
||||
"DB_BACKUP_MAX_COUNT": 30,
|
||||
}
|
||||
|
||||
with patch.object(
|
||||
system_endpoint,
|
||||
"_validate_llm_server_tool_config",
|
||||
return_value=None,
|
||||
), patch.object(type(system_endpoint.settings), "update_settings") as update_settings:
|
||||
response = asyncio.run(system_endpoint.set_env_setting(env=env, _=object()))
|
||||
|
||||
assert response.success is False
|
||||
assert "数据库备份周期格式不正确" in response.message
|
||||
update_settings.assert_not_called()
|
||||
|
||||
|
||||
def test_database_backup_default_path_tracks_config_directory(tmp_path, monkeypatch) -> None:
|
||||
"""未显式配置目录时应跟随当前配置根,而不是写死 Docker 路径。"""
|
||||
monkeypatch.setattr(system_endpoint.settings, "CONFIG_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(system_endpoint.settings, "DB_BACKUP_PATH", None)
|
||||
|
||||
assert system_endpoint.settings.DATABASE_BACKUP_PATH == tmp_path / "database_backup"
|
||||
Reference in New Issue
Block a user