feat(database): add managed backup and offline restore (#6359)

This commit is contained in:
InfinityPacer
2026-08-19 17:34:32 +08:00
committed by GitHub
parent f4a879b3de
commit f700db5ac8
27 changed files with 1576 additions and 80 deletions
+1
View File
@@ -0,0 +1 @@
"""数据库备份的文件系统与数据库技术适配器命名空间。"""
+209
View File
@@ -0,0 +1,209 @@
"""基于活动 SQLAlchemy 引擎的 SQLite 与 PostgreSQL 备份实现。"""
from __future__ import annotations
import os
import shutil
import sqlite3
import subprocess
from contextlib import closing
from dataclasses import dataclass
from pathlib import Path
from typing import Callable, Mapping, Protocol, Sequence
from sqlalchemy.engine import Engine
@dataclass(frozen=True, slots=True)
class DatabaseBackupCheck:
"""数据库适配器返回的基础校验结果。"""
valid: bool
method: str
detail: str | None = None
class ProcessResult(Protocol):
"""数据库命令执行结果的最小合同。"""
returncode: int
stdout: str
stderr: str
class ProcessRunner(Protocol):
"""可替换的数据库命令执行边界。"""
def __call__(
self,
command: Sequence[str],
*,
env: Mapping[str, str],
capture_output: bool,
text: bool,
check: bool,
) -> ProcessResult:
"""执行命令并返回结果。"""
class SQLiteBackupBackend:
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
db_type = "sqlite"
suffix = ".db"
def __init__(self, engine: Engine) -> None:
self._engine = engine
database = engine.url.database
if not database or database == ":memory:":
raise ValueError("SQLite 内存数据库不支持文件备份")
self._database = Path(database)
def create(self, destination: Path) -> None:
"""从活动引擎指向的 SQLite 文件创建一致快照。"""
source = self._engine.raw_connection()
try:
with closing(sqlite3.connect(destination)) as target:
source.driver_connection.backup(target)
target.commit()
finally:
source.close()
def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 SQLite integrity_check 校验备份内容。"""
method = "PRAGMA integrity_check"
try:
# 已发布前的临时快照不会再变化;immutable 避免 WAL 模式为只读校验创建旁路文件。
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
with closing(sqlite3.connect(uri, uri=True)) as connection:
rows = connection.execute("PRAGMA integrity_check").fetchall()
except sqlite3.Error as error:
return DatabaseBackupCheck(False, method, str(error))
valid = bool(rows) and all(row[0] == "ok" for row in rows)
detail = None if valid else "; ".join(str(row[0]) for row in rows)
return DatabaseBackupCheck(valid, method, detail)
def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
temporary = self._database.with_name(f".{self._database.name}.restore")
self._engine.dispose()
try:
shutil.copy2(artifact, temporary)
temporary.chmod(0o600)
self._database.with_name(f"{self._database.name}-wal").unlink(missing_ok=True)
self._database.with_name(f"{self._database.name}-shm").unlink(missing_ok=True)
os.replace(temporary, self._database)
finally:
temporary.unlink(missing_ok=True)
class PostgreSQLBackupBackend:
"""使用 pg_dump 与 pg_restore 管理活动 PostgreSQL 数据库。"""
db_type = "postgresql"
suffix = ".dump"
def __init__(
self,
engine: Engine,
*,
runner: ProcessRunner = subprocess.run,
tool_resolver: Callable[[str], str | None] = shutil.which,
pg_dump: str = "pg_dump",
pg_restore: str = "pg_restore",
) -> None:
self._engine = engine
self._runner = runner
self._tool_resolver = tool_resolver
self._pg_dump = pg_dump
self._pg_restore = pg_restore
def create(self, destination: Path) -> None:
"""创建 PostgreSQL custom-format 在线备份。"""
command = [
self._require_tool(self._pg_dump),
"--format=custom",
"--no-owner",
"--no-acl",
"--file",
str(destination),
*self._connection_arguments(),
]
result = self._run(command, include_password=True)
if result.returncode != 0:
raise RuntimeError(f"pg_dump 执行失败,退出码 {result.returncode}")
if not destination.is_file() or destination.stat().st_size == 0:
raise RuntimeError("pg_dump 未生成有效的备份文件")
def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
method = "pg_restore --list"
result = self._run(
[self._require_tool(self._pg_restore), "--list", str(artifact)],
include_password=False,
)
valid = result.returncode == 0 and bool(result.stdout.strip())
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
return DatabaseBackupCheck(valid, method, detail)
def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
command = [
self._require_tool(self._pg_restore),
"--clean",
"--if-exists",
"--no-owner",
"--no-acl",
"--single-transaction",
"--exit-on-error",
*self._connection_arguments(),
str(artifact),
]
result = self._run(command, include_password=True)
if result.returncode != 0:
raise RuntimeError(f"pg_restore 执行失败,退出码 {result.returncode}")
def _connection_arguments(self) -> list[str]:
url = self._engine.url
host = str(url.query.get("host") or url.host or "")
port = str(url.query.get("port") or url.port or "")
arguments = [
"--username",
str(url.username or ""),
"--dbname",
str(url.database or ""),
]
if host:
arguments.extend(["--host", host])
if port:
arguments.extend(["--port", port])
return arguments
def _run(self, command: Sequence[str], *, include_password: bool) -> ProcessResult:
return self._runner(
command,
env=self._environment(include_password=include_password),
capture_output=True,
text=True,
check=False,
)
def _require_tool(self, executable: str) -> str:
resolved = self._tool_resolver(executable)
if resolved is None:
raise RuntimeError(
f"未找到 {executable},请安装与服务端同主版本或更高的 "
"PostgreSQL client 并加入 PATH"
)
return resolved
def _environment(self, *, include_password: bool) -> dict[str, str]:
environment = dict(os.environ)
environment.pop("PGPASSWORD", None)
environment.pop("PGSSLMODE", None)
if include_password and self._engine.url.password:
environment["PGPASSWORD"] = str(self._engine.url.password)
sslmode = self._engine.url.query.get("sslmode")
if sslmode:
environment["PGSSLMODE"] = str(sslmode)
return environment
+116
View File
@@ -0,0 +1,116 @@
"""数据库备份单文件的受限文件系统操作。"""
from __future__ import annotations
import os
import re
import tempfile
from datetime import datetime
from pathlib import Path
_BACKUP_NAME = re.compile(
r"^(?P<db_type>sqlite|postgresql)_"
r"(?P<timestamp>\d{8}_\d{6})"
r"(?:_(?P<sequence>\d+))?"
r"(?P<suffix>\.db|\.dump)$"
)
class BackupFiles:
"""把备份文件操作限制在一个私有根目录内。"""
def __init__(self, root: Path) -> None:
self.root = Path(root)
def create_temporary(self, suffix: str) -> Path:
"""在最终目录内创建私有临时文件,保证发布可使用原子替换。"""
self._ensure_root()
descriptor, filename = tempfile.mkstemp(
prefix=".database-",
suffix=f"{suffix}.partial",
dir=self.root,
)
os.close(descriptor)
path = Path(filename)
path.chmod(0o600)
return path
def publish(self, temporary: Path, name: str) -> Path:
"""把已校验临时文件发布为正式备份文件。"""
destination = self._resolve_name(name, require_exists=False)
os.replace(temporary, destination)
destination.chmod(0o600)
return destination
def discard(self, temporary: Path) -> None:
"""清理本次操作拥有的未发布临时文件。"""
path = Path(temporary)
if path.parent == self.root and path.name.startswith(".database-"):
path.unlink(missing_ok=True)
def list(self) -> list[Path]:
"""返回当前根目录内格式合法的正式备份文件。"""
if not self.root.is_dir():
return []
paths = [
path
for path in self.root.iterdir()
if path.is_file() and _BACKUP_NAME.fullmatch(path.name)
]
return sorted(paths, key=lambda path: (self.created_at(path.name), path.name), reverse=True)
def resolve(self, name: str) -> Path:
"""按受限文件名解析一个必须存在的备份文件。"""
return self._resolve_name(name, require_exists=True)
def delete(self, name: str) -> None:
"""删除一个已通过名称约束的备份文件。"""
self.resolve(name).unlink()
def available_name(self, *, db_type: str, created_at: datetime, suffix: str) -> str:
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
base = f"{db_type}_{timestamp}"
candidate = f"{base}{suffix}"
sequence = 1
while (self.root / candidate).exists():
candidate = f"{base}_{sequence}{suffix}"
sequence += 1
if not _BACKUP_NAME.fullmatch(candidate):
raise ValueError("数据库备份文件名无效")
return candidate
@staticmethod
def database_type(name: str) -> str:
"""从受管文件名读取数据库类型。"""
return BackupFiles._match(name).group("db_type")
@staticmethod
def created_at(name: str) -> datetime:
"""从受管文件名读取本地创建时间。"""
return datetime.strptime(
BackupFiles._match(name).group("timestamp"),
"%Y%m%d_%H%M%S",
)
def _ensure_root(self) -> None:
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
self.root.chmod(0o700)
def _resolve_name(self, name: str, *, require_exists: bool) -> Path:
normalized = str(name).strip()
self._match(normalized)
if Path(normalized).name != normalized:
raise ValueError("数据库备份文件名不能包含路径")
path = self.root / normalized
if require_exists and not path.is_file():
raise FileNotFoundError(normalized)
return path
@staticmethod
def _match(name: str) -> re.Match[str]:
matched = _BACKUP_NAME.fullmatch(str(name))
if matched is None:
raise ValueError("数据库备份文件名无效")
return matched
+43
View File
@@ -53,6 +53,7 @@ from app.adapters.external.market import (
)
from app.application.messaging.message import MessageHelper
from app.runtime.progress import AsyncProgressHelper
from app.runtime.scheduling import TimerUtils
from app.application.rules import RuleHelper
from app.adapters.external.server import MoviePilotServerHelper
from app.runtime.state import SystemHelper
@@ -86,6 +87,13 @@ _PUBLIC_SYSTEM_CONFIG_KEYS = {
_PUBLIC_SETTINGS_KEYS = {"PLUGIN_MARKET"}
_LOG_DOWNLOAD_LIMIT = 10
_LOG_DOWNLOAD_NAME_PATTERN = re.compile(r"^[A-Za-z0-9_-]+$")
_DATABASE_BACKUP_SETTING_KEYS = {
"DB_BACKUP_ENABLE",
"DB_BACKUP_CRON",
"DB_BACKUP_PATH",
"DB_BACKUP_RETENTION_DAYS",
"DB_BACKUP_MAX_COUNT",
}
def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
@@ -129,6 +137,38 @@ def _validate_llm_server_tool_config(env: dict) -> Optional[str]:
)
def _validate_database_backup_config(env: dict) -> Optional[str]:
"""在批量写入前校验数据库备份策略,避免只保存部分字段。"""
if not _DATABASE_BACKUP_SETTING_KEYS.intersection(env):
return None
cron = str(env.get("DB_BACKUP_CRON", settings.DB_BACKUP_CRON) or "").strip()
if cron:
try:
TimerUtils.normalize_schedule_trigger("cron", cron, settings.TZ)
except (TypeError, ValueError):
return "数据库备份周期格式不正确"
backup_path = env.get("DB_BACKUP_PATH", settings.DB_BACKUP_PATH)
if backup_path is not None and not isinstance(backup_path, str):
return "数据库备份目录必须是路径字符串"
for key, label in (
("DB_BACKUP_RETENTION_DAYS", "数据库备份过期天数"),
("DB_BACKUP_MAX_COUNT", "数据库备份最大保留份数"),
):
value = env.get(key, getattr(settings, key))
if isinstance(value, bool):
return f"{label}必须是大于等于 0 的整数"
try:
converted = int(value)
except (TypeError, ValueError):
return f"{label}必须是大于等于 0 的整数"
if converted < 0 or str(value).strip() != str(converted):
return f"{label}必须是大于等于 0 的整数"
return None
def _is_allowed_plugin_market_wiki_url(wiki_url: str) -> bool:
"""
校验插件市场 Wiki 地址是否属于固定文档源。
@@ -796,6 +836,9 @@ async def set_env_setting(
更新系统环境变量(仅管理员)
"""
validation_error = _validate_llm_server_tool_config(env)
if validation_error:
return _SchemaResponse(success=False, message=validation_error)
validation_error = _validate_database_backup_config(env)
if validation_error:
return _SchemaResponse(success=False, message=validation_error)
+176
View File
@@ -0,0 +1,176 @@
"""数据库备份与离线还原用例。"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timedelta
from pathlib import Path
from typing import Callable, Protocol
from app.adapters.system.backup.files import BackupFiles
from app.runtime.log import logger
@dataclass(frozen=True, slots=True)
class BackupPolicy:
"""一次备份操作使用的目录与保留策略快照。"""
root: Path
retention_days: int = 0
max_count: int = 0
def __post_init__(self) -> None:
if self.retention_days < 0 or self.max_count < 0:
raise ValueError("数据库备份保留策略不能使用负数")
@dataclass(frozen=True, slots=True)
class BackupArtifact:
"""一个已完成校验并发布的数据库备份文件。"""
name: str
db_type: str
created_at: datetime
path: Path
size: int
@dataclass(frozen=True, slots=True)
class BackupVerification:
"""数据库备份文件的基础内容校验结果。"""
valid: bool
method: str
detail: str | None = None
class BackupCheck(Protocol):
"""数据库适配器校验结果的结构合同。"""
valid: bool
method: str
detail: str | None
class DatabaseBackupBackend(Protocol):
"""活动数据库创建、校验和离线还原所需的最小技术端口。"""
db_type: str
suffix: str
def create(self, destination: Path) -> None:
"""把活动数据库的一致快照写入目标文件。"""
def verify(self, artifact: Path) -> BackupCheck:
"""在不修改数据库的前提下校验备份文件。"""
def restore(self, artifact: Path) -> None:
"""把已校验制品还原到离线目标数据库。"""
class DatabaseBackupService:
"""管理单文件数据库备份及明确的离线还原操作。"""
def __init__(
self,
*,
backend: DatabaseBackupBackend,
policy_reader: Callable[[], BackupPolicy],
clock: Callable[[], datetime] = datetime.now,
) -> None:
self._backend = backend
self._policy_reader = policy_reader
self._clock = clock
def create(self) -> BackupArtifact:
"""创建、校验并发布一个在线一致快照。"""
policy = self._policy_reader()
files = BackupFiles(policy.root)
created_at = self._clock()
name = files.available_name(
db_type=self._backend.db_type,
created_at=created_at,
suffix=self._backend.suffix,
)
temporary = files.create_temporary(self._backend.suffix)
try:
self._backend.create(temporary)
verification = self._backend.verify(temporary)
if not verification.valid:
detail = f"{verification.detail}" if verification.detail else ""
raise RuntimeError(
f"数据库备份校验失败({verification.method}{detail}"
)
path = files.publish(temporary, name)
except Exception:
files.discard(temporary)
raise
artifact = self._artifact(path, created_at=created_at)
self._prune(files, policy, keep=artifact.name)
logger.info(
f"数据库备份完成:文件={artifact.name},类型={artifact.db_type}"
f"大小={artifact.size} bytes"
)
return artifact
def list(self) -> tuple[BackupArtifact, ...]:
"""按创建时间倒序列出受管数据库备份文件。"""
files = BackupFiles(self._policy_reader().root)
return tuple(self._artifact(path) for path in files.list())
def verify(self, name: str) -> BackupVerification:
"""按文件名校验一个受管数据库备份。"""
path = BackupFiles(self._policy_reader().root).resolve(name)
self._require_matching_type(path)
result = self._backend.verify(path)
return BackupVerification(result.valid, result.method, result.detail)
def restore(self, name: str) -> BackupArtifact:
"""校验后将受管制品还原到当前 CLI 解析出的离线数据库目标。"""
path = BackupFiles(self._policy_reader().root).resolve(name)
self._require_matching_type(path)
verification = self._backend.verify(path)
if not verification.valid:
detail = f"{verification.detail}" if verification.detail else ""
raise RuntimeError(
f"数据库备份校验失败({verification.method}{detail}"
)
self._backend.restore(path)
logger.info(
f"数据库离线还原完成:文件={path.name},类型={self._backend.db_type}"
)
return self._artifact(path)
def _prune(self, files: BackupFiles, policy: BackupPolicy, *, keep: str) -> None:
"""新备份发布成功后按天数或份数清理同一目录中的旧文件。"""
cutoff = (
self._clock() - timedelta(days=policy.retention_days)
if policy.retention_days > 0
else None
)
for index, path in enumerate(files.list()):
if path.name == keep:
continue
created_at = files.created_at(path.name)
expired = cutoff is not None and created_at < cutoff
exceeds_count = policy.max_count > 0 and index >= policy.max_count
if expired or exceeds_count:
files.delete(path.name)
def _require_matching_type(self, path: Path) -> None:
db_type = BackupFiles.database_type(path.name)
if db_type != self._backend.db_type:
raise ValueError(
f"备份类型 {db_type} 与当前数据库类型 {self._backend.db_type} 不一致"
)
@staticmethod
def _artifact(path: Path, *, created_at: datetime | None = None) -> BackupArtifact:
return BackupArtifact(
name=path.name,
db_type=BackupFiles.database_type(path.name),
created_at=created_at or BackupFiles.created_at(path.name),
path=path,
size=path.stat().st_size,
)
+67 -14
View File
@@ -1,19 +1,25 @@
"""数据库连通性应用服务"""
"""数据库健康、清理、备份与离线还原的统一应用门面"""
from __future__ import annotations
from collections.abc import Callable
from typing import Optional
from typing import Any, Optional
from app.application.backup import (
BackupArtifact,
BackupVerification,
DatabaseBackupService,
)
from app.application.maintenance import DataCleanupService
DatabaseProbe = Callable[[], Optional[str]]
class DatabaseHealthService:
"""为模块和诊断入口提供不暴露会话实现的数据库探测能力。"""
"""提供不暴露会话实现的数据库探测能力。"""
def __init__(self, probe: DatabaseProbe) -> None:
"""保存由组合根提供的数据库探测端口。"""
self._probe = probe
def test(self) -> Optional[str]:
@@ -21,17 +27,64 @@ class DatabaseHealthService:
return self._probe()
_configured_database_health: DatabaseHealthService | None = None
class DatabaseGovernance:
"""向宿主入口提供唯一的数据库治理能力入口。"""
def __init__(
self,
*,
health: DatabaseHealthService,
cleanup: DataCleanupService,
backup: DatabaseBackupService,
) -> None:
self._health = health
self._cleanup = cleanup
self._backup = backup
def test(self) -> Optional[str]:
"""探测当前活动数据库。"""
return self._health.test()
def cleanup(
self,
*,
batch_size: int | None = None,
progress_callback: Callable[..., None] | None = None,
) -> dict[str, Any]:
"""按当前配置执行数据表清理。"""
return self._cleanup.execute(
batch_size=batch_size,
progress_callback=progress_callback,
)
def create_backup(self) -> BackupArtifact:
"""创建一个当前活动数据库的一致备份。"""
return self._backup.create()
def list_backups(self) -> tuple[BackupArtifact, ...]:
"""列出受管数据库备份文件。"""
return self._backup.list()
def verify_backup(self, name: str) -> BackupVerification:
"""校验一个受管数据库备份文件。"""
return self._backup.verify(name)
def restore_backup(self, name: str) -> BackupArtifact:
"""在离线 CLI 进程中还原一个受管数据库备份。"""
return self._backup.restore(name)
def configure_database_health(service: DatabaseHealthService) -> None:
"""由启动组合根登记数据库探测服务。"""
global _configured_database_health
_configured_database_health = service
_DATABASE_GOVERNANCE: list[DatabaseGovernance] = []
def get_configured_database_health() -> DatabaseHealthService:
"""返回启动阶段登记的数据库探测服务"""
if _configured_database_health is None:
raise RuntimeError("数据库探测服务尚未配置")
return _configured_database_health
def configure_database_governance(governance: DatabaseGovernance) -> None:
"""由启动组合根登记宿主唯一的数据库治理门面"""
_DATABASE_GOVERNANCE.clear()
_DATABASE_GOVERNANCE.append(governance)
def get_database_governance() -> DatabaseGovernance:
"""返回启动阶段登记的数据库治理门面。"""
if not _DATABASE_GOVERNANCE:
raise RuntimeError("数据库治理服务尚未配置")
return _DATABASE_GOVERNANCE[0]
-18
View File
@@ -324,24 +324,6 @@ def read_cleanup_policy() -> CleanupPolicy:
)
def build_cleanup_service() -> DataCleanupService:
"""返回启动组合根登记的清理服务。"""
if _configured_cleanup_service_factory is None:
raise RuntimeError("数据清理服务尚未配置")
return _configured_cleanup_service_factory()
_configured_cleanup_service_factory: Callable[[], DataCleanupService] | None = None
def configure_cleanup_service_factory(
factory: Callable[[], DataCleanupService],
) -> None:
"""由启动组合根登记数据清理服务工厂。"""
global _configured_cleanup_service_factory
_configured_cleanup_service_factory = factory
def _normalize_days(retention_days: Any) -> int:
"""把配置保留期规范为非负整数,非法值按关闭单表清理处理。"""
try:
+81 -1
View File
@@ -17,6 +17,8 @@ import psutil
from app.runtime.config import Settings, settings
from app.runtime.state import SystemHelper
from app.application.backup import BackupArtifact
from app.startup.database import build_database_governance
from version import APP_VERSION
BACKEND_RUNTIME_FILE = settings.TEMP_PATH / "moviepilot.runtime.json"
@@ -839,6 +841,84 @@ def cli() -> None:
"""MoviePilot 本地 CLI"""
def _format_backup_artifact(artifact: BackupArtifact) -> str:
"""将制品信息格式化为不含数据库凭据的单行 CLI 输出。"""
return "\t".join(
(
artifact.name,
artifact.db_type,
artifact.created_at.isoformat(),
str(artifact.size),
str(artifact.path),
)
)
@cli.group(context_settings=CONTEXT_SETTINGS)
def database() -> None:
"""创建、列举、校验和离线还原本地数据库备份"""
@database.command("backup", context_settings=CONTEXT_SETTINGS)
def database_backup() -> None:
"""创建并验证一个在线数据库备份"""
try:
artifact = build_database_governance().create_backup()
except Exception as error:
raise click.ClickException(f"数据库备份失败:{error}") from error
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
click.echo(_format_backup_artifact(artifact))
@database.command("list", context_settings=CONTEXT_SETTINGS)
def database_list() -> None:
"""列出当前受管目录中的正式数据库备份文件"""
try:
artifacts = build_database_governance().list_backups()
except Exception as error:
raise click.ClickException(f"数据库备份列表读取失败:{error}") from error
if not artifacts:
click.echo("暂无数据库备份")
return
click.echo("name\tdb_type\tcreated_at\tsize\tpath")
for artifact in artifacts:
click.echo(_format_backup_artifact(artifact))
@database.command("verify", context_settings=CONTEXT_SETTINGS)
@click.argument("name")
def database_verify(name: str) -> None:
"""按文件名重新校验一个数据库备份"""
try:
result = build_database_governance().verify_backup(name)
except Exception as error:
raise click.ClickException(f"数据库备份校验失败:{error}") from error
if not result.valid:
detail = f"{result.detail}" if result.detail else ""
raise click.ClickException(f"数据库备份校验未通过({result.method}{detail}")
click.echo(
f"数据库备份校验通过:name={name} method={result.method}"
)
@database.command("restore", context_settings=CONTEXT_SETTINGS)
@click.argument("name")
@click.option(
"--confirm",
is_flag=True,
help="确认 MoviePilot 已停止,并允许覆盖当前数据库",
)
def database_restore(name: str, confirm: bool) -> None:
"""在 MoviePilot 停止运行时还原一个数据库备份"""
if not confirm:
raise click.ClickException("离线还原必须使用 --confirm 明确确认")
try:
artifact = build_database_governance().restore_backup(name)
except Exception as error:
raise click.ClickException(f"数据库还原失败:{error}") from error
click.echo(f"数据库还原完成:{artifact.name}")
@cli.command(context_settings=CONTEXT_SETTINGS)
@click.option("--timeout", default=60, show_default=True, help="等待后端与前端就绪的秒数")
@click.option("--safe", is_flag=True, help="安全模式启动,仅保留核心 API,跳过插件和后台任务")
@@ -989,7 +1069,7 @@ def logs(lines: int, follow: bool, stdio: bool, frontend_log: bool) -> None:
@click.option("--deep", is_flag=True, help="执行可能较慢的深度检查")
def doctor(json_output: bool, fix: bool, deep: bool) -> None:
"""离线诊断本地 MoviePilot 运行环境,插件日志告警不影响整体状态"""
from app.doctor import run_doctor
from app.doctor import run_doctor # pylint: disable=no-name-in-module
from app.doctor.formatters import format_json_report, format_text_report
report = run_doctor(fix=fix, deep=deep)
+2 -2
View File
@@ -1,7 +1,7 @@
from typing import Tuple, Union
from app.runtime.config import settings
from app.application.database import get_configured_database_health
from app.application.database import get_database_governance
from app.modules import _ModuleBase
from app.schemas.types import ModuleType, OtherModulesType
@@ -51,7 +51,7 @@ class PostgreSQLModule(_ModuleBase):
"""
if settings.DB_TYPE != "postgresql":
return None
error = get_configured_database_health().test()
error = get_database_governance().test()
if error:
return False, f"PostgreSQL连接失败:{error}"
return True, ""
+20
View File
@@ -189,6 +189,17 @@ class ConfigModel(BaseModel):
# 例如经 PgBouncer 事务模式接入时 asyncpg 需要 {"statement_cache_size": 0}
DB_CONNECT_ARGS: dict = Field(default_factory=dict)
# ==================== 数据库备份配置 ====================
# 是否启用主程序数据库自动备份
DB_BACKUP_ENABLE: bool = False
# 定时备份的 Cron 表达式,留空时不注册定时任务
DB_BACKUP_CRON: str = "0 3 * * *"
# 备份根目录;未配置时使用 CONFIG_PATH/database_backup
DB_BACKUP_PATH: Optional[str] = None
# 本地备份的保留天数,0 表示不按时间清理
DB_BACKUP_RETENTION_DAYS: int = 30
# 本地备份的最大保留份数,0 表示不按数量清理
DB_BACKUP_MAX_COUNT: int = 30
# ==================== 数据清理配置 ====================
# 是否启用数据表定时清理
DATA_CLEANUP_ENABLE: bool = False
@@ -1074,6 +1085,15 @@ class Settings(BaseSettings, ConfigModel, LogConfigModel):
"""返回插件持久化数据目录。"""
return self.CONFIG_PATH / "plugins"
@property
def DATABASE_BACKUP_PATH(self) -> Path:
"""返回数据库备份根目录,允许相对当前配置目录进行配置。"""
configured = str(self.DB_BACKUP_PATH or "").strip()
if not configured:
return self.CONFIG_PATH / "database_backup"
path = Path(configured).expanduser()
return path if path.is_absolute() else self.CONFIG_PATH / path
@property
def LOG_PATH(self):
"""返回应用日志目录。"""
+36 -2
View File
@@ -28,7 +28,7 @@ from app.runtime.events import Event, eventmanager
from app.runtime.extensions.plugin_manager import PluginManager
from app.db.oper.agenttask import AgentTaskOper
from app.db.oper.systemconfig import SystemConfigOper
from app.application.maintenance import build_cleanup_service
from app.application.database import get_database_governance
from app.application.image import WallpaperHelper
from app.application.messaging.message import MessageHelper
from app.runtime.progress import AsyncProgressHelper, ProgressHelper
@@ -66,7 +66,7 @@ class SchedulerChain(ChainBase):
"""
按配置保留期执行分批清理。
"""
return build_cleanup_service().execute(
return get_database_governance().cleanup(
batch_size=batch_size,
progress_callback=progress_callback,
)
@@ -94,6 +94,8 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
"DATA_CLEANUP_DOWNLOAD_HISTORY_DAYS",
"DATA_CLEANUP_SITE_USERDATA_DAYS",
"DATA_CLEANUP_TRANSFER_HISTORY_DAYS",
"DB_BACKUP_ENABLE",
"DB_BACKUP_CRON",
"USAGE_STATISTIC_SHARE",
}
@@ -192,6 +194,35 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
"""
return (value or datetime.now()).strftime("%Y-%m-%d %H:%M:%S")
@staticmethod
def database_backup():
"""按当前宿主策略创建一次定时数据库备份。"""
return get_database_governance().create_backup()
def _register_database_backup_job(self) -> None:
"""在共享调度器中按当前配置维护唯一的数据库备份作业。"""
if not settings.DB_BACKUP_ENABLE or not settings.DB_BACKUP_CRON.strip():
return
job_id = "database_backup"
self._jobs[job_id] = {
"name": "数据库备份",
"func": self.database_backup,
"running": False,
}
self._scheduler.add_job(
self.start,
trigger=TimerUtils.build_schedule_trigger(
trigger_type="cron",
trigger_value=settings.DB_BACKUP_CRON,
timezone_name=settings.TZ,
),
id=job_id,
name="数据库备份",
kwargs={"job_id": job_id},
replace_existing=True,
)
def init(self) -> None:
"""
初始化定时服务
@@ -323,6 +354,9 @@ class Scheduler(ConfigReloadMixin, metaclass=SingletonClass):
executors={"default": ThreadPoolExecutor(settings.CONF.scheduler)},
)
# 数据库备份复用宿主调度器,不创建独立定时线程。
self._register_database_backup_job()
# CookieCloud定时同步
if (
settings.COOKIECLOUD_INTERVAL
+19
View File
@@ -0,0 +1,19 @@
"""插件可使用的数据库备份只读门面。"""
from app.application.backup import BackupArtifact, BackupVerification
from app.application.database import get_database_governance as _get_database_governance
def create_backup() -> BackupArtifact:
"""在宿主管理目录中创建当前数据库的一致备份。"""
return _get_database_governance().create_backup()
def list_backups() -> tuple[BackupArtifact, ...]:
"""列出宿主管理目录中的数据库备份。"""
return _get_database_governance().list_backups()
def verify_backup(name: str) -> BackupVerification:
"""按文件名校验一个宿主管理的数据库备份。"""
return _get_database_governance().verify_backup(name)
+61
View File
@@ -0,0 +1,61 @@
"""数据库治理能力的宿主组合根。"""
from __future__ import annotations
from app.adapters.system.backup.database import (
PostgreSQLBackupBackend,
SQLiteBackupBackend,
)
from app.application.backup import BackupPolicy, DatabaseBackupService
from app.application.database import (
DatabaseGovernance,
DatabaseHealthService,
configure_database_governance,
)
from app.application.maintenance import (
DataCleanupService,
read_cleanup_policy,
)
from app.db.engine import get_engine
from app.db.health import probe_database
from app.db.maintenance import DatabaseCleanupRepository
from app.db.session import SessionFactory
from app.runtime.config import settings
def build_database_governance() -> DatabaseGovernance:
"""以缓存同步引擎为事实源构造一个完整数据库治理门面。"""
engine = get_engine()
dialect = engine.dialect.name
if dialect == "sqlite":
backup_backend = SQLiteBackupBackend(engine)
elif dialect == "postgresql":
backup_backend = PostgreSQLBackupBackend(engine)
else:
raise RuntimeError(f"不支持的数据库类型:{dialect}")
return DatabaseGovernance(
health=DatabaseHealthService(probe_database),
cleanup=DataCleanupService(
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
policy_reader=read_cleanup_policy,
),
backup=DatabaseBackupService(
backend=backup_backend,
policy_reader=read_backup_policy,
),
)
def configure_database() -> None:
"""构造并登记宿主进程唯一的数据库治理门面。"""
configure_database_governance(build_database_governance())
def read_backup_policy() -> BackupPolicy:
"""读取一次可热更新的数据库备份目录与保留策略。"""
return BackupPolicy(
root=settings.DATABASE_BACKUP_PATH,
retention_days=settings.DB_BACKUP_RETENTION_DAYS,
max_count=settings.DB_BACKUP_MAX_COUNT,
)
+4 -18
View File
@@ -36,7 +36,7 @@ from app.application.messaging.message import (
stop_message,
)
from app.application.configuration import SystemConfigService, configure_system_config
from app.application.database import DatabaseHealthService, configure_database_health
from app.application.database import configure_database_governance
from app.application.service import configure_service_directory
from app.application.plugin.runtime import configure_plugin_runtime
from app.application.module import configure_module_runtime
@@ -55,19 +55,13 @@ from app.application.workflow import WorkflowQueryService, configure_workflow_qu
from app.application.agentdata import configure_agent_data_ports
from app.api.data import configure_api_data_ports
from app.application.subscription.write import configure_subscribe_writer
from app.application.maintenance import (
DataCleanupService,
configure_cleanup_service_factory,
read_cleanup_policy,
)
from app.adapters.external.server import (
MoviePilotServerHelper,
configure_server_application_services,
)
from app.application.server.report import ServerReportService
from app.application.server.share import ServerSharingService
from app.db import close_database
from app.db.session import get_async_db, get_db
from app.db.session import close_database, get_async_db, get_db
from app.db.uow import SqlAlchemyAsyncUnitOfWork, SqlAlchemyUnitOfWork
from app.db.oper.subscribe import SubscribeOper
from app.db.oper.agentchat import AgentChatOper
@@ -84,9 +78,6 @@ from app.db.oper.site import SiteOper
from app.db.oper.message import MessageOper
from app.db.oper.subscribehistory import SubscribeHistoryOper
from app.db.oper.plugindata import PluginDataOper
from app.db.maintenance import DatabaseCleanupRepository
from app.db.session import SessionFactory
from app.db.health import probe_database
from app.db.oper.systemconfig import SystemConfigOper
from app.db.oper.workflow import WorkflowOper
from app.command import CommandChain
@@ -94,6 +85,7 @@ from app.schemas.message import Message
from app.schemas.message import MessageType
from app.schemas.types import SystemConfigKey
from app.startup.agent_initializer import init_agent, stop_agent
from app.startup.database import build_database_governance
from app.startup.managed_resources_initializer import (
init_managed_resources,
stop_managed_resources,
@@ -441,7 +433,7 @@ async def init_modules():
user=lambda: UserOper(),
)
configure_system_config(SystemConfigService(repository=SystemConfigOper()))
configure_database_health(DatabaseHealthService(probe_database))
configure_database_governance(build_database_governance())
configure_agent_chat_service(AgentChatService(repository=AgentChatOper()))
configure_user_lookups(
by_id=lambda user_id: UserOper().get_by_id(user_id),
@@ -474,12 +466,6 @@ async def init_modules():
plugin_data=lambda: PluginDataOper(),
)
configure_subscribe_writer(lambda: SubscribeOper())
configure_cleanup_service_factory(
lambda: DataCleanupService(
repository=DatabaseCleanupRepository(session_factory=SessionFactory),
policy_reader=read_cleanup_policy,
)
)
# 托管资源只在这里装配声明与 adapter,具体资源仍由首个消费者显式激活。
init_managed_resources()
# 应用服务不反向依赖 Chain,由启动组合层注入壁纸来源。
+19 -3
View File
@@ -17,15 +17,30 @@ ENV LANG="C.UTF-8" \
ENV PATH="${VENV_PATH}/bin:${PATH}"
RUN apt-get update && apt-get install -y --no-install-recommends \
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl \
&& install -d -m 0755 /usr/share/postgresql-common/pgdg \
&& curl -fsSL \
--connect-timeout 10 \
--max-time 30 \
--retry 3 \
--retry-all-errors \
--retry-delay 2 \
--retry-max-time 90 \
"https://www.postgresql.org/media/keys/ACCC4CF8.asc" \
-o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
&& chmod 0644 /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc \
&& printf '%s\n' \
'deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main' \
> /etc/apt/sources.list.d/pgdg.list \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
nginx \
gettext-base \
locales \
procps \
gosu \
bash \
ca-certificates \
curl \
wget \
git \
gh \
@@ -45,6 +60,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
lsof \
nano \
unar \
postgresql-client-18 \
libchromaprint-tools \
libjemalloc2 \
&& dpkg-reconfigure --frontend noninteractive tzdata \
+35
View File
@@ -88,6 +88,7 @@ moviepilot help update
moviepilot help agent
moviepilot help config
moviepilot help config set
moviepilot help database
moviepilot help tool
moviepilot help scheduler
```
@@ -136,6 +137,10 @@ moviepilot config get
moviepilot config set
moviepilot config keys
moviepilot config describe
moviepilot database backup
moviepilot database list
moviepilot database verify <filename>
moviepilot database restore <filename> --confirm
moviepilot tool list
moviepilot tool show
moviepilot tool run
@@ -464,6 +469,36 @@ moviepilot config describe API_TOKEN --show-secrets
- `MUSIC_METADATA_TO_SIMPLIFIED` 默认开启;开启后会将识别结果中的曲名、艺术家、专辑和分类等标准音乐元数据转换为简体中文,不转换歌词与来源原始响应
- `config describe` 显示单个配置项的类型、默认值和当前值
## 数据库备份命令
创建一次在线一致备份:
```shell
moviepilot database backup
```
列出本地备份,并按文件名重新校验:
```shell
moviepilot database list
moviepilot database verify <filename>
```
MoviePilot 停止运行后,可通过明确确认执行离线还原:
```shell
moviepilot database restore <filename> --confirm
```
说明:
- SQLite 使用在线备份 APIPostgreSQL 使用镜像内置的 `pg_dump` custom format
- 源码部署使用 PostgreSQL 时,宿主机需安装 `pg_dump``pg_restore` 并加入 `PATH`Docker 镜像已内置
- 默认目录为配置目录下的 `database_backup/`,可通过 `DB_BACKUP_PATH` 调整
- 文件名包含数据库类型和创建时间,例如 `sqlite_20260819_030000.db`
- 备份、列举和校验可独立通过 CLI 执行
- 还原会覆盖当前数据库,执行前必须停止 MoviePilot;运行中的 Web API 和插件 SDK 不提供还原入口
## Tool 命令
列出所有 MCP 工具:
+43 -7
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6029,
"edge_sha256": "a312bbaffcb68adc5ead36130d0e1e20056cab73273727a55182f2bfed5efd34",
"edge_count": 6059,
"edge_sha256": "85ab4dd2f01f48bae417f0401f272763f15ba7256b58ca0ab5a8256f4946c2f9",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -2115,6 +2115,7 @@
"app.api.endpoints.system -> app.runtime.localization",
"app.api.endpoints.system -> app.runtime.log",
"app.api.endpoints.system -> app.runtime.progress",
"app.api.endpoints.system -> app.runtime.scheduling",
"app.api.endpoints.system -> app.runtime.state",
"app.api.endpoints.system -> app.schemas",
"app.api.endpoints.system -> app.schemas.common",
@@ -2302,11 +2303,20 @@
"app.application.audio -> app.runtime.log",
"app.application.audio -> app.schemas",
"app.application.audio -> app.schemas.types",
"app.application.backup -> app.adapters",
"app.application.backup -> app.adapters.system",
"app.application.backup -> app.adapters.system.backup",
"app.application.backup -> app.adapters.system.backup.files",
"app.application.backup -> app.runtime",
"app.application.backup -> app.runtime.log",
"app.application.chain.context -> app.application",
"app.application.chain.context -> app.application.chain",
"app.application.chain.context -> app.application.chain.data",
"app.application.dashboard -> app.schemas",
"app.application.dashboard -> app.schemas.dashboard",
"app.application.database -> app.application",
"app.application.database -> app.application.backup",
"app.application.database -> app.application.maintenance",
"app.application.directory -> app.adapters",
"app.application.directory -> app.adapters.system",
"app.application.directory -> app.adapters.system.host",
@@ -3214,11 +3224,15 @@
"app.chain.workflow -> app.schemas.types",
"app.chain.workflow -> app.schemas.workflow",
"app.chain.workflow -> app.workflow",
"app.cli -> app.application",
"app.cli -> app.application.backup",
"app.cli -> app.doctor",
"app.cli -> app.doctor.formatters",
"app.cli -> app.runtime",
"app.cli -> app.runtime.config",
"app.cli -> app.runtime.state",
"app.cli -> app.startup",
"app.cli -> app.startup.database",
"app.command -> app.application",
"app.command -> app.application.messaging",
"app.command -> app.application.messaging.message",
@@ -5395,8 +5409,8 @@
"app.scheduler -> app.agent",
"app.scheduler -> app.agent.runtime_loader",
"app.scheduler -> app.application",
"app.scheduler -> app.application.database",
"app.scheduler -> app.application.image",
"app.scheduler -> app.application.maintenance",
"app.scheduler -> app.application.messaging",
"app.scheduler -> app.application.messaging.message",
"app.scheduler -> app.application.scheduling",
@@ -5554,6 +5568,9 @@
"app.sdk.cache -> app.runtime.cache",
"app.sdk.config -> app.runtime",
"app.sdk.config -> app.runtime.config",
"app.sdk.database -> app.application",
"app.sdk.database -> app.application.backup",
"app.sdk.database -> app.application.database",
"app.sdk.events -> app.runtime",
"app.sdk.events -> app.runtime.events",
"app.sdk.logging -> app.runtime",
@@ -5667,6 +5684,21 @@
"app.startup.command_initializer -> app.application",
"app.startup.command_initializer -> app.application.commands",
"app.startup.command_initializer -> app.command",
"app.startup.database -> app.adapters",
"app.startup.database -> app.adapters.system",
"app.startup.database -> app.adapters.system.backup",
"app.startup.database -> app.adapters.system.backup.database",
"app.startup.database -> app.application",
"app.startup.database -> app.application.backup",
"app.startup.database -> app.application.database",
"app.startup.database -> app.application.maintenance",
"app.startup.database -> app.db",
"app.startup.database -> app.db.engine",
"app.startup.database -> app.db.health",
"app.startup.database -> app.db.maintenance",
"app.startup.database -> app.db.session",
"app.startup.database -> app.runtime",
"app.startup.database -> app.runtime.config",
"app.startup.database_initializer -> app.db",
"app.startup.database_initializer -> app.db.engine",
"app.startup.database_initializer -> app.db.models",
@@ -5744,7 +5776,6 @@
"app.startup.modules_initializer -> app.application.database",
"app.startup.modules_initializer -> app.application.history",
"app.startup.modules_initializer -> app.application.image",
"app.startup.modules_initializer -> app.application.maintenance",
"app.startup.modules_initializer -> app.application.messaging",
"app.startup.modules_initializer -> app.application.messaging.chat",
"app.startup.modules_initializer -> app.application.messaging.message",
@@ -5777,8 +5808,6 @@
"app.startup.modules_initializer -> app.chain.workflow",
"app.startup.modules_initializer -> app.command",
"app.startup.modules_initializer -> app.db",
"app.startup.modules_initializer -> app.db.health",
"app.startup.modules_initializer -> app.db.maintenance",
"app.startup.modules_initializer -> app.db.oper",
"app.startup.modules_initializer -> app.db.oper.agentchat",
"app.startup.modules_initializer -> app.db.oper.agenttask",
@@ -5818,6 +5847,7 @@
"app.startup.modules_initializer -> app.schemas.types",
"app.startup.modules_initializer -> app.startup",
"app.startup.modules_initializer -> app.startup.agent_initializer",
"app.startup.modules_initializer -> app.startup.database",
"app.startup.modules_initializer -> app.startup.managed_resources_initializer",
"app.startup.monitor_initializer -> app.monitor",
"app.startup.plugins_initializer -> app.adapters",
@@ -6046,7 +6076,7 @@
"app.workflow.actions.transfer_file -> app.workflow",
"app.workflow.actions.transfer_file -> app.workflow.actions"
],
"module_count": 746,
"module_count": 752,
"modules": [
"app",
"app.adapters",
@@ -6069,6 +6099,9 @@
"app.adapters.network.http",
"app.adapters.network.ip",
"app.adapters.system",
"app.adapters.system.backup",
"app.adapters.system.backup.database",
"app.adapters.system.backup.files",
"app.adapters.system.display",
"app.adapters.system.display.resource",
"app.adapters.system.fsproxy",
@@ -6275,6 +6308,7 @@
"app.application.agent",
"app.application.agentdata",
"app.application.audio",
"app.application.backup",
"app.application.chain",
"app.application.chain.context",
"app.application.chain.data",
@@ -6748,6 +6782,7 @@
"app.sdk.browser",
"app.sdk.cache",
"app.sdk.config",
"app.sdk.database",
"app.sdk.events",
"app.sdk.logging",
"app.sdk.media",
@@ -6761,6 +6796,7 @@
"app.startup.agent_initializer",
"app.startup.cache_initializer",
"app.startup.command_initializer",
"app.startup.database",
"app.startup.database_initializer",
"app.startup.domain_initializer",
"app.startup.lifecycle",
+33 -6
View File
@@ -1210,19 +1210,19 @@
},
{
"caller": "app.api.endpoints.system",
"line": 816
"line": 859
},
{
"caller": "app.api.endpoints.system",
"line": 930
"line": 973
},
{
"caller": "app.api.endpoints.system",
"line": 984
"line": 1027
},
{
"caller": "app.api.endpoints.system",
"line": 998
"line": 1041
},
{
"caller": "app.chain._messaging",
@@ -1652,7 +1652,7 @@
"consumers": [
{
"caller": "app.scheduler",
"line": 1116
"line": 1150
}
],
"producers": []
@@ -1813,7 +1813,7 @@
},
{
"caller": "app.scheduler",
"line": 736
"line": 770
}
]
},
@@ -3729,6 +3729,33 @@
"target": "app.runtime.config.settings"
}
],
"app.sdk.database": [
{
"kind": "import",
"name": "BackupArtifact",
"target": "app.application.backup.BackupArtifact"
},
{
"kind": "import",
"name": "BackupVerification",
"target": "app.application.backup.BackupVerification"
},
{
"kind": "FunctionDef",
"name": "create_backup",
"target": ""
},
{
"kind": "FunctionDef",
"name": "list_backups",
"target": ""
},
{
"kind": "FunctionDef",
"name": "verify_backup",
"target": ""
}
],
"app.sdk.events": [
{
"kind": "import",
+9 -5
View File
@@ -2,7 +2,7 @@ import tempfile
import unittest
from datetime import datetime, timedelta
from pathlib import Path
from unittest.mock import patch
from unittest.mock import MagicMock, patch
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
@@ -16,9 +16,9 @@ from app.runtime.config import settings
from app.scheduler import SchedulerChain
from app.application.maintenance import (
DataCleanupService,
configure_cleanup_service_factory,
read_cleanup_policy,
)
from app.application.database import DatabaseGovernance, configure_database_governance
from app.db.maintenance import DatabaseCleanupRepository
@@ -51,13 +51,17 @@ class DataCleanupChainTest(unittest.TestCase):
return patch.multiple(settings, **defaults)
def _configure_cleanup_service(self):
"""把当前测试数据库注入清理应用服务,替代旧的 SessionFactory 打桩"""
configure_cleanup_service_factory(
lambda: DataCleanupService(
"""把当前测试数据库注入统一数据库治理门面"""
configure_database_governance(
DatabaseGovernance(
health=MagicMock(),
backup=MagicMock(),
cleanup=DataCleanupService(
repository=DatabaseCleanupRepository(
session_factory=self.SessionFactory,
),
policy_reader=read_cleanup_policy,
),
)
)
+4 -4
View File
@@ -117,15 +117,15 @@ def test_cleanup_service_finishes_other_tables_before_raising_partial_failure()
def test_scheduler_cleanup_is_a_compatibility_delegate() -> None:
"""旧 SchedulerChain 入口应原样转发参数和返回值。"""
service = MagicMock()
service.execute.return_value = {"enabled": True}
governance = MagicMock()
governance.cleanup.return_value = {"enabled": True}
progress = MagicMock()
with patch("app.scheduler.build_cleanup_service", return_value=service):
with patch("app.scheduler.get_database_governance", return_value=governance):
result = SchedulerChain().cleanup(batch_size=7, progress_callback=progress)
assert result == {"enabled": True}
service.execute.assert_called_once_with(
governance.cleanup.assert_called_once_with(
batch_size=7,
progress_callback=progress,
)
+134
View File
@@ -0,0 +1,134 @@
from __future__ import annotations
import sqlite3
import subprocess
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from app.adapters.system.backup.database import (
PostgreSQLBackupBackend,
SQLiteBackupBackend,
)
def test_sqlite_backup_includes_committed_wal_data(tmp_path: Path) -> None:
source = tmp_path / "user.db"
with sqlite3.connect(source) as connection:
connection.execute("PRAGMA journal_mode=WAL")
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
connection.execute("INSERT INTO entries VALUES ('from-wal')")
engine = create_engine(f"sqlite:///{source}")
backend = SQLiteBackupBackend(engine)
artifact = tmp_path / "backup.db"
backend.create(artifact)
assert backend.verify(artifact).valid is True
with sqlite3.connect(artifact) as connection:
assert connection.execute("SELECT value FROM entries").fetchone() == ("from-wal",)
engine.dispose()
def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None:
source = tmp_path / "user.db"
backup = tmp_path / "backup.db"
for path, value in ((source, "old"), (backup, "restored")):
with sqlite3.connect(path) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
connection.execute("INSERT INTO entries VALUES (?)", (value,))
source.with_name("user.db-wal").write_bytes(b"old wal")
source.with_name("user.db-shm").write_bytes(b"old shm")
backend = SQLiteBackupBackend(create_engine(f"sqlite:///{source}"))
backend.restore(backup)
with sqlite3.connect(source) as connection:
assert connection.execute("SELECT value FROM entries").fetchone() == ("restored",)
assert not source.with_name("user.db-wal").exists()
assert not source.with_name("user.db-shm").exists()
class _Runner:
def __init__(self) -> None:
self.calls: list[tuple[list[str], dict]] = []
def __call__(self, command, **kwargs):
command = list(command)
self.calls.append((command, kwargs))
if "--file" in command:
Path(command[command.index("--file") + 1]).write_bytes(b"PGDMP")
stdout = "; archive listing" if command[:2] == ["pg_restore", "--list"] else ""
return subprocess.CompletedProcess(command, 0, stdout, "")
def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend:
engine = SimpleNamespace(
url=make_url(
"postgresql://moviepilot:secret@database.internal:5432/moviepilot"
"?sslmode=require"
),
dispose=Mock(),
)
return PostgreSQLBackupBackend(
engine,
runner=runner,
tool_resolver=lambda executable: executable,
)
def test_postgresql_backup_keeps_password_out_of_command_and_file(tmp_path: Path) -> None:
runner = _Runner()
backend = _postgres_backend(runner)
artifact = tmp_path / "backup.dump"
backend.create(artifact)
command, kwargs = runner.calls[0]
assert "--format=custom" in command
assert "secret" not in " ".join(command)
assert kwargs["env"]["PGPASSWORD"] == "secret"
assert kwargs["env"]["PGSSLMODE"] == "require"
assert artifact.read_bytes() == b"PGDMP"
def test_postgresql_verify_and_restore_use_pg_restore(tmp_path: Path) -> None:
runner = _Runner()
backend = _postgres_backend(runner)
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
assert backend.verify(artifact).valid is True
backend.restore(artifact)
verify_command, verify_kwargs = runner.calls[0]
restore_command, restore_kwargs = runner.calls[1]
assert verify_command == ["pg_restore", "--list", str(artifact)]
assert "PGPASSWORD" not in verify_kwargs["env"]
assert "--single-transaction" in restore_command
assert "--clean" in restore_command
assert restore_kwargs["env"]["PGPASSWORD"] == "secret"
def test_postgresql_source_install_reports_missing_native_client() -> None:
runner = _Runner()
engine = SimpleNamespace(
url=make_url("postgresql://moviepilot:secret@database/moviepilot"),
dispose=Mock(),
)
backend = PostgreSQLBackupBackend(
engine,
runner=runner,
tool_resolver=lambda _executable: None,
)
try:
backend.create(Path("unused.dump"))
except RuntimeError as error:
assert "安装与服务端同主版本或更高的 PostgreSQL client" in str(error)
else:
raise AssertionError("缺少 pg_dump 时未给出安装提示")
+77
View File
@@ -0,0 +1,77 @@
from datetime import datetime
from pathlib import Path
from unittest.mock import Mock
import pytest
from click.testing import CliRunner
from app import cli as cli_module
from app.application.backup import BackupArtifact, BackupVerification
from app.cli import cli
from app.sdk import database as database_sdk
NAME = "sqlite_20260819_030000.db"
def _artifact(tmp_path: Path) -> BackupArtifact:
path = tmp_path / NAME
path.write_bytes(b"snapshot")
return BackupArtifact(NAME, "sqlite", datetime(2026, 8, 19, 3, 0), path, 8)
def test_cli_backup_list_and_verify_use_local_governance(tmp_path: Path, monkeypatch) -> None:
governance = Mock()
governance.create_backup.return_value = _artifact(tmp_path)
governance.list_backups.return_value = (governance.create_backup.return_value,)
governance.verify_backup.return_value = BackupVerification(True, "integrity_check")
monkeypatch.setattr(cli_module, "build_database_governance", lambda: governance)
backup = CliRunner().invoke(cli, ["database", "backup"])
listed = CliRunner().invoke(cli, ["database", "list"])
verified = CliRunner().invoke(cli, ["database", "verify", NAME])
assert backup.exit_code == 0, backup.output
assert "name\tdb_type\tcreated_at\tsize\tpath" in backup.output
assert NAME in backup.output
assert listed.exit_code == 0 and NAME in listed.output
assert verified.exit_code == 0 and "校验通过" in verified.output
def test_cli_restore_requires_explicit_offline_confirmation(tmp_path: Path, monkeypatch) -> None:
governance = Mock()
governance.restore_backup.return_value = _artifact(tmp_path)
monkeypatch.setattr(cli_module, "build_database_governance", lambda: governance)
rejected = CliRunner().invoke(cli, ["database", "restore", NAME])
restored = CliRunner().invoke(cli, ["database", "restore", NAME, "--confirm"])
assert rejected.exit_code == 1
governance.restore_backup.assert_called_once_with(NAME)
assert restored.exit_code == 0 and "还原完成" in restored.output
def test_sdk_exposes_backup_without_restore_or_policy_controls(tmp_path: Path, monkeypatch) -> None:
governance = Mock()
artifact = _artifact(tmp_path)
verification = BackupVerification(True, "integrity_check")
governance.create_backup.return_value = artifact
governance.list_backups.return_value = (artifact,)
governance.verify_backup.return_value = verification
monkeypatch.setattr(database_sdk, "_get_database_governance", lambda: governance)
assert database_sdk.create_backup() == artifact
assert database_sdk.list_backups() == (artifact,)
assert database_sdk.verify_backup(NAME) == verification
assert not hasattr(database_sdk, "get_database_governance")
assert not hasattr(database_sdk, "restore_backup")
assert not hasattr(database_sdk, "delete_backup")
def test_sdk_rejects_invalid_name_at_host_boundary(monkeypatch) -> None:
governance = Mock()
governance.verify_backup.side_effect = ValueError("数据库备份文件名无效")
monkeypatch.setattr(database_sdk, "_get_database_governance", lambda: governance)
with pytest.raises(ValueError, match="文件名"):
database_sdk.verify_backup("../../user.db")
+50
View File
@@ -0,0 +1,50 @@
"""数据库备份所需 Docker 运行时工具合同。"""
import re
from pathlib import Path
def test_runtime_image_installs_postgresql_18_client_from_pgdg() -> None:
"""Bookworm 镜像必须从签名的 PGDG 源安装固定主版本客户端。"""
dockerfile = (
Path(__file__).resolve().parents[1] / "docker" / "Dockerfile"
).read_text(encoding="utf-8")
assert re.search(
r"^FROM python:[^\s]+-slim-bookworm AS base$", dockerfile, re.MULTILINE
)
assert "https://www.postgresql.org/media/keys/ACCC4CF8.asc" in dockerfile
for curl_option in (
"--connect-timeout 10",
"--max-time 30",
"--retry 3",
"--retry-all-errors",
"--retry-max-time 90",
):
assert curl_option in dockerfile
keyring = "/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc"
assert f"chmod 0644 {keyring}" in dockerfile
assert f"signed-by={keyring}" in dockerfile
assert "https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" in dockerfile
assert re.search(r"^\s+postgresql-client-18 \\$", dockerfile, re.MULTILINE)
assert not re.search(r"^\s+postgresql-client \\$", dockerfile, re.MULTILINE)
def test_runtime_image_keeps_pgdg_setup_architecture_neutral_and_cleans_apt_cache(
) -> None:
"""PGDG 原生架构解析需同时适用于 amd64/arm64,且不得遗留 APT 索引。"""
dockerfile = (
Path(__file__).resolve().parents[1] / "docker" / "Dockerfile"
).read_text(encoding="utf-8")
pgdg_sources = [
line for line in dockerfile.splitlines() if "apt.postgresql.org/pub/repos/apt" in line
]
assert len(pgdg_sources) == 1
pgdg_source = pgdg_sources[0]
assert "arch=" not in pgdg_source
assert "/var/lib/apt/lists/*" in dockerfile
assert dockerfile.index("postgresql-client-18") < dockerfile.index(
"/var/lib/apt/lists/*"
)
+40
View File
@@ -0,0 +1,40 @@
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
from app.application.database import get_database_governance
from app.startup import database as startup_database
def test_builder_uses_cached_engine_as_database_fact_source(monkeypatch) -> None:
engine = SimpleNamespace(dialect=SimpleNamespace(name="sqlite"))
backend = Mock(db_type="sqlite", suffix=".db")
sqlite_backend = Mock(return_value=backend)
monkeypatch.setattr(startup_database, "get_engine", lambda: engine)
monkeypatch.setattr(startup_database, "SQLiteBackupBackend", sqlite_backend)
monkeypatch.setattr(startup_database.settings, "DB_TYPE", "postgresql")
startup_database.build_database_governance()
sqlite_backend.assert_called_once_with(engine)
def test_backup_policy_reads_current_path_and_retention(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_PATH", str(tmp_path))
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_RETENTION_DAYS", 7)
monkeypatch.setattr(startup_database.settings, "DB_BACKUP_MAX_COUNT", 5)
policy = startup_database.read_backup_policy()
assert policy.root == tmp_path
assert policy.retention_days == 7
assert policy.max_count == 5
def test_configure_registers_one_database_governance(monkeypatch) -> None:
governance = Mock()
monkeypatch.setattr(startup_database, "build_database_governance", lambda: governance)
startup_database.configure_database()
assert get_database_governance() is governance
+93
View File
@@ -0,0 +1,93 @@
"""数据库备份与宿主调度器的接入合同。"""
import ast
from pathlib import Path
from unittest.mock import Mock
from app import scheduler as scheduler_module
from app.scheduler import Scheduler
class _SchedulerStub:
def __init__(self) -> None:
self.jobs = {}
def add_job(self, func, *, trigger, id, **kwargs) -> None:
self.jobs[id] = {"func": func, "trigger": trigger, **kwargs}
def _scheduler() -> Scheduler:
scheduler = object.__new__(Scheduler)
scheduler._scheduler = _SchedulerStub()
scheduler._jobs = {}
return scheduler
def test_database_backup_schedule_only_watches_job_shape() -> None:
assert Scheduler.CONFIG_WATCH.intersection({
"DB_BACKUP_ENABLE",
"DB_BACKUP_CRON",
"DB_BACKUP_PATH",
"DB_BACKUP_RETENTION_DAYS",
"DB_BACKUP_MAX_COUNT",
}) == {"DB_BACKUP_ENABLE", "DB_BACKUP_CRON"}
def test_disabled_database_backup_does_not_register_job(monkeypatch) -> None:
scheduler = _scheduler()
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", False)
scheduler._register_database_backup_job()
assert scheduler._scheduler.jobs == {}
def test_enabled_database_backup_without_cron_does_not_register_job(monkeypatch) -> None:
"""总开关开启但未配置周期时,不启用定时备份。"""
scheduler = _scheduler()
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", True)
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_CRON", "")
scheduler._register_database_backup_job()
assert scheduler._scheduler.jobs == {}
def test_enabled_database_backup_registers_single_replaceable_job(monkeypatch) -> None:
scheduler = _scheduler()
trigger = object()
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_ENABLE", True)
monkeypatch.setattr(scheduler_module.settings, "DB_BACKUP_CRON", "0 3 * * *")
monkeypatch.setattr(scheduler_module.TimerUtils, "build_schedule_trigger", Mock(return_value=trigger))
scheduler._register_database_backup_job()
scheduler._register_database_backup_job()
assert list(scheduler._scheduler.jobs) == ["database_backup"]
assert scheduler._scheduler.jobs["database_backup"]["replace_existing"] is True
def test_scheduled_backup_uses_registered_database_governance(monkeypatch) -> None:
governance = Mock()
monkeypatch.setattr(scheduler_module, "get_database_governance", lambda: governance)
result = Scheduler.database_backup()
assert result is governance.create_backup.return_value
governance.create_backup.assert_called_once_with()
def test_scheduler_database_dependencies_are_explicit_module_imports() -> None:
tree = ast.parse(
(Path(__file__).parents[1] / "app" / "scheduler.py").read_text(encoding="utf-8")
)
function_imports = [
node
for function in ast.walk(tree)
if isinstance(function, (ast.FunctionDef, ast.AsyncFunctionDef))
for node in ast.walk(function)
if isinstance(node, (ast.Import, ast.ImportFrom))
and getattr(node, "module", "")
and str(getattr(node, "module", "")).startswith("app.application.database")
]
assert function_imports == []
+115
View File
@@ -0,0 +1,115 @@
from __future__ import annotations
import stat
from dataclasses import dataclass
from datetime import datetime
from pathlib import Path
import pytest
from app.application.backup import BackupPolicy, DatabaseBackupService
@dataclass(frozen=True, slots=True)
class _Check:
valid: bool
method: str = "test-check"
detail: str | None = None
class _Backend:
db_type = "sqlite"
suffix = ".db"
def __init__(self, *, valid: bool = True) -> None:
self.valid = valid
self.restored: Path | None = None
def create(self, destination: Path) -> None:
destination.write_bytes(b"database snapshot")
def verify(self, artifact: Path) -> _Check:
return _Check(self.valid and artifact.read_bytes() == b"database snapshot")
def restore(self, artifact: Path) -> None:
self.restored = artifact
def _service(
root: Path,
*,
backend: _Backend | None = None,
now: datetime | None = None,
retention_days: int = 0,
max_count: int = 0,
) -> DatabaseBackupService:
return DatabaseBackupService(
backend=backend or _Backend(),
policy_reader=lambda: BackupPolicy(root, retention_days, max_count),
clock=lambda: now or datetime(2026, 8, 19, 13, 45, 26),
)
def test_create_publishes_one_readable_private_file(tmp_path: Path) -> None:
artifact = _service(tmp_path).create()
assert artifact.name == "sqlite_20260819_134526.db"
assert artifact.path.read_bytes() == b"database snapshot"
assert stat.S_IMODE(tmp_path.stat().st_mode) == 0o700
assert stat.S_IMODE(artifact.path.stat().st_mode) == 0o600
assert not list(tmp_path.glob("*.partial"))
def test_failed_verification_does_not_publish_artifact(tmp_path: Path) -> None:
with pytest.raises(RuntimeError, match="数据库备份校验失败"):
_service(tmp_path, backend=_Backend(valid=False)).create()
assert list(tmp_path.iterdir()) == []
def test_same_second_backups_receive_short_sequence_suffix(tmp_path: Path) -> None:
service = _service(tmp_path)
first = service.create()
second = service.create()
assert first.name == "sqlite_20260819_134526.db"
assert second.name == "sqlite_20260819_134526_1.db"
def test_retention_applies_after_new_artifact_is_available(tmp_path: Path) -> None:
old = _service(tmp_path, now=datetime(2026, 8, 1, 3, 0, 0)).create()
current = _service(
tmp_path,
now=datetime(2026, 8, 19, 3, 0, 0),
retention_days=7,
max_count=1,
).create()
assert current.path.exists()
assert not old.path.exists()
def test_list_ignores_unmanaged_files_and_rejects_paths(tmp_path: Path) -> None:
artifact = _service(tmp_path).create()
(tmp_path / "notes.txt").write_text("ignore", encoding="utf-8")
assert [item.name for item in _service(tmp_path).list()] == [artifact.name]
with pytest.raises(ValueError, match="文件名"):
_service(tmp_path).verify("../user.db")
def test_restore_requires_matching_database_type(tmp_path: Path) -> None:
backend = _Backend()
service = _service(tmp_path, backend=backend)
artifact = service.create()
restored = service.restore(artifact.name)
assert restored.name == artifact.name
assert backend.restored == artifact.path
postgres = tmp_path / "postgresql_20260819_134526.dump"
postgres.write_bytes(b"database snapshot")
with pytest.raises(ValueError, match="当前数据库类型"):
service.restore(postgres.name)
@@ -0,0 +1,89 @@
"""系统数据库备份策略配置测试。"""
import asyncio
from unittest.mock import patch
import pytest
from app.api.endpoints import system as system_endpoint
@pytest.mark.parametrize(
("env", "message"),
[
({"DB_BACKUP_CRON": "0 3 * *"}, "数据库备份周期格式不正确"),
({"DB_BACKUP_PATH": 123}, "数据库备份目录必须是路径字符串"),
({"DB_BACKUP_RETENTION_DAYS": -1}, "数据库备份过期天数"),
({"DB_BACKUP_RETENTION_DAYS": 1.5}, "数据库备份过期天数"),
({"DB_BACKUP_MAX_COUNT": True}, "数据库备份最大保留份数"),
({"DB_BACKUP_MAX_COUNT": "many"}, "数据库备份最大保留份数"),
],
)
def test_database_backup_policy_rejects_invalid_values(
env: dict,
message: str,
) -> None:
"""无效策略必须在批量配置写入前被拒绝。"""
env["DB_BACKUP_ENABLE"] = True
assert message in str(system_endpoint._validate_database_backup_config(env))
@pytest.mark.parametrize(
"env",
[
{"DB_BACKUP_CRON": ""},
{"DB_BACKUP_CRON": "0 3 * * *"},
{"DB_BACKUP_PATH": None},
{"DB_BACKUP_PATH": ""},
{"DB_BACKUP_PATH": "database_backup"},
{"DB_BACKUP_RETENTION_DAYS": 0},
{"DB_BACKUP_MAX_COUNT": "0"},
],
)
def test_database_backup_policy_accepts_supported_boundaries(env: dict) -> None:
"""空目录使用默认路径,两个保留值的零均表示不限制。"""
env["DB_BACKUP_ENABLE"] = True
assert system_endpoint._validate_database_backup_config(env) is None
@pytest.mark.parametrize("disabled", [False, "false", "0", "off"])
def test_database_backup_policy_rejects_invalid_hidden_values_when_disabled(disabled) -> None:
"""关闭总开关只暂停调度,不能把无效策略写入持久配置。"""
error = system_endpoint._validate_database_backup_config({
"DB_BACKUP_ENABLE": disabled,
"DB_BACKUP_CRON": "invalid",
"DB_BACKUP_PATH": 123,
"DB_BACKUP_RETENTION_DAYS": -1,
"DB_BACKUP_MAX_COUNT": 1.5,
})
assert error is not None
def test_set_env_rejects_invalid_database_backup_policy_without_partial_write() -> None:
"""备份策略校验失败时不得调用 Settings 的批量写入。"""
env = {
"DB_BACKUP_ENABLE": True,
"DB_BACKUP_CRON": "invalid",
"DB_BACKUP_RETENTION_DAYS": 30,
"DB_BACKUP_MAX_COUNT": 30,
}
with patch.object(
system_endpoint,
"_validate_llm_server_tool_config",
return_value=None,
), patch.object(type(system_endpoint.settings), "update_settings") as update_settings:
response = asyncio.run(system_endpoint.set_env_setting(env=env, _=object()))
assert response.success is False
assert "数据库备份周期格式不正确" in response.message
update_settings.assert_not_called()
def test_database_backup_default_path_tracks_config_directory(tmp_path, monkeypatch) -> None:
"""未显式配置目录时应跟随当前配置根,而不是写死 Docker 路径。"""
monkeypatch.setattr(system_endpoint.settings, "CONFIG_DIR", str(tmp_path))
monkeypatch.setattr(system_endpoint.settings, "DB_BACKUP_PATH", None)
assert system_endpoint.settings.DATABASE_BACKUP_PATH == tmp_path / "database_backup"