feat(doctor): add database backup recovery guidance (#6397)

This commit is contained in:
InfinityPacer
2026-08-22 19:28:20 +08:00
committed by GitHub
parent 50ac120123
commit f2a23b0377
5 changed files with 361 additions and 37 deletions
+84 -33
View File
@@ -46,6 +46,73 @@ class ProcessRunner(Protocol):
"""执行命令并返回结果。"""
def verify_database_backup(
artifact: Path,
*,
db_type: str,
runner: ProcessRunner = subprocess.run,
tool_resolver: Callable[[str], str | None] = shutil.which,
pg_restore: str = "pg_restore",
) -> DatabaseBackupCheck:
"""在不访问活动数据库的前提下校验一个受管备份文件。"""
if db_type == "sqlite":
method = "PRAGMA integrity_check"
try:
# 正式备份不会再变化,immutable 可避免只读校验创建 WAL 旁路文件。
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
with closing(sqlite3.connect(uri, uri=True)) as connection:
rows = connection.execute("PRAGMA integrity_check").fetchall()
except sqlite3.Error as error:
return DatabaseBackupCheck(False, method, str(error))
valid = bool(rows) and all(row[0] == "ok" for row in rows)
detail = None if valid else "; ".join(str(row[0]) for row in rows)
return DatabaseBackupCheck(valid, method, detail)
if db_type == "postgresql":
method = "pg_restore --list"
executable = _require_tool(pg_restore, tool_resolver)
result = runner(
[executable, "--list", str(artifact)],
env=_postgres_environment(),
capture_output=True,
text=True,
check=False,
)
valid = result.returncode == 0 and bool(result.stdout.strip())
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
return DatabaseBackupCheck(valid, method, detail)
raise ValueError(f"不支持的数据库备份类型:{db_type}")
def _require_tool(
executable: str,
tool_resolver: Callable[[str], str | None],
) -> str:
resolved = tool_resolver(executable)
if resolved is None:
raise RuntimeError(
f"未找到 {executable},请安装与服务端同主版本或更高的 "
"PostgreSQL client 并加入 PATH"
)
return resolved
def _postgres_environment(
*,
password: str | None = None,
sslmode: str | None = None,
) -> dict[str, str]:
environment = dict(os.environ)
environment.pop("PGPASSWORD", None)
environment.pop("PGSSLMODE", None)
if password:
environment["PGPASSWORD"] = password
if sslmode:
environment["PGSSLMODE"] = sslmode
return environment
class SQLiteBackupBackend:
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
@@ -71,17 +138,7 @@ class SQLiteBackupBackend:
def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 SQLite integrity_check 校验备份内容。"""
method = "PRAGMA integrity_check"
try:
# 已发布前的临时快照不会再变化;immutable 避免 WAL 模式为只读校验创建旁路文件。
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
with closing(sqlite3.connect(uri, uri=True)) as connection:
rows = connection.execute("PRAGMA integrity_check").fetchall()
except sqlite3.Error as error:
return DatabaseBackupCheck(False, method, str(error))
valid = bool(rows) and all(row[0] == "ok" for row in rows)
detail = None if valid else "; ".join(str(row[0]) for row in rows)
return DatabaseBackupCheck(valid, method, detail)
return verify_database_backup(artifact, db_type=self.db_type)
def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
@@ -137,14 +194,13 @@ class PostgreSQLBackupBackend:
def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
method = "pg_restore --list"
result = self._run(
[self._require_tool(self._pg_restore), "--list", str(artifact)],
include_password=False,
return verify_database_backup(
artifact,
db_type=self.db_type,
runner=self._runner,
tool_resolver=self._tool_resolver,
pg_restore=self._pg_restore,
)
valid = result.returncode == 0 and bool(result.stdout.strip())
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
return DatabaseBackupCheck(valid, method, detail)
def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
@@ -189,21 +245,16 @@ class PostgreSQLBackupBackend:
)
def _require_tool(self, executable: str) -> str:
resolved = self._tool_resolver(executable)
if resolved is None:
raise RuntimeError(
f"未找到 {executable},请安装与服务端同主版本或更高的 "
"PostgreSQL client 并加入 PATH"
)
return resolved
return _require_tool(executable, self._tool_resolver)
def _environment(self, *, include_password: bool) -> dict[str, str]:
environment = dict(os.environ)
environment.pop("PGPASSWORD", None)
environment.pop("PGSSLMODE", None)
if include_password and self._engine.url.password:
environment["PGPASSWORD"] = str(self._engine.url.password)
password = (
str(self._engine.url.password)
if include_password and self._engine.url.password
else None
)
sslmode = self._engine.url.query.get("sslmode")
if sslmode:
environment["PGSSLMODE"] = str(sslmode)
return environment
return _postgres_environment(
password=password,
sslmode=str(sslmode) if sslmode else None,
)
+112
View File
@@ -18,6 +18,8 @@ from urllib.request import Request, urlopen
import psutil
from app.adapters.system.backup.database import verify_database_backup
from app.adapters.system.backup.files import BackupFiles
from app.runtime.config import settings
from app.runtime.topology import process_topology_issue
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorReport, DoctorSeverity
@@ -824,6 +826,116 @@ def _check_database(runner: DoctorRunnerProtocol) -> None:
_check_postgresql_database(runner)
else:
_check_sqlite_database(runner)
_check_database_backups(runner)
def _check_database_backups(runner: DoctorRunnerProtocol) -> None:
"""列举并离线校验与当前数据库类型匹配的受管备份。"""
db_type = "postgresql" if settings.DB_TYPE.lower() == "postgresql" else "sqlite"
try:
paths = BackupFiles(settings.DATABASE_BACKUP_PATH).list()
except OSError as error:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Error,
status=DoctorFindingStatus.Failed,
title="数据库备份目录无法读取",
detail=str(error),
recommendation="检查数据库备份目录是否存在且当前用户具有读取权限。",
context={"db_type": db_type},
)
return
matching = [path for path in paths if BackupFiles.database_type(path.name) == db_type]
mismatched = [path.name for path in paths if path not in matching]
if not paths:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Info,
status=DoctorFindingStatus.Skipped,
title="未找到受管数据库备份",
detail=f"当前数据库类型为 {db_type},备份目录中没有正式备份文件。",
recommendation="可执行 `moviepilot database backup` 创建一次可校验备份。",
affects_report_status=False,
context={"db_type": db_type, "backups": []},
)
return
if not matching:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Warn,
status=DoctorFindingStatus.Degraded,
title="没有匹配当前数据库类型的备份",
detail=f"当前数据库类型为 {db_type},仅找到其他类型备份:{', '.join(mismatched)}",
recommendation="确认数据库类型配置,或执行 `moviepilot database backup` 创建当前类型备份。",
affects_report_status=False,
context={"db_type": db_type, "backups": [], "mismatched": mismatched},
)
return
backups = []
for path in matching:
try:
size = path.stat().st_size
verification = verify_database_backup(path, db_type=db_type)
valid = verification.valid
method = verification.method
detail = verification.detail
except (OSError, RuntimeError, ValueError) as error:
size = None
valid = False
method = "unavailable"
detail = str(error)
backups.append(
{
"name": path.name,
"db_type": db_type,
"size": size,
"valid": valid,
"method": method,
"detail": detail,
}
)
valid_backups = [backup for backup in backups if backup["valid"]]
context = {
"db_type": db_type,
"backups": backups,
"mismatched": mismatched,
}
if valid_backups:
newest = valid_backups[0]
command = f"moviepilot database restore {newest['name']} --confirm"
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Info,
status=DoctorFindingStatus.Ok,
title="存在可还原的数据库备份",
detail=f"已校验 {len(backups)}{db_type} 备份,其中 {len(valid_backups)} 个可用。",
recommendation=f"需要恢复时先停止 MoviePilot,再执行 `{command}`。",
context={**context, "restore_command": command},
)
return
failures = "; ".join(
f"{backup['name']}: {backup['detail'] or backup['method']}"
for backup in backups
)
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Error,
status=DoctorFindingStatus.Failed,
title="数据库备份均未通过校验",
detail=(
f"已检查 {len(backups)}{db_type} 备份,没有可直接还原的文件。"
f"校验结果:{failures}"
),
recommendation="根据校验详情处理备份文件或 PostgreSQL client,再重新运行 doctor。",
affects_report_status=False,
context=context,
)
def _check_frontend_assets(runner: DoctorRunnerProtocol) -> None:
+5 -2
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [],
"workflow_to_db": []
},
"edge_count": 6379,
"edge_sha256": "1c619d72157004590838a85c497330b8ca462d3bc1eb490fd5e7bac97e6190b8",
"edge_count": 6382,
"edge_sha256": "d94f2e60efcc1697fe8dfa0ad23cbc99c38fd4873caf889ae1c44769c69fdcc3",
"edges": [
"app -> app.runtime",
"app -> app.runtime.compat",
@@ -3659,6 +3659,9 @@
"app.db.session -> app.runtime.observability",
"app.doctor.checks -> app.adapters",
"app.doctor.checks -> app.adapters.system",
"app.doctor.checks -> app.adapters.system.backup",
"app.doctor.checks -> app.adapters.system.backup.database",
"app.doctor.checks -> app.adapters.system.backup.files",
"app.doctor.checks -> app.adapters.system.host",
"app.doctor.checks -> app.doctor",
"app.doctor.checks -> app.doctor.models",
+65
View File
@@ -6,12 +6,14 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url
from app.adapters.system.backup.database import (
PostgreSQLBackupBackend,
SQLiteBackupBackend,
verify_database_backup,
)
@@ -34,6 +36,15 @@ def test_sqlite_backup_includes_committed_wal_data(tmp_path: Path) -> None:
engine.dispose()
def test_sqlite_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""Doctor 等离线入口不应为校验备份而构造活动数据库引擎。"""
artifact = tmp_path / "backup.db"
with sqlite3.connect(artifact) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
assert verify_database_backup(artifact, db_type="sqlite").valid is True
def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None:
source = tmp_path / "user.db"
backup = tmp_path / "backup.db"
@@ -66,6 +77,11 @@ class _Runner:
return subprocess.CompletedProcess(command, 0, stdout, "")
class _FailedRunner:
def __call__(self, command, **_kwargs):
return subprocess.CompletedProcess(command, 1, "", "invalid archive")
def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend:
engine = SimpleNamespace(
url=make_url(
@@ -114,6 +130,55 @@ def test_postgresql_verify_and_restore_use_pg_restore(tmp_path: Path) -> None:
assert restore_kwargs["env"]["PGPASSWORD"] == "secret"
def test_postgresql_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""PostgreSQL 归档校验只依赖 pg_restore,不连接活动数据库。"""
runner = _Runner()
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=runner,
tool_resolver=lambda executable: executable,
)
assert result.valid is True
command, kwargs = runner.calls[0]
assert command == ["pg_restore", "--list", str(artifact)]
assert "PGPASSWORD" not in kwargs["env"]
assert "PGSSLMODE" not in kwargs["env"]
def test_postgresql_offline_verify_rejects_invalid_archive(tmp_path: Path) -> None:
"""pg_restore 无法读取归档目录时备份必须判定为无效。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"invalid")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=_FailedRunner(),
tool_resolver=lambda executable: executable,
)
assert result.valid is False
assert result.detail == "pg_restore 退出码 1"
def test_postgresql_offline_verify_reports_missing_client(tmp_path: Path) -> None:
"""缺少 pg_restore 时离线校验应给出可执行的安装提示。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
with pytest.raises(RuntimeError, match="PostgreSQL client"):
verify_database_backup(
artifact,
db_type="postgresql",
tool_resolver=lambda _executable: None,
)
def test_postgresql_source_install_reports_missing_native_client() -> None:
runner = _Runner()
engine = SimpleNamespace(
+95 -2
View File
@@ -1,13 +1,14 @@
from __future__ import annotations
from datetime import datetime, timedelta
import sqlite3
from types import SimpleNamespace
from app.runtime.config import settings
from app.doctor import checks, run_doctor
from app.doctor import checks
from app.doctor.formatters import format_json_report, format_text_report
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorSeverity
from app.doctor.runner import DoctorRunner
from app.doctor.runner import DoctorRunner, run_doctor
def _current_log_timestamp() -> str:
@@ -33,6 +34,98 @@ def test_doctor_report_has_stable_json_shape(tmp_path, monkeypatch):
assert any(item["id"] == "runtime.paths" for item in payload["findings"])
def test_doctor_reports_valid_backup_when_sqlite_database_is_corrupt(
tmp_path,
monkeypatch,
):
"""主数据库损坏时 Doctor 仍应离线校验备份并给出还原命令。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
(tmp_path / "user.db").write_bytes(b"not a sqlite database")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
with sqlite3.connect(backup) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
(backup_dir / "sqlite_20260822_040000.db").write_bytes(b"invalid newer backup")
runner = DoctorRunner()
checks._check_database(runner)
assert runner.report.find("database.sqlite_open_failed") is not None
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Ok
assert finding.context["backups"][0]["valid"] is False
assert finding.context["backups"][1]["valid"] is True
assert finding.context["restore_command"] == (
"moviepilot database restore sqlite_20260822_030000.db --confirm"
)
assert finding.context["restore_command"] in finding.recommendation
def test_doctor_distinguishes_missing_and_mismatched_backups(tmp_path, monkeypatch):
"""无备份与仅存在其他数据库类型备份应生成不同诊断结论。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
runner = DoctorRunner()
checks._check_database_backups(runner)
missing = runner.report.find("database.backup_recovery")
assert missing is not None
assert missing.status == DoctorFindingStatus.Skipped
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
runner = DoctorRunner()
checks._check_database_backups(runner)
mismatched = runner.report.find("database.backup_recovery")
assert mismatched is not None
assert mismatched.status == DoctorFindingStatus.Degraded
assert mismatched.context["mismatched"] == ["postgresql_20260822_030000.dump"]
def test_doctor_reports_invalid_backup_without_modifying_it(tmp_path, monkeypatch):
"""Doctor --fix 也只校验备份,不覆盖或删除无效文件。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
original = b"invalid sqlite backup"
backup.write_bytes(original)
runner = DoctorRunner(fix=True)
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert finding.affects_report_status is False
assert finding.context["backups"][0]["valid"] is False
assert backup.read_bytes() == original
assert runner.report.status.value == "healthy"
def test_doctor_exposes_missing_pg_restore_in_text_finding(tmp_path, monkeypatch):
"""PostgreSQL 离线校验工具缺失时应直接告诉用户如何补齐。"""
def missing_pg_restore(*_args, **_kwargs):
raise RuntimeError("未找到 pg_restore,请安装 PostgreSQL client 并加入 PATH")
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
monkeypatch.setattr(settings, "DB_TYPE", "postgresql")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
monkeypatch.setattr(checks, "verify_database_backup", missing_pg_restore)
runner = DoctorRunner()
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert "未找到 pg_restore" in finding.detail
assert "PostgreSQL client" in format_text_report(runner.report)
def test_doctor_formatters_include_status_and_finding(tmp_path, monkeypatch):
"""doctor 文本和 JSON 格式化应展示状态与诊断项。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))