feat(doctor): add database backup recovery guidance (#6397)

This commit is contained in:
InfinityPacer
2026-08-22 19:28:20 +08:00
committed by GitHub
parent 50ac120123
commit f2a23b0377
5 changed files with 361 additions and 37 deletions
+84 -33
View File
@@ -46,6 +46,73 @@ class ProcessRunner(Protocol):
"""执行命令并返回结果。""" """执行命令并返回结果。"""
def verify_database_backup(
artifact: Path,
*,
db_type: str,
runner: ProcessRunner = subprocess.run,
tool_resolver: Callable[[str], str | None] = shutil.which,
pg_restore: str = "pg_restore",
) -> DatabaseBackupCheck:
"""在不访问活动数据库的前提下校验一个受管备份文件。"""
if db_type == "sqlite":
method = "PRAGMA integrity_check"
try:
# 正式备份不会再变化,immutable 可避免只读校验创建 WAL 旁路文件。
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
with closing(sqlite3.connect(uri, uri=True)) as connection:
rows = connection.execute("PRAGMA integrity_check").fetchall()
except sqlite3.Error as error:
return DatabaseBackupCheck(False, method, str(error))
valid = bool(rows) and all(row[0] == "ok" for row in rows)
detail = None if valid else "; ".join(str(row[0]) for row in rows)
return DatabaseBackupCheck(valid, method, detail)
if db_type == "postgresql":
method = "pg_restore --list"
executable = _require_tool(pg_restore, tool_resolver)
result = runner(
[executable, "--list", str(artifact)],
env=_postgres_environment(),
capture_output=True,
text=True,
check=False,
)
valid = result.returncode == 0 and bool(result.stdout.strip())
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
return DatabaseBackupCheck(valid, method, detail)
raise ValueError(f"不支持的数据库备份类型:{db_type}")
def _require_tool(
executable: str,
tool_resolver: Callable[[str], str | None],
) -> str:
resolved = tool_resolver(executable)
if resolved is None:
raise RuntimeError(
f"未找到 {executable},请安装与服务端同主版本或更高的 "
"PostgreSQL client 并加入 PATH"
)
return resolved
def _postgres_environment(
*,
password: str | None = None,
sslmode: str | None = None,
) -> dict[str, str]:
environment = dict(os.environ)
environment.pop("PGPASSWORD", None)
environment.pop("PGSSLMODE", None)
if password:
environment["PGPASSWORD"] = password
if sslmode:
environment["PGSSLMODE"] = sslmode
return environment
class SQLiteBackupBackend: class SQLiteBackupBackend:
"""使用 SQLite 在线备份 API 管理活动文件数据库。""" """使用 SQLite 在线备份 API 管理活动文件数据库。"""
@@ -71,17 +138,7 @@ class SQLiteBackupBackend:
def verify(self, artifact: Path) -> DatabaseBackupCheck: def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 SQLite integrity_check 校验备份内容。""" """通过 SQLite integrity_check 校验备份内容。"""
method = "PRAGMA integrity_check" return verify_database_backup(artifact, db_type=self.db_type)
try:
# 已发布前的临时快照不会再变化;immutable 避免 WAL 模式为只读校验创建旁路文件。
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
with closing(sqlite3.connect(uri, uri=True)) as connection:
rows = connection.execute("PRAGMA integrity_check").fetchall()
except sqlite3.Error as error:
return DatabaseBackupCheck(False, method, str(error))
valid = bool(rows) and all(row[0] == "ok" for row in rows)
detail = None if valid else "; ".join(str(row[0]) for row in rows)
return DatabaseBackupCheck(valid, method, detail)
def restore(self, artifact: Path) -> None: def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中原子替换活动 SQLite 文件。""" """在 CLI 离线进程中原子替换活动 SQLite 文件。"""
@@ -137,14 +194,13 @@ class PostgreSQLBackupBackend:
def verify(self, artifact: Path) -> DatabaseBackupCheck: def verify(self, artifact: Path) -> DatabaseBackupCheck:
"""通过 pg_restore 目录读取校验 custom-format 归档。""" """通过 pg_restore 目录读取校验 custom-format 归档。"""
method = "pg_restore --list" return verify_database_backup(
result = self._run( artifact,
[self._require_tool(self._pg_restore), "--list", str(artifact)], db_type=self.db_type,
include_password=False, runner=self._runner,
tool_resolver=self._tool_resolver,
pg_restore=self._pg_restore,
) )
valid = result.returncode == 0 and bool(result.stdout.strip())
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
return DatabaseBackupCheck(valid, method, detail)
def restore(self, artifact: Path) -> None: def restore(self, artifact: Path) -> None:
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。""" """在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
@@ -189,21 +245,16 @@ class PostgreSQLBackupBackend:
) )
def _require_tool(self, executable: str) -> str: def _require_tool(self, executable: str) -> str:
resolved = self._tool_resolver(executable) return _require_tool(executable, self._tool_resolver)
if resolved is None:
raise RuntimeError(
f"未找到 {executable},请安装与服务端同主版本或更高的 "
"PostgreSQL client 并加入 PATH"
)
return resolved
def _environment(self, *, include_password: bool) -> dict[str, str]: def _environment(self, *, include_password: bool) -> dict[str, str]:
environment = dict(os.environ) password = (
environment.pop("PGPASSWORD", None) str(self._engine.url.password)
environment.pop("PGSSLMODE", None) if include_password and self._engine.url.password
if include_password and self._engine.url.password: else None
environment["PGPASSWORD"] = str(self._engine.url.password) )
sslmode = self._engine.url.query.get("sslmode") sslmode = self._engine.url.query.get("sslmode")
if sslmode: return _postgres_environment(
environment["PGSSLMODE"] = str(sslmode) password=password,
return environment sslmode=str(sslmode) if sslmode else None,
)
+112
View File
@@ -18,6 +18,8 @@ from urllib.request import Request, urlopen
import psutil import psutil
from app.adapters.system.backup.database import verify_database_backup
from app.adapters.system.backup.files import BackupFiles
from app.runtime.config import settings from app.runtime.config import settings
from app.runtime.topology import process_topology_issue from app.runtime.topology import process_topology_issue
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorReport, DoctorSeverity from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorReport, DoctorSeverity
@@ -824,6 +826,116 @@ def _check_database(runner: DoctorRunnerProtocol) -> None:
_check_postgresql_database(runner) _check_postgresql_database(runner)
else: else:
_check_sqlite_database(runner) _check_sqlite_database(runner)
_check_database_backups(runner)
def _check_database_backups(runner: DoctorRunnerProtocol) -> None:
"""列举并离线校验与当前数据库类型匹配的受管备份。"""
db_type = "postgresql" if settings.DB_TYPE.lower() == "postgresql" else "sqlite"
try:
paths = BackupFiles(settings.DATABASE_BACKUP_PATH).list()
except OSError as error:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Error,
status=DoctorFindingStatus.Failed,
title="数据库备份目录无法读取",
detail=str(error),
recommendation="检查数据库备份目录是否存在且当前用户具有读取权限。",
context={"db_type": db_type},
)
return
matching = [path for path in paths if BackupFiles.database_type(path.name) == db_type]
mismatched = [path.name for path in paths if path not in matching]
if not paths:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Info,
status=DoctorFindingStatus.Skipped,
title="未找到受管数据库备份",
detail=f"当前数据库类型为 {db_type},备份目录中没有正式备份文件。",
recommendation="可执行 `moviepilot database backup` 创建一次可校验备份。",
affects_report_status=False,
context={"db_type": db_type, "backups": []},
)
return
if not matching:
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Warn,
status=DoctorFindingStatus.Degraded,
title="没有匹配当前数据库类型的备份",
detail=f"当前数据库类型为 {db_type},仅找到其他类型备份:{', '.join(mismatched)}",
recommendation="确认数据库类型配置,或执行 `moviepilot database backup` 创建当前类型备份。",
affects_report_status=False,
context={"db_type": db_type, "backups": [], "mismatched": mismatched},
)
return
backups = []
for path in matching:
try:
size = path.stat().st_size
verification = verify_database_backup(path, db_type=db_type)
valid = verification.valid
method = verification.method
detail = verification.detail
except (OSError, RuntimeError, ValueError) as error:
size = None
valid = False
method = "unavailable"
detail = str(error)
backups.append(
{
"name": path.name,
"db_type": db_type,
"size": size,
"valid": valid,
"method": method,
"detail": detail,
}
)
valid_backups = [backup for backup in backups if backup["valid"]]
context = {
"db_type": db_type,
"backups": backups,
"mismatched": mismatched,
}
if valid_backups:
newest = valid_backups[0]
command = f"moviepilot database restore {newest['name']} --confirm"
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Info,
status=DoctorFindingStatus.Ok,
title="存在可还原的数据库备份",
detail=f"已校验 {len(backups)}{db_type} 备份,其中 {len(valid_backups)} 个可用。",
recommendation=f"需要恢复时先停止 MoviePilot,再执行 `{command}`。",
context={**context, "restore_command": command},
)
return
failures = "; ".join(
f"{backup['name']}: {backup['detail'] or backup['method']}"
for backup in backups
)
runner.add(
finding_id="database.backup_recovery",
severity=DoctorSeverity.Error,
status=DoctorFindingStatus.Failed,
title="数据库备份均未通过校验",
detail=(
f"已检查 {len(backups)}{db_type} 备份,没有可直接还原的文件。"
f"校验结果:{failures}"
),
recommendation="根据校验详情处理备份文件或 PostgreSQL client,再重新运行 doctor。",
affects_report_status=False,
context=context,
)
def _check_frontend_assets(runner: DoctorRunnerProtocol) -> None: def _check_frontend_assets(runner: DoctorRunnerProtocol) -> None:
+5 -2
View File
@@ -13,8 +13,8 @@
"runtime_to_db": [], "runtime_to_db": [],
"workflow_to_db": [] "workflow_to_db": []
}, },
"edge_count": 6379, "edge_count": 6382,
"edge_sha256": "1c619d72157004590838a85c497330b8ca462d3bc1eb490fd5e7bac97e6190b8", "edge_sha256": "d94f2e60efcc1697fe8dfa0ad23cbc99c38fd4873caf889ae1c44769c69fdcc3",
"edges": [ "edges": [
"app -> app.runtime", "app -> app.runtime",
"app -> app.runtime.compat", "app -> app.runtime.compat",
@@ -3659,6 +3659,9 @@
"app.db.session -> app.runtime.observability", "app.db.session -> app.runtime.observability",
"app.doctor.checks -> app.adapters", "app.doctor.checks -> app.adapters",
"app.doctor.checks -> app.adapters.system", "app.doctor.checks -> app.adapters.system",
"app.doctor.checks -> app.adapters.system.backup",
"app.doctor.checks -> app.adapters.system.backup.database",
"app.doctor.checks -> app.adapters.system.backup.files",
"app.doctor.checks -> app.adapters.system.host", "app.doctor.checks -> app.adapters.system.host",
"app.doctor.checks -> app.doctor", "app.doctor.checks -> app.doctor",
"app.doctor.checks -> app.doctor.models", "app.doctor.checks -> app.doctor.models",
+65
View File
@@ -6,12 +6,14 @@ from pathlib import Path
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import Mock from unittest.mock import Mock
import pytest
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.engine import make_url from sqlalchemy.engine import make_url
from app.adapters.system.backup.database import ( from app.adapters.system.backup.database import (
PostgreSQLBackupBackend, PostgreSQLBackupBackend,
SQLiteBackupBackend, SQLiteBackupBackend,
verify_database_backup,
) )
@@ -34,6 +36,15 @@ def test_sqlite_backup_includes_committed_wal_data(tmp_path: Path) -> None:
engine.dispose() engine.dispose()
def test_sqlite_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""Doctor 等离线入口不应为校验备份而构造活动数据库引擎。"""
artifact = tmp_path / "backup.db"
with sqlite3.connect(artifact) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
assert verify_database_backup(artifact, db_type="sqlite").valid is True
def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None: def test_sqlite_restore_replaces_database_and_removes_old_wal_files(tmp_path: Path) -> None:
source = tmp_path / "user.db" source = tmp_path / "user.db"
backup = tmp_path / "backup.db" backup = tmp_path / "backup.db"
@@ -66,6 +77,11 @@ class _Runner:
return subprocess.CompletedProcess(command, 0, stdout, "") return subprocess.CompletedProcess(command, 0, stdout, "")
class _FailedRunner:
def __call__(self, command, **_kwargs):
return subprocess.CompletedProcess(command, 1, "", "invalid archive")
def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend: def _postgres_backend(runner: _Runner) -> PostgreSQLBackupBackend:
engine = SimpleNamespace( engine = SimpleNamespace(
url=make_url( url=make_url(
@@ -114,6 +130,55 @@ def test_postgresql_verify_and_restore_use_pg_restore(tmp_path: Path) -> None:
assert restore_kwargs["env"]["PGPASSWORD"] == "secret" assert restore_kwargs["env"]["PGPASSWORD"] == "secret"
def test_postgresql_backup_can_be_verified_without_active_engine(tmp_path: Path) -> None:
"""PostgreSQL 归档校验只依赖 pg_restore,不连接活动数据库。"""
runner = _Runner()
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=runner,
tool_resolver=lambda executable: executable,
)
assert result.valid is True
command, kwargs = runner.calls[0]
assert command == ["pg_restore", "--list", str(artifact)]
assert "PGPASSWORD" not in kwargs["env"]
assert "PGSSLMODE" not in kwargs["env"]
def test_postgresql_offline_verify_rejects_invalid_archive(tmp_path: Path) -> None:
"""pg_restore 无法读取归档目录时备份必须判定为无效。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"invalid")
result = verify_database_backup(
artifact,
db_type="postgresql",
runner=_FailedRunner(),
tool_resolver=lambda executable: executable,
)
assert result.valid is False
assert result.detail == "pg_restore 退出码 1"
def test_postgresql_offline_verify_reports_missing_client(tmp_path: Path) -> None:
"""缺少 pg_restore 时离线校验应给出可执行的安装提示。"""
artifact = tmp_path / "backup.dump"
artifact.write_bytes(b"PGDMP")
with pytest.raises(RuntimeError, match="PostgreSQL client"):
verify_database_backup(
artifact,
db_type="postgresql",
tool_resolver=lambda _executable: None,
)
def test_postgresql_source_install_reports_missing_native_client() -> None: def test_postgresql_source_install_reports_missing_native_client() -> None:
runner = _Runner() runner = _Runner()
engine = SimpleNamespace( engine = SimpleNamespace(
+95 -2
View File
@@ -1,13 +1,14 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime, timedelta from datetime import datetime, timedelta
import sqlite3
from types import SimpleNamespace from types import SimpleNamespace
from app.runtime.config import settings from app.runtime.config import settings
from app.doctor import checks, run_doctor from app.doctor import checks
from app.doctor.formatters import format_json_report, format_text_report from app.doctor.formatters import format_json_report, format_text_report
from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorSeverity from app.doctor.models import DoctorFinding, DoctorFindingStatus, DoctorSeverity
from app.doctor.runner import DoctorRunner from app.doctor.runner import DoctorRunner, run_doctor
def _current_log_timestamp() -> str: def _current_log_timestamp() -> str:
@@ -33,6 +34,98 @@ def test_doctor_report_has_stable_json_shape(tmp_path, monkeypatch):
assert any(item["id"] == "runtime.paths" for item in payload["findings"]) assert any(item["id"] == "runtime.paths" for item in payload["findings"])
def test_doctor_reports_valid_backup_when_sqlite_database_is_corrupt(
tmp_path,
monkeypatch,
):
"""主数据库损坏时 Doctor 仍应离线校验备份并给出还原命令。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
(tmp_path / "user.db").write_bytes(b"not a sqlite database")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
with sqlite3.connect(backup) as connection:
connection.execute("CREATE TABLE entries (value TEXT NOT NULL)")
(backup_dir / "sqlite_20260822_040000.db").write_bytes(b"invalid newer backup")
runner = DoctorRunner()
checks._check_database(runner)
assert runner.report.find("database.sqlite_open_failed") is not None
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Ok
assert finding.context["backups"][0]["valid"] is False
assert finding.context["backups"][1]["valid"] is True
assert finding.context["restore_command"] == (
"moviepilot database restore sqlite_20260822_030000.db --confirm"
)
assert finding.context["restore_command"] in finding.recommendation
def test_doctor_distinguishes_missing_and_mismatched_backups(tmp_path, monkeypatch):
"""无备份与仅存在其他数据库类型备份应生成不同诊断结论。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
runner = DoctorRunner()
checks._check_database_backups(runner)
missing = runner.report.find("database.backup_recovery")
assert missing is not None
assert missing.status == DoctorFindingStatus.Skipped
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
runner = DoctorRunner()
checks._check_database_backups(runner)
mismatched = runner.report.find("database.backup_recovery")
assert mismatched is not None
assert mismatched.status == DoctorFindingStatus.Degraded
assert mismatched.context["mismatched"] == ["postgresql_20260822_030000.dump"]
def test_doctor_reports_invalid_backup_without_modifying_it(tmp_path, monkeypatch):
"""Doctor --fix 也只校验备份,不覆盖或删除无效文件。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
backup = backup_dir / "sqlite_20260822_030000.db"
original = b"invalid sqlite backup"
backup.write_bytes(original)
runner = DoctorRunner(fix=True)
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert finding.affects_report_status is False
assert finding.context["backups"][0]["valid"] is False
assert backup.read_bytes() == original
assert runner.report.status.value == "healthy"
def test_doctor_exposes_missing_pg_restore_in_text_finding(tmp_path, monkeypatch):
"""PostgreSQL 离线校验工具缺失时应直接告诉用户如何补齐。"""
def missing_pg_restore(*_args, **_kwargs):
raise RuntimeError("未找到 pg_restore,请安装 PostgreSQL client 并加入 PATH")
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))
monkeypatch.setattr(settings, "DB_TYPE", "postgresql")
backup_dir = settings.DATABASE_BACKUP_PATH
backup_dir.mkdir(parents=True)
(backup_dir / "postgresql_20260822_030000.dump").write_bytes(b"PGDMP")
monkeypatch.setattr(checks, "verify_database_backup", missing_pg_restore)
runner = DoctorRunner()
checks._check_database_backups(runner)
finding = runner.report.find("database.backup_recovery")
assert finding is not None
assert finding.status == DoctorFindingStatus.Failed
assert "未找到 pg_restore" in finding.detail
assert "PostgreSQL client" in format_text_report(runner.report)
def test_doctor_formatters_include_status_and_finding(tmp_path, monkeypatch): def test_doctor_formatters_include_status_and_finding(tmp_path, monkeypatch):
"""doctor 文本和 JSON 格式化应展示状态与诊断项。""" """doctor 文本和 JSON 格式化应展示状态与诊断项。"""
monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path)) monkeypatch.setattr(settings, "CONFIG_DIR", str(tmp_path))