fix: deduplicate graceful restart fallback

This commit is contained in:
jxxghp
2026-08-24 08:29:07 +08:00
parent e2814a129f
commit 01d29d952f
3 changed files with 191 additions and 101 deletions
+143 -90
View File
@@ -6,9 +6,9 @@ import struct
import subprocess
import sys
import tempfile
import threading
import time
from pathlib import Path
from unittest import TestCase
from unittest.mock import MagicMock, call, patch
import psutil
@@ -19,108 +19,161 @@ from app.runtime.config import ConfigModel, settings
from app.adapters.system.host import SystemUtils
class SystemUtilsTest(TestCase):
def test_get_config_path_uses_repository_config_for_source_runtime():
"""源码运行时应从项目根目录读取配置,不能随适配器目录层级偏移。"""
expected = Path(__file__).resolve().parents[1] / "config"
def test_get_config_path_uses_repository_config_for_source_runtime(self):
"""源码运行时应从项目根目录读取配置,不能随适配器目录层级偏移。"""
expected = Path(__file__).resolve().parents[1] / "config"
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=False), \
patch.object(SystemUtils, "is_frozen", return_value=False):
assert SystemUtils.get_config_path() == expected
assert SystemUtils.get_env_path() == expected / "app.env"
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=False), \
patch.object(SystemUtils, "is_frozen", return_value=False):
self.assertEqual(SystemUtils.get_config_path(), expected)
self.assertEqual(SystemUtils.get_env_path(), expected / "app.env")
def test_get_config_path_preserves_explicit_config_dir(self):
"""显式配置目录始终优先于运行环境推导。"""
explicit_path = Path("/custom/moviepilot-config")
def test_get_config_path_preserves_explicit_config_dir():
"""显式配置目录始终优先于运行环境推导。"""
explicit_path = Path("/custom/moviepilot-config")
with patch.dict(os.environ, {"CONFIG_DIR": "/ignored"}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=True):
self.assertEqual(
SystemUtils.get_config_path(str(explicit_path)),
explicit_path,
)
with patch.dict(os.environ, {"CONFIG_DIR": "/ignored"}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=True):
assert SystemUtils.get_config_path(str(explicit_path)) == explicit_path
def test_get_config_path_preserves_runtime_specific_defaults(self):
"""容器和冻结程序继续使用各自稳定的配置目录。"""
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=True):
self.assertEqual(SystemUtils.get_config_path(), Path("/config"))
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=False), \
patch.object(SystemUtils, "is_frozen", return_value=True), \
patch("app.adapters.system.host.sys.executable", "/opt/moviepilot/moviepilot"):
self.assertEqual(
SystemUtils.get_config_path(),
Path("/opt/moviepilot/config"),
)
def test_get_config_path_preserves_runtime_specific_defaults():
"""容器和冻结程序继续使用各自稳定的配置目录。"""
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=True):
assert SystemUtils.get_config_path() == Path("/config")
def test_execute_with_subprocess_keeps_stdout_when_command_fails(self):
"""
命令失败时如果原因只写入 stdout,也需要回传给调用方用于错误提示。
"""
error = subprocess.CalledProcessError(
returncode=1,
cmd=["pip", "check"],
output="demo requires pkg>=2, but you have pkg 1\n",
stderr="",
with patch.dict(os.environ, {}, clear=True), \
patch.object(SystemUtils, "is_docker", return_value=False), \
patch.object(SystemUtils, "is_frozen", return_value=True), \
patch("app.adapters.system.host.sys.executable", "/opt/moviepilot/moviepilot"):
assert SystemUtils.get_config_path() == Path("/opt/moviepilot/config")
def test_execute_with_subprocess_keeps_stdout_when_command_fails():
"""命令失败时如果原因只写入 stdout,也需要回传给调用方用于错误提示。"""
error = subprocess.CalledProcessError(
returncode=1,
cmd=["pip", "check"],
output="demo requires pkg>=2, but you have pkg 1\n",
stderr="",
)
with patch("app.adapters.system.host.subprocess.run", side_effect=error):
success, message = SystemUtils.execute_with_subprocess(["pip", "check"])
assert not success
assert "返回码:1" in message
assert "标准输出:demo requires pkg>=2, but you have pkg 1" in message
def test_execute_with_subprocess_reports_empty_failure_output():
"""命令失败且没有输出时应给出明确占位信息,避免错误原因看起来被截断。"""
error = subprocess.CalledProcessError(
returncode=2,
cmd=["pip", "check"],
output="",
stderr="",
)
with patch("app.adapters.system.host.subprocess.run", side_effect=error):
success, message = SystemUtils.execute_with_subprocess(["pip", "check"])
assert not success
assert "返回码:2" in message
assert "无标准输出或错误输出" in message
def test_docker_restart_policy_marks_intent_before_sigterm():
"""Docker 优雅重启前应写入意图标记,避免 entrypoint 误进入 doctor 保活。"""
with tempfile.TemporaryDirectory() as temp_dir:
original_config_dir = settings.CONFIG_DIR
original_intent_file = SystemHelper._SystemHelper__docker_restart_intent_file
settings.CONFIG_DIR = temp_dir
SystemHelper._SystemHelper__docker_restart_intent_file = (
settings.TEMP_PATH / "moviepilot.intentional_restart"
)
try:
with patch("app.runtime.state.is_docker", return_value=True), \
patch.object(SystemHelper, "_check_restart_policy", return_value=True), \
patch.object(SystemHelper, "_start_graceful_shutdown_monitor"), \
patch("app.runtime.state.os.kill") as kill_mock:
ret, msg = SystemHelper.restart()
with patch("app.adapters.system.host.subprocess.run", side_effect=error):
success, message = SystemUtils.execute_with_subprocess(["pip", "check"])
self.assertFalse(success)
self.assertIn("返回码:1", message)
self.assertIn("标准输出:demo requires pkg>=2, but you have pkg 1", message)
def test_execute_with_subprocess_reports_empty_failure_output(self):
"""
命令失败且没有任何输出时,给出明确占位信息,避免错误原因看起来被截断。
"""
error = subprocess.CalledProcessError(
returncode=2,
cmd=["pip", "check"],
output="",
stderr="",
)
with patch("app.adapters.system.host.subprocess.run", side_effect=error):
success, message = SystemUtils.execute_with_subprocess(["pip", "check"])
self.assertFalse(success)
self.assertIn("返回码:2", message)
self.assertIn("无标准输出或错误输出", message)
assert ret
assert msg == ""
assert (settings.TEMP_PATH / "moviepilot.intentional_restart").exists()
kill_mock.assert_called_once()
finally:
SystemHelper._SystemHelper__docker_restart_intent_file = original_intent_file
settings.CONFIG_DIR = original_config_dir
class SystemHelperRestartTest(TestCase):
def test_graceful_shutdown_monitor_has_single_owner_and_releases_it(monkeypatch):
"""重复重启请求应共享唯一兜底线程,线程结束后必须释放 owner。"""
sleep_started = threading.Event()
release_sleep = threading.Event()
restart = MagicMock(return_value=(True, ""))
monitor_attr = "_SystemHelper__graceful_shutdown_monitor"
original_monitor = getattr(SystemHelper, monitor_attr)
thread = None
def test_docker_restart_policy_marks_intent_before_sigterm(self):
"""
Docker 内置重启走优雅退出时,应写入意图标记,避免 entrypoint 误进入 doctor 保活。
"""
with tempfile.TemporaryDirectory() as temp_dir:
original_config_dir = settings.CONFIG_DIR
original_intent_file = SystemHelper._SystemHelper__docker_restart_intent_file
settings.CONFIG_DIR = temp_dir
SystemHelper._SystemHelper__docker_restart_intent_file = (
settings.TEMP_PATH / "moviepilot.intentional_restart"
)
try:
with patch("app.runtime.state.is_docker", return_value=True), \
patch.object(SystemHelper, "_check_restart_policy", return_value=True), \
patch.object(SystemHelper, "_start_graceful_shutdown_monitor"), \
patch("app.runtime.state.os.kill") as kill_mock:
ret, msg = SystemHelper.restart()
def wait_for_shutdown(_seconds: float) -> None:
"""用事件屏障模拟 180 秒等待,确保第二次启动发生在首线程存活期间。"""
sleep_started.set()
release_sleep.wait(timeout=1)
self.assertTrue(ret)
self.assertEqual(msg, "")
self.assertTrue((settings.TEMP_PATH / "moviepilot.intentional_restart").exists())
kill_mock.assert_called_once()
finally:
SystemHelper._SystemHelper__docker_restart_intent_file = original_intent_file
settings.CONFIG_DIR = original_config_dir
setattr(SystemHelper, monitor_attr, None)
try:
monkeypatch.setattr("app.runtime.state.time.sleep", wait_for_shutdown)
monkeypatch.setattr(SystemHelper, "_docker_api_restart", restart)
SystemHelper._start_graceful_shutdown_monitor()
assert sleep_started.wait(timeout=1)
thread = getattr(SystemHelper, monitor_attr)
SystemHelper._start_graceful_shutdown_monitor()
assert getattr(SystemHelper, monitor_attr) is thread
release_sleep.set()
thread.join(timeout=1)
assert thread.is_alive() is False
assert getattr(SystemHelper, monitor_attr) is None
restart.assert_called_once_with()
finally:
release_sleep.set()
if thread is not None:
thread.join(timeout=1)
setattr(SystemHelper, monitor_attr, original_monitor)
def test_graceful_shutdown_monitor_releases_owner_when_thread_start_fails(monkeypatch):
"""兜底线程启动失败时必须释放 owner,允许后续请求重试。"""
monitor_attr = "_SystemHelper__graceful_shutdown_monitor"
original_monitor = getattr(SystemHelper, monitor_attr)
class FailingThread:
"""模拟在登记 owner 后启动失败的线程对象。"""
def __init__(self, **_kwargs):
"""接收真实 Thread 构造参数,但不创建系统线程。"""
def start(self):
"""模拟底层线程资源不足导致的启动失败。"""
raise RuntimeError("thread start failed")
setattr(SystemHelper, monitor_attr, None)
try:
monkeypatch.setattr("app.runtime.state.threading.Thread", FailingThread)
with pytest.raises(RuntimeError, match="thread start failed"):
SystemHelper._start_graceful_shutdown_monitor()
assert getattr(SystemHelper, monitor_attr) is None
finally:
setattr(SystemHelper, monitor_attr, original_monitor)
def test_execute_with_subprocess_passes_env_to_subprocess():