From bbd5c12dbc859d839c3291e52f2c7f0569d6b6c8 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:28:22 +0800 Subject: [PATCH] perf(docker): bound automatic update probes (#6357) --- docker/update.sh | 117 ++++++++----- tests/test_docker_bootstrap.py | 295 +++++++++++++++++++++++++++++++++ 2 files changed, 375 insertions(+), 37 deletions(-) diff --git a/docker/update.sh b/docker/update.sh index 0d2b09212..641765c28 100644 --- a/docker/update.sh +++ b/docker/update.sh @@ -91,20 +91,17 @@ function install_backend_and_download_resources() { if [ -f "${TMP_PATH}/App/requirements.in" ]; then if ! cmp -s /app/requirements.in "${TMP_PATH}/App/requirements.in"; then INFO "检测到依赖变化，正在更新虚拟环境..." - # 备份当前requirements.txt - cp /app/requirements.txt /tmp/requirements.txt.backup - # 复制新的requirements.in - cp "${TMP_PATH}/App/requirements.in" /app/requirements.in - # 重新编译依赖 - if ! env "${PIP_ENV[@]}" ${VENV_PATH}/bin/pip-compile /app/requirements.in -o /app/requirements.txt; then - ERROR "依赖编译失败，恢复原依赖" - cp /tmp/requirements.txt.backup /app/requirements.txt + configure_pip_route + INFO "PIP：${PIP_LOG}" + local compiled_requirements="${TMP_PATH}/requirements.txt" + if ! env "${PIP_ENV[@]}" ${VENV_PATH}/bin/pip-compile \ + "${TMP_PATH}/App/requirements.in" -o "${compiled_requirements}"; then + ERROR "依赖编译失败，当前程序依赖未变更" return 1 fi - # 安装新依赖 - if ! env "${PIP_ENV[@]}" ${VENV_PATH}/bin/pip install ${PIP_OPTIONS} -r /app/requirements.txt; then - ERROR "依赖安装失败，恢复原依赖" - cp /tmp/requirements.txt.backup /app/requirements.txt + if ! env "${PIP_ENV[@]}" ${VENV_PATH}/bin/pip install ${PIP_OPTIONS} \ + -r "${compiled_requirements}"; then + ERROR "依赖安装失败，当前程序依赖清单未变更" return 1 fi INFO "依赖更新成功" @@ -224,16 +221,54 @@ function install_backend_and_download_resources() { return 0 } +function probe_pip_package() { + local probe_env=( + "UV_NO_CACHE=1" + "PIP_NO_CACHE_DIR=1" + "UV_HTTP_TIMEOUT=5" + "PIP_DEFAULT_TIMEOUT=5" + "UV_HTTP_RETRIES=0" + "PIP_RETRIES=0" + ) + local package_index="${1:-}" + local use_proxy="${2:-false}" + local probe_dir + local -a probe_args=(install) + + if [[ "${use_proxy}" = "true" ]]; then + probe_env+=( + "HTTP_PROXY=${PROXY_HOST}" + "HTTPS_PROXY=${PROXY_HOST}" + "http_proxy=${PROXY_HOST}" + "https_proxy=${PROXY_HOST}" + ) + fi + probe_dir=$(mktemp -d) || return 1 + # 包源探针必须使用独立目标目录，避免修改主程序与插件共享的虚拟环境。 + probe_args+=(--target "${probe_dir}" --no-deps) + if [[ -n "${package_index}" ]]; then + probe_args+=(-i "${package_index}") + fi + probe_args+=(pip-hello-world) + + ( + trap 'rm -rf "${probe_dir}"' EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + timeout --kill-after=2s 10s env "${probe_env[@]}" \ + "${VENV_PATH}/bin/pip" "${probe_args[@]}" > /dev/null 2>&1 + ) +} + function test_connectivity_pip() { - ${VENV_PATH}/bin/pip uninstall -y pip-hello-world > /dev/null 2>&1 case "$1" in 0) if [[ -n "${PIP_PROXY}" ]]; then if [[ -n "${PROXY_HOST}" ]]; then - HTTP_PROXY="${PROXY_HOST}" HTTPS_PROXY="${PROXY_HOST}" http_proxy="${PROXY_HOST}" https_proxy="${PROXY_HOST}" \ - ${VENV_PATH}/bin/pip install -i ${PIP_PROXY} pip-hello-world > /dev/null 2>&1 + probe_pip_package "${PIP_PROXY}" true else - ${VENV_PATH}/bin/pip install -i ${PIP_PROXY} pip-hello-world > /dev/null 2>&1 + probe_pip_package "${PIP_PROXY}" false fi if [[ $? -eq 0 ]]; then PIP_OPTIONS="-i ${PIP_PROXY}" @@ -246,8 +281,7 @@ function test_connectivity_pip() { ;; 1) if [[ -n "${PROXY_HOST}" ]]; then - if HTTP_PROXY="${PROXY_HOST}" HTTPS_PROXY="${PROXY_HOST}" http_proxy="${PROXY_HOST}" https_proxy="${PROXY_HOST}" \ - ${VENV_PATH}/bin/pip install pip-hello-world > /dev/null 2>&1; then + if probe_pip_package "" true; then PIP_OPTIONS="" PIP_LOG="全局代理模式" set_package_proxy_env @@ -270,7 +304,7 @@ function test_connectivity_github() { case "$1" in 0) if [[ -n "${GITHUB_PROXY}" ]]; then - if curl -sL "${GITHUB_PROXY}https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md" > /dev/null 2>&1; then + if curl -sL --connect-timeout 5 --max-time 10 "${GITHUB_PROXY}https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md" > /dev/null 2>&1; then GITHUB_LOG="镜像代理模式" return 0 fi @@ -279,7 +313,7 @@ function test_connectivity_github() { ;; 1) if [[ -n "${PROXY_HOST}" ]]; then - if curl -sL -x ${PROXY_HOST} https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md > /dev/null 2>&1; then + if curl -sL --connect-timeout 5 --max-time 10 -x ${PROXY_HOST} https://raw.githubusercontent.com/jxxghp/MoviePilot/main/README.md > /dev/null 2>&1; then CURL_OPTIONS="-sL -x ${PROXY_HOST}" GITHUB_LOG="全局代理模式" return 0 @@ -295,6 +329,30 @@ function test_connectivity_github() { esac } +function configure_pip_route() { + local retries=0 + while true; do + if test_connectivity_pip "${retries}"; then + return 0 + fi + retries=$((retries + 1)) + done +} + +function fetch_latest_v3_release() { + local response + local releases + local latest_release + + response=$(curl ${CURL_OPTIONS} --compressed --fail --connect-timeout 5 --max-time 15 \ + "https://api.github.com/repos/jxxghp/MoviePilot/releases" \ + ${CURL_HEADERS}) || return 1 + releases=$(printf '%s\n' "${response}" | jq -r '.[].tag_name') || return 1 + latest_release=$(printf '%s\n' "${releases}" | grep "^v3\." | sort -V | tail -n 1) + [[ -n "${latest_release}" ]] || return 1 + printf '%s\n' "${latest_release}" +} + # 版本号比较 function compare_versions() { local v1="$1" @@ -379,17 +437,6 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}" fi mkdir -p /tmp/mp_update_path fi - # 优先级：镜像站 > 全局 > 不代理 - # pip - retries=0 - while true; do - if test_connectivity_pip ${retries}; then - break - else - retries=$((retries + 1)) - fi - done - # Github retries=0 while true; do if test_connectivity_github ${retries}; then @@ -398,7 +445,7 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}" retries=$((retries + 1)) fi done - INFO "PIP：${PIP_LOG}，Github：${GITHUB_LOG}" + INFO "Github：${GITHUB_LOG}" if [ -n "${GITHUB_TOKEN}" ]; then CURL_HEADERS="--oauth2-bearer ${GITHUB_TOKEN}" else @@ -415,13 +462,9 @@ if [[ "${MOVIEPILOT_AUTO_UPDATE}" = "true" ]] || [[ "${MOVIEPILOT_AUTO_UPDATE}" if [[ "${old_version}" == *APP_VERSION* ]]; then current_version=$(echo "${old_version}" | sed -rn "s/APP_VERSION\s*=\s*['\"](.*)['\"]/\1/gp") INFO "当前版本号：${current_version}" - # 获取所有发布的版本列表，并筛选出以v3开头的版本号 - releases=$(curl ${CURL_OPTIONS} "https://api.github.com/repos/jxxghp/MoviePilot/releases" ${CURL_HEADERS} | jq -r '.[].tag_name' | grep "^v3\.") - if [ -z "$releases" ]; then + if ! latest_v3=$(fetch_latest_v3_release); then WARN "未找到任何v3后端版本，继续启动..." else - # 找到最新的v3版本 - latest_v3=$(echo "$releases" | sort -V | tail -n 1) INFO "最新的v3后端版本号：${latest_v3}" # 使用版本号比较函数进行比较，并下载最新版本 compare_versions "${current_version}" "${latest_v3}" diff --git a/tests/test_docker_bootstrap.py b/tests/test_docker_bootstrap.py index ed0da60a0..a309bcafa 100644 --- a/tests/test_docker_bootstrap.py +++ b/tests/test_docker_bootstrap.py @@ -602,6 +602,301 @@ def test_updater_exposes_explicit_result( assert result.stdout == f"{expected}\n" +def test_release_noop_preserves_prerelease_selection_without_probing_package_index( + tmp_path: Path, +) -> None: + pip_probe = tmp_path / "pip-probe" + curl_log = tmp_path / "curl.log" + comparison_log = tmp_path / "comparison.log" + script = textwrap.dedent( + f"""\ + CONFIG_DIR="$1" + MOVIEPILOT_AUTO_UPDATE=release + PIP_PROXY= PROXY_HOST= GITHUB_PROXY= GITHUB_TOKEN= + PIP_PROBE="$2" + CURL_LOG="$3" + COMPARISON_LOG="$4" + source {UPDATER!s} + INFO() {{ :; }} + WARN() {{ :; }} + ERROR() {{ :; }} + test_connectivity_pip() {{ touch "${{PIP_PROBE}}"; return 0; }} + test_connectivity_github() {{ CURL_OPTIONS=-sL; GITHUB_LOG=test; return 0; }} + compare_versions() {{ printf '%s|%s\n' "$1" "$2" > "${{COMPARISON_LOG}}"; return 1; }} + grep() {{ + if [[ "$*" == *"/app/version.py"* ]]; then + printf '%s\n' "APP_VERSION = 'v3.0.0'" + return 0 + fi + command grep "$@" + }} + sed() {{ + if [[ "$*" == *"APP_VERSION"* ]]; then + printf '%s\n' 'v3.0.0' + return 0 + fi + command sed "$@" + }} + curl() {{ + printf '%s\n' "$*" >> "${{CURL_LOG}}" + printf '%s\n' '[{{"tag_name":"v2.15.6"}},{{"tag_name":"v3.1.0-beta"}},{{"tag_name":"v3.1.0-rc"}},{{"tag_name":"v3.0.0"}}]' + }} + run_moviepilot_update + printf '%s\n' "${{MOVIEPILOT_UPDATE_RESULT}}" + """ + ) + + result = subprocess.run( + [ + "bash", + "-c", + script, + "release-noop-test", + str(tmp_path / "config"), + str(pip_probe), + str(curl_log), + str(comparison_log), + ], + text=True, + capture_output=True, + check=True, + ) + + assert result.stdout == "noop\n" + assert not pip_probe.exists() + curl_args = curl_log.read_text(encoding="utf-8") + assert "/releases" in curl_args + assert "/releases/latest" not in curl_args + assert "--compressed" in curl_args + assert "--fail" in curl_args + assert "--connect-timeout 5" in curl_args + assert "--max-time 15" in curl_args + assert comparison_log.read_text(encoding="utf-8") == "v3.0.0|v3.1.0-rc\n" + + +@pytest.mark.parametrize( + ("dependencies_changed", "expected_route_calls", "expected_install_calls"), + ((False, 0, 0), (True, 1, 1)), +) +def test_package_route_is_only_configured_for_changed_dependencies( + tmp_path: Path, + dependencies_changed: bool, + expected_route_calls: int, + expected_install_calls: int, +) -> None: + venv_bin = tmp_path / "venv" / "bin" + venv_bin.mkdir(parents=True) + pip_log = tmp_path / "pip.log" + route_log = tmp_path / "route.log" + for executable in ("pip", "pip-compile"): + path = venv_bin / executable + path.write_text( + "#!/bin/bash\nprintf '%s\\n' \"$*\" >> \"${PIP_TEST_LOG}\"\n", + encoding="utf-8", + ) + path.chmod(0o755) + update_tree = tmp_path / "update" / "App" + update_tree.mkdir(parents=True) + (update_tree / "requirements.in").write_text("new-package==1\n", encoding="utf-8") + (update_tree / "version.py").write_text("FRONTEND_VERSION = ''\n", encoding="utf-8") + script = textwrap.dedent( + f"""\ + CONFIG_DIR="$1" + VENV_PATH="$2" + TMP_PATH="$3" + ROUTE_LOG="$4" + DEPENDENCIES_CHANGED="$5" + PIP_PROXY= PROXY_HOST= + source {UPDATER!s} + INFO() {{ :; }} + WARN() {{ :; }} + ERROR() {{ :; }} + download_and_unzip() {{ return 0; }} + cmp() {{ [ "${{DEPENDENCIES_CHANGED}}" = false ]; }} + cp() {{ return 0; }} + configure_pip_route() {{ printf 'route\n' >> "${{ROUTE_LOG}}"; PIP_LOG=test; }} + install_backend_and_download_resources tags/v3.0.1.zip || true + """ + ) + + env = {**os.environ, "PIP_TEST_LOG": str(pip_log)} + result = subprocess.run( + [ + "bash", + "-c", + script, + "package-route-test", + str(tmp_path / "config"), + str(tmp_path / "venv"), + str(tmp_path / "update"), + str(route_log), + str(dependencies_changed).lower(), + ], + text=True, + capture_output=True, + check=True, + env=env, + ) + + assert result.stderr == "" + route_calls = route_log.read_text(encoding="utf-8").splitlines() if route_log.exists() else [] + install_calls = pip_log.read_text(encoding="utf-8").splitlines() if pip_log.exists() else [] + assert len(route_calls) == expected_route_calls + compile_calls = [call for call in install_calls if not call.startswith("install ")] + package_install_calls = [call for call in install_calls if call.startswith("install ")] + assert len(package_install_calls) == expected_install_calls + if dependencies_changed: + assert compile_calls == [ + f"{update_tree / 'requirements.in'} -o {tmp_path / 'update' / 'requirements.txt'}" + ] + assert package_install_calls == [ + f"install -r {tmp_path / 'update' / 'requirements.txt'}" + ] + + +@pytest.mark.parametrize("failure", ("compile", "install", "post_install")) +def test_failed_dependency_update_does_not_overwrite_current_manifests( + tmp_path: Path, + failure: str, +) -> None: + venv_bin = tmp_path / "venv" / "bin" + venv_bin.mkdir(parents=True) + command_log = tmp_path / "commands.log" + copy_log = tmp_path / "copies.log" + for executable in ("pip", "pip-compile"): + path = venv_bin / executable + path.write_text( + "#!/bin/bash\n" + 'printf \'%s|%s\\n\' "$(basename "$0")" "$*" >> "${COMMAND_LOG}"\n' + f'[[ "$(basename "$0")" == "pip-compile" && "{failure}" == "compile" ]] && exit 1\n' + f'[[ "$(basename "$0")" == "pip" && "{failure}" == "install" ]] && exit 1\n' + "exit 0\n", + encoding="utf-8", + ) + path.chmod(0o755) + update_tree = tmp_path / "update" / "App" + update_tree.mkdir(parents=True) + (update_tree / "requirements.in").write_text("new-package==1\n", encoding="utf-8") + (update_tree / "version.py").write_text( + "FRONTEND_VERSION = 'v3.0.0'\n", + encoding="utf-8", + ) + script = textwrap.dedent( + f"""\ + CONFIG_DIR="$1" + VENV_PATH="$2" + TMP_PATH="$3" + COPY_LOG="$4" + PIP_PROXY= PROXY_HOST= + FAILURE={failure} + source {UPDATER!s} + INFO() {{ :; }} + WARN() {{ :; }} + ERROR() {{ :; }} + download_and_unzip() {{ + if [[ "${{FAILURE}}" = post_install ]] && [[ "$2" = dist ]]; then + return 1 + fi + return 0 + }} + cmp() {{ return 1; }} + cp() {{ printf '%s\n' "$*" >> "${{COPY_LOG}}"; }} + configure_pip_route() {{ PIP_LOG=test; }} + install_backend_and_download_resources tags/v3.0.1.zip || true + if [[ "${{FAILURE}}" = post_install ]]; then + install_backend_and_download_resources tags/v3.0.1.zip || true + fi + """ + ) + + subprocess.run( + [ + "bash", + "-c", + script, + "dependency-failure-test", + str(tmp_path / "config"), + str(tmp_path / "venv"), + str(tmp_path / "update"), + str(copy_log), + ], + text=True, + capture_output=True, + check=True, + env={**os.environ, "COMMAND_LOG": str(command_log)}, + ) + + assert not copy_log.exists() + commands = command_log.read_text(encoding="utf-8").splitlines() + assert commands[0] == ( + f"pip-compile|{update_tree / 'requirements.in'} " + f"-o {tmp_path / 'update' / 'requirements.txt'}" + ) + assert all("/app/requirements" not in command for command in commands) + if failure == "compile": + assert len(commands) == 1 + elif failure == "install": + assert commands[1] == f"pip|install -r {tmp_path / 'update' / 'requirements.txt'}" + else: + assert commands == [ + ( + f"pip-compile|{update_tree / 'requirements.in'} " + f"-o {tmp_path / 'update' / 'requirements.txt'}" + ), + f"pip|install -r {tmp_path / 'update' / 'requirements.txt'}", + ( + f"pip-compile|{update_tree / 'requirements.in'} " + f"-o {tmp_path / 'update' / 'requirements.txt'}" + ), + f"pip|install -r {tmp_path / 'update' / 'requirements.txt'}", + ] + + +def test_package_index_probe_is_cacheless_and_bounded(tmp_path: Path) -> None: + timeout_log = tmp_path / "timeout.log" + script = textwrap.dedent( + f"""\ + CONFIG_DIR="$1" + VENV_PATH="$2" + TIMEOUT_LOG="$3" + PIP_PROXY=https://packages.example/simple + PROXY_HOST= + source {UPDATER!s} + timeout() {{ printf '%s\n' "$*" > "${{TIMEOUT_LOG}}"; return 124; }} + test_connectivity_pip 0 || true + """ + ) + + subprocess.run( + [ + "bash", + "-c", + script, + "package-probe-test", + str(tmp_path / "config"), + str(tmp_path / "venv"), + str(timeout_log), + ], + text=True, + capture_output=True, + check=True, + ) + + command = timeout_log.read_text(encoding="utf-8") + assert command.startswith("--kill-after=2s 10s env ") + assert "UV_NO_CACHE=1" in command + assert "PIP_NO_CACHE_DIR=1" in command + assert "UV_HTTP_TIMEOUT=5" in command + assert "PIP_DEFAULT_TIMEOUT=5" in command + assert "UV_HTTP_RETRIES=0" in command + assert "PIP_RETRIES=0" in command + assert "pip install --target " in command + assert " --no-deps -i https://packages.example/simple pip-hello-world" in command + assert "uninstall" not in command + probe_dir = Path(command.split("--target ", 1)[1].split(" ", 1)[0]) + assert not probe_dir.exists() + + @pytest.mark.parametrize( ("result", "current", "next_generation", "guard", "expected"), (