feat(deps): add uv-backed package installer (#5987)

* feat(deps): add uv-backed package installer * feat(deps): support package cache root
2026-06-28 19:21:47 +08:00 · 2026-06-23 13:36:15 +08:00
parent 126279c63b
commit 0c53fb86fd
21 changed files with 1654 additions and 62 deletions
--- a/scripts/dev/simulate_docker_package_env.sh
+++ b/scripts/dev/simulate_docker_package_env.sh
@@ -0,0 +1,196 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+TMP_DIR="$(mktemp -d)"
+trap 'rm -rf "${TMP_DIR}"' EXIT
+
+mkdir -p "${TMP_DIR}/venv/bin" "${TMP_DIR}/config"
+
+cat > "${TMP_DIR}/venv/bin/pip" <<'SH'
+#!/usr/bin/env bash
+printf 'argv=%s\n' "$*" >> "${MP_FAKE_PIP_LOG}"
+printf 'HTTP_PROXY=%s\n' "${HTTP_PROXY:-}" >> "${MP_FAKE_PIP_LOG}"
+printf 'HTTPS_PROXY=%s\n' "${HTTPS_PROXY:-}" >> "${MP_FAKE_PIP_LOG}"
+printf 'PACKAGE_CACHE_ROOT=%s\n' "${PACKAGE_CACHE_ROOT:-}" >> "${MP_FAKE_PIP_LOG}"
+printf 'PIP_CACHE_DIR=%s\n' "${PIP_CACHE_DIR:-}" >> "${MP_FAKE_PIP_LOG}"
+printf 'UV_CACHE_DIR=%s\n' "${UV_CACHE_DIR:-}" >> "${MP_FAKE_PIP_LOG}"
+if [ "${MP_FAKE_PIP_FAIL:-}" = "1" ]; then
+  exit 1
+fi
+exit 0
+SH
+chmod +x "${TMP_DIR}/venv/bin/pip"
+
+assert_contains() {
+  local needle="$1"
+  local file="$2"
+  if ! grep -Fq -- "$needle" "$file"; then
+    echo "missing expected text: $needle" >&2
+    cat "$file" >&2
+    exit 1
+  fi
+}
+
+assert_not_contains() {
+  local needle="$1"
+  local file="$2"
+  if grep -Fq -- "$needle" "$file"; then
+    echo "unexpected text: $needle" >&2
+    cat "$file" >&2
+    exit 1
+  fi
+}
+
+UPDATE_FUNCS="${TMP_DIR}/update-functions.sh"
+awk '
+  BEGIN {capture=1}
+  /^if \[\[ "\$\{MOVIEPILOT_AUTO_UPDATE\}"/ {capture=0}
+  capture {print}
+' "${ROOT}/docker/update.sh" > "${UPDATE_FUNCS}"
+
+MP_FAKE_PIP_LOG="${TMP_DIR}/update.log"
+export MP_FAKE_PIP_LOG
+export VENV_PATH="${TMP_DIR}/venv"
+export CONFIG_DIR="${TMP_DIR}/config"
+export MOVIEPILOT_AUTO_UPDATE=false
+export PIP_PROXY="https://mirror.example/simple"
+export PROXY_HOST="http://proxy.example:7890"
+unset PACKAGE_CACHE_ROOT PIP_CACHE_DIR UV_CACHE_DIR
+source "${UPDATE_FUNCS}" >/dev/null
+
+: > "${MP_FAKE_PIP_LOG}"
+test_connectivity_pip 0
+assert_contains "argv=install -i https://mirror.example/simple pip-hello-world" "${MP_FAKE_PIP_LOG}"
+assert_contains "HTTPS_PROXY=http://proxy.example:7890" "${MP_FAKE_PIP_LOG}"
+assert_contains "PACKAGE_CACHE_ROOT=${TMP_DIR}/config/.cache" "${MP_FAKE_PIP_LOG}"
+assert_contains "PIP_CACHE_DIR=${TMP_DIR}/config/.cache/pip" "${MP_FAKE_PIP_LOG}"
+assert_contains "UV_CACHE_DIR=${TMP_DIR}/config/.cache/uv" "${MP_FAKE_PIP_LOG}"
+if [[ "${PIP_OPTIONS}" != "-i ${PIP_PROXY}" ]]; then
+  echo "mirror branch must preserve index option: ${PIP_OPTIONS}" >&2
+  exit 1
+fi
+if [[ "${PIP_OPTIONS}" == *"--proxy"* ]]; then
+  echo "PIP_OPTIONS must not contain --proxy: ${PIP_OPTIONS}" >&2
+  exit 1
+fi
+assert_not_contains "user:pass" "${MP_FAKE_PIP_LOG}"
+
+: > "${MP_FAKE_PIP_LOG}"
+PIP_PROXY=""
+test_connectivity_pip 1
+assert_contains "argv=install pip-hello-world" "${MP_FAKE_PIP_LOG}"
+assert_contains "HTTPS_PROXY=http://proxy.example:7890" "${MP_FAKE_PIP_LOG}"
+if [[ -n "${PIP_OPTIONS}" ]]; then
+  echo "proxy branch must keep PIP_OPTIONS empty: ${PIP_OPTIONS}" >&2
+  exit 1
+fi
+
+MP_FAKE_PIP_LOG="${TMP_DIR}/update-explicit-cache.log"
+export MP_FAKE_PIP_LOG
+(
+  export VENV_PATH="${TMP_DIR}/venv"
+  export CONFIG_DIR="${TMP_DIR}/config"
+  export MOVIEPILOT_AUTO_UPDATE=false
+  export PACKAGE_CACHE_ROOT="${TMP_DIR}/update-custom-package-cache"
+  export PIP_CACHE_DIR="${TMP_DIR}/explicit-pip-cache"
+  export UV_CACHE_DIR="${TMP_DIR}/explicit-uv-cache"
+  export PIP_PROXY="https://mirror.example/simple"
+  export PROXY_HOST="http://proxy.example:7890"
+  source "${UPDATE_FUNCS}" >/dev/null
+  test_connectivity_pip 0
+)
+
+assert_contains "PACKAGE_CACHE_ROOT=${TMP_DIR}/update-custom-package-cache" "${MP_FAKE_PIP_LOG}"
+assert_contains "PIP_CACHE_DIR=${TMP_DIR}/explicit-pip-cache" "${MP_FAKE_PIP_LOG}"
+assert_contains "UV_CACHE_DIR=${TMP_DIR}/explicit-uv-cache" "${MP_FAKE_PIP_LOG}"
+
+MP_FAKE_PIP_LOG="${TMP_DIR}/update-fallback-no-proxy.log"
+export MP_FAKE_PIP_LOG
+(
+  export VENV_PATH="${TMP_DIR}/venv"
+  export CONFIG_DIR="${TMP_DIR}/config"
+  export MOVIEPILOT_AUTO_UPDATE=false
+  export PIP_PROXY="https://mirror.example/simple"
+  export PROXY_HOST="http://proxy.example:7890"
+  unset PACKAGE_CACHE_ROOT PIP_CACHE_DIR UV_CACHE_DIR HTTP_PROXY HTTPS_PROXY http_proxy https_proxy
+  source "${UPDATE_FUNCS}" >/dev/null
+  MP_FAKE_PIP_FAIL=1 test_connectivity_pip 0 && exit 1
+  if [[ -n "${HTTPS_PROXY:-}" || -n "${https_proxy:-}" ]]; then
+    echo "mirror failure must not leak proxy env" >&2
+    env | grep -E '^(HTTP_PROXY|HTTPS_PROXY|http_proxy|https_proxy)=' >&2 || true
+    exit 1
+  fi
+  test_connectivity_pip 2
+  if [[ "${PIP_LOG}" != "不使用代理" ]]; then
+    echo "fallback branch must report direct mode: ${PIP_LOG}" >&2
+    exit 1
+  fi
+)
+
+ENTRYPOINT_FUNCS="${TMP_DIR}/entrypoint-functions.sh"
+awk '
+  BEGIN {capture=1}
+  /^# 使用env配置/ {capture=0}
+  capture {print}
+' "${ROOT}/docker/entrypoint.sh" > "${ENTRYPOINT_FUNCS}"
+
+cat > "${TMP_DIR}/venv/bin/python3" <<'SH'
+#!/usr/bin/env bash
+count_file="${MP_FAKE_PYTHON_COUNT}"
+count=0
+if [ -f "$count_file" ]; then
+  count="$(cat "$count_file")"
+fi
+count=$((count + 1))
+printf '%s' "$count" > "$count_file"
+if [ "$count" -eq 1 ]; then
+  exit 1
+fi
+exit 0
+SH
+chmod +x "${TMP_DIR}/venv/bin/python3"
+
+MP_FAKE_PIP_LOG="${TMP_DIR}/entrypoint.log"
+MP_FAKE_PYTHON_COUNT="${TMP_DIR}/python-count"
+export MP_FAKE_PIP_LOG MP_FAKE_PYTHON_COUNT
+(
+  export VENV_PATH="${TMP_DIR}/venv"
+  export CONFIG_DIR="${TMP_DIR}/config"
+  unset PACKAGE_CACHE_ROOT PIP_CACHE_DIR UV_CACHE_DIR
+  export PIP_PROXY=""
+  export PROXY_HOST="http://proxy.example:7890"
+  source "${ENTRYPOINT_FUNCS}"
+  apply_package_cache_env
+  ensure_backend_runtime_dependencies
+) >/dev/null
+
+assert_contains "argv=install -r /app/requirements.txt" "${MP_FAKE_PIP_LOG}"
+assert_contains "HTTPS_PROXY=http://proxy.example:7890" "${MP_FAKE_PIP_LOG}"
+assert_contains "PACKAGE_CACHE_ROOT=${TMP_DIR}/config/.cache" "${MP_FAKE_PIP_LOG}"
+assert_contains "PIP_CACHE_DIR=${TMP_DIR}/config/.cache/pip" "${MP_FAKE_PIP_LOG}"
+assert_contains "UV_CACHE_DIR=${TMP_DIR}/config/.cache/uv" "${MP_FAKE_PIP_LOG}"
+assert_not_contains "--proxy" "${MP_FAKE_PIP_LOG}"
+
+MP_FAKE_PIP_LOG="${TMP_DIR}/entrypoint-app-env.log"
+MP_FAKE_PYTHON_COUNT="${TMP_DIR}/python-count-app-env"
+cat > "${TMP_DIR}/config/app.env" <<EOF
+PACKAGE_CACHE_ROOT='${TMP_DIR}/app-env-custom-package-cache'
+PROXY_HOST='http://proxy.example:7890'
+EOF
+export MP_FAKE_PIP_LOG MP_FAKE_PYTHON_COUNT
+(
+  export VENV_PATH="${TMP_DIR}/venv"
+  export CONFIG_DIR="${TMP_DIR}/config"
+  unset PACKAGE_CACHE_ROOT PIP_CACHE_DIR UV_CACHE_DIR PIP_PROXY PROXY_HOST
+  source "${ENTRYPOINT_FUNCS}"
+  load_config_from_app_env
+  apply_package_cache_env
+  ensure_backend_runtime_dependencies
+) >/dev/null
+
+assert_contains "PACKAGE_CACHE_ROOT=${TMP_DIR}/app-env-custom-package-cache" "${MP_FAKE_PIP_LOG}"
+assert_contains "PIP_CACHE_DIR=${TMP_DIR}/app-env-custom-package-cache/pip" "${MP_FAKE_PIP_LOG}"
+assert_contains "UV_CACHE_DIR=${TMP_DIR}/app-env-custom-package-cache/uv" "${MP_FAKE_PIP_LOG}"
+
+echo "Docker package env simulation passed"
--- a/scripts/dev/simulate_package_installer.py
+++ b/scripts/dev/simulate_package_installer.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT))
+
+from app.helper.package_installer import PackageInstallRequest, build_package_install_strategies
+
+
+def sample(name: str, request: PackageInstallRequest) -> None:
+    print(f"## {name}")
+    strategies = build_package_install_strategies(request)
+    assert strategies, f"{name}: no strategies generated"
+    for strategy in strategies:
+        rendered = " ".join(strategy.safe_log_command)
+        print(strategy.strategy_name)
+        print(rendered)
+        assert all("--proxy" not in arg for arg in strategy.command)
+        assert "user:pass" not in rendered
+        assert strategy.env["PIP_CACHE_DIR"].endswith("/.cache/pip")
+        assert strategy.env["UV_CACHE_DIR"].endswith("/.cache/uv")
+        if strategy.strategy_name.endswith("代理") or strategy.strategy_name.endswith("镜像+代理"):
+            assert strategy.env["HTTPS_PROXY"] == "http://proxy.example:7890"
+
+
+def main() -> None:
+    root = ROOT
+    config_dir = root / "config"
+    python_bin = root.parent / ".venv-test" / "bin" / "python"
+    requirements = root / "requirements.txt"
+
+    samples = {
+        "plain": PackageInstallRequest(
+            requirements_file=requirements,
+            python_bin=python_bin,
+            config_dir=config_dir,
+        ),
+        "mirror": PackageInstallRequest(
+            requirements_file=requirements,
+            python_bin=python_bin,
+            config_dir=config_dir,
+            pip_index_url="https://user:pass@mirror.example/simple",
+        ),
+        "proxy": PackageInstallRequest(
+            requirements_file=requirements,
+            python_bin=python_bin,
+            config_dir=config_dir,
+            proxy_url="http://proxy.example:7890",
+        ),
+        "mirror_proxy_wheels": PackageInstallRequest(
+            requirements_file=requirements,
+            python_bin=python_bin,
+            config_dir=config_dir,
+            find_links_dirs=[
+                root / "plugins.v2" / "demo" / "wheels",
+                root / "plugins.v2" / "other" / "wheels",
+            ],
+            pip_index_url="https://user:pass@mirror.example/simple",
+            proxy_url="http://proxy.example:7890",
+        ),
+    }
+
+    for name, request in samples.items():
+        sample(name, request)
+
+    print("Package installer simulation passed")
+
+
+if __name__ == "__main__":
+    main()
--- a/scripts/local_setup.py
+++ b/scripts/local_setup.py
@@ -17,6 +17,7 @@ import subprocess
 import sys
 import tarfile
 import textwrap
+import urllib.parse
 import uuid
 import zipfile
 from datetime import datetime
@@ -493,15 +494,60 @@ def print_step(message: str) -> None:
    print(f"==> {message}")


+def _redact_url(value: str) -> str:
+    parsed = urllib.parse.urlsplit(value)
+    if "@" not in parsed.netloc:
+        return value
+    host = parsed.netloc.rsplit("@", 1)[-1]
+    return urllib.parse.urlunsplit(
+        (parsed.scheme, host, parsed.path, parsed.query, parsed.fragment)
+    )
+
+
+def redact_command(command: list[str]) -> list[str]:
+    redacted: list[str] = []
+    for item in command:
+        value = str(item)
+        url_marker = value.find("://")
+        equals_marker = value.find("=")
+        if url_marker >= 0 and 0 <= equals_marker < url_marker:
+            key, separator, url = value.partition("=")
+            value = f"{key}{separator}{_redact_url(url)}"
+        elif url_marker >= 0:
+            value = _redact_url(value)
+        redacted.append(value)
+    return redacted
+
+
+def build_package_install_env() -> dict[str, str]:
+    env = os.environ.copy()
+    package_cache_root = env.get("PACKAGE_CACHE_ROOT", "").strip() or str(CONFIG_DIR / ".cache")
+    env.setdefault("PACKAGE_CACHE_ROOT", package_cache_root)
+    env.setdefault("PIP_CACHE_DIR", os.path.join(package_cache_root, "pip"))
+    env.setdefault("UV_CACHE_DIR", os.path.join(package_cache_root, "uv"))
+
+    index_url = env.get("PIP_PROXY", "").strip()
+    if index_url:
+        env["PIP_INDEX_URL"] = index_url
+        env["UV_DEFAULT_INDEX"] = index_url
+
+    proxy = env.get("PROXY_HOST", "").strip()
+    if proxy:
+        for key in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"):
+            env[key] = proxy
+    return env
+
+
 def run(
    command: list[str],
    cwd: Optional[Path] = None,
    env: Optional[dict[str, str]] = None,
+    safe_command: Optional[list[str]] = None,
 ) -> None:
    """
    执行安装步骤中的外部命令，并在失败时让调用方中断流程。
    """
-    pretty = " ".join(command)
+    pretty = " ".join(safe_command or redact_command(command))
    print(f"+ {pretty}")
    subprocess.run(command, cwd=str(cwd or ROOT), check=True, env=env)

@@ -597,7 +643,8 @@ def _ensure_uv_available_for_venv(venv_dir: Path, venv_python: Path) -> Optional
        return uv_bin

    print_step("当前未检测到 uv，先在虚拟环境内安装 uv")
-    run([str(venv_python), "-m", "pip", "install", "--upgrade", "pip", "uv"])
+    command = [str(venv_python), "-m", "pip", "install", "--upgrade", "pip", "uv"]
+    run(command, env=build_package_install_env(), safe_command=redact_command(command))
    if uv_bin.exists():
        return uv_bin
    raise RuntimeError("uv 安装完成，但虚拟环境中未找到 uv 可执行文件")
@@ -2677,13 +2724,15 @@ def install_deps(*, python_bin: str, venv_dir: Path, recreate: bool) -> Path:

    if os.name == "nt":
        print_step("升级 pip")
-        run([str(venv_python), "-m", "pip", "install", "--upgrade", "pip"])
+        command = [str(venv_python), "-m", "pip", "install", "--upgrade", "pip"]
+        run(command, env=build_package_install_env(), safe_command=redact_command(command))
    else:
        print_step("为虚拟环境配置 uv 兼容 pip 命令")
        venv_pip = configure_venv_pip_compat(venv_dir, venv_python)

    print_step("安装项目依赖")
-    run([str(venv_pip), "install", "-r", str(ROOT / "requirements.txt")])
+    command = [str(venv_pip), "install", "-r", str(ROOT / "requirements.txt")]
+    run(command, env=build_package_install_env(), safe_command=redact_command(command))
    install_browser_runtime(venv_python)
    return venv_python

--- a/scripts/uv-pip-compat.sh
+++ b/scripts/uv-pip-compat.sh
@@ -40,6 +40,49 @@ has_environment_option() {
    return 1
 }

+normalize_pip_proxy_args() {
+    output_file="$1"
+    shift
+    original_args_file=$(mktemp)
+    : > "${output_file}"
+    trap 'rm -f "${original_args_file}"' EXIT HUP INT TERM
+
+    for arg in "$@"; do
+        printf '%s\n' "$arg" >> "${original_args_file}"
+    done
+
+    skip_next=0
+    while IFS= read -r arg; do
+        if [ "${skip_next}" -eq 1 ]; then
+            proxy_value="${arg}"
+            export HTTP_PROXY="${proxy_value}"
+            export HTTPS_PROXY="${proxy_value}"
+            export http_proxy="${proxy_value}"
+            export https_proxy="${proxy_value}"
+            skip_next=0
+            continue
+        fi
+        case "$arg" in
+            --proxy)
+                skip_next=1
+                ;;
+            --proxy=*)
+                proxy_value="${arg#--proxy=}"
+                export HTTP_PROXY="${proxy_value}"
+                export HTTPS_PROXY="${proxy_value}"
+                export http_proxy="${proxy_value}"
+                export https_proxy="${proxy_value}"
+                ;;
+            *)
+                printf '%s\n' "$arg" >> "${output_file}"
+                ;;
+        esac
+    done < "${original_args_file}"
+
+    rm -f "${original_args_file}"
+    trap - EXIT HUP INT TERM
+}
+
 uv_pip_with_venv_python() {
    command_name="$1"
    shift
@@ -69,6 +112,15 @@ case "${COMMAND_NAME}" in
            check|freeze|install|list|show|sync|tree|uninstall)
                pip_command="$1"
                shift
+                if [ "${pip_command}" = "install" ]; then
+                    normalized_file=$(mktemp)
+                    normalize_pip_proxy_args "${normalized_file}" "$@"
+                    set --
+                    while IFS= read -r arg; do
+                        set -- "$@" "$arg"
+                    done < "${normalized_file}"
+                    rm -f "${normalized_file}"
+                fi
                uv_pip_with_venv_python "${pip_command}" "$@"
                ;;
            *)