From 2d80f9f04f439c9bdfd3cd1c4dec1c265d076618 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Thu, 20 Aug 2026 20:51:35 +0800 Subject: [PATCH] fix(plugins): preserve dependency manifest sources (#6370) --- app/adapters/external/market.py | 42 +++++++++----- app/adapters/system/package.py | 12 ++-- app/adapters/system/plugin/dependency.py | 34 ++++++----- scripts/dev/simulate_package_installer.py | 8 +-- tests/test_package_installer.py | 34 +++++++++-- tests/test_plugin_dependency_installer.py | 51 +++++++++++------ tests/test_plugin_helper.py | 69 +++++++++++++++++++++++ 7 files changed, 189 insertions(+), 61 deletions(-) diff --git a/app/adapters/external/market.py b/app/adapters/external/market.py index e89104d5c..0488dab94 100644 --- a/app/adapters/external/market.py +++ b/app/adapters/external/market.py @@ -14,7 +14,7 @@ import time import traceback import zipfile from pathlib import Path, PurePosixPath, PureWindowsPath -from typing import Dict, List, Optional, Tuple, Set, Callable, Awaitable +from typing import Dict, List, Optional, Tuple, Set, Callable, Awaitable, Sequence from urllib.parse import parse_qs, quote, unquote, urlparse, urlsplit import aiofiles @@ -1461,7 +1461,7 @@ class PluginHelper(metaclass=WeakSingleton): @classmethod def __build_package_install_request( cls, - dependency_file: Path, + dependency_files: Path | Sequence[Path], find_links_dirs: Optional[List[Path]] = None, constraints_file: Optional[Path] = None, purpose: str = "plugin", @@ -1469,8 +1469,12 @@ class PluginHelper(metaclass=WeakSingleton): """ 将 MoviePilot 运行配置转换为 uv 安装请求，统一缓存、镜像和代理语义。 """ + if isinstance(dependency_files, Path): + resolved_dependency_files = (dependency_files,) + else: + resolved_dependency_files = tuple(Path(item) for item in dependency_files) return PackageInstallRequest( - dependency_file=dependency_file, + dependency_files=resolved_dependency_files, python_bin=Path(sys.executable), find_links_dirs=find_links_dirs or [], constraints_file=constraints_file, @@ -1583,18 +1587,26 @@ class PluginHelper(metaclass=WeakSingleton): @classmethod def install_packages_with_fallback(cls, - dependency_file: Path, + dependency_files: Path | Sequence[Path], find_links_dirs: Optional[List[Path]] = None) -> Tuple[bool, str]: """ 使用自动降级策略安装依赖，并确保新安装的包可被动态导入 - :param dependency_file: 插件依赖清单路径 + :param dependency_files: 一个或多个插件依赖清单路径 :param find_links_dirs: 额外的本地 wheels 目录列表 :return: (是否成功, 错误信息) """ - wheels_dir = dependency_file.parent / "wheels" + if isinstance(dependency_files, Path): + resolved_dependency_files = (dependency_files,) + else: + resolved_dependency_files = tuple(Path(item) for item in dependency_files) + if not resolved_dependency_files: + return False, "没有传入插件依赖清单" + candidate_dirs = [] - if wheels_dir.is_dir(): - candidate_dirs.append(wheels_dir) + for dependency_file in resolved_dependency_files: + wheels_dir = dependency_file.parent / "wheels" + if wheels_dir.is_dir(): + candidate_dirs.append(wheels_dir) if find_links_dirs: candidate_dirs.extend(find_links_dirs) @@ -1619,10 +1631,14 @@ class PluginHelper(metaclass=WeakSingleton): installed_packages = cls.__get_installed_packages() protected_packages = cls.__get_protected_runtime_packages(installed_packages) - check_ok, check_message = cls.__validate_runtime_dependency_conflicts(dependency_file, protected_packages) - if not check_ok: - logger.error(f"[UV] 运行环境冲突预检失败：{check_message}") - return False, check_message + for dependency_file in resolved_dependency_files: + check_ok, check_message = cls.__validate_runtime_dependency_conflicts( + dependency_file, + protected_packages, + ) + if not check_ok: + logger.error(f"[UV] 运行环境冲突预检失败：{check_message}") + return False, check_message constraints_file = None if protected_packages: @@ -1633,7 +1649,7 @@ class PluginHelper(metaclass=WeakSingleton): return False, f"创建运行环境约束文件失败：{e}" request = cls.__build_package_install_request( - dependency_file, + resolved_dependency_files, find_links_dirs=resolved_dirs, constraints_file=constraints_file, purpose="plugin", diff --git a/app/adapters/system/package.py b/app/adapters/system/package.py index aee6d3401..e57f81c42 100644 --- a/app/adapters/system/package.py +++ b/app/adapters/system/package.py @@ -10,10 +10,10 @@ from urllib.parse import urlsplit, urlunsplit @dataclass(frozen=True) class PackageInstallRequest: """ - Python 包安装请求，集中描述依赖文件、工具缓存、代理和本地 wheels 候选源。 + Python 包安装请求，集中描述依赖清单、工具缓存、代理和本地 wheels 候选源。 """ - dependency_file: Path + dependency_files: tuple[Path, ...] python_bin: Path find_links_dirs: list[Path] = field(default_factory=list) constraints_file: Path | None = None @@ -92,7 +92,8 @@ def _base_install_args(request: PackageInstallRequest) -> list[str]: args.extend(["--find-links", str(directory)]) if request.constraints_file: args.extend(["-c", str(request.constraints_file)]) - args.extend(["-r", str(request.dependency_file)]) + for dependency_file in request.dependency_files: + args.extend(["-r", str(dependency_file)]) return args @@ -119,11 +120,14 @@ def _build_uv_command(uv_bin: Path, request: PackageInstallRequest, use_index: b def _build_uv_sync_command(uv_bin: Path, request: PackageInstallRequest, use_index: bool) -> list[str]: + if len(request.dependency_files) != 1: + raise ValueError("主项目锁定依赖恢复只接受一个 pyproject.toml") + project_file = request.dependency_files[0] command = [ str(uv_bin), "sync", "--project", - str(request.dependency_file.parent), + str(project_file.parent), "--locked", "--no-dev", "--no-install-project", diff --git a/app/adapters/system/plugin/dependency.py b/app/adapters/system/plugin/dependency.py index dd8690e70..e6e7c179d 100644 --- a/app/adapters/system/plugin/dependency.py +++ b/app/adapters/system/plugin/dependency.py @@ -248,9 +248,9 @@ class PluginDependencyInstaller: merged.append(Requirement(target)) return merged - def _plugin_dependencies(self) -> list[Requirement]: - """扫描已安装插件的生效依赖清单并合并版本约束。""" - dependencies: list[Requirement] = [] + def _plugin_manifests(self) -> list[Any]: + """返回已安装插件当前生效的依赖清单。""" + manifests = [] installed_plugins = { plugin_id.lower() for plugin_id in self._installed_plugins_provider() or [] @@ -259,7 +259,7 @@ class PluginDependencyInstaller: plugin_dirs = list(self._plugin_dir.iterdir()) except (FileNotFoundError, OSError): return [] - for plugin_dir in plugin_dirs: + for plugin_dir in sorted(plugin_dirs, key=lambda item: item.name): if not plugin_dir.is_dir(): continue if plugin_dir.name not in installed_plugins: @@ -268,6 +268,13 @@ class PluginDependencyInstaller: manifest = load_dependency_manifest(plugin_dir) if manifest is None: continue + manifests.append(manifest) + return manifests + + def _plugin_dependencies(self) -> list[Requirement]: + """扫描已安装插件的生效依赖清单并合并版本约束。""" + dependencies: list[Requirement] = [] + for manifest in self._plugin_manifests(): for requirement in manifest.dependencies: if requirement.marker and not requirement.marker.evaluate(): continue @@ -304,29 +311,20 @@ class PluginDependencyInstaller: return list(dict.fromkeys(result)) def install(self, dependencies: list[str]) -> tuple[bool, str]: - """把依赖写入临时 requirements 并调用统一包安装策略。""" + """把已安装插件的原始清单交给一次统一包安装。""" if not dependencies: return False, "没有传入需要安装的依赖项" - requirements_file = ( - Path(settings.TEMP_PATH) - / "plugin_dependencies" - / "requirements.txt" - ) try: - requirements_file.parent.mkdir(parents=True, exist_ok=True) - requirements_file.write_text( - "".join(f"{dependency}\n" for dependency in dependencies), - encoding="utf-8", - ) + manifest_paths = [manifest.path for manifest in self._plugin_manifests()] + if not manifest_paths: + return False, "没有找到已安装插件的依赖清单" return self._helper.install_packages_with_fallback( - requirements_file, + manifest_paths, self._wheels_dirs(), ) except Exception as err: logger.error(f"安装依赖项时发生错误：{err}") return False, f"安装依赖项时发生错误：{err}" - finally: - requirements_file.unlink(missing_ok=True) async def async_find_missing(self) -> list[str]: """在线程池中扫描缺失依赖，避免阻塞事件循环。""" diff --git a/scripts/dev/simulate_package_installer.py b/scripts/dev/simulate_package_installer.py index b62582b67..802b8e4fa 100644 --- a/scripts/dev/simulate_package_installer.py +++ b/scripts/dev/simulate_package_installer.py @@ -34,24 +34,24 @@ def main() -> None: samples = { "plain": PackageInstallRequest( - dependency_file=requirements, + dependency_files=(requirements,), python_bin=python_bin, config_dir=config_dir, ), "mirror": PackageInstallRequest( - dependency_file=requirements, + dependency_files=(requirements,), python_bin=python_bin, config_dir=config_dir, package_index_url="https://user:pass@mirror.example/simple", ), "proxy": PackageInstallRequest( - dependency_file=requirements, + dependency_files=(requirements,), python_bin=python_bin, config_dir=config_dir, proxy_url="http://proxy.example:7890", ), "mirror_proxy_wheels": PackageInstallRequest( - dependency_file=requirements, + dependency_files=(requirements,), python_bin=python_bin, config_dir=config_dir, find_links_dirs=[ diff --git a/tests/test_package_installer.py b/tests/test_package_installer.py index e265434cc..4a8aef426 100644 --- a/tests/test_package_installer.py +++ b/tests/test_package_installer.py @@ -17,7 +17,7 @@ def test_build_env_maps_proxy_and_cache(tmp_path, monkeypatch): monkeypatch.delenv("PACKAGE_CACHE_ROOT", raising=False) monkeypatch.setenv("HTTP_PROXY", "http://old.example:8080") request = PackageInstallRequest( - dependency_file=tmp_path / "requirements.txt", + dependency_files=(tmp_path / "requirements.txt",), python_bin=Path("/venv/bin/python"), config_dir=tmp_path / "config", package_index_url="https://user:pass@mirror.example/simple", @@ -38,7 +38,7 @@ def test_build_env_uses_package_cache_root_and_preserves_tool_cache_overrides(tm monkeypatch.setenv("PACKAGE_CACHE_ROOT", str(tmp_path / "custom-package-cache")) monkeypatch.delenv("UV_CACHE_DIR", raising=False) request = PackageInstallRequest( - dependency_file=tmp_path / "requirements.txt", + dependency_files=(tmp_path / "requirements.txt",), python_bin=Path("/venv/bin/python"), config_dir=tmp_path / "config", ) @@ -59,7 +59,7 @@ def test_build_strategies_prefers_uv_network_matrix_and_preserves_find_links(tmp uv_bin.write_text("", encoding="utf-8") request = PackageInstallRequest( - dependency_file=req, + dependency_files=(req,), python_bin=tmp_path / "venv" / "bin" / "python", find_links_dirs=[wheels], config_dir=tmp_path / "config", @@ -92,7 +92,7 @@ def test_build_strategies_fail_closed_when_uv_missing(tmp_path): req = tmp_path / "requirements.txt" req.write_text("demo\n", encoding="utf-8") request = PackageInstallRequest( - dependency_file=req, + dependency_files=(req,), python_bin=tmp_path / "venv" / "bin" / "python", config_dir=tmp_path / "config", ) @@ -103,6 +103,32 @@ def test_build_strategies_fail_closed_when_uv_missing(tmp_path): assert strategies == [] +def test_build_strategies_passes_all_manifests_to_one_uv_process(tmp_path): + """多个插件清单必须进入同一个 uv 命令并保持输入顺序。""" + modern = tmp_path / "modern" / "pyproject.toml" + modern.parent.mkdir() + modern.write_text("[project]\nname='modern'\nversion='1'\n", encoding="utf-8") + legacy = tmp_path / "legacy" / "requirements.txt" + legacy.parent.mkdir() + legacy.write_text("demo\n", encoding="utf-8") + uv_bin = tmp_path / "venv" / "bin" / "uv" + uv_bin.parent.mkdir(parents=True) + uv_bin.write_text("", encoding="utf-8") + request = PackageInstallRequest( + dependency_files=(modern, legacy), + python_bin=tmp_path / "venv" / "bin" / "python", + ) + + strategies = build_package_install_strategies(request) + + command = strategies[0].command + first_requirement = command.index("-r") + second_requirement = command.index("-r", first_requirement + 1) + assert command.count("-r") == 2 + assert command[first_requirement + 1] == str(modern) + assert command[second_requirement + 1] == str(legacy) + + def test_redact_url_removes_userinfo(): assert redact_url("https://user:pass@mirror.example/simple") == "https://mirror.example/simple" diff --git a/tests/test_plugin_dependency_installer.py b/tests/test_plugin_dependency_installer.py index d1d930852..09342bf63 100644 --- a/tests/test_plugin_dependency_installer.py +++ b/tests/test_plugin_dependency_installer.py @@ -408,24 +408,38 @@ def test_load_dependency_file_accepts_custom_legacy_filename(tmp_path): ] -def test_install_uses_adapter_owned_temporary_requirements(tmp_path, monkeypatch): - """批量依赖文件由依赖适配器创建并在安装返回后清理。""" - helper = Mock() - installed_contents = [] +def test_install_passes_all_active_manifests_to_one_install(tmp_path): + """缺失依赖恢复必须保留 modern 与 legacy 清单的原始内容。""" + plugin_root = tmp_path / "plugins" + modern_dir = _write_pyproject( + plugin_root, + "Alpha", + """ +[project] +name = "alpha" +version = "1.0.0" +dependencies = ["demo>=2"] - def _install_packages(dependency_file, _wheels_dirs): - installed_contents.append(dependency_file.read_text(encoding="utf-8")) - return True, "installed" +[[tool.uv.index]] +name = "private" +url = "https://packages.example/simple" +explicit = true - helper.install_packages_with_fallback.side_effect = _install_packages - monkeypatch.setattr( - "app.adapters.system.plugin.dependency.settings", - SimpleNamespace(ROOT_PATH=tmp_path, TEMP_PATH=tmp_path / "temp"), +[tool.uv.sources] +demo = { index = "private" } +""", ) + _write_requirements( + plugin_root, + "Beta", + "--extra-index-url https://legacy.example/simple\nother\n", + ) + helper = Mock() + helper.install_packages_with_fallback.return_value = (True, "installed") installer = PluginDependencyInstaller( helper, - installed_plugins_provider=lambda: [], - plugin_dir=tmp_path / "plugins", + installed_plugins_provider=lambda: ["Alpha", "Beta"], + plugin_dir=plugin_root, ) result = installer.install([ @@ -434,9 +448,10 @@ def test_install_uses_adapter_owned_temporary_requirements(tmp_path, monkeypatch ]) assert result == (True, "installed") - assert installed_contents == [ - "demo[feature] @ https://example.com/demo.whl\nother\n" + manifest_paths = helper.install_packages_with_fallback.call_args.args[0] + assert manifest_paths == [ + modern_dir / "pyproject.toml", + plugin_root / "beta" / "requirements.txt", ] - requirements_file = helper.install_packages_with_fallback.call_args.args[0] - assert requirements_file.name == "requirements.txt" - assert not requirements_file.exists() + assert "[tool.uv.sources]" in manifest_paths[0].read_text(encoding="utf-8") + assert "--extra-index-url" in manifest_paths[1].read_text(encoding="utf-8") diff --git a/tests/test_plugin_helper.py b/tests/test_plugin_helper.py index 9d9e1bfd0..987d5ea20 100644 --- a/tests/test_plugin_helper.py +++ b/tests/test_plugin_helper.py @@ -1062,6 +1062,75 @@ class TestPluginHelper: assert env["HTTPS_PROXY"] == "http://proxy.example:7890" assert "user:pass" not in " ".join(safe_command) + def test_uv_install_keeps_multiple_original_manifests_in_one_command(self): + """批量恢复必须让 uv 直接读取每个插件的原始生效清单。""" + try: + from app.adapters.external.market import PluginHelper + except ModuleNotFoundError as exc: + pytest.skip(f"missing dependency: {exc}") + + seen_commands = [] + + def fake_execute(command, env=None, safe_command=None): + seen_commands.append(command) + return True, "ok" + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + modern = root / "modern" / "pyproject.toml" + modern.parent.mkdir() + modern.write_text( + """ +[project] +name = "modern" +version = "1.0.0" +dependencies = ["demo>=2"] + +[[tool.uv.index]] +name = "private" +url = "https://packages.example/simple" +explicit = true + +[tool.uv.sources] +demo = { index = "private" } +""", + encoding="utf-8", + ) + legacy = root / "legacy" / "requirements.txt" + legacy.parent.mkdir() + legacy.write_text( + "--extra-index-url https://legacy.example/simple\nother\n", + encoding="utf-8", + ) + uv_bin = _create_fake_uv(root) + + with patch("app.adapters.system.package.find_uv", return_value=uv_bin), \ + patch.object(PluginHelper, "_PluginHelper__get_installed_packages", return_value={}), \ + patch.object(PluginHelper, "_PluginHelper__get_protected_runtime_packages", return_value={}), \ + patch.object( + PluginHelper, + "_PluginHelper__run_runtime_healthcheck", + return_value={"uv check": (True, "ok"), "核心依赖导入检查": (True, "ok")}, + ), \ + patch("app.adapters.external.market.SystemUtils.execute_with_subprocess", side_effect=fake_execute): + success, message = PluginHelper.install_packages_with_fallback( + [modern, legacy] + ) + + assert success + assert message == "ok" + install_command = next( + command for command in seen_commands + if command[:3] == [str(uv_bin), "pip", "install"] + ) + requirement_positions = [ + index for index, value in enumerate(install_command) if value == "-r" + ] + assert [install_command[index + 1] for index in requirement_positions] == [ + str(modern), + str(legacy), + ] + def test_uv_install_serializes_concurrent_calls(self): """ 验证多个依赖安装请求会复用同一把锁串行执行 uv。