feat: 使用 uv 锁定主程序依赖并强化插件恢复边界 (#6364)

This commit is contained in:
InfinityPacer
2026-08-20 12:17:19 +08:00
committed by GitHub
parent 27ae1b5290
commit 23f5d59c74
59 changed files with 6804 additions and 1797 deletions
+4 -4
View File
@@ -98,23 +98,23 @@ class SystemUtils:
@staticmethod
def execute_with_subprocess(
pip_command: list,
command: list,
env: Optional[dict[str, str]] = None,
safe_command: Optional[list[str]] = None,
) -> Tuple[bool, str]:
"""
执行命令并捕获标准输出和错误输出,记录日志。
:param pip_command: 要执行的命令,以列表形式提供
:param command: 要执行的命令,以列表形式提供
:param env: 传递给子进程的环境变量
:param safe_command: 用于错误信息展示的脱敏命令
:return: (命令是否成功, 输出信息或错误信息)
"""
display_command = safe_command or pip_command
display_command = safe_command or command
try:
# 使用 subprocess.run 捕获标准输出和标准错误
result = subprocess.run(
pip_command,
command,
check=True,
text=True,
stdout=subprocess.PIPE,
+51 -39
View File
@@ -4,26 +4,22 @@ import os
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Literal
from urllib.parse import urlsplit, urlunsplit
PackageBackend = Literal["uv", "pip"]
@dataclass(frozen=True)
class PackageInstallRequest:
"""
Python 包安装请求,集中描述依赖文件、工具缓存、代理和本地 wheels 候选源。
"""
requirements_file: Path
dependency_file: Path
python_bin: Path
find_links_dirs: list[Path] = field(default_factory=list)
constraints_file: Path | None = None
config_dir: Path = Path("/config")
package_cache_root: Path | None = None
pip_index_url: str | None = None
package_index_url: str | None = None
proxy_url: str | None = None
purpose: str = "plugin"
@@ -35,7 +31,6 @@ class PackageInstallStrategy:
"""
strategy_name: str
backend: PackageBackend
command: list[str]
env: dict[str, str]
safe_log_command: list[str]
@@ -61,7 +56,7 @@ def redact_command(command: list[str]) -> list[str]:
def build_package_install_env(request: PackageInstallRequest, include_moviepilot_proxy: bool = True) -> dict[str, str]:
"""
构造 pip/uv 安装子进程环境,默认把包下载缓存放到持久化配置目录。
构造 uv 安装子进程环境,默认把包下载缓存放到持久化配置目录。
"""
env = os.environ.copy()
config_dir = Path(request.config_dir)
@@ -71,7 +66,6 @@ def build_package_install_env(request: PackageInstallRequest, include_moviepilot
else:
package_cache_root = Path(env.get("PACKAGE_CACHE_ROOT") or config_dir / ".cache")
env.setdefault("PACKAGE_CACHE_ROOT", str(package_cache_root))
env.setdefault("PIP_CACHE_DIR", str(package_cache_root / "pip"))
env.setdefault("UV_CACHE_DIR", str(package_cache_root / "uv"))
proxy = (request.proxy_url or "").strip()
if proxy and include_moviepilot_proxy:
@@ -80,9 +74,9 @@ def build_package_install_env(request: PackageInstallRequest, include_moviepilot
return env
def _find_uv(python_bin: Path) -> Path | None:
def find_uv(python_bin: Path) -> Path | None:
"""
优先使用解释器同目录 uv,保证虚拟环境内 wrapper 与真实安装环境一致
优先使用解释器同目录 uv,保证安装器与目标运行环境使用同一版本
"""
uv_name = "uv.exe" if os.name == "nt" else "uv"
sibling = python_bin.with_name(uv_name)
@@ -98,12 +92,12 @@ def _base_install_args(request: PackageInstallRequest) -> list[str]:
args.extend(["--find-links", str(directory)])
if request.constraints_file:
args.extend(["-c", str(request.constraints_file)])
args.extend(["-r", str(request.requirements_file)])
args.extend(["-r", str(request.dependency_file)])
return args
def _network_variants(request: PackageInstallRequest) -> list[tuple[str, bool, bool]]:
has_index = bool((request.pip_index_url or "").strip())
has_index = bool((request.package_index_url or "").strip())
has_proxy = bool((request.proxy_url or "").strip())
variants: list[tuple[str, bool, bool]] = []
if has_index and has_proxy:
@@ -118,49 +112,67 @@ def _network_variants(request: PackageInstallRequest) -> list[tuple[str, bool, b
def _build_uv_command(uv_bin: Path, request: PackageInstallRequest, use_index: bool) -> list[str]:
command = [str(uv_bin), "pip", "install", "--python", str(request.python_bin)]
if use_index and request.pip_index_url:
command.extend(["--default-index", request.pip_index_url])
if use_index and request.package_index_url:
command.extend(["--default-index", request.package_index_url])
command.extend(_base_install_args(request))
return command
def _build_pip_command(request: PackageInstallRequest, use_index: bool) -> list[str]:
command = [str(request.python_bin), "-m", "pip", "install"]
if use_index and request.pip_index_url:
command.extend(["-i", request.pip_index_url])
command.extend(_base_install_args(request))
def _build_uv_sync_command(uv_bin: Path, request: PackageInstallRequest, use_index: bool) -> list[str]:
command = [
str(uv_bin),
"sync",
"--project",
str(request.dependency_file.parent),
"--locked",
"--no-dev",
"--no-install-project",
"--inexact",
]
if use_index and request.package_index_url:
command.extend(["--default-index", request.package_index_url])
return command
def build_package_install_strategies(request: PackageInstallRequest) -> list[PackageInstallStrategy]:
"""
uv 优先、pip 兜底顺序构造网络降级策略。
为固定 uv 安装器构造镜像、代理和直连降级策略。
"""
strategies: list[PackageInstallStrategy] = []
variants = _network_variants(request)
uv_bin = _find_uv(Path(request.python_bin))
if uv_bin:
for variant_name, use_index, use_proxy in variants:
command = _build_uv_command(uv_bin, request, use_index)
env = build_package_install_env(request, include_moviepilot_proxy=use_proxy)
strategies.append(
PackageInstallStrategy(
strategy_name=f"uv:{variant_name}",
backend="uv",
command=command,
env=env,
safe_log_command=redact_command(command),
)
)
uv_bin = find_uv(Path(request.python_bin))
if not uv_bin:
return strategies
for variant_name, use_index, use_proxy in variants:
command = _build_pip_command(request, use_index)
command = _build_uv_command(uv_bin, request, use_index)
env = build_package_install_env(request, include_moviepilot_proxy=use_proxy)
strategies.append(
PackageInstallStrategy(
strategy_name=f"pip:{variant_name}",
backend="pip",
strategy_name=f"uv:{variant_name}",
command=command,
env=env,
safe_log_command=redact_command(command),
)
)
return strategies
def build_project_sync_strategies(request: PackageInstallRequest) -> list[PackageInstallStrategy]:
"""为主项目锁定依赖恢复构造 uv 网络降级策略。"""
uv_bin = find_uv(Path(request.python_bin))
if not uv_bin:
return []
strategies = []
project_environment = request.python_bin.parent.parent
for variant_name, use_index, use_proxy in _network_variants(request):
command = _build_uv_sync_command(uv_bin, request, use_index)
env = build_package_install_env(request, include_moviepilot_proxy=use_proxy)
env["UV_PROJECT_ENVIRONMENT"] = str(project_environment)
strategies.append(
PackageInstallStrategy(
strategy_name=f"uv:{variant_name}",
command=command,
env=env,
safe_log_command=redact_command(command),
+187 -56
View File
@@ -1,23 +1,41 @@
"""插件 requirements 聚合和 Python 依赖安装适配器。"""
"""插件 Python 依赖聚合和安装适配器。"""
from __future__ import annotations
import asyncio
import json
from collections.abc import Callable
from importlib.metadata import distributions
from dataclasses import dataclass, field
from importlib.metadata import PackageNotFoundError, distribution, distributions
from pathlib import Path
from typing import Any, Optional
from urllib.parse import urlsplit
from packaging.markers import default_environment
from packaging.requirements import Requirement
from packaging.specifiers import InvalidSpecifier, SpecifierSet
from packaging.version import InvalidVersion, Version
from app.adapters.system.plugin.manifest import (
PluginDependencyManifestError,
load_dependency_manifest,
)
from app.runtime.config import settings
from app.runtime.log import logger
@dataclass
class _RequirementGroup:
"""聚合同一包和安装来源的 extras 与版本约束。"""
name: str # PEP 503 规范化后的包名
url: Optional[str] # direct reference 来源;为空表示从索引安装
extras: set[str] = field(default_factory=set) # 所有插件要求启用的 extras
specifiers: set[str] = field(default_factory=set) # 待求交集的版本约束
class PluginDependencyInstaller:
"""独立负责插件依赖扫描、约束合并和 pip 安装。"""
"""独立负责插件依赖扫描、约束合并和安装。"""
def __init__(
self,
@@ -26,7 +44,7 @@ class PluginDependencyInstaller:
installed_plugins_provider: Optional[Callable[[], list[str]]] = None,
plugin_dir: Optional[Path] = None,
) -> None:
"""保存 pip 端口和启动层提供的已安装插件读取器。"""
"""保存包安装端口和启动层提供的已安装插件读取器。"""
if helper is None:
from app.adapters.external.market import PluginHelper
@@ -71,49 +89,168 @@ class PluginDependencyInstaller:
return installed
@classmethod
def _parse_requirements(cls, requirements_file: Path) -> dict[str, list[str]]:
"""解析一个 requirements 文件中的包名和版本约束"""
dependencies: dict[str, list[str]] = {}
def _installed_distribution(cls, package_name: str) -> Any | None:
"""读取一个包的元数据,用于校验 extras 和 direct URL 来源"""
try:
for line in requirements_file.read_text(
encoding="utf-8",
errors="replace",
).splitlines():
line = line.strip()
if not line or line.startswith("#"):
continue
try:
requirement = Requirement(line)
except Exception as err:
logger.debug(f"无法解析依赖项 '{line}'{err}")
continue
package_name = cls._standardize(requirement.name)
dependencies.setdefault(package_name, []).append(
str(requirement.specifier)
return distribution(package_name)
except PackageNotFoundError:
return None
def _requirement_satisfied(
self,
requirement: Requirement,
installed: dict[str, Version],
*,
seen: Optional[set[tuple[str, tuple[str, ...], Optional[str]]]] = None,
) -> bool:
"""同时校验版本、extras 及 direct URL,不把同名包误认为同一制品。"""
package_name = self._standardize(requirement.name)
installed_version = installed.get(package_name)
try:
if installed_version is None or not SpecifierSet(
requirement.specifier
).contains(installed_version, prereleases=True):
return False
except InvalidSpecifier as err:
logger.error(f"依赖 {package_name} 约束无效:{err}")
return False
installed_distribution = self._installed_distribution(package_name)
if installed_distribution is None:
return False if requirement.extras or requirement.url else True
if requirement.url and not self._direct_url_matches(
installed_distribution, requirement.url
):
return False
requested_extras = {
self._standardize_extra(extra) for extra in requirement.extras
}
if requested_extras:
provided_extras = {
self._standardize_extra(extra)
for extra in installed_distribution.metadata.get_all(
"Provides-Extra"
)
except Exception as err:
logger.error(f"解析 requirements.txt 时发生错误:{err}")
return dependencies
or []
}
if not requested_extras.issubset(provided_extras):
return False
marker_key = (package_name, tuple(sorted(requested_extras)), requirement.url)
if seen is None:
seen = set()
if marker_key in seen:
return True
seen.add(marker_key)
for raw_dependency in installed_distribution.metadata.get_all(
"Requires-Dist"
) or []:
try:
extra_dependency = Requirement(raw_dependency)
except Exception as err:
logger.debug(
f"无法解析已安装包 {package_name} 的依赖项 '{raw_dependency}'{err}"
)
continue
if not self._marker_matches_for_extras(
extra_dependency, requested_extras
):
continue
if not self._requirement_satisfied(
extra_dependency, installed, seen=seen
):
return False
return True
@classmethod
def _merge(cls, dependencies: dict[str, set[str]]) -> dict[str, str]:
"""求同一包多来源约束的交集,保留冲突约束供 pip 处理。"""
merged: dict[str, str] = {}
for package_name, specifiers in dependencies.items():
def _marker_matches_for_extras(
cls, requirement: Requirement, extras: set[str]
) -> bool:
"""判断已安装发行版声明的可选依赖是否属于当前请求的 extra。"""
if requirement.marker is None:
return True
environment = default_environment()
if "extra" in str(requirement.marker):
return any(
requirement.marker.evaluate({**environment, "extra": extra})
for extra in extras
)
return requirement.marker.evaluate(environment)
@staticmethod
def _standardize_extra(name: str) -> str:
"""按 PEP 685 兼容规则标准化 extra 名称。"""
return (name or "").lower().replace("-", "_").replace(".", "_")
@staticmethod
def _direct_url_matches(installed_distribution: Any, required_url: str) -> bool:
"""校验安装发行版记录的 PEP 610 URL 与清单来源一致。"""
try:
payload = installed_distribution.read_text("direct_url.json")
if not payload:
return False
direct_url = json.loads(payload).get("url")
if not isinstance(direct_url, str):
return False
return PluginDependencyInstaller._canonical_direct_url(
required_url
) == PluginDependencyInstaller._canonical_direct_url(direct_url)
except (AttributeError, json.JSONDecodeError, TypeError, ValueError):
return False
@staticmethod
def _canonical_direct_url(value: str) -> tuple[str, str, str, str, str]:
"""规范化来源 URL,同时保留 fragment 中可能存在的校验信息。"""
parsed = urlsplit(value)
netloc = parsed.netloc.rsplit("@", 1)[-1].lower()
return (
parsed.scheme.lower(),
netloc,
parsed.path.rstrip("/"),
parsed.query,
parsed.fragment,
)
@classmethod
def _merge(cls, dependencies: list[Requirement]) -> list[Requirement]:
"""按包和安装来源合并 extras 与约束,保留完整安装目标。"""
groups: dict[tuple[str, Optional[str]], _RequirementGroup] = {}
for requirement in dependencies:
package_name = cls._standardize(requirement.name)
key = (package_name, requirement.url)
group = groups.setdefault(
key,
_RequirementGroup(name=package_name, url=requirement.url),
)
group.extras.update(requirement.extras)
group.specifiers.add(str(requirement.specifier))
merged: list[Requirement] = []
for group in groups.values():
spec_set = SpecifierSet()
for specifier in specifiers:
for specifier in group.specifiers:
if not specifier:
continue
try:
spec_set &= SpecifierSet(specifier)
except InvalidSpecifier as err:
logger.error(f"发生版本约束冲突:{err}")
merged[package_name] = str(spec_set) if spec_set else ""
target = group.name
if group.extras:
target += f"[{','.join(sorted(group.extras))}]"
if group.url:
target += f" @ {group.url}"
elif spec_set:
target += str(spec_set)
merged.append(Requirement(target))
return merged
def _plugin_dependencies(self) -> dict[str, str]:
"""扫描已安装插件的 requirements 并合并版本约束。"""
dependencies: dict[str, set[str]] = {}
def _plugin_dependencies(self) -> list[Requirement]:
"""扫描已安装插件的生效依赖清单并合并版本约束。"""
dependencies: list[Requirement] = []
installed_plugins = {
plugin_id.lower()
for plugin_id in self._installed_plugins_provider() or []
@@ -121,20 +258,20 @@ class PluginDependencyInstaller:
try:
plugin_dirs = list(self._plugin_dir.iterdir())
except (FileNotFoundError, OSError):
return {}
return []
for plugin_dir in plugin_dirs:
if not plugin_dir.is_dir():
continue
requirements_file = plugin_dir / "requirements.txt"
if not requirements_file.is_file():
continue
if plugin_dir.name not in installed_plugins:
logger.debug(f"忽略插件 {plugin_dir.name} 的依赖")
continue
for package_name, specifiers in self._parse_requirements(
requirements_file
).items():
dependencies.setdefault(package_name, set()).update(specifiers)
manifest = load_dependency_manifest(plugin_dir)
if manifest is None:
continue
for requirement in manifest.dependencies:
if requirement.marker and not requirement.marker.evaluate():
continue
dependencies.append(requirement)
return self._merge(dependencies)
def find_missing(self) -> list[str]:
@@ -143,18 +280,12 @@ class PluginDependencyInstaller:
required = self._plugin_dependencies()
installed = self._installed_packages()
missing = []
for package_name, specifier in required.items():
installed_version = installed.get(package_name)
try:
satisfied = installed_version is not None and SpecifierSet(
specifier
).contains(installed_version, prereleases=True)
except InvalidSpecifier as err:
logger.error(f"依赖 {package_name} 约束无效:{err}")
satisfied = False
if not satisfied:
missing.append(f"{package_name}{specifier}")
for requirement in required:
if not self._requirement_satisfied(requirement, installed):
missing.append(str(requirement))
return missing
except PluginDependencyManifestError:
raise
except Exception as err:
logger.error(f"收集所有需要安装或更新的依赖项时发生错误:{err}")
return []
@@ -173,7 +304,7 @@ class PluginDependencyInstaller:
return list(dict.fromkeys(result))
def install(self, dependencies: list[str]) -> tuple[bool, str]:
"""把依赖写入临时 requirements 并调用现有 pip 健康检查策略。"""
"""把依赖写入临时 requirements 并调用统一包安装策略。"""
if not dependencies:
return False, "没有传入需要安装的依赖项"
requirements_file = (
@@ -187,7 +318,7 @@ class PluginDependencyInstaller:
"".join(f"{dependency}\n" for dependency in dependencies),
encoding="utf-8",
)
return self._helper.pip_install_with_fallback(
return self._helper.install_packages_with_fallback(
requirements_file,
self._wheels_dirs(),
)
@@ -202,5 +333,5 @@ class PluginDependencyInstaller:
return await asyncio.to_thread(self.find_missing)
async def async_install(self, dependencies: list[str]) -> tuple[bool, str]:
"""在线程池中安装依赖,复用同步 pip 健康检查策略。"""
"""在线程池中安装依赖,复用同步包安装策略。"""
return await asyncio.to_thread(self.install, dependencies)
+162
View File
@@ -0,0 +1,162 @@
"""插件 Python 依赖清单的选择和解析。"""
from __future__ import annotations
import tomllib
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from packaging.requirements import Requirement
from app.runtime.log import logger
PYPROJECT_FILENAME = "pyproject.toml"
REQUIREMENTS_FILENAME = "requirements.txt"
DEPENDENCY_MANIFEST_PRIORITY = (
PYPROJECT_FILENAME,
REQUIREMENTS_FILENAME,
)
DEPENDENCY_MANIFEST_FILENAMES = frozenset(
DEPENDENCY_MANIFEST_PRIORITY
)
class PluginDependencyManifestError(ValueError):
"""表示生效的现代依赖清单无法安全消费。"""
@dataclass(frozen=True)
class PluginDependencyManifest:
"""保存插件当前生效的依赖清单及其已解析依赖。"""
path: Path
dependencies: tuple[Requirement, ...]
def select_dependency_manifest(plugin_dir: Path) -> Path | None:
"""按现代清单优先级返回插件当前生效的依赖文件。"""
pyproject_file = plugin_dir / PYPROJECT_FILENAME
if pyproject_file.is_file():
return pyproject_file
requirements_file = plugin_dir / REQUIREMENTS_FILENAME
if requirements_file.is_file():
return requirements_file
return None
def dependency_manifest_status(event_path: Path) -> bool | None:
"""判断文件事件是否改变生效清单,非清单文件返回 None。"""
if event_path.name not in DEPENDENCY_MANIFEST_FILENAMES:
return None
active_manifest = select_dependency_manifest(event_path.parent)
if event_path.is_file():
return active_manifest == event_path
if active_manifest is None:
return True
return DEPENDENCY_MANIFEST_PRIORITY.index(
event_path.name
) < DEPENDENCY_MANIFEST_PRIORITY.index(active_manifest.name)
def load_dependency_manifest(
plugin_dir: Path,
) -> PluginDependencyManifest | None:
"""读取插件当前生效的依赖清单,现代清单无效时拒绝回退。"""
manifest_path = select_dependency_manifest(plugin_dir)
if manifest_path is None:
return None
return load_dependency_file(manifest_path)
def load_dependency_file(path: Path) -> PluginDependencyManifest:
"""读取指定依赖文件,pyproject 严格校验,其余文件保持旧格式兼容。"""
if path.name == PYPROJECT_FILENAME:
dependencies = _load_pyproject_dependencies(path)
else:
dependencies = _load_requirements_dependencies(path)
return PluginDependencyManifest(
path=path,
dependencies=dependencies,
)
def _load_pyproject_dependencies(path: Path) -> tuple[Requirement, ...]:
"""严格读取 PEP 621 ``project.dependencies``。"""
try:
with path.open("rb") as file:
document = tomllib.load(file)
except (OSError, tomllib.TOMLDecodeError) as err:
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 无法解析:{err}"
) from err
project = document.get("project")
if not isinstance(project, Mapping):
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 缺少 [project] 表"
)
name = project.get("name")
if not isinstance(name, str) or not name.strip():
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 的 project.name 必须是非空字符串"
)
dynamic = project.get("dynamic", [])
if not isinstance(dynamic, list) or not all(
isinstance(item, str) for item in dynamic
):
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 的 project.dynamic 必须是字符串数组"
)
if "dependencies" in dynamic:
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 不支持动态 dependencies"
)
version = project.get("version")
if "version" in dynamic:
if version is not None:
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 不能同时静态和动态声明 version"
)
elif not isinstance(version, str) or not version.strip():
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 必须声明非空 project.version"
"或将 version 加入 project.dynamic"
)
raw_dependencies = project.get("dependencies", [])
if not isinstance(raw_dependencies, list) or not all(
isinstance(item, str) for item in raw_dependencies
):
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 的 project.dependencies 必须是字符串数组"
)
dependencies: list[Requirement] = []
for item in raw_dependencies:
try:
dependencies.append(Requirement(item))
except Exception as err:
raise PluginDependencyManifestError(
f"插件依赖清单 {path.name} 包含无效依赖项 {item!r}{err}"
) from err
return tuple(dependencies)
def _load_requirements_dependencies(path: Path) -> tuple[Requirement, ...]:
"""按旧行为逐行读取 requirements,忽略无法解析的兼容内容。"""
dependencies: list[Requirement] = []
try:
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
except OSError as err:
logger.error(f"解析 requirements.txt 时发生错误:{err}")
return ()
for line in lines:
line = line.strip()
if not line or line.startswith("#"):
continue
try:
dependencies.append(Requirement(line))
except Exception as err:
logger.debug(f"无法解析依赖项 '{line}'{err}")
return tuple(dependencies)