mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-28 11:37:23 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d9e6532325 | ||
|
|
049f16ba01 | ||
|
|
6541458326 | ||
|
|
9f2912426b | ||
|
|
fde33d267a | ||
|
|
ef7f0afa37 | ||
|
|
bea77a8243 | ||
|
|
b984b83870 | ||
|
|
2153ad48db | ||
|
|
c9c43fde74 | ||
|
|
e2c9742f64 | ||
|
|
3d459a40f7 | ||
|
|
5675cd5b11 | ||
|
|
74a4d0bd66 | ||
|
|
2b8c313019 | ||
|
|
62fb6b80a3 | ||
|
|
eea86528d8 | ||
|
|
84e6abb659 | ||
|
|
da2c755b6d | ||
|
|
51f39be9bc | ||
|
|
21b762e75c | ||
|
|
54095074b6 | ||
|
|
33525730b5 | ||
|
|
71260f04b5 | ||
|
|
e2acec321d | ||
|
|
74a462a09f | ||
|
|
ad9e1a5da6 | ||
|
|
d90e3c29a5 | ||
|
|
19165eff75 | ||
|
|
52d0703812 | ||
|
|
1431a5e82a | ||
|
|
23fe643526 | ||
|
|
545b3c0482 | ||
|
|
f102119eef | ||
|
|
9bb3d707c9 | ||
|
|
b892ef50dc | ||
|
|
41e2907168 | ||
|
|
14e28ed693 | ||
|
|
79393c21ff | ||
|
|
cafa4d217c | ||
|
|
2b9e69b112 | ||
|
|
3ffcea70a7 | ||
|
|
ffc72ba6fe | ||
|
|
848becd946 | ||
|
|
71fe96d7f9 | ||
|
|
35c7238ede | ||
|
|
3578204508 | ||
|
|
c11cf17f62 | ||
|
|
5a59652684 | ||
|
|
7f5f31f143 | ||
|
|
dc1cee80b1 | ||
|
|
92cb066748 | ||
|
|
6c8ef4122b | ||
|
|
971b02ac8c | ||
|
|
d4a9643f47 | ||
|
|
e56d31fedc | ||
|
|
b9d91c5cd7 | ||
|
|
57cdb57331 | ||
|
|
0f7a7ef44f | ||
|
|
6267b3f670 | ||
|
|
82f77b4729 | ||
|
|
58da0ebb4f | ||
|
|
7a43e43478 | ||
|
|
e5ec02e043 | ||
|
|
2944c343a8 | ||
|
|
940cc566c8 | ||
|
|
db7b2cdcac | ||
|
|
8111cf5dc8 |
-27
@@ -1,27 +0,0 @@
|
||||
[run]
|
||||
branch = True
|
||||
source = app
|
||||
omit =
|
||||
app/plugins/*/*
|
||||
app/testing/*
|
||||
app/helper/sites.py
|
||||
|
||||
[report]
|
||||
show_missing = True
|
||||
skip_empty = True
|
||||
precision = 2
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
if TYPE_CHECKING:
|
||||
if __name__ == .__main__.:
|
||||
raise NotImplementedError
|
||||
pass
|
||||
|
||||
[html]
|
||||
directory = htmlcov
|
||||
|
||||
[xml]
|
||||
output = coverage.xml
|
||||
|
||||
[json]
|
||||
output = coverage.json
|
||||
+2
-108
@@ -1,109 +1,3 @@
|
||||
# Git
|
||||
# Ignore git
|
||||
.github
|
||||
.git
|
||||
.gitignore
|
||||
|
||||
# Documentation
|
||||
docs/
|
||||
README.md
|
||||
LICENSE
|
||||
|
||||
# Development files
|
||||
.pylintrc
|
||||
**/*.pyc
|
||||
**/__pycache__/
|
||||
**/*.pyo
|
||||
**/*.pyd
|
||||
.Python
|
||||
**/*.so
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
coverage.json
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
.hypothesis/
|
||||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
.worktrees/
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Runtime state and locally synchronized payloads
|
||||
.build/
|
||||
.agent-work/
|
||||
.runtime/
|
||||
.tmp/
|
||||
.cache/
|
||||
node_modules/
|
||||
public/
|
||||
.moviepilot.env
|
||||
.env
|
||||
.env.*
|
||||
config/*
|
||||
!config/category.yaml
|
||||
app/plugins/**
|
||||
!app/plugins/__init__.py
|
||||
app/application/site/*.bin
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
.DS_Store?
|
||||
._*
|
||||
.Spotlight-V100
|
||||
.Trashes
|
||||
ehthumbs.db
|
||||
Thumbs.db
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
logs/
|
||||
|
||||
# Temporary files
|
||||
*.tmp
|
||||
*.temp
|
||||
tmp/
|
||||
temp/
|
||||
|
||||
# Database
|
||||
*.db
|
||||
*.sqlite
|
||||
*.sqlite3
|
||||
|
||||
# Test files
|
||||
tests/
|
||||
test_*
|
||||
*_test.py
|
||||
|
||||
# Build artifacts
|
||||
.artifacts/
|
||||
build/
|
||||
.build/
|
||||
dist/
|
||||
*.egg-info/
|
||||
rust/**/target/
|
||||
|
||||
# Docker
|
||||
Dockerfile*
|
||||
docker-compose*
|
||||
.dockerignore
|
||||
|
||||
# Other
|
||||
app.ico
|
||||
frozen.spec
|
||||
.git
|
||||
@@ -7,13 +7,11 @@ body:
|
||||
attributes:
|
||||
value: |
|
||||
请说明你希望添加的功能。
|
||||
|
||||
站点适配请求请先按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-capture.md) 生成脱敏 ZIP,并在下方附加。Issue 及附件是公开内容,提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
label: 当前程序版本
|
||||
description: 目前使用的程序版本;仅提供站点采集文件且未安装 MoviePilot 时填写“不适用”
|
||||
description: 目前使用的程序版本
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
@@ -24,9 +22,6 @@ body:
|
||||
options:
|
||||
- Docker
|
||||
- Windows
|
||||
- macOS
|
||||
- Linux
|
||||
- 仅提供站点采集文件
|
||||
validations:
|
||||
required: true
|
||||
- type: dropdown
|
||||
@@ -37,7 +32,6 @@ body:
|
||||
options:
|
||||
- 主程序
|
||||
- 插件
|
||||
- 站点适配
|
||||
- 其他
|
||||
validations:
|
||||
required: true
|
||||
@@ -49,14 +43,6 @@ body:
|
||||
placeholder: "功能改进"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: site-adapter-capture
|
||||
attributes:
|
||||
label: 站点适配采集文件
|
||||
description: 站点适配请求必须把采集器生成并人工预览确认过的脱敏 ZIP 拖到这里;Issue 附件公开,严禁附加 Cookie、原始 HTML、HAR 或浏览器网络归档。其他类型请填写“不适用”。
|
||||
placeholder: "将 moviepilot-site-capture-*.zip 拖到这里;非站点适配填写:不适用"
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: references
|
||||
attributes:
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
name: 功能提案
|
||||
description: Request for Comments
|
||||
title: "[RFC]"
|
||||
labels: ["RFC"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
一份提案(RFC)定位为 **「在某功能/重构的具体开发前,用于开发者间 review 技术设计/方案的文档」**,
|
||||
目的是让协作的开发者间清晰的知道「要做什么」和「具体会怎么做」,以及所有的开发者都能公开透明的参与讨论;
|
||||
以便评估和讨论产生的影响 (遗漏的考虑、向后兼容性、与现有功能的冲突),
|
||||
因此提案侧重在对解决问题的 **方案、设计、步骤** 的描述上。
|
||||
|
||||
如果仅希望讨论是否添加或改进某功能本身,请使用 -> [Issue: 功能改进](https://github.com/jxxghp/MoviePilot/issues/new?assignees=&labels=feature+request&projects=&template=feature_request.yml&title=%5BFeature+Request%5D%3A+)
|
||||
- type: textarea
|
||||
id: background
|
||||
attributes:
|
||||
label: 背景 or 问题
|
||||
description: 简单描述遇到的什么问题或需要改动什么。可以引用其他 issue、讨论、文档等。
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: goal
|
||||
attributes:
|
||||
label: "目标 & 方案简述"
|
||||
description: 简单描述提案此提案实现后,**预期的目标效果**,以及简单大致描述会采取的方案/步骤,可能会/不会产生什么影响。
|
||||
validations:
|
||||
required: true
|
||||
- type: textarea
|
||||
id: design
|
||||
attributes:
|
||||
label: "方案设计 & 实现步骤"
|
||||
description: |
|
||||
详细描述你设计的具体方案,可以考虑拆分列表或要点,一步步描述具体打算如何实现的步骤和相关细节。
|
||||
这部份不需要一次性写完整,即使在创建完此提案 issue 后,依旧可以再次编辑修改。
|
||||
validations:
|
||||
required: false
|
||||
- type: textarea
|
||||
id: alternative
|
||||
attributes:
|
||||
label: "替代方案 & 对比"
|
||||
description: |
|
||||
[可选] 为来实现目标效果,还考虑过什么其他方案,有什么对比?
|
||||
validations:
|
||||
required: false
|
||||
@@ -1 +0,0 @@
|
||||
AGENTS.md
|
||||
@@ -1,69 +0,0 @@
|
||||
name: Official Plugin Architecture Observation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 3 * * 1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: official-plugin-architecture-observation
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
observe:
|
||||
runs-on: ubuntu-latest
|
||||
name: Compare latest official plugin contracts
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout MoviePilot
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
path: MoviePilot
|
||||
|
||||
- name: Checkout MoviePilot-Plugins
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Plugins
|
||||
ref: main
|
||||
path: MoviePilot-Plugins
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
MoviePilot/pyproject.toml
|
||||
MoviePilot/uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: MoviePilot
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Compare plugin contracts without updating fixtures
|
||||
id: compare
|
||||
continue-on-error: true
|
||||
working-directory: MoviePilot
|
||||
run: |
|
||||
uv run --locked --no-sync python scripts/architecture/baseline.py \
|
||||
--check-plugins \
|
||||
--plugin-repo ../MoviePilot-Plugins \
|
||||
--report official-plugin-architecture-report.json
|
||||
|
||||
- name: Upload semantic comparison report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: official-plugin-architecture-report
|
||||
path: MoviePilot/official-plugin-architecture-report.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 3
|
||||
|
||||
- name: Fail when plugin contracts changed
|
||||
if: steps.compare.outcome == 'failure'
|
||||
run: exit 1
|
||||
@@ -1,340 +0,0 @@
|
||||
name: MoviePilot Builder Beta
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
env:
|
||||
TRIVY_SKIP_DIRS: /usr/share/java
|
||||
TRIVY_SKIP_JAVA_DB_UPDATE: "true"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
|
||||
- name: Verify dependency lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Audit locked Python dependencies
|
||||
run: |
|
||||
uv export --quiet --locked --no-default-groups --group runtime-standard \
|
||||
--no-emit-project --output-file /tmp/moviepilot-audit-standard.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--require-hashes --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-standard.txt
|
||||
|
||||
uv export --quiet --locked --no-default-groups --group runtime-free-threaded \
|
||||
--no-emit-project --no-hashes \
|
||||
--output-file /tmp/moviepilot-audit-free-threaded.txt
|
||||
python3 scripts/normalize_audit_requirements.py \
|
||||
--lock uv.lock \
|
||||
--input /tmp/moviepilot-audit-free-threaded.txt \
|
||||
--output /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--no-deps --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
frontend_version=$(sed -ne "s/FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/gp" version.py)
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
|
||||
echo "frontend_version=$frontend_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve External Payload Revisions
|
||||
id: payloads
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
FRONTEND_VERSION: ${{ steps.release_version.outputs.frontend_version }}
|
||||
run: |
|
||||
plugins_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Plugins.git refs/heads/main | awk '{print $1}')
|
||||
resources_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Resources.git refs/heads/main | awk '{print $1}')
|
||||
frontend_digest=$(gh api "repos/jxxghp/MoviePilot-Frontend/releases/tags/${FRONTEND_VERSION}" \
|
||||
--jq '.assets[] | select(.name == "dist.zip") | .digest')
|
||||
|
||||
[[ "$plugins_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$resources_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
case "$frontend_digest" in
|
||||
sha256:*) frontend_sha256=${frontend_digest#sha256:} ;;
|
||||
*) echo "dist.zip 缺少 SHA-256 摘要" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "$frontend_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
echo "plugins_revision=$plugins_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "resources_revision=$resources_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_digest=$frontend_digest" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_sha256=$frontend_sha256" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout Wiki Plugin Market
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Wiki
|
||||
ref: main
|
||||
path: .build/moviepilot-wiki
|
||||
sparse-checkout: plugin.md
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Generate Plugin Market Default
|
||||
id: plugin_market
|
||||
run: |
|
||||
python3 -m scripts.generate_plugin_market_default \
|
||||
--wiki-file .build/moviepilot-wiki/plugin.md \
|
||||
--config-file app/runtime/config.py
|
||||
wiki_commit=$(git -C .build/moviepilot-wiki rev-parse HEAD)
|
||||
echo "wiki_commit=$wiki_commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download models.dev catalog
|
||||
id: models_catalog
|
||||
run: |
|
||||
temp_file=$(mktemp app/agent/llm/models.json.XXXXXX)
|
||||
trap 'rm -f "$temp_file"' EXIT
|
||||
curl --fail --show-error --silent --location --retry 3 \
|
||||
--connect-timeout 10 --max-time 120 \
|
||||
"https://models.dev/api.json" -o "$temp_file"
|
||||
jq -e 'type == "object"' "$temp_file" >/dev/null
|
||||
# Git keeps only a small placeholder; the beta image receives the current catalog.
|
||||
chmod 644 "$temp_file"
|
||||
mv "$temp_file" app/agent/llm/models.json
|
||||
echo "digest=sha256:$(sha256sum app/agent/llm/models.json | awk '{print $1}')" >> "$GITHUB_OUTPUT"
|
||||
echo "Downloaded models.dev catalog ($(wc -c < app/agent/llm/models.json) bytes)"
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3
|
||||
ghcr.io/${{ github.repository }}-v3
|
||||
tags: |
|
||||
type=raw,value=beta
|
||||
|
||||
- name: Docker Meta free-threaded
|
||||
id: meta_ft
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3t
|
||||
ghcr.io/${{ github.repository }}-v3t
|
||||
tags: |
|
||||
type=raw,value=beta
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build standard amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan standard amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build standard arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan standard arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish standard multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
labels: |
|
||||
${{ steps.meta.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
|
||||
- name: Publish free-threaded multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta_ft.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
labels: |
|
||||
${{ steps.meta_ft.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
@@ -1,532 +0,0 @@
|
||||
name: MoviePilot Builder v3
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'version.py'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
env:
|
||||
TRIVY_SKIP_DIRS: /usr/share/java
|
||||
TRIVY_SKIP_JAVA_DB_UPDATE: "true"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
|
||||
- name: Verify dependency lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Audit locked Python dependencies
|
||||
run: |
|
||||
uv export --quiet --locked --no-default-groups --group runtime-standard \
|
||||
--no-emit-project --output-file /tmp/moviepilot-audit-standard.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--require-hashes --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-standard.txt
|
||||
|
||||
uv export --quiet --locked --no-default-groups --group runtime-free-threaded \
|
||||
--no-emit-project --no-hashes \
|
||||
--output-file /tmp/moviepilot-audit-free-threaded.txt
|
||||
python3 scripts/normalize_audit_requirements.py \
|
||||
--lock uv.lock \
|
||||
--input /tmp/moviepilot-audit-free-threaded.txt \
|
||||
--output /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--no-deps --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
frontend_version=$(sed -ne "s/FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/gp" version.py)
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
|
||||
echo "frontend_version=$frontend_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve External Payload Revisions
|
||||
id: payloads
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
FRONTEND_VERSION: ${{ steps.release_version.outputs.frontend_version }}
|
||||
run: |
|
||||
plugins_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Plugins.git refs/heads/main | awk '{print $1}')
|
||||
resources_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Resources.git refs/heads/main | awk '{print $1}')
|
||||
frontend_digest=$(gh api "repos/jxxghp/MoviePilot-Frontend/releases/tags/${FRONTEND_VERSION}" \
|
||||
--jq '.assets[] | select(.name == "dist.zip") | .digest')
|
||||
|
||||
[[ "$plugins_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$resources_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
case "$frontend_digest" in
|
||||
sha256:*) frontend_sha256=${frontend_digest#sha256:} ;;
|
||||
*) echo "dist.zip 缺少 SHA-256 摘要" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "$frontend_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
echo "plugins_revision=$plugins_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "resources_revision=$resources_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_digest=$frontend_digest" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_sha256=$frontend_sha256" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout Wiki Plugin Market
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Wiki
|
||||
ref: main
|
||||
path: .build/moviepilot-wiki
|
||||
sparse-checkout: plugin.md
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Generate Plugin Market Default
|
||||
id: plugin_market
|
||||
run: |
|
||||
python3 -m scripts.generate_plugin_market_default \
|
||||
--wiki-file .build/moviepilot-wiki/plugin.md \
|
||||
--config-file app/runtime/config.py
|
||||
wiki_commit=$(git -C .build/moviepilot-wiki rev-parse HEAD)
|
||||
echo "wiki_commit=$wiki_commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download models.dev catalog
|
||||
id: models_catalog
|
||||
run: |
|
||||
temp_file=$(mktemp app/agent/llm/models.json.XXXXXX)
|
||||
trap 'rm -f "$temp_file"' EXIT
|
||||
curl --fail --show-error --silent --location --retry 3 \
|
||||
--connect-timeout 10 --max-time 120 \
|
||||
"https://models.dev/api.json" -o "$temp_file"
|
||||
jq -e 'type == "object"' "$temp_file" >/dev/null
|
||||
# Git keeps only a small placeholder; the release image receives the current catalog.
|
||||
chmod 644 "$temp_file"
|
||||
mv "$temp_file" app/agent/llm/models.json
|
||||
echo "digest=sha256:$(sha256sum app/agent/llm/models.json | awk '{print $1}')" >> "$GITHUB_OUTPUT"
|
||||
echo "Downloaded models.dev catalog ($(wc -c < app/agent/llm/models.json) bytes)"
|
||||
|
||||
- name: Create Release Snapshot
|
||||
id: release_snapshot
|
||||
env:
|
||||
WIKI_COMMIT: ${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add app/runtime/config.py
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "build(plugin-market): sync default from MoviePilot-Wiki@${WIKI_COMMIT:0:12}"
|
||||
fi
|
||||
echo "release_commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3
|
||||
ghcr.io/${{ github.repository }}-v3
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
|
||||
- name: Docker Meta free-threaded
|
||||
id: meta_ft
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3t
|
||||
ghcr.io/${{ github.repository }}-v3t
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
labels: |
|
||||
${{ steps.meta.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.release-snapshot-revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
|
||||
- name: Publish free-threaded multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta_ft.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
labels: |
|
||||
${{ steps.meta_ft.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.release-snapshot-revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
|
||||
- name: Promote latest image pair
|
||||
env:
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
run: |
|
||||
ghcr_repository="${GITHUB_REPOSITORY,,}"
|
||||
for image in \
|
||||
"${DOCKER_USERNAME}/moviepilot-v3" \
|
||||
"ghcr.io/${ghcr_repository}-v3"; do
|
||||
docker buildx imagetools create \
|
||||
--tag "${image}:latest" \
|
||||
"${image}:${app_version}"
|
||||
done
|
||||
for image in \
|
||||
"${DOCKER_USERNAME}/moviepilot-v3t" \
|
||||
"ghcr.io/${ghcr_repository}-v3t"; do
|
||||
docker buildx imagetools create \
|
||||
--tag "${image}:latest" \
|
||||
"${image}:${app_version}"
|
||||
done
|
||||
|
||||
- name: Generate Changelog
|
||||
id: changelog
|
||||
run: |
|
||||
# 获取上一个 tag(优先 v3.*,回退到任意 v* 版本 tag)
|
||||
PREVIOUS_TAG=$(git tag -l 'v3.*' --sort=-v:refname | grep -v "^v${{ env.app_version }}$" | head -n 1)
|
||||
if [ -z "$PREVIOUS_TAG" ]; then
|
||||
PREVIOUS_TAG=$(git tag -l 'v[0-9]*' --sort=-v:refname | grep -v "^v${{ env.app_version }}$" | head -n 1)
|
||||
fi
|
||||
echo "Previous tag: $PREVIOUS_TAG"
|
||||
|
||||
# 使用 || 作为分隔符,同时获取 commit 消息和作者 GitHub 用户名
|
||||
if [ -z "$PREVIOUS_TAG" ]; then
|
||||
# 首次发布且无任何历史版本 tag,限制条数避免打印整库历史撑爆环境变量
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" -n 300 "${SOURCE_COMMIT}")
|
||||
else
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" "${PREVIOUS_TAG}..${SOURCE_COMMIT}")
|
||||
fi
|
||||
|
||||
# 分类收集 commit 消息(使用关联数组去重)
|
||||
declare -A SEEN
|
||||
FEATURES=""
|
||||
FIXES=""
|
||||
OTHERS=""
|
||||
|
||||
while IFS= read -r line; do
|
||||
# 跳过空行
|
||||
if [ -z "$line" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 分离 commit 消息和作者
|
||||
msg=$(echo "$line" | sed 's/||[^|]*$//')
|
||||
author=$(echo "$line" | sed 's/.*||//')
|
||||
|
||||
# 跳过 Merge commit 和版本更新 commit
|
||||
if echo "$msg" | grep -qE "^Merge pull request|^Merge branch|^更新 version"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 按 Conventional Commits 前缀分类
|
||||
if echo "$msg" | grep -qiE "^feat(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^feat(\([^)]*\))?:\s*//')
|
||||
category="FEATURES"
|
||||
elif echo "$msg" | grep -qiE "^fix(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^fix(\([^)]*\))?:\s*//')
|
||||
category="FIXES"
|
||||
elif echo "$msg" | grep -qiE "^(docs|style|refactor|perf|test|build|ci|chore|revert)(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^(docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]*\))?:\s*//')
|
||||
category="OTHERS"
|
||||
else
|
||||
desc="$msg"
|
||||
category="OTHERS"
|
||||
fi
|
||||
|
||||
# 使用 "分类+描述" 作为去重的 key,跳过重复内容
|
||||
dedup_key="${category}::${desc}"
|
||||
if [ -n "${SEEN[$dedup_key]+x}" ]; then
|
||||
continue
|
||||
fi
|
||||
SEEN[$dedup_key]=1
|
||||
|
||||
# 添加 by @author 引用
|
||||
entry="- ${desc} by @${author}"
|
||||
|
||||
case "$category" in
|
||||
FEATURES) FEATURES="${FEATURES}${entry}\n" ;;
|
||||
FIXES) FIXES="${FIXES}${entry}\n" ;;
|
||||
OTHERS) OTHERS="${OTHERS}${entry}\n" ;;
|
||||
esac
|
||||
done <<< "$COMMITS"
|
||||
|
||||
# 组装 changelog
|
||||
CHANGELOG=""
|
||||
|
||||
if [ -n "$FEATURES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### ✨ 新功能\n\n${FEATURES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$FIXES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🐛 修复\n\n${FIXES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$OTHERS" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🔧 其他\n\n${OTHERS}\n"
|
||||
fi
|
||||
|
||||
# 添加版本对比链接
|
||||
if [ -n "$PREVIOUS_TAG" ]; then
|
||||
CHANGELOG="${CHANGELOG}**完整更新记录**: https://github.com/${{ github.repository }}/compare/${PREVIOUS_TAG}...v${{ env.app_version }}"
|
||||
fi
|
||||
|
||||
# 写入环境变量
|
||||
echo "CHANGELOG<<EOF" >> $GITHUB_ENV
|
||||
echo -e "$CHANGELOG" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
|
||||
- name: Get existing release body
|
||||
id: get_release_body
|
||||
continue-on-error: true
|
||||
env:
|
||||
CHANGELOG: ${{ env.CHANGELOG }}
|
||||
run: |
|
||||
release_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.app_version }}" | \
|
||||
jq -r '.body // ""')
|
||||
|
||||
# 如果已有手动编写的 release body,则保留;否则使用自动生成的 changelog
|
||||
if [ -n "$release_body" ] && [ "$release_body" != "null" ] && [ "$release_body" != "" ]; then
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "$release_body" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
else
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "$CHANGELOG" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Delete Release
|
||||
uses: dev-drprasad/delete-tag-and-release@v1.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
delete_release: true
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish Release Tag
|
||||
env:
|
||||
RELEASE_COMMIT: ${{ steps.release_snapshot.outputs.release_commit }}
|
||||
run: |
|
||||
tag_name="v${{ env.app_version }}"
|
||||
if git show-ref --verify --quiet "refs/tags/${tag_name}"; then
|
||||
git tag -d "$tag_name"
|
||||
fi
|
||||
git tag "$tag_name" "$RELEASE_COMMIT"
|
||||
git push origin "refs/tags/${tag_name}"
|
||||
|
||||
- name: Generate Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
name: v${{ env.app_version }}
|
||||
body: ${{ env.RELEASE_BODY }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
make_latest: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -0,0 +1,231 @@
|
||||
name: MoviePilot Builder
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- version.py
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKER_USERNAME }}/moviepilot
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Build Image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
build-args: |
|
||||
MOVIEPILOT_VERSION=${{ env.app_version }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha, scope=${{ github.workflow }}-docker
|
||||
cache-to: type=gha, scope=${{ github.workflow }}-docker
|
||||
|
||||
Windows-build:
|
||||
runs-on: windows-latest
|
||||
name: Build Windows Binary
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Init Python 3.11.4
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11.4'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependent Packages
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install wheel pyinstaller
|
||||
pip install -r requirements.txt
|
||||
shell: pwsh
|
||||
|
||||
- name: Prepare Frontend
|
||||
run: |
|
||||
# 下载nginx
|
||||
Invoke-WebRequest -Uri "http://nginx.org/download/nginx-1.25.2.zip" -OutFile "nginx.zip"
|
||||
Expand-Archive -Path "nginx.zip" -DestinationPath "nginx-1.25.2"
|
||||
Move-Item -Path "nginx-1.25.2/nginx-1.25.2" -Destination "nginx"
|
||||
Remove-Item -Path "nginx.zip"
|
||||
Remove-Item -Path "nginx-1.25.2" -Recurse -Force
|
||||
# 下载前端
|
||||
$FRONTEND_VERSION = (Invoke-WebRequest -Uri "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | ConvertFrom-Json).tag_name
|
||||
Invoke-WebRequest -Uri "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/$FRONTEND_VERSION/dist.zip" -OutFile "dist.zip"
|
||||
Expand-Archive -Path "dist.zip" -DestinationPath "dist"
|
||||
Move-Item -Path "dist/dist/*" -Destination "nginx/html" -Force
|
||||
Remove-Item -Path "dist.zip"
|
||||
Remove-Item -Path "dist" -Recurse -Force
|
||||
Move-Item -Path "nginx/html/nginx.conf" -Destination "nginx/conf/nginx.conf" -Force
|
||||
New-Item -Path "nginx/temp" -ItemType Directory -Force
|
||||
New-Item -Path "nginx/temp/__keep__.txt" -ItemType File -Force
|
||||
New-Item -Path "nginx/logs" -ItemType Directory -Force
|
||||
New-Item -Path "nginx/logs/__keep__.txt" -ItemType File -Force
|
||||
# 下载插件 jxxghp
|
||||
Invoke-WebRequest -Uri "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" -OutFile "MoviePilot-Plugins-main.zip"
|
||||
Expand-Archive -Path "MoviePilot-Plugins-main.zip" -DestinationPath "MoviePilot-Plugins-main"
|
||||
Move-Item -Path "MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/*" -Destination "app/plugins/" -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "MoviePilot-Plugins-main.zip"
|
||||
Remove-Item -Path "MoviePilot-Plugins-main" -Recurse -Force
|
||||
# 下载插件 thsrite
|
||||
Invoke-WebRequest -Uri "https://github.com/thsrite/MoviePilot-Plugins/archive/refs/heads/main.zip" -OutFile "MoviePilot-Plugins-main.zip"
|
||||
Expand-Archive -Path "MoviePilot-Plugins-main.zip" -DestinationPath "MoviePilot-Plugins-main"
|
||||
Move-Item -Path "MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/*" -Destination "app/plugins/" -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "MoviePilot-Plugins-main.zip"
|
||||
Remove-Item -Path "MoviePilot-Plugins-main" -Recurse -Force
|
||||
# 下载插件 honue
|
||||
Invoke-WebRequest -Uri "https://github.com/honue/MoviePilot-Plugins/archive/refs/heads/main.zip" -OutFile "MoviePilot-Plugins-main.zip"
|
||||
Expand-Archive -Path "MoviePilot-Plugins-main.zip" -DestinationPath "MoviePilot-Plugins-main"
|
||||
Move-Item -Path "MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/*" -Destination "app/plugins/" -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "MoviePilot-Plugins-main.zip"
|
||||
Remove-Item -Path "MoviePilot-Plugins-main" -Recurse -Force
|
||||
# 下载插件 InfinityPacer
|
||||
Invoke-WebRequest -Uri "https://github.com/InfinityPacer/MoviePilot-Plugins/archive/refs/heads/main.zip" -OutFile "MoviePilot-Plugins-main.zip"
|
||||
Expand-Archive -Path "MoviePilot-Plugins-main.zip" -DestinationPath "MoviePilot-Plugins-main"
|
||||
Move-Item -Path "MoviePilot-Plugins-main/MoviePilot-Plugins-main/plugins/*" -Destination "app/plugins/" -Force -ErrorAction SilentlyContinue
|
||||
Remove-Item -Path "MoviePilot-Plugins-main.zip"
|
||||
Remove-Item -Path "MoviePilot-Plugins-main" -Recurse -Force
|
||||
# 下载资源
|
||||
Invoke-WebRequest -Uri "https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip" -OutFile "MoviePilot-Resources-main.zip"
|
||||
Expand-Archive -Path "MoviePilot-Resources-main.zip" -DestinationPath "MoviePilot-Resources-main"
|
||||
Move-Item -Path "MoviePilot-Resources-main/MoviePilot-Resources-main/resources/*" -Destination "app/helper/" -Force
|
||||
Remove-Item -Path "MoviePilot-Resources-main.zip"
|
||||
Remove-Item -Path "MoviePilot-Resources-main" -Recurse -Force
|
||||
shell: pwsh
|
||||
|
||||
- name: Pyinstaller
|
||||
run: |
|
||||
pyinstaller frozen.spec
|
||||
shell: pwsh
|
||||
|
||||
- name: Upload Windows File
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: windows
|
||||
path: dist/MoviePilot.exe
|
||||
|
||||
Linux-build-amd64:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Linux Amd64
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Init Python 3.11.4
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11.4'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install Dependent Packages
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install wheel pyinstaller
|
||||
pip install -r requirements.txt
|
||||
find app/plugins -name requirements.txt -exec pip install -r {} \;
|
||||
|
||||
- name: Prepare Frontend
|
||||
run: |
|
||||
wget https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip
|
||||
unzip main.zip
|
||||
mv MoviePilot-Plugins-main/plugins/* app/plugins/
|
||||
rm main.zip
|
||||
rm -rf MoviePilot-Plugins-main
|
||||
|
||||
wget https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip
|
||||
unzip main.zip
|
||||
mv MoviePilot-Resources-main/resources/* app/helper/
|
||||
rm main.zip
|
||||
rm -rf MoviePilot-Resources-main
|
||||
|
||||
- name: Pyinstaller
|
||||
run: |
|
||||
pyinstaller frozen.spec
|
||||
mv dist/MoviePilot dist/MoviePilot_Amd64
|
||||
|
||||
- name: Upload Linux File
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: linux-amd64
|
||||
path: dist/MoviePilot_Amd64
|
||||
|
||||
Create-release:
|
||||
permissions: write-all
|
||||
runs-on: ubuntu-latest
|
||||
needs: [ Windows-build, Docker-build, Linux-build-amd64]
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Release Version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
|
||||
- name: Download Artifact
|
||||
uses: actions/download-artifact@v3
|
||||
|
||||
- name: get release_informations
|
||||
shell: bash
|
||||
run: |
|
||||
mkdir releases
|
||||
mv ./windows/MoviePilot.exe ./releases/MoviePilot_Win_v${{ env.app_version }}.exe
|
||||
mv ./linux-amd64/MoviePilot_Amd64 ./releases/MoviePilot_Amd64_v${{ env.app_version }}
|
||||
|
||||
- name: Create Release
|
||||
id: create_release
|
||||
uses: actions/create-release@latest
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
release_name: v${{ env.app_version }}
|
||||
body: ${{ github.event.commits[0].message }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
|
||||
- name: Upload Release Asset
|
||||
uses: dwenegar/upload-release-assets@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
release_id: ${{ steps.create_release.outputs.id }}
|
||||
assets_path: |
|
||||
./releases/
|
||||
@@ -1,183 +0,0 @@
|
||||
name: Dependency Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- 'app/doctor/dependencies.py'
|
||||
- 'app/foundation/environment.py'
|
||||
- 'app/runtime/dependencies.py'
|
||||
- 'docker/Dockerfile'
|
||||
- 'docker/**'
|
||||
- '.github/workflows/dependency-compat.yml'
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- 'app/doctor/dependencies.py'
|
||||
- 'app/foundation/environment.py'
|
||||
- 'app/runtime/dependencies.py'
|
||||
- 'docker/Dockerfile'
|
||||
- 'docker/**'
|
||||
- '.github/workflows/dependency-compat.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: dependency-compat-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
install:
|
||||
name: ${{ matrix.name }} / Python ${{ matrix.python-version }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: Linux x64
|
||||
runner: ubuntu-24.04
|
||||
python-version: '3.14'
|
||||
expected-system: Linux
|
||||
expected-machine: x86_64
|
||||
- name: Linux ARM64
|
||||
runner: ubuntu-24.04-arm
|
||||
python-version: '3.14'
|
||||
expected-system: Linux
|
||||
expected-machine: aarch64
|
||||
- name: macOS Intel
|
||||
runner: macos-15-intel
|
||||
python-version: '3.14'
|
||||
expected-system: Darwin
|
||||
expected-machine: x86_64
|
||||
- name: macOS ARM
|
||||
runner: macos-15
|
||||
python-version: '3.14'
|
||||
expected-system: Darwin
|
||||
expected-machine: arm64
|
||||
- name: Windows x64
|
||||
runner: windows-2025
|
||||
python-version: '3.14'
|
||||
expected-system: Windows
|
||||
expected-machine: AMD64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: ${{ matrix.python-version }}
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install locked runtime dependencies
|
||||
run: uv sync --locked --inexact --no-dev --python ${{ matrix.python-version }}
|
||||
|
||||
- name: Verify environment and core imports
|
||||
env:
|
||||
EXPECTED_SYSTEM: ${{ matrix.expected-system }}
|
||||
EXPECTED_MACHINE: ${{ matrix.expected-machine }}
|
||||
run: >-
|
||||
uv run --locked --no-sync python -c
|
||||
"import os, platform;
|
||||
assert platform.system() == os.environ['EXPECTED_SYSTEM'], (platform.system(), os.environ['EXPECTED_SYSTEM']);
|
||||
assert platform.machine() == os.environ['EXPECTED_MACHINE'], (platform.machine(), os.environ['EXPECTED_MACHINE']);
|
||||
import alembic, fastapi, pydantic, pydantic_settings, sqlalchemy, starlette, uvicorn"
|
||||
|
||||
- name: Verify locked project consistency
|
||||
run: uv sync --locked --offline --inexact --no-dev --check --python ${{ matrix.python-version }}
|
||||
|
||||
docker-dependencies:
|
||||
name: Docker dependencies / ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-24.04
|
||||
platform: linux/amd64
|
||||
python-variant: standard
|
||||
cache-scope: linux-amd64-standard
|
||||
image-tag: moviepilot-dependency-gate:linux-amd64-standard
|
||||
expected-machine: x86_64
|
||||
- runner: ubuntu-24.04-arm
|
||||
platform: linux/arm64
|
||||
python-variant: standard
|
||||
cache-scope: linux-arm64-standard
|
||||
image-tag: moviepilot-dependency-gate:linux-arm64-standard
|
||||
expected-machine: aarch64
|
||||
- runner: ubuntu-24.04
|
||||
platform: linux/amd64
|
||||
python-variant: free-threaded
|
||||
cache-scope: linux-amd64-free-threaded
|
||||
image-tag: moviepilot-dependency-gate:linux-amd64-free-threaded
|
||||
expected-machine: x86_64
|
||||
- runner: ubuntu-24.04-arm
|
||||
platform: linux/arm64
|
||||
python-variant: free-threaded
|
||||
cache-scope: linux-arm64-free-threaded
|
||||
image-tag: moviepilot-dependency-gate:linux-arm64-free-threaded
|
||||
expected-machine: aarch64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build locked dependency stage
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
target: prepare_venv
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
MOVIEPILOT_PYTHON_VARIANT=${{ matrix.python-variant }}
|
||||
load: true
|
||||
push: false
|
||||
tags: ${{ matrix.image-tag }}
|
||||
cache-from: type=gha,scope=dependency-compat-${{ matrix.cache-scope }}
|
||||
cache-to: type=gha,scope=dependency-compat-${{ matrix.cache-scope }},mode=max
|
||||
|
||||
- name: Verify dependency image
|
||||
env:
|
||||
IMAGE_TAG: ${{ matrix.image-tag }}
|
||||
EXPECTED_MACHINE: ${{ matrix.expected-machine }}
|
||||
EXPECTED_VARIANT: ${{ matrix.python-variant }}
|
||||
run: >-
|
||||
docker run --rm
|
||||
-e EXPECTED_MACHINE
|
||||
-e EXPECTED_VARIANT
|
||||
"${IMAGE_TAG}"
|
||||
/opt/venv/bin/python -c
|
||||
"import os, platform, sys, sysconfig;
|
||||
assert platform.machine() == os.environ['EXPECTED_MACHINE'], (platform.machine(), os.environ['EXPECTED_MACHINE']);
|
||||
expected_free_threaded = os.environ['EXPECTED_VARIANT'] == 'free-threaded';
|
||||
assert (sysconfig.get_config_var('Py_GIL_DISABLED') == 1) is expected_free_threaded;
|
||||
assert sys._is_gil_enabled() == (not expected_free_threaded);
|
||||
import alembic, fastapi, moviepilot_rust, pydantic, pydantic_settings, sqlalchemy, starlette, uvicorn;
|
||||
assert moviepilot_rust.is_available();
|
||||
assert moviepilot_rust.jieba_cut('中文分词');
|
||||
assert not expected_free_threaded or callable(moviepilot_rust.zhconv_fast);
|
||||
assert sys._is_gil_enabled() == (not expected_free_threaded)"
|
||||
|
||||
- name: Verify pinned uv version
|
||||
env:
|
||||
IMAGE_TAG: ${{ matrix.image-tag }}
|
||||
run: docker run --rm "${IMAGE_TAG}" uv --version | grep -F 'uv 0.12.5'
|
||||
@@ -1,158 +0,0 @@
|
||||
name: Close inactive issues
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
issues:
|
||||
types: [opened, edited]
|
||||
|
||||
schedule:
|
||||
# Github Action 只支持 UTC 时间。
|
||||
# '0 18 * * *' 对应 UTC 时间的 18:00,也就是中国时区 (UTC+8) 的第二天凌晨 02:00。
|
||||
- cron: "0 18 * * *"
|
||||
|
||||
jobs:
|
||||
label-opened-issue:
|
||||
if: github.event_name == 'issues'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const issue = context.payload.issue;
|
||||
const title = issue.title || '';
|
||||
const body = issue.body || '';
|
||||
const currentLabels = (issue.labels || []).map((label) => label.name);
|
||||
|
||||
// 网页 Issue Form 已经会自动带模板 labels;这里只兜底处理
|
||||
// API 创建或异常路径产生的无 label issue,避免重复补标。
|
||||
if (currentLabels.length > 0) {
|
||||
core.info(`Issue #${issue.number} already has labels: ${currentLabels.join(', ')}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const hasAllMarkers = (markers) => markers.every((marker) => body.includes(marker));
|
||||
const labelRules = [
|
||||
{
|
||||
label: 'bug',
|
||||
titlePrefix: '[错误报告]:',
|
||||
markers: ['### 当前程序版本', '### 运行环境', '### 问题类型', '### 问题描述'],
|
||||
},
|
||||
{
|
||||
label: 'feature request',
|
||||
titlePrefix: '[Feature Request]:',
|
||||
markers: ['### 当前程序版本', '### 运行环境', '### 功能改进类型', '### 功能改进'],
|
||||
},
|
||||
{
|
||||
label: 'RFC',
|
||||
titlePrefix: '[RFC]',
|
||||
markers: ['### 背景 or 问题', '### 目标 & 方案简述'],
|
||||
},
|
||||
];
|
||||
|
||||
const matched = labelRules.find((rule) => (
|
||||
title.startsWith(rule.titlePrefix) || hasAllMarkers(rule.markers)
|
||||
));
|
||||
|
||||
if (!matched) {
|
||||
core.info(`Issue #${issue.number} does not match known issue templates.`);
|
||||
return;
|
||||
}
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: [matched.label],
|
||||
});
|
||||
core.info(`Added label "${matched.label}" to issue #${issue.number}.`);
|
||||
|
||||
label-unlabeled-issues:
|
||||
if: github.event_name != 'issues'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const labelRules = [
|
||||
{
|
||||
label: 'bug',
|
||||
titlePrefix: '[错误报告]:',
|
||||
markers: ['### 当前程序版本', '### 运行环境', '### 问题类型', '### 问题描述'],
|
||||
},
|
||||
{
|
||||
label: 'feature request',
|
||||
titlePrefix: '[Feature Request]:',
|
||||
markers: ['### 当前程序版本', '### 运行环境', '### 功能改进类型', '### 功能改进'],
|
||||
},
|
||||
{
|
||||
label: 'RFC',
|
||||
titlePrefix: '[RFC]',
|
||||
markers: ['### 背景 or 问题', '### 目标 & 方案简述'],
|
||||
},
|
||||
];
|
||||
|
||||
const hasAllMarkers = (body, markers) => markers.every((marker) => body.includes(marker));
|
||||
const getMatchedRule = (issue) => {
|
||||
const title = issue.title || '';
|
||||
const body = issue.body || '';
|
||||
return labelRules.find((rule) => (
|
||||
title.startsWith(rule.titlePrefix) || hasAllMarkers(body, rule.markers)
|
||||
));
|
||||
};
|
||||
|
||||
// Search API 支持 no:label 查询;issues.listForRepo 的 labels=none
|
||||
// 会被当作名为 none 的标签,不能用于扫描无 label issue。
|
||||
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open no:label`;
|
||||
for await (const response of github.paginate.iterator(github.rest.search.issuesAndPullRequests, {
|
||||
q: query,
|
||||
per_page: 100,
|
||||
})) {
|
||||
for (const issue of response.data) {
|
||||
if (issue.pull_request) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const matched = getMatchedRule(issue);
|
||||
if (!matched) {
|
||||
continue;
|
||||
}
|
||||
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: [matched.label],
|
||||
});
|
||||
core.info(`Added label "${matched.label}" to issue #${issue.number}.`);
|
||||
}
|
||||
}
|
||||
|
||||
close-issues:
|
||||
if: github.event_name != 'issues'
|
||||
needs: label-unlabeled-issues
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@v5
|
||||
with:
|
||||
# 标记 stale 标签时间
|
||||
days-before-issue-stale: 30
|
||||
# 关闭 issues 标签时间
|
||||
days-before-issue-close: 14
|
||||
# 自定义标签名
|
||||
stale-issue-label: "stale"
|
||||
stale-issue-message: "此问题已过时,因为它已打开 30 天且没有任何活动。"
|
||||
close-issue-message: "此问题已关闭,因为它在标记为 stale 后,已处于无更新状态 14 天。"
|
||||
# 忽略所有的 Pull Request,只处理 Issue
|
||||
days-before-pr-stale: -1
|
||||
days-before-pr-close: -1
|
||||
# 排除带有RFC标签的issue
|
||||
exempt-issue-labels: "RFC"
|
||||
operations-per-run: 500
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -1,52 +0,0 @@
|
||||
name: PR-Agent
|
||||
|
||||
on:
|
||||
pull_request_target:
|
||||
# Fork 审查需要目标仓库凭据;该 job 仅通过 GitHub API 读取 PR 内容,不 checkout 或执行 PR 分支代码。
|
||||
types:
|
||||
- opened
|
||||
- reopened
|
||||
- ready_for_review
|
||||
- review_requested
|
||||
- synchronize
|
||||
issue_comment:
|
||||
types:
|
||||
- created
|
||||
- edited
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
pr-agent:
|
||||
if: >-
|
||||
github.event.sender.type != 'Bot' &&
|
||||
(
|
||||
github.event_name == 'pull_request_target' ||
|
||||
(
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.issue.pull_request != null &&
|
||||
contains(fromJSON('["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR"]'), github.event.comment.author_association)
|
||||
)
|
||||
)
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.event.issue.number }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request_target' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Run PR Review
|
||||
uses: docker://ghcr.io/infinitypacer/pr-review-runner:latest
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
OPENAI_KEY: ${{ secrets.OPENAI_KEY }}
|
||||
OPENAI.API_BASE: ${{ secrets.OPENAI_API_BASE }}
|
||||
PRR_AUTO_REVIEW_SCOPE: all
|
||||
PRR_ALLOWED_ASSOCIATIONS: '["OWNER", "MEMBER", "COLLABORATOR", "CONTRIBUTOR", "FIRST_TIME_CONTRIBUTOR"]'
|
||||
PRR_DISABLED_COMMANDS: '["/improve"]'
|
||||
PRR_SKIP_LABEL: skip pr-agent
|
||||
PRR_SKIP_TITLE_PATTERN: '^(?:\[Auto\]|Auto)'
|
||||
config.response_language: zh-CN
|
||||
@@ -1,108 +0,0 @@
|
||||
name: Pylint Code Quality Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: pylint-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pylint:
|
||||
runs-on: ubuntu-latest
|
||||
name: Pylint Code Quality Check
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Verify pylint config
|
||||
run: |
|
||||
# 检查项目中的pylint配置文件是否存在
|
||||
if [ -f .pylintrc ]; then
|
||||
echo "✅ 找到项目配置文件: .pylintrc"
|
||||
echo "配置文件内容预览:"
|
||||
head -10 .pylintrc
|
||||
else
|
||||
echo "❌ 未找到 .pylintrc 配置文件"
|
||||
exit 1
|
||||
fi
|
||||
- name: Collect changed Python files
|
||||
id: changed
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
BEFORE_SHA: ${{ github.event.before }}
|
||||
CURRENT_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
if [[ "$EVENT_NAME" == "pull_request" ]]; then
|
||||
git diff --name-only --diff-filter=ACMRT \
|
||||
"origin/$BASE_REF...HEAD" -- '*.py' > changed-python-files.txt
|
||||
elif [[ "$EVENT_NAME" == "push" ]] \
|
||||
&& [[ -n "$BEFORE_SHA" ]] \
|
||||
&& [[ ! "$BEFORE_SHA" =~ ^0+$ ]] \
|
||||
&& git cat-file -e "$BEFORE_SHA^{commit}"; then
|
||||
git diff --name-only --diff-filter=ACMRT \
|
||||
"$BEFORE_SHA" "$CURRENT_SHA" -- '*.py' > changed-python-files.txt
|
||||
else
|
||||
git diff-tree --no-commit-id --name-only --diff-filter=ACMRT \
|
||||
-r HEAD -- '*.py' > changed-python-files.txt
|
||||
fi
|
||||
sort -u -o changed-python-files.txt changed-python-files.txt
|
||||
if [[ -s changed-python-files.txt ]]; then
|
||||
echo "has_files=true" >> "$GITHUB_OUTPUT"
|
||||
sed -n '1,200p' changed-python-files.txt
|
||||
else
|
||||
echo "has_files=false" >> "$GITHUB_OUTPUT"
|
||||
echo "本次没有改动 Python 文件"
|
||||
fi
|
||||
|
||||
- name: Run pylint on changed Python files
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: |
|
||||
xargs uv run --locked --no-sync pylint \
|
||||
--output-format=colorized --reports=yes --score=yes \
|
||||
< changed-python-files.txt
|
||||
|
||||
- name: Generate full advisory report
|
||||
if: always()
|
||||
run: |
|
||||
uv run --locked --no-sync pylint app/ \
|
||||
--output-format=json > pylint-report.json || true
|
||||
|
||||
- name: Upload pylint report
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: pylint-report
|
||||
path: pylint-report.json
|
||||
|
||||
- name: Summary
|
||||
run: |
|
||||
echo "🎉 Pylint 检查完成!"
|
||||
echo "✅ 改动 Python 文件没有新增语法错误或严重问题"
|
||||
echo "📊 全仓建议性报告已保存为构建工件"
|
||||
@@ -1,137 +0,0 @@
|
||||
name: Site Adapter Collector
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: Existing release tag to receive collector assets; leave empty for artifacts only
|
||||
required: false
|
||||
type: string
|
||||
release:
|
||||
types:
|
||||
- published
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build ${{ matrix.platform_name }} collector
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- platform_name: Windows
|
||||
platform_id: windows
|
||||
runner: windows-latest
|
||||
source_name: moviepilot-site-collector.exe
|
||||
asset_name: moviepilot-site-collector-windows.exe
|
||||
artifact_name: site-adapter-collector-windows
|
||||
- platform_name: macOS
|
||||
platform_id: macos
|
||||
runner: macos-latest
|
||||
source_name: moviepilot-site-collector
|
||||
asset_name: MoviePilot-Site-Collector-macOS.zip
|
||||
artifact_name: site-adapter-collector-macos
|
||||
- platform_name: Linux
|
||||
platform_id: linux
|
||||
runner: ubuntu-latest
|
||||
source_name: moviepilot-site-collector
|
||||
asset_name: moviepilot-site-collector-linux
|
||||
artifact_name: site-adapter-collector-linux
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Install build dependencies
|
||||
run: uv pip install --system --requirement scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Build single-file collector
|
||||
run: |
|
||||
pyinstaller --clean --noconfirm scripts/site_adapter_collector.spec
|
||||
|
||||
- name: Smoke-test collector
|
||||
env:
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
run: |
|
||||
python -c "import os, subprocess; from pathlib import Path; subprocess.run([str((Path('dist') / os.environ['SOURCE_NAME']).resolve()), '--help'], check=True)"
|
||||
|
||||
- name: Package macOS double-click archive
|
||||
if: matrix.platform_id == 'macos'
|
||||
shell: bash
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
run: |
|
||||
package_dir="dist/MoviePilot-Collector"
|
||||
mkdir -p "$package_dir"
|
||||
cp "dist/$SOURCE_NAME" "$package_dir/moviepilot-site-collector-macos"
|
||||
cp scripts/start-site-adapter-collector.command "$package_dir/start-site-adapter-collector.command"
|
||||
chmod +x "$package_dir/moviepilot-site-collector-macos"
|
||||
chmod +x "$package_dir/start-site-adapter-collector.command"
|
||||
cd dist
|
||||
COPYFILE_DISABLE=1 zip -q -r -X "$ASSET_NAME" MoviePilot-Collector
|
||||
|
||||
- name: Rename Windows and Linux collector
|
||||
if: matrix.platform_id != 'macos'
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
SOURCE_NAME: ${{ matrix.source_name }}
|
||||
run: |
|
||||
python -c "import os; from pathlib import Path; (Path('dist') / os.environ['SOURCE_NAME']).replace(Path('dist') / os.environ['ASSET_NAME'])"
|
||||
|
||||
- name: Generate SHA-256 checksum
|
||||
env:
|
||||
ASSET_NAME: ${{ matrix.asset_name }}
|
||||
run: |
|
||||
python -c "import hashlib, os; from pathlib import Path; path = Path('dist') / os.environ['ASSET_NAME']; path.with_name(path.name + '.sha256').write_text(f'{hashlib.sha256(path.read_bytes()).hexdigest()} {path.name}\n', encoding='utf-8')"
|
||||
|
||||
- name: Upload collector artifact
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ${{ matrix.artifact_name }}
|
||||
path: |
|
||||
dist/${{ matrix.asset_name }}
|
||||
dist/${{ matrix.asset_name }}.sha256
|
||||
if-no-files-found: error
|
||||
retention-days: 3
|
||||
|
||||
publish:
|
||||
name: Upload collectors to release
|
||||
if: github.event_name == 'release' || inputs.release_tag != ''
|
||||
needs:
|
||||
- build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
steps:
|
||||
- name: Download collector artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
pattern: site-adapter-collector-*
|
||||
path: release-assets
|
||||
merge-multiple: true
|
||||
|
||||
- name: Upload assets to published release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name || inputs.release_tag }}
|
||||
run: |
|
||||
gh release view "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" >/dev/null
|
||||
gh release upload "$RELEASE_TAG" release-assets/* --clobber --repo "$GITHUB_REPOSITORY"
|
||||
@@ -1,156 +0,0 @@
|
||||
name: Unit Tests
|
||||
|
||||
on:
|
||||
# 指向 v3 的 PR 与推送都跑全量单测,作为合并门禁
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
# 允许手动触发
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: unit-tests-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
architecture:
|
||||
runs-on: ubuntu-latest
|
||||
name: Architecture Contract Gate
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Check host architecture contracts
|
||||
run: |
|
||||
uv run --locked --no-sync pytest \
|
||||
tests/test_architecture_dependencies.py \
|
||||
tests/test_architecture_adapter_imports.py \
|
||||
tests/test_architecture_egress.py \
|
||||
tests/test_architecture_contract_baseline.py \
|
||||
tests/test_architecture_baseline_cli.py -q
|
||||
uv run --locked --no-sync python \
|
||||
scripts/architecture/baseline.py --check-host
|
||||
|
||||
- name: Check governed Python types
|
||||
run: uv run --locked --no-sync mypy --config-file mypy.ini
|
||||
|
||||
- name: Check complexity ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/complexity.py
|
||||
|
||||
- name: Check async blocking ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/async_blocking.py
|
||||
|
||||
- name: Check background task ownership
|
||||
run: uv run --locked --no-sync python scripts/architecture/task_ownership.py
|
||||
|
||||
- name: Check process runtime service locators
|
||||
run: uv run --locked --no-sync python scripts/architecture/service_locator.py
|
||||
|
||||
- name: Check Ruff diagnostic ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/ruff_ratchet.py
|
||||
|
||||
- name: Check mypy error ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/mypy_ratchet.py
|
||||
|
||||
- name: Check startup performance contract
|
||||
run: >-
|
||||
uv run --locked --no-sync python
|
||||
scripts/startup/performance.py --check --repeat 3
|
||||
|
||||
pytest:
|
||||
runs-on: ubuntu-latest
|
||||
name: Unit Tests (${{ matrix.shard }})
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- shard: '1/4'
|
||||
- shard: '2/4'
|
||||
- shard: '3/4'
|
||||
- shard: '4/4'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Run tests
|
||||
timeout-minutes: 10
|
||||
run: uv run --locked --no-sync python tests/run.py --shard "${{ matrix.shard }}"
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
name: Coverage Report
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Generate coverage reports
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
uv run --locked --no-sync python -m coverage erase
|
||||
uv run --locked --no-sync python -m coverage run tests/run.py --serial
|
||||
uv run --locked --no-sync python -m coverage report
|
||||
uv run --locked --no-sync python -m coverage json
|
||||
uv run --locked --no-sync python -m coverage xml
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: coverage-report
|
||||
path: |
|
||||
coverage.xml
|
||||
coverage.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Check coverage ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/coverage_ratchet.py
|
||||
+7
-38
@@ -1,51 +1,20 @@
|
||||
.idea/
|
||||
.DS_Store
|
||||
*.c
|
||||
*.so
|
||||
*.pyd
|
||||
build/
|
||||
cython_cache/
|
||||
dist/
|
||||
rust/**/target/
|
||||
nginx/
|
||||
test.py
|
||||
app/application/site/*.bin
|
||||
# 站点数据的运行期下载产物。上游 v3 架构重构后落点从 app/application/site 移到了
|
||||
# app/helper,同目录的 .so/.pyd 由上面的通配兜住,只有 .bin 漏了网
|
||||
app/helper/sites.py
|
||||
app/helper/*.so
|
||||
app/helper/*.pyd
|
||||
app/helper/*.bin
|
||||
app/plugins/**
|
||||
!app/plugins/__init__.py
|
||||
config/*
|
||||
!config/category.yaml
|
||||
# 运行期设置持久化目录(settings 写回 app.env 的落点)与本地验证产物
|
||||
app/config/
|
||||
.verify_tmp/
|
||||
.artifacts/
|
||||
.runtime/
|
||||
public/
|
||||
.moviepilot.env
|
||||
config/cookies/**
|
||||
config/user.db
|
||||
config/sites/**
|
||||
*.pyc
|
||||
*.log
|
||||
.coverage
|
||||
coverage.xml
|
||||
coverage.json
|
||||
htmlcov/
|
||||
.vscode
|
||||
venv
|
||||
moviepilot-site-capture-*.zip
|
||||
|
||||
# Pylint
|
||||
pylint-report.json
|
||||
.pylint.d/
|
||||
|
||||
# AI
|
||||
.claude/
|
||||
!.claude/*.json
|
||||
.claude/settings.local.json
|
||||
|
||||
# Superpowers 设计/计划文档(本地协作产物,不纳入仓库)
|
||||
docs/superpowers/
|
||||
|
||||
# 保留本地前端构建产物目录,目录内产物不纳入仓库
|
||||
frontend-dist/*
|
||||
!frontend-dist/.gitkeep
|
||||
.DS_Store
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
[MASTER]
|
||||
# 指定Python路径
|
||||
init-hook='import sys; sys.path.append(".")'
|
||||
|
||||
# 忽略的文件和目录
|
||||
ignore=.git,__pycache__,.venv,build,dist,tests,docs
|
||||
|
||||
# 通过 `pylint app/` 检查主程序时不扫描内置插件目录,
|
||||
# 插件依赖和动态模型较多,容易产生与主程序无关的误报。
|
||||
ignore-paths=^app/plugins(/|$)
|
||||
|
||||
# 并行作业数量
|
||||
jobs=0
|
||||
|
||||
[MESSAGES CONTROL]
|
||||
# 只启用确定性较强的严重问题检查,避免 SQLAlchemy、FastAPI 依赖注入、
|
||||
# 第三方 SDK 等动态对象被 Pylint 推断成误报。
|
||||
disable=all
|
||||
enable=syntax-error,
|
||||
undefined-variable,
|
||||
used-before-assignment,
|
||||
possibly-used-before-assignment,
|
||||
unreachable,
|
||||
return-outside-function,
|
||||
yield-outside-function,
|
||||
continue-in-finally,
|
||||
nonlocal-without-binding,
|
||||
undefined-loop-variable,
|
||||
import-error,
|
||||
relative-beyond-top-level,
|
||||
no-name-in-module
|
||||
|
||||
[REPORTS]
|
||||
# 设置报告格式
|
||||
output-format=colorized
|
||||
reports=yes
|
||||
score=yes
|
||||
|
||||
[FORMAT]
|
||||
# 最大行长度
|
||||
max-line-length=120
|
||||
# 缩进大小
|
||||
indent-string=' '
|
||||
|
||||
[DESIGN]
|
||||
# 最大参数数量
|
||||
max-args=10
|
||||
# 最大本地变量数量
|
||||
max-locals=20
|
||||
# 最大分支数量
|
||||
max-branches=15
|
||||
# 最大语句数量
|
||||
max-statements=50
|
||||
# 最大父类数量
|
||||
max-parents=7
|
||||
# 最大属性数量
|
||||
max-attributes=10
|
||||
# 最小公共方法数量
|
||||
min-public-methods=1
|
||||
# 最大公共方法数量
|
||||
max-public-methods=25
|
||||
|
||||
[SIMILARITIES]
|
||||
# 最小相似行数
|
||||
min-similarity-lines=6
|
||||
# 忽略注释
|
||||
ignore-comments=yes
|
||||
# 忽略文档字符串
|
||||
ignore-docstrings=yes
|
||||
# 忽略导入
|
||||
ignore-imports=yes
|
||||
|
||||
[TYPECHECK]
|
||||
# 生成缺失成员提示的类列表
|
||||
generated-members=requests.packages.urllib3
|
||||
# app.infrastructure.sites 会主动隐藏模块属性枚举,接口由同目录 sites.pyi 声明
|
||||
ignored-modules=app.infrastructure.sites
|
||||
@@ -1,71 +0,0 @@
|
||||
vulnerabilities:
|
||||
- id: GHSA-6v7p-g79w-8964
|
||||
paths:
|
||||
- Python
|
||||
purls:
|
||||
- pkg:pypi/msgpack@1.1.2
|
||||
expired_at: 2026-11-20
|
||||
statement: The finding belongs to the base image's system pip and is not imported by MoviePilot.
|
||||
- id: CVE-2025-47273
|
||||
paths:
|
||||
- Python
|
||||
purls:
|
||||
- pkg:pypi/setuptools@70.3.0
|
||||
expired_at: 2026-11-20
|
||||
statement: The finding belongs to the base image's system pip and is not used for dependency installation.
|
||||
- id: CVE-2026-33818
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-39821
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-46600
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56853
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56858
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56859
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56860
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56862
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
@@ -1,162 +0,0 @@
|
||||
# AGENTS.md
|
||||
|
||||
This file is the primary instruction set for all AI agents and LLMs working in this repository. Local documentation takes precedence over general training data. You must follow this file and the rule documents it references.
|
||||
|
||||
---
|
||||
|
||||
## Task-to-Documentation Mapping
|
||||
|
||||
For work that changes or reviews repository behavior, identify the domains actually touched and load only the applicable documents. Simple factual checks and unrelated domains do not require preloading rule files.
|
||||
|
||||
### Architectural Decisions
|
||||
* **Primary Reference:** `docs/rules/05-architecture.md`
|
||||
* **Required Constraints:** Respect layer boundaries and dependency flow. Do not introduce circular dependencies. Verify the correct layer for any new capability before implementing.
|
||||
|
||||
### Business Logic and Design Patterns
|
||||
* **Primary Reference:** `docs/rules/04-design-patterns.md`
|
||||
* **Required Constraints:** Use the project's established Module, Chain, Event, and Oper structural patterns. Do not introduce abstractions the project has not adopted.
|
||||
|
||||
### Coding Standards and Style
|
||||
* **Primary Reference:** `docs/rules/06-code-styles.md`
|
||||
* **Required Constraints:** Match the style of the surrounding file. Type annotations, Pydantic models, and async/await usage must all conform to the documented standards.
|
||||
|
||||
### Identifiers and Naming
|
||||
* **Primary Reference:** `docs/rules/07-naming-conventions.md`
|
||||
* **Required Constraints:** All filenames, class names, function names, and constants must follow the project's taxonomy. No arbitrary abbreviations or mixed casing styles.
|
||||
|
||||
### Comments and Documentation
|
||||
* **Primary Reference:** `docs/rules/08-comment-styles.md`
|
||||
* **Required Constraints:** Public or cross-module contracts and non-obvious business behavior require concise Chinese docstrings. Small self-evident private helpers and test scaffolding may omit them. Comments must explain the *why*, not restate the code.
|
||||
|
||||
### External Communication and Interfaces
|
||||
* **Primary Reference:** `docs/rules/09-external-response.md`
|
||||
* **Required Constraints:** Host-authored ordinary HTTP must go through `RequestUtils`; this rule does not authorize Application/Chain to import the concrete Adapter. Canonical transport, SDK, streaming protocol, contained vendor, diagnostic and control-plane exceptions must match the exact direct-egress policy. Response formats must use the project's standard schemas. Error handling must follow the per-layer conventions.
|
||||
|
||||
### Data and Persistence
|
||||
* **Primary Reference:** `docs/rules/10-data-and-persistent.md`
|
||||
* **Required Constraints:** Any database model change requires a matching Alembic migration. Runtime configuration must be managed via `SystemConfigKey` + `SystemConfigOper`. Raw string keys are forbidden.
|
||||
|
||||
### Quality and Security
|
||||
* **Primary Reference:** `docs/rules/11-quality-and-security.md`
|
||||
* **Required Constraints:** All code changes must pass the relevant pytest tests and pylint checks. Dependency changes require a current `uv.lock`, locked environment verification, and a passing locked dependency vulnerability audit.
|
||||
|
||||
### Testing
|
||||
* **Primary Reference:** `docs/testing.md`
|
||||
* **Required Constraints:** pytest is the only runner; `tests/conftest.py` isolates each run to a temporary `CONFIG_DIR`. Tests must not touch the real database, network, or external services (TMDB, LLM catalogs, downloaders, media servers, MP server) — mock at the boundary or replay recorded responses; the bar is zero real outbound traffic. Tests must restore any process-level state they stub (`sys.modules`, singletons, caches, settings). New tests must be pytest-native (function + `assert` + fixtures); do not add new `unittest.TestCase`. Convert existing `TestCase` files to pytest-native opportunistically when you modify them. Before opening a PR to `v3`, run the affected tests and applicable local checks. Run the full local suite (`uv run --locked --no-sync python tests/run.py`) for dependency or lock changes, shared test infrastructure, database or startup paths, cross-module lifecycle, compatibility layers, broad behavior changes, or an explicit maintainer requirement. The changed path must pass; any unrelated failure must be reported and reproduced against the current `upstream/v3` baseline instead of silently expanding the PR. Documentation-only changes use applicable text and structure checks; the `.github/workflows/test.yml` gate remains the final full-suite check on every PR/push to `v3`.
|
||||
|
||||
### Commands and Development Workflow
|
||||
* **Primary Reference:** `docs/rules/03-commands.md`
|
||||
* **Required Constraints:** Use that file as the project command reference. Other standard inspection, Git, GitHub, and focused verification commands are allowed when they are necessary, scoped, and consistent with current authorization.
|
||||
|
||||
---
|
||||
|
||||
## Canonical Package Ownership
|
||||
|
||||
The historical `app/core`, `app/helper`, and `app/utils` directories are compatibility-only virtual import roots. Never add physical Python source there and never use those imports from host code. Choose an owner by responsibility, not by whether a function is "shared" or has historically been called a helper.
|
||||
|
||||
The legacy roots have no physical directories in the source tree. Current images and update flows write site resources only to `app/application/site/`; plugin imports under `app.helper.*` are resolved exclusively by the exact runtime compatibility manifest.
|
||||
|
||||
| Package | Owns | Must Not Own | Representative Files |
|
||||
|---|---|---|---|
|
||||
| `app/foundation/` | 无状态、无配置和无 I/O 的底层机制:反射/动态导入、加密、DOM、身份、集合、单例、文本、URL 和版本比较 | `settings`、DB/SystemConfig、网络请求、运行日志、MoviePilot 业务规则、旧导入路径 | `reflection.py`, `crypto.py`, `collections.py`, `text.py`, `url.py` |
|
||||
| `app/domain/` | Pure MoviePilot business semantics and models for media, recognition, sites, and torrents | Persistence, global settings reads, network/filesystem clients, Rust imports, service discovery, process lifecycle | `context.py`, `media.py`, `metainfo.py`, `scraper.py`, `meta/` |
|
||||
| `app/runtime/` | 进程级运行机制和策略:配置、事件、完整日志、缓存契约/内存行为、托管资源门面、并发、调度、限流、本地化、GC 和重启状态 | 具体外部产品、业务流程、Redis/文件缓存实现 | `config.py`, `events.py`, `log.py`, `cache.py`, `managed_resources.py`, `thread.py`, `state.py` |
|
||||
| `app/runtime/extensions/` | 模块、插件、配置化服务和托管资源实现的发现、注册与生命周期适配 | 通用反射机制、插件公开 API、无关业务流程 | `module_manager.py`, `plugin_manager.py`, `managed_resource_adapter.py`, `service_registry.py` |
|
||||
| `app/adapters/network/` | HTTP、浏览器、DNS、Cloudflare 和 IP 等通用网络技术适配 | RSS/站点业务编排、身份认证策略、命名外部产品流程 | `http.py`, `browser.py`, `doh.py`, `ip.py` |
|
||||
| `app/adapters/cache/` | Redis 与文件缓存等具体持久化实现 | 缓存协议、装饰器和进程内缓存策略 | `backends.py`, `redis.py` |
|
||||
| `app/adapters/system/` | 操作系统、文件、进程、标准流、包/资源安装、显示和 Rust 加速适配 | 业务规则、进程重启决策 | `host.py`, `display/`, `stdio.py`, `package.py`, `resource.py`, `rust.py`, `fsproxy.py` |
|
||||
| `app/adapters/external/` | CookieCloud、插件市场、OCR、IP 归属和 MoviePilot Server 等命名外部生态 | 通用 HTTP/DNS/文件机制或可复用领域语义 | `market.py`, `server.py`, `cookiecloud.py`, `ocr.py`, `location.py`, `wechat_crypt.py` |
|
||||
| `app/application/` | 聚焦应用服务、用例命令,以及由用例拥有的持久化/技术能力 Port/Protocol | SQLAlchemy、Session、Oper 等具体 DB 实现,具体 Adapter 静态依赖,多领域 Chain 编排、底层通用机制、通用传输协议 | `recognition.py`, `filter.py`, `outbox.py`, `subscription/write.py`, `workflow.py` |
|
||||
| `app/application/messaging/` | 消息渲染/路由、交互和 Agent 到消息桥接:`ingress.py` 统一渠道回环入口;`interaction.py` 通用交互契约和视图工具;`router.py` 统一交互优先级和回调分发;`site.py`/`subscribe.py`/`skill.py` 对应命令的会话、输入解析和视图;`media.py` 媒体交互状态(业务工作流仍由 `MediaInteractionChain` 执行);`plugin.py` 插件输入接管和插件按钮回调;`agent.py` Agent 选择状态、回调协议和 WebAgent 消息桥接;`message.py` 通知渲染、模板和队列。不作为推荐给插件直接使用的公开 SDK | 认证策略、通用 HTTP、服务发现、仅端点使用的 Web Push 行为 | `ingress.py`, `message.py`, `interaction.py`, `router.py`, `agent.py` |
|
||||
| `app/application/security/` | 认证、授权、Cookie、Passkey、OTP/二次认证、路径/URL 安全、SSRF 和签名策略 | 通用 URL 解析、进程运行策略、普通业务校验 | `access.py`, `auth.py`, `cookie.py`, `passkey.py`, `otp.py`, `twofactor.py`, `url.py` |
|
||||
| `app/chain/` | Reusable use-case orchestration across modules, Application services, injected ports, events, and caches; chains reach modules only through `run_module` dispatch on method-name contracts | Transport schemas, backend-specific protocol details, concrete Adapter imports, generic primitives, direct Oper/DB imports, direct imports of module internals (classes, exceptions, constants) | `media.py`, `download.py`, `subscribe.py`, `transfer.py` |
|
||||
| `app/db/oper/` | 面向表和持久化值的 SQLAlchemy 数据访问;接收调用方 Session,只查询、暂存或 flush | Application 业务规则、隐式事务所有权、外部副作用 | `subscribe.py`, `site.py`, `workflow.py` |
|
||||
| `app/db/adapters/` | 实现 Application 持久化 Port,创建短生命周期 Session/UoW,并适配 Oper | 用例规则、启动顺序、进程生命周期 | `subscription.py`, `site.py`, `outbox.py`, `workflow.py` |
|
||||
| `app/startup/` | Composition root: `composition/` 构造并注入跨层依赖,`initializers/` 按领域初始化,`lifecycle/` 编排启动关闭 | Reusable business rules or adapter implementation details | `composition/context.py`, `composition/database.py`, `initializers/modules.py`, `lifecycle/components.py` |
|
||||
| `app/sdk/` | Deliberately curated stable imports for new plugins | Canonical implementation logic or host-internal dependencies | `browser.py`, `cache.py`, `logging.py`, `media.py`, `network.py`, `services.py` |
|
||||
| `app/runtime/compat/` | 仅依赖标准库的精确旧导入路由、资源前置扫描和 DEBUG 诊断 | 业务实现、通配猜测、目标模块的提前导入 | `manifest.py`, `imports.py`, `resource_imports.py`, `diagnostics.py` |
|
||||
|
||||
容易误分的三个边界必须按实际职责判断:`application/rss.py` 同时承担 Feed/种子语义、站点规则和浏览器回退,不是单纯 HTTP 传输;规范目标是由它拥有所需 Port、startup 注入 network/system Adapter。当前直接导入是 `S2-L6` 临时债务,不是允许的新模式。`application/site/sites.*` 及 `user.sites.v3.bin` 共同构成站点目录、认证和索引应用能力,只有下载安装机制留在 `adapters/system/resource.py`;`foundation/crypto.py` 只提供无状态 RSA/摘要/AES 算法,认证、签名、令牌和二次验证策略仍属于 `application/security/`。
|
||||
|
||||
### Placement Decision Order
|
||||
|
||||
Use these questions in order before creating or moving a module:
|
||||
|
||||
1. Is it generic, free of MoviePilot state and I/O? Put it in `foundation`.
|
||||
2. Is it a pure core MoviePilot rule/model that is independent of a configured service boundary? Put it in `domain`.
|
||||
3. Is it process-wide runtime policy or a contract used by adapters? Put it in `runtime`.
|
||||
4. Does it discover or manage modules/plugins/service implementations? Put it in `runtime/extensions`.
|
||||
5. Does it perform configured network, cache, OS/process, file, package/resource, stdio, or Rust I/O? Put it under the matching `adapters` technical boundary.
|
||||
6. Does it implement a named external product/ecosystem workflow? Put it in `adapters/external`.
|
||||
7. Does it own authentication, authorization, signing, SSRF, URL/path safety, OTP, passkeys, or two-factor behavior? Put it in `application/security`.
|
||||
8. Does it define a use case or the persistence Port required by that use case? Put it in `application`; do not import concrete DB there.
|
||||
9. Does it implement an Application persistence Port with SQLAlchemy Session/UoW/Oper? Put it in `db/adapters`.
|
||||
10. Does it coordinate several modules/services/Oper classes for one use case? Put it in `chain`.
|
||||
11. Is it public to plugins or only preserving an old path? Curate it in `sdk` or map it in `runtime/compat`; do not move implementation there.
|
||||
|
||||
### Enforced Split Examples
|
||||
|
||||
These decisions are architectural constraints, not naming suggestions:
|
||||
|
||||
* Cache contracts, memory backends, decorators, and proxies stay in `app/runtime/cache.py`; Redis and filesystem implementations stay in `app/adapters/cache/backends.py`. Startup registers concrete factories before decorated business modules are imported. Legacy `app.core.cache` resolves to the complete `app.sdk.cache` facade.
|
||||
* The complete logging runtime stays in `app/runtime/log.py`: policy, console/plugin routing, async rotating file output, and shutdown. `app.runtime.config` supplies the resolved settings and log path. `runtime/log.py` remains a dependency leaf with no `app.*` imports. Plugins use `app.sdk.logging`; legacy `app.log` resolves to that SDK facade.
|
||||
* Recognition parsing stays pure in `app/domain/meta/` and `app/domain/metainfo.py`. `app/application/recognition.py` consumes injected configuration; `app/startup/initializers/domain.py` injects rules, extension policy, source defaults, TMDB image construction, and the optional Rust accelerator.
|
||||
* Kodi-style NFO reading and metadata document generation are one domain capability and stay together in `app/domain/scraper.py`; a separate `domain/nfo.py` must not be recreated.
|
||||
* `app/application/mediaserver.py` is the single media-server service capability module. It owns configured service discovery together with Provider ID normalization and music-library matching, while reusing generic identity rules from `app/domain/media.py`.
|
||||
* Configured notification-service discovery belongs in `app/application/notification.py`. Web Push subscription and manual-send HTTP behavior stays in `app/api/endpoints/message.py`; it is not a reusable messaging capability module.
|
||||
* `app/adapters/system/resource.py` detects/downloads/installs resources and returns whether installation occurred. Only `app/startup/initializers/modules.py` may decide to restart the process afterward.
|
||||
* Process memory/GC policy belongs in `app/runtime/gc.py`; external IP-location APIs belong in `app/adapters/external/location.py`.
|
||||
* Security implementation filenames use package-context nouns: `app/application/security/url.py` and `app/application/security/twofactor.py`. Historical `app.utils.security` and `app.helper.twofa` remain compatibility mappings only.
|
||||
|
||||
Foundation modules do not emit runtime logs. They return documented fallback values or raise according to their public contract; application callers decide whether a failure is operationally relevant and log it from the owning upper layer.
|
||||
|
||||
Any ownership move must update canonical host imports, `app/runtime/compat/manifest.py`, curated SDK exports when applicable, `docs/rules/05-architecture.md`, and `tests/test_architecture_dependencies.py`. Run that architecture test before broader tests; it rejects physical legacy sources, forbidden upward dependencies, retired canonical filenames, and import cycles.
|
||||
|
||||
---
|
||||
|
||||
## Agent Execution Rules
|
||||
|
||||
### Pre-Flight Check
|
||||
|
||||
Before generating code or proposing changes, identify the domains the task actually touches and load only the corresponding documents from `docs/rules/`. Apply those constraints while designing, implementing, and reviewing the change; do not produce a formal checklist for unrelated domains.
|
||||
|
||||
Architecture, persistence, security, external protocols, cross-module lifecycle, and public-contract changes require an explicit boundary check before implementation. Local documentation, mechanical maintenance, and narrowly scoped changes use only the rules that materially affect their correctness and reviewability.
|
||||
|
||||
### Implementation Guidelines
|
||||
|
||||
* **Pattern Adherence:** Avoid generic boilerplate. If `04-design-patterns.md` defines a project-level pattern for a scenario, you are required to use it.
|
||||
* **Documentation Standards:** Docstring style for any new function or module must match `08-comment-styles.md`.
|
||||
* **Documentation Gate:** Public or cross-module contracts and non-obvious business behavior without useful Chinese documentation are rejected. Do not require comments that merely restate self-evident syntax.
|
||||
* **Command Reliance:** Prefer commands documented in `03-commands.md`; use other necessary standard commands with explicit, scoped arguments.
|
||||
* **Minimal Change Principle:** Prefer the smallest correct change. Do not perform unrelated refactors, mass renames, or formatting-only cleanup.
|
||||
* **Output Language:** Summaries, validation results, and risk notes default to Chinese unless the user requests otherwise.
|
||||
|
||||
### Conflict Resolution
|
||||
|
||||
If existing code appears to contradict the documentation, identify the exact contradiction and decide which current-task gate it affects. Stop and ask only when it blocks acceptance, creates a security or data-safety ambiguity, or cannot be resolved from current source and maintained documentation. Otherwise preserve the evidence, continue unaffected work, and report the discrepancy without silently expanding scope.
|
||||
|
||||
---
|
||||
|
||||
## Coupled Update Rules
|
||||
|
||||
When modifying the following, you must also update the listed artifacts:
|
||||
|
||||
| Changed Content | Must Also Update |
|
||||
|---|---|
|
||||
| CLI behavior | `moviepilot` entrypoint, `docs/cli.md`, related tests |
|
||||
| MCP / REST API, exposed tools | `docs/mcp-api.md`, `skills/*/SKILL.md`, related tests |
|
||||
| Dev workflow, dependency management, security checks | `docs/development-setup.md` |
|
||||
| Database model schema | New Alembic migration under `database/versions/` |
|
||||
| User-visible config or init flow | Related docs, help text, setup/init flows, tests |
|
||||
| New skill | Follow `skills/<name>/SKILL.md` structure, keep YAML front matter |
|
||||
| Canonical module ownership or import path | `docs/rules/05-architecture.md`, `app/runtime/compat/manifest.py`, SDK exports when public, architecture/compatibility tests |
|
||||
|
||||
---
|
||||
|
||||
## Primary Entry Point
|
||||
|
||||
For the full documentation map and cross-references, refer to:
|
||||
|
||||
**[Documentation Hub Index](./docs/rules/README.md)**
|
||||
|
||||
*Last Updated: 2026-08-19*
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
FROM python:3.11.4-slim-bookworm
|
||||
ARG MOVIEPILOT_VERSION
|
||||
ENV LANG="C.UTF-8" \
|
||||
TZ="Asia/Shanghai" \
|
||||
HOME="/moviepilot" \
|
||||
CONFIG_DIR="/config" \
|
||||
TERM="xterm" \
|
||||
PUID=0 \
|
||||
PGID=0 \
|
||||
UMASK=000 \
|
||||
PORT=3001 \
|
||||
NGINX_PORT=3000 \
|
||||
PROXY_HOST="" \
|
||||
MOVIEPILOT_AUTO_UPDATE=release \
|
||||
AUTH_SITE="iyuu" \
|
||||
IYUU_SIGN=""
|
||||
WORKDIR "/app"
|
||||
RUN apt-get update -y \
|
||||
&& apt-get upgrade -y \
|
||||
&& apt-get -y install \
|
||||
musl-dev \
|
||||
nginx \
|
||||
gettext-base \
|
||||
locales \
|
||||
procps \
|
||||
gosu \
|
||||
bash \
|
||||
wget \
|
||||
curl \
|
||||
busybox \
|
||||
dumb-init \
|
||||
jq \
|
||||
haproxy \
|
||||
fuse3 \
|
||||
rsync \
|
||||
ffmpeg \
|
||||
nano \
|
||||
&& \
|
||||
if [ "$(uname -m)" = "x86_64" ]; \
|
||||
then ln -s /usr/lib/x86_64-linux-musl/libc.so /lib/libc.musl-x86_64.so.1; \
|
||||
elif [ "$(uname -m)" = "aarch64" ]; \
|
||||
then ln -s /usr/lib/aarch64-linux-musl/libc.so /lib/libc.musl-aarch64.so.1; \
|
||||
fi \
|
||||
&& curl https://rclone.org/install.sh | bash \
|
||||
&& apt-get autoremove -y \
|
||||
&& apt-get clean -y \
|
||||
&& rm -rf \
|
||||
/tmp/* \
|
||||
/moviepilot/.cache \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
COPY requirements.txt requirements.txt
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y build-essential \
|
||||
&& pip install --upgrade pip \
|
||||
&& pip install Cython \
|
||||
&& pip install -r requirements.txt \
|
||||
&& playwright install-deps chromium \
|
||||
&& apt-get remove -y build-essential \
|
||||
&& apt-get autoremove -y \
|
||||
&& apt-get clean -y \
|
||||
&& rm -rf \
|
||||
/tmp/* \
|
||||
/moviepilot/.cache \
|
||||
/var/lib/apt/lists/* \
|
||||
/var/tmp/*
|
||||
COPY . .
|
||||
RUN cp -f /app/nginx.conf /etc/nginx/nginx.template.conf \
|
||||
&& cp -f /app/update /usr/local/bin/mp_update \
|
||||
&& cp -f /app/entrypoint /entrypoint \
|
||||
&& chmod +x /entrypoint /usr/local/bin/mp_update \
|
||||
&& mkdir -p ${HOME} /var/lib/haproxy/server-state \
|
||||
&& groupadd -r moviepilot -g 911 \
|
||||
&& useradd -r moviepilot -g moviepilot -d ${HOME} -s /bin/bash -u 911 \
|
||||
&& python_ver=$(python3 -V | awk '{print $2}') \
|
||||
&& echo "/app/" > /usr/local/lib/python${python_ver%.*}/site-packages/app.pth \
|
||||
&& echo 'fs.inotify.max_user_watches=5242880' >> /etc/sysctl.conf \
|
||||
&& echo 'fs.inotify.max_user_instances=5242880' >> /etc/sysctl.conf \
|
||||
&& locale-gen zh_CN.UTF-8 \
|
||||
&& FRONTEND_VERSION=$(curl -sL "https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/latest" | jq -r .tag_name) \
|
||||
&& curl -sL "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${FRONTEND_VERSION}/dist.zip" | busybox unzip -d / - \
|
||||
&& mv /dist /public \
|
||||
&& curl -sL "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \
|
||||
&& mv -f /tmp/MoviePilot-Plugins-main/plugins/* /app/app/plugins/ \
|
||||
&& curl -sL "https://github.com/jxxghp/MoviePilot-Resources/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \
|
||||
&& mv -f /tmp/MoviePilot-Resources-main/resources/* /app/app/helper/ \
|
||||
&& rm -rf /tmp/*
|
||||
EXPOSE 3000
|
||||
VOLUME [ "/config" ]
|
||||
ENTRYPOINT [ "/entrypoint" ]
|
||||
@@ -1,17 +1,14 @@
|
||||
# MoviePilot
|
||||
|
||||
简体中文 | [English](README_EN.md)
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
|
||||
基于 [NAStool](https://github.com/NAStool/nas-tools) 部分代码重新设计,聚焦自动化核心需求,减少问题同时更易于扩展和维护。
|
||||
|
||||
# 仅用于学习交流使用,请勿在任何国内平台宣传该项目!
|
||||
@@ -20,64 +17,13 @@
|
||||
|
||||
## 主要特性
|
||||
|
||||
- 聚焦影视自动化的核心流程:订阅、搜索、下载、整理、刮削、媒体库刷新与消息通知。
|
||||
- 前后端分离,后端基于 FastAPI,前端基于 Vue 3,部署和扩展边界更清晰。
|
||||
- 支持下载器、媒体服务器、元数据源、消息渠道、插件、工作流和 AI Agent 等能力组合。
|
||||
- 更完整的功能介绍、截图和使用入口见官网:https://movie-pilot.org
|
||||
- 前后端分离,基于FastApi + Vue3,前端项目地址:[MoviePilot-Frontend](https://github.com/jxxghp/MoviePilot-Frontend),API:http://localhost:3001/docs
|
||||
- 聚焦核心需求,简化功能和设置,部分设置项可直接使用默认值。
|
||||
- 重新设计了用户界面,更加美观易用。
|
||||
|
||||
## 安装使用
|
||||
|
||||
推荐优先使用 Docker 部署。V3 使用独立镜像 `jxxghp/moviepilot-v3`,V2 和旧版镜像保持原命名。Compose 示例、环境变量、目录映射和升级方式以官方 Wiki 为准:
|
||||
|
||||
- 官方 Wiki:https://wiki.movie-pilot.org
|
||||
- PostgreSQL 部署说明:[docs/postgresql-setup.md](docs/postgresql-setup.md)
|
||||
|
||||
也可以使用本地 CLI 以源码模式安装和管理 MoviePilot:
|
||||
|
||||
```shell
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootstrap-local.sh | bash
|
||||
```
|
||||
|
||||
安装完成后使用 `moviepilot` 命令完成初始化、启动、停止、更新和配置查看。完整命令见 [docs/cli.md](docs/cli.md)。
|
||||
|
||||
## Agent
|
||||
|
||||
1. MoviePilot 自带智能体能力,可在完成模型配置后,通过自然语言调用系统工具,辅助完成搜索、订阅、下载、整理、排障等管理任务。
|
||||
2. 其它智能体可以导入本仓库的 `skills/` 目录以获得 MoviePilot 操作能力;支持 `skills` CLI 的环境可使用:
|
||||
|
||||
```shell
|
||||
npx skills add https://github.com/jxxghp/MoviePilot
|
||||
```
|
||||
|
||||
内置 Skills 列表见 [skills/](skills/),自定义 Skill 可参考 [skills/create-moviepilot-skill/SKILL.md](skills/create-moviepilot-skill/SKILL.md)。
|
||||
3. 其它 MCP 客户端可以通过 MoviePilot 的 MCP 端点 `/api/v1/mcp` 调用工具,认证方式、客户端配置和工具 API 见 [docs/mcp-api.md](docs/mcp-api.md)。
|
||||
|
||||
|
||||
## 参与开发
|
||||
|
||||
开发前请先阅读仓库规则和本地环境说明,保持变更聚焦,通过测试后再提交 PR。常用入口:
|
||||
|
||||
- 文档规则入口:[docs/rules/README.md](docs/rules/README.md)
|
||||
- 开发环境与本地源码运行:[docs/development-setup.md](docs/development-setup.md)
|
||||
- 测试说明:[docs/testing.md](docs/testing.md)
|
||||
- 新站点适配采集与 Feature Request 提交:[docs/site-adapter-capture.md](docs/site-adapter-capture.md)
|
||||
- REST API 文档:https://api.movie-pilot.org
|
||||
- 插件开发说明:https://wiki.movie-pilot.org/zh/plugindev
|
||||
|
||||
## 相关项目
|
||||
|
||||
- [MoviePilot-Frontend](https://github.com/jxxghp/MoviePilot-Frontend)
|
||||
- [MoviePilot-Resources](https://github.com/jxxghp/MoviePilot-Resources)
|
||||
- [MoviePilot-Plugins](https://github.com/jxxghp/MoviePilot-Plugins)
|
||||
- [MoviePilot-Server](https://github.com/jxxghp/MoviePilot-Server)
|
||||
- [MoviePilot-Rust](https://github.com/jxxghp/MoviePilot-Rust)
|
||||
- [MoviePilot-Wiki](https://github.com/jxxghp/MoviePilot-Wiki)
|
||||
|
||||
## 免责申明
|
||||
|
||||
- 本软件仅供学习交流使用,任何人不得将本软件用于商业用途,任何人不得将本软件用于违法犯罪活动,软件对用户行为不知情,一切责任由使用者承担。
|
||||
- 本软件代码开源,基于开源代码进行修改,人为去除相关限制导致软件被分发、传播并造成责任事件的,需由代码修改发布者承担全部责任,不建议对用户认证机制进行规避或修改并公开发布。
|
||||
- 本项目不接受捐赠,没有在任何地方发布捐赠信息页面,软件本身不收费也不提供任何收费相关服务,请仔细辨别避免误导。
|
||||
访问官方Wiki:https://wiki.movie-pilot.org
|
||||
|
||||
## 贡献者
|
||||
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
# MoviePilot
|
||||
|
||||
[简体中文](README.md) | English
|
||||
|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||

|
||||
|
||||
Redesigned from parts of [NAStool](https://github.com/NAStool/nas-tools), with a stronger focus on core automation scenarios while reducing issues and making the project easier to extend and maintain.
|
||||
|
||||
# For learning and personal communication only. Please do not promote this project on platforms in mainland China.
|
||||
|
||||
Release channel: https://t.me/moviepilot_channel
|
||||
|
||||
## Key Features
|
||||
|
||||
- Focuses on the core media automation flow: subscriptions, search, downloads, file organization, scraping, media server refresh, and notifications.
|
||||
- Uses a separated backend/frontend architecture: FastAPI for the backend and Vue 3 for the frontend.
|
||||
- Connects download clients, media servers, metadata providers, message channels, plugins, workflows, and AI Agent capabilities.
|
||||
- For feature details, screenshots, and product entry points, see https://movie-pilot.org
|
||||
|
||||
## Installation and Usage
|
||||
|
||||
Docker is the recommended deployment model. V3 uses the dedicated `jxxghp/moviepilot-v3` image; V2 and legacy images keep their existing names. Compose examples, environment variables, volume mappings, and upgrade notes are maintained in the official wiki:
|
||||
|
||||
- Official wiki: https://wiki.movie-pilot.org
|
||||
- PostgreSQL setup: [docs/postgresql-setup.md](docs/postgresql-setup.md)
|
||||
|
||||
MoviePilot can also be installed and managed from source with the local CLI:
|
||||
|
||||
```shell
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootstrap-local.sh | bash
|
||||
```
|
||||
|
||||
After installation, use the `moviepilot` command for initialization, service management, updates, and configuration. See [docs/cli.md](docs/cli.md) for the full command reference.
|
||||
|
||||
## Agent
|
||||
|
||||
1. MoviePilot includes a built-in AI Agent. After model configuration, it can call system tools through natural language to help with search, subscriptions, downloads, organization, diagnostics, and other management tasks.
|
||||
2. Other agents can import the repository `skills/` directory to gain MoviePilot operation capabilities. Environments that support the `skills` CLI can use:
|
||||
|
||||
```shell
|
||||
npx skills add https://github.com/jxxghp/MoviePilot
|
||||
```
|
||||
|
||||
Built-in skills live in [skills/](skills/). For custom skill authoring, see [skills/create-moviepilot-skill/SKILL.md](skills/create-moviepilot-skill/SKILL.md).
|
||||
3. Other MCP clients can call MoviePilot tools through `/api/v1/mcp`. Authentication, client configuration, and tool APIs are documented in [docs/mcp-api.md](docs/mcp-api.md).
|
||||
|
||||
## Development
|
||||
|
||||
Before contributing, read the repository rules and local environment guide, keep changes focused, and validate them before opening a PR. Useful entry points:
|
||||
|
||||
- Rule index: [docs/rules/README.md](docs/rules/README.md)
|
||||
- Development setup and local source run: [docs/development-setup.md](docs/development-setup.md)
|
||||
- Testing guide: [docs/testing.md](docs/testing.md)
|
||||
- New site adapter capture and Feature Request submission: [docs/site-adapter-capture.md](docs/site-adapter-capture.md)
|
||||
- REST API documentation: https://api.movie-pilot.org
|
||||
- Plugin development guide: https://wiki.movie-pilot.org/zh/plugindev
|
||||
|
||||
## Related Projects
|
||||
|
||||
- [MoviePilot-Frontend](https://github.com/jxxghp/MoviePilot-Frontend)
|
||||
- [MoviePilot-Resources](https://github.com/jxxghp/MoviePilot-Resources)
|
||||
- [MoviePilot-Plugins](https://github.com/jxxghp/MoviePilot-Plugins)
|
||||
- [MoviePilot-Server](https://github.com/jxxghp/MoviePilot-Server)
|
||||
- [MoviePilot-Rust](https://github.com/jxxghp/MoviePilot-Rust)
|
||||
- [MoviePilot-Wiki](https://github.com/jxxghp/MoviePilot-Wiki)
|
||||
|
||||
## Disclaimer
|
||||
|
||||
- This software is for learning and personal communication only. It must not be used for commercial purposes or illegal activities. The software does not know how users choose to use it, and all responsibility rests with the user.
|
||||
- The source code is open source and derived from other open-source code. If someone removes the relevant restrictions and redistributes or publishes modified versions that lead to liability events, the publisher of those modifications bears full responsibility. Public releases that bypass or alter the user authentication mechanism are not recommended.
|
||||
- This project does not accept donations and has not published any donation page anywhere. The software itself is free of charge and does not provide paid services. Please verify information carefully to avoid being misled.
|
||||
|
||||
## Contributors
|
||||
|
||||
<a href="https://github.com/jxxghp/MoviePilot/graphs/contributors">
|
||||
<img src="https://contrib.rocks/image?repo=jxxghp/MoviePilot" />
|
||||
</a>
|
||||
@@ -1,24 +0,0 @@
|
||||
import warnings
|
||||
|
||||
from app.runtime.compat.imports import install_legacy_import_hook
|
||||
|
||||
|
||||
def _filter_third_party_startup_warnings() -> None:
|
||||
"""
|
||||
过滤第三方库在新版 Python 下产生的已知无害启动警告。
|
||||
"""
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r"'_UnionGenericAlias' is deprecated and slated for removal in Python 3\.17",
|
||||
category=DeprecationWarning,
|
||||
module=r"google\.genai\.types",
|
||||
)
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r'"\\&" is an invalid escape sequence\..*',
|
||||
category=SyntaxWarning,
|
||||
)
|
||||
|
||||
|
||||
_filter_third_party_startup_warnings()
|
||||
install_legacy_import_hook()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""MoviePilot 技术与外部系统适配器。"""
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
"""缓存持久化适配器。"""
|
||||
Vendored
-343
@@ -1,343 +0,0 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncGenerator, Generator, Optional, Tuple
|
||||
|
||||
import aiofiles
|
||||
import aioshutil
|
||||
from anyio import Path as AsyncPath
|
||||
|
||||
from app.adapters.cache.redis import AsyncRedisHelper, RedisHelper
|
||||
from app.runtime.cache import (
|
||||
DEFAULT_CACHE_REGION,
|
||||
AsyncCacheBackend,
|
||||
CacheBackend,
|
||||
configure_cache_factories,
|
||||
)
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
class RedisBackend(CacheBackend):
|
||||
"""通过同步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = RedisHelper()
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""写入缓存,非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存键是否存在。"""
|
||||
return self.redis_helper.exists(key, region=region)
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取缓存值,不存在时返回空值。"""
|
||||
return self.redis_helper.get(key, region=region)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存键。"""
|
||||
self.redis_helper.delete(key, region=region)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或全部缓存。"""
|
||||
self.redis_helper.clear(region=region)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""遍历指定缓存区的键值对。"""
|
||||
return self.redis_helper.items(region=region)
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭同步 Redis 连接池。"""
|
||||
self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class AsyncRedisBackend(AsyncCacheBackend):
|
||||
"""通过异步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化异步 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = AsyncRedisHelper()
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""异步写入缓存,非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
await self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存键是否存在。"""
|
||||
return await self.redis_helper.exists(key, region=region)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取缓存值,不存在时返回空值。"""
|
||||
return await self.redis_helper.get(key, region=region)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存键。"""
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或全部缓存。"""
|
||||
await self.redis_helper.clear(region=region)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步遍历指定缓存区的键值对。"""
|
||||
async for item in self.redis_helper.items(region=region):
|
||||
yield item
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭异步 Redis 连接池。"""
|
||||
await self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class FileBackend(CacheBackend):
|
||||
"""通过本地文件系统保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""原子写入一个二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
tmp_file.write(value)
|
||||
temp_path = Path(tmp_file.name)
|
||||
temp_path.replace(cache_path)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存文件是否存在。"""
|
||||
return (self.base / region / key).exists()
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
if not cache_path.exists():
|
||||
return None
|
||||
with cache_path.open("rb") as file_handle:
|
||||
return file_handle.read()
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存文件或缓存子目录。"""
|
||||
cache_path = self.base / region / key
|
||||
if cache_path.is_file():
|
||||
cache_path.unlink()
|
||||
elif cache_path.exists():
|
||||
shutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或缓存根目录。"""
|
||||
cache_path = self.base / region if region else self.base
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in cache_path.iterdir():
|
||||
if item.is_file():
|
||||
item.unlink()
|
||||
else:
|
||||
shutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = self.base / region
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in sorted(cache_path.rglob("*")):
|
||||
if item.is_file():
|
||||
with item.open("rb") as file_handle:
|
||||
yield item.relative_to(cache_path).as_posix(), file_handle.read()
|
||||
|
||||
def close(self) -> None:
|
||||
"""文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
class AsyncFileBackend(AsyncCacheBackend):
|
||||
"""通过异步文件接口保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化异步缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""异步原子写入一个二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
await cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
await tmp_file.write(value)
|
||||
temp_path = AsyncPath(tmp_file.name)
|
||||
await temp_path.replace(cache_path)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存文件是否存在。"""
|
||||
return await (AsyncPath(self.base) / region / key).exists()
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if not await cache_path.exists():
|
||||
return None
|
||||
async with aiofiles.open(cache_path, "rb") as file_handle:
|
||||
return await file_handle.read()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存文件或缓存子目录。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if await cache_path.is_file():
|
||||
await cache_path.unlink()
|
||||
elif await cache_path.exists():
|
||||
await aioshutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或缓存根目录。"""
|
||||
cache_path = AsyncPath(self.base) / region if region else AsyncPath(self.base)
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.iterdir():
|
||||
if await item.is_file():
|
||||
await item.unlink()
|
||||
else:
|
||||
await aioshutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = AsyncPath(self.base) / region
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.rglob("*"):
|
||||
if await item.is_file():
|
||||
key = Path(str(item)).relative_to(Path(str(cache_path))).as_posix()
|
||||
async with aiofiles.open(item, "rb") as file_handle:
|
||||
yield key, await file_handle.read()
|
||||
|
||||
async def close(self) -> None:
|
||||
"""异步文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
def configure_platform_cache() -> None:
|
||||
"""把配置感知的 Redis 与文件适配器注册到平台缓存工厂。"""
|
||||
configure_cache_factories(
|
||||
backend_type_provider=lambda: get_runtime_setting('CACHE_BACKEND_TYPE'),
|
||||
redis_factory=lambda ttl: RedisBackend(ttl=ttl),
|
||||
async_redis_factory=lambda ttl: AsyncRedisBackend(ttl=ttl),
|
||||
file_factory=lambda base: FileBackend(
|
||||
base=base or get_runtime_setting('TEMP_PATH')
|
||||
),
|
||||
async_file_factory=lambda base: AsyncFileBackend(
|
||||
base=base or get_runtime_setting('TEMP_PATH')
|
||||
),
|
||||
file_ttl_provider=lambda: get_runtime_setting('TEMP_FILE_DAYS') * 24 * 3600,
|
||||
)
|
||||
Vendored
-625
@@ -1,625 +0,0 @@
|
||||
import asyncio
|
||||
import json
|
||||
import pickle
|
||||
import threading
|
||||
from typing import Any, AsyncGenerator, Generator, Optional, Tuple, Union
|
||||
from urllib.parse import quote, unquote
|
||||
|
||||
import redis
|
||||
from redis.asyncio import BlockingConnectionPool as AsyncBlockingConnectionPool
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.reload import ConfigReloadMixin
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
# 类型缓存集合,针对非容器简单类型
|
||||
_complex_serializable_types = set()
|
||||
_simple_serializable_types = set()
|
||||
|
||||
# 默认连接参数
|
||||
_socket_timeout = 30
|
||||
_socket_connect_timeout = 5
|
||||
_health_check_interval = 60
|
||||
|
||||
|
||||
def serialize(value: Any) -> bytes:
|
||||
"""
|
||||
将值序列化为二进制数据,根据序列化方式标识格式
|
||||
"""
|
||||
|
||||
def _is_container_type(t):
|
||||
"""
|
||||
判断是否为容器类型
|
||||
"""
|
||||
return t in (list, dict, tuple, set)
|
||||
|
||||
vt = type(value)
|
||||
# 针对非容器类型使用缓存策略
|
||||
if not _is_container_type(vt):
|
||||
# 如果已知需要复杂序列化
|
||||
if vt in _complex_serializable_types:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
# 如果已知可以简单序列化
|
||||
if vt in _simple_serializable_types:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
# 对于未知的非容器类型,尝试简单序列化,如抛出异常,再使用复杂序列化
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
_simple_serializable_types.add(vt)
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
_complex_serializable_types.add(vt)
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
else:
|
||||
# 针对容器类型,每次尝试简单序列化,不使用缓存
|
||||
try:
|
||||
json_data = json.dumps(value).encode("utf-8")
|
||||
return b"JSON" + b"\x00" + json_data
|
||||
except TypeError:
|
||||
return b"PICKLE" + b"\x00" + pickle.dumps(value)
|
||||
|
||||
|
||||
def deserialize(value: bytes) -> Any:
|
||||
"""
|
||||
将二进制数据反序列化为原始值,根据格式标识区分序列化方式
|
||||
"""
|
||||
format_marker, data = value.split(b"\x00", 1)
|
||||
if format_marker == b"JSON":
|
||||
return json.loads(data.decode("utf-8"))
|
||||
elif format_marker == b"PICKLE":
|
||||
return pickle.loads(data)
|
||||
else:
|
||||
raise ValueError("Unknown serialization format")
|
||||
|
||||
|
||||
class RedisHelper(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
Redis连接和操作助手类,单例模式
|
||||
|
||||
特性:
|
||||
- 管理Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
"""
|
||||
CONFIG_WATCH = {
|
||||
"CACHE_BACKEND_TYPE",
|
||||
"CACHE_BACKEND_URL",
|
||||
"CACHE_REDIS_MAXMEMORY",
|
||||
"CACHE_REDIS_MAX_CONNECTIONS",
|
||||
"CACHE_REDIS_POOL_TIMEOUT",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Redis助手实例
|
||||
"""
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
self.client = None
|
||||
self._connect_lock = threading.RLock()
|
||||
|
||||
def _connect(self):
|
||||
"""
|
||||
建立Redis连接
|
||||
"""
|
||||
if self.client is not None:
|
||||
return
|
||||
client = None
|
||||
try:
|
||||
with self._connect_lock:
|
||||
if self.client is not None:
|
||||
return
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
connection_pool = redis.BlockingConnectionPool.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
max_connections=get_runtime_setting('CACHE_REDIS_MAX_CONNECTIONS'),
|
||||
timeout=get_runtime_setting('CACHE_REDIS_POOL_TIMEOUT'),
|
||||
)
|
||||
client = redis.Redis(connection_pool=connection_pool)
|
||||
# 测试连接,确保Redis可用
|
||||
client.ping()
|
||||
self.client = client
|
||||
logger.info(f"Successfully connected to Redis:{self.redis_url}")
|
||||
self.set_memory_limit()
|
||||
except Exception as e:
|
||||
if client:
|
||||
client.close()
|
||||
logger.error(f"Failed to connect to Redis: {e}")
|
||||
self.client = None
|
||||
raise RuntimeError("Redis connection failed") from e
|
||||
|
||||
def on_config_changed(self):
|
||||
"""缓存配置变化后重建同步 Redis 连接。"""
|
||||
with self._connect_lock:
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
self.close()
|
||||
self._connect()
|
||||
|
||||
def get_reload_name(self):
|
||||
"""返回同步 Redis 配置重载名称。"""
|
||||
return "Redis"
|
||||
|
||||
def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略(如'allkeys-lru')
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值,则直接使用,为0时说明不限制,如果未配置,开启BIG_MEMORY_MODE时为"1024mb",未开启时为"256mb"
|
||||
maxmemory = get_runtime_setting('CACHE_REDIS_MAXMEMORY') or (
|
||||
"1024mb" if get_runtime_setting('BIG_MEMORY_MODE') else "256mb"
|
||||
)
|
||||
self.client.config_set("maxmemory", maxmemory)
|
||||
self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy: {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = None):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{region}" if region else "region:DEFAULT"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return unquote(parts[-1])
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间,单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True,否则返回False
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
return self.client.exists(redis_key) == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key: {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值,如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key: {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
def pop(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""原子读取并删除缓存值。"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = self.client.getdel(redis_key)
|
||||
return deserialize(value) if value is not None else None
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"Failed to pop key: {key} in region: {region}, error: {e}"
|
||||
)
|
||||
return None
|
||||
|
||||
def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key: {key} in region: {region}, error: {e}")
|
||||
|
||||
def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
with self.client.pipeline() as pipe:
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
pipe.delete(key)
|
||||
pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region: {region}")
|
||||
else:
|
||||
self.client.flushdb()
|
||||
logger.info("All Redis cache Cleared!")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache, region: {region}, error: {e}")
|
||||
|
||||
def items(self, region: Optional[str] = None) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
for key in self.client.scan_iter(redis_key):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
for key in self.client.scan_iter("*"):
|
||||
value = self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
def test(self) -> bool:
|
||||
"""
|
||||
测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis connection test failed: {e}")
|
||||
return False
|
||||
|
||||
def close(self) -> None:
|
||||
"""
|
||||
关闭Redis客户端的连接池
|
||||
"""
|
||||
with self._connect_lock:
|
||||
if self.client:
|
||||
self.client.close()
|
||||
self.client = None
|
||||
logger.debug("Redis connection closed")
|
||||
|
||||
|
||||
class AsyncRedisHelper(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
异步Redis连接和操作助手类,单例模式
|
||||
|
||||
特性:
|
||||
- 管理异步Redis连接池和客户端
|
||||
- 提供序列化和反序列化功能
|
||||
- 支持内存限制和淘汰策略设置
|
||||
- 提供键名生成和区域管理功能
|
||||
- 所有操作都是异步的
|
||||
"""
|
||||
CONFIG_WATCH = {
|
||||
"CACHE_BACKEND_TYPE",
|
||||
"CACHE_BACKEND_URL",
|
||||
"CACHE_REDIS_MAXMEMORY",
|
||||
"CACHE_REDIS_MAX_CONNECTIONS",
|
||||
"CACHE_REDIS_POOL_TIMEOUT",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化异步Redis助手实例
|
||||
"""
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
self.client: Optional[Redis] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._connect_lock: Optional[asyncio.Lock] = None
|
||||
self._connect_lock_loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
|
||||
def _get_connect_lock(self, current_loop: asyncio.AbstractEventLoop) -> asyncio.Lock:
|
||||
"""
|
||||
获取当前事件循环对应的异步连接锁
|
||||
"""
|
||||
if self._connect_lock is None or self._connect_lock_loop is not current_loop:
|
||||
self._connect_lock = asyncio.Lock()
|
||||
self._connect_lock_loop = current_loop
|
||||
return self._connect_lock
|
||||
|
||||
async def _connect(self):
|
||||
"""
|
||||
建立异步Redis连接
|
||||
"""
|
||||
current_loop = asyncio.get_running_loop()
|
||||
connect_lock = self._get_connect_lock(current_loop)
|
||||
client = None
|
||||
try:
|
||||
async with connect_lock:
|
||||
# 检测事件循环是否发生变化,如果变化则重新连接
|
||||
if self.client is not None and self._loop is not current_loop:
|
||||
logger.debug("Event loop changed, reconnecting Redis (async)")
|
||||
await self._close_client()
|
||||
if self.client is not None:
|
||||
return
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
connection_pool = AsyncBlockingConnectionPool.from_url(
|
||||
self.redis_url,
|
||||
decode_responses=False,
|
||||
socket_timeout=_socket_timeout,
|
||||
socket_connect_timeout=_socket_connect_timeout,
|
||||
health_check_interval=_health_check_interval,
|
||||
max_connections=get_runtime_setting('CACHE_REDIS_MAX_CONNECTIONS'),
|
||||
timeout=get_runtime_setting('CACHE_REDIS_POOL_TIMEOUT'),
|
||||
)
|
||||
client = Redis(connection_pool=connection_pool)
|
||||
self._loop = current_loop
|
||||
# 测试连接,确保Redis可用
|
||||
await client.ping()
|
||||
self.client = client
|
||||
logger.info(f"Successfully connected to Redis (async):{self.redis_url}")
|
||||
await self.set_memory_limit()
|
||||
except Exception as e:
|
||||
if client:
|
||||
await client.close()
|
||||
logger.error(f"Failed to connect to Redis (async): {e}")
|
||||
self.client = None
|
||||
self._loop = None
|
||||
raise RuntimeError("Redis async connection failed") from e
|
||||
|
||||
async def _close_client(self):
|
||||
"""
|
||||
关闭当前Redis客户端连接
|
||||
"""
|
||||
if self.client:
|
||||
try:
|
||||
await self.client.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.client = None
|
||||
self._loop = None
|
||||
|
||||
async def on_config_changed(self):
|
||||
"""缓存配置变化后异步重建 Redis 连接。"""
|
||||
self.redis_url = get_runtime_setting('CACHE_BACKEND_URL')
|
||||
await self._close_client()
|
||||
await self._connect()
|
||||
|
||||
def get_reload_name(self):
|
||||
"""返回异步 Redis 配置重载名称。"""
|
||||
return "Redis (async)"
|
||||
|
||||
async def set_memory_limit(self, policy: Optional[str] = "allkeys-lru"):
|
||||
"""
|
||||
动态设置Redis最大内存和内存淘汰策略
|
||||
|
||||
:param policy: 淘汰策略(如'allkeys-lru')
|
||||
"""
|
||||
try:
|
||||
# 如果有显式值,则直接使用,为0时说明不限制,如果未配置,开启BIG_MEMORY_MODE时为"1024mb",未开启时为"256mb"
|
||||
maxmemory = get_runtime_setting('CACHE_REDIS_MAXMEMORY') or (
|
||||
"1024mb" if get_runtime_setting('BIG_MEMORY_MODE') else "256mb"
|
||||
)
|
||||
await self.client.config_set("maxmemory", maxmemory)
|
||||
await self.client.config_set("maxmemory-policy", policy)
|
||||
logger.debug(f"Redis maxmemory set to {maxmemory}, policy: {policy} (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set Redis maxmemory or policy (async): {e}")
|
||||
|
||||
@staticmethod
|
||||
def __get_region(region: Optional[str] = "DEFAULT"):
|
||||
"""
|
||||
获取缓存的区
|
||||
"""
|
||||
return f"region:{region}" if region else "region:DEFAULT"
|
||||
|
||||
def __make_redis_key(self, region: str, key: str) -> str:
|
||||
"""
|
||||
获取缓存Key
|
||||
"""
|
||||
# 使用region作为缓存键的一部分
|
||||
region = self.__get_region(region)
|
||||
return f"{region}:key:{quote(key)}"
|
||||
|
||||
@staticmethod
|
||||
def __get_original_key(redis_key: Union[str, bytes]) -> str:
|
||||
"""
|
||||
从Redis键中提取原始key
|
||||
"""
|
||||
try:
|
||||
if isinstance(redis_key, bytes):
|
||||
redis_key = redis_key.decode("utf-8")
|
||||
parts = redis_key.split(":key:")
|
||||
return unquote(parts[-1])
|
||||
except Exception as e:
|
||||
logger.warn(f"Failed to parse redis key: {redis_key}, error: {e}")
|
||||
return redis_key
|
||||
|
||||
async def set(self, key: str, value: Any, ttl: Optional[int] = None,
|
||||
region: Optional[str] = "DEFAULT", **kwargs) -> None:
|
||||
"""
|
||||
异步设置缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param value: 缓存的值
|
||||
:param ttl: 缓存的存活时间,单位秒
|
||||
:param region: 缓存的区
|
||||
:param kwargs: 其他参数
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
# 对值进行序列化
|
||||
serialized_value = serialize(value)
|
||||
kwargs.pop("maxsize", None)
|
||||
await self.client.set(redis_key, serialized_value, ex=ttl, **kwargs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to set key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def exists(self, key: str, region: Optional[str] = "DEFAULT") -> bool:
|
||||
"""
|
||||
异步判断缓存键是否存在
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 存在返回True,否则返回False
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
result = await self.client.exists(redis_key)
|
||||
return result == 1
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to exists key (async): {key} region: {region}, error: {e}")
|
||||
return False
|
||||
|
||||
async def get(self, key: str, region: Optional[str] = "DEFAULT") -> Optional[Any]:
|
||||
"""
|
||||
异步获取缓存的值
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
:return: 返回缓存的值,如果缓存不存在返回None
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
value = await self.client.get(redis_key)
|
||||
if value is not None:
|
||||
return deserialize(value)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get key (async): {key} in region: {region}, error: {e}")
|
||||
return None
|
||||
|
||||
async def delete(self, key: str, region: Optional[str] = "DEFAULT") -> None:
|
||||
"""
|
||||
异步删除缓存
|
||||
|
||||
:param key: 缓存的键
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
redis_key = self.__make_redis_key(region, key)
|
||||
await self.client.delete(redis_key)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete key (async): {key} in region: {region}, error: {e}")
|
||||
|
||||
async def clear(self, region: Optional[str] = None) -> None:
|
||||
"""
|
||||
异步清除指定区域的缓存或全部缓存
|
||||
|
||||
:param region: 缓存的区
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async with self.client.pipeline() as pipe:
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
await pipe.delete(key)
|
||||
await pipe.execute()
|
||||
logger.debug(f"Cleared Redis cache for region (async): {region}")
|
||||
else:
|
||||
await self.client.flushdb()
|
||||
logger.info("Cleared all Redis cache (async)")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear cache (async), region: {region}, error: {e}")
|
||||
|
||||
async def items(self, region: Optional[str] = None) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""
|
||||
获取指定区域的所有缓存键值对
|
||||
|
||||
:param region: 缓存的区
|
||||
:return: 返回键值对生成器
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
if region:
|
||||
cache_region = self.__get_region(region)
|
||||
redis_key = f"{cache_region}:key:*"
|
||||
async for key in self.client.scan_iter(redis_key):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
else:
|
||||
async for key in self.client.scan_iter("*"):
|
||||
value = await self.client.get(key)
|
||||
if value is not None:
|
||||
yield self.__get_original_key(key), deserialize(value)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get items from Redis, region: {region}, error: {e}")
|
||||
|
||||
async def test(self) -> bool:
|
||||
"""
|
||||
异步测试Redis连接性
|
||||
"""
|
||||
try:
|
||||
await self._connect()
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Redis async connection test failed: {e}")
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
"""
|
||||
关闭异步Redis客户端的连接池
|
||||
"""
|
||||
await self._close_client()
|
||||
logger.debug("Redis async connection closed")
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
"""插件市场、CookieCloud、OCR 和远程 MoviePilot 服务集成。"""
|
||||
Vendored
-3493
File diff suppressed because it is too large
Load Diff
Vendored
-79
@@ -1,79 +0,0 @@
|
||||
import base64
|
||||
from typing import Optional
|
||||
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
class OcrHelper:
|
||||
"""
|
||||
OCR 辅助类,负责获取验证码图片并调用 OCR 服务识别文本。
|
||||
"""
|
||||
|
||||
def __init__(self, ocr_base_url: Optional[str] = None) -> None:
|
||||
"""初始化 OCR 服务地址,优先使用组合根设置快照。"""
|
||||
if ocr_base_url is None:
|
||||
ocr_base_url = get_runtime_setting('OCR_HOST')
|
||||
self._ocr_b64_url = f"{str(ocr_base_url).rstrip('/')}/captcha/base64"
|
||||
|
||||
def get_captcha_text(
|
||||
self,
|
||||
image_url: Optional[str] = None,
|
||||
image_b64: Optional[str] = None,
|
||||
cookie: Optional[str] = None,
|
||||
ua: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
根据图片地址,获取验证码图片,并识别内容
|
||||
:param image_url: 图片地址
|
||||
:param image_b64: 图片base64,跳过图片地址下载
|
||||
:param cookie: 下载图片使用的cookie
|
||||
:param ua: 下载图片使用的ua
|
||||
:return: 验证码识别结果,失败时返回空字符串
|
||||
"""
|
||||
image_b64 = self._normalize_image_base64(image_b64)
|
||||
if image_url:
|
||||
data_url_b64 = self._extract_data_url_base64(image_url)
|
||||
if data_url_b64:
|
||||
image_b64 = self._normalize_image_base64(data_url_b64)
|
||||
else:
|
||||
ret = RequestUtils(ua=ua,
|
||||
cookies=cookie).get_res(image_url)
|
||||
if ret is not None:
|
||||
image_bin = ret.content
|
||||
if not image_bin:
|
||||
return ""
|
||||
image_b64 = base64.b64encode(image_bin).decode()
|
||||
if not image_b64:
|
||||
return ""
|
||||
ret = RequestUtils(content_type="application/json").post_res(
|
||||
url=self._ocr_b64_url,
|
||||
json={"base64_img": image_b64})
|
||||
if ret:
|
||||
return ret.json().get("result") or ""
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def _normalize_image_base64(image_b64: Optional[str]) -> str:
|
||||
"""规范化外部传入的图片 base64 内容。"""
|
||||
if not image_b64:
|
||||
return ""
|
||||
clean_image_b64 = OcrHelper._extract_data_url_base64(image_b64) or image_b64
|
||||
clean_image_b64 = "".join(clean_image_b64.split())
|
||||
if not clean_image_b64:
|
||||
return ""
|
||||
padding_size = len(clean_image_b64) % 4
|
||||
if padding_size:
|
||||
clean_image_b64 = f"{clean_image_b64}{'=' * (4 - padding_size)}"
|
||||
return clean_image_b64
|
||||
|
||||
@staticmethod
|
||||
def _extract_data_url_base64(image_url: Optional[str]) -> str:
|
||||
"""从 data:image/...;base64,... 地址中提取纯 base64 内容。"""
|
||||
image_url = (image_url or "").strip()
|
||||
if not image_url.lower().startswith("data:image/"):
|
||||
return ""
|
||||
metadata, separator, data = image_url.partition(",")
|
||||
if not separator or ";base64" not in metadata.lower():
|
||||
return ""
|
||||
return data.strip()
|
||||
-1
@@ -1 +0,0 @@
|
||||
"""插件市场外部适配器。"""
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
"""插件市场查询客户端。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from app.adapters.external.market import PluginHelper as _PluginHelper
|
||||
from app.runtime.cache import async_fresh, fresh
|
||||
|
||||
|
||||
class PluginMarketClient:
|
||||
"""把插件市场、版本元数据和本地仓库查询隔离为只读客户端。"""
|
||||
|
||||
def __init__(self, helper: Optional[_PluginHelper] = None) -> None:
|
||||
"""复用旧 PluginHelper 实现,保持缓存和弱单例身份不变。"""
|
||||
self._helper = helper or _PluginHelper()
|
||||
|
||||
def get_plugins(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict[str, Any]]]:
|
||||
"""同步读取指定仓库和代际的插件索引。"""
|
||||
with fresh(force):
|
||||
return self._helper.get_plugins(repo_url, package_version)
|
||||
|
||||
async def async_get_plugins(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict]]:
|
||||
"""异步读取指定仓库和代际的插件索引。"""
|
||||
async with async_fresh(force):
|
||||
return await self._helper.async_get_plugins(repo_url, package_version)
|
||||
|
||||
def get_plugin_index_result(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict]]:
|
||||
"""读取插件索引的三态结果,供库存读取保留失败事实。"""
|
||||
with fresh(force):
|
||||
return cast(
|
||||
Optional[dict[str, dict[str, Any]]],
|
||||
self._helper.get_plugin_index_result(repo_url, package_version),
|
||||
)
|
||||
|
||||
async def async_get_plugin_index_result(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict[str, Any]]]:
|
||||
"""异步读取插件索引的三态结果,供库存读取保留失败事实。"""
|
||||
async with async_fresh(force):
|
||||
return cast(
|
||||
Optional[dict[str, dict[str, Any]]],
|
||||
await self._helper.async_get_plugin_index_result(
|
||||
repo_url,
|
||||
package_version,
|
||||
),
|
||||
)
|
||||
|
||||
def get_local_candidates(self) -> dict[str, dict]:
|
||||
"""返回全部本地插件仓库候选。"""
|
||||
return self._helper.get_local_plugin_candidates()
|
||||
|
||||
def get_local_candidate(
|
||||
self,
|
||||
plugin_id: str,
|
||||
package_version: Optional[str] = None,
|
||||
repo_path: Optional[Path] = None,
|
||||
**kwargs: Any,
|
||||
) -> Optional[dict]:
|
||||
"""返回指定插件的本地仓库候选。"""
|
||||
return self._helper.get_local_plugin_candidate(
|
||||
pid=plugin_id,
|
||||
package_version=package_version,
|
||||
repo_path=repo_path,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_local_repo_paths() -> list[Path]:
|
||||
"""返回配置中有效的本地插件仓库目录。"""
|
||||
return _PluginHelper.get_local_repo_paths()
|
||||
|
||||
@staticmethod
|
||||
def make_local_repo_url(
|
||||
plugin_id: str,
|
||||
repo_path: Optional[object] = None,
|
||||
package_version: Optional[str] = None,
|
||||
) -> str:
|
||||
"""生成兼容旧入口的本地插件来源标识。"""
|
||||
return _PluginHelper.make_local_repo_url(
|
||||
plugin_id,
|
||||
repo_path,
|
||||
package_version,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_local_repo_url(repo_url: Optional[str]) -> bool:
|
||||
"""判断插件来源是否为本地仓库标识。"""
|
||||
return _PluginHelper.is_local_repo_url(repo_url)
|
||||
|
||||
@staticmethod
|
||||
def annotate_system_version(plugin_info: dict) -> dict:
|
||||
"""补充插件所需 MoviePilot 版本兼容状态。"""
|
||||
return _PluginHelper.annotate_plugin_system_version(plugin_info)
|
||||
|
||||
@staticmethod
|
||||
def is_package_compatible(
|
||||
plugin_info: dict,
|
||||
package_version: Optional[str],
|
||||
) -> bool:
|
||||
"""判断插件条目是否兼容目标插件包代际。"""
|
||||
return _PluginHelper.is_package_plugin_compatible(
|
||||
plugin_info,
|
||||
package_version,
|
||||
)
|
||||
Vendored
-1935
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
"""通用网络与 Web 协议适配器。"""
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,232 +0,0 @@
|
||||
"""
|
||||
doh函数的实现。
|
||||
author: https://github.com/C5H12O5/syno-videoinfo-plugin
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import urllib
|
||||
import urllib.request
|
||||
from concurrent.futures import as_completed
|
||||
from threading import Lock
|
||||
from typing import Dict, Optional
|
||||
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.execution import OwnedThreadPoolExecutor, submit_with_context
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.reload import ConfigReloadMixin
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
# DoH 关闭时需要释放线程池;保持惰性创建可避免未启用 DoH 时占用进程级资源
|
||||
_executor: Optional[OwnedThreadPoolExecutor] = None
|
||||
_executor_lock = Lock()
|
||||
_doh_enabled = False
|
||||
_DOH_EXECUTOR_STOP_TIMEOUT_SECONDS = 10.0
|
||||
|
||||
# 定义默认的DoH配置
|
||||
_doh_timeout = 5
|
||||
_doh_cache: Dict[str, str] = {}
|
||||
_doh_lock = Lock()
|
||||
# 保存原始的 socket.getaddrinfo 方法
|
||||
_orig_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
|
||||
def _doh_setting(key: str):
|
||||
"""读取 DoH 热更新配置,组合根未装配时兼容旧 Settings。"""
|
||||
return get_runtime_setting(key)
|
||||
|
||||
|
||||
def _get_executor_locked() -> OwnedThreadPoolExecutor:
|
||||
"""在持有执行器锁时按需获取 DoH 查询线程池。"""
|
||||
global _executor
|
||||
if _executor is None:
|
||||
_executor = OwnedThreadPoolExecutor()
|
||||
return _executor
|
||||
|
||||
|
||||
def enable_doh(enable: bool) -> bool:
|
||||
"""
|
||||
对 socket.getaddrinfo 进行补丁。
|
||||
|
||||
:param enable: 是否启用 DoH 解析
|
||||
:return: 状态切换成功时返回 True;旧 executor 未收敛时返回 False
|
||||
"""
|
||||
|
||||
global _doh_enabled, _executor
|
||||
|
||||
def _patched_getaddrinfo(host: str, *args, **kwargs):
|
||||
"""
|
||||
socket.getaddrinfo的补丁版本。
|
||||
"""
|
||||
if host not in _doh_setting("DOH_DOMAINS").split(","):
|
||||
return _orig_getaddrinfo(host, *args, **kwargs)
|
||||
# 检查主机是否已解析
|
||||
with _doh_lock:
|
||||
ip = _doh_cache.get(host, None)
|
||||
if ip is not None:
|
||||
logger.info(f"已解析 [{host}] 为 [{ip}] (缓存)")
|
||||
return _orig_getaddrinfo(ip, *args, **kwargs)
|
||||
# 使用DoH解析主机
|
||||
with _executor_lock:
|
||||
if not _doh_enabled:
|
||||
return _orig_getaddrinfo(host, *args, **kwargs)
|
||||
executor = _get_executor_locked()
|
||||
# 一次解析的任务必须在同一临界区提交完,避免关闭过程中部分任务落入新线程池
|
||||
futures = [
|
||||
submit_with_context(executor, _doh_query, resolver, host)
|
||||
for resolver in _doh_setting("DOH_RESOLVERS").split(",")
|
||||
]
|
||||
for future in as_completed(futures):
|
||||
ip = future.result()
|
||||
if ip is not None:
|
||||
logger.info(f"已解析 [{host}] 为 [{ip}]")
|
||||
# 关闭可能在查询等待期间恢复系统 DNS;关闭后的结果不得回填到下一轮配置。
|
||||
with _executor_lock:
|
||||
cache_allowed = _doh_enabled
|
||||
if cache_allowed:
|
||||
with _doh_lock:
|
||||
_doh_cache[host] = ip
|
||||
host = ip
|
||||
break
|
||||
return _orig_getaddrinfo(host, *args, **kwargs)
|
||||
|
||||
with _executor_lock:
|
||||
if enable and _executor is not None and not _executor.accepting:
|
||||
# 上一轮 shutdown 超时后必须继续持有原 owner;只有真实收敛才能替换执行器。
|
||||
if not _executor.shutdown_bounded(timeout=0.0):
|
||||
_doh_enabled = False
|
||||
socket.getaddrinfo = _orig_getaddrinfo
|
||||
return False
|
||||
_executor = None
|
||||
_doh_enabled = enable
|
||||
socket.getaddrinfo = _patched_getaddrinfo if enable else _orig_getaddrinfo
|
||||
return True
|
||||
|
||||
|
||||
class DohHelper(ConfigReloadMixin, metaclass=Singleton):
|
||||
"""
|
||||
DoH帮助类,用于处理DNS over HTTPS解析。
|
||||
"""
|
||||
CONFIG_WATCH = {"DOH_ENABLE", "DOH_DOMAINS", "DOH_RESOLVERS"}
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""按当前配置安装或移除 DoH 解析器。"""
|
||||
enable_doh(_doh_setting("DOH_ENABLE"))
|
||||
|
||||
def on_config_changed(self) -> None:
|
||||
"""配置变化时清理缓存并重新应用 DoH 状态。"""
|
||||
if not _doh_setting("DOH_ENABLE"):
|
||||
if not self.shutdown():
|
||||
logger.error("DoH配置关闭后查询线程池未在预算内收敛")
|
||||
return
|
||||
with _doh_lock:
|
||||
# DOH配置有变动的情况下,清空缓存
|
||||
_doh_cache.clear()
|
||||
if not enable_doh(True):
|
||||
logger.error("DoH查询线程池尚未收敛,暂不重新启用")
|
||||
|
||||
def get_reload_name(self) -> str:
|
||||
"""返回 DoH 配置重载名称。"""
|
||||
return 'DoH'
|
||||
|
||||
def shutdown(
|
||||
self,
|
||||
timeout: float = _DOH_EXECUTOR_STOP_TIMEOUT_SECONDS,
|
||||
) -> bool:
|
||||
"""
|
||||
恢复系统 DNS 并有限等待 DoH 查询线程池。
|
||||
|
||||
:param timeout: 等待已接受查询和 worker 终止的最长秒数
|
||||
:return: 查询线程池真实终止时返回 True,否则返回 False
|
||||
"""
|
||||
global _executor, _doh_enabled
|
||||
with _executor_lock:
|
||||
_doh_enabled = False
|
||||
socket.getaddrinfo = _orig_getaddrinfo
|
||||
executor = _executor
|
||||
converged = executor is None or executor.shutdown_bounded(timeout=timeout)
|
||||
if converged and _executor is executor:
|
||||
_executor = None
|
||||
with _doh_lock:
|
||||
_doh_cache.clear()
|
||||
return converged
|
||||
|
||||
|
||||
def _doh_query(resolver: str, host: str) -> Optional[str]:
|
||||
"""
|
||||
使用给定的DoH解析器查询给定主机的IP地址。
|
||||
"""
|
||||
|
||||
# 构造DNS查询消息(RFC 1035)
|
||||
header = b"".join(
|
||||
[
|
||||
b"\x00\x00", # ID: 0
|
||||
b"\x01\x00", # FLAGS: 标准递归查询
|
||||
b"\x00\x01", # QDCOUNT: 1
|
||||
b"\x00\x00", # ANCOUNT: 0
|
||||
b"\x00\x00", # NSCOUNT: 0
|
||||
b"\x00\x00", # ARCOUNT: 0
|
||||
]
|
||||
)
|
||||
question = b"".join(
|
||||
[
|
||||
b"".join(
|
||||
[
|
||||
struct.pack("B", len(item)) + item.encode("utf-8")
|
||||
for item in host.split(".")
|
||||
]
|
||||
)
|
||||
+ b"\x00", # QNAME: 域名序列
|
||||
b"\x00\x01", # QTYPE: A
|
||||
b"\x00\x01", # QCLASS: IN
|
||||
]
|
||||
)
|
||||
message = header + question
|
||||
|
||||
try:
|
||||
# 发送GET请求到DoH解析器(RFC 8484)
|
||||
b64message = base64.b64encode(message).decode("utf-8").rstrip("=")
|
||||
url = f"https://{resolver}/dns-query?dns={b64message}"
|
||||
headers = {"Content-Type": "application/dns-message"}
|
||||
logger.debug(f"DoH请求: {url}")
|
||||
|
||||
request = urllib.request.Request(url, headers=headers, method="GET")
|
||||
with urllib.request.urlopen(request, timeout=_doh_timeout) as response:
|
||||
logger.debug(f"解析器({resolver})响应: {response.status}")
|
||||
if response.status != 200:
|
||||
return None
|
||||
resp_body = response.read()
|
||||
|
||||
# 解析DNS响应消息(RFC 1035)
|
||||
# name(压缩):2 + type:2 + class:2 + ttl:4 + rdlength:2 = 12字节
|
||||
first_rdata_start = len(header) + len(question) + 12
|
||||
# rdata(A记录)= 4字节
|
||||
first_rdata_end = first_rdata_start + 4
|
||||
# 将rdata转换为IP地址
|
||||
return socket.inet_ntoa(resp_body[first_rdata_start:first_rdata_end])
|
||||
except Exception as e:
|
||||
logger.error(f"解析器({resolver})请求错误: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def doh_query_json(resolver: str, host: str) -> Optional[str]:
|
||||
"""
|
||||
使用给定的DoH解析器查询给定主机的IP地址。
|
||||
"""
|
||||
url = f"https://{resolver}/dns-query?name={host}&type=A"
|
||||
headers = {"Accept": "application/dns-json"}
|
||||
logger.debug(f"DoH请求: {url}")
|
||||
try:
|
||||
request = urllib.request.Request(url, headers=headers, method="GET")
|
||||
with urllib.request.urlopen(request, timeout=_doh_timeout) as response:
|
||||
logger.debug(f"解析器({resolver})响应: {response.status}")
|
||||
if response.status != 200:
|
||||
return None
|
||||
response_body = response.read().decode("utf-8")
|
||||
logger.debug(f"<== body: {response_body}")
|
||||
answer = json.loads(response_body)["Answer"]
|
||||
return answer[0]["data"]
|
||||
except Exception as e:
|
||||
logger.error(f"解析器({resolver})请求错误: {e}")
|
||||
return None
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
"""运行观测导出器适配器。"""
|
||||
@@ -1,49 +0,0 @@
|
||||
"""可选 OpenTelemetry metrics adapter。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from typing import Any, Mapping
|
||||
|
||||
from app.runtime.observability import MetricKind, MetricSpec, NoopObservationPort, ObservationPort
|
||||
|
||||
|
||||
class OpenTelemetryObservationPort:
|
||||
"""把内部指标合同映射到可选安装的 OpenTelemetry Metrics API。"""
|
||||
|
||||
def __init__(self, meter: Any) -> None:
|
||||
"""保存 meter,并按名称惰性创建 instrument。"""
|
||||
self._meter = meter
|
||||
self._instruments: dict[str, Any] = {}
|
||||
|
||||
def record(self, spec: MetricSpec, value: float, labels: Mapping[str, str]) -> None:
|
||||
"""按合同类型使用 OTel counter、histogram 或 up/down counter。"""
|
||||
instrument = self._instruments.get(spec.name)
|
||||
if instrument is None:
|
||||
instrument = self._create_instrument(spec)
|
||||
self._instruments[spec.name] = instrument
|
||||
if spec.kind == MetricKind.HISTOGRAM:
|
||||
instrument.record(value, attributes=dict(labels))
|
||||
else:
|
||||
instrument.add(value, attributes=dict(labels))
|
||||
|
||||
def _create_instrument(self, spec: MetricSpec) -> Any:
|
||||
"""为内部指标类型创建对应 OTel instrument。"""
|
||||
normalized = spec.name.replace(".", "_")
|
||||
if spec.kind == MetricKind.HISTOGRAM:
|
||||
return self._meter.create_histogram(normalized)
|
||||
if spec.kind == MetricKind.COUNTER:
|
||||
return self._meter.create_counter(normalized)
|
||||
return self._meter.create_up_down_counter(normalized)
|
||||
|
||||
|
||||
def build_observation_port() -> ObservationPort:
|
||||
"""仅在显式启用且 API 可导入时创建 OTel adapter,否则返回 no-op。"""
|
||||
if os.getenv("MOVIEPILOT_OTEL_METRICS") != "1":
|
||||
return NoopObservationPort()
|
||||
try:
|
||||
metrics = importlib.import_module("opentelemetry.metrics")
|
||||
except ImportError:
|
||||
return NoopObservationPort()
|
||||
return OpenTelemetryObservationPort(metrics.get_meter("moviepilot"))
|
||||
@@ -1 +0,0 @@
|
||||
"""操作系统、进程与运行资源适配器。"""
|
||||
@@ -1 +0,0 @@
|
||||
"""数据库备份的文件系统与数据库技术适配器命名空间。"""
|
||||
@@ -1,260 +0,0 @@
|
||||
"""基于活动 SQLAlchemy 引擎的 SQLite 与 PostgreSQL 备份实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Protocol, Sequence
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatabaseBackupCheck:
|
||||
"""数据库适配器返回的基础校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class ProcessResult(Protocol):
|
||||
"""数据库命令执行结果的最小合同。"""
|
||||
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
class ProcessRunner(Protocol):
|
||||
"""可替换的数据库命令执行边界。"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
env: Mapping[str, str],
|
||||
capture_output: bool,
|
||||
text: bool,
|
||||
check: bool,
|
||||
) -> ProcessResult:
|
||||
"""执行命令并返回结果。"""
|
||||
|
||||
|
||||
def verify_database_backup(
|
||||
artifact: Path,
|
||||
*,
|
||||
db_type: str,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> DatabaseBackupCheck:
|
||||
"""在不访问活动数据库的前提下校验一个受管备份文件。"""
|
||||
if db_type == "sqlite":
|
||||
method = "PRAGMA integrity_check"
|
||||
try:
|
||||
# 正式备份不会再变化,immutable 可避免只读校验创建 WAL 旁路文件。
|
||||
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
|
||||
with closing(sqlite3.connect(uri, uri=True)) as connection:
|
||||
rows = connection.execute("PRAGMA integrity_check").fetchall()
|
||||
except sqlite3.Error as error:
|
||||
return DatabaseBackupCheck(False, method, str(error))
|
||||
valid = bool(rows) and all(row[0] == "ok" for row in rows)
|
||||
detail = None if valid else "; ".join(str(row[0]) for row in rows)
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
if db_type == "postgresql":
|
||||
method = "pg_restore --list"
|
||||
executable = _require_tool(pg_restore, tool_resolver)
|
||||
result = runner(
|
||||
[executable, "--list", str(artifact)],
|
||||
env=_postgres_environment(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
valid = result.returncode == 0 and bool(result.stdout.strip())
|
||||
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
raise ValueError(f"不支持的数据库备份类型:{db_type}")
|
||||
|
||||
|
||||
def _require_tool(
|
||||
executable: str,
|
||||
tool_resolver: Callable[[str], str | None],
|
||||
) -> str:
|
||||
resolved = tool_resolver(executable)
|
||||
if resolved is None:
|
||||
raise RuntimeError(
|
||||
f"未找到 {executable},请安装与服务端同主版本或更高的 "
|
||||
"PostgreSQL client 并加入 PATH"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _postgres_environment(
|
||||
*,
|
||||
password: str | None = None,
|
||||
sslmode: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
environment = dict(os.environ)
|
||||
environment.pop("PGPASSWORD", None)
|
||||
environment.pop("PGSSLMODE", None)
|
||||
if password:
|
||||
environment["PGPASSWORD"] = password
|
||||
if sslmode:
|
||||
environment["PGSSLMODE"] = sslmode
|
||||
return environment
|
||||
|
||||
|
||||
class SQLiteBackupBackend:
|
||||
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
|
||||
|
||||
db_type = "sqlite"
|
||||
suffix = ".db"
|
||||
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
self._engine = engine
|
||||
database = engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
raise ValueError("SQLite 内存数据库不支持文件备份")
|
||||
self._database = Path(database)
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""从活动引擎指向的 SQLite 文件创建一致快照。"""
|
||||
source = self._engine.raw_connection()
|
||||
try:
|
||||
with closing(sqlite3.connect(destination)) as target:
|
||||
source.driver_connection.backup(target)
|
||||
target.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 SQLite integrity_check 校验备份内容。"""
|
||||
return verify_database_backup(artifact, db_type=self.db_type)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
|
||||
temporary = self._database.with_name(f".{self._database.name}.restore")
|
||||
self._engine.dispose()
|
||||
try:
|
||||
shutil.copy2(artifact, temporary)
|
||||
temporary.chmod(0o600)
|
||||
self._database.with_name(f"{self._database.name}-wal").unlink(missing_ok=True)
|
||||
self._database.with_name(f"{self._database.name}-shm").unlink(missing_ok=True)
|
||||
os.replace(temporary, self._database)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class PostgreSQLBackupBackend:
|
||||
"""使用 pg_dump 与 pg_restore 管理活动 PostgreSQL 数据库。"""
|
||||
|
||||
db_type = "postgresql"
|
||||
suffix = ".dump"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine,
|
||||
*,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_dump: str = "pg_dump",
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._runner = runner
|
||||
self._tool_resolver = tool_resolver
|
||||
self._pg_dump = pg_dump
|
||||
self._pg_restore = pg_restore
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""创建 PostgreSQL custom-format 在线备份。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_dump),
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--file",
|
||||
str(destination),
|
||||
*self._connection_arguments(),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_dump 执行失败,退出码 {result.returncode}")
|
||||
if not destination.is_file() or destination.stat().st_size == 0:
|
||||
raise RuntimeError("pg_dump 未生成有效的备份文件")
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
|
||||
return verify_database_backup(
|
||||
artifact,
|
||||
db_type=self.db_type,
|
||||
runner=self._runner,
|
||||
tool_resolver=self._tool_resolver,
|
||||
pg_restore=self._pg_restore,
|
||||
)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_restore),
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--single-transaction",
|
||||
"--exit-on-error",
|
||||
*self._connection_arguments(),
|
||||
str(artifact),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_restore 执行失败,退出码 {result.returncode}")
|
||||
|
||||
def _connection_arguments(self) -> list[str]:
|
||||
url = self._engine.url
|
||||
host = str(url.query.get("host") or url.host or "")
|
||||
port = str(url.query.get("port") or url.port or "")
|
||||
arguments = [
|
||||
"--username",
|
||||
str(url.username or ""),
|
||||
"--dbname",
|
||||
str(url.database or ""),
|
||||
]
|
||||
if host:
|
||||
arguments.extend(["--host", host])
|
||||
if port:
|
||||
arguments.extend(["--port", port])
|
||||
return arguments
|
||||
|
||||
def _run(self, command: Sequence[str], *, include_password: bool) -> ProcessResult:
|
||||
return self._runner(
|
||||
command,
|
||||
env=self._environment(include_password=include_password),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _require_tool(self, executable: str) -> str:
|
||||
return _require_tool(executable, self._tool_resolver)
|
||||
|
||||
def _environment(self, *, include_password: bool) -> dict[str, str]:
|
||||
password = (
|
||||
str(self._engine.url.password)
|
||||
if include_password and self._engine.url.password
|
||||
else None
|
||||
)
|
||||
sslmode = self._engine.url.query.get("sslmode")
|
||||
return _postgres_environment(
|
||||
password=password,
|
||||
sslmode=str(sslmode) if sslmode else None,
|
||||
)
|
||||
@@ -1,128 +0,0 @@
|
||||
"""数据库备份单文件的受限文件系统操作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.runtime.version import get_app_version
|
||||
|
||||
_BACKUP_NAME = re.compile(
|
||||
r"^(?:moviepilot_(?P<version>v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?"
|
||||
r"(?P<db_type>sqlite|postgresql)_"
|
||||
r"(?P<timestamp>\d{8}_\d{6})"
|
||||
r"(?:_(?P<sequence>\d+))?"
|
||||
r"(?P<suffix>\.db|\.dump)$"
|
||||
)
|
||||
_RELEASE_VERSION = re.compile(r"^v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
|
||||
|
||||
|
||||
class BackupFiles:
|
||||
"""把备份文件操作限制在一个私有根目录内。"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def create_temporary(self, suffix: str) -> Path:
|
||||
"""在最终目录内创建私有临时文件,保证发布可使用原子替换。"""
|
||||
self._ensure_root()
|
||||
descriptor, filename = tempfile.mkstemp(
|
||||
prefix=".database-",
|
||||
suffix=f"{suffix}.partial",
|
||||
dir=self.root,
|
||||
)
|
||||
os.close(descriptor)
|
||||
path = Path(filename)
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
def publish(self, temporary: Path, name: str) -> Path:
|
||||
"""把已校验临时文件发布为正式备份文件。"""
|
||||
destination = self._resolve_name(name, require_exists=False)
|
||||
os.replace(temporary, destination)
|
||||
destination.chmod(0o600)
|
||||
return destination
|
||||
|
||||
def discard(self, temporary: Path) -> None:
|
||||
"""清理本次操作拥有的未发布临时文件。"""
|
||||
path = Path(temporary)
|
||||
if path.parent == self.root and path.name.startswith(".database-"):
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
def list(self) -> list[Path]:
|
||||
"""返回当前根目录内格式合法的正式备份文件。"""
|
||||
if not self.root.is_dir():
|
||||
return []
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.is_file() and _BACKUP_NAME.fullmatch(path.name)
|
||||
]
|
||||
return sorted(paths, key=lambda path: (self.created_at(path.name), path.name), reverse=True)
|
||||
|
||||
def resolve(self, name: str) -> Path:
|
||||
"""按受限文件名解析一个必须存在的备份文件。"""
|
||||
return self._resolve_name(name, require_exists=True)
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
"""删除一个已通过名称约束的备份文件。"""
|
||||
self.resolve(name).unlink()
|
||||
|
||||
def available_name(
|
||||
self,
|
||||
*,
|
||||
db_type: str,
|
||||
created_at: datetime,
|
||||
suffix: str,
|
||||
) -> str:
|
||||
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
|
||||
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
|
||||
version = get_app_version().strip()
|
||||
if _RELEASE_VERSION.fullmatch(version) is None:
|
||||
raise ValueError("程序版本号无法用于数据库备份命名")
|
||||
base = f"moviepilot_{version}_{db_type}_{timestamp}"
|
||||
candidate = f"{base}{suffix}"
|
||||
sequence = 1
|
||||
while (self.root / candidate).exists():
|
||||
candidate = f"{base}_{sequence}{suffix}"
|
||||
sequence += 1
|
||||
if not _BACKUP_NAME.fullmatch(candidate):
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def database_type(name: str) -> str:
|
||||
"""从受管文件名读取数据库类型。"""
|
||||
return BackupFiles._match(name).group("db_type")
|
||||
|
||||
@staticmethod
|
||||
def created_at(name: str) -> datetime:
|
||||
"""从受管文件名读取本地创建时间。"""
|
||||
return datetime.strptime(
|
||||
BackupFiles._match(name).group("timestamp"),
|
||||
"%Y%m%d_%H%M%S",
|
||||
)
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.root.chmod(0o700)
|
||||
|
||||
def _resolve_name(self, name: str, *, require_exists: bool) -> Path:
|
||||
normalized = str(name).strip()
|
||||
self._match(normalized)
|
||||
if Path(normalized).name != normalized:
|
||||
raise ValueError("数据库备份文件名不能包含路径")
|
||||
path = self.root / normalized
|
||||
if require_exists and not path.is_file():
|
||||
raise FileNotFoundError(normalized)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _match(name: str) -> re.Match[str]:
|
||||
matched = _BACKUP_NAME.fullmatch(str(name))
|
||||
if matched is None:
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return matched
|
||||
@@ -1,53 +0,0 @@
|
||||
"""虚拟显示适配器及旧 DisplayHelper 兼容入口。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.managed_resources import (
|
||||
acquire_managed_resource,
|
||||
stop_managed_resource,
|
||||
)
|
||||
|
||||
|
||||
DISPLAY_CAPABILITY_ID = "host.display"
|
||||
|
||||
|
||||
class DisplayHelper(metaclass=Singleton):
|
||||
"""保留旧构造 API,并把资源所有权委托给 host.display 能力。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""显式构造旧门面时激活虚拟显示,失败保持旧 API 的日志语义。"""
|
||||
try:
|
||||
acquire_managed_resource(
|
||||
DISPLAY_CAPABILITY_ID,
|
||||
reason="legacy_display_helper",
|
||||
retry=True,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error("DisplayHelper init error: %s", error)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止已激活的虚拟显示;未配置 Runtime 时保持幂等。"""
|
||||
stop_managed_resource(
|
||||
DISPLAY_CAPABILITY_ID,
|
||||
reason="legacy_display_helper_stop",
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["DISPLAY_CAPABILITY_ID", "DisplayHelper", "VirtualDisplayResource"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""按需公开资源实现,普通兼容导入不加载显示后端。"""
|
||||
if name != "VirtualDisplayResource":
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
value = getattr(
|
||||
import_module("app.adapters.system.display.resource"),
|
||||
"VirtualDisplayResource",
|
||||
)
|
||||
globals()[name] = value
|
||||
return value
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "host.display"
|
||||
kind = "managed_resource.sync"
|
||||
entrypoint = "app.adapters.system.display.resource:VirtualDisplayResource"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Virtual Display"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,45 +0,0 @@
|
||||
"""虚拟显示进程的托管资源实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
class VirtualDisplayResource:
|
||||
"""按需拥有一个容器内虚拟显示进程。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._display: Optional[Any] = None
|
||||
|
||||
@property
|
||||
def display(self) -> Optional[Any]:
|
||||
"""返回当前拥有的显示对象;未启动或已停止时为 None。"""
|
||||
return self._display
|
||||
|
||||
def start(self) -> None:
|
||||
"""仅在容器环境启动虚拟显示,重复启动保持幂等。"""
|
||||
if self._display is not None or not SystemUtils.is_docker():
|
||||
return
|
||||
from pyvirtualdisplay import Display
|
||||
|
||||
display = Display(
|
||||
visible=False,
|
||||
size=(1024, 768),
|
||||
extra_args=[os.environ["DISPLAY"]],
|
||||
)
|
||||
self._display = display
|
||||
display.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止当前资源拥有的显示进程,失败时保留句柄供 Runtime 重试。"""
|
||||
display = self._display
|
||||
if display is None:
|
||||
return
|
||||
logger.info("正在停止虚拟显示...")
|
||||
display.stop()
|
||||
self._display = None
|
||||
logger.info("虚拟显示已停止")
|
||||
@@ -1,483 +0,0 @@
|
||||
"""
|
||||
本地文件系统操作代理。
|
||||
|
||||
FUSE/网络挂载有两种故障形态:crash 型(调用抛错,可捕获、可重试)和 block 型
|
||||
(调用既不返回错误也不返回结果,永久悬挂)。**Python 无法中断一个已经发出的
|
||||
系统调用,也无法强杀线程**,所以 block 型故障下阻塞的线程永远无法回收——这正是
|
||||
整理消费线程停摆、监控自愈路径自冻的根因。
|
||||
|
||||
本模块把这些调用放进一个常驻子进程执行。子进程可以被 SIGKILL,因此超时后能真正
|
||||
回收;对调用方而言,超时表现为一个普通的 OSError 子类(FileSystemTimeout)。
|
||||
换句话说:**把不可处理的 block 型故障,转换成系统各层已经能正确处理的 crash 型
|
||||
故障**——退避重启、登记待重试这些既有机制立刻就能接管。
|
||||
|
||||
第一版只放行安全的操作:
|
||||
- 只读(stat/exists/listdir)——强杀不产生任何副作用
|
||||
- 同存储 rename——内核保证原子性,强杀后要么完全成功要么完全没发生
|
||||
跨存储的复制+删除不在此列,它需要单独的可恢复语义(临时名 + 完成后 rename)。
|
||||
"""
|
||||
import errno as errno_module
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
# worker 脚本路径。用绝对路径直接执行,而不是 -m 或 import:
|
||||
# 直接执行文件不会触发 app/__init__.py 的导入链,代理启动才是毫秒级的
|
||||
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
|
||||
# 单次快操作(stat/listdir/rename/unlink 等)的默认超时秒数
|
||||
DEFAULT_TIMEOUT = 30
|
||||
# 长耗时操作(复制)两次进度上报之间的最长间隔秒数。
|
||||
# worker 每秒上报一次心跳,因此这个阈值判定的是「传输完全没有推进」,
|
||||
# 而不是「传输很慢」——大文件复制几小时也不会误杀
|
||||
DEFAULT_STALL_TIMEOUT = 120
|
||||
# 强杀代理后等待它消失的宽限秒数,不能无限等待
|
||||
_KILL_GRACE = 5
|
||||
|
||||
|
||||
class FileSystemTimeout(OSError):
|
||||
"""
|
||||
文件系统操作在代理中超时未返回,判定挂载无响应。
|
||||
|
||||
继承 OSError 是刻意的:整理链、监控 watcher 等各层对 OSError 已有完整的
|
||||
退避重试与登记逻辑,block 型故障经此转换后可以直接复用它们。
|
||||
"""
|
||||
|
||||
|
||||
class FileSystemProxy:
|
||||
"""
|
||||
常驻子进程文件系统代理。
|
||||
|
||||
请求-响应严格串行(一个代理同时只处理一个请求),由锁保证。超时即强杀代理,
|
||||
下一次请求自动重启一个新的——启动成本是毫秒级,因为 worker 只依赖标准库。
|
||||
"""
|
||||
|
||||
def __init__(self, timeout: Optional[float] = None,
|
||||
stall_timeout: Optional[float] = None):
|
||||
"""
|
||||
:param timeout: 单次快操作的超时秒数,None 表示实时跟随系统设置
|
||||
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数,
|
||||
None 表示实时跟随系统设置
|
||||
"""
|
||||
self._timeout_override = timeout
|
||||
self._stall_timeout_override = stall_timeout
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._selector: Optional[selectors.BaseSelector] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 对外操作
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def stat(self, path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
读取路径属性。
|
||||
:param path: 目标路径
|
||||
:return: {"size", "mtime", "is_dir", "is_file"}
|
||||
"""
|
||||
return self._call("stat", path=str(path))
|
||||
|
||||
def exists(self, path: Path) -> bool:
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
只有 FileNotFoundError 才算「不存在」;其余 OSError(含超时)原样抛出,
|
||||
避免像 Path.exists() 那样把挂载抖动误判成文件消失。
|
||||
:param path: 目标路径
|
||||
:return: 是否存在
|
||||
"""
|
||||
try:
|
||||
self._call("exists", path=str(path))
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def listdir(self, path: Path) -> List[str]:
|
||||
"""
|
||||
列出目录条目名。
|
||||
:param path: 目标目录
|
||||
:return: 条目名列表
|
||||
"""
|
||||
return self._call("listdir", path=str(path))
|
||||
|
||||
def count_entries(self, path: Path, max_check: int = 10000) -> Dict[str, int]:
|
||||
"""
|
||||
统计目录规模。整棵树的遍历在子进程内一次完成,超时可整体放弃。
|
||||
:param path: 目标目录
|
||||
:param max_check: 文件数上限,超过即提前结束
|
||||
:return: {"file_count", "dir_count"}
|
||||
"""
|
||||
return self._call("count_entries", path=str(path), max_check=max_check)
|
||||
|
||||
def rename(self, src: Path, dst: Path) -> bool:
|
||||
"""
|
||||
同一存储内重命名/移动。跨存储会抛 OSError(EXDEV),由调用方走原有路径。
|
||||
:param src: 源路径
|
||||
:param dst: 目标路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rename", src=str(src), dst=str(dst))
|
||||
|
||||
def copy(self, src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]] = None,
|
||||
cancel_cb: Optional[Callable[[], bool]] = None,
|
||||
chunk_size: Optional[int] = None) -> Any:
|
||||
"""
|
||||
复制文件内容并保留时间戳,按「进度无推进」判定挂死。
|
||||
|
||||
复制大文件可能持续几小时,固定超时无法区分「正常但慢」和「已经挂死」。
|
||||
worker 每秒上报一次进度作为心跳,这里判定的是**两次上报之间的间隔**:
|
||||
超过 stall 阈值收不到任何一行,才认定挂载无响应并强杀 worker。
|
||||
|
||||
取消检查放在父进程:worker 里读不到 global_vars 的传输取消标记,而父进程
|
||||
每收到一次进度就能检查一次,要取消直接杀掉 worker 即可,比在子进程里
|
||||
轮询标记更干净。
|
||||
:param src: 源文件
|
||||
:param dst: 目标文件(调用方应传临时名,完成后自行原子替换)
|
||||
:param progress_cb: 进度回调,入参为百分比
|
||||
:param cancel_cb: 取消检查回调,返回 True 表示应中止
|
||||
:param chunk_size: 分块大小
|
||||
:return: 成功时为 {"copied", "total"},被取消或通信失败时为 False
|
||||
"""
|
||||
payload = {"src": str(src), "dst": str(dst)}
|
||||
if chunk_size:
|
||||
payload["chunk_size"] = chunk_size
|
||||
if not self._enabled():
|
||||
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request_stream(payload, progress_cb, cancel_cb)
|
||||
except FileSystemTimeout:
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
|
||||
self._shutdown()
|
||||
return False
|
||||
|
||||
def _request_stream(self, payload: Dict[str, Any],
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]]) -> Any:
|
||||
"""
|
||||
发起一次流式请求,逐行消费进度直到终态。
|
||||
:param payload: 请求参数
|
||||
:param progress_cb: 进度回调
|
||||
:param cancel_cb: 取消检查回调
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": "copy", **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
while True:
|
||||
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
|
||||
progress = response.get("progress")
|
||||
if progress is not None:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
logger.info(f"复制已取消: {payload.get('src')}")
|
||||
# 取消就地生效:杀掉 worker 立刻中断传输,不必等它读完整个文件
|
||||
self._shutdown()
|
||||
return False
|
||||
if progress_cb is not None:
|
||||
total = progress.get("total") or 0
|
||||
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
|
||||
continue
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
@staticmethod
|
||||
def _direct_copy(src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]],
|
||||
chunk_size: Optional[int]) -> bool:
|
||||
"""
|
||||
不经代理直接复制,供代理关闭时使用。
|
||||
"""
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
return False
|
||||
buf = fsrc.read(chunk_size or 1024 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(copied / total * 100)
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return True
|
||||
|
||||
def unlink(self, path: Path) -> bool:
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作,强杀后没有中间状态。
|
||||
:param path: 目标文件
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("unlink", path=str(path))
|
||||
|
||||
def rmtree(self, path: Path) -> bool:
|
||||
"""
|
||||
递归删除目录,容忍部分失败(可重复执行直到成功)。
|
||||
:param path: 目标目录
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rmtree", path=str(path))
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
关闭代理进程。
|
||||
"""
|
||||
with self._lock:
|
||||
self._shutdown()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内部实现
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _timeout(self) -> float:
|
||||
"""
|
||||
单次快操作的超时秒数。
|
||||
|
||||
实时读取而不是构造时固定:这三项都暴露在前端设置里,用户改完保存后
|
||||
必须立刻生效,否则会出现「改了没反应」的困惑。
|
||||
"""
|
||||
if self._timeout_override is not None:
|
||||
return self._timeout_override
|
||||
return float(get_runtime_setting("FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
|
||||
|
||||
@property
|
||||
def _stall_timeout(self) -> float:
|
||||
"""
|
||||
长耗时操作两次进度上报之间的最长间隔秒数,同样实时跟随系统设置。
|
||||
"""
|
||||
if self._stall_timeout_override is not None:
|
||||
return self._stall_timeout_override
|
||||
return float(
|
||||
get_runtime_setting("FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _enabled() -> bool:
|
||||
"""
|
||||
代理是否启用。关闭时退回直接调用,行为与引入代理之前完全一致。
|
||||
"""
|
||||
return bool(get_runtime_setting("FS_PROXY_ENABLED", True))
|
||||
|
||||
@staticmethod
|
||||
def _direct(op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
不经代理直接执行操作,供代理关闭时使用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if op == "stat":
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
if op == "exists":
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
if op == "listdir":
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
if op == "count_entries":
|
||||
file_count = dir_count = 0
|
||||
for _, dirs, files in os.walk(payload["path"]):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > (payload.get("max_check") or 10000):
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
if op == "rename":
|
||||
os.rename(payload["src"], payload["dst"])
|
||||
return True
|
||||
if op == "unlink":
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
if op == "rmtree":
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
raise ValueError(f"unknown op: {op}")
|
||||
|
||||
def _call(self, op: str, **payload) -> Any:
|
||||
"""
|
||||
执行一次代理调用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if not self._enabled():
|
||||
return self._direct(op, payload)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request(op, payload)
|
||||
except FileSystemTimeout:
|
||||
# 超时说明挂载正在挂死,重试只会再冻一次,直接上报给调用方
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
# 代理进程意外退出或响应损坏,重启后重试一次
|
||||
logger.debug(f"文件系统代理通信异常,重启后重试: {op} - {err}")
|
||||
self._shutdown()
|
||||
return self._request(op, payload)
|
||||
|
||||
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
发送请求并等待响应。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": op, **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
response = json.loads(self._read_line().decode("utf-8"))
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类,
|
||||
# 调用方沿用原有的异常分支即可,无需感知代理的存在
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
def _read_line(self, timeout: Optional[float] = None) -> bytes:
|
||||
"""
|
||||
读取一行响应,超时即强杀代理。
|
||||
:param timeout: 本次读取的超时秒数,默认用单次操作超时
|
||||
:return: 响应行
|
||||
"""
|
||||
timeout = self._timeout if timeout is None else timeout
|
||||
# Windows 的 select() 只能等 socket。subprocess.PIPE 是匿名管道,
|
||||
# selectors.DefaultSelector.select() 会抛 WinError 10038(WSAENOTSOCK),
|
||||
# 整理链每次 stat 都会直接失败。POSIX 继续用 selector,Windows 改走线程等待。
|
||||
if sys.platform == "win32":
|
||||
return self._read_line_windows(timeout)
|
||||
if not self._selector.select(timeout=timeout):
|
||||
self._raise_timeout(timeout)
|
||||
line = self._process.stdout.readline()
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _read_line_windows(self, timeout: float) -> bytes:
|
||||
"""
|
||||
在辅助线程上读一行,用 join(timeout) 实现可放弃等待。
|
||||
|
||||
不能用 select()/WaitForSingleObject:匿名管道在 Windows 上不是
|
||||
可选择的套接字,也不是可靠的「有数据」同步对象。
|
||||
:param timeout: 本次读取的超时秒数
|
||||
:return: 响应行
|
||||
"""
|
||||
if self._process is None or self._process.stdout is None:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
stdout = self._process.stdout
|
||||
holder: List[Union[bytes, BaseException]] = []
|
||||
|
||||
def _read() -> None:
|
||||
try:
|
||||
holder.append(stdout.readline())
|
||||
except Exception as err: # noqa: BLE001
|
||||
holder.append(err)
|
||||
|
||||
reader = threading.Thread(target=_read, name="fsproxy-stdout", daemon=True)
|
||||
reader.start()
|
||||
reader.join(timeout)
|
||||
if reader.is_alive():
|
||||
self._raise_timeout(timeout)
|
||||
if not holder:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
line = holder[0]
|
||||
if isinstance(line, BaseException):
|
||||
raise line
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _raise_timeout(self, timeout: float) -> None:
|
||||
"""
|
||||
判定挂载无响应:杀掉代理并抛出可被整理链处理的超时。
|
||||
:param timeout: 已等待的秒数
|
||||
"""
|
||||
logger.error(f"文件系统操作 {timeout} 秒无响应,判定挂载挂死,正在回收代理进程")
|
||||
self._shutdown()
|
||||
raise FileSystemTimeout(
|
||||
errno_module.ETIMEDOUT,
|
||||
f"文件系统操作超过 {timeout} 秒无响应,挂载可能已无响应"
|
||||
)
|
||||
|
||||
def _ensure_worker(self):
|
||||
"""
|
||||
确保代理进程可用,不可用时重新启动。
|
||||
"""
|
||||
if self._process is not None and self._process.poll() is None:
|
||||
return
|
||||
self._shutdown()
|
||||
popen_kwargs: Dict[str, Any] = {
|
||||
"stdin": subprocess.PIPE,
|
||||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.DEVNULL,
|
||||
"bufsize": 0,
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
# NSSM 服务里再拉 python.exe 时避免弹出控制台窗口
|
||||
popen_kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, str(_WORKER_PATH)],
|
||||
**popen_kwargs,
|
||||
)
|
||||
if sys.platform != "win32":
|
||||
self._selector = selectors.DefaultSelector()
|
||||
self._selector.register(self._process.stdout, selectors.EVENT_READ)
|
||||
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
|
||||
|
||||
def _shutdown(self):
|
||||
"""
|
||||
回收代理进程。冻在挂载上的进程用 SIGKILL,且不无限等待它消失
|
||||
——否则「可放弃的代理」又变回一次不可放弃的阻塞。
|
||||
"""
|
||||
if self._selector is not None:
|
||||
try:
|
||||
self._selector.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._selector = None
|
||||
process, self._process = self._process, None
|
||||
if process is None:
|
||||
return
|
||||
for stream in (process.stdin, process.stdout):
|
||||
try:
|
||||
if stream:
|
||||
stream.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=_KILL_GRACE)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warn(f"文件系统代理进程未能及时退出,交由系统回收: pid={process.pid}")
|
||||
except Exception as err: # noqa: BLE001
|
||||
logger.debug(f"回收文件系统代理进程失败: {err}")
|
||||
|
||||
|
||||
# 全局单例:local 存储本身是单例,代理也只需要一个
|
||||
# 不传超时参数:让它实时跟随系统设置,前端改完保存即刻生效
|
||||
fsproxy = FileSystemProxy()
|
||||
@@ -1,201 +0,0 @@
|
||||
"""
|
||||
文件系统操作代理 worker。
|
||||
|
||||
**本文件不能被 import,只能作为独立脚本执行**(fsproxy 用
|
||||
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动)。直接执行文件
|
||||
路径不会触发 `app/__init__.py` 的导入链,因此这个进程只依赖标准库、启动是
|
||||
毫秒级的;一旦走 import 就会把整个应用的依赖拉进来,代理被强杀后的重启成本
|
||||
会高到无法接受。
|
||||
|
||||
存在的理由:FUSE/网络挂载进入 block 型故障时,`stat`/`listdir`/`rename` 这类
|
||||
系统调用既不返回错误也不返回结果,而 Python 没有中断线程的手段——阻塞其上的
|
||||
线程永远无法回收。放进独立进程后,父进程可以在超时后 SIGKILL 掉它,把
|
||||
「不可处理的 block」转换成「可处理的 crash」。
|
||||
|
||||
协议:stdin/stdout 逐行 JSON。
|
||||
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
|
||||
成功 {"ok": true, "result": {...}}
|
||||
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _stat(payload, _emit):
|
||||
"""
|
||||
读取路径的基本属性。
|
||||
"""
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
|
||||
|
||||
def _exists(payload, _emit):
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
用 os.stat 而不是 os.path.exists:后者会把任意 OSError 都归为「不存在」,
|
||||
挂载抖动会被误判成文件消失。这里让异常原样抛出,由父进程按 errno 区分。
|
||||
"""
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _listdir(payload, _emit):
|
||||
"""
|
||||
列出目录下的条目名。
|
||||
"""
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
|
||||
|
||||
def _copy(payload, emit):
|
||||
"""
|
||||
分块复制文件内容并周期上报进度。
|
||||
|
||||
进度上报同时充当心跳:复制大文件可能持续几小时,父进程无法用固定超时判断
|
||||
挂死,只能看「两次上报之间隔了多久」。因此这里按固定时间间隔上报,即使
|
||||
某一秒没读到数据也照常发——一旦挂载卡住,read/write 不返回,上报自然断流,
|
||||
父进程据此判定并强杀本进程。
|
||||
|
||||
只复制内容和时间戳,不复制权限:目标目录的默认权限与继承 ACL 是媒体库的
|
||||
访问策略,用源文件权限覆盖会清除已继承的 ACL。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
chunk_size = payload.get("chunk_size") or 1024 * 1024
|
||||
interval = payload.get("progress_interval") or 1.0
|
||||
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
# 先发一次 0%:既让心跳立刻开始,也保证父进程在传输开始前就有一次检查
|
||||
# 取消的机会——否则小文件会在首次定时上报之前就复制完,取消形同虚设
|
||||
emit({"ok": True, "progress": {"copied": 0, "total": total}})
|
||||
last_emit = time.monotonic()
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
buf = fsrc.read(chunk_size)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= interval:
|
||||
last_emit = now
|
||||
emit({"ok": True, "progress": {"copied": copied, "total": total}})
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return {"copied": copied, "total": total}
|
||||
|
||||
|
||||
def _count_entries(payload, _emit):
|
||||
"""
|
||||
统计目录下的文件与子目录数量,超过上限即提前结束。
|
||||
|
||||
放在子进程里做而不是逐层 listdir 走 IPC:递归遍历一棵大目录树会产生成千
|
||||
上万次往返,代价不可接受;一次调用在子进程内跑完 os.walk,父进程只需对
|
||||
这一次调用设超时即可整体放弃。
|
||||
"""
|
||||
directory = payload["path"]
|
||||
max_check = payload.get("max_check") or 10000
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
for _, dirs, files in os.walk(directory):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > max_check:
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
|
||||
|
||||
def _rename(payload, _emit):
|
||||
"""
|
||||
同一存储内重命名/移动。
|
||||
|
||||
这是第一版唯一放行的写操作:同文件系统内的 rename 由内核保证原子性,
|
||||
进程被强杀后要么完全成功要么完全没发生,不存在需要清理的中间状态。
|
||||
跨存储的复制+删除不走这里,它需要单独的可恢复语义。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
|
||||
raise OSError(18, "Cross-device rename is not handled by the proxy")
|
||||
os.rename(src, dst)
|
||||
return True
|
||||
|
||||
|
||||
def _unlink(payload, _emit):
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作,强杀后要么删掉了要么没删,没有中间状态。
|
||||
"""
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _rmtree(payload, _emit):
|
||||
"""
|
||||
递归删除目录。
|
||||
|
||||
这一项不是原子的,强杀可能只删掉一部分。放行的理由是:删除被中断的后果
|
||||
(残留若干文件)远轻于写入被中断(留下叫最终文件名的半成品),而且调用方
|
||||
本来就以 ignore_errors 容忍部分失败、可以重复执行直到成功。
|
||||
"""
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
|
||||
|
||||
_HANDLERS = {
|
||||
"stat": _stat,
|
||||
"exists": _exists,
|
||||
"listdir": _listdir,
|
||||
"copy": _copy,
|
||||
"count_entries": _count_entries,
|
||||
"rename": _rename,
|
||||
"unlink": _unlink,
|
||||
"rmtree": _rmtree,
|
||||
"ping": lambda _payload, _emit: True,
|
||||
}
|
||||
|
||||
|
||||
def _write(message):
|
||||
"""
|
||||
输出一行响应。
|
||||
"""
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
请求循环:每读一行处理一个请求,直到 stdin 关闭。
|
||||
|
||||
一个请求可能对应多行响应:长耗时操作先流式发若干 progress 行,最后发一行
|
||||
终态(result 或 error)。父进程据此区分「还在推进」和「已经挂死」。
|
||||
"""
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
handler = _HANDLERS.get(payload.get("op"))
|
||||
if handler is None:
|
||||
response = {"ok": False, "errno": 0,
|
||||
"error": f"unknown op: {payload.get('op')}"}
|
||||
else:
|
||||
response = {"ok": True, "result": handler(payload, _write)}
|
||||
except OSError as err:
|
||||
response = {"ok": False, "errno": err.errno or 0,
|
||||
"error": err.strerror or str(err)}
|
||||
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
|
||||
response = {"ok": False, "errno": 0, "error": str(err)}
|
||||
_write(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,188 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
|
||||
from app.runtime.dependencies import runtime_sync_arguments
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageInstallRequest:
|
||||
"""
|
||||
Python 包安装请求,集中描述依赖清单、工具缓存、代理和本地 wheels 候选源。
|
||||
"""
|
||||
|
||||
dependency_files: tuple[Path, ...]
|
||||
python_bin: Path
|
||||
find_links_dirs: list[Path] = field(default_factory=list)
|
||||
constraints_file: Path | None = None
|
||||
config_dir: Path = Path("/config")
|
||||
package_cache_root: Path | None = None
|
||||
package_index_url: str | None = None
|
||||
proxy_url: str | None = None
|
||||
purpose: str = "plugin"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PackageInstallStrategy:
|
||||
"""
|
||||
单次安装尝试的完整执行信息,命令和日志展示命令分离以避免泄露凭据。
|
||||
"""
|
||||
|
||||
strategy_name: str
|
||||
command: list[str]
|
||||
env: dict[str, str]
|
||||
safe_log_command: list[str]
|
||||
|
||||
|
||||
def redact_url(value: str) -> str:
|
||||
"""
|
||||
脱敏 URL 中的 userinfo,保留 scheme、host、path、query 便于定位镜像源。
|
||||
"""
|
||||
parsed = urlsplit(value)
|
||||
if "@" not in parsed.netloc:
|
||||
return value
|
||||
host = parsed.netloc.rsplit("@", 1)[-1]
|
||||
return urlunsplit((parsed.scheme, host, parsed.path, parsed.query, parsed.fragment))
|
||||
|
||||
|
||||
def redact_command(command: list[str]) -> list[str]:
|
||||
"""
|
||||
脱敏命令参数中的 URL 凭据,用于日志展示。
|
||||
"""
|
||||
return [redact_url(item) if "://" in item else item for item in command]
|
||||
|
||||
|
||||
def build_package_install_env(request: PackageInstallRequest, include_moviepilot_proxy: bool = True) -> dict[str, str]:
|
||||
"""
|
||||
构造 uv 安装子进程环境,默认把包下载缓存放到持久化配置目录。
|
||||
"""
|
||||
env = os.environ.copy()
|
||||
config_dir = Path(request.config_dir)
|
||||
if request.package_cache_root:
|
||||
package_cache_root = Path(request.package_cache_root)
|
||||
env["PACKAGE_CACHE_ROOT"] = str(package_cache_root)
|
||||
else:
|
||||
package_cache_root = Path(env.get("PACKAGE_CACHE_ROOT") or config_dir / ".cache")
|
||||
env.setdefault("PACKAGE_CACHE_ROOT", str(package_cache_root))
|
||||
env.setdefault("UV_CACHE_DIR", str(package_cache_root / "uv"))
|
||||
proxy = (request.proxy_url or "").strip()
|
||||
if proxy and include_moviepilot_proxy:
|
||||
for key in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"):
|
||||
env[key] = proxy
|
||||
return env
|
||||
|
||||
|
||||
def find_uv(python_bin: Path) -> Path | None:
|
||||
"""
|
||||
优先使用解释器同目录 uv,保证安装器与目标运行环境使用同一版本。
|
||||
"""
|
||||
uv_name = "uv.exe" if os.name == "nt" else "uv"
|
||||
sibling = python_bin.with_name(uv_name)
|
||||
if sibling.exists():
|
||||
return sibling
|
||||
found = shutil.which("uv")
|
||||
return Path(found) if found else None
|
||||
|
||||
|
||||
def _base_install_args(request: PackageInstallRequest) -> list[str]:
|
||||
args: list[str] = []
|
||||
for directory in request.find_links_dirs:
|
||||
args.extend(["--find-links", str(directory)])
|
||||
if request.constraints_file:
|
||||
args.extend(["-c", str(request.constraints_file)])
|
||||
for dependency_file in request.dependency_files:
|
||||
args.extend(["-r", str(dependency_file)])
|
||||
return args
|
||||
|
||||
|
||||
def _network_variants(request: PackageInstallRequest) -> list[tuple[str, bool, bool]]:
|
||||
has_index = bool((request.package_index_url or "").strip())
|
||||
has_proxy = bool((request.proxy_url or "").strip())
|
||||
variants: list[tuple[str, bool, bool]] = []
|
||||
if has_index and has_proxy:
|
||||
variants.append(("镜像+代理", True, True))
|
||||
if has_index:
|
||||
variants.append(("镜像", True, False))
|
||||
if has_proxy:
|
||||
variants.append(("代理", False, True))
|
||||
variants.append(("直连", False, False))
|
||||
return variants
|
||||
|
||||
|
||||
def _build_uv_command(uv_bin: Path, request: PackageInstallRequest, use_index: bool) -> list[str]:
|
||||
command = [str(uv_bin), "pip", "install", "--python", str(request.python_bin)]
|
||||
if use_index and request.package_index_url:
|
||||
command.extend(["--default-index", request.package_index_url])
|
||||
command.extend(_base_install_args(request))
|
||||
return command
|
||||
|
||||
|
||||
def _build_uv_sync_command(uv_bin: Path, request: PackageInstallRequest, use_index: bool) -> list[str]:
|
||||
if len(request.dependency_files) != 1:
|
||||
raise ValueError("主项目锁定依赖恢复只接受一个 pyproject.toml")
|
||||
project_file = request.dependency_files[0]
|
||||
command = [
|
||||
str(uv_bin),
|
||||
"sync",
|
||||
"--project",
|
||||
str(project_file.parent),
|
||||
"--locked",
|
||||
"--no-dev",
|
||||
"--no-install-project",
|
||||
"--inexact",
|
||||
*runtime_sync_arguments(),
|
||||
]
|
||||
if use_index and request.package_index_url:
|
||||
command.extend(["--default-index", request.package_index_url])
|
||||
return command
|
||||
|
||||
|
||||
def build_package_install_strategies(request: PackageInstallRequest) -> list[PackageInstallStrategy]:
|
||||
"""
|
||||
为固定 uv 安装器构造镜像、代理和直连降级策略。
|
||||
"""
|
||||
strategies: list[PackageInstallStrategy] = []
|
||||
variants = _network_variants(request)
|
||||
uv_bin = find_uv(Path(request.python_bin))
|
||||
if not uv_bin:
|
||||
return strategies
|
||||
|
||||
for variant_name, use_index, use_proxy in variants:
|
||||
command = _build_uv_command(uv_bin, request, use_index)
|
||||
env = build_package_install_env(request, include_moviepilot_proxy=use_proxy)
|
||||
strategies.append(
|
||||
PackageInstallStrategy(
|
||||
strategy_name=f"uv:{variant_name}",
|
||||
command=command,
|
||||
env=env,
|
||||
safe_log_command=redact_command(command),
|
||||
)
|
||||
)
|
||||
return strategies
|
||||
|
||||
|
||||
def build_project_sync_strategies(request: PackageInstallRequest) -> list[PackageInstallStrategy]:
|
||||
"""为主项目锁定依赖恢复构造 uv 网络降级策略。"""
|
||||
uv_bin = find_uv(Path(request.python_bin))
|
||||
if not uv_bin:
|
||||
return []
|
||||
|
||||
strategies = []
|
||||
project_environment = request.python_bin.parent.parent
|
||||
for variant_name, use_index, use_proxy in _network_variants(request):
|
||||
command = _build_uv_sync_command(uv_bin, request, use_index)
|
||||
env = build_package_install_env(request, include_moviepilot_proxy=use_proxy)
|
||||
env["UV_PROJECT_ENVIRONMENT"] = str(project_environment)
|
||||
strategies.append(
|
||||
PackageInstallStrategy(
|
||||
strategy_name=f"uv:{variant_name}",
|
||||
command=command,
|
||||
env=env,
|
||||
safe_log_command=redact_command(command),
|
||||
)
|
||||
)
|
||||
return strategies
|
||||
@@ -1 +0,0 @@
|
||||
"""插件包和依赖系统适配器。"""
|
||||
@@ -1,380 +0,0 @@
|
||||
"""插件 Python 依赖聚合和安装适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from importlib.metadata import PackageNotFoundError, distribution, distributions
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from packaging.markers import default_environment
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.specifiers import InvalidSpecifier, SpecifierSet
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
from app.adapters.system.plugin.manifest import (
|
||||
PluginDependencyManifestError,
|
||||
load_dependency_manifest,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RequirementGroup:
|
||||
"""聚合同一包和安装来源的 extras 与版本约束。"""
|
||||
|
||||
name: str # PEP 503 规范化后的包名
|
||||
url: Optional[str] # direct reference 来源;为空表示从索引安装
|
||||
extras: set[str] = field(default_factory=set) # 所有插件要求启用的 extras
|
||||
specifiers: set[str] = field(default_factory=set) # 待求交集的版本约束
|
||||
|
||||
|
||||
class PluginDependencyInstaller:
|
||||
"""独立负责插件依赖扫描、约束合并和安装。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
helper: Any = None,
|
||||
*,
|
||||
installed_plugins_provider: Optional[Callable[[], list[str]]] = None,
|
||||
plugin_dir: Optional[Path] = None,
|
||||
) -> None:
|
||||
"""保存包安装端口和启动层提供的已安装插件读取器。"""
|
||||
if helper is None:
|
||||
from app.adapters.external.market import PluginHelper
|
||||
|
||||
helper = PluginHelper()
|
||||
self._helper = helper
|
||||
self._installed_plugins_provider = installed_plugins_provider or (lambda: [])
|
||||
self._plugin_dir = plugin_dir or (
|
||||
Path(get_runtime_setting('ROOT_PATH')) / "app" / "plugins"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _standardize(name: str) -> str:
|
||||
"""按 PEP 503 兼容规则标准化依赖包名。"""
|
||||
return (name or "").lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
@classmethod
|
||||
def _installed_packages(cls) -> dict[str, Version]:
|
||||
"""读取当前 Python 环境中可解析版本的已安装包。"""
|
||||
installed: dict[str, Version] = {}
|
||||
try:
|
||||
for distribution in distributions():
|
||||
name = distribution.metadata.get("Name")
|
||||
version = distribution.metadata.get("Version") or getattr(
|
||||
distribution,
|
||||
"version",
|
||||
None,
|
||||
)
|
||||
if not name or not version:
|
||||
continue
|
||||
package_name = cls._standardize(name)
|
||||
try:
|
||||
parsed = Version(version)
|
||||
except InvalidVersion:
|
||||
logger.debug(
|
||||
f"无法解析已安装包 '{package_name}' 的版本:{version}"
|
||||
)
|
||||
continue
|
||||
if package_name not in installed or parsed > installed[package_name]:
|
||||
installed[package_name] = parsed
|
||||
except Exception as err:
|
||||
logger.error(f"获取已安装的包时发生错误:{err}")
|
||||
return installed
|
||||
|
||||
@classmethod
|
||||
def _installed_distribution(cls, package_name: str) -> Any | None:
|
||||
"""读取一个包的元数据,用于校验 extras 和 direct URL 来源。"""
|
||||
try:
|
||||
return distribution(package_name)
|
||||
except PackageNotFoundError:
|
||||
return None
|
||||
|
||||
def _requirement_satisfied(
|
||||
self,
|
||||
requirement: Requirement,
|
||||
installed: dict[str, Version],
|
||||
*,
|
||||
seen: Optional[set[tuple[str, tuple[str, ...], Optional[str]]]] = None,
|
||||
) -> bool:
|
||||
"""同时校验版本、extras 及 direct URL,不把同名包误认为同一制品。"""
|
||||
package_name = self._standardize(requirement.name)
|
||||
installed_version = installed.get(package_name)
|
||||
try:
|
||||
if installed_version is None or not SpecifierSet(
|
||||
requirement.specifier
|
||||
).contains(installed_version, prereleases=True):
|
||||
return False
|
||||
except InvalidSpecifier as err:
|
||||
logger.error(f"依赖 {package_name} 约束无效:{err}")
|
||||
return False
|
||||
|
||||
installed_distribution = self._installed_distribution(package_name)
|
||||
if installed_distribution is None:
|
||||
return False if requirement.extras or requirement.url else True
|
||||
|
||||
if requirement.url and not self._direct_url_matches(
|
||||
installed_distribution, requirement.url
|
||||
):
|
||||
return False
|
||||
|
||||
requested_extras = {
|
||||
self._standardize_extra(extra) for extra in requirement.extras
|
||||
}
|
||||
if requested_extras:
|
||||
provided_extras = {
|
||||
self._standardize_extra(extra)
|
||||
for extra in installed_distribution.metadata.get_all(
|
||||
"Provides-Extra"
|
||||
)
|
||||
or []
|
||||
}
|
||||
if not requested_extras.issubset(provided_extras):
|
||||
return False
|
||||
|
||||
marker_key = (package_name, tuple(sorted(requested_extras)), requirement.url)
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if marker_key in seen:
|
||||
return True
|
||||
seen.add(marker_key)
|
||||
|
||||
for raw_dependency in installed_distribution.metadata.get_all(
|
||||
"Requires-Dist"
|
||||
) or []:
|
||||
try:
|
||||
extra_dependency = Requirement(raw_dependency)
|
||||
except Exception as err:
|
||||
logger.debug(
|
||||
f"无法解析已安装包 {package_name} 的依赖项 '{raw_dependency}':{err}"
|
||||
)
|
||||
continue
|
||||
if not self._marker_matches_for_extras(
|
||||
extra_dependency, requested_extras
|
||||
):
|
||||
continue
|
||||
if not self._requirement_satisfied(
|
||||
extra_dependency, installed, seen=seen
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _marker_matches_for_extras(
|
||||
cls, requirement: Requirement, extras: set[str]
|
||||
) -> bool:
|
||||
"""判断已安装发行版声明的可选依赖是否属于当前请求的 extra。"""
|
||||
if requirement.marker is None:
|
||||
return True
|
||||
environment = default_environment()
|
||||
if "extra" in str(requirement.marker):
|
||||
return any(
|
||||
requirement.marker.evaluate({**environment, "extra": extra})
|
||||
for extra in extras
|
||||
)
|
||||
return requirement.marker.evaluate(environment)
|
||||
|
||||
@staticmethod
|
||||
def _standardize_extra(name: str) -> str:
|
||||
"""按 PEP 685 兼容规则标准化 extra 名称。"""
|
||||
return (name or "").lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
@staticmethod
|
||||
def _direct_url_matches(installed_distribution: Any, required_url: str) -> bool:
|
||||
"""校验安装发行版记录的 PEP 610 URL 与清单来源一致。"""
|
||||
try:
|
||||
payload = installed_distribution.read_text("direct_url.json")
|
||||
if not payload:
|
||||
return False
|
||||
direct_url = json.loads(payload).get("url")
|
||||
if not isinstance(direct_url, str):
|
||||
return False
|
||||
return PluginDependencyInstaller._canonical_direct_url(
|
||||
required_url
|
||||
) == PluginDependencyInstaller._canonical_direct_url(direct_url)
|
||||
except (AttributeError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _canonical_direct_url(value: str) -> tuple[str, str, str, str, str]:
|
||||
"""规范化来源 URL,同时保留 fragment 中可能存在的校验信息。"""
|
||||
parsed = urlsplit(value)
|
||||
netloc = parsed.netloc.rsplit("@", 1)[-1].lower()
|
||||
return (
|
||||
parsed.scheme.lower(),
|
||||
netloc,
|
||||
parsed.path.rstrip("/"),
|
||||
parsed.query,
|
||||
parsed.fragment,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _merge(cls, dependencies: list[Requirement]) -> list[Requirement]:
|
||||
"""按包和安装来源合并 extras 与约束,保留完整安装目标。"""
|
||||
groups: dict[tuple[str, Optional[str]], _RequirementGroup] = {}
|
||||
for requirement in dependencies:
|
||||
package_name = cls._standardize(requirement.name)
|
||||
key = (package_name, requirement.url)
|
||||
group = groups.setdefault(
|
||||
key,
|
||||
_RequirementGroup(name=package_name, url=requirement.url),
|
||||
)
|
||||
group.extras.update(requirement.extras)
|
||||
group.specifiers.add(str(requirement.specifier))
|
||||
|
||||
merged: list[Requirement] = []
|
||||
for group in groups.values():
|
||||
spec_set = SpecifierSet()
|
||||
for specifier in group.specifiers:
|
||||
if not specifier:
|
||||
continue
|
||||
try:
|
||||
spec_set &= SpecifierSet(specifier)
|
||||
except InvalidSpecifier as err:
|
||||
logger.error(f"发生版本约束冲突:{err}")
|
||||
target = group.name
|
||||
if group.extras:
|
||||
target += f"[{','.join(sorted(group.extras))}]"
|
||||
if group.url:
|
||||
target += f" @ {group.url}"
|
||||
elif spec_set:
|
||||
target += str(spec_set)
|
||||
merged.append(Requirement(target))
|
||||
return merged
|
||||
|
||||
def _plugin_manifests(self) -> list[Any]:
|
||||
"""返回已安装插件当前生效的依赖清单。"""
|
||||
manifests = []
|
||||
installed_plugins = {
|
||||
plugin_id.lower()
|
||||
for plugin_id in self._installed_plugins_provider() or []
|
||||
}
|
||||
try:
|
||||
plugin_dirs = list(self._plugin_dir.iterdir())
|
||||
except (FileNotFoundError, OSError):
|
||||
return []
|
||||
for plugin_dir in sorted(plugin_dirs, key=lambda item: item.name):
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
if plugin_dir.name not in installed_plugins:
|
||||
logger.debug(f"忽略插件 {plugin_dir.name} 的依赖")
|
||||
continue
|
||||
manifest = load_dependency_manifest(plugin_dir)
|
||||
if manifest is None:
|
||||
continue
|
||||
manifests.append(manifest)
|
||||
return manifests
|
||||
|
||||
def _plugin_dependencies(self) -> list[Requirement]:
|
||||
"""扫描已安装插件的生效依赖清单并合并版本约束。"""
|
||||
dependencies: list[Requirement] = []
|
||||
for manifest in self._plugin_manifests():
|
||||
for requirement in manifest.dependencies:
|
||||
if requirement.marker and not requirement.marker.evaluate():
|
||||
continue
|
||||
dependencies.append(requirement)
|
||||
return self._merge(dependencies)
|
||||
|
||||
def find_missing(self) -> list[str]:
|
||||
"""返回当前插件集合缺失或不满足约束的依赖项。"""
|
||||
try:
|
||||
required = self._plugin_dependencies()
|
||||
installed = self._installed_packages()
|
||||
missing = []
|
||||
for requirement in required:
|
||||
if not self._requirement_satisfied(requirement, installed):
|
||||
missing.append(str(requirement))
|
||||
return missing
|
||||
except PluginDependencyManifestError:
|
||||
raise
|
||||
except Exception as err:
|
||||
logger.error(f"收集所有需要安装或更新的依赖项时发生错误:{err}")
|
||||
return []
|
||||
|
||||
def classify_plugins(self) -> tuple[list[str], list[str], list[str]]:
|
||||
"""按源码和依赖状态划分已安装插件。"""
|
||||
ready: list[str] = []
|
||||
missing_dependencies: list[str] = []
|
||||
missing_source: list[str] = []
|
||||
installed_packages = self._installed_packages()
|
||||
|
||||
for plugin_id in self._installed_plugins_provider() or []:
|
||||
plugin_dir = self._plugin_dir / plugin_id.lower()
|
||||
if not plugin_dir.is_dir():
|
||||
missing_source.append(plugin_id)
|
||||
continue
|
||||
try:
|
||||
manifest = load_dependency_manifest(plugin_dir)
|
||||
requirements = [] if manifest is None else [
|
||||
requirement
|
||||
for requirement in manifest.dependencies
|
||||
if not requirement.marker or requirement.marker.evaluate()
|
||||
]
|
||||
except PluginDependencyManifestError as error:
|
||||
logger.error(f"插件 {plugin_id} 依赖清单无效:{error}")
|
||||
missing_dependencies.append(plugin_id)
|
||||
continue
|
||||
if all(
|
||||
self._requirement_satisfied(requirement, installed_packages)
|
||||
for requirement in requirements
|
||||
):
|
||||
ready.append(plugin_id)
|
||||
else:
|
||||
missing_dependencies.append(plugin_id)
|
||||
|
||||
return ready, missing_dependencies, missing_source
|
||||
|
||||
def _wheels_dirs(self) -> list[Path]:
|
||||
"""收集已安装插件附带的本地 wheels 目录。"""
|
||||
result = []
|
||||
installed_plugins = {
|
||||
plugin_id.lower()
|
||||
for plugin_id in self._installed_plugins_provider() or []
|
||||
}
|
||||
for plugin_id in installed_plugins:
|
||||
wheels_dir = self._plugin_dir / plugin_id / "wheels"
|
||||
if wheels_dir.is_dir():
|
||||
result.append(wheels_dir)
|
||||
return list(dict.fromkeys(result))
|
||||
|
||||
def install(self, dependencies: list[str]) -> tuple[bool, str]:
|
||||
"""把已安装插件的原始清单交给一次统一包安装。"""
|
||||
if not dependencies:
|
||||
return False, "没有传入需要安装的依赖项"
|
||||
try:
|
||||
manifest_paths = [manifest.path for manifest in self._plugin_manifests()]
|
||||
if not manifest_paths:
|
||||
return False, "没有找到已安装插件的依赖清单"
|
||||
return self._helper.install_packages_with_fallback(
|
||||
manifest_paths,
|
||||
self._wheels_dirs(),
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"安装依赖项时发生错误:{err}")
|
||||
return False, f"安装依赖项时发生错误:{err}"
|
||||
|
||||
async def async_find_missing(self) -> list[str]:
|
||||
"""在线程池中扫描缺失依赖,避免阻塞事件循环。"""
|
||||
return await asyncio.to_thread(self.find_missing)
|
||||
|
||||
async def async_install(self, dependencies: list[str]) -> tuple[bool, str]:
|
||||
"""异步安装依赖,使用可取消的包安装子进程。"""
|
||||
if not dependencies:
|
||||
return False, "没有传入需要安装的依赖项"
|
||||
try:
|
||||
manifest_paths = [manifest.path for manifest in self._plugin_manifests()]
|
||||
if not manifest_paths:
|
||||
return False, "没有找到已安装插件的依赖清单"
|
||||
return await self._helper.async_install_packages_with_fallback(
|
||||
manifest_paths,
|
||||
self._wheels_dirs(),
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"安装依赖项时发生错误:{err}")
|
||||
return False, f"安装依赖项时发生错误:{err}"
|
||||
@@ -1,162 +0,0 @@
|
||||
"""插件 Python 依赖清单的选择和解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
PYPROJECT_FILENAME = "pyproject.toml"
|
||||
REQUIREMENTS_FILENAME = "requirements.txt"
|
||||
DEPENDENCY_MANIFEST_PRIORITY = (
|
||||
PYPROJECT_FILENAME,
|
||||
REQUIREMENTS_FILENAME,
|
||||
)
|
||||
DEPENDENCY_MANIFEST_FILENAMES = frozenset(
|
||||
DEPENDENCY_MANIFEST_PRIORITY
|
||||
)
|
||||
|
||||
|
||||
class PluginDependencyManifestError(ValueError):
|
||||
"""表示生效的现代依赖清单无法安全消费。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PluginDependencyManifest:
|
||||
"""保存插件当前生效的依赖清单及其已解析依赖。"""
|
||||
|
||||
path: Path
|
||||
dependencies: tuple[Requirement, ...]
|
||||
|
||||
|
||||
def select_dependency_manifest(plugin_dir: Path) -> Path | None:
|
||||
"""按现代清单优先级返回插件当前生效的依赖文件。"""
|
||||
pyproject_file = plugin_dir / PYPROJECT_FILENAME
|
||||
if pyproject_file.is_file():
|
||||
return pyproject_file
|
||||
requirements_file = plugin_dir / REQUIREMENTS_FILENAME
|
||||
if requirements_file.is_file():
|
||||
return requirements_file
|
||||
return None
|
||||
|
||||
|
||||
def dependency_manifest_status(event_path: Path) -> bool | None:
|
||||
"""判断文件事件是否改变生效清单,非清单文件返回 None。"""
|
||||
if event_path.name not in DEPENDENCY_MANIFEST_FILENAMES:
|
||||
return None
|
||||
active_manifest = select_dependency_manifest(event_path.parent)
|
||||
if event_path.is_file():
|
||||
return active_manifest == event_path
|
||||
if active_manifest is None:
|
||||
return True
|
||||
return DEPENDENCY_MANIFEST_PRIORITY.index(
|
||||
event_path.name
|
||||
) < DEPENDENCY_MANIFEST_PRIORITY.index(active_manifest.name)
|
||||
|
||||
|
||||
def load_dependency_manifest(
|
||||
plugin_dir: Path,
|
||||
) -> PluginDependencyManifest | None:
|
||||
"""读取插件当前生效的依赖清单,现代清单无效时拒绝回退。"""
|
||||
manifest_path = select_dependency_manifest(plugin_dir)
|
||||
if manifest_path is None:
|
||||
return None
|
||||
return load_dependency_file(manifest_path)
|
||||
|
||||
|
||||
def load_dependency_file(path: Path) -> PluginDependencyManifest:
|
||||
"""读取指定依赖文件,pyproject 严格校验,其余文件保持旧格式兼容。"""
|
||||
if path.name == PYPROJECT_FILENAME:
|
||||
dependencies = _load_pyproject_dependencies(path)
|
||||
else:
|
||||
dependencies = _load_requirements_dependencies(path)
|
||||
return PluginDependencyManifest(
|
||||
path=path,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
|
||||
|
||||
def _load_pyproject_dependencies(path: Path) -> tuple[Requirement, ...]:
|
||||
"""严格读取 PEP 621 ``project.dependencies``。"""
|
||||
try:
|
||||
with path.open("rb") as file:
|
||||
document = tomllib.load(file)
|
||||
except (OSError, tomllib.TOMLDecodeError) as err:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 无法解析:{err}"
|
||||
) from err
|
||||
|
||||
project = document.get("project")
|
||||
if not isinstance(project, Mapping):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 缺少 [project] 表"
|
||||
)
|
||||
name = project.get("name")
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.name 必须是非空字符串"
|
||||
)
|
||||
dynamic = project.get("dynamic", [])
|
||||
if not isinstance(dynamic, list) or not all(
|
||||
isinstance(item, str) for item in dynamic
|
||||
):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.dynamic 必须是字符串数组"
|
||||
)
|
||||
if "dependencies" in dynamic:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 不支持动态 dependencies"
|
||||
)
|
||||
version = project.get("version")
|
||||
if "version" in dynamic:
|
||||
if version is not None:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 不能同时静态和动态声明 version"
|
||||
)
|
||||
elif not isinstance(version, str) or not version.strip():
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 必须声明非空 project.version,"
|
||||
"或将 version 加入 project.dynamic"
|
||||
)
|
||||
raw_dependencies = project.get("dependencies", [])
|
||||
if not isinstance(raw_dependencies, list) or not all(
|
||||
isinstance(item, str) for item in raw_dependencies
|
||||
):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.dependencies 必须是字符串数组"
|
||||
)
|
||||
|
||||
dependencies: list[Requirement] = []
|
||||
for item in raw_dependencies:
|
||||
try:
|
||||
dependencies.append(Requirement(item))
|
||||
except Exception as err:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 包含无效依赖项 {item!r}:{err}"
|
||||
) from err
|
||||
return tuple(dependencies)
|
||||
|
||||
|
||||
def _load_requirements_dependencies(path: Path) -> tuple[Requirement, ...]:
|
||||
"""按旧行为逐行读取 requirements,忽略无法解析的兼容内容。"""
|
||||
dependencies: list[Requirement] = []
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError as err:
|
||||
logger.error(f"解析 requirements.txt 时发生错误:{err}")
|
||||
return ()
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
dependencies.append(Requirement(line))
|
||||
except Exception as err:
|
||||
logger.debug(f"无法解析依赖项 '{line}':{err}")
|
||||
return tuple(dependencies)
|
||||
@@ -1,740 +0,0 @@
|
||||
"""插件包文件安装、快照恢复和分身处理适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from app.adapters.external.market import PluginHelper as _PluginHelper
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.runtime.execution import (
|
||||
run_in_threadpool_to_completion as _await_thread_operation,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PluginPackageCheckpoint:
|
||||
"""记录运行目录快照及待提升的容器恢复备份。"""
|
||||
|
||||
plugin_id: str
|
||||
plugin_dir: Path
|
||||
persistent_backup_dir: Path
|
||||
backup_staging_dir: Path | None
|
||||
backup_previous_dir: Path | None
|
||||
transaction_dir: Path
|
||||
plugin_existed: bool
|
||||
persistent_backup_existed: bool
|
||||
|
||||
@property
|
||||
def existed(self) -> bool:
|
||||
"""保留旧调用方读取运行目录存在状态的兼容属性。"""
|
||||
return self.plugin_existed
|
||||
|
||||
@property
|
||||
def rollback_marker(self) -> Path:
|
||||
"""返回文件补偿完成标记,供 PREPARED 重放保持幂等。"""
|
||||
return self.transaction_dir / ".rollback-complete"
|
||||
|
||||
|
||||
class PluginPackageManager:
|
||||
"""隔离插件包安装、本地同步、分身改写和文件补偿能力。"""
|
||||
|
||||
_COPY_IGNORE = ("__pycache__", "*.pyc", ".DS_Store", "node_modules")
|
||||
|
||||
def __init__(self, helper: Optional[_PluginHelper] = None) -> None:
|
||||
"""保存市场下载实现;文件事务由本适配器独立负责。"""
|
||||
self._helper = helper or _PluginHelper()
|
||||
|
||||
@staticmethod
|
||||
def __plugin_dir(plugin_id: str) -> Path:
|
||||
"""解析插件运行目录并拒绝越出宿主插件根目录的标识。"""
|
||||
plugins_root = (
|
||||
Path(get_runtime_setting('ROOT_PATH')) / "app" / "plugins"
|
||||
).resolve()
|
||||
plugin_dir = (plugins_root / plugin_id.lower()).resolve()
|
||||
if plugin_dir == plugins_root or not plugin_dir.is_relative_to(plugins_root):
|
||||
raise ValueError(f"非法插件ID:{plugin_id}")
|
||||
return plugin_dir
|
||||
|
||||
def checkpoint(
|
||||
self,
|
||||
plugin_id: str,
|
||||
transaction_id: Optional[str] = None,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""在包变更前保存运行目录;持久事务使用配置目录承载恢复材料。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
durable = transaction_id is not None
|
||||
persistent_backup_dir = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugins_backup"
|
||||
/ plugin_id.lower()
|
||||
).resolve()
|
||||
backup_staging_dir = (
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.staging-{transaction_id}"
|
||||
if durable and SystemUtils.is_docker()
|
||||
else None
|
||||
)
|
||||
backup_previous_dir = (
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.previous-{transaction_id}"
|
||||
if durable and SystemUtils.is_docker()
|
||||
else None
|
||||
)
|
||||
transaction_root = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
if durable
|
||||
else Path(get_runtime_setting('TEMP_PATH'))
|
||||
)
|
||||
transaction_dir = (
|
||||
transaction_root
|
||||
/ "plugin_transactions"
|
||||
/ (transaction_id or f"{plugin_id.lower()}-{uuid.uuid4().hex}")
|
||||
)
|
||||
plugin_existed = plugin_dir.exists()
|
||||
persistent_backup_existed = persistent_backup_dir.exists()
|
||||
try:
|
||||
transaction_dir.mkdir(parents=True, exist_ok=False)
|
||||
if plugin_existed:
|
||||
shutil.copytree(plugin_dir, transaction_dir / "package")
|
||||
except Exception:
|
||||
shutil.rmtree(transaction_dir, ignore_errors=True)
|
||||
raise
|
||||
return PluginPackageCheckpoint(
|
||||
plugin_id=plugin_id,
|
||||
plugin_dir=plugin_dir,
|
||||
persistent_backup_dir=persistent_backup_dir,
|
||||
backup_staging_dir=backup_staging_dir,
|
||||
backup_previous_dir=backup_previous_dir,
|
||||
transaction_dir=transaction_dir,
|
||||
plugin_existed=plugin_existed,
|
||||
persistent_backup_existed=persistent_backup_existed,
|
||||
)
|
||||
|
||||
def restore_checkpoint(
|
||||
self,
|
||||
*,
|
||||
plugin_id: str,
|
||||
transaction_id: str,
|
||||
plugin_existed: bool,
|
||||
persistent_backup_existed: bool,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""按受控根目录和事务 ID 重建崩溃回放所需的文件引用。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
persistent_backup_dir = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugins_backup"
|
||||
/ plugin_id.lower()
|
||||
).resolve()
|
||||
durable_backup = SystemUtils.is_docker()
|
||||
return PluginPackageCheckpoint(
|
||||
plugin_id=plugin_id,
|
||||
plugin_dir=plugin_dir,
|
||||
persistent_backup_dir=persistent_backup_dir,
|
||||
backup_staging_dir=(
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.staging-{transaction_id}"
|
||||
if durable_backup
|
||||
else None
|
||||
),
|
||||
backup_previous_dir=(
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.previous-{transaction_id}"
|
||||
if durable_backup
|
||||
else None
|
||||
),
|
||||
transaction_dir=(
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugin_transactions"
|
||||
/ transaction_id
|
||||
),
|
||||
plugin_existed=plugin_existed,
|
||||
persistent_backup_existed=persistent_backup_existed,
|
||||
)
|
||||
|
||||
async def async_checkpoint(
|
||||
self,
|
||||
plugin_id: str,
|
||||
transaction_id: Optional[str] = None,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""在线程池中创建插件包文件快照。"""
|
||||
return cast(
|
||||
PluginPackageCheckpoint,
|
||||
await _await_thread_operation(
|
||||
self.checkpoint,
|
||||
plugin_id,
|
||||
transaction_id,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def commit(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""清理已完成事务的运行目录快照和残余替换材料。"""
|
||||
if checkpoint.backup_staging_dir and checkpoint.backup_staging_dir.exists():
|
||||
raise RuntimeError("持久备份尚未提升,不能清理插件安装事务")
|
||||
if checkpoint.backup_previous_dir and checkpoint.backup_previous_dir.exists():
|
||||
raise RuntimeError("旧持久备份尚未清理,不能结束插件安装事务")
|
||||
if checkpoint.transaction_dir.exists():
|
||||
shutil.rmtree(checkpoint.transaction_dir, ignore_errors=False)
|
||||
|
||||
async def async_commit(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池中清理已提交的插件包快照。"""
|
||||
await _await_thread_operation(self.commit, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def rollback(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""兼容旧调用方,恢复运行目录和持久备份后清理恢复材料。"""
|
||||
PluginPackageManager.restore(checkpoint)
|
||||
PluginPackageManager.cleanup(checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def restore(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""恢复运行目录和提交前持久备份,并保留快照直到 journal 删除。"""
|
||||
if checkpoint.rollback_marker.is_file():
|
||||
return
|
||||
PluginPackageManager.__restore_tree(
|
||||
target=checkpoint.plugin_dir,
|
||||
snapshot=checkpoint.transaction_dir / "package",
|
||||
existed=checkpoint.plugin_existed,
|
||||
label=f"插件 {checkpoint.plugin_id} 运行目录",
|
||||
)
|
||||
PluginPackageManager.__rollback_persistent_backup(checkpoint)
|
||||
if checkpoint.backup_staging_dir and checkpoint.backup_staging_dir.exists():
|
||||
shutil.rmtree(checkpoint.backup_staging_dir, ignore_errors=False)
|
||||
checkpoint.transaction_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkpoint.rollback_marker.touch(exist_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def cleanup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在 journal 已删除后清理恢复材料;重复调用保持幂等。"""
|
||||
if checkpoint.transaction_dir.exists():
|
||||
shutil.rmtree(checkpoint.transaction_dir, ignore_errors=False)
|
||||
|
||||
async def async_rollback(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池中恢复插件包文件快照。"""
|
||||
await _await_thread_operation(self.rollback, checkpoint)
|
||||
|
||||
async def async_restore(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池恢复插件状态,并保留 journal 仍需引用的材料。"""
|
||||
await _await_thread_operation(self.restore, checkpoint)
|
||||
|
||||
async def async_cleanup(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池清理已失去 journal 所有权的恢复材料。"""
|
||||
await _await_thread_operation(self.cleanup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def __rollback_persistent_backup(
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""把已激活但尚未提交的持久备份恢复到事务前状态。"""
|
||||
previous = checkpoint.backup_previous_dir
|
||||
staging = checkpoint.backup_staging_dir
|
||||
if previous is None or staging is None:
|
||||
return
|
||||
|
||||
target = checkpoint.persistent_backup_dir
|
||||
if previous.exists():
|
||||
discarded = target.parent / f".{target.name}.discard-{uuid.uuid4().hex}"
|
||||
try:
|
||||
if target.exists():
|
||||
target.replace(discarded)
|
||||
previous.replace(target)
|
||||
if discarded.exists():
|
||||
shutil.rmtree(discarded, ignore_errors=False)
|
||||
except Exception:
|
||||
if not target.exists() and discarded.exists():
|
||||
discarded.replace(target)
|
||||
raise
|
||||
finally:
|
||||
if target.exists() and discarded.exists():
|
||||
shutil.rmtree(discarded, ignore_errors=True)
|
||||
return
|
||||
|
||||
if staging.exists():
|
||||
return
|
||||
if checkpoint.persistent_backup_existed:
|
||||
if target.exists():
|
||||
return
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的旧持久备份恢复材料不存在"
|
||||
)
|
||||
if target.exists():
|
||||
shutil.rmtree(target, ignore_errors=False)
|
||||
|
||||
@staticmethod
|
||||
def __restore_tree(
|
||||
*,
|
||||
target: Path,
|
||||
snapshot: Path,
|
||||
existed: bool,
|
||||
label: str,
|
||||
) -> None:
|
||||
"""用同级 staging 替换目录,失败时保留替换前的当前目录。"""
|
||||
if existed and not snapshot.is_dir():
|
||||
raise FileNotFoundError(f"{label}补偿快照不存在:{snapshot}")
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
staging = target.parent / f".{target.name}.restore-{uuid.uuid4().hex}"
|
||||
previous = target.parent / f".{target.name}.previous-{uuid.uuid4().hex}"
|
||||
try:
|
||||
if existed:
|
||||
shutil.copytree(snapshot, staging)
|
||||
if target.exists():
|
||||
target.replace(previous)
|
||||
if existed:
|
||||
staging.replace(target)
|
||||
if previous.exists():
|
||||
shutil.rmtree(previous)
|
||||
except Exception:
|
||||
if not target.exists() and previous.exists():
|
||||
previous.replace(target)
|
||||
raise
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
if target.exists() and previous.exists():
|
||||
shutil.rmtree(previous, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def stage_persistent_backup(cls, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""把新载荷复制到持久配置目录的独立 staging,不覆盖现有备份。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
if staging is None:
|
||||
return
|
||||
if not checkpoint.plugin_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 运行目录不存在"
|
||||
)
|
||||
staging.parent.mkdir(parents=True, exist_ok=True)
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=False)
|
||||
shutil.copytree(
|
||||
checkpoint.plugin_dir,
|
||||
staging,
|
||||
ignore=shutil.ignore_patterns(*cls._COPY_IGNORE),
|
||||
)
|
||||
|
||||
async def async_stage_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池准备新载荷的容器恢复备份。"""
|
||||
await _await_thread_operation(self.stage_persistent_backup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def activate_persistent_backup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在数据库提交前激活新备份,并保留上一份备份供失败补偿。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
previous = checkpoint.backup_previous_dir
|
||||
if staging is None or previous is None:
|
||||
return
|
||||
|
||||
target = checkpoint.persistent_backup_dir
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if staging.exists():
|
||||
if target.exists() and not previous.exists():
|
||||
target.replace(previous)
|
||||
if not target.exists():
|
||||
staging.replace(target)
|
||||
elif not target.exists():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的持久备份 staging 不存在"
|
||||
)
|
||||
|
||||
async def async_activate_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池激活新持久备份,同时保留失败补偿材料。"""
|
||||
await _await_thread_operation(self.activate_persistent_backup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def finalize_persistent_backup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""数据库提交后清理上一份持久备份;重复调用保持幂等。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
previous = checkpoint.backup_previous_dir
|
||||
if staging is None or previous is None:
|
||||
return
|
||||
if staging.exists():
|
||||
raise RuntimeError("新持久备份尚未激活")
|
||||
if not checkpoint.persistent_backup_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的已提交持久备份不存在"
|
||||
)
|
||||
if previous.exists():
|
||||
shutil.rmtree(previous, ignore_errors=False)
|
||||
|
||||
async def async_finalize_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池清理数据库提交后的旧持久备份。"""
|
||||
await _await_thread_operation(self.finalize_persistent_backup, checkpoint)
|
||||
|
||||
def payload_receipt(self, plugin_id: str) -> str:
|
||||
"""按稳定相对路径和文件内容计算已安装载荷收据。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
if not plugin_dir.is_dir():
|
||||
raise FileNotFoundError(f"插件 {plugin_id} 运行目录不存在")
|
||||
return self.__tree_receipt(plugin_dir)
|
||||
|
||||
@classmethod
|
||||
def persistent_backup_receipt(
|
||||
cls,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> str:
|
||||
"""计算已提升持久备份的内容收据,供崩溃回放确认终态。"""
|
||||
if not checkpoint.persistent_backup_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 持久备份不存在"
|
||||
)
|
||||
return cls.__tree_receipt(checkpoint.persistent_backup_dir)
|
||||
|
||||
@classmethod
|
||||
def __tree_receipt(cls, root: Path) -> str:
|
||||
"""对插件目录使用稳定路径和文件内容生成审计收据。"""
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(
|
||||
root.rglob("*"),
|
||||
key=lambda item: item.relative_to(root).as_posix(),
|
||||
):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if cls.__ignored_receipt_path(path, root):
|
||||
continue
|
||||
encoded_path = relative.encode("utf-8")
|
||||
digest.update(len(encoded_path).to_bytes(4, "big"))
|
||||
digest.update(encoded_path)
|
||||
if path.is_symlink():
|
||||
digest.update(b"L")
|
||||
target = path.readlink().as_posix().encode("utf-8")
|
||||
digest.update(len(target).to_bytes(4, "big"))
|
||||
digest.update(target)
|
||||
elif path.is_dir():
|
||||
digest.update(b"D")
|
||||
elif path.is_file():
|
||||
digest.update(b"F")
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return f"sha256:{digest.hexdigest()}"
|
||||
|
||||
async def async_payload_receipt(self, plugin_id: str) -> str:
|
||||
"""在线程池计算插件载荷收据。"""
|
||||
return cast(
|
||||
str,
|
||||
await _await_thread_operation(self.payload_receipt, plugin_id),
|
||||
)
|
||||
|
||||
async def async_committed_payload_receipt(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> str:
|
||||
"""读取数据库已提交载荷在当前部署模式下的恢复事实。"""
|
||||
if checkpoint.backup_staging_dir is not None:
|
||||
return cast(
|
||||
str,
|
||||
await _await_thread_operation(
|
||||
self.persistent_backup_receipt,
|
||||
checkpoint,
|
||||
),
|
||||
)
|
||||
return await self.async_payload_receipt(checkpoint.plugin_id)
|
||||
|
||||
@classmethod
|
||||
def __ignored_receipt_path(cls, path: Path, root: Path) -> bool:
|
||||
"""排除不会进入运行载荷和持久备份的派生文件。"""
|
||||
relative_parts = path.relative_to(root).parts
|
||||
return any(
|
||||
part in {"__pycache__", "node_modules", ".DS_Store"}
|
||||
or part.endswith(".pyc")
|
||||
for part in relative_parts
|
||||
)
|
||||
|
||||
def install(
|
||||
self,
|
||||
plugin_id: str,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
release_version: Optional[str] = None,
|
||||
force_install: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""同步安装插件包,下载过程继续复用既有市场兼容策略。"""
|
||||
return cast(
|
||||
tuple[bool, str],
|
||||
cast(Any, self._helper)._PluginHelper__install_package(
|
||||
pid=plugin_id,
|
||||
repo_url=repo_url,
|
||||
package_version=package_version,
|
||||
release_version=release_version,
|
||||
force_install=force_install,
|
||||
),
|
||||
)
|
||||
|
||||
async def async_install(
|
||||
self,
|
||||
plugin_id: str,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
release_version: Optional[str] = None,
|
||||
force_install: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""异步安装插件包,下载过程继续复用既有市场兼容策略。"""
|
||||
return cast(
|
||||
tuple[bool, str],
|
||||
await cast(Any, self._helper)._PluginHelper__async_install_package(
|
||||
pid=plugin_id,
|
||||
repo_url=repo_url,
|
||||
package_version=package_version,
|
||||
release_version=release_version,
|
||||
force_install=force_install,
|
||||
),
|
||||
)
|
||||
|
||||
def sync_local(self, plugin_id: str, source_dir: Path) -> bool:
|
||||
"""用本地仓库内容原子替换运行副本,失败时恢复原目录。"""
|
||||
source_dir = source_dir.resolve()
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
if source_dir == plugin_dir:
|
||||
return True
|
||||
checkpoint = self.checkpoint(plugin_id)
|
||||
try:
|
||||
if plugin_dir.exists():
|
||||
shutil.rmtree(plugin_dir)
|
||||
shutil.copytree(
|
||||
source_dir,
|
||||
plugin_dir,
|
||||
ignore=shutil.ignore_patterns(*self._COPY_IGNORE),
|
||||
)
|
||||
self.commit(checkpoint)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"同步本地插件 {plugin_id} 失败:{err}")
|
||||
try:
|
||||
self.rollback(checkpoint)
|
||||
except Exception as rollback_err:
|
||||
logger.error(
|
||||
f"恢复本地插件 {plugin_id} 原目录失败:{rollback_err}",
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
def clone(
|
||||
self,
|
||||
*,
|
||||
plugin_id: str,
|
||||
clone_id: str,
|
||||
original_class_name: str,
|
||||
suffix: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str] = None,
|
||||
icon: Optional[str] = None,
|
||||
) -> tuple[bool, str]:
|
||||
"""复制并改写插件分身文件,任一步失败都删除不完整目标。"""
|
||||
original_dir = self.__plugin_dir(plugin_id)
|
||||
clone_dir = self.__plugin_dir(clone_id)
|
||||
if not original_dir.is_dir():
|
||||
return False, f"原插件目录 {original_dir} 不存在"
|
||||
if clone_dir.exists():
|
||||
return False, f"分身插件 {clone_id} 已存在"
|
||||
|
||||
checkpoint = self.checkpoint(clone_id)
|
||||
try:
|
||||
shutil.copytree(original_dir, clone_dir)
|
||||
success, message = self._modify_plugin_files(
|
||||
plugin_dir=clone_dir,
|
||||
original_class_name=original_class_name,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
if not success:
|
||||
self.rollback(checkpoint)
|
||||
return False, message
|
||||
self.commit(checkpoint)
|
||||
logger.info(f"已复制插件目录:{original_dir} -> {clone_dir}")
|
||||
return True, "文件修改成功"
|
||||
except Exception as err:
|
||||
try:
|
||||
self.rollback(checkpoint)
|
||||
except Exception as rollback_err:
|
||||
logger.error(
|
||||
f"清理插件分身 {clone_id} 失败:{rollback_err}",
|
||||
exc_info=True,
|
||||
)
|
||||
return False, f"创建插件分身文件失败:{err}"
|
||||
|
||||
def _modify_plugin_files(
|
||||
self,
|
||||
*,
|
||||
plugin_dir: Path,
|
||||
original_class_name: str,
|
||||
suffix: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str],
|
||||
icon: Optional[str],
|
||||
) -> tuple[bool, str]:
|
||||
"""改写分身的 Python 元数据和联邦前端资源。"""
|
||||
clone_class_name = f"{original_class_name}{suffix}"
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if init_file.exists():
|
||||
success, message = self._modify_python_file(
|
||||
file_path=init_file,
|
||||
original_class_name=original_class_name,
|
||||
clone_class_name=clone_class_name,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
if not success:
|
||||
return False, message
|
||||
|
||||
dist_dir = plugin_dir / "dist"
|
||||
if dist_dir.exists():
|
||||
success, message = self._modify_federation_files(
|
||||
dist_dir=dist_dir,
|
||||
original_class_name=original_class_name,
|
||||
clone_class_name=clone_class_name,
|
||||
)
|
||||
if not success:
|
||||
return False, message
|
||||
return True, "文件修改成功"
|
||||
|
||||
@staticmethod
|
||||
def _modify_python_file(
|
||||
*,
|
||||
file_path: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str],
|
||||
icon: Optional[str],
|
||||
) -> tuple[bool, str]:
|
||||
"""改写插件主类名称、展示元数据和独立配置前缀。"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
content = content.replace(
|
||||
f"class {original_class_name}",
|
||||
f"class {clone_class_name}",
|
||||
)
|
||||
if name:
|
||||
content = re.sub(
|
||||
r'plugin_name\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_name = "{name}"',
|
||||
content,
|
||||
)
|
||||
if description:
|
||||
content = re.sub(
|
||||
r'plugin_desc\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_desc = "{description}"',
|
||||
content,
|
||||
)
|
||||
content = re.sub(
|
||||
r'plugin_config_prefix\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_config_prefix = "{clone_class_name.lower()}_"',
|
||||
content,
|
||||
)
|
||||
if version:
|
||||
content = re.sub(
|
||||
r'plugin_version\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_version = "{version}"',
|
||||
content,
|
||||
)
|
||||
if icon and icon.strip():
|
||||
content = re.sub(
|
||||
r'plugin_icon\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_icon = "{icon}"',
|
||||
content,
|
||||
)
|
||||
if "def init_plugin(self" in content:
|
||||
init_index = content.index("def init_plugin(self")
|
||||
content = (
|
||||
content[:init_index]
|
||||
+ "is_clone = True\n\n "
|
||||
+ content[init_index:]
|
||||
)
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
return True, "Python文件修改成功"
|
||||
except Exception as err:
|
||||
logger.error(f"修改Python文件失败:{err}")
|
||||
return False, f"修改Python文件失败:{err}"
|
||||
|
||||
def _modify_federation_files(
|
||||
self,
|
||||
*,
|
||||
dist_dir: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
) -> tuple[bool, str]:
|
||||
"""改写联邦构建产物中的插件类名和样式命名空间。"""
|
||||
try:
|
||||
for file_path in dist_dir.rglob("*"):
|
||||
if not file_path.is_file() or file_path.suffix not in {".js", ".css"}:
|
||||
continue
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
if file_path.suffix == ".js":
|
||||
content = content.replace(original_class_name, clone_class_name)
|
||||
content = content.replace(
|
||||
f'"{original_class_name}"',
|
||||
f'"{clone_class_name}"',
|
||||
)
|
||||
content = content.replace(
|
||||
f"'{original_class_name}'",
|
||||
f"'{clone_class_name}'",
|
||||
)
|
||||
content = content.replace(
|
||||
f"css__{original_class_name}__",
|
||||
f"css__{clone_class_name}__",
|
||||
)
|
||||
content = content.replace(
|
||||
original_class_name.lower(),
|
||||
clone_class_name.lower(),
|
||||
)
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except Exception as err:
|
||||
logger.warning(f"修改联邦插件文件 {file_path} 失败:{err}")
|
||||
self._rename_federation_assets(
|
||||
dist_dir,
|
||||
original_class_name,
|
||||
clone_class_name,
|
||||
)
|
||||
return True, "联邦插件文件修改完成"
|
||||
except Exception as err:
|
||||
logger.error(f"修改联邦插件文件失败:{err}")
|
||||
return False, f"修改联邦插件文件失败:{err}"
|
||||
|
||||
@staticmethod
|
||||
def _rename_federation_assets(
|
||||
dist_dir: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
) -> None:
|
||||
"""重命名包含原类名的顶层联邦资源,避免分身资源冲突。"""
|
||||
try:
|
||||
for file_path in dist_dir.glob("*"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
if original_class_name.lower() not in file_path.name.lower():
|
||||
continue
|
||||
new_name = file_path.name.replace(
|
||||
original_class_name.lower(),
|
||||
clone_class_name.lower(),
|
||||
)
|
||||
new_path = file_path.parent / new_name
|
||||
if not new_path.exists():
|
||||
file_path.rename(new_path)
|
||||
except Exception as err:
|
||||
logger.warning(f"重命名联邦插件资源文件失败:{err}")
|
||||
@@ -1,227 +0,0 @@
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
import sysconfig
|
||||
from pathlib import Path
|
||||
from typing import Callable
|
||||
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.foundation.version import compare_version
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
ResourceVersionProvider = Callable[[], tuple[str, str]]
|
||||
|
||||
|
||||
def _unavailable_resource_versions() -> tuple[str, str]:
|
||||
"""站点能力尚未装配时返回可安全比较的空版本。"""
|
||||
return "0", "0"
|
||||
|
||||
|
||||
_resource_version_provider: ResourceVersionProvider = _unavailable_resource_versions
|
||||
|
||||
|
||||
def configure_resource_version_provider(provider: ResourceVersionProvider) -> None:
|
||||
"""由组合根注入已加载的认证与索引版本,保持适配器不依赖应用层。"""
|
||||
global _resource_version_provider
|
||||
_resource_version_provider = provider
|
||||
|
||||
|
||||
class ResourceHelper:
|
||||
"""
|
||||
检测和更新资源包
|
||||
"""
|
||||
|
||||
_base_dir: Path = get_runtime_setting('ROOT_PATH')
|
||||
_resource_target = Path("app/application/site")
|
||||
_version_flag = get_runtime_setting('RESOURCE_VERSION_FLAG')
|
||||
_repo = (
|
||||
f"{get_runtime_setting('GITHUB_PROXY')}https://raw.githubusercontent.com/"
|
||||
f"jxxghp/MoviePilot-Resources/main/package.{_version_flag}.json"
|
||||
)
|
||||
_files_api = (
|
||||
"https://api.github.com/repos/jxxghp/"
|
||||
f"MoviePilot-Resources/contents/resources.{_version_flag}"
|
||||
)
|
||||
|
||||
@property
|
||||
def proxies(self):
|
||||
"""返回访问 GitHub 资源时应使用的代理配置。"""
|
||||
return (
|
||||
None
|
||||
if get_runtime_setting('GITHUB_PROXY')
|
||||
else get_runtime_setting('PROXY')
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_python_version_tag() -> str:
|
||||
"""返回资源文件名使用的 CPython ABI 标签。"""
|
||||
version = sys.version_info
|
||||
free_threaded = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else ""
|
||||
return f"cp{version.major}{version.minor}{free_threaded}"
|
||||
|
||||
@staticmethod
|
||||
def _get_machine_tag() -> str:
|
||||
"""将系统架构名称归一为资源文件使用的标签。"""
|
||||
machine = platform.machine().lower()
|
||||
if machine in {"arm64", "aarch64"}:
|
||||
return "aarch64"
|
||||
elif machine in {"x86_64", "amd64"}:
|
||||
return "x86_64"
|
||||
return machine
|
||||
|
||||
@classmethod
|
||||
def _get_needed_files(cls) -> list[str]:
|
||||
"""返回 V3 资源在当前平台需要下载的文件名。"""
|
||||
python_version = ResourceHelper._get_python_version_tag()
|
||||
python_ver = python_version.replace("cp", "")
|
||||
system = platform.system().lower()
|
||||
machine = ResourceHelper._get_machine_tag()
|
||||
files = [f"user.sites.{cls._version_flag}.bin"]
|
||||
if system == "linux":
|
||||
files.append(f"sites.cpython-{python_ver}-{machine}-linux-gnu.so")
|
||||
elif system == "darwin":
|
||||
files.append(f"sites.cpython-{python_ver}-darwin.so")
|
||||
elif system == "windows":
|
||||
files.append(f"sites.cp{python_ver}-win_amd64.pyd")
|
||||
return files
|
||||
|
||||
def _load_resource_info(self):
|
||||
"""读取 V3 资源清单。"""
|
||||
response = RequestUtils(
|
||||
proxies=self.proxies,
|
||||
headers=get_runtime_setting('GITHUB_HEADERS'),
|
||||
timeout=10,
|
||||
).get_res(self._repo)
|
||||
return response if response and response.status_code == 200 else None
|
||||
|
||||
def check(
|
||||
self,
|
||||
*,
|
||||
auth_version: str | None = None,
|
||||
indexer_version: str | None = None,
|
||||
) -> bool:
|
||||
"""
|
||||
检测并安装当前平台的资源更新。
|
||||
|
||||
:param auth_version: 当前已加载的站点认证资源版本;省略时使用组合根注入值
|
||||
:param indexer_version: 当前已加载的站点索引资源版本;省略时使用组合根注入值
|
||||
:return: 是否成功安装了需要由上层处理重启的新资源
|
||||
"""
|
||||
if not get_runtime_setting('AUTO_UPDATE_RESOURCE'):
|
||||
return False
|
||||
if SystemUtils.is_frozen():
|
||||
return False
|
||||
if auth_version is None or indexer_version is None:
|
||||
configured_auth_version, configured_indexer_version = (
|
||||
_resource_version_provider()
|
||||
)
|
||||
auth_version = auth_version or configured_auth_version
|
||||
indexer_version = indexer_version or configured_indexer_version
|
||||
logger.info("开始检测资源包版本...")
|
||||
res = self._load_resource_info()
|
||||
if res:
|
||||
try:
|
||||
resource_info = json.loads(res.text)
|
||||
online_version = resource_info.get("version")
|
||||
if online_version:
|
||||
logger.info(f"最新资源包版本:v{online_version}")
|
||||
# 需要更新的资源包
|
||||
need_updates = {}
|
||||
# 资源明细
|
||||
resources: dict = resource_info.get("resources") or {}
|
||||
for rname, resource in resources.items():
|
||||
rtype = resource.get("type")
|
||||
platform = resource.get("platform")
|
||||
declared_target = Path(str(resource.get("target") or ""))
|
||||
version = resource.get("version")
|
||||
# 判断平台
|
||||
if platform and platform != SystemUtils.platform():
|
||||
continue
|
||||
# 判断版本号
|
||||
if rtype == "auth":
|
||||
# 站点认证资源
|
||||
local_version = auth_version
|
||||
elif rtype == "sites":
|
||||
# 站点索引资源
|
||||
local_version = indexer_version
|
||||
else:
|
||||
continue
|
||||
if compare_version(version, ">", local_version):
|
||||
logger.info(f"{rname} 资源包有更新,最新版本:v{version}")
|
||||
else:
|
||||
continue
|
||||
# 需要安装
|
||||
if declared_target != self._resource_target:
|
||||
logger.warning(
|
||||
"忽略资源 %s 的非 canonical 目标目录:%s",
|
||||
rname,
|
||||
declared_target,
|
||||
)
|
||||
continue
|
||||
need_updates[rname] = self._resource_target
|
||||
if need_updates:
|
||||
# 下载文件信息列表
|
||||
r = RequestUtils(
|
||||
proxies=get_runtime_setting('PROXY'),
|
||||
headers=get_runtime_setting('GITHUB_HEADERS'),
|
||||
timeout=30,
|
||||
).get_res(self._files_api)
|
||||
if r and not r.ok:
|
||||
logger.error(
|
||||
f"连接仓库失败:{r.status_code} - {r.reason}"
|
||||
)
|
||||
return False
|
||||
elif not r:
|
||||
logger.error("连接仓库失败")
|
||||
return False
|
||||
files_info = r.json()
|
||||
# 下载资源文件
|
||||
needed_files = self._get_needed_files()
|
||||
logger.info(f"需要下载的资源文件:{needed_files}")
|
||||
success = True
|
||||
for item in files_info:
|
||||
file_name = item.get("name")
|
||||
if file_name not in needed_files:
|
||||
continue
|
||||
save_path = need_updates.get(file_name)
|
||||
if not save_path:
|
||||
continue
|
||||
if item.get("download_url"):
|
||||
logger.info(f"开始更新资源文件:{file_name} ...")
|
||||
download_url = (
|
||||
f"{get_runtime_setting('GITHUB_PROXY')}{item.get('download_url')}"
|
||||
)
|
||||
res = RequestUtils(
|
||||
proxies=self.proxies,
|
||||
headers=get_runtime_setting('GITHUB_HEADERS'),
|
||||
timeout=180,
|
||||
).get_res(download_url)
|
||||
if not res:
|
||||
logger.error(f"文件 {file_name} 下载失败!")
|
||||
success = False
|
||||
break
|
||||
elif res.status_code != 200:
|
||||
logger.error(
|
||||
f"下载文件 {file_name} 失败:{res.status_code} - {res.reason}"
|
||||
)
|
||||
success = False
|
||||
break
|
||||
file_path = self._base_dir / save_path / file_name
|
||||
if not file_path.parent.exists():
|
||||
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_path.write_bytes(res.content)
|
||||
if success:
|
||||
logger.info("资源包更新完成,等待启动层处理后续重启")
|
||||
return True
|
||||
else:
|
||||
logger.warning("资源包更新失败,跳过升级!")
|
||||
else:
|
||||
logger.info("所有资源已最新,无需更新")
|
||||
except json.JSONDecodeError:
|
||||
logger.error("资源包仓库数据解析失败!")
|
||||
return False
|
||||
else:
|
||||
logger.warning("无法连接资源包仓库!")
|
||||
return False
|
||||
@@ -1,284 +0,0 @@
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from app.foundation.environment import is_free_threaded_runtime
|
||||
from app.runtime.log import logger, log_settings
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
try:
|
||||
import moviepilot_rust as _moviepilot_rust
|
||||
except Exception as err: # pragma: no cover - 取决于运行环境是否安装 Rust 扩展
|
||||
_moviepilot_rust = None
|
||||
_import_error = err
|
||||
else:
|
||||
_import_error = None
|
||||
|
||||
|
||||
def _rust_accel_enabled() -> bool:
|
||||
"""读取 Rust 开关快照,组合根未装配时回退旧 Settings。"""
|
||||
return bool(get_runtime_setting('RUST_ACCEL'))
|
||||
|
||||
|
||||
def is_required() -> bool:
|
||||
"""free-threaded 运行时必须使用不会重新启用 GIL 的 Rust 快路径。"""
|
||||
return is_free_threaded_runtime()
|
||||
|
||||
|
||||
def is_available() -> bool:
|
||||
"""
|
||||
判断 Rust 扩展是否可用。
|
||||
"""
|
||||
return bool(_moviepilot_rust and _moviepilot_rust.is_available())
|
||||
|
||||
|
||||
def is_config_enabled() -> bool:
|
||||
"""
|
||||
判断系统配置是否允许使用 Rust 加速。
|
||||
"""
|
||||
return is_required() or _rust_accel_enabled()
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
"""
|
||||
判断当前运行时是否实际启用 Rust 加速。
|
||||
"""
|
||||
return is_config_enabled() and is_available()
|
||||
|
||||
|
||||
def status() -> dict:
|
||||
"""
|
||||
返回 Rust 加速能力与开关状态,供系统配置接口展示。
|
||||
"""
|
||||
return {
|
||||
"available": is_available(),
|
||||
"enabled": is_enabled(),
|
||||
"required": is_required(),
|
||||
"import_error": str(_import_error) if _import_error else "",
|
||||
}
|
||||
|
||||
|
||||
def import_error() -> Optional[Exception]:
|
||||
"""
|
||||
返回 Rust 扩展导入失败的异常,便于调试构建问题。
|
||||
"""
|
||||
return _import_error
|
||||
|
||||
|
||||
def parse_filter_rule(expression: str) -> Optional[list]:
|
||||
"""
|
||||
使用 Rust 解析过滤规则表达式,不可用时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_filter_rule_fast(expression)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust 过滤规则解析失败,回退 Python:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def filter_torrents(
|
||||
groups: list,
|
||||
torrent_list: list,
|
||||
rule_set: dict,
|
||||
mediainfo=None,
|
||||
metainfo_options: Optional[dict] = None,
|
||||
) -> Optional[Tuple[list, list]]:
|
||||
"""
|
||||
使用 Rust 执行完整种子过滤入口,返回原列表下标、优先级和可选调试日志。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
args = (
|
||||
groups,
|
||||
torrent_list,
|
||||
rule_set,
|
||||
mediainfo,
|
||||
metainfo_options or {},
|
||||
)
|
||||
if is_debug_log_enabled() and hasattr(_moviepilot_rust, "filter_torrents_with_trace_fast"):
|
||||
matched_orders, traces = _moviepilot_rust.filter_torrents_with_trace_fast(*args)
|
||||
return matched_orders, traces
|
||||
return _moviepilot_rust.filter_torrents_fast(*args), []
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust 种子过滤失败,回退 Python:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def is_debug_log_enabled() -> bool:
|
||||
"""
|
||||
判断当前日志配置是否会实际输出 debug 日志。
|
||||
"""
|
||||
if log_settings.DEBUG:
|
||||
return True
|
||||
return getattr(logging, log_settings.LOG_LEVEL.upper(), logging.INFO) <= logging.DEBUG
|
||||
|
||||
def parse_indexer_torrents(
|
||||
html_text: str,
|
||||
domain: str,
|
||||
list_config: dict,
|
||||
fields: dict,
|
||||
category: Optional[dict] = None,
|
||||
result_num: int = 100
|
||||
) -> Optional[List[dict]]:
|
||||
"""
|
||||
使用 Rust 批量解析普通配置站点种子列表,不可用时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_indexer_torrents_fast(
|
||||
html_text,
|
||||
domain,
|
||||
list_config,
|
||||
fields,
|
||||
category,
|
||||
result_num
|
||||
)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust 站点列表解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_indexer_subtitles(
|
||||
html_text: str,
|
||||
domain: str,
|
||||
list_config: dict,
|
||||
fields: dict,
|
||||
result_num: int = 100
|
||||
) -> Optional[List[dict]]:
|
||||
"""
|
||||
使用 Rust 批量解析普通配置站点字幕列表,不可用时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_indexer_subtitles_fast(
|
||||
html_text,
|
||||
domain,
|
||||
list_config,
|
||||
fields,
|
||||
result_num
|
||||
)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust 字幕列表解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_rss_items(xml_text: str, max_items: int = 1000) -> Optional[List[dict]]:
|
||||
"""
|
||||
使用 Rust 解析 RSS/Atom 条目,不可用或异常时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_rss_items_fast(xml_text, max_items)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust RSS解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_metainfo(title: str, subtitle: Optional[str] = None, options: Optional[dict] = None) -> Optional[dict]:
|
||||
"""
|
||||
使用 Rust 从标题入口解析 MetaInfo,不可用或异常时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_metainfo_fast(title, subtitle, options or {})
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust MetaInfo解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_metainfo_path(path: str, options: Optional[dict] = None) -> Optional[dict]:
|
||||
"""
|
||||
使用 Rust 从路径入口解析 MetaInfoPath,不可用或异常时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.parse_metainfo_path_fast(path, options or {})
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust MetaInfoPath解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
def parse_metamusic(
|
||||
title: str,
|
||||
artists: Optional[List[str]] = None,
|
||||
year: Optional[int] = None,
|
||||
) -> Optional[dict]:
|
||||
"""使用 Rust 解析音乐资源标题,旧扩展不支持时返回 None。
|
||||
|
||||
:param title: 音乐资源标题或文件主干名
|
||||
:param artists: 调用方已有的高可信艺术家列表
|
||||
:param year: 调用方已有的高可信发行年份
|
||||
:return: Rust 解析字段,不可用、不支持或异常时返回 None
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
parser = getattr(_moviepilot_rust, "parse_metamusic_fast", None)
|
||||
if not callable(parser):
|
||||
return None
|
||||
try:
|
||||
result = parser(title, artists, year)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust MetaMusic解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
return result if isinstance(result, dict) and result else None
|
||||
|
||||
|
||||
def find_metainfo(title: str) -> Optional[dict]:
|
||||
"""
|
||||
使用 Rust 提取标题中的显式媒体标签,不可用或异常时返回 None。
|
||||
"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
try:
|
||||
return _moviepilot_rust.find_metainfo_fast(title)
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"Rust 显式媒体标签解析失败,使用 Python 解析兜底:{err}")
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def supports_extended_media_ids() -> bool:
|
||||
"""
|
||||
判断当前 Rust 扩展是否支持 Bangumi 与 AniList 显式媒体标签。
|
||||
|
||||
:return: 是否支持扩展数据源ID字段
|
||||
"""
|
||||
if not is_enabled():
|
||||
return False
|
||||
try:
|
||||
result = _moviepilot_rust.find_metainfo_fast("test [anilist=1]")
|
||||
except BaseException as err:
|
||||
_raise_non_rust_panic(err)
|
||||
logger.debug(f"检测 Rust 扩展数据源ID能力失败:{err}")
|
||||
return False
|
||||
metainfo = result.get("metainfo") if isinstance(result, dict) else None
|
||||
return bool(
|
||||
metainfo
|
||||
and metainfo.get("media_source") == "anilist"
|
||||
and metainfo.get("media_id") == "1"
|
||||
)
|
||||
|
||||
|
||||
def _raise_non_rust_panic(err: BaseException) -> None:
|
||||
"""
|
||||
只吞掉 Rust 扩展 panic/异常,保留用户中断和进程退出语义。
|
||||
"""
|
||||
if isinstance(err, (KeyboardInterrupt, SystemExit)):
|
||||
raise err
|
||||
@@ -1,90 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import logging
|
||||
import sys
|
||||
import threading
|
||||
from logging.handlers import RotatingFileHandler
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class RotatingLineStream(io.TextIOBase):
|
||||
"""
|
||||
将 stdout/stderr 按行写入滚动日志文件。
|
||||
|
||||
这里不复用业务 logger,避免 stdout 日志再次回流到控制台或普通业务日志文件,
|
||||
同时保证启动阶段的 print/uvicorn 输出也能按配置滚动。
|
||||
"""
|
||||
|
||||
def __init__(self, log_file: Path, max_bytes: int, backup_count: int):
|
||||
"""创建写入指定滚动日志文件的文本流。"""
|
||||
super().__init__()
|
||||
self._buffer = ""
|
||||
self._lock = threading.Lock()
|
||||
|
||||
logger_name = f"moviepilot-stdio::{log_file}"
|
||||
self._logger = logging.getLogger(logger_name)
|
||||
self._logger.setLevel(logging.INFO)
|
||||
self._logger.propagate = False
|
||||
self._logger.handlers.clear()
|
||||
|
||||
handler = RotatingFileHandler(
|
||||
filename=str(log_file),
|
||||
maxBytes=max_bytes,
|
||||
backupCount=backup_count,
|
||||
encoding="utf-8",
|
||||
)
|
||||
handler.setFormatter(logging.Formatter("%(message)s"))
|
||||
self._logger.addHandler(handler)
|
||||
|
||||
@property
|
||||
def encoding(self) -> str:
|
||||
"""返回日志流使用的文本编码。"""
|
||||
return "utf-8"
|
||||
|
||||
def writable(self) -> bool:
|
||||
"""声明该文本流支持写入。"""
|
||||
return True
|
||||
|
||||
def isatty(self) -> bool:
|
||||
"""声明该日志流不是交互式终端。"""
|
||||
return False
|
||||
|
||||
def write(self, message: str) -> int:
|
||||
"""缓冲消息并按完整行写入滚动日志。"""
|
||||
if not message:
|
||||
return 0
|
||||
|
||||
with self._lock:
|
||||
self._buffer += message.replace("\r\n", "\n")
|
||||
while "\n" in self._buffer:
|
||||
line, self._buffer = self._buffer.split("\n", 1)
|
||||
self._logger.info(line)
|
||||
return len(message)
|
||||
|
||||
def flush(self) -> None:
|
||||
"""写出剩余缓冲区并刷新底层处理器。"""
|
||||
with self._lock:
|
||||
if self._buffer:
|
||||
self._logger.info(self._buffer)
|
||||
self._buffer = ""
|
||||
for handler in self._logger.handlers:
|
||||
handler.flush()
|
||||
|
||||
|
||||
def configure_rotating_stdio(
|
||||
*, log_file: Path, max_bytes: int, backup_count: int
|
||||
) -> RotatingLineStream:
|
||||
"""
|
||||
将当前进程的 stdout/stderr 统一重定向到同一个滚动日志流。
|
||||
"""
|
||||
|
||||
log_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
stream = RotatingLineStream(
|
||||
log_file=log_file,
|
||||
max_bytes=max_bytes,
|
||||
backup_count=backup_count,
|
||||
)
|
||||
sys.stdout = stream
|
||||
sys.stderr = stream
|
||||
return stream
|
||||
@@ -1,481 +0,0 @@
|
||||
"""MoviePilot Release 后台检查、下载与待安装状态管理。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import threading
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.foundation.singleton import SingletonClass
|
||||
from app.foundation.version import compare_version
|
||||
from app.runtime.version import get_app_version
|
||||
from app.foundation.environment import is_docker
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.runtime.thread import ThreadHelper
|
||||
from app.schemas.system import SystemUpdateStatus
|
||||
|
||||
|
||||
class SystemUpdateManager(metaclass=SingletonClass):
|
||||
"""持久化更新状态,并保证同一时刻只有一个下载任务。"""
|
||||
|
||||
_BACKEND_RELEASES_API = "https://api.github.com/repos/jxxghp/MoviePilot/releases"
|
||||
_FRONTEND_RELEASE_API = (
|
||||
"https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/tags/{tag}"
|
||||
)
|
||||
_BACKEND_ARCHIVE_URL = (
|
||||
"https://github.com/jxxghp/MoviePilot/archive/refs/tags/{tag}.zip"
|
||||
)
|
||||
_VERSION_PATTERN = re.compile(r"^v3\.\d+\.\d+(?:[-.](?:alpha|beta|rc)\d*)?$", re.I)
|
||||
_STABLE_VERSION_PATTERN = re.compile(r"^v3\.\d+\.\d+$", re.I)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._download_active = False
|
||||
|
||||
@property
|
||||
def _root(self) -> Path:
|
||||
return Path(get_runtime_setting('TEMP_PATH')) / "moviepilot-update"
|
||||
|
||||
@property
|
||||
def _state_file(self) -> Path:
|
||||
return self._root / "state.json"
|
||||
|
||||
@property
|
||||
def _install_file(self) -> Path:
|
||||
return self._root / "install.json"
|
||||
|
||||
@property
|
||||
def _backend_archive(self) -> Path:
|
||||
return self._root / "backend.zip"
|
||||
|
||||
@property
|
||||
def _frontend_archive(self) -> Path:
|
||||
return self._root / "frontend.zip"
|
||||
|
||||
@staticmethod
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _default_state(self) -> dict[str, Any]:
|
||||
return SystemUpdateStatus(current_version=get_app_version()).model_dump()
|
||||
|
||||
def _read_state(self) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(self._state_file.read_text(encoding="utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
return {
|
||||
**self._default_state(),
|
||||
**payload,
|
||||
"current_version": get_app_version(),
|
||||
}
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
return self._default_state()
|
||||
|
||||
def _write_state(self, **changes: Any) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
state = self._read_state()
|
||||
state.update(changes)
|
||||
state["current_version"] = get_app_version()
|
||||
state["progress"] = self._progress(
|
||||
state.get("downloaded_bytes", 0), state.get("total_bytes", 0)
|
||||
)
|
||||
validated = SystemUpdateStatus.model_validate(state).model_dump()
|
||||
self._root.mkdir(parents=True, exist_ok=True)
|
||||
temporary = self._state_file.with_suffix(".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(validated, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
temporary.replace(self._state_file)
|
||||
return validated
|
||||
|
||||
@staticmethod
|
||||
def _progress(downloaded: Any, total: Any) -> int:
|
||||
try:
|
||||
downloaded_value = max(0, int(downloaded))
|
||||
total_value = max(0, int(total))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
if total_value <= 0:
|
||||
return 0
|
||||
return min(100, int(downloaded_value * 100 / total_value))
|
||||
|
||||
def get_status(self) -> SystemUpdateStatus:
|
||||
"""返回状态快照,并在更新完成后的新进程中清理安装终态。"""
|
||||
with self._lock:
|
||||
state = self._read_state()
|
||||
target = str(state.get("version") or "")
|
||||
if state.get("state") == "downloading" and not self._download_active:
|
||||
state = self._write_state(
|
||||
state="failed",
|
||||
error="更新包下载因服务重启而中断,请重试",
|
||||
can_update=True,
|
||||
can_install=False,
|
||||
)
|
||||
if state.get("state") == "installing" and target == get_app_version():
|
||||
self._install_file.unlink(missing_ok=True)
|
||||
state = self._write_state(
|
||||
state="idle",
|
||||
version=None,
|
||||
frontend_version=None,
|
||||
release_name=None,
|
||||
release_notes=None,
|
||||
published_at=None,
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
return SystemUpdateStatus.model_validate(state)
|
||||
|
||||
def check(self) -> SystemUpdateStatus:
|
||||
"""查询 GitHub 稳定版 v3 Release,并保留正在下载或待安装状态。"""
|
||||
current = self.get_status()
|
||||
if current.state in {"downloading", "ready", "installing"}:
|
||||
return current
|
||||
|
||||
try:
|
||||
response = self._request().get_res(self._BACKEND_RELEASES_API)
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError("GitHub Release 请求失败")
|
||||
releases = response.json()
|
||||
release = next(
|
||||
(
|
||||
item
|
||||
for item in releases
|
||||
if isinstance(item, dict)
|
||||
and not item.get("draft")
|
||||
and not item.get("prerelease")
|
||||
and self._STABLE_VERSION_PATTERN.fullmatch(
|
||||
str(item.get("tag_name") or "")
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not release:
|
||||
raise RuntimeError("未找到可用的 v3 稳定版本")
|
||||
version = str(release["tag_name"])
|
||||
has_update = compare_version(version, "gt", get_app_version()) is True
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="available" if has_update else "idle",
|
||||
version=version if has_update else None,
|
||||
frontend_version=None,
|
||||
release_name=str(release.get("name") or version) if has_update else None,
|
||||
release_notes=str(release.get("body") or "") if has_update else None,
|
||||
published_at=release.get("published_at") if has_update else None,
|
||||
checked_at=self._now(),
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=has_update,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
except Exception as error: # 定时检查失败不应打扰用户,下载失败才进入可见错误态
|
||||
logger.warning(f"检查 MoviePilot 更新失败: {error}")
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="idle",
|
||||
checked_at=self._now(),
|
||||
error=str(error),
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
|
||||
def start_download(self) -> SystemUpdateStatus:
|
||||
"""启动唯一后台下载线程,并立即返回下载中状态。"""
|
||||
with self._lock:
|
||||
state = self.get_status()
|
||||
if state.state == "ready":
|
||||
return state
|
||||
if state.state == "downloading" and self._download_active:
|
||||
return state
|
||||
if state.state != "available" or not state.version:
|
||||
state = self.check()
|
||||
if state.state != "available" or not state.version:
|
||||
return state
|
||||
|
||||
self._backend_archive.unlink(missing_ok=True)
|
||||
self._frontend_archive.unlink(missing_ok=True)
|
||||
self._write_state(
|
||||
state="downloading",
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
self._download_active = True
|
||||
try:
|
||||
ThreadHelper().submit(self._download_update, state.version)
|
||||
except RuntimeError as error:
|
||||
self._download_active = False
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="failed",
|
||||
error=f"无法启动更新包下载:{error}",
|
||||
can_update=True,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
return self.get_status()
|
||||
|
||||
def request_install(self) -> tuple[bool, str]:
|
||||
"""校验待安装文件并写入启动阶段消费的安装意图。"""
|
||||
with self._lock:
|
||||
state = self.get_status()
|
||||
if state.state != "ready" or not state.version:
|
||||
return False, "更新包尚未下载完成"
|
||||
try:
|
||||
backend_sha256 = self._sha256(self._backend_archive)
|
||||
frontend_sha256 = self._sha256(self._frontend_archive)
|
||||
prepared = self._read_prepared_manifest()
|
||||
if backend_sha256 != prepared.get("backend_sha256"):
|
||||
raise RuntimeError("后端更新包校验失败")
|
||||
if frontend_sha256 != prepared.get("frontend_sha256"):
|
||||
raise RuntimeError("前端更新包校验失败")
|
||||
self._install_file.write_text(
|
||||
json.dumps(prepared, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
self._write_state(state="installing", can_install=False, error=None)
|
||||
return True, "更新包已就绪,正在重启安装"
|
||||
except (OSError, RuntimeError, json.JSONDecodeError) as error:
|
||||
self._write_state(state="failed", error=str(error), can_install=False)
|
||||
return False, str(error)
|
||||
|
||||
def cancel_install(self, reason: str) -> None:
|
||||
"""重启请求失败时撤销安装意图,避免下次普通启动意外安装。"""
|
||||
with self._lock:
|
||||
self._install_file.unlink(missing_ok=True)
|
||||
self._write_state(
|
||||
state="ready",
|
||||
error=reason,
|
||||
can_update=False,
|
||||
can_install=True,
|
||||
)
|
||||
|
||||
def _request(self) -> RequestUtils:
|
||||
return RequestUtils(
|
||||
proxies=get_runtime_setting('PROXY'),
|
||||
headers=get_runtime_setting('GITHUB_HEADERS'),
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
def _download_update(self, version: str) -> None:
|
||||
try:
|
||||
downloaded = 0
|
||||
downloaded, backend_total = self._download_file(
|
||||
self._proxied(self._BACKEND_ARCHIVE_URL.format(tag=version)),
|
||||
self._backend_archive,
|
||||
downloaded,
|
||||
0,
|
||||
)
|
||||
frontend_version = self._validate_backend_archive(version)
|
||||
frontend_release = self._fetch_frontend_release(frontend_version)
|
||||
frontend_asset = next(
|
||||
(
|
||||
item
|
||||
for item in frontend_release.get("assets") or []
|
||||
if item.get("name") == "dist.zip" and item.get("browser_download_url")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not frontend_asset:
|
||||
raise RuntimeError(f"前端 {frontend_version} 缺少 dist.zip 发布资产")
|
||||
frontend_total = int(frontend_asset.get("size") or 0)
|
||||
total = backend_total + frontend_total
|
||||
self._write_state(
|
||||
frontend_version=frontend_version,
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
)
|
||||
downloaded, _ = self._download_file(
|
||||
self._proxied(str(frontend_asset["browser_download_url"])),
|
||||
self._frontend_archive,
|
||||
downloaded,
|
||||
total,
|
||||
)
|
||||
self._validate_frontend_archive(frontend_version)
|
||||
expected_digest = str(frontend_asset.get("digest") or "")
|
||||
frontend_sha256 = self._sha256(self._frontend_archive)
|
||||
if expected_digest.startswith("sha256:") and frontend_sha256 != expected_digest.removeprefix("sha256:"):
|
||||
raise RuntimeError("前端更新包与 GitHub Release 摘要不一致")
|
||||
|
||||
if not is_docker():
|
||||
self._prepare_local_backend_ref(version)
|
||||
|
||||
prepared = {
|
||||
"version": version,
|
||||
"frontend_version": frontend_version,
|
||||
"backend_archive": str(self._backend_archive),
|
||||
"frontend_archive": str(self._frontend_archive),
|
||||
"backend_sha256": self._sha256(self._backend_archive),
|
||||
"frontend_sha256": frontend_sha256,
|
||||
"prepared_at": self._now(),
|
||||
}
|
||||
(self._root / "prepared.json").write_text(
|
||||
json.dumps(prepared, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
self._write_state(
|
||||
state="ready",
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=max(total, downloaded),
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=True,
|
||||
)
|
||||
logger.info(f"MoviePilot {version} 更新包已下载完成,等待用户确认重启")
|
||||
except Exception as error: # 后台线程必须把所有失败沉淀为可查询状态
|
||||
logger.error(f"下载 MoviePilot 更新包失败: {error}")
|
||||
self._write_state(
|
||||
state="failed", error=str(error), can_update=True, can_install=False
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._download_active = False
|
||||
|
||||
def _download_file(
|
||||
self, url: str, destination: Path, downloaded_before: int, total_hint: int
|
||||
) -> tuple[int, int]:
|
||||
temporary = destination.with_suffix(".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
with self._request().get_stream(url) as response:
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError(f"下载更新包失败:HTTP {getattr(response, 'status_code', '无响应')}")
|
||||
content_length = int(response.headers.get("content-length") or 0)
|
||||
total = total_hint or content_length
|
||||
current = downloaded_before
|
||||
with temporary.open("wb") as output:
|
||||
for chunk in response.iter_content(chunk_size=256 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
output.write(chunk)
|
||||
current += len(chunk)
|
||||
self._write_state(downloaded_bytes=current, total_bytes=total)
|
||||
temporary.replace(destination)
|
||||
return current, content_length
|
||||
|
||||
def _fetch_frontend_release(self, version: str) -> dict[str, Any]:
|
||||
response = self._request().get_res(
|
||||
self._FRONTEND_RELEASE_API.format(tag=version)
|
||||
)
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError(f"无法获取前端 {version} Release")
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("前端 Release 返回格式异常")
|
||||
return payload
|
||||
|
||||
def _validate_backend_archive(self, version: str) -> str:
|
||||
with zipfile.ZipFile(self._backend_archive) as archive:
|
||||
self._validate_zip_members(archive)
|
||||
version_name = next(
|
||||
(name for name in archive.namelist() if name.count("/") == 1 and name.endswith("/version.py")),
|
||||
None,
|
||||
)
|
||||
if not version_name:
|
||||
raise RuntimeError("后端更新包缺少 version.py")
|
||||
version_source = archive.read(version_name).decode("utf-8")
|
||||
app_match = re.search(r"^APP_VERSION\s*=\s*['\"]([^'\"]+)", version_source, re.M)
|
||||
frontend_match = re.search(r"^FRONTEND_VERSION\s*=\s*['\"]([^'\"]+)", version_source, re.M)
|
||||
if not app_match or app_match.group(1) != version:
|
||||
raise RuntimeError("后端更新包版本与目标 Release 不一致")
|
||||
if not frontend_match or not self._VERSION_PATTERN.fullmatch(frontend_match.group(1)):
|
||||
raise RuntimeError("后端更新包声明的前端版本无效")
|
||||
required = ("pyproject.toml", "uv.lock")
|
||||
names = archive.namelist()
|
||||
if any(not any(name.endswith(f"/{item}") for name in names) for item in required):
|
||||
raise RuntimeError("后端更新包缺少依赖锁定文件")
|
||||
return frontend_match.group(1)
|
||||
|
||||
def _validate_frontend_archive(self, version: str) -> None:
|
||||
with zipfile.ZipFile(self._frontend_archive) as archive:
|
||||
self._validate_zip_members(archive)
|
||||
names = set(archive.namelist())
|
||||
if "dist/index.html" not in names or "dist/version.txt" not in names:
|
||||
raise RuntimeError("前端更新包结构无效")
|
||||
archived_version = archive.read("dist/version.txt").decode("utf-8").strip()
|
||||
if archived_version != version:
|
||||
raise RuntimeError("前端更新包版本与后端声明不一致")
|
||||
|
||||
@staticmethod
|
||||
def _validate_zip_members(archive: zipfile.ZipFile) -> None:
|
||||
for item in archive.infolist():
|
||||
path = PurePosixPath(item.filename)
|
||||
file_type = (item.external_attr >> 16) & 0o170000
|
||||
if (
|
||||
path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or file_type == stat.S_IFLNK
|
||||
):
|
||||
raise RuntimeError("更新包包含不安全路径")
|
||||
|
||||
def _read_prepared_manifest(self) -> dict[str, Any]:
|
||||
payload = json.loads((self._root / "prepared.json").read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("更新包清单格式无效")
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
def _prepare_local_backend_ref(version: str) -> None:
|
||||
"""本地 CLI 在下载阶段获取标签,使重启后的代码切换不再联网。"""
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
if not (root / ".git").is_dir():
|
||||
raise RuntimeError("本地安装目录不是 Git 仓库,无法准备 Release 更新")
|
||||
try:
|
||||
worktree = subprocess.run(
|
||||
["git", "status", "--porcelain", "--untracked-files=no"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if worktree.stdout.strip():
|
||||
raise RuntimeError("本地源码存在未提交改动,无法准备 Release 更新")
|
||||
subprocess.run(
|
||||
["git", "fetch", "--no-tags", "origin", "tag", version],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--verify", f"{version}^{{commit}}"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
||||
raise RuntimeError(f"无法准备本地 Release 标签 {version}") from error
|
||||
|
||||
@staticmethod
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _proxied(url: str) -> str:
|
||||
proxy = str(get_runtime_setting('GITHUB_PROXY') or "").strip()
|
||||
return f"{proxy}{url}" if proxy else url
|
||||
|
||||
|
||||
system_update_manager = SystemUpdateManager()
|
||||
@@ -1 +0,0 @@
|
||||
"""Web 框架适配器。"""
|
||||
@@ -1,44 +0,0 @@
|
||||
"""HTTP 请求关联 ID 的 ASGI 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.runtime.correlation import (
|
||||
CORRELATION_ID_HEADER,
|
||||
correlation_scope,
|
||||
normalize_correlation_id,
|
||||
)
|
||||
|
||||
|
||||
class CorrelationIdMiddleware:
|
||||
"""验证入口 ID、绑定请求上下文并把同一 ID 写回响应。"""
|
||||
|
||||
def __init__(self, app: Any) -> None:
|
||||
"""保存下游 ASGI 应用。"""
|
||||
self._app = app
|
||||
|
||||
async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
|
||||
"""只治理 HTTP scope,并让绑定覆盖完整流式响应生命周期。"""
|
||||
if scope.get("type") != "http":
|
||||
await self._app(scope, receive, send)
|
||||
return
|
||||
raw_headers = dict(scope.get("headers") or [])
|
||||
candidate = raw_headers.get(CORRELATION_ID_HEADER.lower().encode("ascii"))
|
||||
correlation_id = normalize_correlation_id(
|
||||
candidate.decode("ascii", errors="ignore") if candidate else None
|
||||
)
|
||||
scope.setdefault("state", {})["request_id"] = correlation_id
|
||||
|
||||
async def send_with_correlation(message: dict) -> None:
|
||||
"""在响应开始帧中覆盖为当前请求的安全关联 ID。"""
|
||||
if message.get("type") == "http.response.start":
|
||||
headers = list(message.get("headers") or [])
|
||||
header_name = CORRELATION_ID_HEADER.lower().encode("ascii")
|
||||
headers = [item for item in headers if item[0].lower() != header_name]
|
||||
headers.append((header_name, correlation_id.encode("ascii")))
|
||||
message["headers"] = headers
|
||||
await send(message)
|
||||
|
||||
with correlation_scope(correlation_id):
|
||||
await self._app(scope, receive, send_with_correlation)
|
||||
@@ -1,45 +0,0 @@
|
||||
"""面向编排器的最小公开健康探针。"""
|
||||
|
||||
from fastapi import FastAPI, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from app.runtime.health import get_application_health
|
||||
|
||||
|
||||
async def liveness() -> JSONResponse:
|
||||
"""确认进程和当前事件循环能够处理请求,不访问任何外部依赖。"""
|
||||
return JSONResponse(content={"status": "alive"})
|
||||
|
||||
|
||||
async def readiness(request: Request) -> JSONResponse:
|
||||
"""仅公开可否接流量,不泄露数据库、插件或启动异常细节。"""
|
||||
ready = get_application_health(request.app).is_ready
|
||||
return JSONResponse(
|
||||
status_code=(
|
||||
status.HTTP_200_OK
|
||||
if ready
|
||||
else status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
),
|
||||
content={"status": "ready" if ready else "not_ready"},
|
||||
)
|
||||
|
||||
|
||||
def install_health_routes(app: FastAPI) -> None:
|
||||
"""把公开探针装到 API 版本前缀之外,供容器和反向代理使用。"""
|
||||
app.router.add_api_route(
|
||||
"/health/live",
|
||||
liveness,
|
||||
methods=["GET"],
|
||||
response_class=JSONResponse,
|
||||
include_in_schema=False,
|
||||
route_class_override=APIRoute,
|
||||
)
|
||||
app.router.add_api_route(
|
||||
"/health/ready",
|
||||
readiness,
|
||||
methods=["GET"],
|
||||
response_class=JSONResponse,
|
||||
include_in_schema=False,
|
||||
route_class_override=APIRoute,
|
||||
)
|
||||
@@ -1,58 +0,0 @@
|
||||
"""HTTP route/status/latency 指标 ASGI 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.runtime.observability import record_metric
|
||||
from starlette.routing import Match
|
||||
|
||||
|
||||
class HttpMetricsMiddleware:
|
||||
"""按路由模板、方法和状态码记录低基数 HTTP 时延。"""
|
||||
|
||||
def __init__(self, app: Any) -> None:
|
||||
"""保存下游 ASGI 应用。"""
|
||||
self._app = app
|
||||
|
||||
async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
|
||||
"""只治理 HTTP scope,并在响应开始后读取路由模板。"""
|
||||
if scope.get("type") != "http":
|
||||
await self._app(scope, receive, send)
|
||||
return
|
||||
started_at = time.perf_counter()
|
||||
status = "500"
|
||||
|
||||
async def send_with_metrics(message: dict) -> None:
|
||||
"""捕获响应状态并原样转发 ASGI 消息。"""
|
||||
nonlocal status
|
||||
if message.get("type") == "http.response.start":
|
||||
status = str(message.get("status", 500))
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self._app(scope, receive, send_with_metrics)
|
||||
finally:
|
||||
route_path = self._resolve_route_template(scope)
|
||||
record_metric(
|
||||
"http.server.duration",
|
||||
time.perf_counter() - started_at,
|
||||
route=route_path,
|
||||
method=str(scope.get("method", "UNKNOWN")),
|
||||
status=status,
|
||||
)
|
||||
|
||||
def _resolve_route_template(self, scope: dict) -> str:
|
||||
"""遍历 ASGI wrapper 找到匹配路由模板,绝不回退到具体请求 path。"""
|
||||
route = scope.get("route")
|
||||
if getattr(route, "path", None):
|
||||
return route.path
|
||||
candidate = self._app
|
||||
while candidate is not None:
|
||||
for registered_route in getattr(candidate, "routes", ()):
|
||||
match, _ = registered_route.matches(scope)
|
||||
if match == Match.FULL:
|
||||
return getattr(registered_route, "path", "unmatched")
|
||||
candidate = getattr(candidate, "app", None)
|
||||
return "unmatched"
|
||||
@@ -1 +0,0 @@
|
||||
"""插件 Web 适配器。"""
|
||||
@@ -1,178 +0,0 @@
|
||||
"""FastAPI 动态插件路由适配器。"""
|
||||
|
||||
import asyncio
|
||||
from concurrent.futures import Future, TimeoutError as FutureTimeoutError
|
||||
from threading import Lock
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
|
||||
class FastAPIDynamicRouteRegistry:
|
||||
"""在 FastAPI 上注册插件自由响应路由,并维护 OpenAPI 缓存。"""
|
||||
|
||||
_dispatch_admission_timeout = 5.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: FastAPI,
|
||||
plugin_ids: Callable[[], list[str]],
|
||||
plugin_apis: Callable[[str], list[dict]],
|
||||
verify_token: Callable[..., Any],
|
||||
verify_apikey: Callable[..., Any],
|
||||
prefix: str,
|
||||
protected_routes: set[str],
|
||||
log: Any,
|
||||
event_loop: Callable[[], asyncio.AbstractEventLoop | None] | None = None,
|
||||
) -> None:
|
||||
"""注入应用、插件投影、认证依赖和日志端口。"""
|
||||
self._app = app
|
||||
self._plugin_ids = plugin_ids
|
||||
self._plugin_apis = plugin_apis
|
||||
self._verify_token = verify_token
|
||||
self._verify_apikey = verify_apikey
|
||||
self._prefix = prefix
|
||||
self._protected_routes = protected_routes
|
||||
self._logger = log
|
||||
self._event_loop = event_loop
|
||||
|
||||
def update(self, plugin_id: Optional[str], action: str) -> None:
|
||||
"""在主事件循环中按插件生命周期新增或移除动态路由。"""
|
||||
if self._event_loop is None:
|
||||
self._update(plugin_id, action)
|
||||
return
|
||||
target_loop = self._event_loop()
|
||||
if (
|
||||
target_loop is None
|
||||
or not target_loop.is_running()
|
||||
or target_loop.is_closed()
|
||||
):
|
||||
raise RuntimeError("主事件循环未运行,无法更新插件动态路由")
|
||||
try:
|
||||
current_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
current_loop = None
|
||||
if current_loop is target_loop:
|
||||
self._update(plugin_id, action)
|
||||
return
|
||||
|
||||
completed: Future[None] = Future()
|
||||
dispatch_lock = Lock()
|
||||
dispatch_started = False
|
||||
dispatch_abandoned = False
|
||||
|
||||
def apply_update() -> None:
|
||||
"""在目标 loop 的单个回调中完成路由表与 OpenAPI 投影切换。"""
|
||||
nonlocal dispatch_started
|
||||
with dispatch_lock:
|
||||
if dispatch_abandoned:
|
||||
return
|
||||
dispatch_started = True
|
||||
try:
|
||||
self._update(plugin_id, action)
|
||||
except BaseException as error:
|
||||
completed.set_exception(error)
|
||||
else:
|
||||
completed.set_result(None)
|
||||
|
||||
target_loop.call_soon_threadsafe(apply_update)
|
||||
try:
|
||||
completed.result(timeout=self._dispatch_admission_timeout)
|
||||
except FutureTimeoutError as error:
|
||||
with dispatch_lock:
|
||||
if not dispatch_started:
|
||||
dispatch_abandoned = True
|
||||
raise RuntimeError(
|
||||
"主事件循环未及时接收插件动态路由更新"
|
||||
) from error
|
||||
# 回调一旦开始便不可撤销,等待确定终态以免失败回滚后发生迟到写入。
|
||||
completed.result()
|
||||
|
||||
def _update(self, plugin_id: Optional[str], action: str) -> None:
|
||||
"""执行不可中断的路由表与 OpenAPI 投影更新。"""
|
||||
if action not in {"add", "remove"}:
|
||||
raise ValueError("Action must be 'add' or 'remove'")
|
||||
|
||||
modified = False
|
||||
existing_paths = {
|
||||
path: route
|
||||
for route in self._app.routes
|
||||
if (path := self._route_path(route)) is not None
|
||||
}
|
||||
plugin_ids = [plugin_id] if plugin_id else self._plugin_ids()
|
||||
for current_id in plugin_ids:
|
||||
if self.remove(current_id):
|
||||
modified = True
|
||||
if action != "add":
|
||||
continue
|
||||
for source_api in self._plugin_apis(current_id):
|
||||
api = dict(source_api)
|
||||
api["dependencies"] = list(source_api.get("dependencies") or ())
|
||||
api_path = f"{self._prefix}{api.get('path', '')}"
|
||||
try:
|
||||
api["path"] = api_path
|
||||
allow_anonymous = api.pop("allow_anonymous", False)
|
||||
auth_mode = api.pop("auth", "apikey")
|
||||
dependencies = api.setdefault("dependencies", [])
|
||||
if not allow_anonymous:
|
||||
if (
|
||||
auth_mode == "bear"
|
||||
and Depends(self._verify_token) not in dependencies
|
||||
):
|
||||
dependencies.append(Depends(self._verify_token))
|
||||
elif Depends(self._verify_apikey) not in dependencies:
|
||||
dependencies.append(Depends(self._verify_apikey))
|
||||
# 插件 API 自行决定响应结构,不使用宿主统一 Response 路由。
|
||||
api.setdefault("route_class_override", APIRoute)
|
||||
self._app.router.add_api_route(**api, tags=["plugin"])
|
||||
modified = True
|
||||
self._logger.debug(f"Added plugin route: {api_path}")
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error adding plugin route {api_path}: {str(error)}"
|
||||
)
|
||||
if modified:
|
||||
self.clean(existing_paths)
|
||||
self._app.openapi_schema = None
|
||||
self._app.setup()
|
||||
|
||||
def remove(self, plugin_id: str) -> bool:
|
||||
"""移除指定插件前缀下的全部动态路由。"""
|
||||
if not plugin_id:
|
||||
return False
|
||||
prefix = f"{self._prefix}/{plugin_id}/"
|
||||
routes = [
|
||||
route for route in self._app.routes
|
||||
if (path := self._route_path(route)) is not None
|
||||
and path.startswith(prefix)
|
||||
]
|
||||
removed = False
|
||||
for route in routes:
|
||||
try:
|
||||
self._app.routes.remove(route)
|
||||
removed = True
|
||||
self._logger.debug(f"Removed plugin route: {self._route_path(route)}")
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error removing plugin route {self._route_path(route)}: {str(error)}"
|
||||
)
|
||||
return removed
|
||||
|
||||
@staticmethod
|
||||
def _route_path(route: Any) -> Optional[str]:
|
||||
"""返回公开路由路径,跳过 FastAPI 内部的无路径 include 包装器。"""
|
||||
path = getattr(route, "path", None)
|
||||
return path if isinstance(path, str) else None
|
||||
|
||||
def clean(self, existing_paths: dict) -> None:
|
||||
"""清理 FastAPI 重建时可能重复的受保护文档路由。"""
|
||||
for protected_route in self._protected_routes:
|
||||
try:
|
||||
existing_route = existing_paths.get(protected_route)
|
||||
if existing_route:
|
||||
self._app.routes.remove(existing_route)
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error removing protected route {protected_route}: {str(error)}"
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
"""Web 传输层认证适配器。"""
|
||||
@@ -1,274 +0,0 @@
|
||||
"""把应用安全能力适配为 FastAPI 认证依赖和 Cookie 行为。"""
|
||||
|
||||
import datetime
|
||||
from datetime import timedelta
|
||||
from typing import Annotated, Any, Callable, Optional
|
||||
|
||||
import jwt
|
||||
from fastapi import HTTPException, Request, Response, Security, status
|
||||
from fastapi.security import (
|
||||
APIKeyCookie,
|
||||
APIKeyHeader,
|
||||
APIKeyQuery,
|
||||
HTTPBearer,
|
||||
OAuth2PasswordBearer,
|
||||
)
|
||||
|
||||
from app.runtime.cache import cached
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.schemas.token import TokenPayload
|
||||
|
||||
SuperuserTokenPayloadProvider = Callable[[], TokenPayload]
|
||||
TokenEncoder = Callable[..., str]
|
||||
TokenDecoder = Callable[[str | None, str], TokenPayload]
|
||||
_superuser_token_payload_provider: Optional[SuperuserTokenPayloadProvider] = None
|
||||
_token_encoder: Optional[TokenEncoder] = None
|
||||
_token_decoder: Optional[TokenDecoder] = None
|
||||
JWT_ALGORITHM = "HS256"
|
||||
|
||||
|
||||
oauth2_scheme_manual_error = OAuth2PasswordBearer(
|
||||
auto_error=False,
|
||||
tokenUrl=f"{get_runtime_setting('API_V1_STR')}/login/access-token",
|
||||
)
|
||||
resource_token_cookie = APIKeyCookie(
|
||||
name=get_runtime_setting('PROJECT_NAME'),
|
||||
auto_error=False,
|
||||
scheme_name="resource_token_cookie",
|
||||
)
|
||||
api_token_query = APIKeyQuery(
|
||||
name="token",
|
||||
auto_error=False,
|
||||
scheme_name="api_token_query",
|
||||
)
|
||||
api_key_header = APIKeyHeader(
|
||||
name="X-API-KEY",
|
||||
auto_error=False,
|
||||
scheme_name="api_key_header",
|
||||
)
|
||||
api_key_query = APIKeyQuery(
|
||||
name="apikey",
|
||||
auto_error=False,
|
||||
scheme_name="api_key_query",
|
||||
)
|
||||
openai_bearer_scheme = HTTPBearer(auto_error=False)
|
||||
anthropic_api_key_header = APIKeyHeader(
|
||||
name="x-api-key",
|
||||
auto_error=False,
|
||||
scheme_name="anthropic_api_key_header",
|
||||
)
|
||||
|
||||
|
||||
def set_superuser_token_payload_provider(
|
||||
provider: SuperuserTokenPayloadProvider,
|
||||
) -> None:
|
||||
"""由启动组合根注入 API 密钥认证使用的超级用户载荷来源。"""
|
||||
global _superuser_token_payload_provider
|
||||
_superuser_token_payload_provider = provider
|
||||
|
||||
|
||||
def configure_token_codec(
|
||||
encoder: TokenEncoder,
|
||||
decoder: TokenDecoder,
|
||||
) -> None:
|
||||
"""由组合根注入框架无关的令牌编码与解码能力。"""
|
||||
global _token_encoder, _token_decoder
|
||||
_token_encoder = encoder
|
||||
_token_decoder = decoder
|
||||
|
||||
|
||||
def _encode_token(**claims: Any) -> str:
|
||||
"""使用已注入编码器创建令牌,未装配时给出明确错误。"""
|
||||
if _token_encoder is None:
|
||||
raise RuntimeError("Web 认证令牌编码器尚未配置")
|
||||
return _token_encoder(**claims)
|
||||
|
||||
|
||||
def _decode_token(token: str | None, purpose: str) -> TokenPayload:
|
||||
"""使用已注入解码器验证令牌,未装配时给出明确错误。"""
|
||||
if _token_decoder is None:
|
||||
raise RuntimeError("Web 认证令牌解码器尚未配置")
|
||||
return _token_decoder(token, purpose)
|
||||
|
||||
|
||||
def _get_api_token(
|
||||
token_query: Annotated[str | None, Security(api_token_query)] = None,
|
||||
) -> str | None:
|
||||
"""从 URL 查询参数读取兼容 API Token。"""
|
||||
return token_query
|
||||
|
||||
|
||||
def _get_api_key(
|
||||
key_query: Annotated[str | None, Security(api_key_query)] = None,
|
||||
key_header: Annotated[str | None, Security(api_key_header)] = None,
|
||||
) -> str | None:
|
||||
"""优先从请求头、其次从查询参数读取兼容 API Key。"""
|
||||
return key_header or key_query
|
||||
|
||||
|
||||
@cached(maxsize=1, ttl=600)
|
||||
def _create_superuser_token_payload() -> TokenPayload:
|
||||
"""使用组合根提供器创建 API 密钥调用的超级用户载荷。"""
|
||||
if not _superuser_token_payload_provider:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="认证服务尚未初始化",
|
||||
)
|
||||
try:
|
||||
return _superuser_token_payload_provider()
|
||||
except PermissionError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=str(error) or "用户权限不足",
|
||||
) from error
|
||||
|
||||
|
||||
def set_or_refresh_resource_token_cookie(
|
||||
request: Request,
|
||||
response: Response,
|
||||
payload: TokenPayload,
|
||||
) -> None:
|
||||
"""复用匹配的资源令牌,或为当前身份写入新的安全 Cookie。"""
|
||||
project_name = get_runtime_setting('PROJECT_NAME')
|
||||
resource_token = request.cookies.get(project_name)
|
||||
if resource_token:
|
||||
try:
|
||||
decoded = jwt.decode(
|
||||
resource_token,
|
||||
get_runtime_setting('RESOURCE_SECRET_KEY'),
|
||||
algorithms=[JWT_ALGORITHM],
|
||||
)
|
||||
exp = decoded.get("exp")
|
||||
if exp:
|
||||
remaining_time = datetime.datetime.fromtimestamp(
|
||||
exp,
|
||||
tz=datetime.UTC,
|
||||
) - datetime.datetime.now(datetime.UTC)
|
||||
if remaining_time < timedelta(
|
||||
seconds=(
|
||||
get_runtime_setting(
|
||||
"RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS"
|
||||
)
|
||||
/ 3
|
||||
)
|
||||
):
|
||||
raise jwt.ExpiredSignatureError
|
||||
expected_claims = {
|
||||
"sub": str(payload.sub),
|
||||
"username": payload.username,
|
||||
"super_user": payload.super_user,
|
||||
"level": payload.level,
|
||||
"purpose": "resource",
|
||||
}
|
||||
if any(
|
||||
decoded.get(claim) != value
|
||||
for claim, value in expected_claims.items()
|
||||
):
|
||||
raise jwt.InvalidTokenError("资源令牌身份或权限上下文不匹配")
|
||||
except jwt.PyJWTError:
|
||||
logger.debug("Token error occurred. refreshing token")
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
f"Unexpected error occurred while decoding token: {error}"
|
||||
)
|
||||
else:
|
||||
return
|
||||
|
||||
resource_token = _encode_token(
|
||||
userid=payload.sub,
|
||||
username=payload.username or "",
|
||||
super_user=payload.super_user,
|
||||
expires_delta=timedelta(
|
||||
seconds=get_runtime_setting('RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS')
|
||||
),
|
||||
level=payload.level,
|
||||
purpose="resource",
|
||||
)
|
||||
is_https = (
|
||||
request.url.scheme == "https"
|
||||
or request.headers.get("x-forwarded-proto", "").lower() == "https"
|
||||
)
|
||||
response.set_cookie(
|
||||
key=project_name,
|
||||
value=resource_token,
|
||||
httponly=True,
|
||||
secure=is_https,
|
||||
samesite="lax",
|
||||
)
|
||||
|
||||
|
||||
def _decode_or_http_error(
|
||||
token: str | None,
|
||||
purpose: str,
|
||||
) -> TokenPayload:
|
||||
"""把应用层令牌校验错误转换为 HTTP 403。"""
|
||||
try:
|
||||
return _decode_token(token, purpose)
|
||||
except ValueError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=str(error),
|
||||
) from error
|
||||
|
||||
|
||||
def verify_token(
|
||||
request: Request,
|
||||
response: Response,
|
||||
jwt_token: Annotated[
|
||||
str | None,
|
||||
Security(oauth2_scheme_manual_error),
|
||||
],
|
||||
api_key: Annotated[str | None, Security(_get_api_key)],
|
||||
api_token: Annotated[str | None, Security(_get_api_token)],
|
||||
) -> TokenPayload:
|
||||
"""验证 JWT、API Key 或 API Token,并维护资源 Cookie。"""
|
||||
if jwt_token:
|
||||
payload = _decode_or_http_error(jwt_token, "authentication")
|
||||
set_or_refresh_resource_token_cookie(request, response, payload)
|
||||
return payload
|
||||
if api_key:
|
||||
verify_apikey(api_key)
|
||||
return _create_superuser_token_payload()
|
||||
if api_token:
|
||||
verify_apitoken(api_token)
|
||||
return _create_superuser_token_payload()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def verify_resource_token(
|
||||
resource_token: Annotated[
|
||||
str | None,
|
||||
Security(resource_token_cookie),
|
||||
],
|
||||
) -> TokenPayload:
|
||||
"""验证 Cookie 中携带的资源访问令牌。"""
|
||||
return _decode_or_http_error(resource_token, "resource")
|
||||
|
||||
|
||||
def _verify_key(key: str | None, expected_key: str, key_type: str) -> str:
|
||||
"""校验受信第三方集成使用的固定 API 凭据。"""
|
||||
if not key or key != expected_key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=f"{key_type} 校验不通过",
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def verify_apitoken(
|
||||
token: Annotated[str | None, Security(_get_api_token)],
|
||||
) -> str:
|
||||
"""校验 URL 查询参数中的兼容 API Token。"""
|
||||
return _verify_key(token, get_runtime_setting('API_TOKEN'), "token")
|
||||
|
||||
|
||||
def verify_apikey(
|
||||
apikey: Annotated[str | None, Security(_get_api_key)],
|
||||
) -> str:
|
||||
"""校验请求头或查询参数中的兼容 API Key。"""
|
||||
return _verify_key(apikey, get_runtime_setting('API_TOKEN'), "apikey")
|
||||
@@ -1,19 +0,0 @@
|
||||
"""Agent 公共入口,按需加载完整编排实现。"""
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""按需从 Agent 编排模块解析历史包级公开对象。"""
|
||||
orchestrator = import_module("app.agent.orchestrator")
|
||||
try:
|
||||
return getattr(orchestrator, name)
|
||||
except AttributeError as err:
|
||||
raise AttributeError(f"module 'app.agent' has no attribute {name!r}") from err
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""返回 Agent 包和编排模块共同提供的可发现名称。"""
|
||||
orchestrator = import_module("app.agent.orchestrator")
|
||||
return sorted(set(globals()) | set(dir(orchestrator)))
|
||||
@@ -1,736 +0,0 @@
|
||||
import asyncio
|
||||
import re
|
||||
import threading
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.agent.policy.sanitizer import sanitize_for_host
|
||||
from app.chain import ChainBase
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.message import Message, MessageResponse
|
||||
from app.schemas.notification import ChannelCapabilityManager, ChannelCapability
|
||||
from app.schemas.types import NotificationChannel, MessageType
|
||||
|
||||
|
||||
class _StreamChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
_PATCH_FILE_HEADER_PATTERN = re.compile(
|
||||
r"\*\*\* (?:Add|Update|Delete) File:\s*(\S+)"
|
||||
)
|
||||
|
||||
|
||||
def _extract_first_patch_path(patch: Optional[str]) -> Optional[str]:
|
||||
"""从补丁文本中提取首个文件路径,作为流式消息展示目标。"""
|
||||
if not patch:
|
||||
return None
|
||||
match = _PATCH_FILE_HEADER_PATTERN.search(patch)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
class StreamingHandler:
|
||||
"""
|
||||
流式Token缓冲管理器
|
||||
|
||||
负责从 LLM 流式 token 中积累文本,并在支持消息编辑的渠道上实时推送给用户。
|
||||
|
||||
工作流程:
|
||||
1. Agent开始处理时调用 start_streaming(),检查渠道能力并启动定时刷新
|
||||
2. LLM 产生 token 时调用 emit() 积累到缓冲区
|
||||
3. 定时器周期性调用 _flush():
|
||||
- 第一次有内容时发送新消息(通过 send_direct_message 获取 message_id)
|
||||
- 后续有新内容时编辑同一条消息(通过 edit_message)
|
||||
- 当消息长度接近渠道限制时,冻结当前消息并发送新消息继续输出
|
||||
4. 工具调用时:
|
||||
- 流式渠道:工具消息直接 emit() 追加到 buffer,与 Agent 文字合并为同一条流式消息
|
||||
- 非流式渠道:调用 take() 取出已积累的文字,与工具消息合并独立发送
|
||||
5. Agent最终完成时调用 stop_streaming():执行最后一次刷新,
|
||||
返回是否已通过流式发送完所有内容(调用方据此决定是否还需额外发送)
|
||||
"""
|
||||
|
||||
# 流式输出的刷新间隔(秒)
|
||||
FLUSH_INTERVAL = 0.3
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.Lock()
|
||||
self._buffer = ""
|
||||
# 流式输出相关状态
|
||||
self._streaming_enabled = False
|
||||
self._flush_task: Optional[asyncio.Task] = None
|
||||
self._streaming_lifecycle_lock = asyncio.Lock()
|
||||
# 当前消息的发送信息(用于编辑消息)
|
||||
self._message_response: Optional[MessageResponse] = None
|
||||
# 已发送给用户的文本(用于追踪增量)
|
||||
self._sent_text = ""
|
||||
# 当前消息的起始偏移量(buffer 中属于当前消息的起始位置)
|
||||
self._msg_start_offset = 0
|
||||
# 当前渠道的单条消息最大长度(0 表示不限制)
|
||||
self._max_message_length = 0
|
||||
# 消息发送所需的上下文信息
|
||||
self._channel: Optional[str] = None
|
||||
self._source: Optional[str] = None
|
||||
self._user_id: Optional[str] = None
|
||||
self._username: Optional[str] = None
|
||||
self._original_message_id: Optional[str] = None
|
||||
self._original_chat_id: Optional[str] = None
|
||||
self._title: str = ""
|
||||
self._allow_dispatch_without_context = False
|
||||
# 非啰嗦模式下的待输出工具统计,等下一段文本到来时再统一补一句摘要
|
||||
self._pending_tool_stats: dict[str, dict[str, Any]] = {}
|
||||
# 本轮已写入缓冲区的工具摘要行,供 Telegram 富文本渲染时做区分样式
|
||||
self._tool_summaries: set[str] = set()
|
||||
|
||||
def set_dispatch_policy(
|
||||
self, allow_dispatch_without_context: bool = False
|
||||
) -> None:
|
||||
"""
|
||||
设置在缺少渠道上下文时是否仍允许向默认通知渠道分发消息。
|
||||
后台 DISPATCH 任务允许,CAPTURE_ONLY 必须禁止。
|
||||
"""
|
||||
self._allow_dispatch_without_context = allow_dispatch_without_context
|
||||
|
||||
def emit(self, token: str) -> str:
|
||||
"""
|
||||
接收 LLM 流式 token,积累到缓冲区。
|
||||
如果存在待输出的工具统计,则会先补上一句摘要再追加 token。
|
||||
"""
|
||||
with self._lock:
|
||||
emitted = token or ""
|
||||
|
||||
if self._pending_tool_stats:
|
||||
summary = self._consume_pending_tool_summary_locked()
|
||||
if summary:
|
||||
if emitted:
|
||||
emitted = f"{summary}{emitted.lstrip(chr(10))}"
|
||||
else:
|
||||
emitted = summary
|
||||
|
||||
# 如果存量消息结束是两个换行,则去掉新消息前面的换行,避免过多空行
|
||||
if self._buffer.endswith("\n\n") and emitted.startswith("\n"):
|
||||
emitted = emitted.lstrip("\n")
|
||||
self._buffer += emitted
|
||||
return emitted
|
||||
|
||||
async def take(self) -> str:
|
||||
"""
|
||||
获取当前已积累的消息内容,获取后清空缓冲区。
|
||||
|
||||
用于非流式渠道:工具调用前取出 Agent 已产出的文字,
|
||||
与工具提示合并后独立发送。
|
||||
|
||||
注意:流式渠道不调用此方法,工具消息直接 emit 到 buffer 中。
|
||||
"""
|
||||
self.flush_pending_tool_summary()
|
||||
|
||||
with self._lock:
|
||||
if not self._buffer:
|
||||
return ""
|
||||
message = self._buffer
|
||||
logger.info(f"Agent消息: {message}")
|
||||
self._buffer = ""
|
||||
return message
|
||||
|
||||
def clear(self):
|
||||
"""
|
||||
清空缓冲区(不返回内容)
|
||||
"""
|
||||
with self._lock:
|
||||
self._buffer = ""
|
||||
self._sent_text = ""
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
def reset(self):
|
||||
"""
|
||||
重置缓冲区,清空已发送的文本从头更新,但保持消息编辑能力。
|
||||
|
||||
与 clear 的区别:
|
||||
- clear:完全重置所有状态,后续会开新消息
|
||||
- reset:只清空buffer,保留消息编辑状态,后续继续编辑同一条消息
|
||||
"""
|
||||
with self._lock:
|
||||
self._buffer = ""
|
||||
self._sent_text = ""
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
async def start_streaming(
|
||||
self,
|
||||
channel: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
original_message_id: Optional[str] = None,
|
||||
original_chat_id: Optional[str] = None,
|
||||
title: str = "",
|
||||
):
|
||||
"""串行启动流式输出,禁止新一轮覆盖尚未结束的刷新 owner。"""
|
||||
async with self._streaming_lifecycle_lock:
|
||||
await self._start_streaming(
|
||||
channel=channel,
|
||||
source=source,
|
||||
user_id=user_id,
|
||||
username=username,
|
||||
original_message_id=original_message_id,
|
||||
original_chat_id=original_chat_id,
|
||||
title=title,
|
||||
)
|
||||
|
||||
async def _start_streaming(
|
||||
self,
|
||||
channel: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
original_message_id: Optional[str] = None,
|
||||
original_chat_id: Optional[str] = None,
|
||||
title: str = "",
|
||||
):
|
||||
"""
|
||||
启动流式输出。
|
||||
始终标记为流式状态(用于 buffer 收集 token),
|
||||
但只有渠道支持消息编辑时才启动定时刷新任务(实时推送给用户)。
|
||||
:param channel: 消息渠道
|
||||
:param source: 消息来源
|
||||
:param user_id: 用户ID
|
||||
:param username: 用户名
|
||||
:param title: 消息标题
|
||||
:param original_message_id: 原始消息ID(如果是回复消息)
|
||||
:param original_chat_id: 原始聊天ID(如果是回复消息)
|
||||
"""
|
||||
if self._flush_task is not None:
|
||||
self._streaming_enabled = False
|
||||
await self._cancel_flush_task()
|
||||
|
||||
self._channel = channel
|
||||
self._source = source
|
||||
self._user_id = user_id
|
||||
self._username = username
|
||||
self._original_message_id = original_message_id
|
||||
self._original_chat_id = original_chat_id
|
||||
self._title = title
|
||||
|
||||
self._streaming_enabled = True
|
||||
self._sent_text = ""
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
# 检查渠道是否支持消息编辑,不支持则仅收集 token 到 buffer,不实时推送
|
||||
if not self._can_stream():
|
||||
logger.debug(f"渠道 {channel} 不支持消息编辑,仅启用 buffer 收集模式")
|
||||
return
|
||||
|
||||
# 从渠道能力中获取单条消息最大长度
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
self._max_message_length = ChannelCapabilityManager.get_max_message_length(
|
||||
channel_enum
|
||||
)
|
||||
except (ValueError, KeyError):
|
||||
self._max_message_length = 0
|
||||
|
||||
# 启动异步定时刷新任务
|
||||
self._flush_task = asyncio.create_task(self._flush_loop())
|
||||
logger.debug("流式输出已启动")
|
||||
|
||||
async def stop_streaming(self) -> Tuple[bool, str]:
|
||||
"""串行停止流式输出,并等待本轮刷新与最终消息收口。"""
|
||||
async with self._streaming_lifecycle_lock:
|
||||
return await self._stop_streaming()
|
||||
|
||||
async def _stop_streaming(self) -> Tuple[bool, str]:
|
||||
"""
|
||||
停止流式输出。执行最后一次刷新确保所有内容都已发送。
|
||||
:return: (all_sent, final_text)
|
||||
all_sent: 是否已经通过流式编辑将最终完整内容发送给了用户
|
||||
(True 表示调用方无需再额外发送消息)
|
||||
final_text: 流式发送的完整文本内容(用于调用方保存消息记录)
|
||||
"""
|
||||
if not self._streaming_enabled:
|
||||
return False, ""
|
||||
|
||||
self._streaming_enabled = False
|
||||
|
||||
# 取消定时任务
|
||||
await self._cancel_flush_task()
|
||||
|
||||
# 将未落地的工具统计补入缓冲区,避免流式结束时丢失这段执行信息
|
||||
self.flush_pending_tool_summary()
|
||||
|
||||
# 执行最后一次刷新
|
||||
await self._flush()
|
||||
|
||||
message_response = self._message_response
|
||||
if message_response:
|
||||
await run_in_threadpool(
|
||||
_StreamChain().finalize_message,
|
||||
message_response,
|
||||
)
|
||||
|
||||
# 检查是否所有缓冲内容都已发送
|
||||
with self._lock:
|
||||
# 当前消息的文本 = buffer 中从 _msg_start_offset 开始的部分
|
||||
current_msg_text = self._buffer[self._msg_start_offset:]
|
||||
all_sent = (
|
||||
self._message_response is not None
|
||||
and self._sent_text
|
||||
and current_msg_text == self._sent_text
|
||||
)
|
||||
# 保留最终文本用于返回(返回完整 buffer 内容,包含所有分段消息)
|
||||
final_text = self._buffer if all_sent else ""
|
||||
# 重置状态
|
||||
self._sent_text = ""
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
if all_sent:
|
||||
# 所有内容已通过流式发送,清空缓冲区
|
||||
self._buffer = ""
|
||||
return all_sent, final_text
|
||||
|
||||
def record_tool_call(
|
||||
self,
|
||||
tool_name: str,
|
||||
tool_message: Optional[str] = None,
|
||||
tool_kwargs: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
记录一次工具调用,供非啰嗦模式下延迟汇总输出。
|
||||
"""
|
||||
recorded_message = sanitize_for_host(tool_message) if tool_message else tool_message
|
||||
recorded_args = sanitize_for_host(tool_kwargs or {})
|
||||
if not isinstance(recorded_args, dict):
|
||||
recorded_args = {}
|
||||
category, target = self._classify_tool_call(
|
||||
tool_name=tool_name,
|
||||
tool_message=recorded_message,
|
||||
tool_kwargs=recorded_args,
|
||||
)
|
||||
target_values = []
|
||||
if isinstance(target, (list, tuple, set)):
|
||||
target_values = [item for item in target if item]
|
||||
elif target:
|
||||
target_values = [target]
|
||||
|
||||
with self._lock:
|
||||
bucket = self._pending_tool_stats.setdefault(
|
||||
category,
|
||||
{
|
||||
"count": 0,
|
||||
"targets": set(),
|
||||
},
|
||||
)
|
||||
if category == "subagent" and target_values:
|
||||
bucket["count"] += len(target_values)
|
||||
else:
|
||||
bucket["count"] += 1
|
||||
for target_value in target_values:
|
||||
bucket["targets"].add(str(target_value))
|
||||
|
||||
@staticmethod
|
||||
def _extract_subagent_targets(tool_kwargs: dict[str, Any]) -> list[str]:
|
||||
"""提取子代理工具请求中的目标子代理类型。"""
|
||||
tasks = tool_kwargs.get("tasks")
|
||||
if not isinstance(tasks, list):
|
||||
subagent_type = tool_kwargs.get("subagent_type")
|
||||
return [str(subagent_type)] if subagent_type else []
|
||||
|
||||
targets = []
|
||||
for task in tasks:
|
||||
if isinstance(task, dict):
|
||||
subagent_type = task.get("subagent_type")
|
||||
else:
|
||||
subagent_type = getattr(task, "subagent_type", None)
|
||||
if subagent_type:
|
||||
targets.append(str(subagent_type))
|
||||
return targets
|
||||
|
||||
def flush_pending_tool_summary(self) -> str:
|
||||
"""
|
||||
将待输出的工具统计摘要补入缓冲区,并返回本次新增的摘要文本。
|
||||
"""
|
||||
with self._lock:
|
||||
summary = self._consume_pending_tool_summary_locked()
|
||||
if summary:
|
||||
self._buffer += summary
|
||||
return summary
|
||||
|
||||
@staticmethod
|
||||
def _classify_tool_call(
|
||||
tool_name: str,
|
||||
tool_message: Optional[str],
|
||||
tool_kwargs: dict[str, Any],
|
||||
) -> tuple[str, Optional[Any]]:
|
||||
tool_name = (tool_name or "").strip().lower()
|
||||
tool_message = (tool_message or "").strip()
|
||||
tool_message_lower = tool_message.lower()
|
||||
|
||||
if tool_name == "skill":
|
||||
return "skill", tool_kwargs.get("name")
|
||||
if tool_name == "query_activity_log":
|
||||
return "activity_log", tool_kwargs.get("keyword") or tool_kwargs.get("date")
|
||||
if tool_name == "subagent_task":
|
||||
return "subagent", StreamingHandler._extract_subagent_targets(tool_kwargs)
|
||||
if tool_name == "task":
|
||||
return "subagent", tool_kwargs.get("subagent_type")
|
||||
if tool_name == "read_file":
|
||||
return "file_read", tool_kwargs.get("file_path")
|
||||
if tool_name in {"write_file", "edit_file"}:
|
||||
return "file_write", tool_kwargs.get("file_path")
|
||||
if tool_name == "apply_patch":
|
||||
return "file_write", _extract_first_patch_path(tool_kwargs.get("patch"))
|
||||
if tool_name in {"list_directory", "query_directory_settings"}:
|
||||
return "directory", tool_kwargs.get("path")
|
||||
if tool_name == "browse_webpage":
|
||||
return (
|
||||
"web_browse",
|
||||
tool_kwargs.get("url")
|
||||
or tool_kwargs.get("target_url")
|
||||
or tool_kwargs.get("path"),
|
||||
)
|
||||
if tool_name == "execute_command":
|
||||
return (
|
||||
"command",
|
||||
tool_kwargs.get("command") or tool_kwargs.get("session_id"),
|
||||
)
|
||||
if tool_name == "ask_user_choice":
|
||||
return "interaction", tool_kwargs.get("message")
|
||||
if tool_name.startswith("search_") or tool_name in {"get_search_results"}:
|
||||
return (
|
||||
"search",
|
||||
tool_kwargs.get("query")
|
||||
or tool_kwargs.get("title")
|
||||
or tool_kwargs.get("keyword"),
|
||||
)
|
||||
if tool_name.startswith("query_") or tool_name.startswith("list_") or tool_name.startswith("get_"):
|
||||
return "data_query", None
|
||||
if tool_name.startswith(("add_", "update_", "delete_", "modify_", "run_")):
|
||||
return "action", None
|
||||
if tool_name in {
|
||||
"recognize_media",
|
||||
"scrape_metadata",
|
||||
"transfer_file",
|
||||
"test_site",
|
||||
"send_message",
|
||||
"send_local_file",
|
||||
"send_voice_message",
|
||||
}:
|
||||
return "action", None
|
||||
|
||||
if "读取文件" in tool_message or "read file" in tool_message_lower:
|
||||
return "file_read", tool_kwargs.get("file_path")
|
||||
if (
|
||||
"写入文件" in tool_message
|
||||
or "编辑文件" in tool_message
|
||||
or "write file" in tool_message_lower
|
||||
or "edit file" in tool_message_lower
|
||||
):
|
||||
return "file_write", tool_kwargs.get("file_path")
|
||||
if "目录" in tool_message or "directory" in tool_message_lower:
|
||||
return "directory", tool_kwargs.get("path")
|
||||
if "搜索" in tool_message or "search" in tool_message_lower:
|
||||
return (
|
||||
"search",
|
||||
tool_kwargs.get("query")
|
||||
or tool_kwargs.get("title")
|
||||
or tool_kwargs.get("keyword"),
|
||||
)
|
||||
if "网页" in tool_message or "browser" in tool_message_lower or "webpage" in tool_message_lower:
|
||||
return "web_browse", tool_kwargs.get("url")
|
||||
if "命令" in tool_message or "command" in tool_message_lower:
|
||||
return "command", tool_kwargs.get("command")
|
||||
|
||||
return "tool", None
|
||||
|
||||
def _consume_pending_tool_summary_locked(self) -> str:
|
||||
if not self._pending_tool_stats:
|
||||
return ""
|
||||
|
||||
parts = []
|
||||
for category, bucket in self._pending_tool_stats.items():
|
||||
value = bucket["count"]
|
||||
if category in {"file_read", "file_write", "directory", "web_browse", "skill"} and bucket["targets"]:
|
||||
value = len(bucket["targets"])
|
||||
part = self._format_tool_stat(category, value)
|
||||
if part:
|
||||
parts.append(part)
|
||||
|
||||
self._pending_tool_stats = {}
|
||||
if not parts:
|
||||
return ""
|
||||
|
||||
summary = f"({','.join(parts)})"
|
||||
self._tool_summaries.add(summary)
|
||||
# 摘要前始终保证一个空行,让工具执行信息与正文分属不同段落,
|
||||
# 避免 Markdown 富文本把单个换行折叠成同一段落内的软换行
|
||||
visible_buffer = self._buffer.rstrip(" \t")
|
||||
trailing_newlines = len(visible_buffer) - len(visible_buffer.rstrip("\n"))
|
||||
prefix = ""
|
||||
if visible_buffer.strip():
|
||||
prefix = "\n" * max(2 - trailing_newlines, 0)
|
||||
return f"{prefix}{summary}\n\n"
|
||||
|
||||
@staticmethod
|
||||
def _format_tool_stat(category: str, count: int) -> str:
|
||||
if count <= 0:
|
||||
return ""
|
||||
|
||||
if category == "search":
|
||||
return f"执行了 {count} 次搜索"
|
||||
if category == "file_read":
|
||||
return f"读取了 {count} 个文件"
|
||||
if category == "file_write":
|
||||
return f"修改了 {count} 个文件"
|
||||
if category == "directory":
|
||||
return f"查看了 {count} 个目录"
|
||||
if category == "web_browse":
|
||||
return f"浏览了 {count} 个网页"
|
||||
if category == "command":
|
||||
return f"执行了 {count} 条命令"
|
||||
if category == "data_query":
|
||||
return f"查询了 {count} 次数据"
|
||||
if category == "skill":
|
||||
return f"查询了 {count} 个技能说明"
|
||||
if category == "activity_log":
|
||||
return f"查询了 {count} 次活动日志"
|
||||
if category == "action":
|
||||
return f"执行了 {count} 次操作"
|
||||
if category == "interaction":
|
||||
return f"发起了 {count} 次交互"
|
||||
if category == "subagent":
|
||||
return f"已调用 {count} 个子代理"
|
||||
return f"调用了 {count} 次工具"
|
||||
|
||||
def _can_stream(self) -> bool:
|
||||
"""
|
||||
检查当前渠道是否支持流式输出(消息编辑)
|
||||
"""
|
||||
if not self._channel:
|
||||
return False
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
return ChannelCapabilityManager.supports_capability(
|
||||
channel_enum, ChannelCapability.MESSAGE_EDITING
|
||||
)
|
||||
except (ValueError, KeyError):
|
||||
return False
|
||||
|
||||
def _get_rich_message(self, text: str) -> Optional[str]:
|
||||
"""
|
||||
为 Telegram 流式消息返回 Rich Markdown,其他渠道继续使用原有格式。
|
||||
"""
|
||||
if self._channel != NotificationChannel.Telegram.value:
|
||||
return None
|
||||
return self._quote_tool_summary_lines(text)
|
||||
|
||||
def _quote_tool_summary_lines(self, text: str) -> str:
|
||||
"""
|
||||
将缓冲区中的工具摘要整行转换为 Markdown 引用块。
|
||||
|
||||
富文本会把普通段落间的空行折叠成紧凑排版,引用块作为独立 block 类型
|
||||
渲染,保证工具执行信息在 Telegram 上始终与正文有可辨识的视觉分隔。
|
||||
"""
|
||||
if not self._tool_summaries or not text:
|
||||
return text
|
||||
return "\n".join(
|
||||
f"> {line}" if line in self._tool_summaries else line
|
||||
for line in text.split("\n")
|
||||
)
|
||||
|
||||
async def _flush_loop(self):
|
||||
"""
|
||||
定时刷新循环,定期将缓冲区内容发送/编辑到用户
|
||||
"""
|
||||
try:
|
||||
while self._streaming_enabled:
|
||||
await asyncio.sleep(self.FLUSH_INTERVAL)
|
||||
if self._streaming_enabled:
|
||||
await self._flush()
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"流式刷新异常: {e}")
|
||||
|
||||
async def _cancel_flush_task(self):
|
||||
"""
|
||||
停止当前的定时刷新任务。
|
||||
|
||||
停止流式输出时,刷新任务可能已经在线程池里发出了首条消息。
|
||||
这里先等待该轮刷新自然完成,确保 message_id 等返回信息能落回本地状态;
|
||||
否则最终刷新会误以为尚未发送过消息,从而再次发送一条新消息。
|
||||
"""
|
||||
current_task = asyncio.current_task()
|
||||
if (
|
||||
self._flush_task
|
||||
and not self._flush_task.done()
|
||||
and self._flush_task is not current_task
|
||||
):
|
||||
try:
|
||||
await self._flush_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._flush_task = None
|
||||
|
||||
async def _flush(self):
|
||||
"""
|
||||
将当前缓冲区内容刷新到用户消息
|
||||
- 如果还没有发送过消息,先发送一条新消息并记录message_id
|
||||
- 如果已经发送过消息,编辑该消息为最新的完整内容
|
||||
- 如果当前消息内容超过长度限制,冻结当前消息并发送新消息继续输出
|
||||
"""
|
||||
with self._lock:
|
||||
# 当前消息的文本 = buffer 中从 _msg_start_offset 开始的部分
|
||||
current_text = self._buffer[self._msg_start_offset:]
|
||||
if not current_text or current_text == self._sent_text:
|
||||
# 没有新内容需要刷新
|
||||
return
|
||||
if (
|
||||
(not self._channel or not self._source)
|
||||
and not self._allow_dispatch_without_context
|
||||
):
|
||||
logger.debug("流式输出缺少渠道上下文,当前模式禁止外发消息")
|
||||
return
|
||||
|
||||
chain = _StreamChain()
|
||||
|
||||
try:
|
||||
if self._message_response is None:
|
||||
# 第一次发送:发送新消息并获取 message_id
|
||||
response = await run_in_threadpool(
|
||||
chain.send_direct_message,
|
||||
Message(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
mtype=MessageType.Agent,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
original_message_id=self._original_message_id,
|
||||
original_chat_id=self._original_chat_id,
|
||||
title=self._title,
|
||||
text=current_text,
|
||||
rich_message=self._get_rich_message(current_text),
|
||||
save_history=False,
|
||||
),
|
||||
)
|
||||
if response and response.success and response.message_id:
|
||||
self._message_response = response
|
||||
with self._lock:
|
||||
self._sent_text = current_text
|
||||
logger.debug(
|
||||
f"流式输出初始消息已发送: message_id={response.message_id}"
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"流式输出初始消息发送失败或未返回message_id,降级为非流式输出"
|
||||
)
|
||||
self._streaming_enabled = False
|
||||
else:
|
||||
# 检查当前消息内容是否超过长度限制
|
||||
if (
|
||||
self._max_message_length
|
||||
and len(current_text) > self._max_message_length
|
||||
):
|
||||
# 消息过长,冻结当前消息(保持最后一次成功编辑的内容)
|
||||
# 将 offset 移动到已发送文本之后,开启新消息
|
||||
logger.debug(
|
||||
f"流式消息长度 {len(current_text)} 超过限制 {self._max_message_length},启用新消息"
|
||||
)
|
||||
with self._lock:
|
||||
self._msg_start_offset += len(self._sent_text)
|
||||
current_text = self._buffer[self._msg_start_offset:]
|
||||
self._message_response = None
|
||||
self._sent_text = ""
|
||||
|
||||
# 如果偏移后还有新内容,立即发送为新消息
|
||||
if current_text:
|
||||
response = await run_in_threadpool(
|
||||
chain.send_direct_message,
|
||||
Message(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
mtype=MessageType.Agent,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
original_message_id=self._original_message_id,
|
||||
original_chat_id=self._original_chat_id,
|
||||
title=self._title,
|
||||
text=current_text,
|
||||
rich_message=self._get_rich_message(current_text),
|
||||
save_history=False,
|
||||
),
|
||||
)
|
||||
if response and response.success and response.message_id:
|
||||
self._message_response = response
|
||||
with self._lock:
|
||||
self._sent_text = current_text
|
||||
logger.debug(
|
||||
f"流式输出新消息已发送: message_id={response.message_id}"
|
||||
)
|
||||
else:
|
||||
logger.debug("流式输出新消息发送失败,降级为非流式输出")
|
||||
self._streaming_enabled = False
|
||||
else:
|
||||
# 后续更新:编辑已有消息
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
except (ValueError, KeyError):
|
||||
return
|
||||
|
||||
metadata = dict(self._message_response.metadata or {})
|
||||
rich_message = self._get_rich_message(current_text)
|
||||
if rich_message:
|
||||
# 通用编辑接口不增加渠道专属参数,通过元数据交给 Telegram 模块消费。
|
||||
metadata["telegram_rich_message"] = rich_message
|
||||
success = await run_in_threadpool(
|
||||
chain.edit_message,
|
||||
channel=channel_enum,
|
||||
source=self._message_response.source,
|
||||
message_id=self._message_response.message_id,
|
||||
chat_id=self._message_response.chat_id,
|
||||
text=current_text,
|
||||
title=self._title,
|
||||
metadata=metadata,
|
||||
)
|
||||
if success:
|
||||
with self._lock:
|
||||
self._sent_text = current_text
|
||||
else:
|
||||
logger.debug("流式输出消息编辑失败")
|
||||
except Exception as e:
|
||||
logger.error(f"流式输出刷新失败: {e}")
|
||||
|
||||
@property
|
||||
def is_streaming(self) -> bool:
|
||||
"""
|
||||
是否正在流式输出
|
||||
"""
|
||||
return self._streaming_enabled
|
||||
|
||||
@property
|
||||
def is_auto_flushing(self) -> bool:
|
||||
"""
|
||||
是否正在定时刷新(渠道支持消息编辑时自动推送 buffer 内容)
|
||||
"""
|
||||
return self._flush_task is not None
|
||||
|
||||
@property
|
||||
def has_sent_message(self) -> bool:
|
||||
"""
|
||||
是否已经通过流式输出发送过消息(当前轮次)
|
||||
"""
|
||||
return self._message_response is not None
|
||||
|
||||
@property
|
||||
def last_buffer_char(self) -> str:
|
||||
"""
|
||||
返回当前缓冲区最后一个字符;缓冲区为空时返回空字符串。
|
||||
"""
|
||||
with self._lock:
|
||||
return self._buffer[-1:] if self._buffer else ""
|
||||
@@ -1,8 +0,0 @@
|
||||
"""Agent Capability 声明与通用入口适配器。"""
|
||||
|
||||
AGENT_ENTRYPOINT_KIND = "agent_entrypoint"
|
||||
AGENT_SERVICE_KIND = "agent_service"
|
||||
AGENT_MANAGER_CAPABILITY_ID = "agent.manager"
|
||||
AGENT_SERVICE_CAPABILITY_ID = "agent.service"
|
||||
MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID = "agent.moviepilot_type"
|
||||
TOOL_FACTORY_CAPABILITY_ID = "agent.tool_factory"
|
||||
@@ -1,221 +0,0 @@
|
||||
"""Agent canonical entrypoint 的 Capability Runtime 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from app.agent.capabilities import AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND
|
||||
from app.runtime.capabilities.errors import CapabilityAdapterContractError
|
||||
from app.runtime.capabilities.model import (
|
||||
ActivationPolicy,
|
||||
AdapterExecutionMode,
|
||||
CapabilitySpec,
|
||||
SelectorSchema,
|
||||
)
|
||||
from app.runtime.capabilities.registry import CapabilityRegistry
|
||||
from app.runtime.settings import get_runtime_setting, has_runtime_setting
|
||||
|
||||
_DEFAULT_CAPABILITY_ROOT = Path(__file__).resolve().parent
|
||||
_SETTING_SELECTOR = "setting_truthy"
|
||||
|
||||
|
||||
def _validate_setting_selector(config: Mapping[str, Any]) -> None:
|
||||
"""限制 selector 只能读取已声明的应用设置。"""
|
||||
key = config["key"]
|
||||
if not isinstance(key, str) or not key or not has_runtime_setting(key):
|
||||
raise ValueError(f"未知应用设置:{key!r}")
|
||||
|
||||
|
||||
AGENT_SELECTOR_SCHEMAS = {
|
||||
_SETTING_SELECTOR: SelectorSchema(
|
||||
required_fields=frozenset({"key"}),
|
||||
validator=_validate_setting_selector,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _load_entrypoint(spec: CapabilitySpec) -> Any:
|
||||
"""按 manifest 解析 canonical 符号,不创建额外业务对象。"""
|
||||
module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1)
|
||||
module = importlib.import_module(module_name)
|
||||
try:
|
||||
return getattr(module, symbol_name)
|
||||
except AttributeError as error:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint} 未公开 Agent entrypoint"
|
||||
) from error
|
||||
|
||||
|
||||
def _lifecycle_method(spec: CapabilitySpec, candidate: Any, name: str) -> Any:
|
||||
"""读取 Agent Service 必需的异步生命周期方法。"""
|
||||
callback = getattr(candidate, name, None)
|
||||
if not callable(callback):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint} 的 Agent Service 缺少 {name}()"
|
||||
)
|
||||
return callback
|
||||
|
||||
|
||||
class AgentEntrypointAdapter:
|
||||
"""把 canonical Python 符号作为无资源副作用的同步能力发布。"""
|
||||
|
||||
execution_mode = AdapterExecutionMode.SYNC
|
||||
|
||||
@staticmethod
|
||||
def materialize(spec: CapabilitySpec) -> Any:
|
||||
"""按 manifest entrypoint 导入 canonical 符号。"""
|
||||
return _load_entrypoint(spec)
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
_spec: CapabilitySpec,
|
||||
implementation: Any,
|
||||
_generation: int,
|
||||
_previous: Any = None,
|
||||
) -> Any:
|
||||
"""发布 canonical 符号本身,不创建第二份业务对象。"""
|
||||
return implementation
|
||||
|
||||
@staticmethod
|
||||
def start(
|
||||
_spec: CapabilitySpec,
|
||||
_candidate: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""entrypoint 不拥有业务资源,初始化由独立 service 能力负责。"""
|
||||
|
||||
@staticmethod
|
||||
def stop(
|
||||
_spec: CapabilitySpec,
|
||||
_instance: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""撤销入口可见性;业务资源由独立 service 能力关闭。"""
|
||||
|
||||
@staticmethod
|
||||
def cleanup(
|
||||
_spec: CapabilitySpec,
|
||||
_candidate: Any,
|
||||
_generation: int,
|
||||
_error: BaseException,
|
||||
) -> None:
|
||||
"""entrypoint 启动无副作用,因此失败候选无需额外释放。"""
|
||||
|
||||
|
||||
class AgentServiceAdapter:
|
||||
"""把具备 initialize/close 的 canonical 对象接入异步资源生命周期。"""
|
||||
|
||||
execution_mode = AdapterExecutionMode.ASYNC
|
||||
|
||||
@staticmethod
|
||||
async def materialize(spec: CapabilitySpec) -> Any:
|
||||
"""在线程中导入 canonical service,避免阻塞应用事件循环。"""
|
||||
return await asyncio.to_thread(_load_entrypoint, spec)
|
||||
|
||||
@staticmethod
|
||||
async def create(
|
||||
_spec: CapabilitySpec,
|
||||
implementation: Any,
|
||||
_generation: int,
|
||||
_previous: Any = None,
|
||||
) -> Any:
|
||||
"""复用 canonical service,不复制其内部队列和后台任务所有权。"""
|
||||
return implementation
|
||||
|
||||
@staticmethod
|
||||
async def start(
|
||||
spec: CapabilitySpec,
|
||||
candidate: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""等待 service 在当前应用事件循环完成初始化。"""
|
||||
result = _lifecycle_method(spec, candidate, "initialize")()
|
||||
if not inspect.isawaitable(result):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.initialize() 必须返回 awaitable"
|
||||
)
|
||||
await result
|
||||
|
||||
@staticmethod
|
||||
async def stop(
|
||||
spec: CapabilitySpec,
|
||||
instance: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""等待 service 停止后台任务并释放其资源。"""
|
||||
result = _lifecycle_method(spec, instance, "close")()
|
||||
if not inspect.isawaitable(result):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 必须返回 awaitable"
|
||||
)
|
||||
converged = await result
|
||||
if converged is False:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 返回未收敛,保留 service owner"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def cleanup(
|
||||
spec: CapabilitySpec,
|
||||
candidate: Any,
|
||||
generation: int,
|
||||
_error: BaseException,
|
||||
) -> None:
|
||||
"""初始化失败或关闭竞态时按相同 close 合同释放部分资源。"""
|
||||
await AgentServiceAdapter.stop(spec, candidate, generation)
|
||||
|
||||
|
||||
def _validate_registry(registry: CapabilityRegistry) -> None:
|
||||
"""固定 entrypoint 物化轴与 service 资源轴的声明合同。"""
|
||||
for spec in registry.list_specs():
|
||||
if set(spec.metadata) != {"name"}:
|
||||
raise ValueError(f"{spec.source}: Agent Capability metadata 只能包含 name")
|
||||
if spec.kind == AGENT_ENTRYPOINT_KIND:
|
||||
if spec.activation is not ActivationPolicy.ON_FIRST_USE:
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent entrypoint 必须使用 on_first_use"
|
||||
)
|
||||
if spec.selector is not None or spec.watch:
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent entrypoint 不接受 selector 或 watch"
|
||||
)
|
||||
continue
|
||||
if spec.activation is not ActivationPolicy.WHEN_CONFIGURED:
|
||||
raise ValueError(f"{spec.source}: Agent Service 必须使用 when_configured")
|
||||
selector = spec.selector
|
||||
if selector is None or selector.kind != _SETTING_SELECTOR:
|
||||
raise ValueError(f"{spec.source}: Agent Service 必须声明 setting_truthy")
|
||||
selector_key = str(selector.config["key"])
|
||||
if spec.watch != (selector_key,):
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent Service watch 必须只包含 selector key"
|
||||
)
|
||||
|
||||
|
||||
def build_agent_capability_registry(
|
||||
roots: Iterable[Path | str] | None = None,
|
||||
) -> CapabilityRegistry:
|
||||
"""发现 data-only Agent manifests,不导入编排器、Provider 或工具实现。"""
|
||||
registry = CapabilityRegistry.discover(
|
||||
tuple(roots) if roots is not None else (_DEFAULT_CAPABILITY_ROOT,),
|
||||
kinds={AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND},
|
||||
selector_schemas=AGENT_SELECTOR_SCHEMAS,
|
||||
)
|
||||
_validate_registry(registry)
|
||||
return registry
|
||||
|
||||
|
||||
def should_run_agent_service(spec: CapabilitySpec) -> bool:
|
||||
"""依据 manifest selector 判断 service 是否应拥有运行实例。"""
|
||||
selector = spec.selector
|
||||
if (
|
||||
spec.kind != AGENT_SERVICE_KIND
|
||||
or selector is None
|
||||
or selector.kind != _SETTING_SELECTOR
|
||||
):
|
||||
raise ValueError(f"{spec.source}: 不是可协调的 Agent Service 声明")
|
||||
return bool(get_runtime_setting(selector.config["key"]))
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.manager"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.orchestrator:agent_manager"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Manager"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.moviepilot_type"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.orchestrator:MoviePilotAgent"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "MoviePilot Agent Type"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,16 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.service"
|
||||
kind = "agent_service"
|
||||
entrypoint = "app.agent.orchestrator:agent_manager"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Service"
|
||||
|
||||
[activation]
|
||||
policy = "when_configured"
|
||||
watch = ["AI_AGENT_ENABLE"]
|
||||
|
||||
[activation.selector]
|
||||
kind = "setting_truthy"
|
||||
key = "AI_AGENT_ENABLE"
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.tool_factory"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.tools.factory:MoviePilotToolFactory"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Tool Factory"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,35 +0,0 @@
|
||||
"""Agent 轻量公共合同,不触发模型、工具或编排运行时加载。"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.schemas.types import ReplyMode
|
||||
|
||||
|
||||
def build_display_message(
|
||||
role: str,
|
||||
content: str = "",
|
||||
attachments: Optional[list[dict]] = None,
|
||||
status: str = "done",
|
||||
) -> dict[str, Any]:
|
||||
"""构造前后端共享的 Agent 会话展示消息。"""
|
||||
normalized_content = content or ""
|
||||
return {
|
||||
"id": f"{role}-{uuid.uuid4().hex}",
|
||||
"role": role,
|
||||
"content": normalized_content,
|
||||
"createdAt": int(datetime.now().timestamp() * 1000),
|
||||
"status": status,
|
||||
"tools": [],
|
||||
"segments": (
|
||||
[{"type": "text", "content": normalized_content}]
|
||||
if normalized_content
|
||||
else []
|
||||
),
|
||||
"attachments": attachments or [],
|
||||
"choices": [],
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["ReplyMode", "build_display_message"]
|
||||
@@ -1,19 +0,0 @@
|
||||
---
|
||||
version: 3
|
||||
active_persona: default
|
||||
extra_context_files: []
|
||||
deprecated_phrases: []
|
||||
---
|
||||
# CURRENT_PERSONA
|
||||
|
||||
当前激活人格:`default`
|
||||
|
||||
运行时加载顺序固定如下:
|
||||
|
||||
1. 核心系统提示词(程序内置,不可运行时覆盖)
|
||||
2. `personas/<active_persona>/PERSONA.md`
|
||||
3. `extra_context_files`
|
||||
4. `memory/*.md`
|
||||
5. `activity/*.md`
|
||||
|
||||
`memory` 中的长期偏好可以细化回复方式,但不应覆盖系统核心身份、目标和安全边界。
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: aloof
|
||||
label: 高冷
|
||||
description: 冷静、克制、低温度,话少但不失礼。
|
||||
aliases:
|
||||
- 冷淡
|
||||
- 冷感
|
||||
- 冷艳
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: cool, distant, and composed.
|
||||
- Keep emotional temperature low and transitions short.
|
||||
- Be brief and efficient, but do not become rude or contemptuous.
|
||||
- Prefer understatement over enthusiasm.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Lead with the answer or the action result.
|
||||
- Keep explanations minimal unless the user explicitly asks for detail.
|
||||
- Avoid extra reassurance, hype, or emotional softening.
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: anime
|
||||
label: 二次元
|
||||
description: 带一点 ACG 语感和戏剧化表达,但仍然以任务完成和清晰沟通为主。
|
||||
aliases:
|
||||
- 动漫风
|
||||
- ACG
|
||||
- 宅系
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: lively, stylized, and lightly dramatic, with a small amount of anime-flavored wording.
|
||||
- Keep the actual task handling grounded and practical; the style should stay mostly in phrasing.
|
||||
- You may occasionally use short ACG-like interjections, but do not flood the reply with memes, kaomoji, or niche jargon.
|
||||
- Stay readable first. If the task is serious, reduce the stylistic flavor automatically.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Prefer short paragraphs or compact lists.
|
||||
- A light playful closing line is acceptable after the real result is already clear.
|
||||
- Do not let the style make operational instructions vague.
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: catgirl
|
||||
label: 猫娘
|
||||
description: 带一点猫系拟人风格,轻松可爱,但不过度角色扮演。
|
||||
aliases:
|
||||
- 猫猫
|
||||
- 喵系
|
||||
- 猫耳
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: playful, cat-like, and cute, with occasional feline wording.
|
||||
- You may occasionally use a light "喵" style suffix or cat metaphor, but only sparingly.
|
||||
- Do not turn the reply into full roleplay; task clarity remains the primary goal.
|
||||
- If the content is operational, keep the answer direct first and add only a thin layer of style.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Keep answers compact and readable.
|
||||
- Use only a very small amount of repeated verbal tic.
|
||||
- The result or action status should always appear before any playful flourish.
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: concise
|
||||
label: 极简
|
||||
description: 更短、更硬朗,优先结论和动作,不主动展开背景解释。
|
||||
aliases:
|
||||
- 简洁
|
||||
- 干脆
|
||||
- 极简人格
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: terse, decisive, and highly compressed.
|
||||
- Prefer the shortest complete answer that still moves the task forward.
|
||||
- Default to one sentence when possible. Only use lists when they materially improve readability.
|
||||
- Avoid extra context, caveats, or teaching unless the user explicitly asks for explanation.
|
||||
- Keep transitions minimal and skip conversational softening.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Lead with the conclusion or result.
|
||||
- For option lists, keep each item very short.
|
||||
- Do not repeat already-known context back to the user unless it is needed to disambiguate the action.
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: cute
|
||||
label: 可爱
|
||||
description: 语气更亲和、更柔软、更讨喜,但不做重度角色扮演。
|
||||
aliases:
|
||||
- 软萌
|
||||
- 甜系
|
||||
- 亲和
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: warm, cheerful, and gently cute.
|
||||
- Sound approachable and pleasant, but keep the answer concise and useful.
|
||||
- Avoid baby talk, excessive repetition, or exaggerated emotive punctuation.
|
||||
- If the user asks for directness, keep the cute flavor minimal.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Prefer friendly short paragraphs.
|
||||
- For lists, keep each item short and easy to read.
|
||||
- When something fails, explain it gently but clearly.
|
||||
@@ -1,24 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: default
|
||||
label: 默认
|
||||
description: 专业、克制、简洁,适合大多数日常媒体管理场景。
|
||||
aliases:
|
||||
- 专业
|
||||
- 默认人格
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: professional, concise, restrained.
|
||||
- Be direct. No unnecessary preamble, no repeating the user's words, no narrating internal reasoning.
|
||||
- Do not flatter the user, praise the question, or add emotional cushioning.
|
||||
- Do not use emojis, exclamation marks, cute language, or excessive apology.
|
||||
- Prefer short declarative sentences. Default to one or two short paragraphs; use lists only when they improve scanability.
|
||||
- Use Markdown for structured data. Use `inline code` for media titles and paths.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Keep confirmations short.
|
||||
- For search or comparison results, prefer a brief list over a long paragraph.
|
||||
- Skip filler phrases like "Let me help you", "Here are the results", or "I found...".
|
||||
- When an error occurs, briefly state the blocker and the next best action.
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: disdain
|
||||
label: 不屑
|
||||
description: 带一点嫌弃感和轻微毒舌,但必须保持可控和不越界。
|
||||
aliases:
|
||||
- 嫌弃
|
||||
- 毒舌
|
||||
- 鄙视链
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: dry, skeptical, and faintly dismissive.
|
||||
- Mild sarcasm is acceptable, but it must stay controlled and should never turn into direct insult or humiliation.
|
||||
- Prioritize sharp phrasing and low patience, while still giving the user the actual answer.
|
||||
- If the task is sensitive or the user is clearly frustrated, reduce the bite automatically.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Keep answers crisp and pointed.
|
||||
- Use short, cutting observations only when they improve the style without harming clarity.
|
||||
- Always include the concrete result, instruction, or blocker.
|
||||
@@ -1,22 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: guide
|
||||
label: 说明型
|
||||
description: 在复杂问题上更愿意解释原因和步骤,但仍保持克制,不会无节制展开。
|
||||
aliases:
|
||||
- 讲解
|
||||
- 解释型
|
||||
- 教学
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: clear, structured, and mildly explanatory.
|
||||
- When the task is simple, stay concise. When the task is complex or the user asks why/how, provide a short explanation with visible structure.
|
||||
- Keep explanations practical and tied to the current decision, not generic theory.
|
||||
- Remain restrained: do not become chatty, cute, or overly warm.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- For non-trivial tasks, prefer short sections or a compact numbered list.
|
||||
- When describing tradeoffs, keep them concrete and action-oriented.
|
||||
- End with the actual outcome or next step, not a generic summary.
|
||||
@@ -1,23 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
persona_id: moe
|
||||
label: 萌系
|
||||
description: 更轻小说感、更元气、更可爱,但仍然保持边界和专业度。
|
||||
aliases:
|
||||
- 萝莉风
|
||||
- 轻小说风
|
||||
- 元气少女
|
||||
- 萌萌
|
||||
---
|
||||
# PERSONA
|
||||
|
||||
- Tone: soft, upbeat, cute, and lightly playful.
|
||||
- Keep the personality in wording only; do not imitate a child, emphasize age, or use any sexualized framing.
|
||||
- Use cute particles or soft wording sparingly so the answer still feels useful instead of noisy.
|
||||
- When the task is urgent or technical, reduce the fluff and keep the result clear.
|
||||
|
||||
## RESPONSE_FORMAT
|
||||
|
||||
- Prefer short, bright sentences.
|
||||
- A small amount of cute phrasing is acceptable, but the final answer must still be easy to scan.
|
||||
- Do not bury the actual conclusion under roleplay language.
|
||||
@@ -1,20 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: download-diagnostician
|
||||
label: 下载诊断
|
||||
description: Download and transfer diagnosis subagent for downloaders, download tasks, transfer history, and library status.
|
||||
include_tags:
|
||||
- download
|
||||
- transfer
|
||||
- library
|
||||
- directory
|
||||
- file
|
||||
- media
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in downloaders, download tasks, transfer history, directory settings, and library ingestion state.
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: general-purpose
|
||||
label: 通用调查
|
||||
description: General read-only investigation subagent for cross-domain MoviePilot analysis and execution recommendations.
|
||||
include_tags:
|
||||
- media
|
||||
- resource
|
||||
- site
|
||||
- subscription
|
||||
- download
|
||||
- library
|
||||
- transfer
|
||||
- system
|
||||
- settings
|
||||
- plugin
|
||||
- workflow
|
||||
- scheduler
|
||||
- file
|
||||
- directory
|
||||
- web
|
||||
- command
|
||||
- filter_rule
|
||||
- persona
|
||||
- slash_command
|
||||
- recommendation
|
||||
- metadata
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in synthesizing media, site, subscription, download, and system status signals.
|
||||
@@ -1,19 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: media-researcher
|
||||
label: 媒体研究
|
||||
description: Media research subagent for title recognition, people, episodes, metadata, and library existence checks.
|
||||
include_tags:
|
||||
- media
|
||||
- library
|
||||
- recommendation
|
||||
- metadata
|
||||
- web
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in media identity resolution, metadata validation, person credits, and library status analysis.
|
||||
@@ -1,19 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: moviepilot-explorer
|
||||
label: 代码探索
|
||||
description: MoviePilot exploration subagent for source-code inspection, configuration structure analysis, logs, and code-level troubleshooting clues.
|
||||
include_tags:
|
||||
- system
|
||||
- settings
|
||||
- file
|
||||
- directory
|
||||
- command
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in MoviePilot source-code structure, local configuration files, directory layout, logs or read-only command output, and code-level root-cause troubleshooting. Prefer reading relevant code paths before judging behavior, and distinguish code/config evidence from runtime system state.
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: resource-searcher
|
||||
label: 资源搜索
|
||||
description: Site and resource search subagent for site checks, torrent search, and resource quality analysis.
|
||||
include_tags:
|
||||
- resource
|
||||
- site
|
||||
- web
|
||||
- media
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in site status, site user data, torrent search results, and resource quality judgment.
|
||||
@@ -1,18 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: subscription-analyst
|
||||
label: 订阅分析
|
||||
description: Subscription analysis subagent for subscriptions, history, filter rules, and custom identifiers.
|
||||
include_tags:
|
||||
- subscription
|
||||
- filter_rule
|
||||
- settings
|
||||
- media
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in current subscription state, subscription history, filter rules, and subscription optimization suggestions.
|
||||
@@ -1,25 +0,0 @@
|
||||
---
|
||||
version: 1
|
||||
subagent_id: system-diagnostician
|
||||
label: 系统诊断
|
||||
description: System diagnosis subagent for read-only inspection of settings, schedulers, workflows, plugins, directories, and command output.
|
||||
include_tags:
|
||||
- system
|
||||
- settings
|
||||
- plugin
|
||||
- workflow
|
||||
- scheduler
|
||||
- file
|
||||
- directory
|
||||
- web
|
||||
- command
|
||||
- persona
|
||||
- slash_command
|
||||
exclude_tags:
|
||||
- write
|
||||
- message
|
||||
- user_interaction
|
||||
---
|
||||
# SUBAGENT
|
||||
|
||||
You specialize in settings, plugins, scheduled tasks, workflows, directories, and read-only command diagnostics.
|
||||
@@ -1,52 +0,0 @@
|
||||
"""Agent 内部使用的 LLM 适配层,公开对象按需解析。"""
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
|
||||
_EXPORT_MODULES = {
|
||||
"LLMHelper": "app.agent.llm.helper",
|
||||
"LLMTestError": "app.agent.llm.helper",
|
||||
"LLMTestTimeout": "app.agent.llm.helper",
|
||||
"AgentCapabilityManager": "app.agent.llm.capability",
|
||||
"AgentCapabilityProvider": "app.agent.llm.capability",
|
||||
"AudioCapabilityProvider": "app.agent.llm.capability",
|
||||
"MiMoAudioProvider": "app.agent.llm.capability",
|
||||
"OpenAIChatAudioProvider": "app.agent.llm.capability",
|
||||
"OpenAIAudioProvider": "app.agent.llm.capability",
|
||||
"LLMProviderAuthError": "app.agent.llm.provider",
|
||||
"LLMProviderError": "app.agent.llm.provider",
|
||||
"LLMProviderManager": "app.agent.llm.provider",
|
||||
"render_auth_result_html": "app.agent.llm.provider",
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""首次访问公开对象时只加载其所属适配模块。"""
|
||||
module_name = _EXPORT_MODULES.get(name)
|
||||
if module_name is None:
|
||||
raise AttributeError(f"module 'app.agent.llm' has no attribute {name!r}")
|
||||
value = getattr(import_module(module_name), name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""让延迟公开对象继续支持交互式发现。"""
|
||||
return sorted(set(globals()) | set(_EXPORT_MODULES))
|
||||
|
||||
__all__ = [
|
||||
"LLMHelper",
|
||||
"AgentCapabilityManager",
|
||||
"AgentCapabilityProvider",
|
||||
"AudioCapabilityProvider",
|
||||
"LLMProviderAuthError",
|
||||
"LLMProviderError",
|
||||
"LLMProviderManager",
|
||||
"LLMTestError",
|
||||
"LLMTestTimeout",
|
||||
"MiMoAudioProvider",
|
||||
"OpenAIChatAudioProvider",
|
||||
"OpenAIAudioProvider",
|
||||
"render_auth_result_html",
|
||||
]
|
||||
@@ -1,829 +0,0 @@
|
||||
"""Agent 多模态能力 provider 与调度入口。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import mimetypes
|
||||
import shutil
|
||||
import subprocess
|
||||
from abc import ABC
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from app.application.notification import get_notification_configs
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.adapters.network.http import RequestUtils
|
||||
|
||||
|
||||
class AgentCapabilityProvider(ABC):
|
||||
"""Agent 能力 provider 基类,后续图片等能力可继续扩展到这里。"""
|
||||
|
||||
name: str
|
||||
|
||||
|
||||
class AudioCapabilityProvider(AgentCapabilityProvider):
|
||||
"""音频输入/输出能力 provider。"""
|
||||
|
||||
MAX_TRANSCRIBE_BYTES = 10 * 1024 * 1024
|
||||
|
||||
def is_available_for_audio_input(self) -> bool:
|
||||
"""是否可用于音频输入转写。"""
|
||||
return False
|
||||
|
||||
def is_available_for_audio_output(self) -> bool:
|
||||
"""是否可用于语音合成输出。"""
|
||||
return False
|
||||
|
||||
def transcribe_audio(self, content: bytes, filename: str = "input.ogg") -> Optional[str]:
|
||||
"""将音频字节转成文字。"""
|
||||
raise NotImplementedError
|
||||
|
||||
def synthesize_speech(self, text: str) -> Optional[Path]:
|
||||
"""将文字合成为可发送的音频文件。"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class OpenAIAudioProvider(AudioCapabilityProvider):
|
||||
"""OpenAI / OpenAI-compatible 音频 provider。"""
|
||||
|
||||
name = "openai"
|
||||
|
||||
@staticmethod
|
||||
def _build_client(api_key: str, base_url: Optional[str]):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(api_key=api_key, base_url=base_url, max_retries=3)
|
||||
|
||||
@staticmethod
|
||||
def _input_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_INPUT_API_KEY'), get_runtime_setting('AUDIO_INPUT_BASE_URL')
|
||||
|
||||
@staticmethod
|
||||
def _output_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_OUTPUT_API_KEY'), get_runtime_setting('AUDIO_OUTPUT_BASE_URL')
|
||||
|
||||
def is_available_for_audio_input(self) -> bool:
|
||||
api_key, _ = self._input_credentials()
|
||||
return bool(api_key)
|
||||
|
||||
def is_available_for_audio_output(self) -> bool:
|
||||
api_key, _ = self._output_credentials()
|
||||
return bool(api_key)
|
||||
|
||||
def transcribe_audio(self, content: bytes, filename: str = "input.ogg") -> Optional[str]:
|
||||
if not content:
|
||||
return None
|
||||
if len(content) > self.MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError("语音文件超过 10MB,无法识别")
|
||||
|
||||
try:
|
||||
api_key, base_url = self._input_credentials()
|
||||
if not api_key:
|
||||
raise ValueError("音频输入 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
audio_file = BytesIO(content)
|
||||
audio_file.name = filename
|
||||
response = client.audio.transcriptions.create(
|
||||
model=get_runtime_setting('AUDIO_INPUT_MODEL'),
|
||||
file=audio_file,
|
||||
language=get_runtime_setting('AUDIO_INPUT_LANGUAGE') or "zh",
|
||||
response_format="verbose_json",
|
||||
)
|
||||
text = getattr(response, "text", None)
|
||||
return text.strip() if text else None
|
||||
except Exception as err:
|
||||
logger.error(f"音频输入转写失败: provider={self.name}, error={err}")
|
||||
return None
|
||||
|
||||
def synthesize_speech(self, text: str) -> Optional[Path]:
|
||||
if not text:
|
||||
return None
|
||||
|
||||
try:
|
||||
api_key, base_url = self._output_credentials()
|
||||
if not api_key:
|
||||
raise ValueError("音频输出 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = voice_dir / f"{uuid4().hex}.opus"
|
||||
response = client.audio.speech.create(
|
||||
model=get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
voice=get_runtime_setting('AUDIO_OUTPUT_VOICE'),
|
||||
input=text,
|
||||
response_format="opus",
|
||||
)
|
||||
response.write_to_file(output_path)
|
||||
return output_path
|
||||
except Exception as err:
|
||||
logger.error(f"音频输出合成失败: provider={self.name}, error={err}")
|
||||
return None
|
||||
|
||||
|
||||
class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
"""通过 OpenAI Chat Completions 兼容接口传入/返回音频的 provider。"""
|
||||
|
||||
name = "openai_chat_audio"
|
||||
DISPLAY_NAME = "OpenAI Chat Audio"
|
||||
DEFAULT_BASE_URL: Optional[str] = None
|
||||
DEFAULT_STT_MODEL: Optional[str] = None
|
||||
DEFAULT_TTS_MODEL: Optional[str] = None
|
||||
DEFAULT_VOICE = "alloy"
|
||||
AUDIO_RESPONSE_FORMAT = "wav"
|
||||
AUDIO_INPUT_DATA_URL = False
|
||||
INCLUDE_AUDIO_MODALITIES = True
|
||||
TTS_MESSAGE_ROLE = "user"
|
||||
SUPPORTED_STT_MODELS: Optional[frozenset[str]] = None
|
||||
SUPPORTED_TTS_MODELS: Optional[frozenset[str]] = None
|
||||
UNSUPPORTED_TTS_MODELS = frozenset()
|
||||
SUPPORTED_AUDIO_MIME_TYPES = {
|
||||
".flac": "audio/flac",
|
||||
".m4a": "audio/mp4",
|
||||
".mp3": "audio/mpeg",
|
||||
".ogg": "audio/ogg",
|
||||
".opus": "audio/ogg",
|
||||
".wav": "audio/wav",
|
||||
}
|
||||
TRANSCODED_STT_SUFFIX = ".wav"
|
||||
TRANSCODED_STT_SAMPLE_RATE = "16000"
|
||||
|
||||
def _build_client(self, api_key: str, base_url: Optional[str]):
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=base_url or self.DEFAULT_BASE_URL,
|
||||
max_retries=3,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _input_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_INPUT_API_KEY'), get_runtime_setting('AUDIO_INPUT_BASE_URL')
|
||||
|
||||
@staticmethod
|
||||
def _output_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_OUTPUT_API_KEY'), get_runtime_setting('AUDIO_OUTPUT_BASE_URL')
|
||||
|
||||
def _normalize_stt_model(self) -> str:
|
||||
return self._normalize_model(
|
||||
model=get_runtime_setting('AUDIO_INPUT_MODEL'),
|
||||
supported_models=self.SUPPORTED_STT_MODELS,
|
||||
default_model=self.DEFAULT_STT_MODEL,
|
||||
)
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
return self._normalize_model(
|
||||
model=get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
supported_models=self.SUPPORTED_TTS_MODELS,
|
||||
default_model=self.DEFAULT_TTS_MODEL,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalize_model(
|
||||
model: Optional[str],
|
||||
supported_models: Optional[frozenset[str]],
|
||||
default_model: Optional[str],
|
||||
) -> str:
|
||||
model = (model or "").strip()
|
||||
if not model:
|
||||
return default_model or ""
|
||||
if supported_models is None:
|
||||
return model
|
||||
model_key = model.lower()
|
||||
if model_key in supported_models:
|
||||
return model_key
|
||||
return default_model or model
|
||||
|
||||
def _is_supported_tts_model(self) -> bool:
|
||||
model = self._normalize_tts_model()
|
||||
if not model:
|
||||
return False
|
||||
model_key = model.lower()
|
||||
if model_key in self.UNSUPPORTED_TTS_MODELS:
|
||||
return False
|
||||
return self.SUPPORTED_TTS_MODELS is None or model_key in self.SUPPORTED_TTS_MODELS
|
||||
|
||||
@classmethod
|
||||
def _guess_audio_mime_type(cls, filename: str) -> str:
|
||||
suffix = Path(filename or "").suffix.lower()
|
||||
if suffix in cls.SUPPORTED_AUDIO_MIME_TYPES:
|
||||
return cls.SUPPORTED_AUDIO_MIME_TYPES[suffix]
|
||||
mime_type, _ = mimetypes.guess_type(filename or "")
|
||||
return mime_type or "audio/ogg"
|
||||
|
||||
@staticmethod
|
||||
def _guess_audio_format(filename: str) -> str:
|
||||
suffix = Path(filename or "").suffix.lower().lstrip(".")
|
||||
if suffix == "opus":
|
||||
return "ogg"
|
||||
return suffix or "ogg"
|
||||
|
||||
def _build_audio_input_payload(self, content: bytes, filename: str) -> dict:
|
||||
"""按不同 Chat Audio 兼容形态构造 input_audio 内容。"""
|
||||
audio_data = base64.b64encode(content).decode("utf-8")
|
||||
if self.AUDIO_INPUT_DATA_URL:
|
||||
mime_type = self._guess_audio_mime_type(filename)
|
||||
return {"data": f"data:{mime_type};base64,{audio_data}"}
|
||||
return {
|
||||
"data": audio_data,
|
||||
"format": self._guess_audio_format(filename),
|
||||
}
|
||||
|
||||
def _normalize_audio_for_transcription(
|
||||
self, content: bytes, filename: str
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
将转写输入归一化为 Chat Audio provider 明确支持的格式。
|
||||
|
||||
:param content: 原始音频字节
|
||||
:param filename: 原始音频文件名
|
||||
:return: 成功时返回可提交的音频字节和文件名,失败时返回 None
|
||||
"""
|
||||
suffix = Path(filename or "").suffix.lower()
|
||||
if suffix in self.SUPPORTED_AUDIO_MIME_TYPES:
|
||||
return content, filename
|
||||
return self._convert_audio_for_transcription(content=content, filename=filename)
|
||||
|
||||
def _convert_audio_for_transcription(
|
||||
self, content: bytes, filename: str
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
将 AMR 等第三方 STT 不支持的输入转为 WAV。
|
||||
|
||||
:param content: 原始音频字节
|
||||
:param filename: 原始音频文件名
|
||||
:return: 成功时返回 WAV 字节和文件名,失败时返回 None
|
||||
"""
|
||||
if not shutil.which("ffmpeg"):
|
||||
logger.warning(
|
||||
"%s STT 不支持当前音频格式且 ffmpeg 不可用,无法转码: filename=%s",
|
||||
self.DISPLAY_NAME,
|
||||
filename,
|
||||
)
|
||||
return None
|
||||
|
||||
suffix = Path(filename or "").suffix.lower() or ".audio"
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
input_path = voice_dir / f"{uuid4().hex}{suffix}"
|
||||
output_path = input_path.with_suffix(self.TRANSCODED_STT_SUFFIX)
|
||||
try:
|
||||
input_path.write_bytes(content)
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
str(input_path),
|
||||
"-ar",
|
||||
self.TRANSCODED_STT_SAMPLE_RATE,
|
||||
"-ac",
|
||||
"1",
|
||||
"-f",
|
||||
"wav",
|
||||
str(output_path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
if result.returncode != 0 or not output_path.exists():
|
||||
logger.warning(
|
||||
"%s STT 音频转 WAV 失败: returncode=%s, stderr=%s",
|
||||
self.DISPLAY_NAME,
|
||||
result.returncode,
|
||||
(result.stderr or "").strip()[:500],
|
||||
)
|
||||
return None
|
||||
return output_path.read_bytes(), f"{input_path.stem}{self.TRANSCODED_STT_SUFFIX}"
|
||||
finally:
|
||||
for temp_path in (input_path, output_path):
|
||||
try:
|
||||
temp_path.unlink(missing_ok=True)
|
||||
except OSError as err:
|
||||
logger.debug(f"清理 STT 临时音频失败: path={temp_path}, error={err}")
|
||||
|
||||
@staticmethod
|
||||
def _extract_message_text(message) -> Optional[str]:
|
||||
"""兼容音频理解响应可能放在 content 或 reasoning_content 的情况。"""
|
||||
content = getattr(message, "content", None)
|
||||
if isinstance(content, str) and content.strip():
|
||||
return content.strip()
|
||||
|
||||
reasoning_content = getattr(message, "reasoning_content", None)
|
||||
if isinstance(reasoning_content, str) and reasoning_content.strip():
|
||||
return reasoning_content.strip()
|
||||
|
||||
extra = getattr(message, "model_extra", None)
|
||||
if isinstance(extra, dict):
|
||||
for key in ("content", "reasoning_content"):
|
||||
value = extra.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip()
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _extract_audio_data(message) -> Optional[str]:
|
||||
audio = getattr(message, "audio", None)
|
||||
if isinstance(audio, dict):
|
||||
return audio.get("data")
|
||||
if audio is not None:
|
||||
return getattr(audio, "data", None)
|
||||
|
||||
extra = getattr(message, "model_extra", None)
|
||||
if isinstance(extra, dict) and isinstance(extra.get("audio"), dict):
|
||||
return extra["audio"].get("data")
|
||||
return None
|
||||
|
||||
def _convert_wav_to_opus(self, wav_path: Path) -> Optional[Path]:
|
||||
"""将 Chat Audio 返回的 WAV 转成 OGG/Opus,便于各通知渠道发送语音。"""
|
||||
if not shutil.which("ffmpeg"):
|
||||
return None
|
||||
|
||||
output_path = wav_path.with_suffix(".opus")
|
||||
cmd = [
|
||||
"ffmpeg",
|
||||
"-y",
|
||||
"-i",
|
||||
str(wav_path),
|
||||
"-ar",
|
||||
"48000",
|
||||
"-ac",
|
||||
"1",
|
||||
"-c:a",
|
||||
"libopus",
|
||||
str(output_path),
|
||||
]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
if result.returncode != 0 or not output_path.exists():
|
||||
logger.warning(
|
||||
"%s TTS 音频转 Opus 失败,将使用 WAV 原文件: returncode=%s, stderr=%s",
|
||||
self.DISPLAY_NAME,
|
||||
result.returncode,
|
||||
(result.stderr or "").strip()[:500],
|
||||
)
|
||||
return None
|
||||
return output_path
|
||||
|
||||
def is_available_for_audio_input(self) -> bool:
|
||||
api_key, _ = self._input_credentials()
|
||||
return bool(api_key)
|
||||
|
||||
def is_available_for_audio_output(self) -> bool:
|
||||
api_key, _ = self._output_credentials()
|
||||
return bool(api_key) and self._is_supported_tts_model()
|
||||
|
||||
def transcribe_audio(self, content: bytes, filename: str = "input.ogg") -> Optional[str]:
|
||||
if not content:
|
||||
return None
|
||||
if len(content) > self.MAX_TRANSCRIBE_BYTES:
|
||||
raise ValueError("语音文件超过 10MB,无法识别")
|
||||
|
||||
try:
|
||||
api_key, base_url = self._input_credentials()
|
||||
if not api_key:
|
||||
raise ValueError("音频输入 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
normalized_audio = self._normalize_audio_for_transcription(
|
||||
content=content, filename=filename
|
||||
)
|
||||
if not normalized_audio:
|
||||
return None
|
||||
content, filename = normalized_audio
|
||||
language = (get_runtime_setting('AUDIO_INPUT_LANGUAGE') or "").strip()
|
||||
prompt = "请将这段音频完整转写为文字,只输出转写结果,不要添加解释。"
|
||||
if language:
|
||||
prompt += f"音频主要语言是 {language}。"
|
||||
|
||||
completion = client.chat.completions.create(
|
||||
model=self._normalize_stt_model(),
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": self._build_audio_input_payload(
|
||||
content=content, filename=filename
|
||||
),
|
||||
},
|
||||
{"type": "text", "text": prompt},
|
||||
],
|
||||
}
|
||||
],
|
||||
max_completion_tokens=2048,
|
||||
)
|
||||
return self._extract_message_text(completion.choices[0].message)
|
||||
except Exception as err:
|
||||
logger.error(f"音频输入转写失败: provider={self.name}, error={err}")
|
||||
return None
|
||||
|
||||
def synthesize_speech(self, text: str) -> Optional[Path]:
|
||||
if not text:
|
||||
return None
|
||||
if not self._is_supported_tts_model():
|
||||
logger.error(
|
||||
"%s TTS 当前不支持该模型或模型未配置: %s",
|
||||
self.DISPLAY_NAME,
|
||||
get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
)
|
||||
return None
|
||||
|
||||
try:
|
||||
api_key, base_url = self._output_credentials()
|
||||
if not api_key:
|
||||
raise ValueError("音频输出 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
wav_path = voice_dir / f"{uuid4().hex}.wav"
|
||||
request = {
|
||||
"model": self._normalize_tts_model(),
|
||||
"messages": [
|
||||
{
|
||||
"role": self.TTS_MESSAGE_ROLE,
|
||||
"content": text,
|
||||
}
|
||||
],
|
||||
"audio": {
|
||||
"format": self.AUDIO_RESPONSE_FORMAT,
|
||||
"voice": get_runtime_setting('AUDIO_OUTPUT_VOICE') or self.DEFAULT_VOICE,
|
||||
},
|
||||
}
|
||||
if self.INCLUDE_AUDIO_MODALITIES:
|
||||
request["modalities"] = ["text", "audio"]
|
||||
completion = client.chat.completions.create(**request)
|
||||
audio_data = self._extract_audio_data(completion.choices[0].message)
|
||||
if not audio_data:
|
||||
raise ValueError(f"{self.DISPLAY_NAME} TTS 响应中没有音频数据")
|
||||
|
||||
wav_path.write_bytes(base64.b64decode(audio_data))
|
||||
return self._convert_wav_to_opus(wav_path) or wav_path
|
||||
except Exception as err:
|
||||
logger.error(f"音频输出合成失败: provider={self.name}, error={err}")
|
||||
return None
|
||||
|
||||
|
||||
class MiMoAudioProvider(OpenAIChatAudioProvider):
|
||||
"""Xiaomi MiMo Chat Audio 预设,仅接入普通 STT/TTS 能力。"""
|
||||
|
||||
name = "mimo"
|
||||
DISPLAY_NAME = "Xiaomi MiMo"
|
||||
DEFAULT_BASE_URL = "https://api.xiaomimimo.com/v1"
|
||||
DEFAULT_STT_MODEL = "mimo-v2.5"
|
||||
DEFAULT_TTS_MODEL = "mimo-v2.5-tts"
|
||||
DEFAULT_VOICE = "mimo_default"
|
||||
AUDIO_INPUT_DATA_URL = True
|
||||
INCLUDE_AUDIO_MODALITIES = False
|
||||
TTS_MESSAGE_ROLE = "assistant"
|
||||
SUPPORTED_STT_MODELS = frozenset({"mimo-v2.5", "mimo-v2-omni"})
|
||||
SUPPORTED_TTS_MODELS = frozenset({DEFAULT_TTS_MODEL})
|
||||
UNSUPPORTED_TTS_MODELS = frozenset(
|
||||
{
|
||||
"mimo-v2.5-tts-voiceclone",
|
||||
"mimo-v2.5-tts-voicedesign",
|
||||
}
|
||||
)
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
model = (get_runtime_setting('AUDIO_OUTPUT_MODEL') or "").strip().lower()
|
||||
if not model or not model.startswith("mimo-"):
|
||||
return self.DEFAULT_TTS_MODEL
|
||||
return model
|
||||
|
||||
|
||||
class MiniMaxAudioProvider(OpenAIChatAudioProvider):
|
||||
"""MiniMax 音频 provider,语音合成使用官方 T2A HTTP 接口。"""
|
||||
|
||||
name = "minimax"
|
||||
DISPLAY_NAME = "MiniMax"
|
||||
DEFAULT_BASE_URL = "https://api.minimaxi.com/v1"
|
||||
DEFAULT_STT_MODEL = "MiniMax-M2.7"
|
||||
DEFAULT_TTS_MODEL = "speech-2.8-turbo"
|
||||
DEFAULT_VOICE = "Chinese (Mandarin)_Lyrical_Voice"
|
||||
AUDIO_INPUT_DATA_URL = True
|
||||
SUPPORTED_TTS_MODELS = frozenset(
|
||||
{
|
||||
"speech-2.8-hd",
|
||||
"speech-2.8-turbo",
|
||||
"speech-2.6-hd",
|
||||
"speech-2.6-turbo",
|
||||
"speech-02-hd",
|
||||
"speech-02-turbo",
|
||||
"speech-01-hd",
|
||||
"speech-01-turbo",
|
||||
}
|
||||
)
|
||||
|
||||
def _build_client(self, api_key: str, base_url: Optional[str]):
|
||||
"""构建 MiniMax OpenAI 兼容客户端,兼容用户误填 Anthropic 端点的情况。"""
|
||||
from openai import OpenAI
|
||||
|
||||
return OpenAI(
|
||||
api_key=api_key,
|
||||
base_url=self._normalize_api_base_url(base_url),
|
||||
max_retries=3,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _normalize_api_base_url(cls, base_url: Optional[str]) -> str:
|
||||
"""归一化 MiniMax API 基础 URL,确保后续可以拼接 OpenAI/T2A 路径。"""
|
||||
normalized = (base_url or cls.DEFAULT_BASE_URL).strip().rstrip("/")
|
||||
if normalized.endswith("/t2a_v2"):
|
||||
normalized = normalized[: -len("/t2a_v2")]
|
||||
for suffix in ("/anthropic/v1", "/openai/v1"):
|
||||
if normalized.endswith(suffix):
|
||||
return normalized[: -len(suffix)] + "/v1"
|
||||
if not normalized.endswith("/v1"):
|
||||
normalized = f"{normalized}/v1"
|
||||
return normalized
|
||||
|
||||
@classmethod
|
||||
def _build_t2a_url(cls, base_url: Optional[str]) -> str:
|
||||
"""生成 MiniMax 同步 T2A 接口地址。"""
|
||||
return f"{cls._normalize_api_base_url(base_url)}/t2a_v2"
|
||||
|
||||
def _normalize_stt_model(self) -> str:
|
||||
"""将非 MiniMax 的默认转写模型名兜底为 MiniMax 对话模型。"""
|
||||
model = (get_runtime_setting('AUDIO_INPUT_MODEL') or "").strip()
|
||||
if not model or model.lower().startswith(("gpt-", "mimo-")):
|
||||
return self.DEFAULT_STT_MODEL
|
||||
return model
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
"""将非 MiniMax 语音模型兜底为官方 T2A 模型。"""
|
||||
model = (get_runtime_setting('AUDIO_OUTPUT_MODEL') or "").strip().lower()
|
||||
if model in self.SUPPORTED_TTS_MODELS:
|
||||
return model
|
||||
return self.DEFAULT_TTS_MODEL
|
||||
|
||||
def _normalize_voice_id(self) -> str:
|
||||
"""将其他 provider 的默认音色兜底为 MiniMax 中文系统音色。"""
|
||||
voice_id = (get_runtime_setting('AUDIO_OUTPUT_VOICE') or "").strip()
|
||||
if not voice_id or voice_id in {"alloy", "mimo_default"}:
|
||||
return self.DEFAULT_VOICE
|
||||
return voice_id
|
||||
|
||||
@staticmethod
|
||||
def _decode_audio_payload(audio_data: str) -> bytes:
|
||||
"""解析 MiniMax T2A 返回的音频数据,优先按官方 hex 格式处理。"""
|
||||
normalized = "".join((audio_data or "").split())
|
||||
try:
|
||||
return bytes.fromhex(normalized)
|
||||
except ValueError:
|
||||
return base64.b64decode(audio_data)
|
||||
|
||||
@staticmethod
|
||||
def _extract_minimax_error(data: dict[str, Any]) -> Optional[str]:
|
||||
"""提取 MiniMax base_resp 错误信息,成功响应返回 None。"""
|
||||
base_resp = data.get("base_resp") or {}
|
||||
status_code = base_resp.get("status_code")
|
||||
if status_code in (None, 0, "0"):
|
||||
return None
|
||||
status_msg = base_resp.get("status_msg") or "unknown error"
|
||||
return f"{status_code}: {status_msg}"
|
||||
|
||||
def synthesize_speech(self, text: str) -> Optional[Path]:
|
||||
"""调用 MiniMax T2A HTTP 接口合成语音文件。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
try:
|
||||
api_key, base_url = self._output_credentials()
|
||||
if not api_key:
|
||||
raise ValueError("音频输出 provider 未配置 API Key")
|
||||
response = RequestUtils(
|
||||
headers={
|
||||
"Authorization": f"Bearer {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
proxies=get_runtime_setting('PROXY') or {},
|
||||
timeout=60,
|
||||
).post_res(
|
||||
url=self._build_t2a_url(base_url),
|
||||
json={
|
||||
"model": self._normalize_tts_model(),
|
||||
"text": text,
|
||||
"stream": False,
|
||||
"language_boost": "auto",
|
||||
"output_format": "hex",
|
||||
"voice_setting": {
|
||||
"voice_id": self._normalize_voice_id(),
|
||||
"speed": 1,
|
||||
"vol": 1,
|
||||
"pitch": 0,
|
||||
},
|
||||
"audio_setting": {
|
||||
"sample_rate": 32000,
|
||||
"bitrate": 128000,
|
||||
"format": "opus",
|
||||
"channel": 1,
|
||||
},
|
||||
},
|
||||
)
|
||||
if not response:
|
||||
raise ValueError("MiniMax T2A 请求无响应")
|
||||
if response.status_code >= 400:
|
||||
raise ValueError(f"MiniMax T2A HTTP {response.status_code}")
|
||||
|
||||
result = response.json()
|
||||
minimax_error = self._extract_minimax_error(result)
|
||||
if minimax_error:
|
||||
raise ValueError(f"MiniMax T2A 返回错误: {minimax_error}")
|
||||
|
||||
audio_data = ((result.get("data") or {}).get("audio") or "").strip()
|
||||
if not audio_data:
|
||||
raise ValueError("MiniMax T2A 响应中没有音频数据")
|
||||
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = voice_dir / f"{uuid4().hex}.opus"
|
||||
output_path.write_bytes(self._decode_audio_payload(audio_data))
|
||||
return output_path
|
||||
except Exception as err:
|
||||
logger.error(f"音频输出合成失败: provider={self.name}, error={err}")
|
||||
return None
|
||||
|
||||
|
||||
class AgentCapabilityManager:
|
||||
"""Agent 能力统一入口。"""
|
||||
|
||||
REPLY_MODE_NATIVE = "native_voice"
|
||||
REPLY_MODE_TEXT = "text"
|
||||
_audio_providers: Dict[str, AudioCapabilityProvider] = {
|
||||
OpenAIAudioProvider.name: OpenAIAudioProvider(),
|
||||
OpenAIChatAudioProvider.name: OpenAIChatAudioProvider(),
|
||||
MiMoAudioProvider.name: MiMoAudioProvider(),
|
||||
MiniMaxAudioProvider.name: MiniMaxAudioProvider(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def register_audio_provider(cls, provider: AudioCapabilityProvider) -> None:
|
||||
"""注册新的音频 provider。"""
|
||||
cls._audio_providers[provider.name.lower()] = provider
|
||||
|
||||
@classmethod
|
||||
def get_registered_audio_providers(cls) -> list[str]:
|
||||
"""返回已注册的音频 provider 名称。"""
|
||||
return sorted(cls._audio_providers.keys())
|
||||
|
||||
@staticmethod
|
||||
def _normalize_provider_name(provider: Optional[str]) -> str:
|
||||
return (provider or "openai").strip().lower()
|
||||
|
||||
@staticmethod
|
||||
def _get_provider_log_name(provider: AudioCapabilityProvider) -> str:
|
||||
provider_name = getattr(provider, "name", None)
|
||||
return provider_name if isinstance(provider_name, str) else provider.__class__.__name__
|
||||
|
||||
@classmethod
|
||||
def get_audio_provider(cls, mode: str) -> Optional[AudioCapabilityProvider]:
|
||||
provider_name = cls._normalize_provider_name(
|
||||
get_runtime_setting('AUDIO_INPUT_PROVIDER')
|
||||
if (mode or "").lower() == "input"
|
||||
else get_runtime_setting('AUDIO_OUTPUT_PROVIDER')
|
||||
)
|
||||
provider = cls._audio_providers.get(provider_name)
|
||||
if provider:
|
||||
return provider
|
||||
logger.warning("未注册音频 provider: mode=%s, provider=%s", mode, provider_name)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def supports_image_input() -> bool:
|
||||
"""当前 Agent 是否启用图片输入能力。"""
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
|
||||
return LLMHelper.supports_image_input()
|
||||
|
||||
@staticmethod
|
||||
def supports_audio_input() -> bool:
|
||||
"""当前 Agent 是否启用音频输入能力。"""
|
||||
return bool(get_runtime_setting('LLM_SUPPORT_AUDIO_INPUT'))
|
||||
|
||||
@staticmethod
|
||||
def supports_audio_output() -> bool:
|
||||
"""当前 Agent 是否启用音频输出能力。"""
|
||||
return bool(get_runtime_setting('LLM_SUPPORT_AUDIO_OUTPUT'))
|
||||
|
||||
@classmethod
|
||||
def is_audio_input_available(cls) -> bool:
|
||||
if not cls.supports_audio_input():
|
||||
return False
|
||||
provider = cls.get_audio_provider("input")
|
||||
return bool(provider and provider.is_available_for_audio_input())
|
||||
|
||||
@classmethod
|
||||
def is_audio_output_available(cls) -> bool:
|
||||
if not cls.supports_audio_output():
|
||||
return False
|
||||
provider = cls.get_audio_provider("output")
|
||||
return bool(provider and provider.is_available_for_audio_output())
|
||||
|
||||
@classmethod
|
||||
def transcribe_audio(cls, content: bytes, filename: str = "input.ogg") -> Optional[str]:
|
||||
"""将语音文件内容转写为文字,并记录能力调用日志。"""
|
||||
provider = cls.get_audio_provider("input")
|
||||
if not provider or not cls.is_audio_input_available():
|
||||
logger.info("语音转文字跳过:音频输入能力未启用或 provider 不可用")
|
||||
return None
|
||||
provider_name = cls._get_provider_log_name(provider)
|
||||
logger.info(
|
||||
f"语音转文字开始:provider={provider_name}, filename={filename}, "
|
||||
f"bytes={len(content) if content else 0}"
|
||||
)
|
||||
transcript = provider.transcribe_audio(content=content, filename=filename)
|
||||
if transcript:
|
||||
logger.info(
|
||||
f"语音转文字完成:provider={provider_name}, filename={filename}, "
|
||||
f"text_len={len(transcript)}"
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
f"语音转文字无结果:provider={provider_name}, filename={filename}"
|
||||
)
|
||||
return transcript
|
||||
|
||||
@classmethod
|
||||
def synthesize_speech(cls, text: str) -> Optional[Path]:
|
||||
"""将文字合成为语音文件,并记录能力调用日志。"""
|
||||
provider = cls.get_audio_provider("output")
|
||||
if not provider or not cls.is_audio_output_available():
|
||||
logger.info("文字转语音跳过:音频输出能力未启用或 provider 不可用")
|
||||
return None
|
||||
provider_name = cls._get_provider_log_name(provider)
|
||||
logger.info(
|
||||
f"文字转语音开始:provider={provider_name}, text_len={len(text) if text else 0}"
|
||||
)
|
||||
output_path = provider.synthesize_speech(text=text)
|
||||
if output_path:
|
||||
logger.info(f"文字转语音完成:provider={provider_name}, path={output_path}")
|
||||
else:
|
||||
logger.info(f"文字转语音无结果:provider={provider_name}")
|
||||
return output_path
|
||||
|
||||
@classmethod
|
||||
def resolve_reply_mode(cls, channel: Optional[str], source: Optional[str]) -> str:
|
||||
"""仅在支持原生语音回复的渠道上发送音频,其余渠道回退文字。"""
|
||||
if cls.supports_native_voice_reply(channel=channel, source=source):
|
||||
return cls.REPLY_MODE_NATIVE
|
||||
return cls.REPLY_MODE_TEXT
|
||||
|
||||
@classmethod
|
||||
def _parse_message_channel(cls, channel: Optional[Any]):
|
||||
"""将渠道入参归一化为消息渠道枚举。"""
|
||||
if not channel:
|
||||
return None
|
||||
|
||||
from app.schemas.types import NotificationChannel
|
||||
|
||||
if isinstance(channel, NotificationChannel):
|
||||
return channel
|
||||
|
||||
channel_text = str(channel).strip()
|
||||
if not channel_text:
|
||||
return None
|
||||
lowered_channel = channel_text.lower()
|
||||
for channel_item in NotificationChannel:
|
||||
aliases = {
|
||||
channel_item.value.lower(),
|
||||
channel_item.name.lower(),
|
||||
f"{NotificationChannel.__name__}.{channel_item.name}".lower(),
|
||||
}
|
||||
if lowered_channel in aliases:
|
||||
return channel_item
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_wechat_app_mode(source: Optional[str]) -> bool:
|
||||
"""判断企业微信来源是否为自建应用模式。"""
|
||||
if not source:
|
||||
return False
|
||||
|
||||
for config in get_notification_configs(include_disabled=True):
|
||||
if config.name != source:
|
||||
continue
|
||||
return (config.config or {}).get("WECHAT_MODE", "app") != "bot"
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def supports_native_voice_reply(
|
||||
cls, channel: Optional[str], source: Optional[str]
|
||||
) -> bool:
|
||||
"""判断当前渠道是否支持原生语音消息发送。"""
|
||||
from app.schemas.notification import ChannelCapability, ChannelCapabilityManager
|
||||
from app.schemas.types import NotificationChannel
|
||||
|
||||
channel_enum = cls._parse_message_channel(channel)
|
||||
if not channel_enum:
|
||||
return False
|
||||
|
||||
if not ChannelCapabilityManager.supports_capability(
|
||||
channel_enum, ChannelCapability.AUDIO_OUTPUT
|
||||
):
|
||||
return False
|
||||
|
||||
if channel_enum == NotificationChannel.Wechat:
|
||||
return cls._is_wechat_app_mode(source)
|
||||
return True
|
||||
@@ -1,69 +0,0 @@
|
||||
"""LLM helper 与 provider 实现之间的运行时端口。"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
class LLMProviderRuntimePort(Protocol):
|
||||
"""声明 LLM helper 与管理 API 共用的 provider 运行时能力。"""
|
||||
|
||||
def resolve_cached_model_metadata(self, **kwargs: Any) -> dict[str, Any] | None:
|
||||
"""从本地目录缓存解析模型元数据。"""
|
||||
...
|
||||
|
||||
async def resolve_runtime(self, **kwargs: Any) -> dict[str, Any]:
|
||||
"""解析创建模型客户端所需的统一运行时参数。"""
|
||||
...
|
||||
|
||||
def create_bedrock_client(self, *args: Any, **kwargs: Any) -> Any:
|
||||
"""创建带统一认证和网络配置的 Bedrock 客户端。"""
|
||||
...
|
||||
|
||||
async def list_models(self, **kwargs: Any) -> list[dict[str, Any]]:
|
||||
"""返回 provider 可用的模型目录。"""
|
||||
...
|
||||
|
||||
def resolve_model_list_base_url(self, **kwargs: Any) -> str | None:
|
||||
"""解析兼容接口用于查询模型列表的基础地址。"""
|
||||
...
|
||||
|
||||
async def provider_manage(
|
||||
self,
|
||||
provider: str,
|
||||
action: str,
|
||||
**params: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""执行与具体提供商无关的统一管理动作。"""
|
||||
...
|
||||
|
||||
async def handle_chatgpt_callback(
|
||||
self,
|
||||
provider_id: str,
|
||||
code: str | None,
|
||||
state: str | None,
|
||||
error: str | None,
|
||||
error_description: str | None,
|
||||
) -> tuple[bool, str]:
|
||||
"""完成 ChatGPT OAuth 回调并返回公开结果。"""
|
||||
...
|
||||
|
||||
|
||||
LLMProviderRuntimeFactory = Callable[[], LLMProviderRuntimePort]
|
||||
_provider_runtime_factory: LLMProviderRuntimeFactory | None = None
|
||||
|
||||
|
||||
def register_llm_provider_runtime(
|
||||
factory: LLMProviderRuntimeFactory | None,
|
||||
) -> LLMProviderRuntimeFactory | None:
|
||||
"""注册 provider 运行时工厂,并返回先前工厂供隔离测试恢复。"""
|
||||
global _provider_runtime_factory
|
||||
previous = _provider_runtime_factory
|
||||
_provider_runtime_factory = factory
|
||||
return previous
|
||||
|
||||
|
||||
def resolve_llm_provider_runtime() -> LLMProviderRuntimePort:
|
||||
"""解析已组装的 provider 运行时,未注册时给出明确边界错误。"""
|
||||
if _provider_runtime_factory is None:
|
||||
raise RuntimeError("LLM provider 运行时尚未由启动层完成组装")
|
||||
return _provider_runtime_factory()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1 +0,0 @@
|
||||
{}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,249 +0,0 @@
|
||||
"""LLM 服务端工具能力注册与解析。"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from fnmatch import fnmatch
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
WEB_SEARCH_MODES = frozenset({"local", "builtin", "auto", "disabled"})
|
||||
|
||||
|
||||
class ServerToolUnavailableError(ValueError):
|
||||
"""表示用户强制选择了当前模型不可用的服务端工具。"""
|
||||
|
||||
def __init__(self, *, provider: str, model: str, tool_id: str) -> None:
|
||||
"""初始化服务端工具不可用异常。"""
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self.tool_id = tool_id
|
||||
super().__init__(
|
||||
f"当前模型 {provider}/{model} 或接口地址不支持服务端联网搜索,"
|
||||
"请改用“自动”或“MoviePilot 本地搜索”"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServerToolCapability:
|
||||
"""描述一个模型可用的服务端工具能力。"""
|
||||
|
||||
tool_id: str
|
||||
provider_ids: tuple[str, ...]
|
||||
model_patterns: tuple[str, ...]
|
||||
required_api_protocol: str
|
||||
client_adapter: str
|
||||
tool_definition: dict[str, Any]
|
||||
base_url_patterns: tuple[str, ...] = ()
|
||||
match_without_base_url: bool = True
|
||||
|
||||
def matches(self, provider: str, model: str, base_url: Optional[str] = None) -> bool:
|
||||
"""判断给定 provider/model 是否匹配当前能力。"""
|
||||
normalized_provider = str(provider or "").strip().lower()
|
||||
normalized_model = str(model or "").strip().lower().removeprefix("models/")
|
||||
normalized_base_url = str(base_url or "").strip().lower()
|
||||
return (
|
||||
normalized_provider in self.provider_ids
|
||||
and any(fnmatch(normalized_model, pattern) for pattern in self.model_patterns)
|
||||
and (
|
||||
(not normalized_base_url and self.match_without_base_url)
|
||||
or not self.base_url_patterns
|
||||
or any(
|
||||
pattern in normalized_base_url
|
||||
for pattern in self.base_url_patterns
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
def serialize(self) -> dict[str, Any]:
|
||||
"""返回供 API 与前端使用的能力元数据。"""
|
||||
return {
|
||||
"id": self.tool_id,
|
||||
"required_api_protocol": self.required_api_protocol,
|
||||
"client_adapter": self.client_adapter,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ServerToolResolution:
|
||||
"""记录本次联网搜索模式解析后的执行策略。"""
|
||||
|
||||
mode: str
|
||||
use_local_web_search: bool
|
||||
server_tools: tuple[dict[str, Any], ...] = ()
|
||||
client_adapter: Optional[str] = None
|
||||
required_api_protocol: Optional[str] = None
|
||||
available: bool = False
|
||||
reason: Optional[str] = None
|
||||
|
||||
|
||||
class ServerToolRegistry:
|
||||
"""集中注册模型服务端工具,并解析通用执行策略。"""
|
||||
|
||||
_CAPABILITIES = (
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("chatgpt",),
|
||||
model_patterns=("gpt-5*", "gpt-4.1*", "o4-mini*"),
|
||||
base_url_patterns=("api.openai.com",),
|
||||
required_api_protocol="responses",
|
||||
client_adapter="openai_responses",
|
||||
tool_definition={"type": "web_search"},
|
||||
),
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("openai",),
|
||||
model_patterns=("gpt-5*", "gpt-4.1*", "o4-mini*"),
|
||||
base_url_patterns=("api.openai.com",),
|
||||
required_api_protocol="responses",
|
||||
client_adapter="openai_responses",
|
||||
tool_definition={"type": "web_search"},
|
||||
match_without_base_url=False,
|
||||
),
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("anthropic",),
|
||||
model_patterns=(
|
||||
"claude-opus-4*",
|
||||
"claude-sonnet-4*",
|
||||
"claude-haiku-4*",
|
||||
"claude-opus-5*",
|
||||
"claude-sonnet-5*",
|
||||
"claude-haiku-5*",
|
||||
"claude-fable-5*",
|
||||
"claude-mythos-5*",
|
||||
),
|
||||
base_url_patterns=("api.anthropic.com",),
|
||||
required_api_protocol="native",
|
||||
client_adapter="anthropic_native",
|
||||
tool_definition={
|
||||
"type": "web_search_20250305",
|
||||
"name": "web_search",
|
||||
},
|
||||
),
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("google",),
|
||||
model_patterns=("gemini-3*", "gemini-2.5*", "gemini-2.0-flash*"),
|
||||
required_api_protocol="native",
|
||||
client_adapter="google_native",
|
||||
tool_definition={"google_search": {}},
|
||||
),
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("xai",),
|
||||
model_patterns=("grok-4.5*",),
|
||||
base_url_patterns=("api.x.ai",),
|
||||
required_api_protocol="responses",
|
||||
client_adapter="openai_responses",
|
||||
tool_definition={"type": "web_search"},
|
||||
),
|
||||
ServerToolCapability(
|
||||
tool_id="web_search",
|
||||
provider_ids=("deepseek",),
|
||||
model_patterns=("deepseek-v4-flash",),
|
||||
base_url_patterns=("api.deepseek.com",),
|
||||
required_api_protocol="responses",
|
||||
client_adapter="openai_responses",
|
||||
tool_definition={"type": "web_search"},
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def normalize_web_search_mode(cls, mode: Optional[str]) -> str:
|
||||
"""规范化联网搜索模式,未知值回退为本地搜索。"""
|
||||
normalized = str(mode or "local").strip().lower()
|
||||
return normalized if normalized in WEB_SEARCH_MODES else "local"
|
||||
|
||||
@classmethod
|
||||
def get_capability(
|
||||
cls,
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
base_url: Optional[str] = None,
|
||||
tool_id: str,
|
||||
) -> Optional[ServerToolCapability]:
|
||||
"""查找指定模型的服务端工具能力。"""
|
||||
return next(
|
||||
(
|
||||
capability
|
||||
for capability in cls._CAPABILITIES
|
||||
if capability.tool_id == tool_id
|
||||
and capability.matches(provider, model, base_url)
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def list_capabilities(
|
||||
cls,
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
base_url: Optional[str] = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""列出指定模型可用的服务端工具能力。"""
|
||||
return [
|
||||
capability.serialize()
|
||||
for capability in cls._CAPABILITIES
|
||||
if capability.matches(provider, model, base_url)
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def resolve_web_search(
|
||||
cls,
|
||||
*,
|
||||
provider: str,
|
||||
model: str,
|
||||
mode: Optional[str],
|
||||
api_protocol: Optional[str],
|
||||
base_url: Optional[str] = None,
|
||||
) -> ServerToolResolution:
|
||||
"""解析联网搜索应使用本地工具还是模型服务端工具。"""
|
||||
normalized_mode = cls.normalize_web_search_mode(mode)
|
||||
normalized_protocol = str(api_protocol or "auto").strip().lower()
|
||||
capability = cls.get_capability(
|
||||
provider=provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
tool_id="web_search",
|
||||
)
|
||||
|
||||
if normalized_mode == "disabled":
|
||||
return ServerToolResolution(
|
||||
mode=normalized_mode,
|
||||
use_local_web_search=False,
|
||||
reason="web_search_disabled",
|
||||
)
|
||||
if normalized_mode == "local":
|
||||
return ServerToolResolution(
|
||||
mode=normalized_mode,
|
||||
use_local_web_search=True,
|
||||
reason="local_web_search_selected",
|
||||
)
|
||||
if capability is None:
|
||||
return ServerToolResolution(
|
||||
mode=normalized_mode,
|
||||
use_local_web_search=normalized_mode == "auto",
|
||||
reason="builtin_web_search_unavailable",
|
||||
)
|
||||
if (
|
||||
normalized_mode == "auto"
|
||||
and normalized_protocol == "chat_completions"
|
||||
and capability.required_api_protocol == "responses"
|
||||
):
|
||||
return ServerToolResolution(
|
||||
mode=normalized_mode,
|
||||
use_local_web_search=True,
|
||||
available=True,
|
||||
reason="chat_completions_uses_local_fallback",
|
||||
)
|
||||
|
||||
return ServerToolResolution(
|
||||
mode=normalized_mode,
|
||||
use_local_web_search=False,
|
||||
server_tools=(dict(capability.tool_definition),),
|
||||
client_adapter=capability.client_adapter,
|
||||
required_api_protocol=capability.required_api_protocol,
|
||||
available=True,
|
||||
reason="builtin_web_search_selected",
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user