mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-28 11:37:23 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 LinFei83andCursor
|
6a02e7de21 | ||
|
jxxghp
|
458c08a137 | ||
|
千石
|
7012e0e305 | ||
|
ngcat
|
91ce365f78 | ||
|
jxxghp
|
17be4304c1 |
@@ -0,0 +1 @@
|
||||
AGENTS.md
|
||||
+5
-28
@@ -10,17 +10,14 @@ LICENSE
|
||||
|
||||
# Development files
|
||||
.pylintrc
|
||||
**/*.pyc
|
||||
**/__pycache__/
|
||||
**/*.pyo
|
||||
**/*.pyd
|
||||
*.pyc
|
||||
__pycache__/
|
||||
*.pyo
|
||||
*.pyd
|
||||
.Python
|
||||
**/*.so
|
||||
*.so
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
.coverage
|
||||
coverage.json
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
.tox/
|
||||
.nox/
|
||||
@@ -30,31 +27,12 @@ htmlcov/
|
||||
dmypy.json
|
||||
|
||||
# Virtual environments
|
||||
.venv/
|
||||
.worktrees/
|
||||
venv/
|
||||
env/
|
||||
ENV/
|
||||
env.bak/
|
||||
venv.bak/
|
||||
|
||||
# Runtime state and locally synchronized payloads
|
||||
.build/
|
||||
.agent-work/
|
||||
.runtime/
|
||||
.tmp/
|
||||
.cache/
|
||||
node_modules/
|
||||
public/
|
||||
.moviepilot.env
|
||||
.env
|
||||
.env.*
|
||||
config/*
|
||||
!config/category.yaml
|
||||
app/plugins/**
|
||||
!app/plugins/__init__.py
|
||||
app/application/site/*.bin
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
@@ -92,7 +70,6 @@ test_*
|
||||
*_test.py
|
||||
|
||||
# Build artifacts
|
||||
.artifacts/
|
||||
build/
|
||||
.build/
|
||||
dist/
|
||||
|
||||
@@ -8,7 +8,7 @@ body:
|
||||
value: |
|
||||
请说明你希望添加的功能。
|
||||
|
||||
站点适配请求请先按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot/blob/v3/docs/site-adapter-capture.md) 生成脱敏 ZIP，并在下方附加。Issue 及附件是公开内容，提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
|
||||
站点适配请求请先按 [站点适配采集说明](https://github.com/jxxghp/MoviePilot/blob/v2/docs/site-adapter-capture.md) 生成脱敏 ZIP，并在下方附加。Issue 及附件是公开内容，提交前必须解压预览四个文件。不要上传 Cookie、Authorization、通行密钥、会话字段或任何原始数据。
|
||||
- type: input
|
||||
id: version
|
||||
attributes:
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
name: Official Plugin Architecture Observation
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: '17 3 * * 1'
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: official-plugin-architecture-observation
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
observe:
|
||||
runs-on: ubuntu-latest
|
||||
name: Compare latest official plugin contracts
|
||||
timeout-minutes: 15
|
||||
|
||||
steps:
|
||||
- name: Checkout MoviePilot
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
path: MoviePilot
|
||||
|
||||
- name: Checkout MoviePilot-Plugins
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Plugins
|
||||
ref: main
|
||||
path: MoviePilot-Plugins
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
MoviePilot/pyproject.toml
|
||||
MoviePilot/uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
working-directory: MoviePilot
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Compare plugin contracts without updating fixtures
|
||||
id: compare
|
||||
continue-on-error: true
|
||||
working-directory: MoviePilot
|
||||
run: |
|
||||
uv run --locked --no-sync python scripts/architecture/baseline.py \
|
||||
--check-plugins \
|
||||
--plugin-repo ../MoviePilot-Plugins \
|
||||
--report official-plugin-architecture-report.json
|
||||
|
||||
- name: Upload semantic comparison report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: official-plugin-architecture-report
|
||||
path: MoviePilot/official-plugin-architecture-report.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 3
|
||||
|
||||
- name: Fail when plugin contracts changed
|
||||
if: steps.compare.outcome == 'failure'
|
||||
run: exit 1
|
||||
+7
-262
@@ -10,72 +10,15 @@ jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
env:
|
||||
TRIVY_SKIP_DIRS: /usr/share/java
|
||||
TRIVY_SKIP_JAVA_DB_UPDATE: "true"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
|
||||
- name: Verify dependency lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Audit locked Python dependencies
|
||||
run: |
|
||||
uv export --quiet --locked --no-default-groups --group runtime-standard \
|
||||
--no-emit-project --output-file /tmp/moviepilot-audit-standard.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--require-hashes --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-standard.txt
|
||||
|
||||
uv export --quiet --locked --no-default-groups --group runtime-free-threaded \
|
||||
--no-emit-project --no-hashes \
|
||||
--output-file /tmp/moviepilot-audit-free-threaded.txt
|
||||
python3 scripts/normalize_audit_requirements.py \
|
||||
--lock uv.lock \
|
||||
--input /tmp/moviepilot-audit-free-threaded.txt \
|
||||
--output /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--no-deps --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
frontend_version=$(sed -ne "s/FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/gp" version.py)
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
|
||||
echo "frontend_version=$frontend_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve External Payload Revisions
|
||||
id: payloads
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
FRONTEND_VERSION: ${{ steps.release_version.outputs.frontend_version }}
|
||||
run: |
|
||||
plugins_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Plugins.git refs/heads/main | awk '{print $1}')
|
||||
resources_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Resources.git refs/heads/main | awk '{print $1}')
|
||||
frontend_digest=$(gh api "repos/jxxghp/MoviePilot-Frontend/releases/tags/${FRONTEND_VERSION}" \
|
||||
--jq '.assets[] | select(.name == "dist.zip") | .digest')
|
||||
|
||||
[[ "$plugins_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$resources_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
case "$frontend_digest" in
|
||||
sha256:*) frontend_sha256=${frontend_digest#sha256:} ;;
|
||||
*) echo "dist.zip 缺少 SHA-256 摘要" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "$frontend_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
echo "plugins_revision=$plugins_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "resources_revision=$resources_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_digest=$frontend_digest" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_sha256=$frontend_sha256" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout Wiki Plugin Market
|
||||
uses: actions/checkout@v4
|
||||
@@ -92,42 +35,17 @@ jobs:
|
||||
run: |
|
||||
python3 -m scripts.generate_plugin_market_default \
|
||||
--wiki-file .build/moviepilot-wiki/plugin.md \
|
||||
--config-file app/runtime/config.py
|
||||
--config-file app/core/config.py
|
||||
wiki_commit=$(git -C .build/moviepilot-wiki rev-parse HEAD)
|
||||
echo "wiki_commit=$wiki_commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download models.dev catalog
|
||||
id: models_catalog
|
||||
run: |
|
||||
temp_file=$(mktemp app/agent/llm/models.json.XXXXXX)
|
||||
trap 'rm -f "$temp_file"' EXIT
|
||||
curl --fail --show-error --silent --location --retry 3 \
|
||||
--connect-timeout 10 --max-time 120 \
|
||||
"https://models.dev/api.json" -o "$temp_file"
|
||||
jq -e 'type == "object"' "$temp_file" >/dev/null
|
||||
# Git keeps only a small placeholder; the beta image receives the current catalog.
|
||||
chmod 644 "$temp_file"
|
||||
mv "$temp_file" app/agent/llm/models.json
|
||||
echo "digest=sha256:$(sha256sum app/agent/llm/models.json | awk '{print $1}')" >> "$GITHUB_OUTPUT"
|
||||
echo "Downloaded models.dev catalog ($(wc -c < app/agent/llm/models.json) bytes)"
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3
|
||||
ghcr.io/${{ github.repository }}-v3
|
||||
tags: |
|
||||
type=raw,value=beta
|
||||
|
||||
- name: Docker Meta free-threaded
|
||||
id: meta_ft
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3t
|
||||
ghcr.io/${{ github.repository }}-v3t
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v2
|
||||
ghcr.io/${{ github.repository }}
|
||||
tags: |
|
||||
type=raw,value=beta
|
||||
|
||||
@@ -137,134 +55,6 @@ jobs:
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build standard amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan standard amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build standard arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan standard arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -278,8 +68,8 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish standard multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
- name: Build Image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
@@ -288,53 +78,8 @@ jobs:
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
labels: |
|
||||
${{ steps.meta.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
|
||||
- name: Publish free-threaded multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta_ft.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
labels: |
|
||||
${{ steps.meta_ft.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
cache-from: type=gha,scope=moviepilot-docker,version=2
|
||||
cache-to: type=gha,scope=moviepilot-docker,mode=max,version=2
|
||||
|
||||
@@ -1,532 +1,14 @@
|
||||
name: MoviePilot Builder v3
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'version.py'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
select-v3:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
env:
|
||||
TRIVY_SKIP_DIRS: /usr/share/java
|
||||
TRIVY_SKIP_JAVA_DB_UPDATE: "true"
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
|
||||
- name: Verify dependency lock
|
||||
run: uv lock --check
|
||||
|
||||
- name: Audit locked Python dependencies
|
||||
# GitHub 仅从默认分支登记手动工作流；选择 v3 后会加载 v3 分支的完整构建配置。
|
||||
- name: Require v3 branch
|
||||
run: |
|
||||
uv export --quiet --locked --no-default-groups --group runtime-standard \
|
||||
--no-emit-project --output-file /tmp/moviepilot-audit-standard.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--require-hashes --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-standard.txt
|
||||
|
||||
uv export --quiet --locked --no-default-groups --group runtime-free-threaded \
|
||||
--no-emit-project --no-hashes \
|
||||
--output-file /tmp/moviepilot-audit-free-threaded.txt
|
||||
python3 scripts/normalize_audit_requirements.py \
|
||||
--lock uv.lock \
|
||||
--input /tmp/moviepilot-audit-free-threaded.txt \
|
||||
--output /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
uvx --from pip-audit==2.10.1 pip-audit \
|
||||
--no-deps --disable-pip --strict --progress-spinner off \
|
||||
--requirement /tmp/moviepilot-audit-free-threaded-normalized.txt
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
frontend_version=$(sed -ne "s/FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/gp" version.py)
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
|
||||
echo "frontend_version=$frontend_version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Resolve External Payload Revisions
|
||||
id: payloads
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
FRONTEND_VERSION: ${{ steps.release_version.outputs.frontend_version }}
|
||||
run: |
|
||||
plugins_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Plugins.git refs/heads/main | awk '{print $1}')
|
||||
resources_revision=$(git ls-remote https://github.com/jxxghp/MoviePilot-Resources.git refs/heads/main | awk '{print $1}')
|
||||
frontend_digest=$(gh api "repos/jxxghp/MoviePilot-Frontend/releases/tags/${FRONTEND_VERSION}" \
|
||||
--jq '.assets[] | select(.name == "dist.zip") | .digest')
|
||||
|
||||
[[ "$plugins_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$resources_revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
case "$frontend_digest" in
|
||||
sha256:*) frontend_sha256=${frontend_digest#sha256:} ;;
|
||||
*) echo "dist.zip 缺少 SHA-256 摘要" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "$frontend_sha256" =~ ^[0-9a-f]{64}$ ]]
|
||||
|
||||
echo "plugins_revision=$plugins_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "resources_revision=$resources_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_digest=$frontend_digest" >> "$GITHUB_OUTPUT"
|
||||
echo "frontend_sha256=$frontend_sha256" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Checkout Wiki Plugin Market
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Wiki
|
||||
ref: main
|
||||
path: .build/moviepilot-wiki
|
||||
sparse-checkout: plugin.md
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Generate Plugin Market Default
|
||||
id: plugin_market
|
||||
run: |
|
||||
python3 -m scripts.generate_plugin_market_default \
|
||||
--wiki-file .build/moviepilot-wiki/plugin.md \
|
||||
--config-file app/runtime/config.py
|
||||
wiki_commit=$(git -C .build/moviepilot-wiki rev-parse HEAD)
|
||||
echo "wiki_commit=$wiki_commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download models.dev catalog
|
||||
id: models_catalog
|
||||
run: |
|
||||
temp_file=$(mktemp app/agent/llm/models.json.XXXXXX)
|
||||
trap 'rm -f "$temp_file"' EXIT
|
||||
curl --fail --show-error --silent --location --retry 3 \
|
||||
--connect-timeout 10 --max-time 120 \
|
||||
"https://models.dev/api.json" -o "$temp_file"
|
||||
jq -e 'type == "object"' "$temp_file" >/dev/null
|
||||
# Git keeps only a small placeholder; the release image receives the current catalog.
|
||||
chmod 644 "$temp_file"
|
||||
mv "$temp_file" app/agent/llm/models.json
|
||||
echo "digest=sha256:$(sha256sum app/agent/llm/models.json | awk '{print $1}')" >> "$GITHUB_OUTPUT"
|
||||
echo "Downloaded models.dev catalog ($(wc -c < app/agent/llm/models.json) bytes)"
|
||||
|
||||
- name: Create Release Snapshot
|
||||
id: release_snapshot
|
||||
env:
|
||||
WIKI_COMMIT: ${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add app/runtime/config.py
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "build(plugin-market): sync default from MoviePilot-Wiki@${WIKI_COMMIT:0:12}"
|
||||
fi
|
||||
echo "release_commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3
|
||||
ghcr.io/${{ github.repository }}-v3
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
|
||||
- name: Docker Meta free-threaded
|
||||
id: meta_ft
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v3t
|
||||
ghcr.io/${{ github.repository }}-v3t
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
cache-from: type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3-standard-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded amd64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/amd64
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-amd64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-amd64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded amd64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-amd64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Build free-threaded arm64 candidate
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: linux/arm64/v8
|
||||
load: true
|
||||
push: false
|
||||
pull: true
|
||||
tags: moviepilot-v3t-candidate:linux-arm64
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
cache-from: type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
cache-to: type=gha,scope=moviepilot-v3t-docker-arm64,mode=max,version=2
|
||||
|
||||
- name: Scan free-threaded arm64 candidate vulnerabilities
|
||||
uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0
|
||||
with:
|
||||
image-ref: moviepilot-v3t-candidate:linux-arm64
|
||||
version: v0.70.0
|
||||
cache-dir: ${{ runner.temp }}/trivy
|
||||
scanners: vuln
|
||||
vuln-type: os,library
|
||||
severity: HIGH,CRITICAL
|
||||
ignore-unfixed: true
|
||||
trivyignores: .trivyignore.yaml
|
||||
exit-code: 1
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=standard
|
||||
labels: |
|
||||
${{ steps.meta.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.release-snapshot-revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3-standard-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3-standard-docker-arm64,version=2
|
||||
|
||||
- name: Publish free-threaded multi-architecture image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
pull: false
|
||||
tags: ${{ steps.meta_ft.outputs.tags }}
|
||||
build-args: |
|
||||
MOVIEPILOT_FRONTEND_VERSION=${{ steps.release_version.outputs.frontend_version }}
|
||||
MOVIEPILOT_FRONTEND_SHA256=${{ steps.payloads.outputs.frontend_sha256 }}
|
||||
MOVIEPILOT_PLUGINS_REF=${{ steps.payloads.outputs.plugins_revision }}
|
||||
MOVIEPILOT_RESOURCES_REF=${{ steps.payloads.outputs.resources_revision }}
|
||||
MOVIEPILOT_PYTHON_VARIANT=free-threaded
|
||||
labels: |
|
||||
${{ steps.meta_ft.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.source-revision=${{ env.SOURCE_COMMIT }}
|
||||
org.moviepilot.release-snapshot-revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.frontend-version=${{ steps.release_version.outputs.frontend_version }}
|
||||
org.moviepilot.frontend-digest=${{ steps.payloads.outputs.frontend_digest }}
|
||||
org.moviepilot.plugins-revision=${{ steps.payloads.outputs.plugins_revision }}
|
||||
org.moviepilot.resources-revision=${{ steps.payloads.outputs.resources_revision }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
org.moviepilot.models-catalog-digest=${{ steps.models_catalog.outputs.digest }}
|
||||
cache-from: |
|
||||
type=gha,scope=moviepilot-v3t-docker-amd64,version=2
|
||||
type=gha,scope=moviepilot-v3t-docker-arm64,version=2
|
||||
|
||||
- name: Promote latest image pair
|
||||
env:
|
||||
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
|
||||
run: |
|
||||
ghcr_repository="${GITHUB_REPOSITORY,,}"
|
||||
for image in \
|
||||
"${DOCKER_USERNAME}/moviepilot-v3" \
|
||||
"ghcr.io/${ghcr_repository}-v3"; do
|
||||
docker buildx imagetools create \
|
||||
--tag "${image}:latest" \
|
||||
"${image}:${app_version}"
|
||||
done
|
||||
for image in \
|
||||
"${DOCKER_USERNAME}/moviepilot-v3t" \
|
||||
"ghcr.io/${ghcr_repository}-v3t"; do
|
||||
docker buildx imagetools create \
|
||||
--tag "${image}:latest" \
|
||||
"${image}:${app_version}"
|
||||
done
|
||||
|
||||
- name: Generate Changelog
|
||||
id: changelog
|
||||
run: |
|
||||
# 获取上一个 tag（优先 v3.*，回退到任意 v* 版本 tag）
|
||||
PREVIOUS_TAG=$(git tag -l 'v3.*' --sort=-v:refname | grep -v "^v${{ env.app_version }}$" | head -n 1)
|
||||
if [ -z "$PREVIOUS_TAG" ]; then
|
||||
PREVIOUS_TAG=$(git tag -l 'v[0-9]*' --sort=-v:refname | grep -v "^v${{ env.app_version }}$" | head -n 1)
|
||||
fi
|
||||
echo "Previous tag: $PREVIOUS_TAG"
|
||||
|
||||
# 使用 || 作为分隔符，同时获取 commit 消息和作者 GitHub 用户名
|
||||
if [ -z "$PREVIOUS_TAG" ]; then
|
||||
# 首次发布且无任何历史版本 tag，限制条数避免打印整库历史撑爆环境变量
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" -n 300 "${SOURCE_COMMIT}")
|
||||
else
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" "${PREVIOUS_TAG}..${SOURCE_COMMIT}")
|
||||
fi
|
||||
|
||||
# 分类收集 commit 消息（使用关联数组去重）
|
||||
declare -A SEEN
|
||||
FEATURES=""
|
||||
FIXES=""
|
||||
OTHERS=""
|
||||
|
||||
while IFS= read -r line; do
|
||||
# 跳过空行
|
||||
if [ -z "$line" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 分离 commit 消息和作者
|
||||
msg=$(echo "$line" | sed 's/||[^|]*$//')
|
||||
author=$(echo "$line" | sed 's/.*||//')
|
||||
|
||||
# 跳过 Merge commit 和版本更新 commit
|
||||
if echo "$msg" | grep -qE "^Merge pull request|^Merge branch|^更新 version"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 按 Conventional Commits 前缀分类
|
||||
if echo "$msg" | grep -qiE "^feat(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^feat(\([^)]*\))?:\s*//')
|
||||
category="FEATURES"
|
||||
elif echo "$msg" | grep -qiE "^fix(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^fix(\([^)]*\))?:\s*//')
|
||||
category="FIXES"
|
||||
elif echo "$msg" | grep -qiE "^(docs|style|refactor|perf|test|build|ci|chore|revert)(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^(docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]*\))?:\s*//')
|
||||
category="OTHERS"
|
||||
else
|
||||
desc="$msg"
|
||||
category="OTHERS"
|
||||
fi
|
||||
|
||||
# 使用 "分类+描述" 作为去重的 key，跳过重复内容
|
||||
dedup_key="${category}::${desc}"
|
||||
if [ -n "${SEEN[$dedup_key]+x}" ]; then
|
||||
continue
|
||||
fi
|
||||
SEEN[$dedup_key]=1
|
||||
|
||||
# 添加 by @author 引用
|
||||
entry="- ${desc} by @${author}"
|
||||
|
||||
case "$category" in
|
||||
FEATURES) FEATURES="${FEATURES}${entry}\n" ;;
|
||||
FIXES) FIXES="${FIXES}${entry}\n" ;;
|
||||
OTHERS) OTHERS="${OTHERS}${entry}\n" ;;
|
||||
esac
|
||||
done <<< "$COMMITS"
|
||||
|
||||
# 组装 changelog
|
||||
CHANGELOG=""
|
||||
|
||||
if [ -n "$FEATURES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### ✨ 新功能\n\n${FEATURES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$FIXES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🐛 修复\n\n${FIXES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$OTHERS" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🔧 其他\n\n${OTHERS}\n"
|
||||
fi
|
||||
|
||||
# 添加版本对比链接
|
||||
if [ -n "$PREVIOUS_TAG" ]; then
|
||||
CHANGELOG="${CHANGELOG}**完整更新记录**: https://github.com/${{ github.repository }}/compare/${PREVIOUS_TAG}...v${{ env.app_version }}"
|
||||
fi
|
||||
|
||||
# 写入环境变量
|
||||
echo "CHANGELOG<<EOF" >> $GITHUB_ENV
|
||||
echo -e "$CHANGELOG" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
|
||||
- name: Get existing release body
|
||||
id: get_release_body
|
||||
continue-on-error: true
|
||||
env:
|
||||
CHANGELOG: ${{ env.CHANGELOG }}
|
||||
run: |
|
||||
release_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.app_version }}" | \
|
||||
jq -r '.body // ""')
|
||||
|
||||
# 如果已有手动编写的 release body，则保留；否则使用自动生成的 changelog
|
||||
if [ -n "$release_body" ] && [ "$release_body" != "null" ] && [ "$release_body" != "" ]; then
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "$release_body" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
else
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "$CHANGELOG" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Delete Release
|
||||
uses: dev-drprasad/delete-tag-and-release@v1.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
delete_release: true
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish Release Tag
|
||||
env:
|
||||
RELEASE_COMMIT: ${{ steps.release_snapshot.outputs.release_commit }}
|
||||
run: |
|
||||
tag_name="v${{ env.app_version }}"
|
||||
if git show-ref --verify --quiet "refs/tags/${tag_name}"; then
|
||||
git tag -d "$tag_name"
|
||||
fi
|
||||
git tag "$tag_name" "$RELEASE_COMMIT"
|
||||
git push origin "refs/tags/${tag_name}"
|
||||
|
||||
- name: Generate Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
name: v${{ env.app_version }}
|
||||
body: ${{ env.RELEASE_BODY }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
make_latest: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
echo "::error::请在 Run workflow 中选择 v3 分支"
|
||||
exit 1
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
name: MoviePilot Builder v2
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- v2
|
||||
paths:
|
||||
- 'version.py'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
Docker-build:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Docker Image
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
fetch-tags: true
|
||||
|
||||
- name: Release version
|
||||
id: release_version
|
||||
run: |
|
||||
app_version=$(cat version.py |sed -ne "s/APP_VERSION\s=\s'v\(.*\)'/\1/gp")
|
||||
echo "app_version=$app_version" >> $GITHUB_ENV
|
||||
echo "SOURCE_COMMIT=$(git rev-parse HEAD)" >> $GITHUB_ENV
|
||||
|
||||
- name: Checkout Wiki Plugin Market
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: jxxghp/MoviePilot-Wiki
|
||||
ref: main
|
||||
path: .build/moviepilot-wiki
|
||||
sparse-checkout: plugin.md
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
|
||||
- name: Generate Plugin Market Default
|
||||
id: plugin_market
|
||||
run: |
|
||||
python3 -m scripts.generate_plugin_market_default \
|
||||
--wiki-file .build/moviepilot-wiki/plugin.md \
|
||||
--config-file app/core/config.py
|
||||
wiki_commit=$(git -C .build/moviepilot-wiki rev-parse HEAD)
|
||||
echo "wiki_commit=$wiki_commit" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create Release Snapshot
|
||||
id: release_snapshot
|
||||
env:
|
||||
WIKI_COMMIT: ${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add app/core/config.py
|
||||
if ! git diff --cached --quiet; then
|
||||
git commit -m "build(plugin-market): sync default from MoviePilot-Wiki@${WIKI_COMMIT:0:12}"
|
||||
fi
|
||||
echo "release_commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: |
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot-v2
|
||||
${{ secrets.DOCKER_USERNAME }}/moviepilot
|
||||
ghcr.io/${{ github.repository }}
|
||||
tags: |
|
||||
type=raw,value=${{ env.app_version }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Set Up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set Up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Login DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build Image
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64/v8
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: |
|
||||
${{ steps.meta.outputs.labels }}
|
||||
org.opencontainers.image.revision=${{ steps.release_snapshot.outputs.release_commit }}
|
||||
org.moviepilot.plugin-market-wiki-revision=${{ steps.plugin_market.outputs.wiki_commit }}
|
||||
cache-from: type=gha,scope=moviepilot-docker,version=2
|
||||
cache-to: type=gha,scope=moviepilot-docker,mode=max,version=2
|
||||
|
||||
- name: Generate Changelog
|
||||
id: changelog
|
||||
run: |
|
||||
# 获取上一个 tag（排除当前版本的 tag）
|
||||
PREVIOUS_TAG=$(git tag -l 'v*' --sort=-v:refname | grep -v "^v${{ env.app_version }}$" | head -n 1)
|
||||
echo "Previous tag: $PREVIOUS_TAG"
|
||||
|
||||
# 使用 || 作为分隔符，同时获取 commit 消息和作者 GitHub 用户名
|
||||
if [ -z "$PREVIOUS_TAG" ]; then
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" "${SOURCE_COMMIT}")
|
||||
else
|
||||
COMMITS=$(git log --pretty=format:"%s||%an" "${PREVIOUS_TAG}..${SOURCE_COMMIT}")
|
||||
fi
|
||||
|
||||
# 分类收集 commit 消息（使用关联数组去重）
|
||||
declare -A SEEN
|
||||
FEATURES=""
|
||||
FIXES=""
|
||||
OTHERS=""
|
||||
|
||||
while IFS= read -r line; do
|
||||
# 跳过空行
|
||||
if [ -z "$line" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 分离 commit 消息和作者
|
||||
msg=$(echo "$line" | sed 's/||[^|]*$//')
|
||||
author=$(echo "$line" | sed 's/.*||//')
|
||||
|
||||
# 跳过 Merge commit 和版本更新 commit
|
||||
if echo "$msg" | grep -qE "^Merge pull request|^Merge branch|^更新 version"; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# 按 Conventional Commits 前缀分类
|
||||
if echo "$msg" | grep -qiE "^feat(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^feat(\([^)]*\))?:\s*//')
|
||||
category="FEATURES"
|
||||
elif echo "$msg" | grep -qiE "^fix(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^fix(\([^)]*\))?:\s*//')
|
||||
category="FIXES"
|
||||
elif echo "$msg" | grep -qiE "^(docs|style|refactor|perf|test|build|ci|chore|revert)(\(.+\))?:"; then
|
||||
desc=$(echo "$msg" | sed -E 's/^(docs|style|refactor|perf|test|build|ci|chore|revert)(\([^)]*\))?:\s*//')
|
||||
category="OTHERS"
|
||||
else
|
||||
desc="$msg"
|
||||
category="OTHERS"
|
||||
fi
|
||||
|
||||
# 使用 "分类+描述" 作为去重的 key，跳过重复内容
|
||||
dedup_key="${category}::${desc}"
|
||||
if [ -n "${SEEN[$dedup_key]+x}" ]; then
|
||||
continue
|
||||
fi
|
||||
SEEN[$dedup_key]=1
|
||||
|
||||
# 添加 by @author 引用
|
||||
entry="- ${desc} by @${author}"
|
||||
|
||||
case "$category" in
|
||||
FEATURES) FEATURES="${FEATURES}${entry}\n" ;;
|
||||
FIXES) FIXES="${FIXES}${entry}\n" ;;
|
||||
OTHERS) OTHERS="${OTHERS}${entry}\n" ;;
|
||||
esac
|
||||
done <<< "$COMMITS"
|
||||
|
||||
# 组装 changelog
|
||||
CHANGELOG=""
|
||||
|
||||
if [ -n "$FEATURES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### ✨ 新功能\n\n${FEATURES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$FIXES" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🐛 修复\n\n${FIXES}\n"
|
||||
fi
|
||||
|
||||
if [ -n "$OTHERS" ]; then
|
||||
CHANGELOG="${CHANGELOG}### 🔧 其他\n\n${OTHERS}\n"
|
||||
fi
|
||||
|
||||
# 添加版本对比链接
|
||||
if [ -n "$PREVIOUS_TAG" ]; then
|
||||
CHANGELOG="${CHANGELOG}**完整更新记录**: https://github.com/${{ github.repository }}/compare/${PREVIOUS_TAG}...v${{ env.app_version }}"
|
||||
fi
|
||||
|
||||
# 写入环境变量
|
||||
echo "CHANGELOG<<EOF" >> $GITHUB_ENV
|
||||
echo -e "$CHANGELOG" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
|
||||
- name: Get existing release body
|
||||
id: get_release_body
|
||||
continue-on-error: true
|
||||
run: |
|
||||
release_body=$(curl -s -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
|
||||
"https://api.github.com/repos/${{ github.repository }}/releases/tags/v${{ env.app_version }}" | \
|
||||
jq -r '.body // ""')
|
||||
|
||||
# 如果已有手动编写的 release body，则保留；否则使用自动生成的 changelog
|
||||
if [ -n "$release_body" ] && [ "$release_body" != "null" ] && [ "$release_body" != "" ]; then
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "$release_body" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
else
|
||||
echo "RELEASE_BODY<<EOF" >> $GITHUB_ENV
|
||||
echo "${{ env.CHANGELOG }}" >> $GITHUB_ENV
|
||||
echo "EOF" >> $GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Delete Release
|
||||
uses: dev-drprasad/delete-tag-and-release@v1.1
|
||||
continue-on-error: true
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
delete_release: true
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Publish Release Tag
|
||||
env:
|
||||
RELEASE_COMMIT: ${{ steps.release_snapshot.outputs.release_commit }}
|
||||
run: |
|
||||
tag_name="v${{ env.app_version }}"
|
||||
if git show-ref --verify --quiet "refs/tags/${tag_name}"; then
|
||||
git tag -d "$tag_name"
|
||||
fi
|
||||
git tag "$tag_name" "$RELEASE_COMMIT"
|
||||
git push origin "refs/tags/${tag_name}"
|
||||
|
||||
- name: Generate Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: v${{ env.app_version }}
|
||||
name: v${{ env.app_version }}
|
||||
body: ${{ env.RELEASE_BODY }}
|
||||
draft: false
|
||||
prerelease: false
|
||||
make_latest: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -1,183 +0,0 @@
|
||||
name: Dependency Compatibility
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- 'app/doctor/dependencies.py'
|
||||
- 'app/foundation/environment.py'
|
||||
- 'app/runtime/dependencies.py'
|
||||
- 'docker/Dockerfile'
|
||||
- 'docker/**'
|
||||
- '.github/workflows/dependency-compat.yml'
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
paths:
|
||||
- 'pyproject.toml'
|
||||
- 'uv.lock'
|
||||
- 'app/doctor/dependencies.py'
|
||||
- 'app/foundation/environment.py'
|
||||
- 'app/runtime/dependencies.py'
|
||||
- 'docker/Dockerfile'
|
||||
- 'docker/**'
|
||||
- '.github/workflows/dependency-compat.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: dependency-compat-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
install:
|
||||
name: ${{ matrix.name }} / Python ${{ matrix.python-version }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- name: Linux x64
|
||||
runner: ubuntu-24.04
|
||||
python-version: '3.14'
|
||||
expected-system: Linux
|
||||
expected-machine: x86_64
|
||||
- name: Linux ARM64
|
||||
runner: ubuntu-24.04-arm
|
||||
python-version: '3.14'
|
||||
expected-system: Linux
|
||||
expected-machine: aarch64
|
||||
- name: macOS Intel
|
||||
runner: macos-15-intel
|
||||
python-version: '3.14'
|
||||
expected-system: Darwin
|
||||
expected-machine: x86_64
|
||||
- name: macOS ARM
|
||||
runner: macos-15
|
||||
python-version: '3.14'
|
||||
expected-system: Darwin
|
||||
expected-machine: arm64
|
||||
- name: Windows x64
|
||||
runner: windows-2025
|
||||
python-version: '3.14'
|
||||
expected-system: Windows
|
||||
expected-machine: AMD64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: ${{ matrix.python-version }}
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install locked runtime dependencies
|
||||
run: uv sync --locked --inexact --no-dev --python ${{ matrix.python-version }}
|
||||
|
||||
- name: Verify environment and core imports
|
||||
env:
|
||||
EXPECTED_SYSTEM: ${{ matrix.expected-system }}
|
||||
EXPECTED_MACHINE: ${{ matrix.expected-machine }}
|
||||
run: >-
|
||||
uv run --locked --no-sync python -c
|
||||
"import os, platform;
|
||||
assert platform.system() == os.environ['EXPECTED_SYSTEM'], (platform.system(), os.environ['EXPECTED_SYSTEM']);
|
||||
assert platform.machine() == os.environ['EXPECTED_MACHINE'], (platform.machine(), os.environ['EXPECTED_MACHINE']);
|
||||
import alembic, fastapi, pydantic, pydantic_settings, sqlalchemy, starlette, uvicorn"
|
||||
|
||||
- name: Verify locked project consistency
|
||||
run: uv sync --locked --offline --inexact --no-dev --check --python ${{ matrix.python-version }}
|
||||
|
||||
docker-dependencies:
|
||||
name: Docker dependencies / ${{ matrix.platform }}
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- runner: ubuntu-24.04
|
||||
platform: linux/amd64
|
||||
python-variant: standard
|
||||
cache-scope: linux-amd64-standard
|
||||
image-tag: moviepilot-dependency-gate:linux-amd64-standard
|
||||
expected-machine: x86_64
|
||||
- runner: ubuntu-24.04-arm
|
||||
platform: linux/arm64
|
||||
python-variant: standard
|
||||
cache-scope: linux-arm64-standard
|
||||
image-tag: moviepilot-dependency-gate:linux-arm64-standard
|
||||
expected-machine: aarch64
|
||||
- runner: ubuntu-24.04
|
||||
platform: linux/amd64
|
||||
python-variant: free-threaded
|
||||
cache-scope: linux-amd64-free-threaded
|
||||
image-tag: moviepilot-dependency-gate:linux-amd64-free-threaded
|
||||
expected-machine: x86_64
|
||||
- runner: ubuntu-24.04-arm
|
||||
platform: linux/arm64
|
||||
python-variant: free-threaded
|
||||
cache-scope: linux-arm64-free-threaded
|
||||
image-tag: moviepilot-dependency-gate:linux-arm64-free-threaded
|
||||
expected-machine: aarch64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Build locked dependency stage
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
context: .
|
||||
file: docker/Dockerfile
|
||||
target: prepare_venv
|
||||
platforms: ${{ matrix.platform }}
|
||||
build-args: |
|
||||
MOVIEPILOT_PYTHON_VARIANT=${{ matrix.python-variant }}
|
||||
load: true
|
||||
push: false
|
||||
tags: ${{ matrix.image-tag }}
|
||||
cache-from: type=gha,scope=dependency-compat-${{ matrix.cache-scope }}
|
||||
cache-to: type=gha,scope=dependency-compat-${{ matrix.cache-scope }},mode=max
|
||||
|
||||
- name: Verify dependency image
|
||||
env:
|
||||
IMAGE_TAG: ${{ matrix.image-tag }}
|
||||
EXPECTED_MACHINE: ${{ matrix.expected-machine }}
|
||||
EXPECTED_VARIANT: ${{ matrix.python-variant }}
|
||||
run: >-
|
||||
docker run --rm
|
||||
-e EXPECTED_MACHINE
|
||||
-e EXPECTED_VARIANT
|
||||
"${IMAGE_TAG}"
|
||||
/opt/venv/bin/python -c
|
||||
"import os, platform, sys, sysconfig;
|
||||
assert platform.machine() == os.environ['EXPECTED_MACHINE'], (platform.machine(), os.environ['EXPECTED_MACHINE']);
|
||||
expected_free_threaded = os.environ['EXPECTED_VARIANT'] == 'free-threaded';
|
||||
assert (sysconfig.get_config_var('Py_GIL_DISABLED') == 1) is expected_free_threaded;
|
||||
assert sys._is_gil_enabled() == (not expected_free_threaded);
|
||||
import alembic, fastapi, moviepilot_rust, pydantic, pydantic_settings, sqlalchemy, starlette, uvicorn;
|
||||
assert moviepilot_rust.is_available();
|
||||
assert moviepilot_rust.jieba_cut('中文分词');
|
||||
assert not expected_free_threaded or callable(moviepilot_rust.zhconv_fast);
|
||||
assert sys._is_gil_enabled() == (not expected_free_threaded)"
|
||||
|
||||
- name: Verify pinned uv version
|
||||
env:
|
||||
IMAGE_TAG: ${{ matrix.image-tag }}
|
||||
run: docker run --rm "${IMAGE_TAG}" uv --version | grep -F 'uv 0.12.5'
|
||||
@@ -1,21 +1,9 @@
|
||||
name: Pylint Code Quality Check
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
# 允许手动触发
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: pylint-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
pylint:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -23,22 +11,27 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
fetch-depth: 0
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
python-version: '3.12'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.in', '**/requirements-dev.in', '**/requirements.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
run: |
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
# Pylint 属于开发/静态检查依赖，统一通过 dev 入口安装。
|
||||
pip install -r requirements-dev.in
|
||||
|
||||
- name: Verify pylint config
|
||||
run: |
|
||||
@@ -51,51 +44,32 @@ jobs:
|
||||
echo "❌ 未找到 .pylintrc 配置文件"
|
||||
exit 1
|
||||
fi
|
||||
- name: Collect changed Python files
|
||||
id: changed
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BASE_REF: ${{ github.base_ref }}
|
||||
BEFORE_SHA: ${{ github.event.before }}
|
||||
CURRENT_SHA: ${{ github.sha }}
|
||||
- name: Run pylint
|
||||
run: |
|
||||
if [[ "$EVENT_NAME" == "pull_request" ]]; then
|
||||
git diff --name-only --diff-filter=ACMRT \
|
||||
"origin/$BASE_REF...HEAD" -- '*.py' > changed-python-files.txt
|
||||
elif [[ "$EVENT_NAME" == "push" ]] \
|
||||
&& [[ -n "$BEFORE_SHA" ]] \
|
||||
&& [[ ! "$BEFORE_SHA" =~ ^0+$ ]] \
|
||||
&& git cat-file -e "$BEFORE_SHA^{commit}"; then
|
||||
git diff --name-only --diff-filter=ACMRT \
|
||||
"$BEFORE_SHA" "$CURRENT_SHA" -- '*.py' > changed-python-files.txt
|
||||
else
|
||||
git diff-tree --no-commit-id --name-only --diff-filter=ACMRT \
|
||||
-r HEAD -- '*.py' > changed-python-files.txt
|
||||
fi
|
||||
sort -u -o changed-python-files.txt changed-python-files.txt
|
||||
if [[ -s changed-python-files.txt ]]; then
|
||||
echo "has_files=true" >> "$GITHUB_OUTPUT"
|
||||
sed -n '1,200p' changed-python-files.txt
|
||||
else
|
||||
echo "has_files=false" >> "$GITHUB_OUTPUT"
|
||||
echo "本次没有改动 Python 文件"
|
||||
fi
|
||||
# 运行pylint，检查主要的Python文件
|
||||
echo "🚀 运行 Pylint 错误检查..."
|
||||
|
||||
- name: Run pylint on changed Python files
|
||||
if: steps.changed.outputs.has_files == 'true'
|
||||
run: |
|
||||
xargs uv run --locked --no-sync pylint \
|
||||
--output-format=colorized --reports=yes --score=yes \
|
||||
< changed-python-files.txt
|
||||
# 检查主要目录 - 只关注错误，如果有错误则退出
|
||||
echo "📂 检查 app/ 目录..."
|
||||
pylint app/ --output-format=colorized --reports=yes --score=yes
|
||||
|
||||
- name: Generate full advisory report
|
||||
if: always()
|
||||
run: |
|
||||
uv run --locked --no-sync pylint app/ \
|
||||
--output-format=json > pylint-report.json || true
|
||||
# 检查根目录的Python文件
|
||||
echo "📂 检查根目录 Python 文件..."
|
||||
for file in $(find . -name "*.py" -not -path "./.*" -not -path "./.venv/*" -not -path "./build/*" -not -path "./dist/*" -not -path "./tests/*" -not -path "./docs/*" -not -path "./__pycache__/*" -maxdepth 1); do
|
||||
echo "检查文件: $file"
|
||||
pylint "$file" --output-format=colorized || exit 1
|
||||
done
|
||||
|
||||
# 生成详细报告
|
||||
echo "📊 生成 Pylint 详细报告..."
|
||||
pylint app/ --output-format=json > pylint-report.json || true
|
||||
|
||||
# 显示评分（仅供参考）
|
||||
echo "📈 Pylint 评分（仅供参考）:"
|
||||
pylint app/ --score=yes --reports=no | tail -2 || true
|
||||
|
||||
- name: Upload pylint report
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: pylint-report
|
||||
@@ -104,5 +78,5 @@ jobs:
|
||||
- name: Summary
|
||||
run: |
|
||||
echo "🎉 Pylint 检查完成！"
|
||||
echo "✅ 改动 Python 文件没有新增语法错误或严重问题"
|
||||
echo "📊 全仓建议性报告已保存为构建工件"
|
||||
echo "✅ 没有发现语法错误或严重问题"
|
||||
echo "📊 详细报告已保存为构建工件"
|
||||
|
||||
@@ -49,17 +49,14 @@ jobs:
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: scripts/site_adapter_collector_requirements.txt
|
||||
python-version: '3.12'
|
||||
cache: pip
|
||||
cache-dependency-path: scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Install build dependencies
|
||||
run: uv pip install --system --requirement scripts/site_adapter_collector_requirements.txt
|
||||
run: |
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
pip install -r scripts/site_adapter_collector_requirements.txt
|
||||
|
||||
- name: Build single-file collector
|
||||
run: |
|
||||
|
||||
+29
-115
@@ -1,13 +1,13 @@
|
||||
name: Unit Tests
|
||||
|
||||
on:
|
||||
# 指向 v3 的 PR 与推送都跑全量单测，作为合并门禁
|
||||
# 指向 v2 的 PR 与推送都跑全量单测，作为合并门禁
|
||||
pull_request:
|
||||
branches:
|
||||
- v3
|
||||
- v2
|
||||
push:
|
||||
branches:
|
||||
- v3
|
||||
- v2
|
||||
# 允许手动触发
|
||||
workflow_dispatch:
|
||||
|
||||
@@ -19,138 +19,52 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
architecture:
|
||||
runs-on: ubuntu-latest
|
||||
name: Architecture Contract Gate
|
||||
timeout-minutes: 10
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Check host architecture contracts
|
||||
run: |
|
||||
uv run --locked --no-sync pytest \
|
||||
tests/test_architecture_dependencies.py \
|
||||
tests/test_architecture_adapter_imports.py \
|
||||
tests/test_architecture_egress.py \
|
||||
tests/test_architecture_contract_baseline.py \
|
||||
tests/test_architecture_baseline_cli.py -q
|
||||
uv run --locked --no-sync python \
|
||||
scripts/architecture/baseline.py --check-host
|
||||
|
||||
- name: Check governed Python types
|
||||
run: uv run --locked --no-sync mypy --config-file mypy.ini
|
||||
|
||||
- name: Check complexity ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/complexity.py
|
||||
|
||||
- name: Check async blocking ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/async_blocking.py
|
||||
|
||||
- name: Check background task ownership
|
||||
run: uv run --locked --no-sync python scripts/architecture/task_ownership.py
|
||||
|
||||
- name: Check process runtime service locators
|
||||
run: uv run --locked --no-sync python scripts/architecture/service_locator.py
|
||||
|
||||
- name: Check Ruff diagnostic ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/ruff_ratchet.py
|
||||
|
||||
- name: Check mypy error ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/mypy_ratchet.py
|
||||
|
||||
- name: Check startup performance contract
|
||||
run: >-
|
||||
uv run --locked --no-sync python
|
||||
scripts/startup/performance.py --check --repeat 3
|
||||
|
||||
pytest:
|
||||
runs-on: ubuntu-latest
|
||||
name: Unit Tests (${{ matrix.shard }})
|
||||
name: Unit Tests
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- shard: '1/4'
|
||||
- shard: '2/4'
|
||||
- shard: '3/4'
|
||||
- shard: '4/4'
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
python-version: '3.12'
|
||||
cache: 'pip'
|
||||
|
||||
- name: Cache pip dependencies
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ~/.cache/pip
|
||||
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.in', '**/requirements-dev.in', '**/requirements.txt') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-pip-
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
run: |
|
||||
python -m pip install --upgrade pip setuptools wheel
|
||||
# 单测需要开发/测试依赖；运行时入口 requirements.in 不携带测试与构建辅助工具。
|
||||
pip install -r requirements-dev.in
|
||||
|
||||
- name: Run tests
|
||||
timeout-minutes: 10
|
||||
run: uv run --locked --no-sync python tests/run.py --shard "${{ matrix.shard }}"
|
||||
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
name: Coverage Report
|
||||
timeout-minutes: 20
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v5
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
|
||||
with:
|
||||
version: '0.12.5'
|
||||
python-version: '3.14'
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
uv.lock
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --locked
|
||||
|
||||
- name: Generate coverage reports
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
uv run --locked --no-sync python -m coverage erase
|
||||
uv run --locked --no-sync python -m coverage run tests/run.py --serial
|
||||
uv run --locked --no-sync python -m coverage report
|
||||
uv run --locked --no-sync python -m coverage json
|
||||
uv run --locked --no-sync python -m coverage xml
|
||||
# tests/run.py 以 pytest 跑 tests 全量；tests/conftest.py 在收集前把 CONFIG_DIR
|
||||
# 指向临时库并建表；CI 额外生成覆盖率报告，便于后续补测和回归分析。
|
||||
python -m coverage erase
|
||||
python -m coverage run tests/run.py
|
||||
python -m coverage report
|
||||
python -m coverage json
|
||||
python -m coverage xml
|
||||
|
||||
- name: Upload coverage report
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: coverage-report
|
||||
path: |
|
||||
coverage.xml
|
||||
coverage.json
|
||||
retention-days: 7
|
||||
|
||||
- name: Check coverage ratchet
|
||||
run: uv run --locked --no-sync python scripts/architecture/coverage_ratchet.py
|
||||
|
||||
+14
-13
@@ -9,18 +9,23 @@ dist/
|
||||
rust/**/target/
|
||||
nginx/
|
||||
test.py
|
||||
app/application/site/*.bin
|
||||
# 站点数据的运行期下载产物。上游 v3 架构重构后落点从 app/application/site 移到了
|
||||
# app/helper，同目录的 .so/.pyd 由上面的通配兜住，只有 .bin 漏了网
|
||||
safety_report.txt
|
||||
app/helper/sites.py
|
||||
app/helper/*.so
|
||||
app/helper/*.pyd
|
||||
app/helper/*.bin
|
||||
app/plugins/**
|
||||
!app/plugins/__init__.py
|
||||
config/*
|
||||
!config/category.yaml
|
||||
# 运行期设置持久化目录（settings 写回 app.env 的落点）与本地验证产物
|
||||
app/config/
|
||||
.verify_tmp/
|
||||
.artifacts/
|
||||
config/cookies/
|
||||
config/app.env
|
||||
config/user.db*
|
||||
config/sites/**
|
||||
config/agent/
|
||||
config/logs/
|
||||
config/plugins/
|
||||
config/temp/
|
||||
config/cache/
|
||||
config/.cache/
|
||||
.runtime/
|
||||
public/
|
||||
.moviepilot.env
|
||||
@@ -45,7 +50,3 @@ pylint-report.json
|
||||
|
||||
# Superpowers 设计/计划文档（本地协作产物，不纳入仓库）
|
||||
docs/superpowers/
|
||||
|
||||
# 保留本地前端构建产物目录，目录内产物不纳入仓库
|
||||
frontend-dist/*
|
||||
!frontend-dist/.gitkeep
|
||||
|
||||
@@ -73,5 +73,5 @@ ignore-imports=yes
|
||||
[TYPECHECK]
|
||||
# 生成缺失成员提示的类列表
|
||||
generated-members=requests.packages.urllib3
|
||||
# app.infrastructure.sites 会主动隐藏模块属性枚举，接口由同目录 sites.pyi 声明
|
||||
ignored-modules=app.infrastructure.sites
|
||||
# app.helper.sites 会主动隐藏模块属性枚举，避免误报 no-name-in-module
|
||||
ignored-modules=app.helper.sites
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
vulnerabilities:
|
||||
- id: GHSA-6v7p-g79w-8964
|
||||
paths:
|
||||
- Python
|
||||
purls:
|
||||
- pkg:pypi/msgpack@1.1.2
|
||||
expired_at: 2026-11-20
|
||||
statement: The finding belongs to the base image's system pip and is not imported by MoviePilot.
|
||||
- id: CVE-2025-47273
|
||||
paths:
|
||||
- Python
|
||||
purls:
|
||||
- pkg:pypi/setuptools@70.3.0
|
||||
expired_at: 2026-11-20
|
||||
statement: The finding belongs to the base image's system pip and is not used for dependency installation.
|
||||
- id: CVE-2026-33818
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-39821
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-46600
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56853
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56858
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56859
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56860
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
- id: CVE-2026-56862
|
||||
paths:
|
||||
- usr/bin/rclone
|
||||
purls:
|
||||
- pkg:golang/stdlib@v1.26.5
|
||||
expired_at: 2026-11-20
|
||||
statement: The official rclone binary has no patched release for this embedded Go runtime yet.
|
||||
@@ -6,7 +6,7 @@ This file is the primary instruction set for all AI agents and LLMs working in t
|
||||
|
||||
## Task-to-Documentation Mapping
|
||||
|
||||
For work that changes or reviews repository behavior, identify the domains actually touched and load only the applicable documents. Simple factual checks and unrelated domains do not require preloading rule files.
|
||||
Before executing any task, identify the domain and load the corresponding document.
|
||||
|
||||
### Architectural Decisions
|
||||
* **Primary Reference:** `docs/rules/05-architecture.md`
|
||||
@@ -26,11 +26,12 @@ For work that changes or reviews repository behavior, identify the domains actua
|
||||
|
||||
### Comments and Documentation
|
||||
* **Primary Reference:** `docs/rules/08-comment-styles.md`
|
||||
* **Required Constraints:** Public or cross-module contracts and non-obvious business behavior require concise Chinese docstrings. Small self-evident private helpers and test scaffolding may omit them. Comments must explain the *why*, not restate the code.
|
||||
* **Required Constraints:** All public classes and methods require Chinese docstrings. Comments must explain the *why*, not restate the code.
|
||||
* **⚠️ MANDATORY GATE:** Code that is missing proper Chinese docstrings on public interfaces is **REJECTED** at review. No exceptions.
|
||||
|
||||
### External Communication and Interfaces
|
||||
* **Primary Reference:** `docs/rules/09-external-response.md`
|
||||
* **Required Constraints:** Host-authored ordinary HTTP must go through `RequestUtils`; this rule does not authorize Application/Chain to import the concrete Adapter. Canonical transport, SDK, streaming protocol, contained vendor, diagnostic and control-plane exceptions must match the exact direct-egress policy. Response formats must use the project's standard schemas. Error handling must follow the per-layer conventions.
|
||||
* **Required Constraints:** All third-party HTTP requests must go through `RequestUtils`. Response formats must use the project's standard schemas. Error handling must follow the per-layer conventions.
|
||||
|
||||
### Data and Persistence
|
||||
* **Primary Reference:** `docs/rules/10-data-and-persistent.md`
|
||||
@@ -38,79 +39,15 @@ For work that changes or reviews repository behavior, identify the domains actua
|
||||
|
||||
### Quality and Security
|
||||
* **Primary Reference:** `docs/rules/11-quality-and-security.md`
|
||||
* **Required Constraints:** All code changes must pass the relevant pytest tests and pylint checks. Dependency changes require a current `uv.lock`, locked environment verification, and a passing locked dependency vulnerability audit.
|
||||
* **Required Constraints:** All code changes must pass the relevant pytest tests and pylint checks. Dependency changes require a passing safety scan.
|
||||
|
||||
### Testing
|
||||
* **Primary Reference:** `docs/testing.md`
|
||||
* **Required Constraints:** pytest is the only runner; `tests/conftest.py` isolates each run to a temporary `CONFIG_DIR`. Tests must not touch the real database, network, or external services (TMDB, LLM catalogs, downloaders, media servers, MP server) — mock at the boundary or replay recorded responses; the bar is zero real outbound traffic. Tests must restore any process-level state they stub (`sys.modules`, singletons, caches, settings). New tests must be pytest-native (function + `assert` + fixtures); do not add new `unittest.TestCase`. Convert existing `TestCase` files to pytest-native opportunistically when you modify them. Before opening a PR to `v3`, run the affected tests and applicable local checks. Run the full local suite (`uv run --locked --no-sync python tests/run.py`) for dependency or lock changes, shared test infrastructure, database or startup paths, cross-module lifecycle, compatibility layers, broad behavior changes, or an explicit maintainer requirement. The changed path must pass; any unrelated failure must be reported and reproduced against the current `upstream/v3` baseline instead of silently expanding the PR. Documentation-only changes use applicable text and structure checks; the `.github/workflows/test.yml` gate remains the final full-suite check on every PR/push to `v3`.
|
||||
* **Required Constraints:** pytest is the only runner; `tests/conftest.py` isolates each run to a temporary `CONFIG_DIR`. Tests must not touch the real database, network, or external services (TMDB, LLM catalogs, downloaders, media servers, MP server) — mock at the boundary or replay recorded responses; the bar is zero real outbound traffic. Tests must restore any process-level state they stub (`sys.modules`, singletons, caches, settings). New tests must be pytest-native (function + `assert` + fixtures); do not add new `unittest.TestCase`. Convert existing `TestCase` files to pytest-native opportunistically when you modify them. Before opening a PR to `v2`, run the full suite locally (`python tests/run.py`) and confirm it is green with zero real network calls; the `.github/workflows/test.yml` gate runs the same suite on every PR/push to `v2`.
|
||||
|
||||
### Commands and Development Workflow
|
||||
* **Primary Reference:** `docs/rules/03-commands.md`
|
||||
* **Required Constraints:** Use that file as the project command reference. Other standard inspection, Git, GitHub, and focused verification commands are allowed when they are necessary, scoped, and consistent with current authorization.
|
||||
|
||||
---
|
||||
|
||||
## Canonical Package Ownership
|
||||
|
||||
The historical `app/core`, `app/helper`, and `app/utils` directories are compatibility-only virtual import roots. Never add physical Python source there and never use those imports from host code. Choose an owner by responsibility, not by whether a function is "shared" or has historically been called a helper.
|
||||
|
||||
The legacy roots have no physical directories in the source tree. Current images and update flows write site resources only to `app/application/site/`; plugin imports under `app.helper.*` are resolved exclusively by the exact runtime compatibility manifest.
|
||||
|
||||
| Package | Owns | Must Not Own | Representative Files |
|
||||
|---|---|---|---|
|
||||
| `app/foundation/` | 无状态、无配置和无 I/O 的底层机制：反射/动态导入、加密、DOM、身份、集合、单例、文本、URL 和版本比较 | `settings`、DB/SystemConfig、网络请求、运行日志、MoviePilot 业务规则、旧导入路径 | `reflection.py`, `crypto.py`, `collections.py`, `text.py`, `url.py` |
|
||||
| `app/domain/` | Pure MoviePilot business semantics and models for media, recognition, sites, and torrents | Persistence, global settings reads, network/filesystem clients, Rust imports, service discovery, process lifecycle | `context.py`, `media.py`, `metainfo.py`, `scraper.py`, `meta/` |
|
||||
| `app/runtime/` | 进程级运行机制和策略：配置、事件、完整日志、缓存契约/内存行为、托管资源门面、并发、调度、限流、本地化、GC 和重启状态 | 具体外部产品、业务流程、Redis/文件缓存实现 | `config.py`, `events.py`, `log.py`, `cache.py`, `managed_resources.py`, `thread.py`, `state.py` |
|
||||
| `app/runtime/extensions/` | 模块、插件、配置化服务和托管资源实现的发现、注册与生命周期适配 | 通用反射机制、插件公开 API、无关业务流程 | `module_manager.py`, `plugin_manager.py`, `managed_resource_adapter.py`, `service_registry.py` |
|
||||
| `app/adapters/network/` | HTTP、浏览器、DNS、Cloudflare 和 IP 等通用网络技术适配 | RSS/站点业务编排、身份认证策略、命名外部产品流程 | `http.py`, `browser.py`, `doh.py`, `ip.py` |
|
||||
| `app/adapters/cache/` | Redis 与文件缓存等具体持久化实现 | 缓存协议、装饰器和进程内缓存策略 | `backends.py`, `redis.py` |
|
||||
| `app/adapters/system/` | 操作系统、文件、进程、标准流、包/资源安装、显示和 Rust 加速适配 | 业务规则、进程重启决策 | `host.py`, `display/`, `stdio.py`, `package.py`, `resource.py`, `rust.py`, `fsproxy.py` |
|
||||
| `app/adapters/external/` | CookieCloud、插件市场、OCR、IP 归属和 MoviePilot Server 等命名外部生态 | 通用 HTTP/DNS/文件机制或可复用领域语义 | `market.py`, `server.py`, `cookiecloud.py`, `ocr.py`, `location.py`, `wechat_crypt.py` |
|
||||
| `app/application/` | 聚焦应用服务、用例命令，以及由用例拥有的持久化/技术能力 Port/Protocol | SQLAlchemy、Session、Oper 等具体 DB 实现，具体 Adapter 静态依赖，多领域 Chain 编排、底层通用机制、通用传输协议 | `recognition.py`, `filter.py`, `outbox.py`, `subscription/write.py`, `workflow.py` |
|
||||
| `app/application/messaging/` | 消息渲染/路由、交互和 Agent 到消息桥接：`ingress.py` 统一渠道回环入口；`interaction.py` 通用交互契约和视图工具；`router.py` 统一交互优先级和回调分发；`site.py`/`subscribe.py`/`skill.py` 对应命令的会话、输入解析和视图；`media.py` 媒体交互状态（业务工作流仍由 `MediaInteractionChain` 执行）；`plugin.py` 插件输入接管和插件按钮回调；`agent.py` Agent 选择状态、回调协议和 WebAgent 消息桥接；`message.py` 通知渲染、模板和队列。不作为推荐给插件直接使用的公开 SDK | 认证策略、通用 HTTP、服务发现、仅端点使用的 Web Push 行为 | `ingress.py`, `message.py`, `interaction.py`, `router.py`, `agent.py` |
|
||||
| `app/application/security/` | 认证、授权、Cookie、Passkey、OTP/二次认证、路径/URL 安全、SSRF 和签名策略 | 通用 URL 解析、进程运行策略、普通业务校验 | `access.py`, `auth.py`, `cookie.py`, `passkey.py`, `otp.py`, `twofactor.py`, `url.py` |
|
||||
| `app/chain/` | Reusable use-case orchestration across modules, Application services, injected ports, events, and caches; chains reach modules only through `run_module` dispatch on method-name contracts | Transport schemas, backend-specific protocol details, concrete Adapter imports, generic primitives, direct Oper/DB imports, direct imports of module internals (classes, exceptions, constants) | `media.py`, `download.py`, `subscribe.py`, `transfer.py` |
|
||||
| `app/db/oper/` | 面向表和持久化值的 SQLAlchemy 数据访问；接收调用方 Session，只查询、暂存或 flush | Application 业务规则、隐式事务所有权、外部副作用 | `subscribe.py`, `site.py`, `workflow.py` |
|
||||
| `app/db/adapters/` | 实现 Application 持久化 Port，创建短生命周期 Session/UoW，并适配 Oper | 用例规则、启动顺序、进程生命周期 | `subscription.py`, `site.py`, `outbox.py`, `workflow.py` |
|
||||
| `app/startup/` | Composition root: `composition/` 构造并注入跨层依赖，`initializers/` 按领域初始化，`lifecycle/` 编排启动关闭 | Reusable business rules or adapter implementation details | `composition/context.py`, `composition/database.py`, `initializers/modules.py`, `lifecycle/components.py` |
|
||||
| `app/sdk/` | Deliberately curated stable imports for new plugins | Canonical implementation logic or host-internal dependencies | `browser.py`, `cache.py`, `logging.py`, `media.py`, `network.py`, `services.py` |
|
||||
| `app/runtime/compat/` | 仅依赖标准库的精确旧导入路由、资源前置扫描和 DEBUG 诊断 | 业务实现、通配猜测、目标模块的提前导入 | `manifest.py`, `imports.py`, `resource_imports.py`, `diagnostics.py` |
|
||||
|
||||
容易误分的三个边界必须按实际职责判断：`application/rss.py` 同时承担 Feed/种子语义、站点规则和浏览器回退，不是单纯 HTTP 传输；规范目标是由它拥有所需 Port、startup 注入 network/system Adapter。当前直接导入是 `S2-L6` 临时债务，不是允许的新模式。`application/site/sites.*` 及 `user.sites.v3.bin` 共同构成站点目录、认证和索引应用能力，只有下载安装机制留在 `adapters/system/resource.py`；`foundation/crypto.py` 只提供无状态 RSA/摘要/AES 算法，认证、签名、令牌和二次验证策略仍属于 `application/security/`。
|
||||
|
||||
### Placement Decision Order
|
||||
|
||||
Use these questions in order before creating or moving a module:
|
||||
|
||||
1. Is it generic, free of MoviePilot state and I/O? Put it in `foundation`.
|
||||
2. Is it a pure core MoviePilot rule/model that is independent of a configured service boundary? Put it in `domain`.
|
||||
3. Is it process-wide runtime policy or a contract used by adapters? Put it in `runtime`.
|
||||
4. Does it discover or manage modules/plugins/service implementations? Put it in `runtime/extensions`.
|
||||
5. Does it perform configured network, cache, OS/process, file, package/resource, stdio, or Rust I/O? Put it under the matching `adapters` technical boundary.
|
||||
6. Does it implement a named external product/ecosystem workflow? Put it in `adapters/external`.
|
||||
7. Does it own authentication, authorization, signing, SSRF, URL/path safety, OTP, passkeys, or two-factor behavior? Put it in `application/security`.
|
||||
8. Does it define a use case or the persistence Port required by that use case? Put it in `application`; do not import concrete DB there.
|
||||
9. Does it implement an Application persistence Port with SQLAlchemy Session/UoW/Oper? Put it in `db/adapters`.
|
||||
10. Does it coordinate several modules/services/Oper classes for one use case? Put it in `chain`.
|
||||
11. Is it public to plugins or only preserving an old path? Curate it in `sdk` or map it in `runtime/compat`; do not move implementation there.
|
||||
|
||||
### Enforced Split Examples
|
||||
|
||||
These decisions are architectural constraints, not naming suggestions:
|
||||
|
||||
* Cache contracts, memory backends, decorators, and proxies stay in `app/runtime/cache.py`; Redis and filesystem implementations stay in `app/adapters/cache/backends.py`. Startup registers concrete factories before decorated business modules are imported. Legacy `app.core.cache` resolves to the complete `app.sdk.cache` facade.
|
||||
* The complete logging runtime stays in `app/runtime/log.py`: policy, console/plugin routing, async rotating file output, and shutdown. `app.runtime.config` supplies the resolved settings and log path. `runtime/log.py` remains a dependency leaf with no `app.*` imports. Plugins use `app.sdk.logging`; legacy `app.log` resolves to that SDK facade.
|
||||
* Recognition parsing stays pure in `app/domain/meta/` and `app/domain/metainfo.py`. `app/application/recognition.py` consumes injected configuration; `app/startup/initializers/domain.py` injects rules, extension policy, source defaults, TMDB image construction, and the optional Rust accelerator.
|
||||
* Kodi-style NFO reading and metadata document generation are one domain capability and stay together in `app/domain/scraper.py`; a separate `domain/nfo.py` must not be recreated.
|
||||
* `app/application/mediaserver.py` is the single media-server service capability module. It owns configured service discovery together with Provider ID normalization and music-library matching, while reusing generic identity rules from `app/domain/media.py`.
|
||||
* Configured notification-service discovery belongs in `app/application/notification.py`. Web Push subscription and manual-send HTTP behavior stays in `app/api/endpoints/message.py`; it is not a reusable messaging capability module.
|
||||
* `app/adapters/system/resource.py` detects/downloads/installs resources and returns whether installation occurred. Only `app/startup/initializers/modules.py` may decide to restart the process afterward.
|
||||
* Process memory/GC policy belongs in `app/runtime/gc.py`; external IP-location APIs belong in `app/adapters/external/location.py`.
|
||||
* Security implementation filenames use package-context nouns: `app/application/security/url.py` and `app/application/security/twofactor.py`. Historical `app.utils.security` and `app.helper.twofa` remain compatibility mappings only.
|
||||
|
||||
Foundation modules do not emit runtime logs. They return documented fallback values or raise according to their public contract; application callers decide whether a failure is operationally relevant and log it from the owning upper layer.
|
||||
|
||||
Any ownership move must update canonical host imports, `app/runtime/compat/manifest.py`, curated SDK exports when applicable, `docs/rules/05-architecture.md`, and `tests/test_architecture_dependencies.py`. Run that architecture test before broader tests; it rejects physical legacy sources, forbidden upward dependencies, retired canonical filenames, and import cycles.
|
||||
* **Required Constraints:** Only suggest or execute commands documented in that file. Do not assume tool defaults or global flags.
|
||||
|
||||
---
|
||||
|
||||
@@ -118,22 +55,31 @@ Any ownership move must update canonical host imports, `app/runtime/compat/manif
|
||||
|
||||
### Pre-Flight Check
|
||||
|
||||
Before generating code or proposing changes, identify the domains the task actually touches and load only the corresponding documents from `docs/rules/`. Apply those constraints while designing, implementing, and reviewing the change; do not produce a formal checklist for unrelated domains.
|
||||
Before generating any code or proposing changes, you must:
|
||||
|
||||
Architecture, persistence, security, external protocols, cross-module lifecycle, and public-contract changes require an explicit boundary check before implementation. Local documentation, mechanical maintenance, and narrowly scoped changes use only the rules that materially affect their correctness and reviewability.
|
||||
1. Identify the task domain (architecture / business logic / coding style / naming / comments / external interfaces / data / quality).
|
||||
2. Load the corresponding document from `docs/rules/`.
|
||||
3. Explicitly verify that your proposed solution does not violate the following three mandatory constraints:
|
||||
- **Naming Conventions (07):** Are all files, classes, functions, and constants named correctly?
|
||||
- **Architecture Boundaries (05):** Is the code placed in the correct layer? Are all call directions valid?
|
||||
- **Comment Standards (08):** Do all new public classes and methods include Chinese docstrings?
|
||||
|
||||
### Implementation Guidelines
|
||||
|
||||
* **Pattern Adherence:** Avoid generic boilerplate. If `04-design-patterns.md` defines a project-level pattern for a scenario, you are required to use it.
|
||||
* **Documentation Standards:** Docstring style for any new function or module must match `08-comment-styles.md`.
|
||||
* **Documentation Gate:** Public or cross-module contracts and non-obvious business behavior without useful Chinese documentation are rejected. Do not require comments that merely restate self-evident syntax.
|
||||
* **Command Reliance:** Prefer commands documented in `03-commands.md`; use other necessary standard commands with explicit, scoped arguments.
|
||||
* **⚠️ MANDATORY GATE:** Public classes, methods, and functions without proper Chinese docstrings are **REJECTED**. No exceptions.
|
||||
* **Command Reliance:** Only suggest commands listed in `03-commands.md`. Do not rely on inferred tool defaults.
|
||||
* **Minimal Change Principle:** Prefer the smallest correct change. Do not perform unrelated refactors, mass renames, or formatting-only cleanup.
|
||||
* **Output Language:** Summaries, validation results, and risk notes default to Chinese unless the user requests otherwise.
|
||||
|
||||
### Conflict Resolution
|
||||
|
||||
If existing code appears to contradict the documentation, identify the exact contradiction and decide which current-task gate it affects. Stop and ask only when it blocks acceptance, creates a security or data-safety ambiguity, or cannot be resolved from current source and maintained documentation. Otherwise preserve the evidence, continue unaffected work, and report the discrepancy without silently expanding scope.
|
||||
If existing code appears to contradict the documentation:
|
||||
|
||||
1. Stop implementation immediately.
|
||||
2. Identify the specific file and line of the contradiction.
|
||||
3. Prompt the user: "The documentation in `[File]` requires Pattern A, but the current implementation uses Pattern B. Which is the current standard?"
|
||||
|
||||
---
|
||||
|
||||
@@ -149,7 +95,6 @@ When modifying the following, you must also update the listed artifacts:
|
||||
| Database model schema | New Alembic migration under `database/versions/` |
|
||||
| User-visible config or init flow | Related docs, help text, setup/init flows, tests |
|
||||
| New skill | Follow `skills/<name>/SKILL.md` structure, keep YAML front matter |
|
||||
| Canonical module ownership or import path | `docs/rules/05-architecture.md`, `app/runtime/compat/manifest.py`, SDK exports when public, architecture/compatibility tests |
|
||||
|
||||
---
|
||||
|
||||
@@ -159,4 +104,4 @@ For the full documentation map and cross-references, refer to:
|
||||
|
||||
**[Documentation Hub Index](./docs/rules/README.md)**
|
||||
|
||||
*Last Updated: 2026-08-19*
|
||||
*Last Updated: 2026-05-25*
|
||||
|
||||
@@ -9,7 +9,6 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
基于 [NAStool](https://github.com/NAStool/nas-tools) 部分代码重新设计，聚焦自动化核心需求，减少问题同时更易于扩展和维护。
|
||||
@@ -27,7 +26,7 @@
|
||||
|
||||
## 安装使用
|
||||
|
||||
推荐优先使用 Docker 部署。V3 使用独立镜像 `jxxghp/moviepilot-v3`，V2 和旧版镜像保持原命名。Compose 示例、环境变量、目录映射和升级方式以官方 Wiki 为准：
|
||||
推荐优先使用 Docker 部署，常用镜像包括 `jxxghp/moviepilot-v2` 和 `jxxghp/moviepilot`。Compose 示例、环境变量、目录映射和升级方式以官方 Wiki 为准：
|
||||
|
||||
- 官方 Wiki：https://wiki.movie-pilot.org
|
||||
- PostgreSQL 部署说明：[docs/postgresql-setup.md](docs/postgresql-setup.md)
|
||||
@@ -35,7 +34,7 @@
|
||||
也可以使用本地 CLI 以源码模式安装和管理 MoviePilot：
|
||||
|
||||
```shell
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootstrap-local.sh | bash
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v2/scripts/bootstrap-local.sh | bash
|
||||
```
|
||||
|
||||
安装完成后使用 `moviepilot` 命令完成初始化、启动、停止、更新和配置查看。完整命令见 [docs/cli.md](docs/cli.md)。
|
||||
|
||||
+2
-3
@@ -9,7 +9,6 @@
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
Redesigned from parts of [NAStool](https://github.com/NAStool/nas-tools), with a stronger focus on core automation scenarios while reducing issues and making the project easier to extend and maintain.
|
||||
@@ -27,7 +26,7 @@ Release channel: https://t.me/moviepilot_channel
|
||||
|
||||
## Installation and Usage
|
||||
|
||||
Docker is the recommended deployment model. V3 uses the dedicated `jxxghp/moviepilot-v3` image; V2 and legacy images keep their existing names. Compose examples, environment variables, volume mappings, and upgrade notes are maintained in the official wiki:
|
||||
Docker is the recommended deployment model. Common images include `jxxghp/moviepilot-v2` and `jxxghp/moviepilot`. Compose examples, environment variables, volume mappings, and upgrade notes are maintained in the official wiki:
|
||||
|
||||
- Official wiki: https://wiki.movie-pilot.org
|
||||
- PostgreSQL setup: [docs/postgresql-setup.md](docs/postgresql-setup.md)
|
||||
@@ -35,7 +34,7 @@ Docker is the recommended deployment model. V3 uses the dedicated `jxxghp/moviep
|
||||
MoviePilot can also be installed and managed from source with the local CLI:
|
||||
|
||||
```shell
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v3/scripts/bootstrap-local.sh | bash
|
||||
curl -fsSL https://raw.githubusercontent.com/jxxghp/MoviePilot/v2/scripts/bootstrap-local.sh | bash
|
||||
```
|
||||
|
||||
After installation, use the `moviepilot` command for initialization, service management, updates, and configuration. See [docs/cli.md](docs/cli.md) for the full command reference.
|
||||
|
||||
+1
-10
@@ -1,7 +1,5 @@
|
||||
import warnings
|
||||
|
||||
from app.runtime.compat.imports import install_legacy_import_hook
|
||||
|
||||
|
||||
def _filter_third_party_startup_warnings() -> None:
|
||||
"""
|
||||
@@ -9,16 +7,9 @@ def _filter_third_party_startup_warnings() -> None:
|
||||
"""
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r"'_UnionGenericAlias' is deprecated and slated for removal in Python 3\.17",
|
||||
category=DeprecationWarning,
|
||||
module=r"google\.genai\.types",
|
||||
)
|
||||
warnings.filterwarnings(
|
||||
"ignore",
|
||||
message=r'"\\&" is an invalid escape sequence\..*',
|
||||
message=r"invalid escape sequence '\\&'",
|
||||
category=SyntaxWarning,
|
||||
)
|
||||
|
||||
|
||||
_filter_third_party_startup_warnings()
|
||||
install_legacy_import_hook()
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""MoviePilot 技术与外部系统适配器。"""
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
"""缓存持久化适配器。"""
|
||||
Vendored
-343
@@ -1,343 +0,0 @@
|
||||
import shutil
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, AsyncGenerator, Generator, Optional, Tuple
|
||||
|
||||
import aiofiles
|
||||
import aioshutil
|
||||
from anyio import Path as AsyncPath
|
||||
|
||||
from app.adapters.cache.redis import AsyncRedisHelper, RedisHelper
|
||||
from app.runtime.cache import (
|
||||
DEFAULT_CACHE_REGION,
|
||||
AsyncCacheBackend,
|
||||
CacheBackend,
|
||||
configure_cache_factories,
|
||||
)
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
class RedisBackend(CacheBackend):
|
||||
"""通过同步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = RedisHelper()
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""写入缓存，非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存键是否存在。"""
|
||||
return self.redis_helper.exists(key, region=region)
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取缓存值，不存在时返回空值。"""
|
||||
return self.redis_helper.get(key, region=region)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存键。"""
|
||||
self.redis_helper.delete(key, region=region)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或全部缓存。"""
|
||||
self.redis_helper.clear(region=region)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""遍历指定缓存区的键值对。"""
|
||||
return self.redis_helper.items(region=region)
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭同步 Redis 连接池。"""
|
||||
self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class AsyncRedisBackend(AsyncCacheBackend):
|
||||
"""通过异步 Redis 客户端实现缓存后端。"""
|
||||
|
||||
def __init__(self, ttl: Optional[int] = None) -> None:
|
||||
"""初始化异步 Redis 缓存并保存默认 TTL。"""
|
||||
self.ttl = ttl
|
||||
self.redis_helper = AsyncRedisHelper()
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
ttl: Optional[int] = None,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""异步写入缓存，非正 TTL 视为立即删除。"""
|
||||
ttl = self.ttl if ttl is None else ttl
|
||||
if ttl is not None and ttl <= 0:
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
return
|
||||
await self.redis_helper.set(key, value, ttl=ttl, region=region, **kwargs)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存键是否存在。"""
|
||||
return await self.redis_helper.exists(key, region=region)
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取缓存值，不存在时返回空值。"""
|
||||
return await self.redis_helper.get(key, region=region)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存键。"""
|
||||
await self.redis_helper.delete(key, region=region)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或全部缓存。"""
|
||||
await self.redis_helper.clear(region=region)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步遍历指定缓存区的键值对。"""
|
||||
async for item in self.redis_helper.items(region=region):
|
||||
yield item
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭异步 Redis 连接池。"""
|
||||
await self.redis_helper.close()
|
||||
|
||||
@staticmethod
|
||||
def is_redis() -> bool:
|
||||
"""标记当前后端为 Redis。"""
|
||||
return True
|
||||
|
||||
|
||||
class FileBackend(CacheBackend):
|
||||
"""通过本地文件系统保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""原子写入一个二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
tmp_file.write(value)
|
||||
temp_path = Path(tmp_file.name)
|
||||
temp_path.replace(cache_path)
|
||||
|
||||
def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""判断缓存文件是否存在。"""
|
||||
return (self.base / region / key).exists()
|
||||
|
||||
def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""读取二进制缓存文件。"""
|
||||
cache_path = self.base / region / key
|
||||
if not cache_path.exists():
|
||||
return None
|
||||
with cache_path.open("rb") as file_handle:
|
||||
return file_handle.read()
|
||||
|
||||
def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""删除缓存文件或缓存子目录。"""
|
||||
cache_path = self.base / region / key
|
||||
if cache_path.is_file():
|
||||
cache_path.unlink()
|
||||
elif cache_path.exists():
|
||||
shutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""清空指定缓存区或缓存根目录。"""
|
||||
cache_path = self.base / region if region else self.base
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in cache_path.iterdir():
|
||||
if item.is_file():
|
||||
item.unlink()
|
||||
else:
|
||||
shutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Generator[Tuple[str, Any], None, None]:
|
||||
"""按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = self.base / region
|
||||
if not cache_path.exists():
|
||||
return
|
||||
for item in sorted(cache_path.rglob("*")):
|
||||
if item.is_file():
|
||||
with item.open("rb") as file_handle:
|
||||
yield item.relative_to(cache_path).as_posix(), file_handle.read()
|
||||
|
||||
def close(self) -> None:
|
||||
"""文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
class AsyncFileBackend(AsyncCacheBackend):
|
||||
"""通过异步文件接口保存二进制缓存。"""
|
||||
|
||||
def __init__(self, base: Path) -> None:
|
||||
"""初始化异步缓存根目录。"""
|
||||
self.base = base
|
||||
self.base.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
async def set(
|
||||
self,
|
||||
key: str,
|
||||
value: Any,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
**_kwargs,
|
||||
) -> None:
|
||||
"""异步原子写入一个二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
await cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
async with aiofiles.tempfile.NamedTemporaryFile(
|
||||
dir=cache_path.parent,
|
||||
delete=False,
|
||||
) as tmp_file:
|
||||
await tmp_file.write(value)
|
||||
temp_path = AsyncPath(tmp_file.name)
|
||||
await temp_path.replace(cache_path)
|
||||
|
||||
async def exists(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> bool:
|
||||
"""异步判断缓存文件是否存在。"""
|
||||
return await (AsyncPath(self.base) / region / key).exists()
|
||||
|
||||
async def get(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> Optional[Any]:
|
||||
"""异步读取二进制缓存文件。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if not await cache_path.exists():
|
||||
return None
|
||||
async with aiofiles.open(cache_path, "rb") as file_handle:
|
||||
return await file_handle.read()
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
key: str,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> None:
|
||||
"""异步删除缓存文件或缓存子目录。"""
|
||||
cache_path = AsyncPath(self.base) / region / key
|
||||
if await cache_path.is_file():
|
||||
await cache_path.unlink()
|
||||
elif await cache_path.exists():
|
||||
await aioshutil.rmtree(cache_path, ignore_errors=True)
|
||||
|
||||
async def clear(self, region: Optional[str] = DEFAULT_CACHE_REGION) -> None:
|
||||
"""异步清空指定缓存区或缓存根目录。"""
|
||||
cache_path = AsyncPath(self.base) / region if region else AsyncPath(self.base)
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.iterdir():
|
||||
if await item.is_file():
|
||||
await item.unlink()
|
||||
else:
|
||||
await aioshutil.rmtree(item, ignore_errors=True)
|
||||
|
||||
async def items(
|
||||
self,
|
||||
region: Optional[str] = DEFAULT_CACHE_REGION,
|
||||
) -> AsyncGenerator[Tuple[str, Any], None]:
|
||||
"""异步按相对键遍历指定缓存区中的二进制文件。"""
|
||||
cache_path = AsyncPath(self.base) / region
|
||||
if not await cache_path.exists():
|
||||
return
|
||||
async for item in cache_path.rglob("*"):
|
||||
if await item.is_file():
|
||||
key = Path(str(item)).relative_to(Path(str(cache_path))).as_posix()
|
||||
async with aiofiles.open(item, "rb") as file_handle:
|
||||
yield key, await file_handle.read()
|
||||
|
||||
async def close(self) -> None:
|
||||
"""异步文件缓存没有需要关闭的持久连接。"""
|
||||
|
||||
|
||||
def configure_platform_cache() -> None:
|
||||
"""把配置感知的 Redis 与文件适配器注册到平台缓存工厂。"""
|
||||
configure_cache_factories(
|
||||
backend_type_provider=lambda: get_runtime_setting('CACHE_BACKEND_TYPE'),
|
||||
redis_factory=lambda ttl: RedisBackend(ttl=ttl),
|
||||
async_redis_factory=lambda ttl: AsyncRedisBackend(ttl=ttl),
|
||||
file_factory=lambda base: FileBackend(
|
||||
base=base or get_runtime_setting('TEMP_PATH')
|
||||
),
|
||||
async_file_factory=lambda base: AsyncFileBackend(
|
||||
base=base or get_runtime_setting('TEMP_PATH')
|
||||
),
|
||||
file_ttl_provider=lambda: get_runtime_setting('TEMP_FILE_DAYS') * 24 * 3600,
|
||||
)
|
||||
Vendored
-1
@@ -1 +0,0 @@
|
||||
"""插件市场、CookieCloud、OCR 和远程 MoviePilot 服务集成。"""
|
||||
-1
@@ -1 +0,0 @@
|
||||
"""插件市场外部适配器。"""
|
||||
-124
@@ -1,124 +0,0 @@
|
||||
"""插件市场查询客户端。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from app.adapters.external.market import PluginHelper as _PluginHelper
|
||||
from app.runtime.cache import async_fresh, fresh
|
||||
|
||||
|
||||
class PluginMarketClient:
|
||||
"""把插件市场、版本元数据和本地仓库查询隔离为只读客户端。"""
|
||||
|
||||
def __init__(self, helper: Optional[_PluginHelper] = None) -> None:
|
||||
"""复用旧 PluginHelper 实现，保持缓存和弱单例身份不变。"""
|
||||
self._helper = helper or _PluginHelper()
|
||||
|
||||
def get_plugins(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict[str, Any]]]:
|
||||
"""同步读取指定仓库和代际的插件索引。"""
|
||||
with fresh(force):
|
||||
return self._helper.get_plugins(repo_url, package_version)
|
||||
|
||||
async def async_get_plugins(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict]]:
|
||||
"""异步读取指定仓库和代际的插件索引。"""
|
||||
async with async_fresh(force):
|
||||
return await self._helper.async_get_plugins(repo_url, package_version)
|
||||
|
||||
def get_plugin_index_result(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict]]:
|
||||
"""读取插件索引的三态结果，供库存读取保留失败事实。"""
|
||||
with fresh(force):
|
||||
return cast(
|
||||
Optional[dict[str, dict[str, Any]]],
|
||||
self._helper.get_plugin_index_result(repo_url, package_version),
|
||||
)
|
||||
|
||||
async def async_get_plugin_index_result(
|
||||
self,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Optional[dict[str, dict[str, Any]]]:
|
||||
"""异步读取插件索引的三态结果，供库存读取保留失败事实。"""
|
||||
async with async_fresh(force):
|
||||
return cast(
|
||||
Optional[dict[str, dict[str, Any]]],
|
||||
await self._helper.async_get_plugin_index_result(
|
||||
repo_url,
|
||||
package_version,
|
||||
),
|
||||
)
|
||||
|
||||
def get_local_candidates(self) -> dict[str, dict]:
|
||||
"""返回全部本地插件仓库候选。"""
|
||||
return self._helper.get_local_plugin_candidates()
|
||||
|
||||
def get_local_candidate(
|
||||
self,
|
||||
plugin_id: str,
|
||||
package_version: Optional[str] = None,
|
||||
repo_path: Optional[Path] = None,
|
||||
**kwargs: Any,
|
||||
) -> Optional[dict]:
|
||||
"""返回指定插件的本地仓库候选。"""
|
||||
return self._helper.get_local_plugin_candidate(
|
||||
pid=plugin_id,
|
||||
package_version=package_version,
|
||||
repo_path=repo_path,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_local_repo_paths() -> list[Path]:
|
||||
"""返回配置中有效的本地插件仓库目录。"""
|
||||
return _PluginHelper.get_local_repo_paths()
|
||||
|
||||
@staticmethod
|
||||
def make_local_repo_url(
|
||||
plugin_id: str,
|
||||
repo_path: Optional[object] = None,
|
||||
package_version: Optional[str] = None,
|
||||
) -> str:
|
||||
"""生成兼容旧入口的本地插件来源标识。"""
|
||||
return _PluginHelper.make_local_repo_url(
|
||||
plugin_id,
|
||||
repo_path,
|
||||
package_version,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_local_repo_url(repo_url: Optional[str]) -> bool:
|
||||
"""判断插件来源是否为本地仓库标识。"""
|
||||
return _PluginHelper.is_local_repo_url(repo_url)
|
||||
|
||||
@staticmethod
|
||||
def annotate_system_version(plugin_info: dict) -> dict:
|
||||
"""补充插件所需 MoviePilot 版本兼容状态。"""
|
||||
return _PluginHelper.annotate_plugin_system_version(plugin_info)
|
||||
|
||||
@staticmethod
|
||||
def is_package_compatible(
|
||||
plugin_info: dict,
|
||||
package_version: Optional[str],
|
||||
) -> bool:
|
||||
"""判断插件条目是否兼容目标插件包代际。"""
|
||||
return _PluginHelper.is_package_plugin_compatible(
|
||||
plugin_info,
|
||||
package_version,
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
"""通用网络与 Web 协议适配器。"""
|
||||
@@ -1 +0,0 @@
|
||||
"""运行观测导出器适配器。"""
|
||||
@@ -1,49 +0,0 @@
|
||||
"""可选 OpenTelemetry metrics adapter。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from typing import Any, Mapping
|
||||
|
||||
from app.runtime.observability import MetricKind, MetricSpec, NoopObservationPort, ObservationPort
|
||||
|
||||
|
||||
class OpenTelemetryObservationPort:
|
||||
"""把内部指标合同映射到可选安装的 OpenTelemetry Metrics API。"""
|
||||
|
||||
def __init__(self, meter: Any) -> None:
|
||||
"""保存 meter，并按名称惰性创建 instrument。"""
|
||||
self._meter = meter
|
||||
self._instruments: dict[str, Any] = {}
|
||||
|
||||
def record(self, spec: MetricSpec, value: float, labels: Mapping[str, str]) -> None:
|
||||
"""按合同类型使用 OTel counter、histogram 或 up/down counter。"""
|
||||
instrument = self._instruments.get(spec.name)
|
||||
if instrument is None:
|
||||
instrument = self._create_instrument(spec)
|
||||
self._instruments[spec.name] = instrument
|
||||
if spec.kind == MetricKind.HISTOGRAM:
|
||||
instrument.record(value, attributes=dict(labels))
|
||||
else:
|
||||
instrument.add(value, attributes=dict(labels))
|
||||
|
||||
def _create_instrument(self, spec: MetricSpec) -> Any:
|
||||
"""为内部指标类型创建对应 OTel instrument。"""
|
||||
normalized = spec.name.replace(".", "_")
|
||||
if spec.kind == MetricKind.HISTOGRAM:
|
||||
return self._meter.create_histogram(normalized)
|
||||
if spec.kind == MetricKind.COUNTER:
|
||||
return self._meter.create_counter(normalized)
|
||||
return self._meter.create_up_down_counter(normalized)
|
||||
|
||||
|
||||
def build_observation_port() -> ObservationPort:
|
||||
"""仅在显式启用且 API 可导入时创建 OTel adapter，否则返回 no-op。"""
|
||||
if os.getenv("MOVIEPILOT_OTEL_METRICS") != "1":
|
||||
return NoopObservationPort()
|
||||
try:
|
||||
metrics = importlib.import_module("opentelemetry.metrics")
|
||||
except ImportError:
|
||||
return NoopObservationPort()
|
||||
return OpenTelemetryObservationPort(metrics.get_meter("moviepilot"))
|
||||
@@ -1 +0,0 @@
|
||||
"""操作系统、进程与运行资源适配器。"""
|
||||
@@ -1 +0,0 @@
|
||||
"""数据库备份的文件系统与数据库技术适配器命名空间。"""
|
||||
@@ -1,260 +0,0 @@
|
||||
"""基于活动 SQLAlchemy 引擎的 SQLite 与 PostgreSQL 备份实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
from contextlib import closing
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Callable, Mapping, Protocol, Sequence
|
||||
|
||||
from sqlalchemy.engine import Engine
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DatabaseBackupCheck:
|
||||
"""数据库适配器返回的基础校验结果。"""
|
||||
|
||||
valid: bool
|
||||
method: str
|
||||
detail: str | None = None
|
||||
|
||||
|
||||
class ProcessResult(Protocol):
|
||||
"""数据库命令执行结果的最小合同。"""
|
||||
|
||||
returncode: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
|
||||
|
||||
class ProcessRunner(Protocol):
|
||||
"""可替换的数据库命令执行边界。"""
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
command: Sequence[str],
|
||||
*,
|
||||
env: Mapping[str, str],
|
||||
capture_output: bool,
|
||||
text: bool,
|
||||
check: bool,
|
||||
) -> ProcessResult:
|
||||
"""执行命令并返回结果。"""
|
||||
|
||||
|
||||
def verify_database_backup(
|
||||
artifact: Path,
|
||||
*,
|
||||
db_type: str,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> DatabaseBackupCheck:
|
||||
"""在不访问活动数据库的前提下校验一个受管备份文件。"""
|
||||
if db_type == "sqlite":
|
||||
method = "PRAGMA integrity_check"
|
||||
try:
|
||||
# 正式备份不会再变化，immutable 可避免只读校验创建 WAL 旁路文件。
|
||||
uri = f"{artifact.resolve().as_uri()}?mode=ro&immutable=1"
|
||||
with closing(sqlite3.connect(uri, uri=True)) as connection:
|
||||
rows = connection.execute("PRAGMA integrity_check").fetchall()
|
||||
except sqlite3.Error as error:
|
||||
return DatabaseBackupCheck(False, method, str(error))
|
||||
valid = bool(rows) and all(row[0] == "ok" for row in rows)
|
||||
detail = None if valid else "; ".join(str(row[0]) for row in rows)
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
if db_type == "postgresql":
|
||||
method = "pg_restore --list"
|
||||
executable = _require_tool(pg_restore, tool_resolver)
|
||||
result = runner(
|
||||
[executable, "--list", str(artifact)],
|
||||
env=_postgres_environment(),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
valid = result.returncode == 0 and bool(result.stdout.strip())
|
||||
detail = None if valid else f"pg_restore 退出码 {result.returncode}"
|
||||
return DatabaseBackupCheck(valid, method, detail)
|
||||
|
||||
raise ValueError(f"不支持的数据库备份类型：{db_type}")
|
||||
|
||||
|
||||
def _require_tool(
|
||||
executable: str,
|
||||
tool_resolver: Callable[[str], str | None],
|
||||
) -> str:
|
||||
resolved = tool_resolver(executable)
|
||||
if resolved is None:
|
||||
raise RuntimeError(
|
||||
f"未找到 {executable}，请安装与服务端同主版本或更高的 "
|
||||
"PostgreSQL client 并加入 PATH"
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def _postgres_environment(
|
||||
*,
|
||||
password: str | None = None,
|
||||
sslmode: str | None = None,
|
||||
) -> dict[str, str]:
|
||||
environment = dict(os.environ)
|
||||
environment.pop("PGPASSWORD", None)
|
||||
environment.pop("PGSSLMODE", None)
|
||||
if password:
|
||||
environment["PGPASSWORD"] = password
|
||||
if sslmode:
|
||||
environment["PGSSLMODE"] = sslmode
|
||||
return environment
|
||||
|
||||
|
||||
class SQLiteBackupBackend:
|
||||
"""使用 SQLite 在线备份 API 管理活动文件数据库。"""
|
||||
|
||||
db_type = "sqlite"
|
||||
suffix = ".db"
|
||||
|
||||
def __init__(self, engine: Engine) -> None:
|
||||
self._engine = engine
|
||||
database = engine.url.database
|
||||
if not database or database == ":memory:":
|
||||
raise ValueError("SQLite 内存数据库不支持文件备份")
|
||||
self._database = Path(database)
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""从活动引擎指向的 SQLite 文件创建一致快照。"""
|
||||
source = self._engine.raw_connection()
|
||||
try:
|
||||
with closing(sqlite3.connect(destination)) as target:
|
||||
source.driver_connection.backup(target)
|
||||
target.commit()
|
||||
finally:
|
||||
source.close()
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 SQLite integrity_check 校验备份内容。"""
|
||||
return verify_database_backup(artifact, db_type=self.db_type)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中原子替换活动 SQLite 文件。"""
|
||||
temporary = self._database.with_name(f".{self._database.name}.restore")
|
||||
self._engine.dispose()
|
||||
try:
|
||||
shutil.copy2(artifact, temporary)
|
||||
temporary.chmod(0o600)
|
||||
self._database.with_name(f"{self._database.name}-wal").unlink(missing_ok=True)
|
||||
self._database.with_name(f"{self._database.name}-shm").unlink(missing_ok=True)
|
||||
os.replace(temporary, self._database)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
class PostgreSQLBackupBackend:
|
||||
"""使用 pg_dump 与 pg_restore 管理活动 PostgreSQL 数据库。"""
|
||||
|
||||
db_type = "postgresql"
|
||||
suffix = ".dump"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Engine,
|
||||
*,
|
||||
runner: ProcessRunner = subprocess.run,
|
||||
tool_resolver: Callable[[str], str | None] = shutil.which,
|
||||
pg_dump: str = "pg_dump",
|
||||
pg_restore: str = "pg_restore",
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._runner = runner
|
||||
self._tool_resolver = tool_resolver
|
||||
self._pg_dump = pg_dump
|
||||
self._pg_restore = pg_restore
|
||||
|
||||
def create(self, destination: Path) -> None:
|
||||
"""创建 PostgreSQL custom-format 在线备份。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_dump),
|
||||
"--format=custom",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--file",
|
||||
str(destination),
|
||||
*self._connection_arguments(),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_dump 执行失败，退出码 {result.returncode}")
|
||||
if not destination.is_file() or destination.stat().st_size == 0:
|
||||
raise RuntimeError("pg_dump 未生成有效的备份文件")
|
||||
|
||||
def verify(self, artifact: Path) -> DatabaseBackupCheck:
|
||||
"""通过 pg_restore 目录读取校验 custom-format 归档。"""
|
||||
return verify_database_backup(
|
||||
artifact,
|
||||
db_type=self.db_type,
|
||||
runner=self._runner,
|
||||
tool_resolver=self._tool_resolver,
|
||||
pg_restore=self._pg_restore,
|
||||
)
|
||||
|
||||
def restore(self, artifact: Path) -> None:
|
||||
"""在 CLI 离线进程中覆盖当前 PostgreSQL 数据库内容。"""
|
||||
command = [
|
||||
self._require_tool(self._pg_restore),
|
||||
"--clean",
|
||||
"--if-exists",
|
||||
"--no-owner",
|
||||
"--no-acl",
|
||||
"--single-transaction",
|
||||
"--exit-on-error",
|
||||
*self._connection_arguments(),
|
||||
str(artifact),
|
||||
]
|
||||
result = self._run(command, include_password=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"pg_restore 执行失败，退出码 {result.returncode}")
|
||||
|
||||
def _connection_arguments(self) -> list[str]:
|
||||
url = self._engine.url
|
||||
host = str(url.query.get("host") or url.host or "")
|
||||
port = str(url.query.get("port") or url.port or "")
|
||||
arguments = [
|
||||
"--username",
|
||||
str(url.username or ""),
|
||||
"--dbname",
|
||||
str(url.database or ""),
|
||||
]
|
||||
if host:
|
||||
arguments.extend(["--host", host])
|
||||
if port:
|
||||
arguments.extend(["--port", port])
|
||||
return arguments
|
||||
|
||||
def _run(self, command: Sequence[str], *, include_password: bool) -> ProcessResult:
|
||||
return self._runner(
|
||||
command,
|
||||
env=self._environment(include_password=include_password),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def _require_tool(self, executable: str) -> str:
|
||||
return _require_tool(executable, self._tool_resolver)
|
||||
|
||||
def _environment(self, *, include_password: bool) -> dict[str, str]:
|
||||
password = (
|
||||
str(self._engine.url.password)
|
||||
if include_password and self._engine.url.password
|
||||
else None
|
||||
)
|
||||
sslmode = self._engine.url.query.get("sslmode")
|
||||
return _postgres_environment(
|
||||
password=password,
|
||||
sslmode=str(sslmode) if sslmode else None,
|
||||
)
|
||||
@@ -1,128 +0,0 @@
|
||||
"""数据库备份单文件的受限文件系统操作。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.runtime.version import get_app_version
|
||||
|
||||
_BACKUP_NAME = re.compile(
|
||||
r"^(?:moviepilot_(?P<version>v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?)_)?"
|
||||
r"(?P<db_type>sqlite|postgresql)_"
|
||||
r"(?P<timestamp>\d{8}_\d{6})"
|
||||
r"(?:_(?P<sequence>\d+))?"
|
||||
r"(?P<suffix>\.db|\.dump)$"
|
||||
)
|
||||
_RELEASE_VERSION = re.compile(r"^v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$")
|
||||
|
||||
|
||||
class BackupFiles:
|
||||
"""把备份文件操作限制在一个私有根目录内。"""
|
||||
|
||||
def __init__(self, root: Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def create_temporary(self, suffix: str) -> Path:
|
||||
"""在最终目录内创建私有临时文件，保证发布可使用原子替换。"""
|
||||
self._ensure_root()
|
||||
descriptor, filename = tempfile.mkstemp(
|
||||
prefix=".database-",
|
||||
suffix=f"{suffix}.partial",
|
||||
dir=self.root,
|
||||
)
|
||||
os.close(descriptor)
|
||||
path = Path(filename)
|
||||
path.chmod(0o600)
|
||||
return path
|
||||
|
||||
def publish(self, temporary: Path, name: str) -> Path:
|
||||
"""把已校验临时文件发布为正式备份文件。"""
|
||||
destination = self._resolve_name(name, require_exists=False)
|
||||
os.replace(temporary, destination)
|
||||
destination.chmod(0o600)
|
||||
return destination
|
||||
|
||||
def discard(self, temporary: Path) -> None:
|
||||
"""清理本次操作拥有的未发布临时文件。"""
|
||||
path = Path(temporary)
|
||||
if path.parent == self.root and path.name.startswith(".database-"):
|
||||
path.unlink(missing_ok=True)
|
||||
|
||||
def list(self) -> list[Path]:
|
||||
"""返回当前根目录内格式合法的正式备份文件。"""
|
||||
if not self.root.is_dir():
|
||||
return []
|
||||
paths = [
|
||||
path
|
||||
for path in self.root.iterdir()
|
||||
if path.is_file() and _BACKUP_NAME.fullmatch(path.name)
|
||||
]
|
||||
return sorted(paths, key=lambda path: (self.created_at(path.name), path.name), reverse=True)
|
||||
|
||||
def resolve(self, name: str) -> Path:
|
||||
"""按受限文件名解析一个必须存在的备份文件。"""
|
||||
return self._resolve_name(name, require_exists=True)
|
||||
|
||||
def delete(self, name: str) -> None:
|
||||
"""删除一个已通过名称约束的备份文件。"""
|
||||
self.resolve(name).unlink()
|
||||
|
||||
def available_name(
|
||||
self,
|
||||
*,
|
||||
db_type: str,
|
||||
created_at: datetime,
|
||||
suffix: str,
|
||||
) -> str:
|
||||
"""生成包含数据库类型和秒级时间的简短可读文件名。"""
|
||||
timestamp = created_at.strftime("%Y%m%d_%H%M%S")
|
||||
version = get_app_version().strip()
|
||||
if _RELEASE_VERSION.fullmatch(version) is None:
|
||||
raise ValueError("程序版本号无法用于数据库备份命名")
|
||||
base = f"moviepilot_{version}_{db_type}_{timestamp}"
|
||||
candidate = f"{base}{suffix}"
|
||||
sequence = 1
|
||||
while (self.root / candidate).exists():
|
||||
candidate = f"{base}_{sequence}{suffix}"
|
||||
sequence += 1
|
||||
if not _BACKUP_NAME.fullmatch(candidate):
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def database_type(name: str) -> str:
|
||||
"""从受管文件名读取数据库类型。"""
|
||||
return BackupFiles._match(name).group("db_type")
|
||||
|
||||
@staticmethod
|
||||
def created_at(name: str) -> datetime:
|
||||
"""从受管文件名读取本地创建时间。"""
|
||||
return datetime.strptime(
|
||||
BackupFiles._match(name).group("timestamp"),
|
||||
"%Y%m%d_%H%M%S",
|
||||
)
|
||||
|
||||
def _ensure_root(self) -> None:
|
||||
self.root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.root.chmod(0o700)
|
||||
|
||||
def _resolve_name(self, name: str, *, require_exists: bool) -> Path:
|
||||
normalized = str(name).strip()
|
||||
self._match(normalized)
|
||||
if Path(normalized).name != normalized:
|
||||
raise ValueError("数据库备份文件名不能包含路径")
|
||||
path = self.root / normalized
|
||||
if require_exists and not path.is_file():
|
||||
raise FileNotFoundError(normalized)
|
||||
return path
|
||||
|
||||
@staticmethod
|
||||
def _match(name: str) -> re.Match[str]:
|
||||
matched = _BACKUP_NAME.fullmatch(str(name))
|
||||
if matched is None:
|
||||
raise ValueError("数据库备份文件名无效")
|
||||
return matched
|
||||
@@ -1,53 +0,0 @@
|
||||
"""虚拟显示适配器及旧 DisplayHelper 兼容入口。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.managed_resources import (
|
||||
acquire_managed_resource,
|
||||
stop_managed_resource,
|
||||
)
|
||||
|
||||
|
||||
DISPLAY_CAPABILITY_ID = "host.display"
|
||||
|
||||
|
||||
class DisplayHelper(metaclass=Singleton):
|
||||
"""保留旧构造 API，并把资源所有权委托给 host.display 能力。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""显式构造旧门面时激活虚拟显示，失败保持旧 API 的日志语义。"""
|
||||
try:
|
||||
acquire_managed_resource(
|
||||
DISPLAY_CAPABILITY_ID,
|
||||
reason="legacy_display_helper",
|
||||
retry=True,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.error("DisplayHelper init error: %s", error)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止已激活的虚拟显示；未配置 Runtime 时保持幂等。"""
|
||||
stop_managed_resource(
|
||||
DISPLAY_CAPABILITY_ID,
|
||||
reason="legacy_display_helper_stop",
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["DISPLAY_CAPABILITY_ID", "DisplayHelper", "VirtualDisplayResource"]
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""按需公开资源实现，普通兼容导入不加载显示后端。"""
|
||||
if name != "VirtualDisplayResource":
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
value = getattr(
|
||||
import_module("app.adapters.system.display.resource"),
|
||||
"VirtualDisplayResource",
|
||||
)
|
||||
globals()[name] = value
|
||||
return value
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "host.display"
|
||||
kind = "managed_resource.sync"
|
||||
entrypoint = "app.adapters.system.display.resource:VirtualDisplayResource"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Virtual Display"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,45 +0,0 @@
|
||||
"""虚拟显示进程的托管资源实现。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
class VirtualDisplayResource:
|
||||
"""按需拥有一个容器内虚拟显示进程。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._display: Optional[Any] = None
|
||||
|
||||
@property
|
||||
def display(self) -> Optional[Any]:
|
||||
"""返回当前拥有的显示对象；未启动或已停止时为 None。"""
|
||||
return self._display
|
||||
|
||||
def start(self) -> None:
|
||||
"""仅在容器环境启动虚拟显示，重复启动保持幂等。"""
|
||||
if self._display is not None or not SystemUtils.is_docker():
|
||||
return
|
||||
from pyvirtualdisplay import Display
|
||||
|
||||
display = Display(
|
||||
visible=False,
|
||||
size=(1024, 768),
|
||||
extra_args=[os.environ["DISPLAY"]],
|
||||
)
|
||||
self._display = display
|
||||
display.start()
|
||||
|
||||
def stop(self) -> None:
|
||||
"""停止当前资源拥有的显示进程，失败时保留句柄供 Runtime 重试。"""
|
||||
display = self._display
|
||||
if display is None:
|
||||
return
|
||||
logger.info("正在停止虚拟显示...")
|
||||
display.stop()
|
||||
self._display = None
|
||||
logger.info("虚拟显示已停止")
|
||||
@@ -1,483 +0,0 @@
|
||||
"""
|
||||
本地文件系统操作代理。
|
||||
|
||||
FUSE/网络挂载有两种故障形态：crash 型（调用抛错，可捕获、可重试）和 block 型
|
||||
（调用既不返回错误也不返回结果，永久悬挂）。**Python 无法中断一个已经发出的
|
||||
系统调用，也无法强杀线程**，所以 block 型故障下阻塞的线程永远无法回收——这正是
|
||||
整理消费线程停摆、监控自愈路径自冻的根因。
|
||||
|
||||
本模块把这些调用放进一个常驻子进程执行。子进程可以被 SIGKILL，因此超时后能真正
|
||||
回收；对调用方而言，超时表现为一个普通的 OSError 子类（FileSystemTimeout）。
|
||||
换句话说：**把不可处理的 block 型故障，转换成系统各层已经能正确处理的 crash 型
|
||||
故障**——退避重启、登记待重试这些既有机制立刻就能接管。
|
||||
|
||||
第一版只放行安全的操作：
|
||||
- 只读（stat/exists/listdir）——强杀不产生任何副作用
|
||||
- 同存储 rename——内核保证原子性，强杀后要么完全成功要么完全没发生
|
||||
跨存储的复制+删除不在此列，它需要单独的可恢复语义（临时名 + 完成后 rename）。
|
||||
"""
|
||||
import errno as errno_module
|
||||
import json
|
||||
import os
|
||||
import selectors
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
# worker 脚本路径。用绝对路径直接执行，而不是 -m 或 import：
|
||||
# 直接执行文件不会触发 app/__init__.py 的导入链，代理启动才是毫秒级的
|
||||
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
|
||||
# 单次快操作（stat/listdir/rename/unlink 等）的默认超时秒数
|
||||
DEFAULT_TIMEOUT = 30
|
||||
# 长耗时操作（复制）两次进度上报之间的最长间隔秒数。
|
||||
# worker 每秒上报一次心跳，因此这个阈值判定的是「传输完全没有推进」，
|
||||
# 而不是「传输很慢」——大文件复制几小时也不会误杀
|
||||
DEFAULT_STALL_TIMEOUT = 120
|
||||
# 强杀代理后等待它消失的宽限秒数，不能无限等待
|
||||
_KILL_GRACE = 5
|
||||
|
||||
|
||||
class FileSystemTimeout(OSError):
|
||||
"""
|
||||
文件系统操作在代理中超时未返回，判定挂载无响应。
|
||||
|
||||
继承 OSError 是刻意的：整理链、监控 watcher 等各层对 OSError 已有完整的
|
||||
退避重试与登记逻辑，block 型故障经此转换后可以直接复用它们。
|
||||
"""
|
||||
|
||||
|
||||
class FileSystemProxy:
|
||||
"""
|
||||
常驻子进程文件系统代理。
|
||||
|
||||
请求-响应严格串行（一个代理同时只处理一个请求），由锁保证。超时即强杀代理，
|
||||
下一次请求自动重启一个新的——启动成本是毫秒级，因为 worker 只依赖标准库。
|
||||
"""
|
||||
|
||||
def __init__(self, timeout: Optional[float] = None,
|
||||
stall_timeout: Optional[float] = None):
|
||||
"""
|
||||
:param timeout: 单次快操作的超时秒数，None 表示实时跟随系统设置
|
||||
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数，
|
||||
None 表示实时跟随系统设置
|
||||
"""
|
||||
self._timeout_override = timeout
|
||||
self._stall_timeout_override = stall_timeout
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
self._selector: Optional[selectors.BaseSelector] = None
|
||||
self._lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 对外操作
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
def stat(self, path: Path) -> Dict[str, Any]:
|
||||
"""
|
||||
读取路径属性。
|
||||
:param path: 目标路径
|
||||
:return: {"size", "mtime", "is_dir", "is_file"}
|
||||
"""
|
||||
return self._call("stat", path=str(path))
|
||||
|
||||
def exists(self, path: Path) -> bool:
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
只有 FileNotFoundError 才算「不存在」；其余 OSError（含超时）原样抛出，
|
||||
避免像 Path.exists() 那样把挂载抖动误判成文件消失。
|
||||
:param path: 目标路径
|
||||
:return: 是否存在
|
||||
"""
|
||||
try:
|
||||
self._call("exists", path=str(path))
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def listdir(self, path: Path) -> List[str]:
|
||||
"""
|
||||
列出目录条目名。
|
||||
:param path: 目标目录
|
||||
:return: 条目名列表
|
||||
"""
|
||||
return self._call("listdir", path=str(path))
|
||||
|
||||
def count_entries(self, path: Path, max_check: int = 10000) -> Dict[str, int]:
|
||||
"""
|
||||
统计目录规模。整棵树的遍历在子进程内一次完成，超时可整体放弃。
|
||||
:param path: 目标目录
|
||||
:param max_check: 文件数上限，超过即提前结束
|
||||
:return: {"file_count", "dir_count"}
|
||||
"""
|
||||
return self._call("count_entries", path=str(path), max_check=max_check)
|
||||
|
||||
def rename(self, src: Path, dst: Path) -> bool:
|
||||
"""
|
||||
同一存储内重命名/移动。跨存储会抛 OSError(EXDEV)，由调用方走原有路径。
|
||||
:param src: 源路径
|
||||
:param dst: 目标路径
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rename", src=str(src), dst=str(dst))
|
||||
|
||||
def copy(self, src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]] = None,
|
||||
cancel_cb: Optional[Callable[[], bool]] = None,
|
||||
chunk_size: Optional[int] = None) -> Any:
|
||||
"""
|
||||
复制文件内容并保留时间戳，按「进度无推进」判定挂死。
|
||||
|
||||
复制大文件可能持续几小时，固定超时无法区分「正常但慢」和「已经挂死」。
|
||||
worker 每秒上报一次进度作为心跳，这里判定的是**两次上报之间的间隔**：
|
||||
超过 stall 阈值收不到任何一行，才认定挂载无响应并强杀 worker。
|
||||
|
||||
取消检查放在父进程：worker 里读不到 global_vars 的传输取消标记，而父进程
|
||||
每收到一次进度就能检查一次，要取消直接杀掉 worker 即可，比在子进程里
|
||||
轮询标记更干净。
|
||||
:param src: 源文件
|
||||
:param dst: 目标文件（调用方应传临时名，完成后自行原子替换）
|
||||
:param progress_cb: 进度回调，入参为百分比
|
||||
:param cancel_cb: 取消检查回调，返回 True 表示应中止
|
||||
:param chunk_size: 分块大小
|
||||
:return: 成功时为 {"copied", "total"}，被取消或通信失败时为 False
|
||||
"""
|
||||
payload = {"src": str(src), "dst": str(dst)}
|
||||
if chunk_size:
|
||||
payload["chunk_size"] = chunk_size
|
||||
if not self._enabled():
|
||||
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request_stream(payload, progress_cb, cancel_cb)
|
||||
except FileSystemTimeout:
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
|
||||
self._shutdown()
|
||||
return False
|
||||
|
||||
def _request_stream(self, payload: Dict[str, Any],
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]]) -> Any:
|
||||
"""
|
||||
发起一次流式请求，逐行消费进度直到终态。
|
||||
:param payload: 请求参数
|
||||
:param progress_cb: 进度回调
|
||||
:param cancel_cb: 取消检查回调
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": "copy", **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
while True:
|
||||
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
|
||||
progress = response.get("progress")
|
||||
if progress is not None:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
logger.info(f"复制已取消: {payload.get('src')}")
|
||||
# 取消就地生效：杀掉 worker 立刻中断传输，不必等它读完整个文件
|
||||
self._shutdown()
|
||||
return False
|
||||
if progress_cb is not None:
|
||||
total = progress.get("total") or 0
|
||||
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
|
||||
continue
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
@staticmethod
|
||||
def _direct_copy(src: Path, dst: Path,
|
||||
progress_cb: Optional[Callable[[float], None]],
|
||||
cancel_cb: Optional[Callable[[], bool]],
|
||||
chunk_size: Optional[int]) -> bool:
|
||||
"""
|
||||
不经代理直接复制，供代理关闭时使用。
|
||||
"""
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_cb is not None and cancel_cb():
|
||||
return False
|
||||
buf = fsrc.read(chunk_size or 1024 * 1024)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(copied / total * 100)
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return True
|
||||
|
||||
def unlink(self, path: Path) -> bool:
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作，强杀后没有中间状态。
|
||||
:param path: 目标文件
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("unlink", path=str(path))
|
||||
|
||||
def rmtree(self, path: Path) -> bool:
|
||||
"""
|
||||
递归删除目录，容忍部分失败（可重复执行直到成功）。
|
||||
:param path: 目标目录
|
||||
:return: 是否成功
|
||||
"""
|
||||
return self._call("rmtree", path=str(path))
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
关闭代理进程。
|
||||
"""
|
||||
with self._lock:
|
||||
self._shutdown()
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# 内部实现
|
||||
# ------------------------------------------------------------------ #
|
||||
|
||||
@property
|
||||
def _timeout(self) -> float:
|
||||
"""
|
||||
单次快操作的超时秒数。
|
||||
|
||||
实时读取而不是构造时固定：这三项都暴露在前端设置里，用户改完保存后
|
||||
必须立刻生效，否则会出现「改了没反应」的困惑。
|
||||
"""
|
||||
if self._timeout_override is not None:
|
||||
return self._timeout_override
|
||||
return float(get_runtime_setting("FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
|
||||
|
||||
@property
|
||||
def _stall_timeout(self) -> float:
|
||||
"""
|
||||
长耗时操作两次进度上报之间的最长间隔秒数，同样实时跟随系统设置。
|
||||
"""
|
||||
if self._stall_timeout_override is not None:
|
||||
return self._stall_timeout_override
|
||||
return float(
|
||||
get_runtime_setting("FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _enabled() -> bool:
|
||||
"""
|
||||
代理是否启用。关闭时退回直接调用，行为与引入代理之前完全一致。
|
||||
"""
|
||||
return bool(get_runtime_setting("FS_PROXY_ENABLED", True))
|
||||
|
||||
@staticmethod
|
||||
def _direct(op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
不经代理直接执行操作，供代理关闭时使用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if op == "stat":
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
if op == "exists":
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
if op == "listdir":
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
if op == "count_entries":
|
||||
file_count = dir_count = 0
|
||||
for _, dirs, files in os.walk(payload["path"]):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > (payload.get("max_check") or 10000):
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
if op == "rename":
|
||||
os.rename(payload["src"], payload["dst"])
|
||||
return True
|
||||
if op == "unlink":
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
if op == "rmtree":
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
raise ValueError(f"unknown op: {op}")
|
||||
|
||||
def _call(self, op: str, **payload) -> Any:
|
||||
"""
|
||||
执行一次代理调用。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
if not self._enabled():
|
||||
return self._direct(op, payload)
|
||||
with self._lock:
|
||||
try:
|
||||
return self._request(op, payload)
|
||||
except FileSystemTimeout:
|
||||
# 超时说明挂载正在挂死，重试只会再冻一次，直接上报给调用方
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||
# 代理进程意外退出或响应损坏，重启后重试一次
|
||||
logger.debug(f"文件系统代理通信异常，重启后重试: {op} - {err}")
|
||||
self._shutdown()
|
||||
return self._request(op, payload)
|
||||
|
||||
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
|
||||
"""
|
||||
发送请求并等待响应。
|
||||
:param op: 操作名
|
||||
:param payload: 操作参数
|
||||
:return: 操作结果
|
||||
"""
|
||||
self._ensure_worker()
|
||||
message = json.dumps({"op": op, **payload}) + "\n"
|
||||
self._process.stdin.write(message.encode("utf-8"))
|
||||
self._process.stdin.flush()
|
||||
|
||||
response = json.loads(self._read_line().decode("utf-8"))
|
||||
if response.get("ok"):
|
||||
return response.get("result")
|
||||
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类，
|
||||
# 调用方沿用原有的异常分支即可，无需感知代理的存在
|
||||
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||
|
||||
def _read_line(self, timeout: Optional[float] = None) -> bytes:
|
||||
"""
|
||||
读取一行响应，超时即强杀代理。
|
||||
:param timeout: 本次读取的超时秒数，默认用单次操作超时
|
||||
:return: 响应行
|
||||
"""
|
||||
timeout = self._timeout if timeout is None else timeout
|
||||
# Windows 的 select() 只能等 socket。subprocess.PIPE 是匿名管道，
|
||||
# selectors.DefaultSelector.select() 会抛 WinError 10038（WSAENOTSOCK），
|
||||
# 整理链每次 stat 都会直接失败。POSIX 继续用 selector，Windows 改走线程等待。
|
||||
if sys.platform == "win32":
|
||||
return self._read_line_windows(timeout)
|
||||
if not self._selector.select(timeout=timeout):
|
||||
self._raise_timeout(timeout)
|
||||
line = self._process.stdout.readline()
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _read_line_windows(self, timeout: float) -> bytes:
|
||||
"""
|
||||
在辅助线程上读一行，用 join(timeout) 实现可放弃等待。
|
||||
|
||||
不能用 select()/WaitForSingleObject：匿名管道在 Windows 上不是
|
||||
可选择的套接字，也不是可靠的「有数据」同步对象。
|
||||
:param timeout: 本次读取的超时秒数
|
||||
:return: 响应行
|
||||
"""
|
||||
if self._process is None or self._process.stdout is None:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
stdout = self._process.stdout
|
||||
holder: List[Union[bytes, BaseException]] = []
|
||||
|
||||
def _read() -> None:
|
||||
try:
|
||||
holder.append(stdout.readline())
|
||||
except Exception as err: # noqa: BLE001
|
||||
holder.append(err)
|
||||
|
||||
reader = threading.Thread(target=_read, name="fsproxy-stdout", daemon=True)
|
||||
reader.start()
|
||||
reader.join(timeout)
|
||||
if reader.is_alive():
|
||||
self._raise_timeout(timeout)
|
||||
if not holder:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
line = holder[0]
|
||||
if isinstance(line, BaseException):
|
||||
raise line
|
||||
if not line:
|
||||
raise BrokenPipeError("文件系统代理进程已退出")
|
||||
return line
|
||||
|
||||
def _raise_timeout(self, timeout: float) -> None:
|
||||
"""
|
||||
判定挂载无响应：杀掉代理并抛出可被整理链处理的超时。
|
||||
:param timeout: 已等待的秒数
|
||||
"""
|
||||
logger.error(f"文件系统操作 {timeout} 秒无响应，判定挂载挂死，正在回收代理进程")
|
||||
self._shutdown()
|
||||
raise FileSystemTimeout(
|
||||
errno_module.ETIMEDOUT,
|
||||
f"文件系统操作超过 {timeout} 秒无响应，挂载可能已无响应"
|
||||
)
|
||||
|
||||
def _ensure_worker(self):
|
||||
"""
|
||||
确保代理进程可用，不可用时重新启动。
|
||||
"""
|
||||
if self._process is not None and self._process.poll() is None:
|
||||
return
|
||||
self._shutdown()
|
||||
popen_kwargs: Dict[str, Any] = {
|
||||
"stdin": subprocess.PIPE,
|
||||
"stdout": subprocess.PIPE,
|
||||
"stderr": subprocess.DEVNULL,
|
||||
"bufsize": 0,
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
# NSSM 服务里再拉 python.exe 时避免弹出控制台窗口
|
||||
popen_kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
||||
self._process = subprocess.Popen(
|
||||
[sys.executable, str(_WORKER_PATH)],
|
||||
**popen_kwargs,
|
||||
)
|
||||
if sys.platform != "win32":
|
||||
self._selector = selectors.DefaultSelector()
|
||||
self._selector.register(self._process.stdout, selectors.EVENT_READ)
|
||||
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
|
||||
|
||||
def _shutdown(self):
|
||||
"""
|
||||
回收代理进程。冻在挂载上的进程用 SIGKILL，且不无限等待它消失
|
||||
——否则「可放弃的代理」又变回一次不可放弃的阻塞。
|
||||
"""
|
||||
if self._selector is not None:
|
||||
try:
|
||||
self._selector.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
self._selector = None
|
||||
process, self._process = self._process, None
|
||||
if process is None:
|
||||
return
|
||||
for stream in (process.stdin, process.stdout):
|
||||
try:
|
||||
if stream:
|
||||
stream.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
process.kill()
|
||||
process.wait(timeout=_KILL_GRACE)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warn(f"文件系统代理进程未能及时退出，交由系统回收: pid={process.pid}")
|
||||
except Exception as err: # noqa: BLE001
|
||||
logger.debug(f"回收文件系统代理进程失败: {err}")
|
||||
|
||||
|
||||
# 全局单例：local 存储本身是单例，代理也只需要一个
|
||||
# 不传超时参数：让它实时跟随系统设置，前端改完保存即刻生效
|
||||
fsproxy = FileSystemProxy()
|
||||
@@ -1,201 +0,0 @@
|
||||
"""
|
||||
文件系统操作代理 worker。
|
||||
|
||||
**本文件不能被 import，只能作为独立脚本执行**（fsproxy 用
|
||||
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动）。直接执行文件
|
||||
路径不会触发 `app/__init__.py` 的导入链，因此这个进程只依赖标准库、启动是
|
||||
毫秒级的；一旦走 import 就会把整个应用的依赖拉进来，代理被强杀后的重启成本
|
||||
会高到无法接受。
|
||||
|
||||
存在的理由：FUSE/网络挂载进入 block 型故障时，`stat`/`listdir`/`rename` 这类
|
||||
系统调用既不返回错误也不返回结果，而 Python 没有中断线程的手段——阻塞其上的
|
||||
线程永远无法回收。放进独立进程后，父进程可以在超时后 SIGKILL 掉它，把
|
||||
「不可处理的 block」转换成「可处理的 crash」。
|
||||
|
||||
协议：stdin/stdout 逐行 JSON。
|
||||
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
|
||||
成功 {"ok": true, "result": {...}}
|
||||
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def _stat(payload, _emit):
|
||||
"""
|
||||
读取路径的基本属性。
|
||||
"""
|
||||
path = payload["path"]
|
||||
info = os.stat(path)
|
||||
return {
|
||||
"size": info.st_size,
|
||||
"mtime": info.st_mtime,
|
||||
"is_dir": os.path.isdir(path),
|
||||
"is_file": os.path.isfile(path),
|
||||
}
|
||||
|
||||
|
||||
def _exists(payload, _emit):
|
||||
"""
|
||||
判断路径是否存在。
|
||||
|
||||
用 os.stat 而不是 os.path.exists：后者会把任意 OSError 都归为「不存在」，
|
||||
挂载抖动会被误判成文件消失。这里让异常原样抛出，由父进程按 errno 区分。
|
||||
"""
|
||||
os.stat(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _listdir(payload, _emit):
|
||||
"""
|
||||
列出目录下的条目名。
|
||||
"""
|
||||
return sorted(os.listdir(payload["path"]))
|
||||
|
||||
|
||||
def _copy(payload, emit):
|
||||
"""
|
||||
分块复制文件内容并周期上报进度。
|
||||
|
||||
进度上报同时充当心跳：复制大文件可能持续几小时，父进程无法用固定超时判断
|
||||
挂死，只能看「两次上报之间隔了多久」。因此这里按固定时间间隔上报，即使
|
||||
某一秒没读到数据也照常发——一旦挂载卡住，read/write 不返回，上报自然断流，
|
||||
父进程据此判定并强杀本进程。
|
||||
|
||||
只复制内容和时间戳，不复制权限：目标目录的默认权限与继承 ACL 是媒体库的
|
||||
访问策略，用源文件权限覆盖会清除已继承的 ACL。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
chunk_size = payload.get("chunk_size") or 1024 * 1024
|
||||
interval = payload.get("progress_interval") or 1.0
|
||||
|
||||
info = os.stat(src)
|
||||
total = info.st_size
|
||||
copied = 0
|
||||
# 先发一次 0%：既让心跳立刻开始，也保证父进程在传输开始前就有一次检查
|
||||
# 取消的机会——否则小文件会在首次定时上报之前就复制完，取消形同虚设
|
||||
emit({"ok": True, "progress": {"copied": 0, "total": total}})
|
||||
last_emit = time.monotonic()
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
buf = fsrc.read(chunk_size)
|
||||
if not buf:
|
||||
break
|
||||
fdst.write(buf)
|
||||
copied += len(buf)
|
||||
now = time.monotonic()
|
||||
if now - last_emit >= interval:
|
||||
last_emit = now
|
||||
emit({"ok": True, "progress": {"copied": copied, "total": total}})
|
||||
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||
return {"copied": copied, "total": total}
|
||||
|
||||
|
||||
def _count_entries(payload, _emit):
|
||||
"""
|
||||
统计目录下的文件与子目录数量，超过上限即提前结束。
|
||||
|
||||
放在子进程里做而不是逐层 listdir 走 IPC：递归遍历一棵大目录树会产生成千
|
||||
上万次往返，代价不可接受；一次调用在子进程内跑完 os.walk，父进程只需对
|
||||
这一次调用设超时即可整体放弃。
|
||||
"""
|
||||
directory = payload["path"]
|
||||
max_check = payload.get("max_check") or 10000
|
||||
file_count = 0
|
||||
dir_count = 0
|
||||
for _, dirs, files in os.walk(directory):
|
||||
file_count += len(files)
|
||||
dir_count += len(dirs)
|
||||
if file_count > max_check:
|
||||
break
|
||||
return {"file_count": file_count, "dir_count": dir_count}
|
||||
|
||||
|
||||
def _rename(payload, _emit):
|
||||
"""
|
||||
同一存储内重命名/移动。
|
||||
|
||||
这是第一版唯一放行的写操作：同文件系统内的 rename 由内核保证原子性，
|
||||
进程被强杀后要么完全成功要么完全没发生，不存在需要清理的中间状态。
|
||||
跨存储的复制+删除不走这里，它需要单独的可恢复语义。
|
||||
"""
|
||||
src, dst = payload["src"], payload["dst"]
|
||||
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
|
||||
raise OSError(18, "Cross-device rename is not handled by the proxy")
|
||||
os.rename(src, dst)
|
||||
return True
|
||||
|
||||
|
||||
def _unlink(payload, _emit):
|
||||
"""
|
||||
删除单个文件。unlink 是原子操作，强杀后要么删掉了要么没删，没有中间状态。
|
||||
"""
|
||||
os.unlink(payload["path"])
|
||||
return True
|
||||
|
||||
|
||||
def _rmtree(payload, _emit):
|
||||
"""
|
||||
递归删除目录。
|
||||
|
||||
这一项不是原子的，强杀可能只删掉一部分。放行的理由是：删除被中断的后果
|
||||
（残留若干文件）远轻于写入被中断（留下叫最终文件名的半成品），而且调用方
|
||||
本来就以 ignore_errors 容忍部分失败、可以重复执行直到成功。
|
||||
"""
|
||||
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||
return True
|
||||
|
||||
|
||||
_HANDLERS = {
|
||||
"stat": _stat,
|
||||
"exists": _exists,
|
||||
"listdir": _listdir,
|
||||
"copy": _copy,
|
||||
"count_entries": _count_entries,
|
||||
"rename": _rename,
|
||||
"unlink": _unlink,
|
||||
"rmtree": _rmtree,
|
||||
"ping": lambda _payload, _emit: True,
|
||||
}
|
||||
|
||||
|
||||
def _write(message):
|
||||
"""
|
||||
输出一行响应。
|
||||
"""
|
||||
sys.stdout.write(json.dumps(message) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
请求循环：每读一行处理一个请求，直到 stdin 关闭。
|
||||
|
||||
一个请求可能对应多行响应：长耗时操作先流式发若干 progress 行，最后发一行
|
||||
终态（result 或 error）。父进程据此区分「还在推进」和「已经挂死」。
|
||||
"""
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
handler = _HANDLERS.get(payload.get("op"))
|
||||
if handler is None:
|
||||
response = {"ok": False, "errno": 0,
|
||||
"error": f"unknown op: {payload.get('op')}"}
|
||||
else:
|
||||
response = {"ok": True, "result": handler(payload, _write)}
|
||||
except OSError as err:
|
||||
response = {"ok": False, "errno": err.errno or 0,
|
||||
"error": err.strerror or str(err)}
|
||||
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
|
||||
response = {"ok": False, "errno": 0, "error": str(err)}
|
||||
_write(response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1 +0,0 @@
|
||||
"""插件包和依赖系统适配器。"""
|
||||
@@ -1,380 +0,0 @@
|
||||
"""插件 Python 依赖聚合和安装适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from importlib.metadata import PackageNotFoundError, distribution, distributions
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from packaging.markers import default_environment
|
||||
from packaging.requirements import Requirement
|
||||
from packaging.specifiers import InvalidSpecifier, SpecifierSet
|
||||
from packaging.version import InvalidVersion, Version
|
||||
|
||||
from app.adapters.system.plugin.manifest import (
|
||||
PluginDependencyManifestError,
|
||||
load_dependency_manifest,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RequirementGroup:
|
||||
"""聚合同一包和安装来源的 extras 与版本约束。"""
|
||||
|
||||
name: str # PEP 503 规范化后的包名
|
||||
url: Optional[str] # direct reference 来源；为空表示从索引安装
|
||||
extras: set[str] = field(default_factory=set) # 所有插件要求启用的 extras
|
||||
specifiers: set[str] = field(default_factory=set) # 待求交集的版本约束
|
||||
|
||||
|
||||
class PluginDependencyInstaller:
|
||||
"""独立负责插件依赖扫描、约束合并和安装。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
helper: Any = None,
|
||||
*,
|
||||
installed_plugins_provider: Optional[Callable[[], list[str]]] = None,
|
||||
plugin_dir: Optional[Path] = None,
|
||||
) -> None:
|
||||
"""保存包安装端口和启动层提供的已安装插件读取器。"""
|
||||
if helper is None:
|
||||
from app.adapters.external.market import PluginHelper
|
||||
|
||||
helper = PluginHelper()
|
||||
self._helper = helper
|
||||
self._installed_plugins_provider = installed_plugins_provider or (lambda: [])
|
||||
self._plugin_dir = plugin_dir or (
|
||||
Path(get_runtime_setting('ROOT_PATH')) / "app" / "plugins"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _standardize(name: str) -> str:
|
||||
"""按 PEP 503 兼容规则标准化依赖包名。"""
|
||||
return (name or "").lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
@classmethod
|
||||
def _installed_packages(cls) -> dict[str, Version]:
|
||||
"""读取当前 Python 环境中可解析版本的已安装包。"""
|
||||
installed: dict[str, Version] = {}
|
||||
try:
|
||||
for distribution in distributions():
|
||||
name = distribution.metadata.get("Name")
|
||||
version = distribution.metadata.get("Version") or getattr(
|
||||
distribution,
|
||||
"version",
|
||||
None,
|
||||
)
|
||||
if not name or not version:
|
||||
continue
|
||||
package_name = cls._standardize(name)
|
||||
try:
|
||||
parsed = Version(version)
|
||||
except InvalidVersion:
|
||||
logger.debug(
|
||||
f"无法解析已安装包 '{package_name}' 的版本：{version}"
|
||||
)
|
||||
continue
|
||||
if package_name not in installed or parsed > installed[package_name]:
|
||||
installed[package_name] = parsed
|
||||
except Exception as err:
|
||||
logger.error(f"获取已安装的包时发生错误：{err}")
|
||||
return installed
|
||||
|
||||
@classmethod
|
||||
def _installed_distribution(cls, package_name: str) -> Any | None:
|
||||
"""读取一个包的元数据，用于校验 extras 和 direct URL 来源。"""
|
||||
try:
|
||||
return distribution(package_name)
|
||||
except PackageNotFoundError:
|
||||
return None
|
||||
|
||||
def _requirement_satisfied(
|
||||
self,
|
||||
requirement: Requirement,
|
||||
installed: dict[str, Version],
|
||||
*,
|
||||
seen: Optional[set[tuple[str, tuple[str, ...], Optional[str]]]] = None,
|
||||
) -> bool:
|
||||
"""同时校验版本、extras 及 direct URL，不把同名包误认为同一制品。"""
|
||||
package_name = self._standardize(requirement.name)
|
||||
installed_version = installed.get(package_name)
|
||||
try:
|
||||
if installed_version is None or not SpecifierSet(
|
||||
requirement.specifier
|
||||
).contains(installed_version, prereleases=True):
|
||||
return False
|
||||
except InvalidSpecifier as err:
|
||||
logger.error(f"依赖 {package_name} 约束无效：{err}")
|
||||
return False
|
||||
|
||||
installed_distribution = self._installed_distribution(package_name)
|
||||
if installed_distribution is None:
|
||||
return False if requirement.extras or requirement.url else True
|
||||
|
||||
if requirement.url and not self._direct_url_matches(
|
||||
installed_distribution, requirement.url
|
||||
):
|
||||
return False
|
||||
|
||||
requested_extras = {
|
||||
self._standardize_extra(extra) for extra in requirement.extras
|
||||
}
|
||||
if requested_extras:
|
||||
provided_extras = {
|
||||
self._standardize_extra(extra)
|
||||
for extra in installed_distribution.metadata.get_all(
|
||||
"Provides-Extra"
|
||||
)
|
||||
or []
|
||||
}
|
||||
if not requested_extras.issubset(provided_extras):
|
||||
return False
|
||||
|
||||
marker_key = (package_name, tuple(sorted(requested_extras)), requirement.url)
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if marker_key in seen:
|
||||
return True
|
||||
seen.add(marker_key)
|
||||
|
||||
for raw_dependency in installed_distribution.metadata.get_all(
|
||||
"Requires-Dist"
|
||||
) or []:
|
||||
try:
|
||||
extra_dependency = Requirement(raw_dependency)
|
||||
except Exception as err:
|
||||
logger.debug(
|
||||
f"无法解析已安装包 {package_name} 的依赖项 '{raw_dependency}'：{err}"
|
||||
)
|
||||
continue
|
||||
if not self._marker_matches_for_extras(
|
||||
extra_dependency, requested_extras
|
||||
):
|
||||
continue
|
||||
if not self._requirement_satisfied(
|
||||
extra_dependency, installed, seen=seen
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def _marker_matches_for_extras(
|
||||
cls, requirement: Requirement, extras: set[str]
|
||||
) -> bool:
|
||||
"""判断已安装发行版声明的可选依赖是否属于当前请求的 extra。"""
|
||||
if requirement.marker is None:
|
||||
return True
|
||||
environment = default_environment()
|
||||
if "extra" in str(requirement.marker):
|
||||
return any(
|
||||
requirement.marker.evaluate({**environment, "extra": extra})
|
||||
for extra in extras
|
||||
)
|
||||
return requirement.marker.evaluate(environment)
|
||||
|
||||
@staticmethod
|
||||
def _standardize_extra(name: str) -> str:
|
||||
"""按 PEP 685 兼容规则标准化 extra 名称。"""
|
||||
return (name or "").lower().replace("-", "_").replace(".", "_")
|
||||
|
||||
@staticmethod
|
||||
def _direct_url_matches(installed_distribution: Any, required_url: str) -> bool:
|
||||
"""校验安装发行版记录的 PEP 610 URL 与清单来源一致。"""
|
||||
try:
|
||||
payload = installed_distribution.read_text("direct_url.json")
|
||||
if not payload:
|
||||
return False
|
||||
direct_url = json.loads(payload).get("url")
|
||||
if not isinstance(direct_url, str):
|
||||
return False
|
||||
return PluginDependencyInstaller._canonical_direct_url(
|
||||
required_url
|
||||
) == PluginDependencyInstaller._canonical_direct_url(direct_url)
|
||||
except (AttributeError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _canonical_direct_url(value: str) -> tuple[str, str, str, str, str]:
|
||||
"""规范化来源 URL，同时保留 fragment 中可能存在的校验信息。"""
|
||||
parsed = urlsplit(value)
|
||||
netloc = parsed.netloc.rsplit("@", 1)[-1].lower()
|
||||
return (
|
||||
parsed.scheme.lower(),
|
||||
netloc,
|
||||
parsed.path.rstrip("/"),
|
||||
parsed.query,
|
||||
parsed.fragment,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _merge(cls, dependencies: list[Requirement]) -> list[Requirement]:
|
||||
"""按包和安装来源合并 extras 与约束，保留完整安装目标。"""
|
||||
groups: dict[tuple[str, Optional[str]], _RequirementGroup] = {}
|
||||
for requirement in dependencies:
|
||||
package_name = cls._standardize(requirement.name)
|
||||
key = (package_name, requirement.url)
|
||||
group = groups.setdefault(
|
||||
key,
|
||||
_RequirementGroup(name=package_name, url=requirement.url),
|
||||
)
|
||||
group.extras.update(requirement.extras)
|
||||
group.specifiers.add(str(requirement.specifier))
|
||||
|
||||
merged: list[Requirement] = []
|
||||
for group in groups.values():
|
||||
spec_set = SpecifierSet()
|
||||
for specifier in group.specifiers:
|
||||
if not specifier:
|
||||
continue
|
||||
try:
|
||||
spec_set &= SpecifierSet(specifier)
|
||||
except InvalidSpecifier as err:
|
||||
logger.error(f"发生版本约束冲突：{err}")
|
||||
target = group.name
|
||||
if group.extras:
|
||||
target += f"[{','.join(sorted(group.extras))}]"
|
||||
if group.url:
|
||||
target += f" @ {group.url}"
|
||||
elif spec_set:
|
||||
target += str(spec_set)
|
||||
merged.append(Requirement(target))
|
||||
return merged
|
||||
|
||||
def _plugin_manifests(self) -> list[Any]:
|
||||
"""返回已安装插件当前生效的依赖清单。"""
|
||||
manifests = []
|
||||
installed_plugins = {
|
||||
plugin_id.lower()
|
||||
for plugin_id in self._installed_plugins_provider() or []
|
||||
}
|
||||
try:
|
||||
plugin_dirs = list(self._plugin_dir.iterdir())
|
||||
except (FileNotFoundError, OSError):
|
||||
return []
|
||||
for plugin_dir in sorted(plugin_dirs, key=lambda item: item.name):
|
||||
if not plugin_dir.is_dir():
|
||||
continue
|
||||
if plugin_dir.name not in installed_plugins:
|
||||
logger.debug(f"忽略插件 {plugin_dir.name} 的依赖")
|
||||
continue
|
||||
manifest = load_dependency_manifest(plugin_dir)
|
||||
if manifest is None:
|
||||
continue
|
||||
manifests.append(manifest)
|
||||
return manifests
|
||||
|
||||
def _plugin_dependencies(self) -> list[Requirement]:
|
||||
"""扫描已安装插件的生效依赖清单并合并版本约束。"""
|
||||
dependencies: list[Requirement] = []
|
||||
for manifest in self._plugin_manifests():
|
||||
for requirement in manifest.dependencies:
|
||||
if requirement.marker and not requirement.marker.evaluate():
|
||||
continue
|
||||
dependencies.append(requirement)
|
||||
return self._merge(dependencies)
|
||||
|
||||
def find_missing(self) -> list[str]:
|
||||
"""返回当前插件集合缺失或不满足约束的依赖项。"""
|
||||
try:
|
||||
required = self._plugin_dependencies()
|
||||
installed = self._installed_packages()
|
||||
missing = []
|
||||
for requirement in required:
|
||||
if not self._requirement_satisfied(requirement, installed):
|
||||
missing.append(str(requirement))
|
||||
return missing
|
||||
except PluginDependencyManifestError:
|
||||
raise
|
||||
except Exception as err:
|
||||
logger.error(f"收集所有需要安装或更新的依赖项时发生错误：{err}")
|
||||
return []
|
||||
|
||||
def classify_plugins(self) -> tuple[list[str], list[str], list[str]]:
|
||||
"""按源码和依赖状态划分已安装插件。"""
|
||||
ready: list[str] = []
|
||||
missing_dependencies: list[str] = []
|
||||
missing_source: list[str] = []
|
||||
installed_packages = self._installed_packages()
|
||||
|
||||
for plugin_id in self._installed_plugins_provider() or []:
|
||||
plugin_dir = self._plugin_dir / plugin_id.lower()
|
||||
if not plugin_dir.is_dir():
|
||||
missing_source.append(plugin_id)
|
||||
continue
|
||||
try:
|
||||
manifest = load_dependency_manifest(plugin_dir)
|
||||
requirements = [] if manifest is None else [
|
||||
requirement
|
||||
for requirement in manifest.dependencies
|
||||
if not requirement.marker or requirement.marker.evaluate()
|
||||
]
|
||||
except PluginDependencyManifestError as error:
|
||||
logger.error(f"插件 {plugin_id} 依赖清单无效：{error}")
|
||||
missing_dependencies.append(plugin_id)
|
||||
continue
|
||||
if all(
|
||||
self._requirement_satisfied(requirement, installed_packages)
|
||||
for requirement in requirements
|
||||
):
|
||||
ready.append(plugin_id)
|
||||
else:
|
||||
missing_dependencies.append(plugin_id)
|
||||
|
||||
return ready, missing_dependencies, missing_source
|
||||
|
||||
def _wheels_dirs(self) -> list[Path]:
|
||||
"""收集已安装插件附带的本地 wheels 目录。"""
|
||||
result = []
|
||||
installed_plugins = {
|
||||
plugin_id.lower()
|
||||
for plugin_id in self._installed_plugins_provider() or []
|
||||
}
|
||||
for plugin_id in installed_plugins:
|
||||
wheels_dir = self._plugin_dir / plugin_id / "wheels"
|
||||
if wheels_dir.is_dir():
|
||||
result.append(wheels_dir)
|
||||
return list(dict.fromkeys(result))
|
||||
|
||||
def install(self, dependencies: list[str]) -> tuple[bool, str]:
|
||||
"""把已安装插件的原始清单交给一次统一包安装。"""
|
||||
if not dependencies:
|
||||
return False, "没有传入需要安装的依赖项"
|
||||
try:
|
||||
manifest_paths = [manifest.path for manifest in self._plugin_manifests()]
|
||||
if not manifest_paths:
|
||||
return False, "没有找到已安装插件的依赖清单"
|
||||
return self._helper.install_packages_with_fallback(
|
||||
manifest_paths,
|
||||
self._wheels_dirs(),
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"安装依赖项时发生错误：{err}")
|
||||
return False, f"安装依赖项时发生错误：{err}"
|
||||
|
||||
async def async_find_missing(self) -> list[str]:
|
||||
"""在线程池中扫描缺失依赖，避免阻塞事件循环。"""
|
||||
return await asyncio.to_thread(self.find_missing)
|
||||
|
||||
async def async_install(self, dependencies: list[str]) -> tuple[bool, str]:
|
||||
"""异步安装依赖，使用可取消的包安装子进程。"""
|
||||
if not dependencies:
|
||||
return False, "没有传入需要安装的依赖项"
|
||||
try:
|
||||
manifest_paths = [manifest.path for manifest in self._plugin_manifests()]
|
||||
if not manifest_paths:
|
||||
return False, "没有找到已安装插件的依赖清单"
|
||||
return await self._helper.async_install_packages_with_fallback(
|
||||
manifest_paths,
|
||||
self._wheels_dirs(),
|
||||
)
|
||||
except Exception as err:
|
||||
logger.error(f"安装依赖项时发生错误：{err}")
|
||||
return False, f"安装依赖项时发生错误：{err}"
|
||||
@@ -1,162 +0,0 @@
|
||||
"""插件 Python 依赖清单的选择和解析。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import tomllib
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from packaging.requirements import Requirement
|
||||
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
PYPROJECT_FILENAME = "pyproject.toml"
|
||||
REQUIREMENTS_FILENAME = "requirements.txt"
|
||||
DEPENDENCY_MANIFEST_PRIORITY = (
|
||||
PYPROJECT_FILENAME,
|
||||
REQUIREMENTS_FILENAME,
|
||||
)
|
||||
DEPENDENCY_MANIFEST_FILENAMES = frozenset(
|
||||
DEPENDENCY_MANIFEST_PRIORITY
|
||||
)
|
||||
|
||||
|
||||
class PluginDependencyManifestError(ValueError):
|
||||
"""表示生效的现代依赖清单无法安全消费。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PluginDependencyManifest:
|
||||
"""保存插件当前生效的依赖清单及其已解析依赖。"""
|
||||
|
||||
path: Path
|
||||
dependencies: tuple[Requirement, ...]
|
||||
|
||||
|
||||
def select_dependency_manifest(plugin_dir: Path) -> Path | None:
|
||||
"""按现代清单优先级返回插件当前生效的依赖文件。"""
|
||||
pyproject_file = plugin_dir / PYPROJECT_FILENAME
|
||||
if pyproject_file.is_file():
|
||||
return pyproject_file
|
||||
requirements_file = plugin_dir / REQUIREMENTS_FILENAME
|
||||
if requirements_file.is_file():
|
||||
return requirements_file
|
||||
return None
|
||||
|
||||
|
||||
def dependency_manifest_status(event_path: Path) -> bool | None:
|
||||
"""判断文件事件是否改变生效清单，非清单文件返回 None。"""
|
||||
if event_path.name not in DEPENDENCY_MANIFEST_FILENAMES:
|
||||
return None
|
||||
active_manifest = select_dependency_manifest(event_path.parent)
|
||||
if event_path.is_file():
|
||||
return active_manifest == event_path
|
||||
if active_manifest is None:
|
||||
return True
|
||||
return DEPENDENCY_MANIFEST_PRIORITY.index(
|
||||
event_path.name
|
||||
) < DEPENDENCY_MANIFEST_PRIORITY.index(active_manifest.name)
|
||||
|
||||
|
||||
def load_dependency_manifest(
|
||||
plugin_dir: Path,
|
||||
) -> PluginDependencyManifest | None:
|
||||
"""读取插件当前生效的依赖清单，现代清单无效时拒绝回退。"""
|
||||
manifest_path = select_dependency_manifest(plugin_dir)
|
||||
if manifest_path is None:
|
||||
return None
|
||||
return load_dependency_file(manifest_path)
|
||||
|
||||
|
||||
def load_dependency_file(path: Path) -> PluginDependencyManifest:
|
||||
"""读取指定依赖文件，pyproject 严格校验，其余文件保持旧格式兼容。"""
|
||||
if path.name == PYPROJECT_FILENAME:
|
||||
dependencies = _load_pyproject_dependencies(path)
|
||||
else:
|
||||
dependencies = _load_requirements_dependencies(path)
|
||||
return PluginDependencyManifest(
|
||||
path=path,
|
||||
dependencies=dependencies,
|
||||
)
|
||||
|
||||
|
||||
def _load_pyproject_dependencies(path: Path) -> tuple[Requirement, ...]:
|
||||
"""严格读取 PEP 621 ``project.dependencies``。"""
|
||||
try:
|
||||
with path.open("rb") as file:
|
||||
document = tomllib.load(file)
|
||||
except (OSError, tomllib.TOMLDecodeError) as err:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 无法解析：{err}"
|
||||
) from err
|
||||
|
||||
project = document.get("project")
|
||||
if not isinstance(project, Mapping):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 缺少 [project] 表"
|
||||
)
|
||||
name = project.get("name")
|
||||
if not isinstance(name, str) or not name.strip():
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.name 必须是非空字符串"
|
||||
)
|
||||
dynamic = project.get("dynamic", [])
|
||||
if not isinstance(dynamic, list) or not all(
|
||||
isinstance(item, str) for item in dynamic
|
||||
):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.dynamic 必须是字符串数组"
|
||||
)
|
||||
if "dependencies" in dynamic:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 不支持动态 dependencies"
|
||||
)
|
||||
version = project.get("version")
|
||||
if "version" in dynamic:
|
||||
if version is not None:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 不能同时静态和动态声明 version"
|
||||
)
|
||||
elif not isinstance(version, str) or not version.strip():
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 必须声明非空 project.version，"
|
||||
"或将 version 加入 project.dynamic"
|
||||
)
|
||||
raw_dependencies = project.get("dependencies", [])
|
||||
if not isinstance(raw_dependencies, list) or not all(
|
||||
isinstance(item, str) for item in raw_dependencies
|
||||
):
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 的 project.dependencies 必须是字符串数组"
|
||||
)
|
||||
|
||||
dependencies: list[Requirement] = []
|
||||
for item in raw_dependencies:
|
||||
try:
|
||||
dependencies.append(Requirement(item))
|
||||
except Exception as err:
|
||||
raise PluginDependencyManifestError(
|
||||
f"插件依赖清单 {path.name} 包含无效依赖项 {item!r}：{err}"
|
||||
) from err
|
||||
return tuple(dependencies)
|
||||
|
||||
|
||||
def _load_requirements_dependencies(path: Path) -> tuple[Requirement, ...]:
|
||||
"""按旧行为逐行读取 requirements，忽略无法解析的兼容内容。"""
|
||||
dependencies: list[Requirement] = []
|
||||
try:
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except OSError as err:
|
||||
logger.error(f"解析 requirements.txt 时发生错误：{err}")
|
||||
return ()
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
try:
|
||||
dependencies.append(Requirement(line))
|
||||
except Exception as err:
|
||||
logger.debug(f"无法解析依赖项 '{line}'：{err}")
|
||||
return tuple(dependencies)
|
||||
@@ -1,740 +0,0 @@
|
||||
"""插件包文件安装、快照恢复和分身处理适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
from app.adapters.external.market import PluginHelper as _PluginHelper
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.runtime.execution import (
|
||||
run_in_threadpool_to_completion as _await_thread_operation,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PluginPackageCheckpoint:
|
||||
"""记录运行目录快照及待提升的容器恢复备份。"""
|
||||
|
||||
plugin_id: str
|
||||
plugin_dir: Path
|
||||
persistent_backup_dir: Path
|
||||
backup_staging_dir: Path | None
|
||||
backup_previous_dir: Path | None
|
||||
transaction_dir: Path
|
||||
plugin_existed: bool
|
||||
persistent_backup_existed: bool
|
||||
|
||||
@property
|
||||
def existed(self) -> bool:
|
||||
"""保留旧调用方读取运行目录存在状态的兼容属性。"""
|
||||
return self.plugin_existed
|
||||
|
||||
@property
|
||||
def rollback_marker(self) -> Path:
|
||||
"""返回文件补偿完成标记，供 PREPARED 重放保持幂等。"""
|
||||
return self.transaction_dir / ".rollback-complete"
|
||||
|
||||
|
||||
class PluginPackageManager:
|
||||
"""隔离插件包安装、本地同步、分身改写和文件补偿能力。"""
|
||||
|
||||
_COPY_IGNORE = ("__pycache__", "*.pyc", ".DS_Store", "node_modules")
|
||||
|
||||
def __init__(self, helper: Optional[_PluginHelper] = None) -> None:
|
||||
"""保存市场下载实现；文件事务由本适配器独立负责。"""
|
||||
self._helper = helper or _PluginHelper()
|
||||
|
||||
@staticmethod
|
||||
def __plugin_dir(plugin_id: str) -> Path:
|
||||
"""解析插件运行目录并拒绝越出宿主插件根目录的标识。"""
|
||||
plugins_root = (
|
||||
Path(get_runtime_setting('ROOT_PATH')) / "app" / "plugins"
|
||||
).resolve()
|
||||
plugin_dir = (plugins_root / plugin_id.lower()).resolve()
|
||||
if plugin_dir == plugins_root or not plugin_dir.is_relative_to(plugins_root):
|
||||
raise ValueError(f"非法插件ID：{plugin_id}")
|
||||
return plugin_dir
|
||||
|
||||
def checkpoint(
|
||||
self,
|
||||
plugin_id: str,
|
||||
transaction_id: Optional[str] = None,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""在包变更前保存运行目录；持久事务使用配置目录承载恢复材料。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
durable = transaction_id is not None
|
||||
persistent_backup_dir = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugins_backup"
|
||||
/ plugin_id.lower()
|
||||
).resolve()
|
||||
backup_staging_dir = (
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.staging-{transaction_id}"
|
||||
if durable and SystemUtils.is_docker()
|
||||
else None
|
||||
)
|
||||
backup_previous_dir = (
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.previous-{transaction_id}"
|
||||
if durable and SystemUtils.is_docker()
|
||||
else None
|
||||
)
|
||||
transaction_root = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
if durable
|
||||
else Path(get_runtime_setting('TEMP_PATH'))
|
||||
)
|
||||
transaction_dir = (
|
||||
transaction_root
|
||||
/ "plugin_transactions"
|
||||
/ (transaction_id or f"{plugin_id.lower()}-{uuid.uuid4().hex}")
|
||||
)
|
||||
plugin_existed = plugin_dir.exists()
|
||||
persistent_backup_existed = persistent_backup_dir.exists()
|
||||
try:
|
||||
transaction_dir.mkdir(parents=True, exist_ok=False)
|
||||
if plugin_existed:
|
||||
shutil.copytree(plugin_dir, transaction_dir / "package")
|
||||
except Exception:
|
||||
shutil.rmtree(transaction_dir, ignore_errors=True)
|
||||
raise
|
||||
return PluginPackageCheckpoint(
|
||||
plugin_id=plugin_id,
|
||||
plugin_dir=plugin_dir,
|
||||
persistent_backup_dir=persistent_backup_dir,
|
||||
backup_staging_dir=backup_staging_dir,
|
||||
backup_previous_dir=backup_previous_dir,
|
||||
transaction_dir=transaction_dir,
|
||||
plugin_existed=plugin_existed,
|
||||
persistent_backup_existed=persistent_backup_existed,
|
||||
)
|
||||
|
||||
def restore_checkpoint(
|
||||
self,
|
||||
*,
|
||||
plugin_id: str,
|
||||
transaction_id: str,
|
||||
plugin_existed: bool,
|
||||
persistent_backup_existed: bool,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""按受控根目录和事务 ID 重建崩溃回放所需的文件引用。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
persistent_backup_dir = (
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugins_backup"
|
||||
/ plugin_id.lower()
|
||||
).resolve()
|
||||
durable_backup = SystemUtils.is_docker()
|
||||
return PluginPackageCheckpoint(
|
||||
plugin_id=plugin_id,
|
||||
plugin_dir=plugin_dir,
|
||||
persistent_backup_dir=persistent_backup_dir,
|
||||
backup_staging_dir=(
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.staging-{transaction_id}"
|
||||
if durable_backup
|
||||
else None
|
||||
),
|
||||
backup_previous_dir=(
|
||||
persistent_backup_dir.parent
|
||||
/ f".{plugin_id.lower()}.previous-{transaction_id}"
|
||||
if durable_backup
|
||||
else None
|
||||
),
|
||||
transaction_dir=(
|
||||
Path(get_runtime_setting('CONFIG_PATH'))
|
||||
/ "plugin_transactions"
|
||||
/ transaction_id
|
||||
),
|
||||
plugin_existed=plugin_existed,
|
||||
persistent_backup_existed=persistent_backup_existed,
|
||||
)
|
||||
|
||||
async def async_checkpoint(
|
||||
self,
|
||||
plugin_id: str,
|
||||
transaction_id: Optional[str] = None,
|
||||
) -> PluginPackageCheckpoint:
|
||||
"""在线程池中创建插件包文件快照。"""
|
||||
return cast(
|
||||
PluginPackageCheckpoint,
|
||||
await _await_thread_operation(
|
||||
self.checkpoint,
|
||||
plugin_id,
|
||||
transaction_id,
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def commit(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""清理已完成事务的运行目录快照和残余替换材料。"""
|
||||
if checkpoint.backup_staging_dir and checkpoint.backup_staging_dir.exists():
|
||||
raise RuntimeError("持久备份尚未提升，不能清理插件安装事务")
|
||||
if checkpoint.backup_previous_dir and checkpoint.backup_previous_dir.exists():
|
||||
raise RuntimeError("旧持久备份尚未清理，不能结束插件安装事务")
|
||||
if checkpoint.transaction_dir.exists():
|
||||
shutil.rmtree(checkpoint.transaction_dir, ignore_errors=False)
|
||||
|
||||
async def async_commit(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池中清理已提交的插件包快照。"""
|
||||
await _await_thread_operation(self.commit, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def rollback(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""兼容旧调用方，恢复运行目录和持久备份后清理恢复材料。"""
|
||||
PluginPackageManager.restore(checkpoint)
|
||||
PluginPackageManager.cleanup(checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def restore(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""恢复运行目录和提交前持久备份，并保留快照直到 journal 删除。"""
|
||||
if checkpoint.rollback_marker.is_file():
|
||||
return
|
||||
PluginPackageManager.__restore_tree(
|
||||
target=checkpoint.plugin_dir,
|
||||
snapshot=checkpoint.transaction_dir / "package",
|
||||
existed=checkpoint.plugin_existed,
|
||||
label=f"插件 {checkpoint.plugin_id} 运行目录",
|
||||
)
|
||||
PluginPackageManager.__rollback_persistent_backup(checkpoint)
|
||||
if checkpoint.backup_staging_dir and checkpoint.backup_staging_dir.exists():
|
||||
shutil.rmtree(checkpoint.backup_staging_dir, ignore_errors=False)
|
||||
checkpoint.transaction_dir.mkdir(parents=True, exist_ok=True)
|
||||
checkpoint.rollback_marker.touch(exist_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def cleanup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在 journal 已删除后清理恢复材料；重复调用保持幂等。"""
|
||||
if checkpoint.transaction_dir.exists():
|
||||
shutil.rmtree(checkpoint.transaction_dir, ignore_errors=False)
|
||||
|
||||
async def async_rollback(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池中恢复插件包文件快照。"""
|
||||
await _await_thread_operation(self.rollback, checkpoint)
|
||||
|
||||
async def async_restore(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池恢复插件状态，并保留 journal 仍需引用的材料。"""
|
||||
await _await_thread_operation(self.restore, checkpoint)
|
||||
|
||||
async def async_cleanup(self, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在线程池清理已失去 journal 所有权的恢复材料。"""
|
||||
await _await_thread_operation(self.cleanup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def __rollback_persistent_backup(
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""把已激活但尚未提交的持久备份恢复到事务前状态。"""
|
||||
previous = checkpoint.backup_previous_dir
|
||||
staging = checkpoint.backup_staging_dir
|
||||
if previous is None or staging is None:
|
||||
return
|
||||
|
||||
target = checkpoint.persistent_backup_dir
|
||||
if previous.exists():
|
||||
discarded = target.parent / f".{target.name}.discard-{uuid.uuid4().hex}"
|
||||
try:
|
||||
if target.exists():
|
||||
target.replace(discarded)
|
||||
previous.replace(target)
|
||||
if discarded.exists():
|
||||
shutil.rmtree(discarded, ignore_errors=False)
|
||||
except Exception:
|
||||
if not target.exists() and discarded.exists():
|
||||
discarded.replace(target)
|
||||
raise
|
||||
finally:
|
||||
if target.exists() and discarded.exists():
|
||||
shutil.rmtree(discarded, ignore_errors=True)
|
||||
return
|
||||
|
||||
if staging.exists():
|
||||
return
|
||||
if checkpoint.persistent_backup_existed:
|
||||
if target.exists():
|
||||
return
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的旧持久备份恢复材料不存在"
|
||||
)
|
||||
if target.exists():
|
||||
shutil.rmtree(target, ignore_errors=False)
|
||||
|
||||
@staticmethod
|
||||
def __restore_tree(
|
||||
*,
|
||||
target: Path,
|
||||
snapshot: Path,
|
||||
existed: bool,
|
||||
label: str,
|
||||
) -> None:
|
||||
"""用同级 staging 替换目录，失败时保留替换前的当前目录。"""
|
||||
if existed and not snapshot.is_dir():
|
||||
raise FileNotFoundError(f"{label}补偿快照不存在：{snapshot}")
|
||||
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
staging = target.parent / f".{target.name}.restore-{uuid.uuid4().hex}"
|
||||
previous = target.parent / f".{target.name}.previous-{uuid.uuid4().hex}"
|
||||
try:
|
||||
if existed:
|
||||
shutil.copytree(snapshot, staging)
|
||||
if target.exists():
|
||||
target.replace(previous)
|
||||
if existed:
|
||||
staging.replace(target)
|
||||
if previous.exists():
|
||||
shutil.rmtree(previous)
|
||||
except Exception:
|
||||
if not target.exists() and previous.exists():
|
||||
previous.replace(target)
|
||||
raise
|
||||
finally:
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
if target.exists() and previous.exists():
|
||||
shutil.rmtree(previous, ignore_errors=True)
|
||||
|
||||
@classmethod
|
||||
def stage_persistent_backup(cls, checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""把新载荷复制到持久配置目录的独立 staging，不覆盖现有备份。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
if staging is None:
|
||||
return
|
||||
if not checkpoint.plugin_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 运行目录不存在"
|
||||
)
|
||||
staging.parent.mkdir(parents=True, exist_ok=True)
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging, ignore_errors=False)
|
||||
shutil.copytree(
|
||||
checkpoint.plugin_dir,
|
||||
staging,
|
||||
ignore=shutil.ignore_patterns(*cls._COPY_IGNORE),
|
||||
)
|
||||
|
||||
async def async_stage_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池准备新载荷的容器恢复备份。"""
|
||||
await _await_thread_operation(self.stage_persistent_backup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def activate_persistent_backup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""在数据库提交前激活新备份，并保留上一份备份供失败补偿。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
previous = checkpoint.backup_previous_dir
|
||||
if staging is None or previous is None:
|
||||
return
|
||||
|
||||
target = checkpoint.persistent_backup_dir
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
if staging.exists():
|
||||
if target.exists() and not previous.exists():
|
||||
target.replace(previous)
|
||||
if not target.exists():
|
||||
staging.replace(target)
|
||||
elif not target.exists():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的持久备份 staging 不存在"
|
||||
)
|
||||
|
||||
async def async_activate_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池激活新持久备份，同时保留失败补偿材料。"""
|
||||
await _await_thread_operation(self.activate_persistent_backup, checkpoint)
|
||||
|
||||
@staticmethod
|
||||
def finalize_persistent_backup(checkpoint: PluginPackageCheckpoint) -> None:
|
||||
"""数据库提交后清理上一份持久备份；重复调用保持幂等。"""
|
||||
staging = checkpoint.backup_staging_dir
|
||||
previous = checkpoint.backup_previous_dir
|
||||
if staging is None or previous is None:
|
||||
return
|
||||
if staging.exists():
|
||||
raise RuntimeError("新持久备份尚未激活")
|
||||
if not checkpoint.persistent_backup_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 的已提交持久备份不存在"
|
||||
)
|
||||
if previous.exists():
|
||||
shutil.rmtree(previous, ignore_errors=False)
|
||||
|
||||
async def async_finalize_persistent_backup(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> None:
|
||||
"""在线程池清理数据库提交后的旧持久备份。"""
|
||||
await _await_thread_operation(self.finalize_persistent_backup, checkpoint)
|
||||
|
||||
def payload_receipt(self, plugin_id: str) -> str:
|
||||
"""按稳定相对路径和文件内容计算已安装载荷收据。"""
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
if not plugin_dir.is_dir():
|
||||
raise FileNotFoundError(f"插件 {plugin_id} 运行目录不存在")
|
||||
return self.__tree_receipt(plugin_dir)
|
||||
|
||||
@classmethod
|
||||
def persistent_backup_receipt(
|
||||
cls,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> str:
|
||||
"""计算已提升持久备份的内容收据，供崩溃回放确认终态。"""
|
||||
if not checkpoint.persistent_backup_dir.is_dir():
|
||||
raise FileNotFoundError(
|
||||
f"插件 {checkpoint.plugin_id} 持久备份不存在"
|
||||
)
|
||||
return cls.__tree_receipt(checkpoint.persistent_backup_dir)
|
||||
|
||||
@classmethod
|
||||
def __tree_receipt(cls, root: Path) -> str:
|
||||
"""对插件目录使用稳定路径和文件内容生成审计收据。"""
|
||||
|
||||
digest = hashlib.sha256()
|
||||
for path in sorted(
|
||||
root.rglob("*"),
|
||||
key=lambda item: item.relative_to(root).as_posix(),
|
||||
):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
if cls.__ignored_receipt_path(path, root):
|
||||
continue
|
||||
encoded_path = relative.encode("utf-8")
|
||||
digest.update(len(encoded_path).to_bytes(4, "big"))
|
||||
digest.update(encoded_path)
|
||||
if path.is_symlink():
|
||||
digest.update(b"L")
|
||||
target = path.readlink().as_posix().encode("utf-8")
|
||||
digest.update(len(target).to_bytes(4, "big"))
|
||||
digest.update(target)
|
||||
elif path.is_dir():
|
||||
digest.update(b"D")
|
||||
elif path.is_file():
|
||||
digest.update(b"F")
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return f"sha256:{digest.hexdigest()}"
|
||||
|
||||
async def async_payload_receipt(self, plugin_id: str) -> str:
|
||||
"""在线程池计算插件载荷收据。"""
|
||||
return cast(
|
||||
str,
|
||||
await _await_thread_operation(self.payload_receipt, plugin_id),
|
||||
)
|
||||
|
||||
async def async_committed_payload_receipt(
|
||||
self,
|
||||
checkpoint: PluginPackageCheckpoint,
|
||||
) -> str:
|
||||
"""读取数据库已提交载荷在当前部署模式下的恢复事实。"""
|
||||
if checkpoint.backup_staging_dir is not None:
|
||||
return cast(
|
||||
str,
|
||||
await _await_thread_operation(
|
||||
self.persistent_backup_receipt,
|
||||
checkpoint,
|
||||
),
|
||||
)
|
||||
return await self.async_payload_receipt(checkpoint.plugin_id)
|
||||
|
||||
@classmethod
|
||||
def __ignored_receipt_path(cls, path: Path, root: Path) -> bool:
|
||||
"""排除不会进入运行载荷和持久备份的派生文件。"""
|
||||
relative_parts = path.relative_to(root).parts
|
||||
return any(
|
||||
part in {"__pycache__", "node_modules", ".DS_Store"}
|
||||
or part.endswith(".pyc")
|
||||
for part in relative_parts
|
||||
)
|
||||
|
||||
def install(
|
||||
self,
|
||||
plugin_id: str,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
release_version: Optional[str] = None,
|
||||
force_install: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""同步安装插件包，下载过程继续复用既有市场兼容策略。"""
|
||||
return cast(
|
||||
tuple[bool, str],
|
||||
cast(Any, self._helper)._PluginHelper__install_package(
|
||||
pid=plugin_id,
|
||||
repo_url=repo_url,
|
||||
package_version=package_version,
|
||||
release_version=release_version,
|
||||
force_install=force_install,
|
||||
),
|
||||
)
|
||||
|
||||
async def async_install(
|
||||
self,
|
||||
plugin_id: str,
|
||||
repo_url: str,
|
||||
package_version: Optional[str] = None,
|
||||
release_version: Optional[str] = None,
|
||||
force_install: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
"""异步安装插件包，下载过程继续复用既有市场兼容策略。"""
|
||||
return cast(
|
||||
tuple[bool, str],
|
||||
await cast(Any, self._helper)._PluginHelper__async_install_package(
|
||||
pid=plugin_id,
|
||||
repo_url=repo_url,
|
||||
package_version=package_version,
|
||||
release_version=release_version,
|
||||
force_install=force_install,
|
||||
),
|
||||
)
|
||||
|
||||
def sync_local(self, plugin_id: str, source_dir: Path) -> bool:
|
||||
"""用本地仓库内容原子替换运行副本，失败时恢复原目录。"""
|
||||
source_dir = source_dir.resolve()
|
||||
plugin_dir = self.__plugin_dir(plugin_id)
|
||||
if source_dir == plugin_dir:
|
||||
return True
|
||||
checkpoint = self.checkpoint(plugin_id)
|
||||
try:
|
||||
if plugin_dir.exists():
|
||||
shutil.rmtree(plugin_dir)
|
||||
shutil.copytree(
|
||||
source_dir,
|
||||
plugin_dir,
|
||||
ignore=shutil.ignore_patterns(*self._COPY_IGNORE),
|
||||
)
|
||||
self.commit(checkpoint)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.error(f"同步本地插件 {plugin_id} 失败：{err}")
|
||||
try:
|
||||
self.rollback(checkpoint)
|
||||
except Exception as rollback_err:
|
||||
logger.error(
|
||||
f"恢复本地插件 {plugin_id} 原目录失败：{rollback_err}",
|
||||
exc_info=True,
|
||||
)
|
||||
return False
|
||||
|
||||
def clone(
|
||||
self,
|
||||
*,
|
||||
plugin_id: str,
|
||||
clone_id: str,
|
||||
original_class_name: str,
|
||||
suffix: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str] = None,
|
||||
icon: Optional[str] = None,
|
||||
) -> tuple[bool, str]:
|
||||
"""复制并改写插件分身文件，任一步失败都删除不完整目标。"""
|
||||
original_dir = self.__plugin_dir(plugin_id)
|
||||
clone_dir = self.__plugin_dir(clone_id)
|
||||
if not original_dir.is_dir():
|
||||
return False, f"原插件目录 {original_dir} 不存在"
|
||||
if clone_dir.exists():
|
||||
return False, f"分身插件 {clone_id} 已存在"
|
||||
|
||||
checkpoint = self.checkpoint(clone_id)
|
||||
try:
|
||||
shutil.copytree(original_dir, clone_dir)
|
||||
success, message = self._modify_plugin_files(
|
||||
plugin_dir=clone_dir,
|
||||
original_class_name=original_class_name,
|
||||
suffix=suffix,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
if not success:
|
||||
self.rollback(checkpoint)
|
||||
return False, message
|
||||
self.commit(checkpoint)
|
||||
logger.info(f"已复制插件目录：{original_dir} -> {clone_dir}")
|
||||
return True, "文件修改成功"
|
||||
except Exception as err:
|
||||
try:
|
||||
self.rollback(checkpoint)
|
||||
except Exception as rollback_err:
|
||||
logger.error(
|
||||
f"清理插件分身 {clone_id} 失败：{rollback_err}",
|
||||
exc_info=True,
|
||||
)
|
||||
return False, f"创建插件分身文件失败：{err}"
|
||||
|
||||
def _modify_plugin_files(
|
||||
self,
|
||||
*,
|
||||
plugin_dir: Path,
|
||||
original_class_name: str,
|
||||
suffix: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str],
|
||||
icon: Optional[str],
|
||||
) -> tuple[bool, str]:
|
||||
"""改写分身的 Python 元数据和联邦前端资源。"""
|
||||
clone_class_name = f"{original_class_name}{suffix}"
|
||||
init_file = plugin_dir / "__init__.py"
|
||||
if init_file.exists():
|
||||
success, message = self._modify_python_file(
|
||||
file_path=init_file,
|
||||
original_class_name=original_class_name,
|
||||
clone_class_name=clone_class_name,
|
||||
name=name,
|
||||
description=description,
|
||||
version=version,
|
||||
icon=icon,
|
||||
)
|
||||
if not success:
|
||||
return False, message
|
||||
|
||||
dist_dir = plugin_dir / "dist"
|
||||
if dist_dir.exists():
|
||||
success, message = self._modify_federation_files(
|
||||
dist_dir=dist_dir,
|
||||
original_class_name=original_class_name,
|
||||
clone_class_name=clone_class_name,
|
||||
)
|
||||
if not success:
|
||||
return False, message
|
||||
return True, "文件修改成功"
|
||||
|
||||
@staticmethod
|
||||
def _modify_python_file(
|
||||
*,
|
||||
file_path: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
name: str,
|
||||
description: str,
|
||||
version: Optional[str],
|
||||
icon: Optional[str],
|
||||
) -> tuple[bool, str]:
|
||||
"""改写插件主类名称、展示元数据和独立配置前缀。"""
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
content = content.replace(
|
||||
f"class {original_class_name}",
|
||||
f"class {clone_class_name}",
|
||||
)
|
||||
if name:
|
||||
content = re.sub(
|
||||
r'plugin_name\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_name = "{name}"',
|
||||
content,
|
||||
)
|
||||
if description:
|
||||
content = re.sub(
|
||||
r'plugin_desc\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_desc = "{description}"',
|
||||
content,
|
||||
)
|
||||
content = re.sub(
|
||||
r'plugin_config_prefix\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_config_prefix = "{clone_class_name.lower()}_"',
|
||||
content,
|
||||
)
|
||||
if version:
|
||||
content = re.sub(
|
||||
r'plugin_version\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_version = "{version}"',
|
||||
content,
|
||||
)
|
||||
if icon and icon.strip():
|
||||
content = re.sub(
|
||||
r'plugin_icon\s*=\s*["\'][^"\']*["\']',
|
||||
f'plugin_icon = "{icon}"',
|
||||
content,
|
||||
)
|
||||
if "def init_plugin(self" in content:
|
||||
init_index = content.index("def init_plugin(self")
|
||||
content = (
|
||||
content[:init_index]
|
||||
+ "is_clone = True\n\n "
|
||||
+ content[init_index:]
|
||||
)
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
return True, "Python文件修改成功"
|
||||
except Exception as err:
|
||||
logger.error(f"修改Python文件失败：{err}")
|
||||
return False, f"修改Python文件失败：{err}"
|
||||
|
||||
def _modify_federation_files(
|
||||
self,
|
||||
*,
|
||||
dist_dir: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
) -> tuple[bool, str]:
|
||||
"""改写联邦构建产物中的插件类名和样式命名空间。"""
|
||||
try:
|
||||
for file_path in dist_dir.rglob("*"):
|
||||
if not file_path.is_file() or file_path.suffix not in {".js", ".css"}:
|
||||
continue
|
||||
try:
|
||||
content = file_path.read_text(encoding="utf-8", errors="replace")
|
||||
if file_path.suffix == ".js":
|
||||
content = content.replace(original_class_name, clone_class_name)
|
||||
content = content.replace(
|
||||
f'"{original_class_name}"',
|
||||
f'"{clone_class_name}"',
|
||||
)
|
||||
content = content.replace(
|
||||
f"'{original_class_name}'",
|
||||
f"'{clone_class_name}'",
|
||||
)
|
||||
content = content.replace(
|
||||
f"css__{original_class_name}__",
|
||||
f"css__{clone_class_name}__",
|
||||
)
|
||||
content = content.replace(
|
||||
original_class_name.lower(),
|
||||
clone_class_name.lower(),
|
||||
)
|
||||
file_path.write_text(content, encoding="utf-8")
|
||||
except Exception as err:
|
||||
logger.warning(f"修改联邦插件文件 {file_path} 失败：{err}")
|
||||
self._rename_federation_assets(
|
||||
dist_dir,
|
||||
original_class_name,
|
||||
clone_class_name,
|
||||
)
|
||||
return True, "联邦插件文件修改完成"
|
||||
except Exception as err:
|
||||
logger.error(f"修改联邦插件文件失败：{err}")
|
||||
return False, f"修改联邦插件文件失败：{err}"
|
||||
|
||||
@staticmethod
|
||||
def _rename_federation_assets(
|
||||
dist_dir: Path,
|
||||
original_class_name: str,
|
||||
clone_class_name: str,
|
||||
) -> None:
|
||||
"""重命名包含原类名的顶层联邦资源，避免分身资源冲突。"""
|
||||
try:
|
||||
for file_path in dist_dir.glob("*"):
|
||||
if not file_path.is_file():
|
||||
continue
|
||||
if original_class_name.lower() not in file_path.name.lower():
|
||||
continue
|
||||
new_name = file_path.name.replace(
|
||||
original_class_name.lower(),
|
||||
clone_class_name.lower(),
|
||||
)
|
||||
new_path = file_path.parent / new_name
|
||||
if not new_path.exists():
|
||||
file_path.rename(new_path)
|
||||
except Exception as err:
|
||||
logger.warning(f"重命名联邦插件资源文件失败：{err}")
|
||||
@@ -1,481 +0,0 @@
|
||||
"""MoviePilot Release 后台检查、下载与待安装状态管理。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import threading
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.foundation.singleton import SingletonClass
|
||||
from app.foundation.version import compare_version
|
||||
from app.runtime.version import get_app_version
|
||||
from app.foundation.environment import is_docker
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.runtime.thread import ThreadHelper
|
||||
from app.schemas.system import SystemUpdateStatus
|
||||
|
||||
|
||||
class SystemUpdateManager(metaclass=SingletonClass):
|
||||
"""持久化更新状态，并保证同一时刻只有一个下载任务。"""
|
||||
|
||||
_BACKEND_RELEASES_API = "https://api.github.com/repos/jxxghp/MoviePilot/releases"
|
||||
_FRONTEND_RELEASE_API = (
|
||||
"https://api.github.com/repos/jxxghp/MoviePilot-Frontend/releases/tags/{tag}"
|
||||
)
|
||||
_BACKEND_ARCHIVE_URL = (
|
||||
"https://github.com/jxxghp/MoviePilot/archive/refs/tags/{tag}.zip"
|
||||
)
|
||||
_VERSION_PATTERN = re.compile(r"^v3\.\d+\.\d+(?:[-.](?:alpha|beta|rc)\d*)?$", re.I)
|
||||
_STABLE_VERSION_PATTERN = re.compile(r"^v3\.\d+\.\d+$", re.I)
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._lock = threading.RLock()
|
||||
self._download_active = False
|
||||
|
||||
@property
|
||||
def _root(self) -> Path:
|
||||
return Path(get_runtime_setting('TEMP_PATH')) / "moviepilot-update"
|
||||
|
||||
@property
|
||||
def _state_file(self) -> Path:
|
||||
return self._root / "state.json"
|
||||
|
||||
@property
|
||||
def _install_file(self) -> Path:
|
||||
return self._root / "install.json"
|
||||
|
||||
@property
|
||||
def _backend_archive(self) -> Path:
|
||||
return self._root / "backend.zip"
|
||||
|
||||
@property
|
||||
def _frontend_archive(self) -> Path:
|
||||
return self._root / "frontend.zip"
|
||||
|
||||
@staticmethod
|
||||
def _now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
def _default_state(self) -> dict[str, Any]:
|
||||
return SystemUpdateStatus(current_version=get_app_version()).model_dump()
|
||||
|
||||
def _read_state(self) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(self._state_file.read_text(encoding="utf-8"))
|
||||
if isinstance(payload, dict):
|
||||
return {
|
||||
**self._default_state(),
|
||||
**payload,
|
||||
"current_version": get_app_version(),
|
||||
}
|
||||
except (OSError, json.JSONDecodeError):
|
||||
pass
|
||||
return self._default_state()
|
||||
|
||||
def _write_state(self, **changes: Any) -> dict[str, Any]:
|
||||
with self._lock:
|
||||
state = self._read_state()
|
||||
state.update(changes)
|
||||
state["current_version"] = get_app_version()
|
||||
state["progress"] = self._progress(
|
||||
state.get("downloaded_bytes", 0), state.get("total_bytes", 0)
|
||||
)
|
||||
validated = SystemUpdateStatus.model_validate(state).model_dump()
|
||||
self._root.mkdir(parents=True, exist_ok=True)
|
||||
temporary = self._state_file.with_suffix(".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(validated, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
temporary.replace(self._state_file)
|
||||
return validated
|
||||
|
||||
@staticmethod
|
||||
def _progress(downloaded: Any, total: Any) -> int:
|
||||
try:
|
||||
downloaded_value = max(0, int(downloaded))
|
||||
total_value = max(0, int(total))
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
if total_value <= 0:
|
||||
return 0
|
||||
return min(100, int(downloaded_value * 100 / total_value))
|
||||
|
||||
def get_status(self) -> SystemUpdateStatus:
|
||||
"""返回状态快照，并在更新完成后的新进程中清理安装终态。"""
|
||||
with self._lock:
|
||||
state = self._read_state()
|
||||
target = str(state.get("version") or "")
|
||||
if state.get("state") == "downloading" and not self._download_active:
|
||||
state = self._write_state(
|
||||
state="failed",
|
||||
error="更新包下载因服务重启而中断，请重试",
|
||||
can_update=True,
|
||||
can_install=False,
|
||||
)
|
||||
if state.get("state") == "installing" and target == get_app_version():
|
||||
self._install_file.unlink(missing_ok=True)
|
||||
state = self._write_state(
|
||||
state="idle",
|
||||
version=None,
|
||||
frontend_version=None,
|
||||
release_name=None,
|
||||
release_notes=None,
|
||||
published_at=None,
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
return SystemUpdateStatus.model_validate(state)
|
||||
|
||||
def check(self) -> SystemUpdateStatus:
|
||||
"""查询 GitHub 稳定版 v3 Release，并保留正在下载或待安装状态。"""
|
||||
current = self.get_status()
|
||||
if current.state in {"downloading", "ready", "installing"}:
|
||||
return current
|
||||
|
||||
try:
|
||||
response = self._request().get_res(self._BACKEND_RELEASES_API)
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError("GitHub Release 请求失败")
|
||||
releases = response.json()
|
||||
release = next(
|
||||
(
|
||||
item
|
||||
for item in releases
|
||||
if isinstance(item, dict)
|
||||
and not item.get("draft")
|
||||
and not item.get("prerelease")
|
||||
and self._STABLE_VERSION_PATTERN.fullmatch(
|
||||
str(item.get("tag_name") or "")
|
||||
)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not release:
|
||||
raise RuntimeError("未找到可用的 v3 稳定版本")
|
||||
version = str(release["tag_name"])
|
||||
has_update = compare_version(version, "gt", get_app_version()) is True
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="available" if has_update else "idle",
|
||||
version=version if has_update else None,
|
||||
frontend_version=None,
|
||||
release_name=str(release.get("name") or version) if has_update else None,
|
||||
release_notes=str(release.get("body") or "") if has_update else None,
|
||||
published_at=release.get("published_at") if has_update else None,
|
||||
checked_at=self._now(),
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=has_update,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
except Exception as error: # 定时检查失败不应打扰用户，下载失败才进入可见错误态
|
||||
logger.warning(f"检查 MoviePilot 更新失败: {error}")
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="idle",
|
||||
checked_at=self._now(),
|
||||
error=str(error),
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
|
||||
def start_download(self) -> SystemUpdateStatus:
|
||||
"""启动唯一后台下载线程，并立即返回下载中状态。"""
|
||||
with self._lock:
|
||||
state = self.get_status()
|
||||
if state.state == "ready":
|
||||
return state
|
||||
if state.state == "downloading" and self._download_active:
|
||||
return state
|
||||
if state.state != "available" or not state.version:
|
||||
state = self.check()
|
||||
if state.state != "available" or not state.version:
|
||||
return state
|
||||
|
||||
self._backend_archive.unlink(missing_ok=True)
|
||||
self._frontend_archive.unlink(missing_ok=True)
|
||||
self._write_state(
|
||||
state="downloading",
|
||||
downloaded_bytes=0,
|
||||
total_bytes=0,
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=False,
|
||||
)
|
||||
self._download_active = True
|
||||
try:
|
||||
ThreadHelper().submit(self._download_update, state.version)
|
||||
except RuntimeError as error:
|
||||
self._download_active = False
|
||||
return SystemUpdateStatus.model_validate(
|
||||
self._write_state(
|
||||
state="failed",
|
||||
error=f"无法启动更新包下载：{error}",
|
||||
can_update=True,
|
||||
can_install=False,
|
||||
)
|
||||
)
|
||||
return self.get_status()
|
||||
|
||||
def request_install(self) -> tuple[bool, str]:
|
||||
"""校验待安装文件并写入启动阶段消费的安装意图。"""
|
||||
with self._lock:
|
||||
state = self.get_status()
|
||||
if state.state != "ready" or not state.version:
|
||||
return False, "更新包尚未下载完成"
|
||||
try:
|
||||
backend_sha256 = self._sha256(self._backend_archive)
|
||||
frontend_sha256 = self._sha256(self._frontend_archive)
|
||||
prepared = self._read_prepared_manifest()
|
||||
if backend_sha256 != prepared.get("backend_sha256"):
|
||||
raise RuntimeError("后端更新包校验失败")
|
||||
if frontend_sha256 != prepared.get("frontend_sha256"):
|
||||
raise RuntimeError("前端更新包校验失败")
|
||||
self._install_file.write_text(
|
||||
json.dumps(prepared, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
self._write_state(state="installing", can_install=False, error=None)
|
||||
return True, "更新包已就绪，正在重启安装"
|
||||
except (OSError, RuntimeError, json.JSONDecodeError) as error:
|
||||
self._write_state(state="failed", error=str(error), can_install=False)
|
||||
return False, str(error)
|
||||
|
||||
def cancel_install(self, reason: str) -> None:
|
||||
"""重启请求失败时撤销安装意图，避免下次普通启动意外安装。"""
|
||||
with self._lock:
|
||||
self._install_file.unlink(missing_ok=True)
|
||||
self._write_state(
|
||||
state="ready",
|
||||
error=reason,
|
||||
can_update=False,
|
||||
can_install=True,
|
||||
)
|
||||
|
||||
def _request(self) -> RequestUtils:
|
||||
return RequestUtils(
|
||||
proxies=get_runtime_setting('PROXY'),
|
||||
headers=get_runtime_setting('GITHUB_HEADERS'),
|
||||
timeout=60,
|
||||
)
|
||||
|
||||
def _download_update(self, version: str) -> None:
|
||||
try:
|
||||
downloaded = 0
|
||||
downloaded, backend_total = self._download_file(
|
||||
self._proxied(self._BACKEND_ARCHIVE_URL.format(tag=version)),
|
||||
self._backend_archive,
|
||||
downloaded,
|
||||
0,
|
||||
)
|
||||
frontend_version = self._validate_backend_archive(version)
|
||||
frontend_release = self._fetch_frontend_release(frontend_version)
|
||||
frontend_asset = next(
|
||||
(
|
||||
item
|
||||
for item in frontend_release.get("assets") or []
|
||||
if item.get("name") == "dist.zip" and item.get("browser_download_url")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not frontend_asset:
|
||||
raise RuntimeError(f"前端 {frontend_version} 缺少 dist.zip 发布资产")
|
||||
frontend_total = int(frontend_asset.get("size") or 0)
|
||||
total = backend_total + frontend_total
|
||||
self._write_state(
|
||||
frontend_version=frontend_version,
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=total,
|
||||
)
|
||||
downloaded, _ = self._download_file(
|
||||
self._proxied(str(frontend_asset["browser_download_url"])),
|
||||
self._frontend_archive,
|
||||
downloaded,
|
||||
total,
|
||||
)
|
||||
self._validate_frontend_archive(frontend_version)
|
||||
expected_digest = str(frontend_asset.get("digest") or "")
|
||||
frontend_sha256 = self._sha256(self._frontend_archive)
|
||||
if expected_digest.startswith("sha256:") and frontend_sha256 != expected_digest.removeprefix("sha256:"):
|
||||
raise RuntimeError("前端更新包与 GitHub Release 摘要不一致")
|
||||
|
||||
if not is_docker():
|
||||
self._prepare_local_backend_ref(version)
|
||||
|
||||
prepared = {
|
||||
"version": version,
|
||||
"frontend_version": frontend_version,
|
||||
"backend_archive": str(self._backend_archive),
|
||||
"frontend_archive": str(self._frontend_archive),
|
||||
"backend_sha256": self._sha256(self._backend_archive),
|
||||
"frontend_sha256": frontend_sha256,
|
||||
"prepared_at": self._now(),
|
||||
}
|
||||
(self._root / "prepared.json").write_text(
|
||||
json.dumps(prepared, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
self._write_state(
|
||||
state="ready",
|
||||
downloaded_bytes=downloaded,
|
||||
total_bytes=max(total, downloaded),
|
||||
error=None,
|
||||
can_update=False,
|
||||
can_install=True,
|
||||
)
|
||||
logger.info(f"MoviePilot {version} 更新包已下载完成，等待用户确认重启")
|
||||
except Exception as error: # 后台线程必须把所有失败沉淀为可查询状态
|
||||
logger.error(f"下载 MoviePilot 更新包失败: {error}")
|
||||
self._write_state(
|
||||
state="failed", error=str(error), can_update=True, can_install=False
|
||||
)
|
||||
finally:
|
||||
with self._lock:
|
||||
self._download_active = False
|
||||
|
||||
def _download_file(
|
||||
self, url: str, destination: Path, downloaded_before: int, total_hint: int
|
||||
) -> tuple[int, int]:
|
||||
temporary = destination.with_suffix(".part")
|
||||
temporary.unlink(missing_ok=True)
|
||||
with self._request().get_stream(url) as response:
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError(f"下载更新包失败：HTTP {getattr(response, 'status_code', '无响应')}")
|
||||
content_length = int(response.headers.get("content-length") or 0)
|
||||
total = total_hint or content_length
|
||||
current = downloaded_before
|
||||
with temporary.open("wb") as output:
|
||||
for chunk in response.iter_content(chunk_size=256 * 1024):
|
||||
if not chunk:
|
||||
continue
|
||||
output.write(chunk)
|
||||
current += len(chunk)
|
||||
self._write_state(downloaded_bytes=current, total_bytes=total)
|
||||
temporary.replace(destination)
|
||||
return current, content_length
|
||||
|
||||
def _fetch_frontend_release(self, version: str) -> dict[str, Any]:
|
||||
response = self._request().get_res(
|
||||
self._FRONTEND_RELEASE_API.format(tag=version)
|
||||
)
|
||||
if response is None or response.status_code != 200:
|
||||
raise RuntimeError(f"无法获取前端 {version} Release")
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("前端 Release 返回格式异常")
|
||||
return payload
|
||||
|
||||
def _validate_backend_archive(self, version: str) -> str:
|
||||
with zipfile.ZipFile(self._backend_archive) as archive:
|
||||
self._validate_zip_members(archive)
|
||||
version_name = next(
|
||||
(name for name in archive.namelist() if name.count("/") == 1 and name.endswith("/version.py")),
|
||||
None,
|
||||
)
|
||||
if not version_name:
|
||||
raise RuntimeError("后端更新包缺少 version.py")
|
||||
version_source = archive.read(version_name).decode("utf-8")
|
||||
app_match = re.search(r"^APP_VERSION\s*=\s*['\"]([^'\"]+)", version_source, re.M)
|
||||
frontend_match = re.search(r"^FRONTEND_VERSION\s*=\s*['\"]([^'\"]+)", version_source, re.M)
|
||||
if not app_match or app_match.group(1) != version:
|
||||
raise RuntimeError("后端更新包版本与目标 Release 不一致")
|
||||
if not frontend_match or not self._VERSION_PATTERN.fullmatch(frontend_match.group(1)):
|
||||
raise RuntimeError("后端更新包声明的前端版本无效")
|
||||
required = ("pyproject.toml", "uv.lock")
|
||||
names = archive.namelist()
|
||||
if any(not any(name.endswith(f"/{item}") for name in names) for item in required):
|
||||
raise RuntimeError("后端更新包缺少依赖锁定文件")
|
||||
return frontend_match.group(1)
|
||||
|
||||
def _validate_frontend_archive(self, version: str) -> None:
|
||||
with zipfile.ZipFile(self._frontend_archive) as archive:
|
||||
self._validate_zip_members(archive)
|
||||
names = set(archive.namelist())
|
||||
if "dist/index.html" not in names or "dist/version.txt" not in names:
|
||||
raise RuntimeError("前端更新包结构无效")
|
||||
archived_version = archive.read("dist/version.txt").decode("utf-8").strip()
|
||||
if archived_version != version:
|
||||
raise RuntimeError("前端更新包版本与后端声明不一致")
|
||||
|
||||
@staticmethod
|
||||
def _validate_zip_members(archive: zipfile.ZipFile) -> None:
|
||||
for item in archive.infolist():
|
||||
path = PurePosixPath(item.filename)
|
||||
file_type = (item.external_attr >> 16) & 0o170000
|
||||
if (
|
||||
path.is_absolute()
|
||||
or ".." in path.parts
|
||||
or file_type == stat.S_IFLNK
|
||||
):
|
||||
raise RuntimeError("更新包包含不安全路径")
|
||||
|
||||
def _read_prepared_manifest(self) -> dict[str, Any]:
|
||||
payload = json.loads((self._root / "prepared.json").read_text(encoding="utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("更新包清单格式无效")
|
||||
return payload
|
||||
|
||||
@staticmethod
|
||||
def _prepare_local_backend_ref(version: str) -> None:
|
||||
"""本地 CLI 在下载阶段获取标签，使重启后的代码切换不再联网。"""
|
||||
root = Path(__file__).resolve().parents[3]
|
||||
if not (root / ".git").is_dir():
|
||||
raise RuntimeError("本地安装目录不是 Git 仓库，无法准备 Release 更新")
|
||||
try:
|
||||
worktree = subprocess.run(
|
||||
["git", "status", "--porcelain", "--untracked-files=no"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
if worktree.stdout.strip():
|
||||
raise RuntimeError("本地源码存在未提交改动，无法准备 Release 更新")
|
||||
subprocess.run(
|
||||
["git", "fetch", "--no-tags", "origin", "tag", version],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=180,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "rev-parse", "--verify", f"{version}^{{commit}}"],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
||||
raise RuntimeError(f"无法准备本地 Release 标签 {version}") from error
|
||||
|
||||
@staticmethod
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as file_handle:
|
||||
for chunk in iter(lambda: file_handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _proxied(url: str) -> str:
|
||||
proxy = str(get_runtime_setting('GITHUB_PROXY') or "").strip()
|
||||
return f"{proxy}{url}" if proxy else url
|
||||
|
||||
|
||||
system_update_manager = SystemUpdateManager()
|
||||
@@ -1 +0,0 @@
|
||||
"""Web 框架适配器。"""
|
||||
@@ -1,44 +0,0 @@
|
||||
"""HTTP 请求关联 ID 的 ASGI 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.runtime.correlation import (
|
||||
CORRELATION_ID_HEADER,
|
||||
correlation_scope,
|
||||
normalize_correlation_id,
|
||||
)
|
||||
|
||||
|
||||
class CorrelationIdMiddleware:
|
||||
"""验证入口 ID、绑定请求上下文并把同一 ID 写回响应。"""
|
||||
|
||||
def __init__(self, app: Any) -> None:
|
||||
"""保存下游 ASGI 应用。"""
|
||||
self._app = app
|
||||
|
||||
async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
|
||||
"""只治理 HTTP scope，并让绑定覆盖完整流式响应生命周期。"""
|
||||
if scope.get("type") != "http":
|
||||
await self._app(scope, receive, send)
|
||||
return
|
||||
raw_headers = dict(scope.get("headers") or [])
|
||||
candidate = raw_headers.get(CORRELATION_ID_HEADER.lower().encode("ascii"))
|
||||
correlation_id = normalize_correlation_id(
|
||||
candidate.decode("ascii", errors="ignore") if candidate else None
|
||||
)
|
||||
scope.setdefault("state", {})["request_id"] = correlation_id
|
||||
|
||||
async def send_with_correlation(message: dict) -> None:
|
||||
"""在响应开始帧中覆盖为当前请求的安全关联 ID。"""
|
||||
if message.get("type") == "http.response.start":
|
||||
headers = list(message.get("headers") or [])
|
||||
header_name = CORRELATION_ID_HEADER.lower().encode("ascii")
|
||||
headers = [item for item in headers if item[0].lower() != header_name]
|
||||
headers.append((header_name, correlation_id.encode("ascii")))
|
||||
message["headers"] = headers
|
||||
await send(message)
|
||||
|
||||
with correlation_scope(correlation_id):
|
||||
await self._app(scope, receive, send_with_correlation)
|
||||
@@ -1,45 +0,0 @@
|
||||
"""面向编排器的最小公开健康探针。"""
|
||||
|
||||
from fastapi import FastAPI, Request, status
|
||||
from fastapi.responses import JSONResponse
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from app.runtime.health import get_application_health
|
||||
|
||||
|
||||
async def liveness() -> JSONResponse:
|
||||
"""确认进程和当前事件循环能够处理请求，不访问任何外部依赖。"""
|
||||
return JSONResponse(content={"status": "alive"})
|
||||
|
||||
|
||||
async def readiness(request: Request) -> JSONResponse:
|
||||
"""仅公开可否接流量，不泄露数据库、插件或启动异常细节。"""
|
||||
ready = get_application_health(request.app).is_ready
|
||||
return JSONResponse(
|
||||
status_code=(
|
||||
status.HTTP_200_OK
|
||||
if ready
|
||||
else status.HTTP_503_SERVICE_UNAVAILABLE
|
||||
),
|
||||
content={"status": "ready" if ready else "not_ready"},
|
||||
)
|
||||
|
||||
|
||||
def install_health_routes(app: FastAPI) -> None:
|
||||
"""把公开探针装到 API 版本前缀之外，供容器和反向代理使用。"""
|
||||
app.router.add_api_route(
|
||||
"/health/live",
|
||||
liveness,
|
||||
methods=["GET"],
|
||||
response_class=JSONResponse,
|
||||
include_in_schema=False,
|
||||
route_class_override=APIRoute,
|
||||
)
|
||||
app.router.add_api_route(
|
||||
"/health/ready",
|
||||
readiness,
|
||||
methods=["GET"],
|
||||
response_class=JSONResponse,
|
||||
include_in_schema=False,
|
||||
route_class_override=APIRoute,
|
||||
)
|
||||
@@ -1,58 +0,0 @@
|
||||
"""HTTP route/status/latency 指标 ASGI 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from app.runtime.observability import record_metric
|
||||
from starlette.routing import Match
|
||||
|
||||
|
||||
class HttpMetricsMiddleware:
|
||||
"""按路由模板、方法和状态码记录低基数 HTTP 时延。"""
|
||||
|
||||
def __init__(self, app: Any) -> None:
|
||||
"""保存下游 ASGI 应用。"""
|
||||
self._app = app
|
||||
|
||||
async def __call__(self, scope: dict, receive: Any, send: Any) -> None:
|
||||
"""只治理 HTTP scope，并在响应开始后读取路由模板。"""
|
||||
if scope.get("type") != "http":
|
||||
await self._app(scope, receive, send)
|
||||
return
|
||||
started_at = time.perf_counter()
|
||||
status = "500"
|
||||
|
||||
async def send_with_metrics(message: dict) -> None:
|
||||
"""捕获响应状态并原样转发 ASGI 消息。"""
|
||||
nonlocal status
|
||||
if message.get("type") == "http.response.start":
|
||||
status = str(message.get("status", 500))
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self._app(scope, receive, send_with_metrics)
|
||||
finally:
|
||||
route_path = self._resolve_route_template(scope)
|
||||
record_metric(
|
||||
"http.server.duration",
|
||||
time.perf_counter() - started_at,
|
||||
route=route_path,
|
||||
method=str(scope.get("method", "UNKNOWN")),
|
||||
status=status,
|
||||
)
|
||||
|
||||
def _resolve_route_template(self, scope: dict) -> str:
|
||||
"""遍历 ASGI wrapper 找到匹配路由模板，绝不回退到具体请求 path。"""
|
||||
route = scope.get("route")
|
||||
if getattr(route, "path", None):
|
||||
return route.path
|
||||
candidate = self._app
|
||||
while candidate is not None:
|
||||
for registered_route in getattr(candidate, "routes", ()):
|
||||
match, _ = registered_route.matches(scope)
|
||||
if match == Match.FULL:
|
||||
return getattr(registered_route, "path", "unmatched")
|
||||
candidate = getattr(candidate, "app", None)
|
||||
return "unmatched"
|
||||
@@ -1 +0,0 @@
|
||||
"""插件 Web 适配器。"""
|
||||
@@ -1,178 +0,0 @@
|
||||
"""FastAPI 动态插件路由适配器。"""
|
||||
|
||||
import asyncio
|
||||
from concurrent.futures import Future, TimeoutError as FutureTimeoutError
|
||||
from threading import Lock
|
||||
from typing import Any, Callable, Optional
|
||||
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
|
||||
class FastAPIDynamicRouteRegistry:
|
||||
"""在 FastAPI 上注册插件自由响应路由，并维护 OpenAPI 缓存。"""
|
||||
|
||||
_dispatch_admission_timeout = 5.0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
app: FastAPI,
|
||||
plugin_ids: Callable[[], list[str]],
|
||||
plugin_apis: Callable[[str], list[dict]],
|
||||
verify_token: Callable[..., Any],
|
||||
verify_apikey: Callable[..., Any],
|
||||
prefix: str,
|
||||
protected_routes: set[str],
|
||||
log: Any,
|
||||
event_loop: Callable[[], asyncio.AbstractEventLoop | None] | None = None,
|
||||
) -> None:
|
||||
"""注入应用、插件投影、认证依赖和日志端口。"""
|
||||
self._app = app
|
||||
self._plugin_ids = plugin_ids
|
||||
self._plugin_apis = plugin_apis
|
||||
self._verify_token = verify_token
|
||||
self._verify_apikey = verify_apikey
|
||||
self._prefix = prefix
|
||||
self._protected_routes = protected_routes
|
||||
self._logger = log
|
||||
self._event_loop = event_loop
|
||||
|
||||
def update(self, plugin_id: Optional[str], action: str) -> None:
|
||||
"""在主事件循环中按插件生命周期新增或移除动态路由。"""
|
||||
if self._event_loop is None:
|
||||
self._update(plugin_id, action)
|
||||
return
|
||||
target_loop = self._event_loop()
|
||||
if (
|
||||
target_loop is None
|
||||
or not target_loop.is_running()
|
||||
or target_loop.is_closed()
|
||||
):
|
||||
raise RuntimeError("主事件循环未运行，无法更新插件动态路由")
|
||||
try:
|
||||
current_loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
current_loop = None
|
||||
if current_loop is target_loop:
|
||||
self._update(plugin_id, action)
|
||||
return
|
||||
|
||||
completed: Future[None] = Future()
|
||||
dispatch_lock = Lock()
|
||||
dispatch_started = False
|
||||
dispatch_abandoned = False
|
||||
|
||||
def apply_update() -> None:
|
||||
"""在目标 loop 的单个回调中完成路由表与 OpenAPI 投影切换。"""
|
||||
nonlocal dispatch_started
|
||||
with dispatch_lock:
|
||||
if dispatch_abandoned:
|
||||
return
|
||||
dispatch_started = True
|
||||
try:
|
||||
self._update(plugin_id, action)
|
||||
except BaseException as error:
|
||||
completed.set_exception(error)
|
||||
else:
|
||||
completed.set_result(None)
|
||||
|
||||
target_loop.call_soon_threadsafe(apply_update)
|
||||
try:
|
||||
completed.result(timeout=self._dispatch_admission_timeout)
|
||||
except FutureTimeoutError as error:
|
||||
with dispatch_lock:
|
||||
if not dispatch_started:
|
||||
dispatch_abandoned = True
|
||||
raise RuntimeError(
|
||||
"主事件循环未及时接收插件动态路由更新"
|
||||
) from error
|
||||
# 回调一旦开始便不可撤销，等待确定终态以免失败回滚后发生迟到写入。
|
||||
completed.result()
|
||||
|
||||
def _update(self, plugin_id: Optional[str], action: str) -> None:
|
||||
"""执行不可中断的路由表与 OpenAPI 投影更新。"""
|
||||
if action not in {"add", "remove"}:
|
||||
raise ValueError("Action must be 'add' or 'remove'")
|
||||
|
||||
modified = False
|
||||
existing_paths = {
|
||||
path: route
|
||||
for route in self._app.routes
|
||||
if (path := self._route_path(route)) is not None
|
||||
}
|
||||
plugin_ids = [plugin_id] if plugin_id else self._plugin_ids()
|
||||
for current_id in plugin_ids:
|
||||
if self.remove(current_id):
|
||||
modified = True
|
||||
if action != "add":
|
||||
continue
|
||||
for source_api in self._plugin_apis(current_id):
|
||||
api = dict(source_api)
|
||||
api["dependencies"] = list(source_api.get("dependencies") or ())
|
||||
api_path = f"{self._prefix}{api.get('path', '')}"
|
||||
try:
|
||||
api["path"] = api_path
|
||||
allow_anonymous = api.pop("allow_anonymous", False)
|
||||
auth_mode = api.pop("auth", "apikey")
|
||||
dependencies = api.setdefault("dependencies", [])
|
||||
if not allow_anonymous:
|
||||
if (
|
||||
auth_mode == "bear"
|
||||
and Depends(self._verify_token) not in dependencies
|
||||
):
|
||||
dependencies.append(Depends(self._verify_token))
|
||||
elif Depends(self._verify_apikey) not in dependencies:
|
||||
dependencies.append(Depends(self._verify_apikey))
|
||||
# 插件 API 自行决定响应结构，不使用宿主统一 Response 路由。
|
||||
api.setdefault("route_class_override", APIRoute)
|
||||
self._app.router.add_api_route(**api, tags=["plugin"])
|
||||
modified = True
|
||||
self._logger.debug(f"Added plugin route: {api_path}")
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error adding plugin route {api_path}: {str(error)}"
|
||||
)
|
||||
if modified:
|
||||
self.clean(existing_paths)
|
||||
self._app.openapi_schema = None
|
||||
self._app.setup()
|
||||
|
||||
def remove(self, plugin_id: str) -> bool:
|
||||
"""移除指定插件前缀下的全部动态路由。"""
|
||||
if not plugin_id:
|
||||
return False
|
||||
prefix = f"{self._prefix}/{plugin_id}/"
|
||||
routes = [
|
||||
route for route in self._app.routes
|
||||
if (path := self._route_path(route)) is not None
|
||||
and path.startswith(prefix)
|
||||
]
|
||||
removed = False
|
||||
for route in routes:
|
||||
try:
|
||||
self._app.routes.remove(route)
|
||||
removed = True
|
||||
self._logger.debug(f"Removed plugin route: {self._route_path(route)}")
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error removing plugin route {self._route_path(route)}: {str(error)}"
|
||||
)
|
||||
return removed
|
||||
|
||||
@staticmethod
|
||||
def _route_path(route: Any) -> Optional[str]:
|
||||
"""返回公开路由路径，跳过 FastAPI 内部的无路径 include 包装器。"""
|
||||
path = getattr(route, "path", None)
|
||||
return path if isinstance(path, str) else None
|
||||
|
||||
def clean(self, existing_paths: dict) -> None:
|
||||
"""清理 FastAPI 重建时可能重复的受保护文档路由。"""
|
||||
for protected_route in self._protected_routes:
|
||||
try:
|
||||
existing_route = existing_paths.get(protected_route)
|
||||
if existing_route:
|
||||
self._app.routes.remove(existing_route)
|
||||
except Exception as error:
|
||||
self._logger.error(
|
||||
f"Error removing protected route {protected_route}: {str(error)}"
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
"""Web 传输层认证适配器。"""
|
||||
@@ -1,274 +0,0 @@
|
||||
"""把应用安全能力适配为 FastAPI 认证依赖和 Cookie 行为。"""
|
||||
|
||||
import datetime
|
||||
from datetime import timedelta
|
||||
from typing import Annotated, Any, Callable, Optional
|
||||
|
||||
import jwt
|
||||
from fastapi import HTTPException, Request, Response, Security, status
|
||||
from fastapi.security import (
|
||||
APIKeyCookie,
|
||||
APIKeyHeader,
|
||||
APIKeyQuery,
|
||||
HTTPBearer,
|
||||
OAuth2PasswordBearer,
|
||||
)
|
||||
|
||||
from app.runtime.cache import cached
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.schemas.token import TokenPayload
|
||||
|
||||
SuperuserTokenPayloadProvider = Callable[[], TokenPayload]
|
||||
TokenEncoder = Callable[..., str]
|
||||
TokenDecoder = Callable[[str | None, str], TokenPayload]
|
||||
_superuser_token_payload_provider: Optional[SuperuserTokenPayloadProvider] = None
|
||||
_token_encoder: Optional[TokenEncoder] = None
|
||||
_token_decoder: Optional[TokenDecoder] = None
|
||||
JWT_ALGORITHM = "HS256"
|
||||
|
||||
|
||||
oauth2_scheme_manual_error = OAuth2PasswordBearer(
|
||||
auto_error=False,
|
||||
tokenUrl=f"{get_runtime_setting('API_V1_STR')}/login/access-token",
|
||||
)
|
||||
resource_token_cookie = APIKeyCookie(
|
||||
name=get_runtime_setting('PROJECT_NAME'),
|
||||
auto_error=False,
|
||||
scheme_name="resource_token_cookie",
|
||||
)
|
||||
api_token_query = APIKeyQuery(
|
||||
name="token",
|
||||
auto_error=False,
|
||||
scheme_name="api_token_query",
|
||||
)
|
||||
api_key_header = APIKeyHeader(
|
||||
name="X-API-KEY",
|
||||
auto_error=False,
|
||||
scheme_name="api_key_header",
|
||||
)
|
||||
api_key_query = APIKeyQuery(
|
||||
name="apikey",
|
||||
auto_error=False,
|
||||
scheme_name="api_key_query",
|
||||
)
|
||||
openai_bearer_scheme = HTTPBearer(auto_error=False)
|
||||
anthropic_api_key_header = APIKeyHeader(
|
||||
name="x-api-key",
|
||||
auto_error=False,
|
||||
scheme_name="anthropic_api_key_header",
|
||||
)
|
||||
|
||||
|
||||
def set_superuser_token_payload_provider(
|
||||
provider: SuperuserTokenPayloadProvider,
|
||||
) -> None:
|
||||
"""由启动组合根注入 API 密钥认证使用的超级用户载荷来源。"""
|
||||
global _superuser_token_payload_provider
|
||||
_superuser_token_payload_provider = provider
|
||||
|
||||
|
||||
def configure_token_codec(
|
||||
encoder: TokenEncoder,
|
||||
decoder: TokenDecoder,
|
||||
) -> None:
|
||||
"""由组合根注入框架无关的令牌编码与解码能力。"""
|
||||
global _token_encoder, _token_decoder
|
||||
_token_encoder = encoder
|
||||
_token_decoder = decoder
|
||||
|
||||
|
||||
def _encode_token(**claims: Any) -> str:
|
||||
"""使用已注入编码器创建令牌，未装配时给出明确错误。"""
|
||||
if _token_encoder is None:
|
||||
raise RuntimeError("Web 认证令牌编码器尚未配置")
|
||||
return _token_encoder(**claims)
|
||||
|
||||
|
||||
def _decode_token(token: str | None, purpose: str) -> TokenPayload:
|
||||
"""使用已注入解码器验证令牌，未装配时给出明确错误。"""
|
||||
if _token_decoder is None:
|
||||
raise RuntimeError("Web 认证令牌解码器尚未配置")
|
||||
return _token_decoder(token, purpose)
|
||||
|
||||
|
||||
def _get_api_token(
|
||||
token_query: Annotated[str | None, Security(api_token_query)] = None,
|
||||
) -> str | None:
|
||||
"""从 URL 查询参数读取兼容 API Token。"""
|
||||
return token_query
|
||||
|
||||
|
||||
def _get_api_key(
|
||||
key_query: Annotated[str | None, Security(api_key_query)] = None,
|
||||
key_header: Annotated[str | None, Security(api_key_header)] = None,
|
||||
) -> str | None:
|
||||
"""优先从请求头、其次从查询参数读取兼容 API Key。"""
|
||||
return key_header or key_query
|
||||
|
||||
|
||||
@cached(maxsize=1, ttl=600)
|
||||
def _create_superuser_token_payload() -> TokenPayload:
|
||||
"""使用组合根提供器创建 API 密钥调用的超级用户载荷。"""
|
||||
if not _superuser_token_payload_provider:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="认证服务尚未初始化",
|
||||
)
|
||||
try:
|
||||
return _superuser_token_payload_provider()
|
||||
except PermissionError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=str(error) or "用户权限不足",
|
||||
) from error
|
||||
|
||||
|
||||
def set_or_refresh_resource_token_cookie(
|
||||
request: Request,
|
||||
response: Response,
|
||||
payload: TokenPayload,
|
||||
) -> None:
|
||||
"""复用匹配的资源令牌，或为当前身份写入新的安全 Cookie。"""
|
||||
project_name = get_runtime_setting('PROJECT_NAME')
|
||||
resource_token = request.cookies.get(project_name)
|
||||
if resource_token:
|
||||
try:
|
||||
decoded = jwt.decode(
|
||||
resource_token,
|
||||
get_runtime_setting('RESOURCE_SECRET_KEY'),
|
||||
algorithms=[JWT_ALGORITHM],
|
||||
)
|
||||
exp = decoded.get("exp")
|
||||
if exp:
|
||||
remaining_time = datetime.datetime.fromtimestamp(
|
||||
exp,
|
||||
tz=datetime.UTC,
|
||||
) - datetime.datetime.now(datetime.UTC)
|
||||
if remaining_time < timedelta(
|
||||
seconds=(
|
||||
get_runtime_setting(
|
||||
"RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS"
|
||||
)
|
||||
/ 3
|
||||
)
|
||||
):
|
||||
raise jwt.ExpiredSignatureError
|
||||
expected_claims = {
|
||||
"sub": str(payload.sub),
|
||||
"username": payload.username,
|
||||
"super_user": payload.super_user,
|
||||
"level": payload.level,
|
||||
"purpose": "resource",
|
||||
}
|
||||
if any(
|
||||
decoded.get(claim) != value
|
||||
for claim, value in expected_claims.items()
|
||||
):
|
||||
raise jwt.InvalidTokenError("资源令牌身份或权限上下文不匹配")
|
||||
except jwt.PyJWTError:
|
||||
logger.debug("Token error occurred. refreshing token")
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
f"Unexpected error occurred while decoding token: {error}"
|
||||
)
|
||||
else:
|
||||
return
|
||||
|
||||
resource_token = _encode_token(
|
||||
userid=payload.sub,
|
||||
username=payload.username or "",
|
||||
super_user=payload.super_user,
|
||||
expires_delta=timedelta(
|
||||
seconds=get_runtime_setting('RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS')
|
||||
),
|
||||
level=payload.level,
|
||||
purpose="resource",
|
||||
)
|
||||
is_https = (
|
||||
request.url.scheme == "https"
|
||||
or request.headers.get("x-forwarded-proto", "").lower() == "https"
|
||||
)
|
||||
response.set_cookie(
|
||||
key=project_name,
|
||||
value=resource_token,
|
||||
httponly=True,
|
||||
secure=is_https,
|
||||
samesite="lax",
|
||||
)
|
||||
|
||||
|
||||
def _decode_or_http_error(
|
||||
token: str | None,
|
||||
purpose: str,
|
||||
) -> TokenPayload:
|
||||
"""把应用层令牌校验错误转换为 HTTP 403。"""
|
||||
try:
|
||||
return _decode_token(token, purpose)
|
||||
except ValueError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=str(error),
|
||||
) from error
|
||||
|
||||
|
||||
def verify_token(
|
||||
request: Request,
|
||||
response: Response,
|
||||
jwt_token: Annotated[
|
||||
str | None,
|
||||
Security(oauth2_scheme_manual_error),
|
||||
],
|
||||
api_key: Annotated[str | None, Security(_get_api_key)],
|
||||
api_token: Annotated[str | None, Security(_get_api_token)],
|
||||
) -> TokenPayload:
|
||||
"""验证 JWT、API Key 或 API Token，并维护资源 Cookie。"""
|
||||
if jwt_token:
|
||||
payload = _decode_or_http_error(jwt_token, "authentication")
|
||||
set_or_refresh_resource_token_cookie(request, response, payload)
|
||||
return payload
|
||||
if api_key:
|
||||
verify_apikey(api_key)
|
||||
return _create_superuser_token_payload()
|
||||
if api_token:
|
||||
verify_apitoken(api_token)
|
||||
return _create_superuser_token_payload()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def verify_resource_token(
|
||||
resource_token: Annotated[
|
||||
str | None,
|
||||
Security(resource_token_cookie),
|
||||
],
|
||||
) -> TokenPayload:
|
||||
"""验证 Cookie 中携带的资源访问令牌。"""
|
||||
return _decode_or_http_error(resource_token, "resource")
|
||||
|
||||
|
||||
def _verify_key(key: str | None, expected_key: str, key_type: str) -> str:
|
||||
"""校验受信第三方集成使用的固定 API 凭据。"""
|
||||
if not key or key != expected_key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=f"{key_type} 校验不通过",
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def verify_apitoken(
|
||||
token: Annotated[str | None, Security(_get_api_token)],
|
||||
) -> str:
|
||||
"""校验 URL 查询参数中的兼容 API Token。"""
|
||||
return _verify_key(token, get_runtime_setting('API_TOKEN'), "token")
|
||||
|
||||
|
||||
def verify_apikey(
|
||||
apikey: Annotated[str | None, Security(_get_api_key)],
|
||||
) -> str:
|
||||
"""校验请求头或查询参数中的兼容 API Key。"""
|
||||
return _verify_key(apikey, get_runtime_setting('API_TOKEN'), "apikey")
|
||||
+2361
-14
File diff suppressed because it is too large
Load Diff
+23
-109
@@ -1,34 +1,24 @@
|
||||
import asyncio
|
||||
import re
|
||||
import threading
|
||||
from typing import Any, Optional, Tuple
|
||||
|
||||
from app.runtime.execution import run_in_threadpool
|
||||
from app.agent.policy.sanitizer import sanitize_for_host
|
||||
from fastapi.concurrency import run_in_threadpool
|
||||
|
||||
from app.chain import ChainBase
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.message import Message, MessageResponse
|
||||
from app.schemas.notification import ChannelCapabilityManager, ChannelCapability
|
||||
from app.schemas.types import NotificationChannel, MessageType
|
||||
from app.log import logger
|
||||
from app.schemas import Notification
|
||||
from app.schemas.message import (
|
||||
MessageResponse,
|
||||
ChannelCapabilityManager,
|
||||
ChannelCapability,
|
||||
)
|
||||
from app.schemas.types import MessageChannel, NotificationType
|
||||
|
||||
|
||||
class _StreamChain(ChainBase):
|
||||
pass
|
||||
|
||||
|
||||
_PATCH_FILE_HEADER_PATTERN = re.compile(
|
||||
r"\*\*\* (?:Add|Update|Delete) File:\s*(\S+)"
|
||||
)
|
||||
|
||||
|
||||
def _extract_first_patch_path(patch: Optional[str]) -> Optional[str]:
|
||||
"""从补丁文本中提取首个文件路径，作为流式消息展示目标。"""
|
||||
if not patch:
|
||||
return None
|
||||
match = _PATCH_FILE_HEADER_PATTERN.search(patch)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
class StreamingHandler:
|
||||
"""
|
||||
流式Token缓冲管理器
|
||||
@@ -58,7 +48,6 @@ class StreamingHandler:
|
||||
# 流式输出相关状态
|
||||
self._streaming_enabled = False
|
||||
self._flush_task: Optional[asyncio.Task] = None
|
||||
self._streaming_lifecycle_lock = asyncio.Lock()
|
||||
# 当前消息的发送信息（用于编辑消息）
|
||||
self._message_response: Optional[MessageResponse] = None
|
||||
# 已发送给用户的文本（用于追踪增量）
|
||||
@@ -78,8 +67,6 @@ class StreamingHandler:
|
||||
self._allow_dispatch_without_context = False
|
||||
# 非啰嗦模式下的待输出工具统计，等下一段文本到来时再统一补一句摘要
|
||||
self._pending_tool_stats: dict[str, dict[str, Any]] = {}
|
||||
# 本轮已写入缓冲区的工具摘要行，供 Telegram 富文本渲染时做区分样式
|
||||
self._tool_summaries: set[str] = set()
|
||||
|
||||
def set_dispatch_policy(
|
||||
self, allow_dispatch_without_context: bool = False
|
||||
@@ -141,7 +128,6 @@ class StreamingHandler:
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
def reset(self):
|
||||
"""
|
||||
@@ -156,7 +142,6 @@ class StreamingHandler:
|
||||
self._sent_text = ""
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
async def start_streaming(
|
||||
self,
|
||||
@@ -167,28 +152,6 @@ class StreamingHandler:
|
||||
original_message_id: Optional[str] = None,
|
||||
original_chat_id: Optional[str] = None,
|
||||
title: str = "",
|
||||
):
|
||||
"""串行启动流式输出，禁止新一轮覆盖尚未结束的刷新 owner。"""
|
||||
async with self._streaming_lifecycle_lock:
|
||||
await self._start_streaming(
|
||||
channel=channel,
|
||||
source=source,
|
||||
user_id=user_id,
|
||||
username=username,
|
||||
original_message_id=original_message_id,
|
||||
original_chat_id=original_chat_id,
|
||||
title=title,
|
||||
)
|
||||
|
||||
async def _start_streaming(
|
||||
self,
|
||||
channel: Optional[str] = None,
|
||||
source: Optional[str] = None,
|
||||
user_id: Optional[str] = None,
|
||||
username: Optional[str] = None,
|
||||
original_message_id: Optional[str] = None,
|
||||
original_chat_id: Optional[str] = None,
|
||||
title: str = "",
|
||||
):
|
||||
"""
|
||||
启动流式输出。
|
||||
@@ -202,10 +165,6 @@ class StreamingHandler:
|
||||
:param original_message_id: 原始消息ID（如果是回复消息）
|
||||
:param original_chat_id: 原始聊天ID（如果是回复消息）
|
||||
"""
|
||||
if self._flush_task is not None:
|
||||
self._streaming_enabled = False
|
||||
await self._cancel_flush_task()
|
||||
|
||||
self._channel = channel
|
||||
self._source = source
|
||||
self._user_id = user_id
|
||||
@@ -219,7 +178,6 @@ class StreamingHandler:
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
|
||||
# 检查渠道是否支持消息编辑，不支持则仅收集 token 到 buffer，不实时推送
|
||||
if not self._can_stream():
|
||||
@@ -228,7 +186,7 @@ class StreamingHandler:
|
||||
|
||||
# 从渠道能力中获取单条消息最大长度
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
channel_enum = MessageChannel(self._channel)
|
||||
self._max_message_length = ChannelCapabilityManager.get_max_message_length(
|
||||
channel_enum
|
||||
)
|
||||
@@ -240,11 +198,6 @@ class StreamingHandler:
|
||||
logger.debug("流式输出已启动")
|
||||
|
||||
async def stop_streaming(self) -> Tuple[bool, str]:
|
||||
"""串行停止流式输出，并等待本轮刷新与最终消息收口。"""
|
||||
async with self._streaming_lifecycle_lock:
|
||||
return await self._stop_streaming()
|
||||
|
||||
async def _stop_streaming(self) -> Tuple[bool, str]:
|
||||
"""
|
||||
停止流式输出。执行最后一次刷新确保所有内容都已发送。
|
||||
:return: (all_sent, final_text)
|
||||
@@ -289,7 +242,6 @@ class StreamingHandler:
|
||||
self._message_response = None
|
||||
self._msg_start_offset = 0
|
||||
self._pending_tool_stats = {}
|
||||
self._tool_summaries = set()
|
||||
if all_sent:
|
||||
# 所有内容已通过流式发送，清空缓冲区
|
||||
self._buffer = ""
|
||||
@@ -304,14 +256,10 @@ class StreamingHandler:
|
||||
"""
|
||||
记录一次工具调用，供非啰嗦模式下延迟汇总输出。
|
||||
"""
|
||||
recorded_message = sanitize_for_host(tool_message) if tool_message else tool_message
|
||||
recorded_args = sanitize_for_host(tool_kwargs or {})
|
||||
if not isinstance(recorded_args, dict):
|
||||
recorded_args = {}
|
||||
category, target = self._classify_tool_call(
|
||||
tool_name=tool_name,
|
||||
tool_message=recorded_message,
|
||||
tool_kwargs=recorded_args,
|
||||
tool_message=tool_message,
|
||||
tool_kwargs=tool_kwargs or {},
|
||||
)
|
||||
target_values = []
|
||||
if isinstance(target, (list, tuple, set)):
|
||||
@@ -384,8 +332,6 @@ class StreamingHandler:
|
||||
return "file_read", tool_kwargs.get("file_path")
|
||||
if tool_name in {"write_file", "edit_file"}:
|
||||
return "file_write", tool_kwargs.get("file_path")
|
||||
if tool_name == "apply_patch":
|
||||
return "file_write", _extract_first_patch_path(tool_kwargs.get("patch"))
|
||||
if tool_name in {"list_directory", "query_directory_settings"}:
|
||||
return "directory", tool_kwargs.get("path")
|
||||
if tool_name == "browse_webpage":
|
||||
@@ -467,14 +413,11 @@ class StreamingHandler:
|
||||
return ""
|
||||
|
||||
summary = f"（{'，'.join(parts)}）"
|
||||
self._tool_summaries.add(summary)
|
||||
# 摘要前始终保证一个空行，让工具执行信息与正文分属不同段落，
|
||||
# 避免 Markdown 富文本把单个换行折叠成同一段落内的软换行
|
||||
visible_buffer = self._buffer.rstrip(" \t")
|
||||
trailing_newlines = len(visible_buffer) - len(visible_buffer.rstrip("\n"))
|
||||
last_char = visible_buffer[-1:] if visible_buffer.strip() else ""
|
||||
prefix = ""
|
||||
if visible_buffer.strip():
|
||||
prefix = "\n" * max(2 - trailing_newlines, 0)
|
||||
if self._buffer and last_char != "\n":
|
||||
prefix = "\n\n"
|
||||
return f"{prefix}{summary}\n\n"
|
||||
|
||||
@staticmethod
|
||||
@@ -515,35 +458,13 @@ class StreamingHandler:
|
||||
if not self._channel:
|
||||
return False
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
channel_enum = MessageChannel(self._channel)
|
||||
return ChannelCapabilityManager.supports_capability(
|
||||
channel_enum, ChannelCapability.MESSAGE_EDITING
|
||||
)
|
||||
except (ValueError, KeyError):
|
||||
return False
|
||||
|
||||
def _get_rich_message(self, text: str) -> Optional[str]:
|
||||
"""
|
||||
为 Telegram 流式消息返回 Rich Markdown，其他渠道继续使用原有格式。
|
||||
"""
|
||||
if self._channel != NotificationChannel.Telegram.value:
|
||||
return None
|
||||
return self._quote_tool_summary_lines(text)
|
||||
|
||||
def _quote_tool_summary_lines(self, text: str) -> str:
|
||||
"""
|
||||
将缓冲区中的工具摘要整行转换为 Markdown 引用块。
|
||||
|
||||
富文本会把普通段落间的空行折叠成紧凑排版，引用块作为独立 block 类型
|
||||
渲染，保证工具执行信息在 Telegram 上始终与正文有可辨识的视觉分隔。
|
||||
"""
|
||||
if not self._tool_summaries or not text:
|
||||
return text
|
||||
return "\n".join(
|
||||
f"> {line}" if line in self._tool_summaries else line
|
||||
for line in text.split("\n")
|
||||
)
|
||||
|
||||
async def _flush_loop(self):
|
||||
"""
|
||||
定时刷新循环，定期将缓冲区内容发送/编辑到用户
|
||||
@@ -605,17 +526,16 @@ class StreamingHandler:
|
||||
# 第一次发送：发送新消息并获取 message_id
|
||||
response = await run_in_threadpool(
|
||||
chain.send_direct_message,
|
||||
Message(
|
||||
Notification(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
mtype=MessageType.Agent,
|
||||
mtype=NotificationType.Agent,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
original_message_id=self._original_message_id,
|
||||
original_chat_id=self._original_chat_id,
|
||||
title=self._title,
|
||||
text=current_text,
|
||||
rich_message=self._get_rich_message(current_text),
|
||||
save_history=False,
|
||||
),
|
||||
)
|
||||
@@ -652,17 +572,16 @@ class StreamingHandler:
|
||||
if current_text:
|
||||
response = await run_in_threadpool(
|
||||
chain.send_direct_message,
|
||||
Message(
|
||||
Notification(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
mtype=MessageType.Agent,
|
||||
mtype=NotificationType.Agent,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
original_message_id=self._original_message_id,
|
||||
original_chat_id=self._original_chat_id,
|
||||
title=self._title,
|
||||
text=current_text,
|
||||
rich_message=self._get_rich_message(current_text),
|
||||
save_history=False,
|
||||
),
|
||||
)
|
||||
@@ -679,15 +598,10 @@ class StreamingHandler:
|
||||
else:
|
||||
# 后续更新：编辑已有消息
|
||||
try:
|
||||
channel_enum = NotificationChannel(self._channel)
|
||||
channel_enum = MessageChannel(self._channel)
|
||||
except (ValueError, KeyError):
|
||||
return
|
||||
|
||||
metadata = dict(self._message_response.metadata or {})
|
||||
rich_message = self._get_rich_message(current_text)
|
||||
if rich_message:
|
||||
# 通用编辑接口不增加渠道专属参数，通过元数据交给 Telegram 模块消费。
|
||||
metadata["telegram_rich_message"] = rich_message
|
||||
success = await run_in_threadpool(
|
||||
chain.edit_message,
|
||||
channel=channel_enum,
|
||||
@@ -696,7 +610,7 @@ class StreamingHandler:
|
||||
chat_id=self._message_response.chat_id,
|
||||
text=current_text,
|
||||
title=self._title,
|
||||
metadata=metadata,
|
||||
metadata=self._message_response.metadata,
|
||||
)
|
||||
if success:
|
||||
with self._lock:
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
"""Agent Capability 声明与通用入口适配器。"""
|
||||
|
||||
AGENT_ENTRYPOINT_KIND = "agent_entrypoint"
|
||||
AGENT_SERVICE_KIND = "agent_service"
|
||||
AGENT_MANAGER_CAPABILITY_ID = "agent.manager"
|
||||
AGENT_SERVICE_CAPABILITY_ID = "agent.service"
|
||||
MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID = "agent.moviepilot_type"
|
||||
TOOL_FACTORY_CAPABILITY_ID = "agent.tool_factory"
|
||||
@@ -1,221 +0,0 @@
|
||||
"""Agent canonical entrypoint 的 Capability Runtime 适配器。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping
|
||||
|
||||
from app.agent.capabilities import AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND
|
||||
from app.runtime.capabilities.errors import CapabilityAdapterContractError
|
||||
from app.runtime.capabilities.model import (
|
||||
ActivationPolicy,
|
||||
AdapterExecutionMode,
|
||||
CapabilitySpec,
|
||||
SelectorSchema,
|
||||
)
|
||||
from app.runtime.capabilities.registry import CapabilityRegistry
|
||||
from app.runtime.settings import get_runtime_setting, has_runtime_setting
|
||||
|
||||
_DEFAULT_CAPABILITY_ROOT = Path(__file__).resolve().parent
|
||||
_SETTING_SELECTOR = "setting_truthy"
|
||||
|
||||
|
||||
def _validate_setting_selector(config: Mapping[str, Any]) -> None:
|
||||
"""限制 selector 只能读取已声明的应用设置。"""
|
||||
key = config["key"]
|
||||
if not isinstance(key, str) or not key or not has_runtime_setting(key):
|
||||
raise ValueError(f"未知应用设置：{key!r}")
|
||||
|
||||
|
||||
AGENT_SELECTOR_SCHEMAS = {
|
||||
_SETTING_SELECTOR: SelectorSchema(
|
||||
required_fields=frozenset({"key"}),
|
||||
validator=_validate_setting_selector,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _load_entrypoint(spec: CapabilitySpec) -> Any:
|
||||
"""按 manifest 解析 canonical 符号，不创建额外业务对象。"""
|
||||
module_name, symbol_name = spec.entrypoint.split(":", maxsplit=1)
|
||||
module = importlib.import_module(module_name)
|
||||
try:
|
||||
return getattr(module, symbol_name)
|
||||
except AttributeError as error:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint} 未公开 Agent entrypoint"
|
||||
) from error
|
||||
|
||||
|
||||
def _lifecycle_method(spec: CapabilitySpec, candidate: Any, name: str) -> Any:
|
||||
"""读取 Agent Service 必需的异步生命周期方法。"""
|
||||
callback = getattr(candidate, name, None)
|
||||
if not callable(callback):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint} 的 Agent Service 缺少 {name}()"
|
||||
)
|
||||
return callback
|
||||
|
||||
|
||||
class AgentEntrypointAdapter:
|
||||
"""把 canonical Python 符号作为无资源副作用的同步能力发布。"""
|
||||
|
||||
execution_mode = AdapterExecutionMode.SYNC
|
||||
|
||||
@staticmethod
|
||||
def materialize(spec: CapabilitySpec) -> Any:
|
||||
"""按 manifest entrypoint 导入 canonical 符号。"""
|
||||
return _load_entrypoint(spec)
|
||||
|
||||
@staticmethod
|
||||
def create(
|
||||
_spec: CapabilitySpec,
|
||||
implementation: Any,
|
||||
_generation: int,
|
||||
_previous: Any = None,
|
||||
) -> Any:
|
||||
"""发布 canonical 符号本身，不创建第二份业务对象。"""
|
||||
return implementation
|
||||
|
||||
@staticmethod
|
||||
def start(
|
||||
_spec: CapabilitySpec,
|
||||
_candidate: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""entrypoint 不拥有业务资源，初始化由独立 service 能力负责。"""
|
||||
|
||||
@staticmethod
|
||||
def stop(
|
||||
_spec: CapabilitySpec,
|
||||
_instance: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""撤销入口可见性；业务资源由独立 service 能力关闭。"""
|
||||
|
||||
@staticmethod
|
||||
def cleanup(
|
||||
_spec: CapabilitySpec,
|
||||
_candidate: Any,
|
||||
_generation: int,
|
||||
_error: BaseException,
|
||||
) -> None:
|
||||
"""entrypoint 启动无副作用，因此失败候选无需额外释放。"""
|
||||
|
||||
|
||||
class AgentServiceAdapter:
|
||||
"""把具备 initialize/close 的 canonical 对象接入异步资源生命周期。"""
|
||||
|
||||
execution_mode = AdapterExecutionMode.ASYNC
|
||||
|
||||
@staticmethod
|
||||
async def materialize(spec: CapabilitySpec) -> Any:
|
||||
"""在线程中导入 canonical service，避免阻塞应用事件循环。"""
|
||||
return await asyncio.to_thread(_load_entrypoint, spec)
|
||||
|
||||
@staticmethod
|
||||
async def create(
|
||||
_spec: CapabilitySpec,
|
||||
implementation: Any,
|
||||
_generation: int,
|
||||
_previous: Any = None,
|
||||
) -> Any:
|
||||
"""复用 canonical service，不复制其内部队列和后台任务所有权。"""
|
||||
return implementation
|
||||
|
||||
@staticmethod
|
||||
async def start(
|
||||
spec: CapabilitySpec,
|
||||
candidate: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""等待 service 在当前应用事件循环完成初始化。"""
|
||||
result = _lifecycle_method(spec, candidate, "initialize")()
|
||||
if not inspect.isawaitable(result):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.initialize() 必须返回 awaitable"
|
||||
)
|
||||
await result
|
||||
|
||||
@staticmethod
|
||||
async def stop(
|
||||
spec: CapabilitySpec,
|
||||
instance: Any,
|
||||
_generation: int,
|
||||
) -> None:
|
||||
"""等待 service 停止后台任务并释放其资源。"""
|
||||
result = _lifecycle_method(spec, instance, "close")()
|
||||
if not inspect.isawaitable(result):
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 必须返回 awaitable"
|
||||
)
|
||||
converged = await result
|
||||
if converged is False:
|
||||
raise CapabilityAdapterContractError(
|
||||
f"{spec.entrypoint}.close() 返回未收敛，保留 service owner"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def cleanup(
|
||||
spec: CapabilitySpec,
|
||||
candidate: Any,
|
||||
generation: int,
|
||||
_error: BaseException,
|
||||
) -> None:
|
||||
"""初始化失败或关闭竞态时按相同 close 合同释放部分资源。"""
|
||||
await AgentServiceAdapter.stop(spec, candidate, generation)
|
||||
|
||||
|
||||
def _validate_registry(registry: CapabilityRegistry) -> None:
|
||||
"""固定 entrypoint 物化轴与 service 资源轴的声明合同。"""
|
||||
for spec in registry.list_specs():
|
||||
if set(spec.metadata) != {"name"}:
|
||||
raise ValueError(f"{spec.source}: Agent Capability metadata 只能包含 name")
|
||||
if spec.kind == AGENT_ENTRYPOINT_KIND:
|
||||
if spec.activation is not ActivationPolicy.ON_FIRST_USE:
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent entrypoint 必须使用 on_first_use"
|
||||
)
|
||||
if spec.selector is not None or spec.watch:
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent entrypoint 不接受 selector 或 watch"
|
||||
)
|
||||
continue
|
||||
if spec.activation is not ActivationPolicy.WHEN_CONFIGURED:
|
||||
raise ValueError(f"{spec.source}: Agent Service 必须使用 when_configured")
|
||||
selector = spec.selector
|
||||
if selector is None or selector.kind != _SETTING_SELECTOR:
|
||||
raise ValueError(f"{spec.source}: Agent Service 必须声明 setting_truthy")
|
||||
selector_key = str(selector.config["key"])
|
||||
if spec.watch != (selector_key,):
|
||||
raise ValueError(
|
||||
f"{spec.source}: Agent Service watch 必须只包含 selector key"
|
||||
)
|
||||
|
||||
|
||||
def build_agent_capability_registry(
|
||||
roots: Iterable[Path | str] | None = None,
|
||||
) -> CapabilityRegistry:
|
||||
"""发现 data-only Agent manifests，不导入编排器、Provider 或工具实现。"""
|
||||
registry = CapabilityRegistry.discover(
|
||||
tuple(roots) if roots is not None else (_DEFAULT_CAPABILITY_ROOT,),
|
||||
kinds={AGENT_ENTRYPOINT_KIND, AGENT_SERVICE_KIND},
|
||||
selector_schemas=AGENT_SELECTOR_SCHEMAS,
|
||||
)
|
||||
_validate_registry(registry)
|
||||
return registry
|
||||
|
||||
|
||||
def should_run_agent_service(spec: CapabilitySpec) -> bool:
|
||||
"""依据 manifest selector 判断 service 是否应拥有运行实例。"""
|
||||
selector = spec.selector
|
||||
if (
|
||||
spec.kind != AGENT_SERVICE_KIND
|
||||
or selector is None
|
||||
or selector.kind != _SETTING_SELECTOR
|
||||
):
|
||||
raise ValueError(f"{spec.source}: 不是可协调的 Agent Service 声明")
|
||||
return bool(get_runtime_setting(selector.config["key"]))
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.manager"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.orchestrator:agent_manager"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Manager"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.moviepilot_type"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.orchestrator:MoviePilotAgent"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "MoviePilot Agent Type"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,16 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.service"
|
||||
kind = "agent_service"
|
||||
entrypoint = "app.agent.orchestrator:agent_manager"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Service"
|
||||
|
||||
[activation]
|
||||
policy = "when_configured"
|
||||
watch = ["AI_AGENT_ENABLE"]
|
||||
|
||||
[activation.selector]
|
||||
kind = "setting_truthy"
|
||||
key = "AI_AGENT_ENABLE"
|
||||
@@ -1,12 +0,0 @@
|
||||
schema_version = 1
|
||||
id = "agent.tool_factory"
|
||||
kind = "agent_entrypoint"
|
||||
entrypoint = "app.agent.tools.factory:MoviePilotToolFactory"
|
||||
depends_on = []
|
||||
|
||||
[metadata]
|
||||
name = "Agent Tool Factory"
|
||||
|
||||
[activation]
|
||||
policy = "on_first_use"
|
||||
watch = []
|
||||
@@ -1,35 +0,0 @@
|
||||
"""Agent 轻量公共合同，不触发模型、工具或编排运行时加载。"""
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.schemas.types import ReplyMode
|
||||
|
||||
|
||||
def build_display_message(
|
||||
role: str,
|
||||
content: str = "",
|
||||
attachments: Optional[list[dict]] = None,
|
||||
status: str = "done",
|
||||
) -> dict[str, Any]:
|
||||
"""构造前后端共享的 Agent 会话展示消息。"""
|
||||
normalized_content = content or ""
|
||||
return {
|
||||
"id": f"{role}-{uuid.uuid4().hex}",
|
||||
"role": role,
|
||||
"content": normalized_content,
|
||||
"createdAt": int(datetime.now().timestamp() * 1000),
|
||||
"status": status,
|
||||
"tools": [],
|
||||
"segments": (
|
||||
[{"type": "text", "content": normalized_content}]
|
||||
if normalized_content
|
||||
else []
|
||||
),
|
||||
"attachments": attachments or [],
|
||||
"choices": [],
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["ReplyMode", "build_display_message"]
|
||||
+16
-35
@@ -1,39 +1,20 @@
|
||||
"""Agent 内部使用的 LLM 适配层，公开对象按需解析。"""
|
||||
"""Agent 内部使用的 LLM 适配层。"""
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
|
||||
_EXPORT_MODULES = {
|
||||
"LLMHelper": "app.agent.llm.helper",
|
||||
"LLMTestError": "app.agent.llm.helper",
|
||||
"LLMTestTimeout": "app.agent.llm.helper",
|
||||
"AgentCapabilityManager": "app.agent.llm.capability",
|
||||
"AgentCapabilityProvider": "app.agent.llm.capability",
|
||||
"AudioCapabilityProvider": "app.agent.llm.capability",
|
||||
"MiMoAudioProvider": "app.agent.llm.capability",
|
||||
"OpenAIChatAudioProvider": "app.agent.llm.capability",
|
||||
"OpenAIAudioProvider": "app.agent.llm.capability",
|
||||
"LLMProviderAuthError": "app.agent.llm.provider",
|
||||
"LLMProviderError": "app.agent.llm.provider",
|
||||
"LLMProviderManager": "app.agent.llm.provider",
|
||||
"render_auth_result_html": "app.agent.llm.provider",
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""首次访问公开对象时只加载其所属适配模块。"""
|
||||
module_name = _EXPORT_MODULES.get(name)
|
||||
if module_name is None:
|
||||
raise AttributeError(f"module 'app.agent.llm' has no attribute {name!r}")
|
||||
value = getattr(import_module(module_name), name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""让延迟公开对象继续支持交互式发现。"""
|
||||
return sorted(set(globals()) | set(_EXPORT_MODULES))
|
||||
from app.agent.llm.helper import LLMHelper, LLMTestError, LLMTestTimeout
|
||||
from app.agent.llm.capability import (
|
||||
AgentCapabilityManager,
|
||||
AgentCapabilityProvider,
|
||||
AudioCapabilityProvider,
|
||||
MiMoAudioProvider,
|
||||
OpenAIChatAudioProvider,
|
||||
OpenAIAudioProvider,
|
||||
)
|
||||
from app.agent.llm.provider import (
|
||||
LLMProviderAuthError,
|
||||
LLMProviderError,
|
||||
LLMProviderManager,
|
||||
render_auth_result_html,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"LLMHelper",
|
||||
|
||||
+39
-39
@@ -12,11 +12,9 @@ from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
from uuid import uuid4
|
||||
|
||||
from app.application.notification import get_notification_configs
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.utils.http import RequestUtils
|
||||
|
||||
|
||||
class AgentCapabilityProvider(ABC):
|
||||
@@ -60,11 +58,11 @@ class OpenAIAudioProvider(AudioCapabilityProvider):
|
||||
|
||||
@staticmethod
|
||||
def _input_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_INPUT_API_KEY'), get_runtime_setting('AUDIO_INPUT_BASE_URL')
|
||||
return settings.AUDIO_INPUT_API_KEY, settings.AUDIO_INPUT_BASE_URL
|
||||
|
||||
@staticmethod
|
||||
def _output_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_OUTPUT_API_KEY'), get_runtime_setting('AUDIO_OUTPUT_BASE_URL')
|
||||
return settings.AUDIO_OUTPUT_API_KEY, settings.AUDIO_OUTPUT_BASE_URL
|
||||
|
||||
def is_available_for_audio_input(self) -> bool:
|
||||
api_key, _ = self._input_credentials()
|
||||
@@ -88,9 +86,9 @@ class OpenAIAudioProvider(AudioCapabilityProvider):
|
||||
audio_file = BytesIO(content)
|
||||
audio_file.name = filename
|
||||
response = client.audio.transcriptions.create(
|
||||
model=get_runtime_setting('AUDIO_INPUT_MODEL'),
|
||||
model=settings.AUDIO_INPUT_MODEL,
|
||||
file=audio_file,
|
||||
language=get_runtime_setting('AUDIO_INPUT_LANGUAGE') or "zh",
|
||||
language=settings.AUDIO_INPUT_LANGUAGE or "zh",
|
||||
response_format="verbose_json",
|
||||
)
|
||||
text = getattr(response, "text", None)
|
||||
@@ -108,12 +106,12 @@ class OpenAIAudioProvider(AudioCapabilityProvider):
|
||||
if not api_key:
|
||||
raise ValueError("音频输出 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir = settings.TEMP_PATH / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = voice_dir / f"{uuid4().hex}.opus"
|
||||
response = client.audio.speech.create(
|
||||
model=get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
voice=get_runtime_setting('AUDIO_OUTPUT_VOICE'),
|
||||
model=settings.AUDIO_OUTPUT_MODEL,
|
||||
voice=settings.AUDIO_OUTPUT_VOICE,
|
||||
input=text,
|
||||
response_format="opus",
|
||||
)
|
||||
@@ -162,22 +160,22 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
|
||||
@staticmethod
|
||||
def _input_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_INPUT_API_KEY'), get_runtime_setting('AUDIO_INPUT_BASE_URL')
|
||||
return settings.AUDIO_INPUT_API_KEY, settings.AUDIO_INPUT_BASE_URL
|
||||
|
||||
@staticmethod
|
||||
def _output_credentials() -> tuple[Optional[str], Optional[str]]:
|
||||
return get_runtime_setting('AUDIO_OUTPUT_API_KEY'), get_runtime_setting('AUDIO_OUTPUT_BASE_URL')
|
||||
return settings.AUDIO_OUTPUT_API_KEY, settings.AUDIO_OUTPUT_BASE_URL
|
||||
|
||||
def _normalize_stt_model(self) -> str:
|
||||
return self._normalize_model(
|
||||
model=get_runtime_setting('AUDIO_INPUT_MODEL'),
|
||||
model=settings.AUDIO_INPUT_MODEL,
|
||||
supported_models=self.SUPPORTED_STT_MODELS,
|
||||
default_model=self.DEFAULT_STT_MODEL,
|
||||
)
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
return self._normalize_model(
|
||||
model=get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
model=settings.AUDIO_OUTPUT_MODEL,
|
||||
supported_models=self.SUPPORTED_TTS_MODELS,
|
||||
default_model=self.DEFAULT_TTS_MODEL,
|
||||
)
|
||||
@@ -267,7 +265,7 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
return None
|
||||
|
||||
suffix = Path(filename or "").suffix.lower() or ".audio"
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir = settings.TEMP_PATH / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
input_path = voice_dir / f"{uuid4().hex}{suffix}"
|
||||
output_path = input_path.with_suffix(self.TRANSCODED_STT_SUFFIX)
|
||||
@@ -390,7 +388,7 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
if not normalized_audio:
|
||||
return None
|
||||
content, filename = normalized_audio
|
||||
language = (get_runtime_setting('AUDIO_INPUT_LANGUAGE') or "").strip()
|
||||
language = (settings.AUDIO_INPUT_LANGUAGE or "").strip()
|
||||
prompt = "请将这段音频完整转写为文字，只输出转写结果，不要添加解释。"
|
||||
if language:
|
||||
prompt += f"音频主要语言是 {language}。"
|
||||
@@ -425,7 +423,7 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
logger.error(
|
||||
"%s TTS 当前不支持该模型或模型未配置: %s",
|
||||
self.DISPLAY_NAME,
|
||||
get_runtime_setting('AUDIO_OUTPUT_MODEL'),
|
||||
settings.AUDIO_OUTPUT_MODEL,
|
||||
)
|
||||
return None
|
||||
|
||||
@@ -434,7 +432,7 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
if not api_key:
|
||||
raise ValueError("音频输出 provider 未配置 API Key")
|
||||
client = self._build_client(api_key=api_key, base_url=base_url)
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir = settings.TEMP_PATH / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
wav_path = voice_dir / f"{uuid4().hex}.wav"
|
||||
request = {
|
||||
@@ -447,7 +445,7 @@ class OpenAIChatAudioProvider(AudioCapabilityProvider):
|
||||
],
|
||||
"audio": {
|
||||
"format": self.AUDIO_RESPONSE_FORMAT,
|
||||
"voice": get_runtime_setting('AUDIO_OUTPUT_VOICE') or self.DEFAULT_VOICE,
|
||||
"voice": settings.AUDIO_OUTPUT_VOICE or self.DEFAULT_VOICE,
|
||||
},
|
||||
}
|
||||
if self.INCLUDE_AUDIO_MODALITIES:
|
||||
@@ -486,7 +484,7 @@ class MiMoAudioProvider(OpenAIChatAudioProvider):
|
||||
)
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
model = (get_runtime_setting('AUDIO_OUTPUT_MODEL') or "").strip().lower()
|
||||
model = (settings.AUDIO_OUTPUT_MODEL or "").strip().lower()
|
||||
if not model or not model.startswith("mimo-"):
|
||||
return self.DEFAULT_TTS_MODEL
|
||||
return model
|
||||
@@ -545,21 +543,21 @@ class MiniMaxAudioProvider(OpenAIChatAudioProvider):
|
||||
|
||||
def _normalize_stt_model(self) -> str:
|
||||
"""将非 MiniMax 的默认转写模型名兜底为 MiniMax 对话模型。"""
|
||||
model = (get_runtime_setting('AUDIO_INPUT_MODEL') or "").strip()
|
||||
model = (settings.AUDIO_INPUT_MODEL or "").strip()
|
||||
if not model or model.lower().startswith(("gpt-", "mimo-")):
|
||||
return self.DEFAULT_STT_MODEL
|
||||
return model
|
||||
|
||||
def _normalize_tts_model(self) -> str:
|
||||
"""将非 MiniMax 语音模型兜底为官方 T2A 模型。"""
|
||||
model = (get_runtime_setting('AUDIO_OUTPUT_MODEL') or "").strip().lower()
|
||||
model = (settings.AUDIO_OUTPUT_MODEL or "").strip().lower()
|
||||
if model in self.SUPPORTED_TTS_MODELS:
|
||||
return model
|
||||
return self.DEFAULT_TTS_MODEL
|
||||
|
||||
def _normalize_voice_id(self) -> str:
|
||||
"""将其他 provider 的默认音色兜底为 MiniMax 中文系统音色。"""
|
||||
voice_id = (get_runtime_setting('AUDIO_OUTPUT_VOICE') or "").strip()
|
||||
voice_id = (settings.AUDIO_OUTPUT_VOICE or "").strip()
|
||||
if not voice_id or voice_id in {"alloy", "mimo_default"}:
|
||||
return self.DEFAULT_VOICE
|
||||
return voice_id
|
||||
@@ -598,7 +596,7 @@ class MiniMaxAudioProvider(OpenAIChatAudioProvider):
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
},
|
||||
proxies=get_runtime_setting('PROXY') or {},
|
||||
proxies=settings.PROXY or {},
|
||||
timeout=60,
|
||||
).post_res(
|
||||
url=self._build_t2a_url(base_url),
|
||||
@@ -636,7 +634,7 @@ class MiniMaxAudioProvider(OpenAIChatAudioProvider):
|
||||
if not audio_data:
|
||||
raise ValueError("MiniMax T2A 响应中没有音频数据")
|
||||
|
||||
voice_dir = get_runtime_setting('TEMP_PATH') / "voice"
|
||||
voice_dir = settings.TEMP_PATH / "voice"
|
||||
voice_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_path = voice_dir / f"{uuid4().hex}.opus"
|
||||
output_path.write_bytes(self._decode_audio_payload(audio_data))
|
||||
@@ -680,9 +678,9 @@ class AgentCapabilityManager:
|
||||
@classmethod
|
||||
def get_audio_provider(cls, mode: str) -> Optional[AudioCapabilityProvider]:
|
||||
provider_name = cls._normalize_provider_name(
|
||||
get_runtime_setting('AUDIO_INPUT_PROVIDER')
|
||||
settings.AUDIO_INPUT_PROVIDER
|
||||
if (mode or "").lower() == "input"
|
||||
else get_runtime_setting('AUDIO_OUTPUT_PROVIDER')
|
||||
else settings.AUDIO_OUTPUT_PROVIDER
|
||||
)
|
||||
provider = cls._audio_providers.get(provider_name)
|
||||
if provider:
|
||||
@@ -700,12 +698,12 @@ class AgentCapabilityManager:
|
||||
@staticmethod
|
||||
def supports_audio_input() -> bool:
|
||||
"""当前 Agent 是否启用音频输入能力。"""
|
||||
return bool(get_runtime_setting('LLM_SUPPORT_AUDIO_INPUT'))
|
||||
return bool(settings.LLM_SUPPORT_AUDIO_INPUT)
|
||||
|
||||
@staticmethod
|
||||
def supports_audio_output() -> bool:
|
||||
"""当前 Agent 是否启用音频输出能力。"""
|
||||
return bool(get_runtime_setting('LLM_SUPPORT_AUDIO_OUTPUT'))
|
||||
return bool(settings.LLM_SUPPORT_AUDIO_OUTPUT)
|
||||
|
||||
@classmethod
|
||||
def is_audio_input_available(cls) -> bool:
|
||||
@@ -776,20 +774,20 @@ class AgentCapabilityManager:
|
||||
if not channel:
|
||||
return None
|
||||
|
||||
from app.schemas.types import NotificationChannel
|
||||
from app.schemas.types import MessageChannel
|
||||
|
||||
if isinstance(channel, NotificationChannel):
|
||||
if isinstance(channel, MessageChannel):
|
||||
return channel
|
||||
|
||||
channel_text = str(channel).strip()
|
||||
if not channel_text:
|
||||
return None
|
||||
lowered_channel = channel_text.lower()
|
||||
for channel_item in NotificationChannel:
|
||||
for channel_item in MessageChannel:
|
||||
aliases = {
|
||||
channel_item.value.lower(),
|
||||
channel_item.name.lower(),
|
||||
f"{NotificationChannel.__name__}.{channel_item.name}".lower(),
|
||||
f"{MessageChannel.__name__}.{channel_item.name}".lower(),
|
||||
}
|
||||
if lowered_channel in aliases:
|
||||
return channel_item
|
||||
@@ -801,7 +799,9 @@ class AgentCapabilityManager:
|
||||
if not source:
|
||||
return False
|
||||
|
||||
for config in get_notification_configs(include_disabled=True):
|
||||
from app.helper.service import ServiceConfigHelper
|
||||
|
||||
for config in ServiceConfigHelper.get_notification_configs():
|
||||
if config.name != source:
|
||||
continue
|
||||
return (config.config or {}).get("WECHAT_MODE", "app") != "bot"
|
||||
@@ -812,8 +812,8 @@ class AgentCapabilityManager:
|
||||
cls, channel: Optional[str], source: Optional[str]
|
||||
) -> bool:
|
||||
"""判断当前渠道是否支持原生语音消息发送。"""
|
||||
from app.schemas.notification import ChannelCapability, ChannelCapabilityManager
|
||||
from app.schemas.types import NotificationChannel
|
||||
from app.schemas.message import ChannelCapability, ChannelCapabilityManager
|
||||
from app.schemas.types import MessageChannel
|
||||
|
||||
channel_enum = cls._parse_message_channel(channel)
|
||||
if not channel_enum:
|
||||
@@ -824,6 +824,6 @@ class AgentCapabilityManager:
|
||||
):
|
||||
return False
|
||||
|
||||
if channel_enum == NotificationChannel.Wechat:
|
||||
if channel_enum == MessageChannel.Wechat:
|
||||
return cls._is_wechat_app_mode(source)
|
||||
return True
|
||||
|
||||
@@ -1,69 +0,0 @@
|
||||
"""LLM helper 与 provider 实现之间的运行时端口。"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol
|
||||
|
||||
|
||||
class LLMProviderRuntimePort(Protocol):
|
||||
"""声明 LLM helper 与管理 API 共用的 provider 运行时能力。"""
|
||||
|
||||
def resolve_cached_model_metadata(self, **kwargs: Any) -> dict[str, Any] | None:
|
||||
"""从本地目录缓存解析模型元数据。"""
|
||||
...
|
||||
|
||||
async def resolve_runtime(self, **kwargs: Any) -> dict[str, Any]:
|
||||
"""解析创建模型客户端所需的统一运行时参数。"""
|
||||
...
|
||||
|
||||
def create_bedrock_client(self, *args: Any, **kwargs: Any) -> Any:
|
||||
"""创建带统一认证和网络配置的 Bedrock 客户端。"""
|
||||
...
|
||||
|
||||
async def list_models(self, **kwargs: Any) -> list[dict[str, Any]]:
|
||||
"""返回 provider 可用的模型目录。"""
|
||||
...
|
||||
|
||||
def resolve_model_list_base_url(self, **kwargs: Any) -> str | None:
|
||||
"""解析兼容接口用于查询模型列表的基础地址。"""
|
||||
...
|
||||
|
||||
async def provider_manage(
|
||||
self,
|
||||
provider: str,
|
||||
action: str,
|
||||
**params: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""执行与具体提供商无关的统一管理动作。"""
|
||||
...
|
||||
|
||||
async def handle_chatgpt_callback(
|
||||
self,
|
||||
provider_id: str,
|
||||
code: str | None,
|
||||
state: str | None,
|
||||
error: str | None,
|
||||
error_description: str | None,
|
||||
) -> tuple[bool, str]:
|
||||
"""完成 ChatGPT OAuth 回调并返回公开结果。"""
|
||||
...
|
||||
|
||||
|
||||
LLMProviderRuntimeFactory = Callable[[], LLMProviderRuntimePort]
|
||||
_provider_runtime_factory: LLMProviderRuntimeFactory | None = None
|
||||
|
||||
|
||||
def register_llm_provider_runtime(
|
||||
factory: LLMProviderRuntimeFactory | None,
|
||||
) -> LLMProviderRuntimeFactory | None:
|
||||
"""注册 provider 运行时工厂，并返回先前工厂供隔离测试恢复。"""
|
||||
global _provider_runtime_factory
|
||||
previous = _provider_runtime_factory
|
||||
_provider_runtime_factory = factory
|
||||
return previous
|
||||
|
||||
|
||||
def resolve_llm_provider_runtime() -> LLMProviderRuntimePort:
|
||||
"""解析已组装的 provider 运行时，未注册时给出明确边界错误。"""
|
||||
if _provider_runtime_factory is None:
|
||||
raise RuntimeError("LLM provider 运行时尚未由启动层完成组装")
|
||||
return _provider_runtime_factory()
|
||||
+64
-128
@@ -10,9 +10,8 @@ from urllib.parse import urlsplit
|
||||
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk
|
||||
|
||||
from app.agent.llm.gateway import resolve_llm_provider_runtime
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.agent.llm.server_tools import ServerToolResolution
|
||||
@@ -64,7 +63,7 @@ def _patch_gemini_thought_signature():
|
||||
logger.error(
|
||||
f"langchain-google-genai 版本 {_version or '未知'} 过旧，"
|
||||
f"不支持 Gemini 2.5+ 模型的 thought_signature 处理，"
|
||||
f"请恢复 MoviePilot 锁定依赖或更新主程序"
|
||||
f"请升级到 4.2.3+：pip install langchain-google-genai~=4.2.3"
|
||||
)
|
||||
return
|
||||
|
||||
@@ -146,8 +145,8 @@ def _resolve_llm_proxy(use_proxy: bool | None = None) -> str | None:
|
||||
"""
|
||||
解析本次 LLM 调用应使用的系统代理地址。
|
||||
"""
|
||||
should_use_proxy = get_runtime_setting('LLM_USE_PROXY') if use_proxy is None else use_proxy
|
||||
return get_runtime_setting('PROXY_HOST') if should_use_proxy and get_runtime_setting('PROXY_HOST') else None
|
||||
should_use_proxy = settings.LLM_USE_PROXY if use_proxy is None else use_proxy
|
||||
return settings.PROXY_HOST if should_use_proxy and settings.PROXY_HOST else None
|
||||
|
||||
|
||||
def _build_httpx_proxy_kwargs(proxy_url: str | None) -> dict[str, str]:
|
||||
@@ -501,94 +500,6 @@ def _patch_openai_responses_empty_output_support():
|
||||
class LLMHelper:
|
||||
"""LLM模型相关辅助功能"""
|
||||
|
||||
_DEFAULT_MAX_INPUT_TOKENS = 256_000
|
||||
|
||||
@staticmethod
|
||||
def _positive_token_limit(value: Any) -> int | None:
|
||||
"""只接受可直接作为模型窗口上限的正整数。"""
|
||||
return value if type(value) is int and value > 0 else None
|
||||
|
||||
@classmethod
|
||||
def _source_input_limit(cls, source: dict[str, Any]) -> int | None:
|
||||
"""合并同一事实源的 input/context 上限，采用更严格的约束。"""
|
||||
candidates = [
|
||||
cls._positive_token_limit(source.get("input_tokens")),
|
||||
cls._positive_token_limit(source.get("context_tokens")),
|
||||
]
|
||||
valid = [candidate for candidate in candidates if candidate is not None]
|
||||
return min(valid) if valid else None
|
||||
|
||||
@classmethod
|
||||
def _normalize_model_profile(
|
||||
cls,
|
||||
model_profile: Any,
|
||||
runtime: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""把当前端点的窗口事实合并到 LangChain model profile。"""
|
||||
profile = dict(model_profile) if isinstance(model_profile, dict) else {}
|
||||
model_record = runtime.get("model_record") or {}
|
||||
model_metadata = runtime.get("model_metadata") or {}
|
||||
metadata_limit = model_metadata.get("limit") or {}
|
||||
metadata_source = {
|
||||
"input_tokens": metadata_limit.get("input"),
|
||||
"context_tokens": metadata_limit.get("context"),
|
||||
}
|
||||
|
||||
record_input = cls._source_input_limit(model_record)
|
||||
metadata_input = cls._source_input_limit(metadata_source)
|
||||
profile_input = cls._positive_token_limit(profile.get("max_input_tokens"))
|
||||
configured_k = cls._positive_token_limit(get_runtime_setting('LLM_MAX_CONTEXT_TOKENS'))
|
||||
configured_input = configured_k * 1000 if configured_k else None
|
||||
|
||||
endpoint_matched = runtime.get("model_profile_endpoint_matched") is True
|
||||
|
||||
if endpoint_matched:
|
||||
max_input_tokens = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in (
|
||||
record_input,
|
||||
metadata_input,
|
||||
profile_input,
|
||||
configured_input,
|
||||
)
|
||||
if candidate is not None
|
||||
),
|
||||
cls._DEFAULT_MAX_INPUT_TOKENS,
|
||||
)
|
||||
else:
|
||||
constraints = [
|
||||
candidate
|
||||
for candidate in (
|
||||
record_input,
|
||||
metadata_input,
|
||||
profile_input,
|
||||
configured_input,
|
||||
cls._DEFAULT_MAX_INPUT_TOKENS,
|
||||
)
|
||||
if candidate is not None
|
||||
]
|
||||
max_input_tokens = min(constraints)
|
||||
profile["max_input_tokens"] = max_input_tokens
|
||||
|
||||
record_output = (
|
||||
cls._positive_token_limit(model_record.get("output_tokens"))
|
||||
if endpoint_matched
|
||||
else None
|
||||
)
|
||||
metadata_output = (
|
||||
cls._positive_token_limit(metadata_limit.get("output"))
|
||||
if endpoint_matched
|
||||
else None
|
||||
)
|
||||
profile_output = cls._positive_token_limit(profile.get("max_output_tokens"))
|
||||
max_output_tokens = record_output or metadata_output or profile_output
|
||||
if max_output_tokens is not None:
|
||||
profile["max_output_tokens"] = max_output_tokens
|
||||
else:
|
||||
profile.pop("max_output_tokens", None)
|
||||
return profile
|
||||
|
||||
_SUPPORTED_THINKING_LEVELS = frozenset(
|
||||
{"off", "auto", "minimal", "low", "medium", "high", "max", "xhigh"}
|
||||
)
|
||||
@@ -788,20 +699,22 @@ class LLMHelper:
|
||||
base_url_preset: Optional[str] = None,
|
||||
) -> Optional[bool]:
|
||||
"""复用 provider 目录缓存解析当前模型是否支持图片输入。"""
|
||||
provider_name = str(provider if provider is not None else get_runtime_setting('LLM_PROVIDER')).strip()
|
||||
model_name = str(model if model is not None else get_runtime_setting('LLM_MODEL')).strip()
|
||||
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).strip()
|
||||
model_name = str(model if model is not None else settings.LLM_MODEL).strip()
|
||||
if not provider_name or not model_name:
|
||||
return None
|
||||
|
||||
try:
|
||||
metadata = resolve_llm_provider_runtime().resolve_cached_model_metadata(
|
||||
from app.agent.llm.provider import LLMProviderManager
|
||||
|
||||
metadata = LLMProviderManager().resolve_cached_model_metadata(
|
||||
provider_id=provider_name,
|
||||
model_id=model_name,
|
||||
base_url=base_url if base_url is not None else get_runtime_setting('LLM_BASE_URL'),
|
||||
base_url=base_url if base_url is not None else settings.LLM_BASE_URL,
|
||||
base_url_preset_id=(
|
||||
base_url_preset
|
||||
if base_url_preset is not None
|
||||
else get_runtime_setting('LLM_BASE_URL_PRESET')
|
||||
else settings.LLM_BASE_URL_PRESET
|
||||
),
|
||||
)
|
||||
except Exception as err:
|
||||
@@ -826,7 +739,7 @@ class LLMHelper:
|
||||
被兼容端点以 400 拒绝。无参调用保持旧版“只读总开关”语义，
|
||||
未知自定义模型也保持原有开关语义。
|
||||
"""
|
||||
if not get_runtime_setting('LLM_SUPPORT_IMAGE_INPUT'):
|
||||
if not settings.LLM_SUPPORT_IMAGE_INPUT:
|
||||
return False
|
||||
if provider is None and model is None:
|
||||
return True
|
||||
@@ -855,8 +768,8 @@ class LLMHelper:
|
||||
这主要用于单测 stub 环境以及极端的最小运行环境，正常生产路径仍优先
|
||||
走 `LLMProviderManager.resolve_runtime()`。
|
||||
"""
|
||||
api_key_value = api_key if api_key is not None else get_runtime_setting('LLM_API_KEY')
|
||||
base_url_value = base_url if base_url is not None else get_runtime_setting('LLM_BASE_URL')
|
||||
api_key_value = api_key if api_key is not None else settings.LLM_API_KEY
|
||||
base_url_value = base_url if base_url is not None else settings.LLM_BASE_URL
|
||||
if not api_key_value:
|
||||
raise ValueError("未配置LLM API Key")
|
||||
|
||||
@@ -1035,7 +948,7 @@ class LLMHelper:
|
||||
"""
|
||||
规范化 API 协议配置，未知值统一回退为 ``auto`` 以保持兼容。
|
||||
"""
|
||||
normalized = str(api_protocol or get_runtime_setting('LLM_API_PROTOCOL') or "").strip().lower()
|
||||
normalized = str(api_protocol or settings.LLM_API_PROTOCOL or "").strip().lower()
|
||||
if normalized in {"auto", "chat_completions", "responses"}:
|
||||
return normalized
|
||||
if normalized:
|
||||
@@ -1183,20 +1096,24 @@ class LLMHelper:
|
||||
:param prompt_cache_key: 同一 Agent 会话内稳定且脱敏的提示词缓存路由键。
|
||||
:return: LLM实例
|
||||
"""
|
||||
provider_name = str(provider if provider is not None else get_runtime_setting('LLM_PROVIDER')).lower()
|
||||
model_name = model if model is not None else get_runtime_setting('LLM_MODEL')
|
||||
api_key_value = api_key if api_key is not None else get_runtime_setting('LLM_API_KEY')
|
||||
base_url_value = base_url if base_url is not None else get_runtime_setting('LLM_BASE_URL')
|
||||
provider_name = str(provider if provider is not None else settings.LLM_PROVIDER).lower()
|
||||
model_name = model if model is not None else settings.LLM_MODEL
|
||||
api_key_value = api_key if api_key is not None else settings.LLM_API_KEY
|
||||
base_url_value = base_url if base_url is not None else settings.LLM_BASE_URL
|
||||
base_url_preset_value = (
|
||||
base_url_preset if base_url_preset is not None else get_runtime_setting('LLM_BASE_URL_PRESET')
|
||||
base_url_preset if base_url_preset is not None else settings.LLM_BASE_URL_PRESET
|
||||
)
|
||||
user_agent_value = user_agent if user_agent is not None else get_runtime_setting('LLM_USER_AGENT')
|
||||
temperature_value = temperature if temperature is not None else get_runtime_setting('LLM_TEMPERATURE')
|
||||
user_agent_value = user_agent if user_agent is not None else settings.LLM_USER_AGENT
|
||||
temperature_value = temperature if temperature is not None else settings.LLM_TEMPERATURE
|
||||
normalized_thinking_level = cls._resolve_thinking_level(
|
||||
thinking_level=thinking_level,
|
||||
)
|
||||
try:
|
||||
runtime = await resolve_llm_provider_runtime().resolve_runtime(
|
||||
# 延迟导入，避免单测在最小 stub 环境下 import `llm.py` 时被 provider
|
||||
# 目录依赖链拖住。
|
||||
from app.agent.llm.provider import LLMProviderManager
|
||||
|
||||
runtime = await LLMProviderManager().resolve_runtime(
|
||||
provider_id=provider_name,
|
||||
model=model_name,
|
||||
api_key=api_key_value,
|
||||
@@ -1226,12 +1143,12 @@ class LLMHelper:
|
||||
mode=(
|
||||
web_search_mode
|
||||
if web_search_mode is not None
|
||||
else get_runtime_setting("LLM_WEB_SEARCH_MODE", "local")
|
||||
else getattr(settings, "LLM_WEB_SEARCH_MODE", "local")
|
||||
),
|
||||
api_protocol=(
|
||||
api_protocol
|
||||
if api_protocol is not None
|
||||
else get_runtime_setting('LLM_API_PROTOCOL')
|
||||
else settings.LLM_API_PROTOCOL
|
||||
),
|
||||
base_url=runtime.get("base_url"),
|
||||
)
|
||||
@@ -1323,6 +1240,8 @@ class LLMHelper:
|
||||
elif runtime["runtime"] == "bedrock":
|
||||
from langchain_aws import ChatBedrockConverse
|
||||
|
||||
from app.agent.llm.provider import LLMProviderManager
|
||||
|
||||
bedrock_model_cls = ChatBedrockConverse
|
||||
if (
|
||||
str(prompt_cache_key or "").strip()
|
||||
@@ -1337,13 +1256,13 @@ class LLMHelper:
|
||||
aws_auth = runtime.get("aws_auth") or {}
|
||||
# Bearer 认证需要跳过 SigV4 签名并注入 Authorization 头，SigV4 认证
|
||||
# 直接以 AK/SK 签名；两种方式统一由 provider 管理器构造 boto3 客户端。
|
||||
bedrock_client = resolve_llm_provider_runtime().create_bedrock_client(
|
||||
bedrock_client = LLMProviderManager().create_bedrock_client(
|
||||
"bedrock-runtime",
|
||||
region=aws_region,
|
||||
credentials=aws_auth,
|
||||
base_url=runtime.get("base_url"),
|
||||
use_proxy=use_proxy,
|
||||
read_timeout=get_runtime_setting('LLM_TOOL_TIMEOUT'),
|
||||
read_timeout=settings.LLM_TOOL_TIMEOUT,
|
||||
)
|
||||
model = bedrock_model_cls(
|
||||
model_id=model_name,
|
||||
@@ -1409,15 +1328,28 @@ class LLMHelper:
|
||||
**openai_model_kwargs,
|
||||
)
|
||||
|
||||
model.profile = cls._normalize_model_profile(
|
||||
model_profile=getattr(model, "profile", None),
|
||||
runtime=runtime,
|
||||
)
|
||||
# ChatBedrockConverse 等模型类没有 model 属性，模型名存放在 model_id。
|
||||
logged_model_name = getattr(model, "model", None) or getattr(
|
||||
model, "model_id", model_name
|
||||
)
|
||||
logger.debug(f"使用LLM模型: {logged_model_name}，Profile: {model.profile}")
|
||||
# 优先使用 provider / models.dev 目录中的上下文上限，减少用户手填成本。
|
||||
model_profile = getattr(model, "profile", None)
|
||||
if model_profile:
|
||||
# ChatBedrockConverse 等模型类没有 model 属性，模型名存放在 model_id。
|
||||
logged_model_name = getattr(model, "model", None) or getattr(
|
||||
model, "model_id", model_name
|
||||
)
|
||||
logger.debug(f"使用LLM模型: {logged_model_name}，Profile: {model_profile}")
|
||||
else:
|
||||
model_record = runtime.get("model_record") or {}
|
||||
model_metadata = runtime.get("model_metadata") or {}
|
||||
metadata_limit = model_metadata.get("limit") or {}
|
||||
max_input_tokens = (
|
||||
model_record.get("input_tokens")
|
||||
or model_record.get("context_tokens")
|
||||
or metadata_limit.get("input")
|
||||
or metadata_limit.get("context")
|
||||
or settings.LLM_MAX_CONTEXT_TOKENS * 1000
|
||||
)
|
||||
model.profile = {
|
||||
"max_input_tokens": int(max_input_tokens),
|
||||
}
|
||||
|
||||
cls._attach_runtime_metadata(model, runtime)
|
||||
cls._attach_server_tool_metadata(model, server_tool_resolution)
|
||||
@@ -1490,8 +1422,8 @@ class LLMHelper:
|
||||
:param api_protocol: OpenAI 兼容接口 API 协议，未显式传入时沿用已保存配置。
|
||||
:param web_search_mode: 联网搜索模式，未显式传入时沿用已保存配置。
|
||||
"""
|
||||
provider_name = provider if provider is not None else get_runtime_setting('LLM_PROVIDER')
|
||||
model_name = model if model is not None else get_runtime_setting('LLM_MODEL')
|
||||
provider_name = provider if provider is not None else settings.LLM_PROVIDER
|
||||
model_name = model if model is not None else settings.LLM_MODEL
|
||||
start = time.perf_counter()
|
||||
llm_kwargs = {
|
||||
"streaming": False,
|
||||
@@ -1551,7 +1483,9 @@ class LLMHelper:
|
||||
"""
|
||||
logger.info(f"获取 {provider} 模型列表...")
|
||||
try:
|
||||
models = await resolve_llm_provider_runtime().list_models(
|
||||
from app.agent.llm.provider import LLMProviderManager
|
||||
|
||||
models = await LLMProviderManager().list_models(
|
||||
provider_id=provider,
|
||||
api_key=api_key,
|
||||
base_url=base_url,
|
||||
@@ -1580,8 +1514,10 @@ class LLMHelper:
|
||||
base_url=base_url,
|
||||
)
|
||||
try:
|
||||
from app.agent.llm.provider import LLMProviderManager
|
||||
|
||||
model_list_base_url = (
|
||||
resolve_llm_provider_runtime().resolve_model_list_base_url(
|
||||
LLMProviderManager().resolve_model_list_base_url(
|
||||
provider_id=provider,
|
||||
base_url=base_url,
|
||||
base_url_preset_id=base_url_preset,
|
||||
|
||||
File diff suppressed because one or more lines are too long
+25
-209
@@ -20,12 +20,11 @@ import aiofiles
|
||||
import httpx
|
||||
import jwt
|
||||
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.application.configuration import get_configured_system_config
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import LlmProviderAction, SystemConfigKey
|
||||
from app.foundation.singleton import Singleton
|
||||
from app.core.config import settings
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.utils.singleton import Singleton
|
||||
|
||||
|
||||
class LLMProviderError(RuntimeError):
|
||||
@@ -267,7 +266,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
self._models_dev_data: dict[str, Any] | None = None
|
||||
self._models_dev_loaded_at: float = 0
|
||||
self._models_dev_cache_path = (
|
||||
Path(get_runtime_setting('TEMP_PATH')) / "llm_provider_models_dev_cache.json"
|
||||
Path(settings.TEMP_PATH) / "llm_provider_models_dev_cache.json"
|
||||
)
|
||||
|
||||
def _cleanup_auth_sessions_locked(self, now: Optional[float] = None) -> None:
|
||||
@@ -418,7 +417,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
base_url_editable=True,
|
||||
requires_base_url=True,
|
||||
supports_api_key=True,
|
||||
api_key_hint="填写 OpenAI-compatible 服务的 API Key；如服务未启用鉴权，可填写任意占位值。",
|
||||
api_key_hint="通用 OpenAI-compatible 兜底入口，需要手动填写 Base URL。",
|
||||
description="通用 OpenAI-compatible 模型服务。",
|
||||
sort_order=1,
|
||||
),
|
||||
@@ -1448,33 +1447,6 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
|
||||
return spec.models_dev_provider_id
|
||||
|
||||
@classmethod
|
||||
def _is_model_profile_endpoint_matched(
|
||||
cls,
|
||||
spec: ProviderSpec,
|
||||
base_url: Optional[str],
|
||||
base_url_preset_id: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""判断模型目录上限是否与当前 provider 端点具有明确对应关系。"""
|
||||
if spec.id == "openai":
|
||||
return False
|
||||
|
||||
preset = cls._resolve_provider_preset(spec, base_url, base_url_preset_id)
|
||||
if preset:
|
||||
effective_base_url = (
|
||||
cls._sanitize_base_url(base_url)
|
||||
or cls._default_base_url_for_provider(spec)
|
||||
)
|
||||
preset_base_url = cls._sanitize_base_url(preset.value)
|
||||
return effective_base_url == preset_base_url
|
||||
|
||||
default_base_url = cls._default_base_url_for_provider(spec)
|
||||
effective_base_url = cls._sanitize_base_url(base_url)
|
||||
if not effective_base_url and not default_base_url:
|
||||
return bool(spec.models_dev_provider_id)
|
||||
effective_base_url = effective_base_url or default_base_url
|
||||
return bool(default_base_url and effective_base_url == default_base_url)
|
||||
|
||||
def resolve_model_list_base_url(
|
||||
self,
|
||||
provider_id: str,
|
||||
@@ -1497,19 +1469,19 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
|
||||
def _build_httpx_kwargs(self, use_proxy: Optional[bool] = None) -> dict[str, Any]:
|
||||
"""构造用于 httpx 客户端的参数，如代理等。"""
|
||||
should_use_proxy = get_runtime_setting('LLM_USE_PROXY') if use_proxy is None else use_proxy
|
||||
should_use_proxy = settings.LLM_USE_PROXY if use_proxy is None else use_proxy
|
||||
kwargs: dict[str, Any] = {
|
||||
"timeout": self._DEFAULT_TIMEOUT,
|
||||
"trust_env": False,
|
||||
}
|
||||
if should_use_proxy and get_runtime_setting('PROXY_HOST'):
|
||||
kwargs[self._httpx_proxy_key()] = get_runtime_setting('PROXY_HOST')
|
||||
if should_use_proxy and settings.PROXY_HOST:
|
||||
kwargs[self._httpx_proxy_key()] = settings.PROXY_HOST
|
||||
return kwargs
|
||||
|
||||
@staticmethod
|
||||
def _read_agent_config() -> dict[str, Any]:
|
||||
"""读取 AI Agent 配置信息。"""
|
||||
config = get_configured_system_config().get(SystemConfigKey.AIAgentConfig)
|
||||
config = SystemConfigOper().get(SystemConfigKey.AIAgentConfig)
|
||||
if isinstance(config, dict):
|
||||
return config
|
||||
return {}
|
||||
@@ -1519,10 +1491,10 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
"""
|
||||
使用异步持久化写回 provider 鉴权配置。
|
||||
|
||||
`get_configured_system_config().get()` 读取的是内存缓存，这里保留同步调用；
|
||||
`SystemConfigOper().get()` 读取的是内存缓存，这里保留同步调用；
|
||||
但写入需要落库，因此统一走 `async_set()`。
|
||||
"""
|
||||
await get_configured_system_config().async_set(
|
||||
await SystemConfigOper().async_set(
|
||||
SystemConfigKey.AIAgentConfig,
|
||||
copy.deepcopy(value) or None,
|
||||
)
|
||||
@@ -1615,7 +1587,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
|
||||
async def _fetch_models_dev(self, use_proxy: Optional[bool] = None) -> dict[str, Any]:
|
||||
"""通过网络请求获取最新 models.dev 数据。"""
|
||||
headers = {"User-Agent": get_runtime_setting('USER_AGENT')}
|
||||
headers = {"User-Agent": settings.USER_AGENT}
|
||||
async with httpx.AsyncClient(**self._build_httpx_kwargs(use_proxy)) as client:
|
||||
response = await client.get(self._MODELS_DEV_URL, headers=headers)
|
||||
response.raise_for_status()
|
||||
@@ -2042,10 +2014,10 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
from google import genai
|
||||
from google.genai.types import HttpOptions
|
||||
|
||||
should_use_proxy = get_runtime_setting('LLM_USE_PROXY') if use_proxy is None else use_proxy
|
||||
should_use_proxy = settings.LLM_USE_PROXY if use_proxy is None else use_proxy
|
||||
client_args: dict[str, Any] = {"trust_env": False}
|
||||
if should_use_proxy and get_runtime_setting('PROXY_HOST'):
|
||||
client_args[self._httpx_proxy_key()] = get_runtime_setting('PROXY_HOST')
|
||||
if should_use_proxy and settings.PROXY_HOST:
|
||||
client_args[self._httpx_proxy_key()] = settings.PROXY_HOST
|
||||
http_options = HttpOptions(
|
||||
client_args=client_args,
|
||||
async_client_args=client_args,
|
||||
@@ -2159,10 +2131,10 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
"""
|
||||
from botocore.config import Config
|
||||
|
||||
should_use_proxy = get_runtime_setting('LLM_USE_PROXY') if use_proxy is None else use_proxy
|
||||
should_use_proxy = settings.LLM_USE_PROXY if use_proxy is None else use_proxy
|
||||
proxies = None
|
||||
if should_use_proxy and get_runtime_setting('PROXY_HOST'):
|
||||
proxies = {"http": get_runtime_setting('PROXY_HOST'), "https": get_runtime_setting('PROXY_HOST')}
|
||||
if should_use_proxy and settings.PROXY_HOST:
|
||||
proxies = {"http": settings.PROXY_HOST, "https": settings.PROXY_HOST}
|
||||
return Config(
|
||||
connect_timeout=10,
|
||||
read_timeout=60,
|
||||
@@ -2387,7 +2359,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
仅补充 Copilot 必需的意图头，避免重复覆盖。
|
||||
"""
|
||||
headers = {
|
||||
"User-Agent": get_runtime_setting('USER_AGENT'),
|
||||
"User-Agent": settings.USER_AGENT,
|
||||
"Openai-Intent": "conversation-edits",
|
||||
"x-initiator": "user",
|
||||
}
|
||||
@@ -2768,7 +2740,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
f"{self._CHATGPT_ISSUER}/api/accounts/deviceauth/usercode",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": get_runtime_setting('USER_AGENT'),
|
||||
"User-Agent": settings.USER_AGENT,
|
||||
},
|
||||
json={"client_id": self._CHATGPT_CLIENT_ID},
|
||||
)
|
||||
@@ -2805,7 +2777,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": get_runtime_setting('USER_AGENT'),
|
||||
"User-Agent": settings.USER_AGENT,
|
||||
},
|
||||
json={
|
||||
"client_id": self._COPILOT_CLIENT_ID,
|
||||
@@ -2960,155 +2932,6 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
self._mark_session_error(session, str(err))
|
||||
return self.get_session_status(session_id)
|
||||
|
||||
async def provider_manage(self, provider: str, action: str, **params: Any) -> Dict[str, Any]:
|
||||
"""
|
||||
LLM 提供商统一管理入口。
|
||||
|
||||
按公共动作词汇表分发，统一返回 {"success", "message", "data"}，
|
||||
临时配置默认值填充、密钥脱敏与错误归因改写均封闭在此，
|
||||
上层透传时无需感知任何提供商特色。
|
||||
"""
|
||||
normalized = action.value if isinstance(action, LlmProviderAction) else str(action)
|
||||
try:
|
||||
if normalized == LlmProviderAction.LIST_PROVIDERS.value:
|
||||
return {"success": True, "message": "", "data": await self.list_providers_async()}
|
||||
if normalized == LlmProviderAction.LIST_MODELS.value:
|
||||
return await self._manage_list_models(provider, **params)
|
||||
if normalized == LlmProviderAction.START_AUTH.value:
|
||||
data = await self.start_auth(
|
||||
provider, str(params.get("method") or ""), params.get("callback_url")
|
||||
)
|
||||
return {"success": True, "message": "", "data": data}
|
||||
if normalized == LlmProviderAction.AUTH_STATUS.value:
|
||||
data = self.get_session_status(str(params.get("session_id") or ""))
|
||||
return {"success": True, "message": "", "data": data}
|
||||
if normalized == LlmProviderAction.POLL_AUTH.value:
|
||||
data = await self.poll_auth_session(str(params.get("session_id") or ""))
|
||||
return {"success": True, "message": "", "data": data}
|
||||
if normalized == LlmProviderAction.DISCONNECT.value:
|
||||
await self.clear_auth(provider)
|
||||
return {"success": True, "message": "", "data": None}
|
||||
if normalized == LlmProviderAction.TEST.value:
|
||||
return await self._manage_test(provider, **params)
|
||||
return {"success": False, "message": f"不支持的管理动作：{normalized}", "data": None}
|
||||
except Exception as err:
|
||||
return {"success": False, "message": self._sanitize_error(str(err)), "data": None}
|
||||
|
||||
async def _manage_list_models(self, provider: str, **params: Any) -> Dict[str, Any]:
|
||||
"""管理动作：查询模型目录，附带授权状态摘要。"""
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
|
||||
api_key = params.get("api_key")
|
||||
try:
|
||||
models = await LLMHelper().get_models(
|
||||
provider=provider,
|
||||
api_key=api_key,
|
||||
base_url=params.get("base_url"),
|
||||
base_url_preset=params.get("base_url_preset"),
|
||||
user_agent=params.get("user_agent"),
|
||||
use_proxy=params.get("use_proxy"),
|
||||
force_refresh=bool(params.get("force_refresh", False)),
|
||||
)
|
||||
except Exception as err:
|
||||
return {"success": False, "message": self._sanitize_error(str(err), api_key), "data": None}
|
||||
return {
|
||||
"success": True,
|
||||
"message": "",
|
||||
"data": {
|
||||
"provider": provider,
|
||||
"models": models,
|
||||
"auth_status": self.get_auth_status(provider),
|
||||
},
|
||||
}
|
||||
|
||||
def _requires_api_key(self, provider_id: str) -> bool:
|
||||
"""判断测试调用是否必须 API Key：支持 OAuth 授权或已有保存凭据的提供商可豁免。"""
|
||||
try:
|
||||
spec = self.get_provider(provider_id)
|
||||
except Exception:
|
||||
return True
|
||||
if self.get_saved_auth(provider_id):
|
||||
return False
|
||||
return not spec.oauth_methods
|
||||
|
||||
async def _manage_test(self, provider: str, **params: Any) -> Dict[str, Any]:
|
||||
"""管理动作：使用传入配置或当前已保存配置执行一次最小 LLM 调用。"""
|
||||
from app.agent.llm.helper import LLMHelper, LLMTestTimeout
|
||||
|
||||
provider_name = provider or get_runtime_setting('LLM_PROVIDER')
|
||||
model = params.get("model") if params.get("model") is not None else get_runtime_setting('LLM_MODEL')
|
||||
enabled = params.get("enabled")
|
||||
enabled = bool(enabled) if enabled is not None else bool(get_runtime_setting('AI_AGENT_ENABLE'))
|
||||
api_key = params.get("api_key") if params.get("api_key") is not None else get_runtime_setting('LLM_API_KEY')
|
||||
|
||||
data = {"provider": provider_name, "model": model}
|
||||
if not provider_name:
|
||||
return {"success": False, "message": "请配置LLM提供商和模型", "data": None}
|
||||
if not model or not model.strip():
|
||||
return {"success": False, "message": "请先配置 LLM 模型", "data": None}
|
||||
if not enabled:
|
||||
return {"success": False, "message": "请先启用智能助手", "data": data}
|
||||
if self._requires_api_key(provider_name) and (not api_key or not api_key.strip()):
|
||||
return {"success": False, "message": "请先配置 LLM API Key", "data": data}
|
||||
|
||||
test_kwargs: Dict[str, Any] = {
|
||||
"provider": provider_name,
|
||||
"model": model,
|
||||
"thinking_level": params.get("thinking_level"),
|
||||
"api_key": api_key,
|
||||
"base_url": params.get("base_url"),
|
||||
"base_url_preset": params.get("base_url_preset"),
|
||||
"user_agent": params.get("user_agent"),
|
||||
"use_proxy": params.get("use_proxy"),
|
||||
"api_protocol": params.get("api_protocol"),
|
||||
"web_search_mode": params.get("web_search_mode"),
|
||||
}
|
||||
if params.get("temperature") is not None:
|
||||
test_kwargs["temperature"] = params.get("temperature")
|
||||
|
||||
try:
|
||||
result = await LLMHelper.test_current_settings(**test_kwargs)
|
||||
except (LLMTestTimeout, TimeoutError) as err:
|
||||
logger.warning(err)
|
||||
return {"success": False, "message": "LLM 调用超时", "data": None}
|
||||
except Exception as err:
|
||||
return {"success": False, "message": self._sanitize_error(str(err), api_key), "data": None}
|
||||
if not result.get("reply_preview"):
|
||||
return {"success": False, "message": "模型响应为空", "data": result}
|
||||
return {"success": True, "message": "", "data": result}
|
||||
|
||||
@staticmethod
|
||||
def _sanitize_error(message: str, api_key: Optional[str] = None) -> str:
|
||||
"""清理错误信息中的敏感字段，并把 SDK 内部解析错误改写为可定位的基础地址提示。"""
|
||||
if not message:
|
||||
return "LLM 没有返回任何内容"
|
||||
|
||||
sanitized = message
|
||||
if api_key:
|
||||
sanitized = sanitized.replace(api_key, "***")
|
||||
sanitized = re.sub(
|
||||
r"(?i)(api[_-]?key\s*[:=]\s*)([^\s,;]+)",
|
||||
r"\1***",
|
||||
sanitized,
|
||||
)
|
||||
sanitized = re.sub(
|
||||
r"(?i)authorization\s*:\s*bearer\s+[^\s,;]+",
|
||||
"Authorization: ***",
|
||||
sanitized,
|
||||
)
|
||||
|
||||
normalized_message = sanitized.lower().replace("_", "").replace(" ", "")
|
||||
if "str" in normalized_message and (
|
||||
"modeldump" in normalized_message
|
||||
or "setprivateattributes" in normalized_message
|
||||
):
|
||||
return (
|
||||
"服务返回内容不是兼容的模型响应，请检查基础地址是否填写为 "
|
||||
"API Base URL，如果服务要求 /v1 等版本路径，请包含在基础地址中，"
|
||||
"不要填写网页地址或完整的 chat/completions 路径"
|
||||
)
|
||||
return sanitized
|
||||
|
||||
async def _exchange_chatgpt_code_for_tokens(
|
||||
self, code: str, redirect_uri: str, code_verifier: str
|
||||
) -> dict[str, Any]:
|
||||
@@ -3150,7 +2973,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
f"{self._CHATGPT_ISSUER}/api/accounts/deviceauth/token",
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": get_runtime_setting('USER_AGENT'),
|
||||
"User-Agent": settings.USER_AGENT,
|
||||
},
|
||||
json={
|
||||
"device_auth_id": session.context["device_auth_id"],
|
||||
@@ -3195,7 +3018,7 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": get_runtime_setting('USER_AGENT'),
|
||||
"User-Agent": settings.USER_AGENT,
|
||||
},
|
||||
json={
|
||||
"client_id": self._COPILOT_CLIENT_ID,
|
||||
@@ -3316,13 +3139,6 @@ class LLMProviderManager(metaclass=Singleton):
|
||||
"model_id": model,
|
||||
"model_record": model_record,
|
||||
"model_metadata": model_metadata,
|
||||
"model_profile_endpoint_matched": (
|
||||
self._is_model_profile_endpoint_matched(
|
||||
spec,
|
||||
base_url,
|
||||
base_url_preset_id=normalized_base_url_preset_id,
|
||||
)
|
||||
),
|
||||
"supports_prompt_cache": self._metadata_supports_prompt_cache(
|
||||
model_metadata
|
||||
),
|
||||
|
||||
+13
-11
@@ -12,15 +12,15 @@ from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from app.application.configuration import get_configured_system_config
|
||||
from app.runtime.log import logger
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.log import logger
|
||||
from app.schemas.agent import (
|
||||
AgentMcpServerConfig,
|
||||
AgentMcpServerTestResult,
|
||||
AgentMcpServerToolInfo,
|
||||
)
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.adapters.network.http import AsyncRequestUtils
|
||||
from app.utils.http import AsyncRequestUtils
|
||||
|
||||
MCP_PROTOCOL_VERSION = "2025-11-25"
|
||||
MCP_CLIENT_NAME = "MoviePilot Agent"
|
||||
@@ -217,11 +217,8 @@ class _StdioMcpSession:
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb) -> None:
|
||||
"""结束 stdio MCP 子进程。"""
|
||||
stderr_task = self.stderr_task
|
||||
self.stderr_task = None
|
||||
if stderr_task:
|
||||
stderr_task.cancel()
|
||||
await asyncio.gather(stderr_task, return_exceptions=True)
|
||||
if self.stderr_task:
|
||||
self.stderr_task.cancel()
|
||||
if not self.process:
|
||||
return
|
||||
if self.process.returncode is None:
|
||||
@@ -456,7 +453,7 @@ class AgentMcpManager:
|
||||
|
||||
def get_servers(self) -> list[AgentMcpServerConfig]:
|
||||
"""读取已保存的外部 MCP 服务器配置。"""
|
||||
raw_servers = get_configured_system_config().get(SystemConfigKey.AIAgentMcpServers) or []
|
||||
raw_servers = SystemConfigOper().get(SystemConfigKey.AIAgentMcpServers) or []
|
||||
if not isinstance(raw_servers, list):
|
||||
return []
|
||||
servers: list[AgentMcpServerConfig] = []
|
||||
@@ -470,7 +467,7 @@ class AgentMcpManager:
|
||||
async def save_servers(self, servers: list[AgentMcpServerConfig]) -> bool:
|
||||
"""保存外部 MCP 服务器配置。"""
|
||||
normalized_servers = [self.normalize_server(server).model_dump() for server in servers]
|
||||
return await get_configured_system_config().async_set(
|
||||
return await SystemConfigOper().async_set(
|
||||
SystemConfigKey.AIAgentMcpServers,
|
||||
normalized_servers or None,
|
||||
)
|
||||
@@ -544,14 +541,19 @@ class AgentMcpManager:
|
||||
return tool_specs
|
||||
|
||||
async def list_enabled_tool_specs(self) -> list[AgentMcpToolSpec]:
|
||||
"""读取所有启用 MCP 服务器暴露的工具定义并保留同名冲突。"""
|
||||
"""读取所有启用 MCP 服务器暴露的工具定义。"""
|
||||
tool_specs: list[AgentMcpToolSpec] = []
|
||||
seen_names: set[str] = set()
|
||||
for server in self.get_servers():
|
||||
if not server.enabled:
|
||||
continue
|
||||
try:
|
||||
for spec in await self.list_server_tools(server):
|
||||
if spec.agent_tool_name in seen_names:
|
||||
logger.warning(f"跳过重复的 MCP Agent 工具名: {spec.agent_tool_name}")
|
||||
continue
|
||||
tool_specs.append(spec)
|
||||
seen_names.add(spec.agent_tool_name)
|
||||
except Exception as err:
|
||||
logger.warning(f"读取 MCP 服务器 {server.name} 工具失败: {err}")
|
||||
return tool_specs
|
||||
|
||||
@@ -6,13 +6,9 @@ from typing import Dict, List, Optional
|
||||
|
||||
from langchain_core.messages import BaseMessage, messages_from_dict, messages_to_dict
|
||||
|
||||
from app.application.agentdata import get_agent_chat_port
|
||||
from app.application.messaging.chat import (
|
||||
get_configured_agent_chat_persistence,
|
||||
get_configured_agent_chat_service,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.core.config import settings
|
||||
from app.db.agentchat_oper import AgentChatOper
|
||||
from app.log import logger
|
||||
from app.schemas.agent import ConversationMemory
|
||||
|
||||
|
||||
@@ -84,45 +80,9 @@ class MemoryManager:
|
||||
return memory.messages
|
||||
|
||||
try:
|
||||
chat = get_agent_chat_port().get(session_id=session_id, user_id=user_id)
|
||||
chat = AgentChatOper().get(session_id=session_id, user_id=user_id)
|
||||
if not chat:
|
||||
chat = get_agent_chat_port().get(session_id=session_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"读取持久化Agent会话失败: {e}")
|
||||
return []
|
||||
if not chat or not chat.agent_messages:
|
||||
return []
|
||||
|
||||
try:
|
||||
messages = messages_from_dict(chat.agent_messages)
|
||||
except Exception as e:
|
||||
logger.debug(f"恢复持久化Agent消息失败: {e}")
|
||||
return []
|
||||
|
||||
memory = ConversationMemory(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
messages=messages,
|
||||
)
|
||||
self.save_memory(memory)
|
||||
return memory.messages
|
||||
|
||||
async def async_get_agent_messages(
|
||||
self, session_id: str, user_id: str
|
||||
) -> List[BaseMessage]:
|
||||
"""异步恢复 Agent 消息，查询与会话应用服务保持同一异步端口。"""
|
||||
memory = self.get_memory(session_id, user_id)
|
||||
if memory:
|
||||
return memory.messages
|
||||
|
||||
try:
|
||||
service = get_configured_agent_chat_service()
|
||||
chat = await service.get(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
)
|
||||
if not chat:
|
||||
chat = await service.get(session_id=session_id)
|
||||
chat = AgentChatOper().get(session_id=session_id)
|
||||
except Exception as e:
|
||||
logger.debug(f"读取持久化Agent会话失败: {e}")
|
||||
return []
|
||||
@@ -159,28 +119,7 @@ class MemoryManager:
|
||||
# 更新内存缓存
|
||||
self.save_memory(memory)
|
||||
try:
|
||||
get_agent_chat_port().save_agent_messages(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
messages=messages_to_dict(messages),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"持久化Agent消息失败: {e}")
|
||||
|
||||
async def async_save_agent_messages(
|
||||
self, session_id: str, user_id: str, messages: List[BaseMessage]
|
||||
) -> None:
|
||||
"""异步保存 Agent 消息，持久化写入经有界数据库 worker 承接。"""
|
||||
memory = self.get_memory(session_id, user_id)
|
||||
if not memory:
|
||||
memory = ConversationMemory(session_id=session_id, user_id=user_id)
|
||||
|
||||
memory.messages = messages
|
||||
memory.updated_at = datetime.now()
|
||||
self.save_memory(memory)
|
||||
try:
|
||||
persistence = get_configured_agent_chat_persistence()
|
||||
await persistence.async_save_agent_messages(
|
||||
AgentChatOper().save_agent_messages(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
messages=messages_to_dict(messages),
|
||||
@@ -226,7 +165,7 @@ class MemoryManager:
|
||||
for cache_key, memory in self.memory_cache.items():
|
||||
if (
|
||||
current_time - memory.updated_at
|
||||
).days > get_runtime_setting('LLM_MEMORY_RETENTION_DAYS'):
|
||||
).days > settings.LLM_MEMORY_RETENTION_DAYS:
|
||||
expired_sessions.append(cache_key)
|
||||
|
||||
# 只清理内存缓存，不删除Redis中的键（Redis会自动过期）
|
||||
|
||||
@@ -33,14 +33,8 @@ from langgraph.runtime import Runtime
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.middleware.utils import append_to_system_message
|
||||
from app.agent.policy.sanitizer import (
|
||||
sanitize_for_host,
|
||||
summarize_error,
|
||||
summarize_result,
|
||||
)
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.tasks import TaskRegistry, get_task_registry
|
||||
from app.log import logger
|
||||
|
||||
# 活动日志保留天数
|
||||
DEFAULT_RETENTION_DAYS = 7
|
||||
@@ -92,17 +86,6 @@ SUMMARY_PROMPT = """请判断以下 AI 助手与用户的对话是否值得写
|
||||
ACTIVITY_ENTRY_PATTERN = re.compile(r"^-\s+\*\*(?P<time>\d{2}:\d{2})\*\*\s+(?P<summary>.+)$")
|
||||
|
||||
|
||||
def _write_activity_log_exclusive(path: Path, content: str) -> bool:
|
||||
"""同步独占创建日志文件；调用方必须在线程池中执行本函数。"""
|
||||
try:
|
||||
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
return False
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
stream.write(content)
|
||||
return True
|
||||
|
||||
|
||||
class QueryActivityLogInput(BaseModel):
|
||||
"""查询活动日志工具的输入参数模型。"""
|
||||
|
||||
@@ -198,9 +181,7 @@ def load_activity_log_index(activity_dir: str, days: int = PROMPT_LOAD_DAYS) ->
|
||||
try:
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"读取活动日志索引失败 {log_path}: {summarize_error(e)}"
|
||||
)
|
||||
logger.warning(f"读取活动日志索引失败 {log_path}: {e}")
|
||||
continue
|
||||
entry_count = len(_parse_activity_entries(date_str, content))
|
||||
if entry_count:
|
||||
@@ -264,7 +245,7 @@ def query_activity_logs(
|
||||
try:
|
||||
content = log_path.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as e:
|
||||
logger.warning(f"读取活动日志失败 {log_path}: {summarize_error(e)}")
|
||||
logger.warning(f"读取活动日志失败 {log_path}: {e}")
|
||||
continue
|
||||
for entry in _parse_activity_entries(date_str, content):
|
||||
if normalized_keyword and not _activity_summary_matches_keyword(
|
||||
@@ -306,16 +287,14 @@ class _ActivityLogToolProvider:
|
||||
limit: Optional[int] = DEFAULT_QUERY_LIMIT,
|
||||
) -> str:
|
||||
"""查询活动日志并返回 JSON 字符串。"""
|
||||
logged_args = sanitize_for_host(
|
||||
{
|
||||
"keyword": keyword,
|
||||
"use_regex": use_regex,
|
||||
"date": date,
|
||||
"days": days,
|
||||
"limit": limit,
|
||||
}
|
||||
logger.info(
|
||||
"查询活动日志: keyword=%s, use_regex=%s, date=%s, days=%s, limit=%s",
|
||||
keyword,
|
||||
use_regex,
|
||||
date,
|
||||
days,
|
||||
limit,
|
||||
)
|
||||
logger.info(f"查询活动日志: args={logged_args}")
|
||||
try:
|
||||
payload = query_activity_logs(
|
||||
self._activity_dir,
|
||||
@@ -327,12 +306,11 @@ class _ActivityLogToolProvider:
|
||||
)
|
||||
return json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
except Exception as err:
|
||||
error_summary = summarize_error(err)
|
||||
logger.error(f"查询活动日志失败: {error_summary}")
|
||||
logger.error(f"查询活动日志失败: {err}", exc_info=True)
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"查询活动日志时发生错误: {error_summary}",
|
||||
"message": f"查询活动日志时发生错误: {str(err)}",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -464,7 +442,7 @@ async def _summarize_with_llm(conversation_text: str) -> Optional[str]:
|
||||
LLM 生成的摘要字符串，失败时返回 None。
|
||||
"""
|
||||
try:
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
from app.agent.llm import LLMHelper
|
||||
|
||||
llm = await LLMHelper.get_llm(streaming=False)
|
||||
prompt = SUMMARY_PROMPT.format(conversation=conversation_text)
|
||||
@@ -476,7 +454,7 @@ async def _summarize_with_llm(conversation_text: str) -> Optional[str]:
|
||||
return None
|
||||
return summary if summary else None
|
||||
except Exception as e:
|
||||
logger.debug(f"LLM 活动摘要生成失败: {summarize_error(e)}")
|
||||
logger.debug(f"LLM 活动摘要生成失败: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -512,14 +490,12 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
retention_days: int = DEFAULT_RETENTION_DAYS,
|
||||
prompt_load_days: int = PROMPT_LOAD_DAYS,
|
||||
stream_handler: Optional[Any] = None,
|
||||
task_registry: Optional[TaskRegistry] = None,
|
||||
) -> None:
|
||||
"""初始化活动日志中间件，并绑定宿主后台任务 owner。"""
|
||||
"""初始化活动日志中间件。"""
|
||||
self.activity_dir = activity_dir
|
||||
self.retention_days = retention_days
|
||||
self.prompt_load_days = prompt_load_days
|
||||
self.stream_handler = stream_handler
|
||||
self._task_registry = task_registry or get_task_registry()
|
||||
self._background_tasks: set[asyncio.Task[None]] = set()
|
||||
self._tool_provider = _ActivityLogToolProvider(activity_dir=activity_dir)
|
||||
self.tools = [
|
||||
@@ -583,21 +559,21 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
await stream.write(entry)
|
||||
else:
|
||||
header = f"# {today_str} 活动日志\n\n"
|
||||
created = await anyio.to_thread.run_sync(
|
||||
_write_activity_log_exclusive,
|
||||
Path(log_path),
|
||||
header + entry,
|
||||
)
|
||||
if not created:
|
||||
try:
|
||||
fd = os.open(log_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
||||
except FileExistsError:
|
||||
async with await anyio.open_file(
|
||||
log_path,
|
||||
mode="a",
|
||||
encoding="utf-8",
|
||||
) as stream:
|
||||
await stream.write(entry)
|
||||
logger.debug(f"Activity logged: {summarize_result(summary, max_chars=80)}")
|
||||
else:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as stream:
|
||||
stream.write(header + entry)
|
||||
logger.debug(f"Activity logged: {summary[:80]}")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to append activity log: {summarize_error(e)}")
|
||||
logger.warning(f"Failed to append activity log: {e}")
|
||||
|
||||
async def _cleanup_old_logs(self) -> None:
|
||||
"""清理超过保留天数的旧日志文件。"""
|
||||
@@ -623,16 +599,11 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
except ValueError:
|
||||
continue
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed to cleanup old activity logs: {summarize_error(e)}"
|
||||
)
|
||||
logger.warning(f"Failed to cleanup old activity logs: {e}")
|
||||
|
||||
def _schedule_activity_recording(self, messages: list) -> None:
|
||||
"""提交后台活动记录任务，不阻塞当前 Agent 会话结束。"""
|
||||
task = self._task_registry.create(
|
||||
self._record_activity(messages),
|
||||
owner="agent.activity_log.record",
|
||||
)
|
||||
task = asyncio.create_task(self._record_activity(messages))
|
||||
self._background_tasks.add(task)
|
||||
task.add_done_callback(self._on_activity_recording_done)
|
||||
|
||||
@@ -644,7 +615,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
except asyncio.CancelledError:
|
||||
logger.debug("活动日志后台记录任务已取消")
|
||||
except Exception as err:
|
||||
logger.warning(f"活动日志后台记录任务失败: {summarize_error(err)}")
|
||||
logger.warning(f"活动日志后台记录任务失败: {err}")
|
||||
|
||||
async def _record_activity(self, messages: list) -> None:
|
||||
"""在后台生成本轮活动摘要并写入活动日志。"""
|
||||
@@ -666,7 +637,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
if summary:
|
||||
await self._append_activity(summary)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to record activity: {summarize_error(e)}")
|
||||
logger.warning(f"Failed to record activity: {e}")
|
||||
|
||||
async def abefore_agent(
|
||||
self, state: ActivityLogState, runtime: Runtime
|
||||
@@ -715,12 +686,9 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
tool_args = tool_call.get("args") or {}
|
||||
if not isinstance(tool_args, dict):
|
||||
tool_args = {}
|
||||
logged_args = sanitize_for_host(tool_args)
|
||||
if not isinstance(logged_args, dict):
|
||||
logged_args = {}
|
||||
logger.info(
|
||||
f"开始执行活动日志查询工具: keyword={logged_args.get('keyword') or '-'}, "
|
||||
f"date={logged_args.get('date') or '-'}"
|
||||
f"开始执行活动日志查询工具: keyword={tool_args.get('keyword') or '-'}, "
|
||||
f"date={tool_args.get('date') or '-'}"
|
||||
)
|
||||
if self.stream_handler and getattr(self.stream_handler, "is_streaming", False):
|
||||
self.stream_handler.record_tool_call(
|
||||
@@ -731,9 +699,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
try:
|
||||
result = await handler(request)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"活动日志查询工具执行失败: error={summarize_error(err)}"
|
||||
)
|
||||
logger.error(f"活动日志查询工具执行失败: error={err}")
|
||||
raise
|
||||
logger.info("活动日志查询工具执行完成")
|
||||
return result
|
||||
@@ -748,7 +714,7 @@ class ActivityLogMiddleware(AgentMiddleware[ActivityLogState, ContextT, Response
|
||||
return None
|
||||
self._schedule_activity_recording(list(messages))
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to record activity: {summarize_error(e)}")
|
||||
logger.warning(f"Failed to record activity: {e}")
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from app.agent.middleware.utils import append_to_system_message
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
# JOB.md 文件最大限制为 1MB
|
||||
MAX_JOB_FILE_SIZE = 1 * 1024 * 1024
|
||||
|
||||
@@ -15,7 +15,7 @@ from langchain_core.runnables import RunnableConfig
|
||||
from langgraph.runtime import Runtime
|
||||
|
||||
from app.agent.middleware.utils import append_to_system_message
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
# 记忆文件最大限制为 100KB，防止单文件过大导致上下文溢出
|
||||
MAX_MEMORY_FILE_SIZE = 100 * 1024
|
||||
|
||||
@@ -1,195 +0,0 @@
|
||||
"""LangChain 工具调用的 MoviePilot 宿主策略中间件。"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from langchain.agents.middleware import AgentMiddleware, ToolCallRequest, hook_config
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
|
||||
from app.agent.policy.contracts import (
|
||||
ToolOrigin,
|
||||
ToolPolicyContext,
|
||||
)
|
||||
from app.agent.policy.orchestrator import (
|
||||
DEFAULT_TOOL_POLICY_ORCHESTRATOR,
|
||||
AgentToolPolicyOrchestrator,
|
||||
call_policy_hook,
|
||||
)
|
||||
from app.agent.tools.catalog import ToolCatalogSnapshot
|
||||
from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool
|
||||
|
||||
|
||||
POLICY_DENIED_MESSAGE = "当前宿主策略不允许执行该工具。"
|
||||
POLICY_UNAVAILABLE_MESSAGE = "宿主策略暂时不可用，未执行该工具。"
|
||||
TOOL_TIMEOUT_MESSAGE = (
|
||||
"工具执行超时，已停止等待结果；"
|
||||
"若工具包含外部写操作，操作可能仍在继续，请先确认实际状态再重试。"
|
||||
)
|
||||
|
||||
|
||||
class AgentPolicyMiddleware(AgentMiddleware):
|
||||
"""观测进入本地 ToolNode 的 client-side 工具调用和结果。
|
||||
|
||||
模型供应商原生 server tools 在供应商侧执行，不经过本地 middleware，
|
||||
因而不具备这里生成的 start/finish/fail 回执。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
context: ToolPolicyContext,
|
||||
orchestrator: AgentToolPolicyOrchestrator = DEFAULT_TOOL_POLICY_ORCHESTRATOR,
|
||||
catalog: ToolCatalogSnapshot | None = None,
|
||||
tools: list[Any] | None = None,
|
||||
) -> None:
|
||||
"""绑定宿主可信上下文和共享策略编排器。"""
|
||||
self.context = context
|
||||
self.orchestrator = orchestrator
|
||||
self.catalog = catalog
|
||||
self._tools = {
|
||||
tool.name: tool
|
||||
for tool in (tools or [])
|
||||
if getattr(tool, "name", None)
|
||||
}
|
||||
|
||||
@hook_config(can_jump_to=["end"])
|
||||
async def aafter_model(self, state: dict[str, Any], runtime: Any) -> Any:
|
||||
"""在 ToolNode 前暂停需要用户确认的敏感设置读取。"""
|
||||
messages = state.get("messages") or []
|
||||
if not messages or not isinstance(messages[-1], AIMessage):
|
||||
return None
|
||||
|
||||
tool_calls = messages[-1].tool_calls or []
|
||||
sensitive_call = None
|
||||
sensitive_tool = None
|
||||
for tool_call in tool_calls:
|
||||
arguments = tool_call.get("args")
|
||||
tool = self._tools.get(tool_call.get("name"))
|
||||
if (
|
||||
isinstance(tool, QuerySystemSettingsTool)
|
||||
and isinstance(arguments, dict)
|
||||
and arguments.get("show_secrets") is True
|
||||
):
|
||||
sensitive_call = tool_call
|
||||
sensitive_tool = tool
|
||||
break
|
||||
if sensitive_call is None or sensitive_tool is None:
|
||||
return None
|
||||
|
||||
confirmation_handler = (
|
||||
self.context.agent_context.get("secret_confirmation_handler")
|
||||
if self.context.origin is ToolOrigin.AGENT_INTERACTIVE
|
||||
else None
|
||||
)
|
||||
if not callable(confirmation_handler):
|
||||
confirmation_message = "当前入口不支持敏感设置确认，未执行任何工具。"
|
||||
else:
|
||||
confirmation_message = await confirmation_handler(
|
||||
sensitive_tool,
|
||||
sensitive_call.get("args") or {},
|
||||
)
|
||||
|
||||
paused_messages = [
|
||||
ToolMessage(
|
||||
content=(
|
||||
"本轮工具调用已暂停，未执行任何操作；"
|
||||
"请等待用户确认或取消敏感设置读取。"
|
||||
),
|
||||
tool_call_id=str(tool_call.get("id") or ""),
|
||||
name=str(tool_call.get("name") or "unknown"),
|
||||
)
|
||||
for tool_call in tool_calls
|
||||
]
|
||||
paused_messages.append(AIMessage(content=confirmation_message))
|
||||
return {"messages": paused_messages, "jump_to": "end"}
|
||||
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
request: ToolCallRequest,
|
||||
handler: Callable[[ToolCallRequest], Awaitable[Any]],
|
||||
) -> Any:
|
||||
"""在 handler 外层生成 shadow 决策和 secret-safe 回执摘要。"""
|
||||
tool_call = request.tool_call or {}
|
||||
arguments = tool_call.get("args") or {}
|
||||
if not isinstance(arguments, dict):
|
||||
arguments = {}
|
||||
try:
|
||||
_, result = await self.execute_tool_call(
|
||||
tool=request.tool,
|
||||
arguments=arguments,
|
||||
invocation_id=tool_call.get("id"),
|
||||
handler=lambda: handler(request),
|
||||
enforce_decision=False,
|
||||
)
|
||||
except TimeoutError:
|
||||
tool_name = str(getattr(request.tool, "name", None) or "unknown")
|
||||
return ToolMessage(
|
||||
content=TOOL_TIMEOUT_MESSAGE,
|
||||
tool_call_id=str(tool_call.get("id") or ""),
|
||||
name=tool_name,
|
||||
status="error",
|
||||
)
|
||||
# 普通 ToolNode 保持 shadow 观测；已确认调用使用默认的强制决策语义。
|
||||
return result
|
||||
|
||||
async def execute_tool_call(
|
||||
self,
|
||||
*,
|
||||
tool: Any,
|
||||
arguments: dict[str, Any],
|
||||
handler: Callable[[], Awaitable[Any]],
|
||||
invocation_id: str | None = None,
|
||||
enforce_decision: bool = True,
|
||||
) -> tuple[bool, Any]:
|
||||
"""执行一次本地工具调用，并复用 ToolNode 的策略生命周期。"""
|
||||
observation = call_policy_hook(
|
||||
"start",
|
||||
self.orchestrator.start,
|
||||
context=self.context,
|
||||
tool=tool,
|
||||
arguments=arguments,
|
||||
invocation_id=invocation_id,
|
||||
)
|
||||
if enforce_decision and observation is None:
|
||||
return False, POLICY_UNAVAILABLE_MESSAGE
|
||||
if (
|
||||
enforce_decision
|
||||
and observation.decision.allowed is False
|
||||
):
|
||||
return False, POLICY_DENIED_MESSAGE
|
||||
try:
|
||||
result = await handler()
|
||||
except asyncio.CancelledError as error:
|
||||
if observation is not None:
|
||||
call_policy_hook(
|
||||
"cancel",
|
||||
self.orchestrator.fail,
|
||||
observation,
|
||||
error,
|
||||
)
|
||||
raise
|
||||
except Exception as error:
|
||||
if observation is not None:
|
||||
call_policy_hook(
|
||||
"fail",
|
||||
self.orchestrator.fail,
|
||||
observation,
|
||||
error,
|
||||
)
|
||||
raise
|
||||
if observation is not None:
|
||||
call_policy_hook(
|
||||
"finish",
|
||||
self.orchestrator.finish,
|
||||
observation,
|
||||
result,
|
||||
)
|
||||
return True, result
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AgentPolicyMiddleware",
|
||||
"POLICY_DENIED_MESSAGE",
|
||||
"POLICY_UNAVAILABLE_MESSAGE",
|
||||
]
|
||||
+175
-31
@@ -24,19 +24,58 @@ from langgraph.runtime import Runtime
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.middleware.utils import append_to_system_message
|
||||
from app.agent.policy.sanitizer import sanitize_for_host, summarize_error
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.agent.skills.metadata import (
|
||||
MAX_SKILL_FILE_SIZE,
|
||||
SkillMetadata,
|
||||
parse_skill_metadata,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
# 模型返回上限独立于领域层的磁盘读取上限，避免异常内容撑爆上下文。
|
||||
# 磁盘读取上限与模型返回上限分离，避免异常大的 Skill 文件撑爆内存或上下文。
|
||||
MAX_SKILL_FILE_SIZE = 1 * 1024 * 1024
|
||||
MAX_SKILL_RESULT_CHARS = 64 * 1024
|
||||
SKILL_CONTENT_TRUNCATION_SUFFIX = "\n...(Skill 内容已截断)"
|
||||
|
||||
# Agent Skills 规范约束 (https://agentskills.io/specification)
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_SKILL_DESCRIPTION_LENGTH = 1024
|
||||
MAX_SKILL_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
|
||||
class SkillMetadata(TypedDict):
|
||||
"""Skill 元数据，符合 Agent Skills 规范。"""
|
||||
|
||||
path: str
|
||||
"""SKILL.md 文件路径。"""
|
||||
|
||||
id: str
|
||||
"""Skill 标识符。
|
||||
约束: 1-64 字符，仅限小写字母/数字/连字符，不能以连字符开头或结尾，无连续连字符，需与父目录名一致。
|
||||
"""
|
||||
|
||||
name: str
|
||||
"""Skill 名称。
|
||||
约束: Skill中文描述。
|
||||
"""
|
||||
|
||||
version: int
|
||||
"""Skill 版本号。
|
||||
用于内置技能的版本管理，同步时比较版本号决定是否覆盖用户目录中的旧版本。
|
||||
"""
|
||||
|
||||
description: str
|
||||
"""Skill 功能描述。
|
||||
约束: 1-1024 字符，应说明功能及适用场景。
|
||||
"""
|
||||
|
||||
license: str | None
|
||||
"""许可证信息。"""
|
||||
|
||||
compatibility: str | None
|
||||
"""环境依赖或兼容性要求 (最多 500 字符)。"""
|
||||
|
||||
metadata: dict[str, str]
|
||||
"""附加元数据。"""
|
||||
|
||||
allowed_tools: list[str]
|
||||
"""(实验性) Skill 建议使用的工具列表。"""
|
||||
|
||||
|
||||
class SkillsState(AgentState):
|
||||
"""skills 中间件状态。"""
|
||||
@@ -61,6 +100,123 @@ class SkillToolInput(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
def _parse_skill_metadata( # noqa: C901
|
||||
content: str,
|
||||
skill_path: str,
|
||||
skill_id: str,
|
||||
) -> SkillMetadata | None:
|
||||
"""从 SKILL.md 内容中解析 YAML 前言并验证元数据。"""
|
||||
if len(content) > MAX_SKILL_FILE_SIZE:
|
||||
logger.warning(
|
||||
"Skipping %s: content too large (%d bytes)", skill_path, len(content)
|
||||
)
|
||||
return None
|
||||
|
||||
# 匹配 --- 分隔的 YAML 前言
|
||||
frontmatter_pattern = r"^---\s*\n(.*?)\n---\s*\n"
|
||||
match = re.match(frontmatter_pattern, content, re.DOTALL)
|
||||
if not match:
|
||||
logger.warning("Skipping %s: no valid YAML frontmatter found", skill_path)
|
||||
return None
|
||||
frontmatter_str = match.group(1)
|
||||
|
||||
# 解析 YAML
|
||||
try:
|
||||
frontmatter_data = yaml.safe_load(frontmatter_str)
|
||||
except yaml.YAMLError as e:
|
||||
logger.warning("Invalid YAML in %s: %s", skill_path, e)
|
||||
return None
|
||||
|
||||
if not isinstance(frontmatter_data, dict):
|
||||
logger.warning("Skipping %s: frontmatter is not a mapping", skill_path)
|
||||
return None
|
||||
|
||||
# SKill名称和描述
|
||||
name = str(frontmatter_data.get("name", "")).strip()
|
||||
description = str(frontmatter_data.get("description", "")).strip()
|
||||
if not name or not description:
|
||||
logger.warning(
|
||||
"Skipping %s: missing required 'name' or 'description'", skill_path
|
||||
)
|
||||
return None
|
||||
description_str = description
|
||||
if len(description_str) > MAX_SKILL_DESCRIPTION_LENGTH:
|
||||
logger.warning(
|
||||
"Description exceeds %d characters in %s, truncating",
|
||||
MAX_SKILL_DESCRIPTION_LENGTH,
|
||||
skill_path,
|
||||
)
|
||||
description_str = description_str[:MAX_SKILL_DESCRIPTION_LENGTH]
|
||||
|
||||
# 可选的工具列表，支持空格或逗号分隔
|
||||
raw_tools = frontmatter_data.get("allowed-tools")
|
||||
if isinstance(raw_tools, str):
|
||||
allowed_tools = [
|
||||
t.strip(",") # 兼容 Claude Code 风格的逗号分隔
|
||||
for t in raw_tools.split()
|
||||
if t.strip(",")
|
||||
]
|
||||
else:
|
||||
if raw_tools is not None:
|
||||
logger.warning(
|
||||
"Ignoring non-string 'allowed-tools' in %s (got %s)",
|
||||
skill_path,
|
||||
type(raw_tools).__name__,
|
||||
)
|
||||
allowed_tools = []
|
||||
|
||||
# 能力或环境兼容性说明，最多 500 字符
|
||||
compatibility_str = str(frontmatter_data.get("compatibility", "")).strip() or None
|
||||
if compatibility_str and len(compatibility_str) > MAX_SKILL_COMPATIBILITY_LENGTH:
|
||||
logger.warning(
|
||||
"Compatibility exceeds %d characters in %s, truncating",
|
||||
MAX_SKILL_COMPATIBILITY_LENGTH,
|
||||
skill_path,
|
||||
)
|
||||
compatibility_str = str(compatibility_str)[:MAX_SKILL_COMPATIBILITY_LENGTH]
|
||||
|
||||
# 版本号，默认为 0（表示未设置版本）
|
||||
raw_version = frontmatter_data.get("version")
|
||||
version = 0
|
||||
if raw_version is not None:
|
||||
try:
|
||||
version = int(raw_version)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning(
|
||||
"Invalid 'version' in %s (got %r), defaulting to 0",
|
||||
skill_path,
|
||||
raw_version,
|
||||
)
|
||||
|
||||
return SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
version=version,
|
||||
description=description_str,
|
||||
path=skill_path,
|
||||
metadata=_validate_metadata(frontmatter_data.get("metadata", {}), skill_path),
|
||||
license=str(frontmatter_data.get("license", "")).strip() or None,
|
||||
compatibility=compatibility_str,
|
||||
allowed_tools=allowed_tools,
|
||||
)
|
||||
|
||||
|
||||
def _validate_metadata(
|
||||
raw: object,
|
||||
skill_path: str,
|
||||
) -> dict[str, str]:
|
||||
"""验证并规范化 YAML 前言中的元数据字段，确保为 dict[str, str] 类型。"""
|
||||
if not isinstance(raw, dict):
|
||||
if raw:
|
||||
logger.warning(
|
||||
"Ignoring non-dict metadata in %s (got %s)",
|
||||
skill_path,
|
||||
type(raw).__name__,
|
||||
)
|
||||
return {}
|
||||
return {str(k): str(v) for k, v in raw.items()}
|
||||
|
||||
|
||||
def _format_skill_annotations(skill: SkillMetadata) -> str:
|
||||
"""构建许可证和兼容性说明字符串。"""
|
||||
parts: list[str] = []
|
||||
@@ -107,7 +263,7 @@ async def _alist_skills(source_path: AsyncPath) -> list[SkillMetadata]:
|
||||
)
|
||||
|
||||
# 解析元数据
|
||||
skill_metadata = parse_skill_metadata(
|
||||
skill_metadata = _parse_skill_metadata(
|
||||
content=skill_content,
|
||||
skill_path=str(skill_md_path),
|
||||
skill_id=skill_path.name,
|
||||
@@ -146,7 +302,7 @@ def _list_skills(source_path: Path) -> list[SkillMetadata]:
|
||||
skill_content = skill_md_path.read_bytes().decode(
|
||||
"utf-8", errors="replace"
|
||||
)
|
||||
skill_metadata = parse_skill_metadata(
|
||||
skill_metadata = _parse_skill_metadata(
|
||||
content=skill_content,
|
||||
skill_path=str(skill_md_path),
|
||||
skill_id=skill_path.name,
|
||||
@@ -183,7 +339,7 @@ def _extract_version(skill_md: Path) -> int:
|
||||
try:
|
||||
content = skill_md.read_text(encoding="utf-8", errors="replace")
|
||||
except Exception as err:
|
||||
logger.debug(f"读取技能版本失败: {summarize_error(err)}")
|
||||
logger.debug(f"读取技能版本失败: {err}")
|
||||
return 0
|
||||
match = re.match(r"^---\s*\n(.*?)\n---\s*\n", content, re.DOTALL)
|
||||
if not match:
|
||||
@@ -241,11 +397,7 @@ def _sync_bundled_skills(bundled_dir: Path, target_dir: Path) -> None:
|
||||
"已自动复制内置技能 '%s' -> '%s'", skill_src.name, skill_dst
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"复制内置技能 '%s' 失败: %s",
|
||||
sanitize_for_host(skill_src.name),
|
||||
summarize_error(e),
|
||||
)
|
||||
logger.warning("复制内置技能 '%s' 失败: %s", skill_src.name, e)
|
||||
continue
|
||||
|
||||
# 目标已存在，比较版本号
|
||||
@@ -272,11 +424,7 @@ def _sync_bundled_skills(bundled_dir: Path, target_dir: Path) -> None:
|
||||
bundled_version,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"更新内置技能 '%s' 失败: %s",
|
||||
sanitize_for_host(skill_src.name),
|
||||
summarize_error(e),
|
||||
)
|
||||
logger.warning("更新内置技能 '%s' 失败: %s", skill_src.name, e)
|
||||
|
||||
|
||||
class _SkillToolProvider:
|
||||
@@ -371,7 +519,7 @@ class _SkillToolProvider:
|
||||
|
||||
async def load_skill(self, name: str) -> str:
|
||||
"""加载指定 Skill 的完整说明并返回 JSON 字符串。"""
|
||||
logger.info(f"加载 Skill: name={sanitize_for_host(name)}")
|
||||
logger.info(f"加载 Skill: name={name}")
|
||||
try:
|
||||
skill = await self._find_skill(name)
|
||||
if not skill:
|
||||
@@ -399,12 +547,11 @@ class _SkillToolProvider:
|
||||
}
|
||||
)
|
||||
except Exception as err:
|
||||
error_summary = summarize_error(err)
|
||||
logger.error(f"加载 Skill 失败: {error_summary}")
|
||||
logger.error(f"加载 Skill 失败: {err}", exc_info=True)
|
||||
return json.dumps(
|
||||
{
|
||||
"success": False,
|
||||
"message": f"加载 Skill 时发生错误: {error_summary}",
|
||||
"message": f"加载 Skill 时发生错误: {str(err)}",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
@@ -476,7 +623,7 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no
|
||||
try:
|
||||
_sync_bundled_skills(bundled, target)
|
||||
except Exception as e:
|
||||
logger.warning(f"同步内置技能失败: {summarize_error(e)}")
|
||||
logger.warning("同步内置技能失败: %s", e)
|
||||
|
||||
def _load_skills_metadata(self) -> list[SkillMetadata]:
|
||||
"""同步加载当前配置目录中的 Skill 元数据。"""
|
||||
@@ -581,11 +728,8 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no
|
||||
tool_args = tool_call.get("args") or {}
|
||||
if not isinstance(tool_args, dict):
|
||||
tool_args = {}
|
||||
logged_args = sanitize_for_host(tool_args)
|
||||
if not isinstance(logged_args, dict):
|
||||
logged_args = {}
|
||||
logger.info(
|
||||
f"开始执行 Skill 工具: name={logged_args.get('name') or '-'}"
|
||||
f"开始执行 Skill 工具: name={tool_args.get('name') or '-'}"
|
||||
)
|
||||
if self.stream_handler and getattr(self.stream_handler, "is_streaming", False):
|
||||
self.stream_handler.record_tool_call(
|
||||
@@ -596,7 +740,7 @@ class SkillsMiddleware(AgentMiddleware[SkillsState, ContextT, ResponseT]): # no
|
||||
try:
|
||||
result = await handler(request)
|
||||
except Exception as err:
|
||||
logger.error(f"Skill 工具执行失败: error={summarize_error(err)}")
|
||||
logger.error(f"Skill 工具执行失败: error={err}")
|
||||
raise
|
||||
logger.info("Skill 工具执行完成")
|
||||
return result
|
||||
|
||||
@@ -23,23 +23,11 @@ from langchain_core.messages import AIMessage, HumanMessage
|
||||
from langchain_core.tools import BaseTool, StructuredTool
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
from app.agent.middleware.policy import AgentPolicyMiddleware
|
||||
from app.agent.llm import LLMHelper
|
||||
from app.agent.middleware.utils import append_to_system_message
|
||||
from app.agent.policy.contracts import (
|
||||
AuthSource,
|
||||
PrincipalType,
|
||||
ToolOrigin,
|
||||
ToolPolicyContext,
|
||||
)
|
||||
from app.agent.policy.sanitizer import (
|
||||
sanitize_for_host,
|
||||
summarize_error,
|
||||
)
|
||||
from app.agent.runtime import SubAgentDefinition, agent_runtime_manager
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.agent.tools.catalog import ToolCatalogSnapshot
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
|
||||
SUBAGENT_TASK_TOOL_NAME = "task"
|
||||
@@ -48,7 +36,6 @@ SUBAGENT_STREAM_MARKER_KEY = "ls_agent_type"
|
||||
SUBAGENT_STREAM_MARKER_VALUE = "subagent"
|
||||
SUBAGENT_DEFAULT_WAIT_TIMEOUT_MS = 60000
|
||||
SUBAGENT_MAX_WAIT_TIMEOUT_MS = 300000
|
||||
SUBAGENT_CANCEL_GRACE_SECONDS = 5.0
|
||||
SUBAGENT_MAX_ACTIVE_TASKS = 8
|
||||
SUBAGENT_MAX_CONCURRENT_TASKS = 4
|
||||
SUBAGENT_RESULT_MAX_CHARS = 12000
|
||||
@@ -106,36 +93,6 @@ Requirements:
|
||||
"""
|
||||
|
||||
|
||||
def _default_subagent_policy_context(tools: list[BaseTool]) -> ToolPolicyContext:
|
||||
"""从已注入工具继承会话归属，确保独立构造的子图也经过宿主策略。"""
|
||||
for tool in tools:
|
||||
session_id = getattr(tool, "_session_id", None)
|
||||
user_id = getattr(tool, "_user_id", None)
|
||||
if not session_id and not user_id:
|
||||
continue
|
||||
agent_context = getattr(tool, "_agent_context", None)
|
||||
if not isinstance(agent_context, dict):
|
||||
agent_context = {}
|
||||
return ToolPolicyContext(
|
||||
session_id=str(session_id or "subagent"),
|
||||
user_id=str(user_id or "subagent"),
|
||||
origin=ToolOrigin.SUBAGENT,
|
||||
principal_type=PrincipalType.SUBAGENT,
|
||||
auth_source=AuthSource.INTERNAL,
|
||||
agent_context=agent_context,
|
||||
channel=getattr(tool, "_channel", None),
|
||||
source=getattr(tool, "_source", None),
|
||||
)
|
||||
return ToolPolicyContext(
|
||||
session_id="subagent",
|
||||
user_id="subagent",
|
||||
origin=ToolOrigin.SUBAGENT,
|
||||
principal_type=PrincipalType.SUBAGENT,
|
||||
auth_source=AuthSource.INTERNAL,
|
||||
agent_context={},
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _SubAgentProfile:
|
||||
"""子代理运行时定义。"""
|
||||
@@ -421,16 +378,12 @@ class _SubAgentAgentProvider:
|
||||
profiles: tuple[_SubAgentProfile, ...],
|
||||
tools: list[BaseTool],
|
||||
server_tools: Optional[list[dict[str, Any]]] = None,
|
||||
policy_context: Optional[ToolPolicyContext] = None,
|
||||
catalog: Optional[ToolCatalogSnapshot] = None,
|
||||
) -> None:
|
||||
"""初始化子代理执行器。"""
|
||||
self._model = model
|
||||
self._profiles = {profile.name: profile for profile in profiles}
|
||||
self._tools = tools
|
||||
self._server_tools = server_tools or []
|
||||
self._policy_context = policy_context or _default_subagent_policy_context(tools)
|
||||
self._catalog = catalog
|
||||
self._agents = {}
|
||||
self._default_agent_name = "general-purpose"
|
||||
|
||||
@@ -448,9 +401,6 @@ class _SubAgentAgentProvider:
|
||||
return profile.name, cached_agent
|
||||
|
||||
subagent_tools = _select_tools(self._tools, profile)
|
||||
subagent_catalog = (
|
||||
self._catalog.select(subagent_tools) if self._catalog is not None else None
|
||||
)
|
||||
logger.info(
|
||||
f"创建子代理图: subagent_type={profile.name}, tools={len(subagent_tools)}"
|
||||
)
|
||||
@@ -459,12 +409,6 @@ class _SubAgentAgentProvider:
|
||||
tools=[*subagent_tools, *self._server_tools],
|
||||
system_prompt=profile.prompt,
|
||||
name=profile.name,
|
||||
middleware=[
|
||||
AgentPolicyMiddleware(
|
||||
context=self._policy_context,
|
||||
catalog=subagent_catalog,
|
||||
)
|
||||
],
|
||||
)
|
||||
self._agents[profile.name] = agent
|
||||
return profile.name, agent
|
||||
@@ -500,7 +444,7 @@ class _SubAgentAgentProvider:
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"子代理调用失败: subagent_type={agent_name}, "
|
||||
f"task_id={log_task_id}, error={summarize_error(err)}"
|
||||
f"task_id={log_task_id}, error={err}"
|
||||
)
|
||||
raise
|
||||
final_text = _extract_final_text(result)
|
||||
@@ -524,8 +468,6 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware):
|
||||
system_prompt: str = SUBAGENT_PARENT_PROMPT,
|
||||
task_description: str = SUBAGENT_TASK_DESCRIPTION,
|
||||
stream_handler: Any = None,
|
||||
policy_context: Optional[ToolPolicyContext] = None,
|
||||
catalog: Optional[ToolCatalogSnapshot] = None,
|
||||
) -> None:
|
||||
"""初始化同步子代理中间件。"""
|
||||
self.system_prompt = system_prompt
|
||||
@@ -535,8 +477,6 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware):
|
||||
profiles=profiles,
|
||||
tools=tools,
|
||||
server_tools=server_tools,
|
||||
policy_context=policy_context,
|
||||
catalog=catalog,
|
||||
)
|
||||
self.tools = [
|
||||
StructuredTool.from_function(
|
||||
@@ -587,12 +527,9 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware):
|
||||
return await handler(request)
|
||||
|
||||
tool_args = _extract_tool_call_args(request)
|
||||
logged_args = sanitize_for_host(tool_args)
|
||||
if not isinstance(logged_args, dict):
|
||||
logged_args = {}
|
||||
logger.info(
|
||||
f"开始执行子代理工具: tool_name={tool_name}, "
|
||||
f"subagent_type={logged_args.get('subagent_type') or '-'}"
|
||||
f"subagent_type={tool_args.get('subagent_type') or '-'}"
|
||||
)
|
||||
_record_subagent_tool_call(
|
||||
stream_handler=self.stream_handler,
|
||||
@@ -602,10 +539,7 @@ class MoviePilotSubAgentMiddleware(AgentMiddleware):
|
||||
try:
|
||||
result = await handler(request)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"子代理工具执行失败: tool_name={tool_name}, "
|
||||
f"error={summarize_error(err)}"
|
||||
)
|
||||
logger.error(f"子代理工具执行失败: tool_name={tool_name}, error={err}")
|
||||
raise
|
||||
logger.info(f"子代理工具执行完成: tool_name={tool_name}")
|
||||
return result
|
||||
@@ -623,8 +557,6 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
server_tools: Optional[list[dict[str, Any]]] = None,
|
||||
task_description: str = SUBAGENT_CONTROL_DESCRIPTION,
|
||||
stream_handler: Any = None,
|
||||
policy_context: Optional[ToolPolicyContext] = None,
|
||||
catalog: Optional[ToolCatalogSnapshot] = None,
|
||||
) -> None:
|
||||
"""初始化异步子代理调度中间件。"""
|
||||
self.stream_handler = stream_handler
|
||||
@@ -633,13 +565,9 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
profiles=profiles,
|
||||
tools=tools,
|
||||
server_tools=server_tools,
|
||||
policy_context=policy_context,
|
||||
catalog=catalog,
|
||||
)
|
||||
self._semaphore = asyncio.Semaphore(SUBAGENT_MAX_CONCURRENT_TASKS)
|
||||
self._tasks: dict[str, _SubAgentRuntimeTask] = {}
|
||||
self._accepting_tasks = True
|
||||
self._close_cancel_requested: set[asyncio.Task] = set()
|
||||
self.tools = [
|
||||
StructuredTool.from_function(
|
||||
coroutine=self._control_task,
|
||||
@@ -700,7 +628,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
|
||||
error = record.task.exception()
|
||||
if error:
|
||||
payload["error"] = summarize_error(error)
|
||||
payload["error"] = str(error)
|
||||
return payload
|
||||
|
||||
result, result_truncated = _clip_text(
|
||||
@@ -805,17 +733,11 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
raise
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"子代理任务执行失败: task_id={record.task_id}, "
|
||||
f"error={summarize_error(err)}"
|
||||
)
|
||||
logger.error(f"子代理任务执行失败: task_id={record.task_id}, error={err}")
|
||||
raise
|
||||
|
||||
def _mark_task_finished(self, task_id: str, task: asyncio.Task) -> None:
|
||||
"""记录任务完成时间并取出异常避免未读取告警。"""
|
||||
cancel_requested = getattr(self, "_close_cancel_requested", None)
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.discard(task)
|
||||
record = self._tasks.get(task_id)
|
||||
if record:
|
||||
record.finished_at = datetime.now()
|
||||
@@ -898,70 +820,18 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _cancel_records(
|
||||
records: list[_SubAgentRuntimeTask],
|
||||
*,
|
||||
cancel_requested: Optional[set[asyncio.Task]] = None,
|
||||
) -> list[_SubAgentRuntimeTask]:
|
||||
"""取消一组任务，并返回等待上限内仍未收敛的记录。"""
|
||||
async def _cancel_records(records: list[_SubAgentRuntimeTask]) -> None:
|
||||
"""取消一组尚未完成的任务。"""
|
||||
cancellable_tasks = [
|
||||
record.task for record in records if not record.task.done()
|
||||
]
|
||||
if cancellable_tasks:
|
||||
logger.info(f"开始取消子代理任务: tasks={len(cancellable_tasks)}")
|
||||
for task in cancellable_tasks:
|
||||
if cancel_requested is not None and task in cancel_requested:
|
||||
continue
|
||||
task.cancel()
|
||||
if cancel_requested is not None:
|
||||
cancel_requested.add(task)
|
||||
if not cancellable_tasks:
|
||||
return []
|
||||
|
||||
done, pending = await asyncio.wait(
|
||||
cancellable_tasks,
|
||||
timeout=SUBAGENT_CANCEL_GRACE_SECONDS,
|
||||
)
|
||||
if done:
|
||||
await asyncio.gather(*done, return_exceptions=True)
|
||||
if pending:
|
||||
logger.warning(
|
||||
f"子代理任务取消等待超时: pending={len(pending)}, "
|
||||
f"timeout={SUBAGENT_CANCEL_GRACE_SECONDS}s"
|
||||
)
|
||||
else:
|
||||
if cancellable_tasks:
|
||||
await asyncio.gather(*cancellable_tasks, return_exceptions=True)
|
||||
logger.info(f"子代理任务取消完成: tasks={len(cancellable_tasks)}")
|
||||
return [record for record in records if record.task in pending]
|
||||
|
||||
def seal(self) -> None:
|
||||
"""封住新的 detached 子代理提交，既有任务继续由记录表持有。"""
|
||||
self._accepting_tasks = False
|
||||
|
||||
async def close(self) -> bool:
|
||||
"""有限等待 detached 子代理；超时保留记录并返回 False。"""
|
||||
self.seal()
|
||||
if not hasattr(self, "_close_cancel_requested"):
|
||||
self._close_cancel_requested = set()
|
||||
unfinished_records = [
|
||||
record for record in self._tasks.values() if not record.task.done()
|
||||
]
|
||||
if unfinished_records:
|
||||
logger.info(
|
||||
f"关闭子代理任务控制器，取消未完成任务: tasks={len(unfinished_records)}"
|
||||
)
|
||||
pending_records = await self._cancel_records(
|
||||
unfinished_records,
|
||||
cancel_requested=self._close_cancel_requested,
|
||||
)
|
||||
if pending_records:
|
||||
self._tasks = {
|
||||
record.task_id: record
|
||||
for record in pending_records
|
||||
}
|
||||
return False
|
||||
self._tasks.clear()
|
||||
self._close_cancel_requested.clear()
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _pipeline_description(
|
||||
@@ -1031,10 +901,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
)
|
||||
raise
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"管道子代理任务执行失败: task_id={record.task_id}, "
|
||||
f"error={summarize_error(err)}"
|
||||
)
|
||||
logger.error(f"管道子代理任务执行失败: task_id={record.task_id}, error={err}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
@@ -1080,8 +947,6 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
previous_results: list[tuple[_SubAgentRuntimeTask, str]] = []
|
||||
timeout = normalized_timeout_ms / 1000
|
||||
for step_index, spec in enumerate(specs, start=1):
|
||||
if not self._accepting_tasks:
|
||||
return records, "子代理任务控制器正在关闭，不能再启动新任务。"
|
||||
record = self._create_pipeline_record(spec)
|
||||
records.append(record)
|
||||
pipeline_description = self._pipeline_description(
|
||||
@@ -1101,24 +966,17 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
f"subagent_type={record.subagent_type}"
|
||||
)
|
||||
|
||||
done, pending = await asyncio.wait({task}, timeout=timeout)
|
||||
if pending:
|
||||
task.cancel()
|
||||
try:
|
||||
result = await asyncio.wait_for(task, timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
error = f"第 {step_index} 个管道子代理任务等待超时。"
|
||||
logger.info(
|
||||
f"{error} task_id={record.task_id}, timeout_ms={normalized_timeout_ms}"
|
||||
)
|
||||
return records, error
|
||||
try:
|
||||
result = next(iter(done)).result()
|
||||
except Exception as err:
|
||||
error = (
|
||||
f"第 {step_index} 个管道子代理任务执行失败: "
|
||||
f"{summarize_error(err)}"
|
||||
)
|
||||
logger.info(
|
||||
f"{error} task_id={record.task_id}"
|
||||
)
|
||||
error = f"第 {step_index} 个管道子代理任务执行失败: {err}"
|
||||
logger.info(f"{error} task_id={record.task_id}")
|
||||
return records, error
|
||||
|
||||
previous_results.append((record, result))
|
||||
@@ -1139,19 +997,13 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
"""管理异步子代理任务。"""
|
||||
logger.info(f"收到子代理管控操作: action={action}")
|
||||
if action in {"start", "run", "pipeline"}:
|
||||
if not self._accepting_tasks:
|
||||
error = "子代理任务控制器正在关闭，不能再启动新任务。"
|
||||
return self._json_response({"success": False, "error": error})
|
||||
specs, error = self._normalize_specs(
|
||||
description=description,
|
||||
subagent_type=subagent_type,
|
||||
tasks=tasks,
|
||||
)
|
||||
if error:
|
||||
logger.info(
|
||||
f"子代理管控操作未启动任务: action={action}, "
|
||||
f"error={sanitize_for_host(error)}"
|
||||
)
|
||||
logger.info(f"子代理管控操作未启动任务: action={action}, error={error}")
|
||||
return self._json_response({"success": False, "error": error})
|
||||
|
||||
logger.info(f"准备启动子代理任务: action={action}, tasks={len(specs)}")
|
||||
@@ -1192,7 +1044,6 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
active_only=action in {"wait", "cancel"} and not task_ids and not task_id,
|
||||
)
|
||||
|
||||
cancellation_pending: list[_SubAgentRuntimeTask] = []
|
||||
if action == "wait":
|
||||
logger.info(
|
||||
f"准备等待子代理任务: selected={len(records)}, missing={len(missing_ids)}"
|
||||
@@ -1206,28 +1057,31 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
logger.info(
|
||||
f"准备取消子代理任务: selected={len(records)}, missing={len(missing_ids)}"
|
||||
)
|
||||
cancellation_pending = await self._cancel_records(records)
|
||||
await self._cancel_records(records)
|
||||
elif action == "status":
|
||||
logger.info(
|
||||
f"查询子代理任务状态: selected={len(records)}, missing={len(missing_ids)}"
|
||||
)
|
||||
|
||||
response = {
|
||||
"success": not cancellation_pending,
|
||||
"action": action,
|
||||
"wait_mode": wait_mode if action == "wait" else None,
|
||||
"missing_task_ids": missing_ids,
|
||||
"tasks": [self._task_output(record) for record in records],
|
||||
}
|
||||
if action == "cancel":
|
||||
response["cancel_pending_task_ids"] = [
|
||||
record.task_id for record in cancellation_pending
|
||||
]
|
||||
return self._json_response(response)
|
||||
return self._json_response(
|
||||
{
|
||||
"success": True,
|
||||
"action": action,
|
||||
"wait_mode": wait_mode if action == "wait" else None,
|
||||
"missing_task_ids": missing_ids,
|
||||
"tasks": [self._task_output(record) for record in records],
|
||||
}
|
||||
)
|
||||
|
||||
async def aafter_agent(self, state: Any, runtime: Any) -> None:
|
||||
"""Agent 结束时取消未完成的子代理任务，避免后台泄漏。"""
|
||||
await self.close()
|
||||
unfinished_records = [
|
||||
record for record in self._tasks.values() if not record.task.done()
|
||||
]
|
||||
if unfinished_records:
|
||||
logger.info(f"Agent 结束，取消未完成子代理任务: tasks={len(unfinished_records)}")
|
||||
await self._cancel_records(unfinished_records)
|
||||
self._tasks.clear()
|
||||
|
||||
async def awrap_tool_call(
|
||||
self,
|
||||
@@ -1241,13 +1095,10 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
return await handler(request)
|
||||
|
||||
tool_args = _extract_tool_call_args(request)
|
||||
logged_args = sanitize_for_host(tool_args)
|
||||
if not isinstance(logged_args, dict):
|
||||
logged_args = {}
|
||||
logger.info(
|
||||
f"开始执行子代理工具: tool_name={tool_name}, "
|
||||
f"action={logged_args.get('action') or '-'}, "
|
||||
f"subagent_type={logged_args.get('subagent_type') or '-'}"
|
||||
f"action={tool_args.get('action') or '-'}, "
|
||||
f"subagent_type={tool_args.get('subagent_type') or '-'}"
|
||||
)
|
||||
_record_subagent_tool_call(
|
||||
stream_handler=self.stream_handler,
|
||||
@@ -1257,10 +1108,7 @@ class SubAgentTaskControlMiddleware(AgentMiddleware):
|
||||
try:
|
||||
result = await handler(request)
|
||||
except Exception as err:
|
||||
logger.error(
|
||||
f"子代理工具执行失败: tool_name={tool_name}, "
|
||||
f"error={summarize_error(err)}"
|
||||
)
|
||||
logger.error(f"子代理工具执行失败: tool_name={tool_name}, error={err}")
|
||||
raise
|
||||
logger.info(f"子代理工具执行完成: tool_name={tool_name}")
|
||||
return result
|
||||
@@ -1272,8 +1120,6 @@ def create_subagent_middlewares(
|
||||
tools: list[BaseTool],
|
||||
server_tools: Optional[list[dict[str, Any]]] = None,
|
||||
stream_handler: Any = None,
|
||||
policy_context: Optional[ToolPolicyContext] = None,
|
||||
catalog: Optional[ToolCatalogSnapshot] = None,
|
||||
) -> tuple[list[AgentMiddleware], list[BaseTool]]:
|
||||
"""创建子代理中间件列表和任务工具列表。"""
|
||||
runtime_signature = agent_runtime_manager.current_signature()
|
||||
@@ -1284,8 +1130,6 @@ def create_subagent_middlewares(
|
||||
tools=tools,
|
||||
server_tools=server_tools or [],
|
||||
stream_handler=stream_handler,
|
||||
policy_context=policy_context,
|
||||
catalog=catalog,
|
||||
)
|
||||
control_middleware = SubAgentTaskControlMiddleware(
|
||||
model=model,
|
||||
@@ -1293,8 +1137,6 @@ def create_subagent_middlewares(
|
||||
tools=tools,
|
||||
server_tools=server_tools or [],
|
||||
stream_handler=stream_handler,
|
||||
policy_context=policy_context,
|
||||
catalog=catalog,
|
||||
)
|
||||
|
||||
task_tools = [
|
||||
|
||||
@@ -1,554 +0,0 @@
|
||||
"""Agent 会话上下文压缩中间件。"""
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
from langchain.agents.middleware.summarization import (
|
||||
DEFAULT_SUMMARY_PROMPT,
|
||||
ContextSize,
|
||||
SummarizationMiddleware,
|
||||
TokenCounter,
|
||||
TriggerClause,
|
||||
)
|
||||
from langchain.agents.middleware.types import (
|
||||
AgentMiddleware,
|
||||
ExtendedModelResponse,
|
||||
ModelRequest,
|
||||
ModelResponse,
|
||||
)
|
||||
from langchain.chat_models import BaseChatModel
|
||||
from langchain_core.messages import AnyMessage, HumanMessage, RemoveMessage, ToolMessage
|
||||
from langchain_core.messages.utils import count_tokens_approximately, get_buffer_string
|
||||
from langgraph.graph.message import REMOVE_ALL_MESSAGES
|
||||
from langgraph.types import Command
|
||||
|
||||
from app.agent.middleware.usage import UsageMiddleware
|
||||
from app.runtime.log import logger
|
||||
|
||||
try:
|
||||
_internal_call_metadata = import_module(
|
||||
"langchain.agents.middleware.internal_call_transformer"
|
||||
).internal_call_metadata
|
||||
except ImportError:
|
||||
|
||||
def _internal_call_metadata() -> dict[str, Any]:
|
||||
"""旧版 LangChain 没有内部模型调用的流式过滤标记。"""
|
||||
return {}
|
||||
|
||||
|
||||
class ContextSummarizationError(RuntimeError):
|
||||
"""摘要不可用且原有会话上下文未被替换。"""
|
||||
|
||||
|
||||
class ContextPreservingSummarizationMiddleware(SummarizationMiddleware):
|
||||
"""摘要失败时中止状态更新，避免永久丢失既有会话上下文。"""
|
||||
|
||||
_ERROR_MESSAGE = "会话上下文压缩失败，原有上下文已保留，请稍后重试"
|
||||
_UNSUMMARIZABLE_MESSAGE = (
|
||||
"会话历史中存在无法压缩的超长内容，原有上下文已保留，"
|
||||
"请新建或清空会话后继续"
|
||||
)
|
||||
# LangChain 默认按 4000 token 裁剪待摘要消息，单条超长工具结果或用户输入
|
||||
# （超长无换行文本、非文本多模态块）会被整体丢弃，直接触发"无法压缩"报错。
|
||||
# 调大该上限可容纳更大单条消息，减少误报；16k 相对常见模型窗口仍然安全，
|
||||
# 摘要模型与主模型同窗口，过大会抬高摘要成本并挤占主模型预算。
|
||||
_DEFAULT_TRIM_TOKENS_TO_SUMMARIZE = 16000
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str | BaseChatModel,
|
||||
*,
|
||||
trigger: (
|
||||
ContextSize | TriggerClause | list[ContextSize | TriggerClause] | None
|
||||
) = None,
|
||||
keep: ContextSize = ("messages", 20),
|
||||
token_counter: TokenCounter = count_tokens_approximately,
|
||||
summary_prompt: str = DEFAULT_SUMMARY_PROMPT,
|
||||
trim_tokens_to_summarize: int | None = _DEFAULT_TRIM_TOKENS_TO_SUMMARIZE,
|
||||
**deprecated_kwargs: Any,
|
||||
) -> None:
|
||||
"""按 MoviePilot 的默认压缩策略构建摘要中间件。"""
|
||||
super().__init__(
|
||||
model=model,
|
||||
trigger=trigger,
|
||||
keep=keep,
|
||||
token_counter=token_counter,
|
||||
summary_prompt=summary_prompt,
|
||||
trim_tokens_to_summarize=trim_tokens_to_summarize,
|
||||
**deprecated_kwargs,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _require_valid_summary(cls, summary: str) -> str:
|
||||
"""拒绝无法继续承载会话上下文的空摘要。"""
|
||||
if not summary:
|
||||
raise ContextSummarizationError(cls._ERROR_MESSAGE)
|
||||
return summary
|
||||
|
||||
def _prepare_summary_input(
|
||||
self, messages_to_summarize: list[AnyMessage]
|
||||
) -> str:
|
||||
"""复用 LangChain 裁剪策略生成摘要模型输入。"""
|
||||
trimmed_messages = self._trim_messages_for_summary(messages_to_summarize)
|
||||
if not trimmed_messages:
|
||||
raise ContextSummarizationError(self._UNSUMMARIZABLE_MESSAGE)
|
||||
return get_buffer_string(trimmed_messages, format="xml")
|
||||
|
||||
def _create_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
"""同步摘要失败时保持原图状态。"""
|
||||
formatted_messages = self._prepare_summary_input(messages_to_summarize)
|
||||
summary_model = getattr(self, "_summary_model", self.model)
|
||||
try:
|
||||
response = summary_model.invoke(
|
||||
self.summary_prompt.format(messages=formatted_messages).rstrip(),
|
||||
config={
|
||||
"metadata": {
|
||||
"lc_source": "summarization",
|
||||
**_internal_call_metadata(),
|
||||
}
|
||||
},
|
||||
)
|
||||
except Exception as err:
|
||||
raise ContextSummarizationError(self._ERROR_MESSAGE) from err
|
||||
return self._require_valid_summary(response.text.strip())
|
||||
|
||||
async def _acreate_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
"""异步摘要失败时保持原图状态。"""
|
||||
formatted_messages = self._prepare_summary_input(messages_to_summarize)
|
||||
summary_model = getattr(self, "_summary_model", self.model)
|
||||
try:
|
||||
response = await summary_model.ainvoke(
|
||||
self.summary_prompt.format(messages=formatted_messages).rstrip(),
|
||||
config={
|
||||
"metadata": {
|
||||
"lc_source": "summarization",
|
||||
**_internal_call_metadata(),
|
||||
}
|
||||
},
|
||||
)
|
||||
except Exception as err:
|
||||
raise ContextSummarizationError(self._ERROR_MESSAGE) from err
|
||||
return self._require_valid_summary(response.text.strip())
|
||||
|
||||
def partition_for_token_limit(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
token_limit: int,
|
||||
*,
|
||||
force: bool = False,
|
||||
minimum_cutoff: int = 1,
|
||||
strict_token_limit: bool = False,
|
||||
) -> tuple[list[AnyMessage], list[AnyMessage]] | None:
|
||||
"""按 token 上限拆分历史，并保持 LangChain 的工具调用事务边界。"""
|
||||
self._ensure_message_ids(messages)
|
||||
if self.token_counter(messages) <= token_limit:
|
||||
return (
|
||||
self._minimum_safe_partition(
|
||||
messages,
|
||||
minimum_cutoff=minimum_cutoff,
|
||||
token_limit=token_limit if strict_token_limit else None,
|
||||
)
|
||||
if force
|
||||
else None
|
||||
)
|
||||
|
||||
left, right = 0, len(messages)
|
||||
cutoff_candidate = len(messages)
|
||||
while left < right:
|
||||
midpoint = (left + right) // 2
|
||||
if self._partial_token_counter(messages[midpoint:]) <= token_limit:
|
||||
cutoff_candidate = midpoint
|
||||
right = midpoint
|
||||
else:
|
||||
left = midpoint + 1
|
||||
|
||||
if cutoff_candidate >= len(messages):
|
||||
cutoff_candidate = len(messages)
|
||||
cutoff_index = self._find_safe_cutoff_point(messages, cutoff_candidate)
|
||||
if (
|
||||
cutoff_index <= 0
|
||||
or cutoff_index >= len(messages)
|
||||
or cutoff_index < minimum_cutoff
|
||||
or not self._contains_unsummarized_message(messages[:cutoff_index])
|
||||
or (
|
||||
strict_token_limit
|
||||
and self._partial_token_counter(messages[cutoff_index:]) > token_limit
|
||||
)
|
||||
):
|
||||
if cutoff_candidate >= len(messages):
|
||||
if strict_token_limit:
|
||||
return None
|
||||
return self._latest_safe_partition(
|
||||
messages,
|
||||
minimum_cutoff=minimum_cutoff,
|
||||
)
|
||||
return self._minimum_safe_partition(
|
||||
messages,
|
||||
minimum_cutoff=max(minimum_cutoff, cutoff_candidate),
|
||||
token_limit=token_limit if strict_token_limit else None,
|
||||
)
|
||||
return self._partition_messages(messages, cutoff_index)
|
||||
|
||||
def partition_for_retention(
|
||||
self, messages: list[AnyMessage]
|
||||
) -> tuple[list[AnyMessage], list[AnyMessage]] | None:
|
||||
"""按摘要器既有触发和保留策略拆分历史。"""
|
||||
self._ensure_message_ids(messages)
|
||||
total_tokens = self.token_counter(messages)
|
||||
if not self._should_summarize(messages, total_tokens):
|
||||
return None
|
||||
cutoff_index = self._determine_cutoff_index(messages)
|
||||
if cutoff_index <= 0:
|
||||
return None
|
||||
return self._partition_messages(messages, cutoff_index)
|
||||
|
||||
def _minimum_safe_partition(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
minimum_cutoff: int = 1,
|
||||
token_limit: int | None = None,
|
||||
) -> tuple[list[AnyMessage], list[AnyMessage]] | None:
|
||||
"""至少摘要一段旧历史，同时保留最新完整消息事务。"""
|
||||
for candidate in range(max(1, minimum_cutoff), len(messages)):
|
||||
cutoff_index = self._find_safe_cutoff_point(messages, candidate)
|
||||
if (
|
||||
minimum_cutoff <= cutoff_index < len(messages)
|
||||
and self._contains_unsummarized_message(messages[:cutoff_index])
|
||||
and (
|
||||
token_limit is None
|
||||
or self._partial_token_counter(messages[cutoff_index:])
|
||||
<= token_limit
|
||||
)
|
||||
):
|
||||
return self._partition_messages(messages, cutoff_index)
|
||||
return None
|
||||
|
||||
def _latest_safe_partition(
|
||||
self,
|
||||
messages: list[AnyMessage],
|
||||
*,
|
||||
minimum_cutoff: int,
|
||||
) -> tuple[list[AnyMessage], list[AnyMessage]] | None:
|
||||
"""保留无法满足软预算时的最新完整消息事务。"""
|
||||
for candidate in range(len(messages) - 1, minimum_cutoff - 1, -1):
|
||||
cutoff_index = self._find_safe_cutoff_point(messages, candidate)
|
||||
if (
|
||||
minimum_cutoff <= cutoff_index < len(messages)
|
||||
and self._contains_unsummarized_message(messages[:cutoff_index])
|
||||
):
|
||||
return self._partition_messages(messages, cutoff_index)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _contains_unsummarized_message(messages: list[AnyMessage]) -> bool:
|
||||
"""确认待摘要段包含可推进上下文的原始消息。"""
|
||||
return any(
|
||||
message.additional_kwargs.get("lc_source") != "summarization"
|
||||
for message in messages
|
||||
)
|
||||
|
||||
def build_summary_messages(self, summary: str) -> list[AnyMessage]:
|
||||
"""将摘要转换为 LangChain 约定的可识别历史消息。"""
|
||||
return self._build_new_messages(summary)
|
||||
|
||||
def ensure_message_ids(self, messages: list[AnyMessage]) -> None:
|
||||
"""为压缩后消息补齐 LangGraph reducer 所需的稳定 ID。"""
|
||||
self._ensure_message_ids(messages)
|
||||
|
||||
def create_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
"""通过 MoviePilot 的失败保护合同生成同步摘要。"""
|
||||
return self._create_summary(messages_to_summarize)
|
||||
|
||||
async def acreate_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
|
||||
"""通过 MoviePilot 的失败保护合同生成异步摘要。"""
|
||||
return await self._acreate_summary(messages_to_summarize)
|
||||
|
||||
|
||||
class FinalRequestCompactionMiddleware(AgentMiddleware):
|
||||
"""按最终模型请求预算压缩历史，并在模型成功后原子提交新状态。"""
|
||||
|
||||
_COMPACTION_ANCHOR_KEY = "moviepilot_compaction_anchor_id"
|
||||
_UNCOMPRESSIBLE_REQUEST = (
|
||||
"最终模型请求压缩后仍超出上下文窗口，原有上下文已保留，"
|
||||
"请减少启用工具或切换更大上下文模型"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
summarizer: ContextPreservingSummarizationMiddleware,
|
||||
trigger_fraction: float = 0.85,
|
||||
keep_fraction: float = 0.10,
|
||||
) -> None:
|
||||
self.summarizer = summarizer
|
||||
self.trigger_fraction = trigger_fraction
|
||||
self.keep_fraction = keep_fraction
|
||||
|
||||
def _should_compact(self, budget: dict[str, Any]) -> bool:
|
||||
"""以最终请求实际模型窗口判断是否需要压缩。"""
|
||||
estimated_tokens = budget.get("estimated_input_tokens")
|
||||
context_window = budget.get("context_window_tokens")
|
||||
return (
|
||||
isinstance(estimated_tokens, int)
|
||||
and isinstance(context_window, int)
|
||||
and estimated_tokens >= context_window * self.trigger_fraction
|
||||
)
|
||||
|
||||
def _compaction_partition(
|
||||
self, request: ModelRequest
|
||||
) -> tuple[list[AnyMessage], list[AnyMessage]] | None:
|
||||
"""最终输入达到阈值时，拆分需要摘要和需要原样保留的消息。"""
|
||||
messages = list(request.messages)
|
||||
try:
|
||||
budget = UsageMiddleware.estimate_request(request)
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"最终模型请求预算评估失败，继续原请求: error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
return None
|
||||
|
||||
context_window = budget.get("context_window_tokens")
|
||||
if self._should_skip_after_current_turn_compaction(messages, budget):
|
||||
return None
|
||||
if not self._should_compact(budget) or not isinstance(context_window, int):
|
||||
return None
|
||||
try:
|
||||
partition = self.summarizer.partition_for_retention(messages)
|
||||
if partition is None:
|
||||
partition = self.summarizer.partition_for_token_limit(
|
||||
messages,
|
||||
max(1, int(context_window * self.keep_fraction)),
|
||||
force=budget["estimated_input_tokens"] > context_window,
|
||||
)
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"最终请求历史拆分失败，继续原请求: error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
if budget["estimated_input_tokens"] > context_window:
|
||||
raise ContextSummarizationError(
|
||||
self._UNCOMPRESSIBLE_REQUEST
|
||||
) from error
|
||||
return None
|
||||
if partition is None:
|
||||
if budget["estimated_input_tokens"] > context_window:
|
||||
raise ContextSummarizationError(self._UNCOMPRESSIBLE_REQUEST)
|
||||
return None
|
||||
messages_to_summarize, preserved_messages = partition
|
||||
if all(
|
||||
message.additional_kwargs.get("lc_source") == "summarization"
|
||||
for message in messages_to_summarize
|
||||
):
|
||||
if budget["estimated_input_tokens"] > context_window:
|
||||
raise ContextSummarizationError(self._UNCOMPRESSIBLE_REQUEST)
|
||||
return None
|
||||
return messages_to_summarize, preserved_messages
|
||||
|
||||
@classmethod
|
||||
def _should_skip_after_current_turn_compaction(
|
||||
cls, messages: list[AnyMessage], budget: dict[str, Any]
|
||||
) -> bool:
|
||||
"""同轮只在新工具结果已使请求超窗时再次压缩。"""
|
||||
for message in reversed(messages):
|
||||
anchor_id = message.additional_kwargs.get(cls._COMPACTION_ANCHOR_KEY)
|
||||
if not isinstance(anchor_id, str):
|
||||
continue
|
||||
anchor_index = next(
|
||||
(
|
||||
index
|
||||
for index, candidate in enumerate(messages)
|
||||
if candidate.id == anchor_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if anchor_index is None:
|
||||
return False
|
||||
messages_after_anchor = messages[anchor_index + 1 :]
|
||||
if any(
|
||||
isinstance(candidate, HumanMessage)
|
||||
for candidate in messages_after_anchor
|
||||
):
|
||||
return False
|
||||
if any(isinstance(candidate, ToolMessage) for candidate in messages_after_anchor):
|
||||
estimated_tokens = budget.get("estimated_input_tokens")
|
||||
context_window = budget.get("context_window_tokens")
|
||||
return not (
|
||||
isinstance(estimated_tokens, int)
|
||||
and isinstance(context_window, int)
|
||||
and estimated_tokens > context_window
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
def _build_compacted_messages(
|
||||
self, summary: str, preserved_messages: list[AnyMessage]
|
||||
) -> list[AnyMessage]:
|
||||
"""构造摘要与近期历史，并记录本轮压缩输入边界。"""
|
||||
summary_messages = self.summarizer.build_summary_messages(summary)
|
||||
if summary_messages:
|
||||
self.summarizer.ensure_message_ids(summary_messages)
|
||||
anchor_id = (
|
||||
preserved_messages[-1].id
|
||||
if preserved_messages
|
||||
else summary_messages[0].id
|
||||
)
|
||||
first_summary = summary_messages[0]
|
||||
summary_messages[0] = first_summary.model_copy(
|
||||
update={
|
||||
"additional_kwargs": {
|
||||
**first_summary.additional_kwargs,
|
||||
self._COMPACTION_ANCHOR_KEY: anchor_id,
|
||||
}
|
||||
}
|
||||
)
|
||||
return [*summary_messages, *preserved_messages]
|
||||
|
||||
def _validate_or_repartition(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
messages_to_summarize: list[AnyMessage],
|
||||
preserved_messages: list[AnyMessage],
|
||||
summary: str,
|
||||
) -> tuple[list[AnyMessage], tuple[list[AnyMessage], list[AnyMessage]] | None]:
|
||||
"""复核压缩后的最终预算，并计算一次更小的近期历史分区。"""
|
||||
compacted_messages = self._build_compacted_messages(summary, preserved_messages)
|
||||
compacted_budget = UsageMiddleware.estimate_request(
|
||||
request.override(messages=compacted_messages)
|
||||
)
|
||||
context_window = compacted_budget.get("context_window_tokens")
|
||||
estimated_tokens = compacted_budget.get("estimated_input_tokens")
|
||||
if not isinstance(context_window, int) or not isinstance(estimated_tokens, int):
|
||||
return compacted_messages, None
|
||||
|
||||
if estimated_tokens <= context_window:
|
||||
return compacted_messages, None
|
||||
|
||||
summary_messages = self.summarizer.build_summary_messages(summary)
|
||||
fixed_summary_budget = UsageMiddleware.estimate_request(
|
||||
request.override(messages=summary_messages)
|
||||
)
|
||||
available_recent_tokens = (
|
||||
context_window - fixed_summary_budget["estimated_input_tokens"]
|
||||
)
|
||||
if available_recent_tokens <= 0:
|
||||
raise ContextSummarizationError(self._UNCOMPRESSIBLE_REQUEST)
|
||||
repartition = self.summarizer.partition_for_token_limit(
|
||||
list(request.messages),
|
||||
available_recent_tokens,
|
||||
force=True,
|
||||
minimum_cutoff=len(messages_to_summarize) + 1,
|
||||
strict_token_limit=True,
|
||||
)
|
||||
if (
|
||||
repartition is None
|
||||
or len(repartition[0]) <= len(messages_to_summarize)
|
||||
):
|
||||
raise ContextSummarizationError(self._UNCOMPRESSIBLE_REQUEST)
|
||||
return compacted_messages, repartition
|
||||
|
||||
def _require_within_window(
|
||||
self, request: ModelRequest, compacted_messages: list[AnyMessage]
|
||||
) -> None:
|
||||
"""禁止把已知仍超过主模型窗口的请求发送给 provider。"""
|
||||
budget = UsageMiddleware.estimate_request(
|
||||
request.override(messages=compacted_messages)
|
||||
)
|
||||
estimated_tokens = budget.get("estimated_input_tokens")
|
||||
context_window = budget.get("context_window_tokens")
|
||||
if (
|
||||
isinstance(estimated_tokens, int)
|
||||
and isinstance(context_window, int)
|
||||
and estimated_tokens > context_window
|
||||
):
|
||||
raise ContextSummarizationError(self._UNCOMPRESSIBLE_REQUEST)
|
||||
|
||||
def _prepare_messages(self, request: ModelRequest) -> list[AnyMessage] | None:
|
||||
"""同步生成摘要与需要原样保留的近期消息。"""
|
||||
partition = self._compaction_partition(request)
|
||||
if partition is None:
|
||||
return None
|
||||
messages_to_summarize, preserved_messages = partition
|
||||
summary = self.summarizer.create_summary(messages_to_summarize)
|
||||
compacted_messages, repartition = self._validate_or_repartition(
|
||||
request,
|
||||
messages_to_summarize,
|
||||
preserved_messages,
|
||||
summary,
|
||||
)
|
||||
if repartition is not None:
|
||||
messages_to_summarize, preserved_messages = repartition
|
||||
summary = self.summarizer.create_summary(messages_to_summarize)
|
||||
compacted_messages = self._build_compacted_messages(
|
||||
summary, preserved_messages
|
||||
)
|
||||
self._require_within_window(request, compacted_messages)
|
||||
return compacted_messages
|
||||
|
||||
async def _aprepare_messages(
|
||||
self, request: ModelRequest
|
||||
) -> list[AnyMessage] | None:
|
||||
"""异步生成摘要与需要原样保留的近期消息。"""
|
||||
partition = self._compaction_partition(request)
|
||||
if partition is None:
|
||||
return None
|
||||
messages_to_summarize, preserved_messages = partition
|
||||
summary = await self.summarizer.acreate_summary(messages_to_summarize)
|
||||
compacted_messages, repartition = self._validate_or_repartition(
|
||||
request,
|
||||
messages_to_summarize,
|
||||
preserved_messages,
|
||||
summary,
|
||||
)
|
||||
if repartition is not None:
|
||||
messages_to_summarize, preserved_messages = repartition
|
||||
summary = await self.summarizer.acreate_summary(messages_to_summarize)
|
||||
compacted_messages = self._build_compacted_messages(
|
||||
summary, preserved_messages
|
||||
)
|
||||
self._require_within_window(request, compacted_messages)
|
||||
return compacted_messages
|
||||
|
||||
@staticmethod
|
||||
def _with_state_update(
|
||||
response: ModelResponse, compacted_messages: list[AnyMessage]
|
||||
) -> ExtendedModelResponse:
|
||||
"""主模型成功后一次性替换历史，同时保留本次模型结果。"""
|
||||
return ExtendedModelResponse(
|
||||
model_response=response,
|
||||
command=Command(
|
||||
update={
|
||||
"messages": [
|
||||
RemoveMessage(id=REMOVE_ALL_MESSAGES),
|
||||
*compacted_messages,
|
||||
*response.result,
|
||||
]
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
def wrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], ModelResponse],
|
||||
) -> ModelResponse | ExtendedModelResponse:
|
||||
"""同步压缩最终请求；模型失败时不提交摘要状态。"""
|
||||
compacted_messages = self._prepare_messages(request)
|
||||
if compacted_messages is None:
|
||||
return handler(request)
|
||||
response = handler(request.override(messages=compacted_messages))
|
||||
return self._with_state_update(response, compacted_messages)
|
||||
|
||||
async def awrap_model_call(
|
||||
self,
|
||||
request: ModelRequest,
|
||||
handler: Callable[[ModelRequest], Awaitable[ModelResponse]],
|
||||
) -> ModelResponse | ExtendedModelResponse:
|
||||
"""异步压缩最终请求；模型失败时不提交摘要状态。"""
|
||||
compacted_messages = await self._aprepare_messages(request)
|
||||
if compacted_messages is None:
|
||||
return await handler(request)
|
||||
response = await handler(request.override(messages=compacted_messages))
|
||||
return self._with_state_update(response, compacted_messages)
|
||||
@@ -26,9 +26,9 @@ from langchain_core.tools import BaseTool
|
||||
from langgraph.runtime import Runtime
|
||||
from typing_extensions import TypedDict # noqa
|
||||
|
||||
from app.agent.llm.helper import LLMHelper
|
||||
from app.agent.llm import LLMHelper
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
MIN_SELECTED_TOOL_COUNT = 4
|
||||
RECENT_SELECTION_CONTEXT_MESSAGE_LIMIT = 6
|
||||
|
||||
+10
-230
@@ -9,25 +9,19 @@ from langchain.agents.middleware.types import (
|
||||
ResponseT,
|
||||
)
|
||||
from langchain_core.messages import AIMessage
|
||||
from langchain_core.messages.utils import count_tokens_approximately
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
|
||||
|
||||
class UsageMiddleware(AgentMiddleware):
|
||||
"""观察最终模型请求预算，并记录模型返回的真实 usage。"""
|
||||
"""记录模型调用 usage 信息并回传给外部会话。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
on_usage: Callable[[dict[str, Any]], None] | None = None,
|
||||
on_request_budget: Callable[[dict[str, Any]], None] | None = None,
|
||||
next_request_sequence: Callable[[], int] | None = None,
|
||||
) -> None:
|
||||
self.on_usage = on_usage
|
||||
self.on_request_budget = on_request_budget
|
||||
self.next_request_sequence = next_request_sequence
|
||||
self._request_sequence = 0
|
||||
|
||||
@staticmethod
|
||||
def _coerce_int(value: Any) -> int | None:
|
||||
@@ -38,37 +32,6 @@ class UsageMiddleware(AgentMiddleware):
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _coerce_positive_int(value: Any) -> int | None:
|
||||
"""仅接受模型 profile 和请求设置声明的非 bool 正整数。"""
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
|
||||
return None
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _lookup_positive_int(cls, container: Any, *keys: str) -> int | None:
|
||||
"""按字段优先级读取 token 上限，拒绝隐式数值转换。"""
|
||||
if not container:
|
||||
return None
|
||||
|
||||
getter = getattr(container, "get", None)
|
||||
if callable(getter):
|
||||
for key in keys:
|
||||
value = getter(key)
|
||||
if value is not None:
|
||||
normalized = cls._coerce_positive_int(value)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
|
||||
for key in keys:
|
||||
value = getattr(container, key, None)
|
||||
if value is not None:
|
||||
normalized = cls._coerce_positive_int(value)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _lookup_int(cls, container: Any, *keys: str) -> int | None:
|
||||
if not container:
|
||||
@@ -102,158 +65,18 @@ class UsageMiddleware(AgentMiddleware):
|
||||
|
||||
@classmethod
|
||||
def _extract_model_name(cls, model: Any) -> str | None:
|
||||
for field in ("model", "model_name", "model_id"):
|
||||
try:
|
||||
value = getattr(model, field, None)
|
||||
except Exception:
|
||||
continue
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
return (
|
||||
getattr(model, "model", None)
|
||||
or getattr(model, "model_name", None)
|
||||
or getattr(model, "model_id", None)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _extract_context_window_tokens(cls, model: Any) -> int | None:
|
||||
try:
|
||||
profile = getattr(model, "profile", None)
|
||||
except Exception:
|
||||
return None
|
||||
profile = getattr(model, "profile", None)
|
||||
if not profile:
|
||||
return None
|
||||
try:
|
||||
return cls._lookup_positive_int(
|
||||
profile, "max_input_tokens", "input_token_limit"
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _extract_model_max_output_tokens(cls, model: Any) -> int | None:
|
||||
"""读取模型输出能力上限；该值不代表单次请求已经预留的输出空间。"""
|
||||
try:
|
||||
profile = getattr(model, "profile", None)
|
||||
except Exception:
|
||||
return None
|
||||
if not profile:
|
||||
return None
|
||||
try:
|
||||
return cls._lookup_positive_int(
|
||||
profile, "max_output_tokens", "output_token_limit"
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _next_request_sequence(self) -> int | None:
|
||||
"""优先使用会话级序号，使图重建后的请求仍保持单调顺序。"""
|
||||
if callable(self.next_request_sequence):
|
||||
try:
|
||||
return self.next_request_sequence()
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"分配会话级模型请求序号失败: error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
# 无法证明顺序的请求仍可累计 usage，但不能参与最近请求快照竞争。
|
||||
return None
|
||||
self._request_sequence += 1
|
||||
return self._request_sequence
|
||||
|
||||
@classmethod
|
||||
def _extract_configured_output_limit_tokens(
|
||||
cls, request: ModelRequest
|
||||
) -> int | None:
|
||||
"""读取最终请求显式配置的单次输出上限。"""
|
||||
model_settings = request.model_settings or {}
|
||||
value = cls._lookup_positive_int(
|
||||
model_settings,
|
||||
"max_completion_tokens",
|
||||
"max_tokens",
|
||||
"max_output_tokens",
|
||||
)
|
||||
return value
|
||||
|
||||
@staticmethod
|
||||
def _count_multimodal_blocks(messages: list[Any]) -> tuple[int, int]:
|
||||
"""统计图片和未知多模态块，不保留块内容或资源地址。"""
|
||||
image_count = 0
|
||||
unknown_count = 0
|
||||
for message in messages:
|
||||
content = getattr(message, "content", None)
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
continue
|
||||
if not isinstance(block, dict):
|
||||
unknown_count += 1
|
||||
continue
|
||||
block_type = block.get("type")
|
||||
if block_type in {"image", "image_url"}:
|
||||
image_count += 1
|
||||
elif block_type != "text":
|
||||
unknown_count += 1
|
||||
return image_count, unknown_count
|
||||
|
||||
@classmethod
|
||||
def estimate_request(cls, request: ModelRequest) -> dict[str, Any]:
|
||||
"""估算最终模型输入组成，仅返回可安全暴露的聚合数字。"""
|
||||
messages = list(request.messages or [])
|
||||
system_messages = [request.system_message] if request.system_message else []
|
||||
tools = list(request.tools or [])
|
||||
message_tokens = count_tokens_approximately(
|
||||
messages,
|
||||
use_usage_metadata_scaling=False,
|
||||
)
|
||||
system_tokens = count_tokens_approximately(
|
||||
system_messages,
|
||||
use_usage_metadata_scaling=False,
|
||||
)
|
||||
tool_tokens = count_tokens_approximately(
|
||||
[],
|
||||
tools=tools,
|
||||
use_usage_metadata_scaling=False,
|
||||
)
|
||||
estimated_input_tokens = message_tokens + system_tokens + tool_tokens
|
||||
context_window_tokens = cls._extract_context_window_tokens(request.model)
|
||||
model_max_output_tokens = cls._extract_model_max_output_tokens(request.model)
|
||||
configured_output_limit_tokens = cls._extract_configured_output_limit_tokens(
|
||||
request
|
||||
)
|
||||
image_count, unknown_multimodal_count = cls._count_multimodal_blocks(
|
||||
[*system_messages, *messages]
|
||||
)
|
||||
estimated_input_ratio = (
|
||||
estimated_input_tokens / context_window_tokens
|
||||
if context_window_tokens
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"has_estimate": True,
|
||||
"model": cls._extract_model_name(request.model),
|
||||
"message_count": len(messages),
|
||||
"tool_count": len(tools),
|
||||
"image_count": image_count,
|
||||
"unknown_multimodal_count": unknown_multimodal_count,
|
||||
"message_tokens": message_tokens,
|
||||
"system_tokens": system_tokens,
|
||||
"tool_tokens": tool_tokens,
|
||||
# 该成本已经包含在 message_tokens 中，只单独暴露组成，不能再次汇总。
|
||||
"multimodal_tokens": image_count * 85,
|
||||
"estimated_input_tokens": estimated_input_tokens,
|
||||
"context_window_tokens": context_window_tokens,
|
||||
"estimated_remaining_input_tokens": (
|
||||
context_window_tokens - estimated_input_tokens
|
||||
if context_window_tokens
|
||||
else None
|
||||
),
|
||||
"estimated_input_ratio": estimated_input_ratio,
|
||||
"estimated_over_input_limit": (
|
||||
estimated_input_tokens > context_window_tokens
|
||||
if context_window_tokens
|
||||
else None
|
||||
),
|
||||
"model_max_output_tokens": model_max_output_tokens,
|
||||
"configured_output_limit_tokens": configured_output_limit_tokens,
|
||||
}
|
||||
return cls._lookup_int(profile, "max_input_tokens", "input_token_limit")
|
||||
|
||||
@classmethod
|
||||
def _extract_usage(cls, ai_message: AIMessage) -> dict[str, Any]:
|
||||
@@ -467,7 +290,6 @@ class UsageMiddleware(AgentMiddleware):
|
||||
cache_miss_tokens,
|
||||
)
|
||||
)
|
||||
input_usage_available = input_tokens is not None
|
||||
resolved_input = input_tokens or 0
|
||||
resolved_output = output_tokens or 0
|
||||
resolved_total = (
|
||||
@@ -493,7 +315,6 @@ class UsageMiddleware(AgentMiddleware):
|
||||
|
||||
return {
|
||||
"has_usage": has_usage,
|
||||
"input_usage_available": input_usage_available,
|
||||
"cache_usage_available": has_cache_usage,
|
||||
"input_tokens": resolved_input,
|
||||
"output_tokens": resolved_output,
|
||||
@@ -511,39 +332,6 @@ class UsageMiddleware(AgentMiddleware):
|
||||
[ModelRequest[ContextT]], Awaitable[ModelResponse[ResponseT]]
|
||||
],
|
||||
) -> ModelResponse[ResponseT]:
|
||||
request_sequence = self._next_request_sequence()
|
||||
request_budget = None
|
||||
try:
|
||||
request_budget = {
|
||||
"request_sequence": request_sequence,
|
||||
**self.estimate_request(request),
|
||||
}
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"估算最终模型请求预算失败: error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
request_budget = {
|
||||
"request_sequence": request_sequence,
|
||||
"has_estimate": False,
|
||||
"model": self._extract_model_name(request.model),
|
||||
"context_window_tokens": self._extract_context_window_tokens(
|
||||
request.model
|
||||
),
|
||||
}
|
||||
if callable(self.on_request_budget):
|
||||
request_budget_recorded = False
|
||||
try:
|
||||
self.on_request_budget(request_budget)
|
||||
request_budget_recorded = True
|
||||
except Exception as error:
|
||||
logger.debug(
|
||||
"记录最终模型请求预算失败: error_type=%s",
|
||||
type(error).__name__,
|
||||
)
|
||||
else:
|
||||
request_budget_recorded = False
|
||||
|
||||
response = await handler(request)
|
||||
|
||||
if not callable(self.on_usage):
|
||||
@@ -563,7 +351,6 @@ class UsageMiddleware(AgentMiddleware):
|
||||
if ai_message
|
||||
else {
|
||||
"has_usage": False,
|
||||
"input_usage_available": False,
|
||||
"cache_usage_available": False,
|
||||
"input_tokens": 0,
|
||||
"output_tokens": 0,
|
||||
@@ -576,18 +363,11 @@ class UsageMiddleware(AgentMiddleware):
|
||||
)
|
||||
context_window_tokens = self._extract_context_window_tokens(request.model)
|
||||
context_usage_ratio = None
|
||||
if context_window_tokens and usage["input_usage_available"]:
|
||||
if context_window_tokens and usage["has_usage"]:
|
||||
context_usage_ratio = usage["input_tokens"] / context_window_tokens
|
||||
|
||||
self.on_usage(
|
||||
{
|
||||
"request_sequence": request_sequence,
|
||||
"request_budget_recorded": request_budget_recorded,
|
||||
"estimated_input_tokens": (
|
||||
request_budget.get("estimated_input_tokens")
|
||||
if request_budget
|
||||
else None
|
||||
),
|
||||
"model": self._extract_model_name(request.model),
|
||||
"context_window_tokens": context_window_tokens,
|
||||
"context_usage_ratio": context_usage_ratio,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,55 +0,0 @@
|
||||
"""MoviePilot Agent 宿主策略公共内部入口。"""
|
||||
|
||||
from importlib import import_module
|
||||
from typing import Any
|
||||
|
||||
|
||||
_EXPORT_MODULES = {
|
||||
"ActionEffect": "app.agent.policy.contracts",
|
||||
"ActionPolicy": "app.agent.policy.contracts",
|
||||
"AuthSource": "app.agent.policy.contracts",
|
||||
"ConfirmationMode": "app.agent.policy.contracts",
|
||||
"ExecutionOutcome": "app.agent.policy.contracts",
|
||||
"ExecutionReceipt": "app.agent.policy.contracts",
|
||||
"MigrationState": "app.agent.policy.contracts",
|
||||
"PolicyDecision": "app.agent.policy.contracts",
|
||||
"PolicyObservation": "app.agent.policy.contracts",
|
||||
"PolicyPrincipal": "app.agent.policy.contracts",
|
||||
"PrincipalRole": "app.agent.policy.contracts",
|
||||
"PrincipalType": "app.agent.policy.contracts",
|
||||
"RecoveryMode": "app.agent.policy.contracts",
|
||||
"ResultSensitivity": "app.agent.policy.contracts",
|
||||
"ToolInvocation": "app.agent.policy.contracts",
|
||||
"ToolOrigin": "app.agent.policy.contracts",
|
||||
"ToolPolicyContext": "app.agent.policy.contracts",
|
||||
"ToolRevision": "app.agent.policy.contracts",
|
||||
"AgentToolPolicyOrchestrator": "app.agent.policy.orchestrator",
|
||||
"DEFAULT_TOOL_POLICY_ORCHESTRATOR": "app.agent.policy.orchestrator",
|
||||
"call_policy_hook": "app.agent.policy.orchestrator",
|
||||
"DEFAULT_TOOL_POLICY_REGISTRY": "app.agent.policy.registry",
|
||||
"ToolPolicyRegistry": "app.agent.policy.registry",
|
||||
"REDACTED_VALUE": "app.agent.policy.sanitizer",
|
||||
"sanitize_for_host": "app.agent.policy.sanitizer",
|
||||
"stable_type_name": "app.agent.policy.sanitizer",
|
||||
"summarize_error": "app.agent.policy.sanitizer",
|
||||
"summarize_input": "app.agent.policy.sanitizer",
|
||||
"summarize_result": "app.agent.policy.sanitizer",
|
||||
}
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""首次访问公开策略对象时只加载其所属模块。"""
|
||||
module_name = _EXPORT_MODULES.get(name)
|
||||
if module_name is None:
|
||||
raise AttributeError(f"module 'app.agent.policy' has no attribute {name!r}")
|
||||
value = getattr(import_module(module_name), name)
|
||||
globals()[name] = value
|
||||
return value
|
||||
|
||||
|
||||
def __dir__() -> list[str]:
|
||||
"""让惰性公开对象继续支持交互式发现。"""
|
||||
return sorted(set(globals()) | set(_EXPORT_MODULES))
|
||||
|
||||
|
||||
__all__ = list(_EXPORT_MODULES)
|
||||
@@ -1,259 +0,0 @@
|
||||
"""MoviePilot Agent 宿主策略的内部契约。"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Mapping, MutableMapping, Optional
|
||||
|
||||
|
||||
class ToolOrigin(str, Enum):
|
||||
"""工具调用的宿主可信入口。"""
|
||||
|
||||
AGENT_INTERACTIVE = "agent_interactive"
|
||||
AGENT_API = "agent_api"
|
||||
OPERATOR_DIRECT = "operator_direct"
|
||||
BACKGROUND = "background"
|
||||
SUBAGENT = "subagent"
|
||||
|
||||
|
||||
class PrincipalType(str, Enum):
|
||||
"""调用主体类型，用于区分人、管理员集成和内部运行时。"""
|
||||
|
||||
HUMAN = "human"
|
||||
SYSTEM_ADMIN_INTEGRATION = "system_admin_integration"
|
||||
SCOPED_AGENT = "scoped_agent"
|
||||
BACKGROUND = "background"
|
||||
SUBAGENT = "subagent"
|
||||
|
||||
|
||||
class AuthSource(str, Enum):
|
||||
"""主体身份的宿主认证来源。"""
|
||||
|
||||
CHANNEL = "channel"
|
||||
WEB_SESSION = "web_session"
|
||||
API_TOKEN = "api_token"
|
||||
INTERNAL = "internal"
|
||||
AGENT_TOKEN = "agent_token"
|
||||
|
||||
|
||||
class PrincipalRole(str, Enum):
|
||||
"""策略授权使用的角色层级。"""
|
||||
|
||||
USER = "user"
|
||||
CHANNEL_ADMIN = "channel_admin"
|
||||
SYSTEM_ADMIN = "system_admin"
|
||||
SYSTEM_INTERNAL = "system_internal"
|
||||
|
||||
|
||||
class ActionEffect(str, Enum):
|
||||
"""工具调用的实际副作用类别。"""
|
||||
|
||||
SAFE_READ = "safe_read"
|
||||
SENSITIVE_READ = "sensitive_read"
|
||||
REVERSIBLE_WRITE = "reversible_write"
|
||||
DESTRUCTIVE_WRITE = "destructive_write"
|
||||
EXTERNAL_SIDE_EFFECT = "external_side_effect"
|
||||
ARBITRARY_EXECUTION = "arbitrary_execution"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ConfirmationMode(str, Enum):
|
||||
"""动作在完成授权后所需的确认方式。"""
|
||||
|
||||
NONE = "none"
|
||||
REQUIRED = "required"
|
||||
UNSUPPORTED = "unsupported"
|
||||
|
||||
|
||||
class RecoveryMode(str, Enum):
|
||||
"""动作可提供的执行恢复保证。"""
|
||||
|
||||
NONE = "none"
|
||||
TRANSACTION = "transaction"
|
||||
BEFORE_STATE = "before_state"
|
||||
RECOVERABLE_DELETE = "recoverable_delete"
|
||||
IDEMPOTENT = "idempotent"
|
||||
RECONCILE = "reconcile"
|
||||
MANUAL_ONLY = "manual_only"
|
||||
|
||||
|
||||
class ResultSensitivity(str, Enum):
|
||||
"""工具结果进入模型、记忆和日志时的敏感等级。"""
|
||||
|
||||
NORMAL = "normal"
|
||||
PRIVATE = "private"
|
||||
SECRET = "secret"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class MigrationState(str, Enum):
|
||||
"""工具策略当前采用宿主强制还是兼容观测。"""
|
||||
|
||||
ENFORCED = "enforced"
|
||||
LEGACY_SHADOW = "legacy_shadow"
|
||||
|
||||
|
||||
class ExecutionOutcome(str, Enum):
|
||||
"""工具 handler 观测终态；成功不代表业务授权或副作用已完成。"""
|
||||
|
||||
SUCCEEDED = "succeeded"
|
||||
FAILED = "failed"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyPrincipal:
|
||||
"""由可信入口建立、不可由工具参数覆盖的调用主体。"""
|
||||
|
||||
principal_id: str
|
||||
principal_type: PrincipalType
|
||||
auth_source: AuthSource
|
||||
role: PrincipalRole
|
||||
scopes: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolInvocation:
|
||||
"""一次进入宿主策略层的规范化工具调用。"""
|
||||
|
||||
invocation_id: str
|
||||
tool_name: str
|
||||
arguments: Mapping[str, Any]
|
||||
principal: PolicyPrincipal
|
||||
session_id: str
|
||||
origin: ToolOrigin
|
||||
channel: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolRevision:
|
||||
"""记录目录项绑定的工具实现、工厂和插件目录版本。"""
|
||||
|
||||
implementation: str
|
||||
factory: str
|
||||
plugin: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ActionPolicy:
|
||||
"""参数级动作策略及其兼容迁移状态。"""
|
||||
|
||||
effect: ActionEffect
|
||||
required_role: PrincipalRole
|
||||
confirmation: ConfirmationMode
|
||||
recovery: RecoveryMode
|
||||
result_sensitivity: ResultSensitivity
|
||||
migration_state: MigrationState
|
||||
policy_version: str = "p1-g1-v1"
|
||||
interactive_allowed: bool = True
|
||||
machine_allowed: bool = True
|
||||
background_allowed: bool = True
|
||||
subagent_allowed: bool = True
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyDecision:
|
||||
"""宿主策略层决定；shadow allow 仅表示新策略不拦截，旧门禁仍是授权事实源。"""
|
||||
|
||||
allowed: bool
|
||||
confirmation_required: bool
|
||||
shadow: bool
|
||||
reason_code: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PolicyObservation:
|
||||
"""调用开始时生成、供完成或失败回执复用的观测对象。"""
|
||||
|
||||
invocation: ToolInvocation
|
||||
policy: ActionPolicy
|
||||
decision: PolicyDecision
|
||||
input_summary: str
|
||||
started_at: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExecutionReceipt:
|
||||
"""工具策略生命周期生成的非持久化脱敏回执。"""
|
||||
|
||||
invocation_id: str
|
||||
tool_name: str
|
||||
origin: ToolOrigin
|
||||
decision: PolicyDecision
|
||||
outcome: ExecutionOutcome
|
||||
input_summary: str
|
||||
result_summary: Optional[str] = None
|
||||
error_summary: Optional[str] = None
|
||||
duration_ms: int = 0
|
||||
external_may_continue: bool = False # 中断后外部操作仍可能继续，不能视为已停止。
|
||||
needs_reconcile: bool = False # 调用方需查询外部实际状态后再决定补偿或重试。
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolPolicyContext:
|
||||
"""宿主入口上下文；管理员状态引用会随缓存图的每轮执行刷新。"""
|
||||
|
||||
session_id: str
|
||||
user_id: str
|
||||
origin: ToolOrigin
|
||||
principal_type: PrincipalType
|
||||
auth_source: AuthSource
|
||||
agent_context: MutableMapping[str, Any] = field(repr=False, compare=False)
|
||||
channel: Optional[str] = None
|
||||
source: Optional[str] = None
|
||||
|
||||
@property
|
||||
def principal(self) -> PolicyPrincipal:
|
||||
"""根据当前宿主上下文生成本次调用主体。"""
|
||||
if self.principal_type in {PrincipalType.BACKGROUND, PrincipalType.SUBAGENT}:
|
||||
default_role = PrincipalRole.SYSTEM_INTERNAL
|
||||
else:
|
||||
default_role = PrincipalRole.USER
|
||||
role = (
|
||||
PrincipalRole.SYSTEM_ADMIN
|
||||
if bool(self.agent_context.get("is_admin"))
|
||||
else default_role
|
||||
)
|
||||
raw_scopes = self.agent_context.get("policy_scopes") or ()
|
||||
scopes = tuple(str(scope) for scope in raw_scopes if scope)
|
||||
return PolicyPrincipal(
|
||||
principal_id=str(self.user_id or self.principal_type.value),
|
||||
principal_type=self.principal_type,
|
||||
auth_source=self.auth_source,
|
||||
role=role,
|
||||
scopes=scopes,
|
||||
)
|
||||
|
||||
def for_subagent(self) -> "ToolPolicyContext":
|
||||
"""保留用户与会话归属，并切换为子代理可信来源。"""
|
||||
return ToolPolicyContext(
|
||||
session_id=self.session_id,
|
||||
user_id=self.user_id,
|
||||
origin=ToolOrigin.SUBAGENT,
|
||||
principal_type=PrincipalType.SUBAGENT,
|
||||
auth_source=AuthSource.INTERNAL,
|
||||
agent_context=self.agent_context,
|
||||
channel=self.channel,
|
||||
source=self.source,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ActionEffect",
|
||||
"ActionPolicy",
|
||||
"AuthSource",
|
||||
"ConfirmationMode",
|
||||
"ExecutionOutcome",
|
||||
"ExecutionReceipt",
|
||||
"MigrationState",
|
||||
"PolicyDecision",
|
||||
"PolicyObservation",
|
||||
"PolicyPrincipal",
|
||||
"PrincipalRole",
|
||||
"PrincipalType",
|
||||
"RecoveryMode",
|
||||
"ResultSensitivity",
|
||||
"ToolInvocation",
|
||||
"ToolOrigin",
|
||||
"ToolPolicyContext",
|
||||
"ToolRevision",
|
||||
]
|
||||
@@ -1,229 +0,0 @@
|
||||
"""Agent 工具策略观测、脱敏回执与共享执行边界。"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Mapping, Optional, TypeVar
|
||||
|
||||
from langchain_core.messages import ToolMessage
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.agent.policy.contracts import (
|
||||
ActionEffect,
|
||||
ConfirmationMode,
|
||||
ExecutionOutcome,
|
||||
ExecutionReceipt,
|
||||
MigrationState,
|
||||
PolicyDecision,
|
||||
PolicyObservation,
|
||||
RecoveryMode,
|
||||
ToolInvocation,
|
||||
ToolPolicyContext,
|
||||
)
|
||||
from app.agent.policy.registry import DEFAULT_TOOL_POLICY_REGISTRY, ToolPolicyRegistry
|
||||
from app.agent.policy.sanitizer import (
|
||||
stable_type_name,
|
||||
summarize_error,
|
||||
summarize_input,
|
||||
summarize_result,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
_HookResult = TypeVar("_HookResult")
|
||||
|
||||
|
||||
def call_policy_hook(
|
||||
phase: str,
|
||||
hook: Callable[..., _HookResult],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Optional[_HookResult]:
|
||||
"""以 fail-open 方式调用兼容观测 hook，故障只记录稳定类型。"""
|
||||
try:
|
||||
return hook(*args, **kwargs)
|
||||
except Exception as error:
|
||||
try:
|
||||
logger.warning(
|
||||
f"Agent工具策略观测失败: phase={phase}, "
|
||||
f"error_type={stable_type_name(error)}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_policy_arguments(tool: Any, arguments: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""为策略生成 Pydantic 规范化副本，不改变真实执行参数。"""
|
||||
raw_arguments = dict(arguments or {})
|
||||
args_schema = getattr(tool, "args_schema", None)
|
||||
if not args_schema:
|
||||
return raw_arguments
|
||||
try:
|
||||
validated = args_schema.model_validate(raw_arguments)
|
||||
return validated.model_dump(mode="json")
|
||||
except (AttributeError, TypeError, ValueError, ValidationError):
|
||||
# 实际 handler 仍负责既有参数错误语义；策略观测按原始值保守处理。
|
||||
return raw_arguments
|
||||
|
||||
|
||||
def _result_payload(result: Any) -> Any:
|
||||
"""从 LangChain 工具消息中提取模型可见结果供脱敏摘要使用。"""
|
||||
if isinstance(result, ToolMessage):
|
||||
return result.content
|
||||
return result
|
||||
|
||||
|
||||
class AgentToolPolicyOrchestrator:
|
||||
"""让 Agent middleware 与 direct manager 复用同一策略生命周期。"""
|
||||
|
||||
def __init__(self, registry: ToolPolicyRegistry = DEFAULT_TOOL_POLICY_REGISTRY) -> None:
|
||||
"""绑定工具策略解析表。"""
|
||||
self.registry = registry
|
||||
|
||||
def start(
|
||||
self,
|
||||
*,
|
||||
context: ToolPolicyContext,
|
||||
tool: Any,
|
||||
arguments: Mapping[str, Any],
|
||||
invocation_id: Optional[str] = None,
|
||||
) -> PolicyObservation:
|
||||
"""解析调用策略，并创建不影响现有 allow 行为的观测对象。"""
|
||||
tool_name = str(getattr(tool, "name", None) or "unknown_tool")
|
||||
normalized_arguments = _normalize_policy_arguments(tool, arguments)
|
||||
policy = self.registry.resolve(
|
||||
tool_name=tool_name,
|
||||
arguments=normalized_arguments,
|
||||
requires_admin=bool(getattr(tool, "_require_admin", False)),
|
||||
)
|
||||
if policy.migration_state is MigrationState.LEGACY_SHADOW:
|
||||
decision = PolicyDecision(
|
||||
allowed=True,
|
||||
confirmation_required=False,
|
||||
shadow=True,
|
||||
reason_code="legacy_shadow_allow",
|
||||
)
|
||||
elif policy.confirmation is ConfirmationMode.REQUIRED:
|
||||
# 通用编排器保持 shadow；支持的 Agent 入口会在 ToolNode 前独立完成确认。
|
||||
decision = PolicyDecision(
|
||||
allowed=True,
|
||||
confirmation_required=False,
|
||||
shadow=True,
|
||||
reason_code="confirmation_policy_shadow_allow",
|
||||
)
|
||||
else:
|
||||
decision = PolicyDecision(
|
||||
allowed=True,
|
||||
confirmation_required=False,
|
||||
shadow=False,
|
||||
reason_code="safe_read_allow",
|
||||
)
|
||||
invocation = ToolInvocation(
|
||||
invocation_id=invocation_id or uuid.uuid4().hex,
|
||||
tool_name=tool_name,
|
||||
arguments=normalized_arguments,
|
||||
principal=context.principal,
|
||||
session_id=context.session_id,
|
||||
origin=context.origin,
|
||||
channel=context.channel,
|
||||
source=context.source,
|
||||
)
|
||||
input_summary = summarize_input(normalized_arguments)
|
||||
observation = PolicyObservation(
|
||||
invocation=invocation,
|
||||
policy=policy,
|
||||
decision=decision,
|
||||
input_summary=input_summary,
|
||||
started_at=time.monotonic(),
|
||||
)
|
||||
logger.debug(
|
||||
f"Agent工具策略: tool={tool_name}, origin={context.origin.value}, "
|
||||
f"decision={decision.reason_code}, input={input_summary}"
|
||||
)
|
||||
return observation
|
||||
|
||||
@staticmethod
|
||||
def finish(observation: PolicyObservation, result: Any) -> ExecutionReceipt:
|
||||
"""生成成功回执 envelope，并只记录脱敏结果摘要。"""
|
||||
result_summary = summarize_result(_result_payload(result))
|
||||
receipt = ExecutionReceipt(
|
||||
invocation_id=observation.invocation.invocation_id,
|
||||
tool_name=observation.invocation.tool_name,
|
||||
origin=observation.invocation.origin,
|
||||
decision=observation.decision,
|
||||
outcome=ExecutionOutcome.SUCCEEDED,
|
||||
input_summary=observation.input_summary,
|
||||
result_summary=result_summary,
|
||||
duration_ms=max(
|
||||
0,
|
||||
int((time.monotonic() - observation.started_at) * 1000),
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
f"Agent工具执行完成: tool={receipt.tool_name}, "
|
||||
f"origin={receipt.origin.value}, shadow={receipt.decision.shadow}, "
|
||||
f"duration_ms={receipt.duration_ms}, result={result_summary}"
|
||||
)
|
||||
return receipt
|
||||
|
||||
@staticmethod
|
||||
def _uncertain_external_state(
|
||||
observation: PolicyObservation,
|
||||
error: BaseException,
|
||||
) -> tuple[bool, bool]:
|
||||
"""标记取消或超时后无法确认的写操作终态。"""
|
||||
interrupted = isinstance(error, (asyncio.CancelledError, TimeoutError))
|
||||
read_only = observation.policy.effect in {
|
||||
ActionEffect.SAFE_READ,
|
||||
ActionEffect.SENSITIVE_READ,
|
||||
}
|
||||
if not interrupted or read_only:
|
||||
return False, False
|
||||
needs_reconcile = observation.policy.recovery not in {
|
||||
RecoveryMode.TRANSACTION,
|
||||
RecoveryMode.IDEMPOTENT,
|
||||
}
|
||||
return True, needs_reconcile
|
||||
|
||||
@staticmethod
|
||||
def fail(observation: PolicyObservation, error: BaseException) -> ExecutionReceipt:
|
||||
"""生成失败回执 envelope，不把异常中的凭据写入日志。"""
|
||||
error_summary = summarize_error(error)
|
||||
external_may_continue, needs_reconcile = (
|
||||
AgentToolPolicyOrchestrator._uncertain_external_state(observation, error)
|
||||
)
|
||||
receipt = ExecutionReceipt(
|
||||
invocation_id=observation.invocation.invocation_id,
|
||||
tool_name=observation.invocation.tool_name,
|
||||
origin=observation.invocation.origin,
|
||||
decision=observation.decision,
|
||||
outcome=ExecutionOutcome.FAILED,
|
||||
input_summary=observation.input_summary,
|
||||
error_summary=error_summary,
|
||||
duration_ms=max(
|
||||
0,
|
||||
int((time.monotonic() - observation.started_at) * 1000),
|
||||
),
|
||||
external_may_continue=external_may_continue,
|
||||
needs_reconcile=needs_reconcile,
|
||||
)
|
||||
logger.error(
|
||||
f"Agent工具执行失败: tool={receipt.tool_name}, "
|
||||
f"origin={receipt.origin.value}, shadow={receipt.decision.shadow}, "
|
||||
f"duration_ms={receipt.duration_ms}, error={error_summary}, "
|
||||
f"external_may_continue={receipt.external_may_continue}, "
|
||||
f"needs_reconcile={receipt.needs_reconcile}"
|
||||
)
|
||||
return receipt
|
||||
|
||||
|
||||
DEFAULT_TOOL_POLICY_ORCHESTRATOR = AgentToolPolicyOrchestrator()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"AgentToolPolicyOrchestrator",
|
||||
"DEFAULT_TOOL_POLICY_ORCHESTRATOR",
|
||||
"call_policy_hook",
|
||||
]
|
||||
@@ -1,201 +0,0 @@
|
||||
"""工具策略例外与参数级策略解析。"""
|
||||
|
||||
from typing import Any, Mapping
|
||||
|
||||
from app.agent.policy.contracts import (
|
||||
ActionEffect,
|
||||
ActionPolicy,
|
||||
ConfirmationMode,
|
||||
MigrationState,
|
||||
PrincipalRole,
|
||||
RecoveryMode,
|
||||
ResultSensitivity,
|
||||
)
|
||||
|
||||
|
||||
# 这些非管理员读取在运行时解析为强制 SAFE_READ；管理员门禁仍沿用原有授权事实源。
|
||||
SAFE_READ_TOOL_NAMES = frozenset(
|
||||
{
|
||||
"list_slash_commands",
|
||||
"query_installed_plugins",
|
||||
"query_personas",
|
||||
"query_schedulers",
|
||||
"query_workflows",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# 该清单只校验固定工具 inventory；未命中的固定或动态工具同样默认 LEGACY_SHADOW。
|
||||
BUILTIN_LEGACY_SHADOW_INVENTORY = frozenset(
|
||||
{
|
||||
"add_custom_filter_rule",
|
||||
"add_download_tasks",
|
||||
"add_rule_group",
|
||||
"add_subscribe",
|
||||
"apply_patch",
|
||||
"ask_user_choice",
|
||||
"browse_webpage",
|
||||
"create_agent_task",
|
||||
"delete_agent_task",
|
||||
"delete_custom_filter_rule",
|
||||
"delete_download_history",
|
||||
"delete_download_tasks",
|
||||
"delete_rule_group",
|
||||
"delete_subscribe",
|
||||
"delete_transfer_history",
|
||||
"edit_file",
|
||||
"execute_command",
|
||||
"get_recommendations",
|
||||
"get_search_results",
|
||||
"install_plugin",
|
||||
"list_directory",
|
||||
"query_agent_tasks",
|
||||
"query_builtin_filter_rules",
|
||||
"query_custom_filter_rules",
|
||||
"query_custom_identifiers",
|
||||
"query_directory_settings",
|
||||
"query_doctor_report",
|
||||
"query_download_tasks",
|
||||
"query_downloaders",
|
||||
"query_episode_schedule",
|
||||
"query_library_exists",
|
||||
"query_library_latest",
|
||||
"query_market_plugins",
|
||||
"query_media_detail",
|
||||
"query_plugin_capabilities",
|
||||
"query_plugin_config",
|
||||
"query_plugin_data",
|
||||
"query_popular_subscribes",
|
||||
"query_rule_groups",
|
||||
"query_site_userdata",
|
||||
"query_sites",
|
||||
"query_subscribe_history",
|
||||
"query_subscribe_shares",
|
||||
"query_subscribes",
|
||||
"query_system_settings",
|
||||
"query_transfer_history",
|
||||
"read_file",
|
||||
"recognize_captcha",
|
||||
"recognize_media",
|
||||
"reload_plugin",
|
||||
"run_agent_task",
|
||||
"run_scheduler",
|
||||
"run_slash_command",
|
||||
"run_workflow",
|
||||
"scrape_metadata",
|
||||
"search_media",
|
||||
"search_person",
|
||||
"search_person_credits",
|
||||
"search_subscribe",
|
||||
"search_torrents",
|
||||
"search_web",
|
||||
"send_local_file",
|
||||
"send_message",
|
||||
"send_voice_message",
|
||||
"switch_persona",
|
||||
"test_site",
|
||||
"transfer_file",
|
||||
"uninstall_plugin",
|
||||
"update_agent_task",
|
||||
"update_custom_filter_rule",
|
||||
"update_custom_identifiers",
|
||||
"update_download_tasks",
|
||||
"update_persona_definition",
|
||||
"update_plugin_config",
|
||||
"update_rule_group",
|
||||
"update_site",
|
||||
"update_site_cookie",
|
||||
"update_subscribe",
|
||||
"update_system_settings",
|
||||
"write_file",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ToolPolicyRegistry:
|
||||
"""解析固定和动态工具的宿主策略。"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
safe_read_tool_names: frozenset[str] = SAFE_READ_TOOL_NAMES,
|
||||
builtin_legacy_shadow_inventory: frozenset[str] = (
|
||||
BUILTIN_LEGACY_SHADOW_INVENTORY
|
||||
),
|
||||
) -> None:
|
||||
"""建立 SAFE_READ 例外与固定工具 inventory。"""
|
||||
overlap = safe_read_tool_names & builtin_legacy_shadow_inventory
|
||||
if overlap:
|
||||
raise ValueError(f"工具策略 inventory 存在重复项: {sorted(overlap)}")
|
||||
self._safe_read_tool_names = safe_read_tool_names
|
||||
self._builtin_legacy_shadow_inventory = builtin_legacy_shadow_inventory
|
||||
|
||||
@property
|
||||
def builtin_tool_inventory(self) -> set[str]:
|
||||
"""返回用于测试校验的固定工具 inventory。"""
|
||||
return set(
|
||||
self._safe_read_tool_names | self._builtin_legacy_shadow_inventory
|
||||
)
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
*,
|
||||
tool_name: str,
|
||||
arguments: Mapping[str, Any],
|
||||
requires_admin: bool,
|
||||
) -> ActionPolicy:
|
||||
"""根据工具名和宿主权限声明解析当前参数级策略。"""
|
||||
required_role = (
|
||||
PrincipalRole.SYSTEM_ADMIN if requires_admin else PrincipalRole.USER
|
||||
)
|
||||
if (
|
||||
tool_name == "query_system_settings"
|
||||
and arguments.get("show_secrets") is True
|
||||
):
|
||||
return ActionPolicy(
|
||||
effect=ActionEffect.SENSITIVE_READ,
|
||||
required_role=PrincipalRole.SYSTEM_ADMIN,
|
||||
confirmation=ConfirmationMode.REQUIRED,
|
||||
recovery=RecoveryMode.NONE,
|
||||
result_sensitivity=ResultSensitivity.SECRET,
|
||||
migration_state=MigrationState.ENFORCED,
|
||||
policy_version="p1-g2a2-v1",
|
||||
machine_allowed=True,
|
||||
background_allowed=False,
|
||||
subagent_allowed=False,
|
||||
)
|
||||
if tool_name in self._safe_read_tool_names:
|
||||
return ActionPolicy(
|
||||
effect=ActionEffect.SAFE_READ,
|
||||
required_role=required_role,
|
||||
confirmation=ConfirmationMode.NONE,
|
||||
recovery=RecoveryMode.NONE,
|
||||
result_sensitivity=ResultSensitivity.NORMAL,
|
||||
# 角色门禁仍由既有授权事实源判断，管理员读取保持兼容观测。
|
||||
migration_state=(
|
||||
MigrationState.LEGACY_SHADOW
|
||||
if requires_admin
|
||||
else MigrationState.ENFORCED
|
||||
),
|
||||
)
|
||||
|
||||
# 除明确例外外，固定和动态工具都保持现有能力，但不得被视为安全读取。
|
||||
return ActionPolicy(
|
||||
effect=ActionEffect.UNKNOWN,
|
||||
required_role=required_role,
|
||||
confirmation=ConfirmationMode.REQUIRED,
|
||||
recovery=RecoveryMode.MANUAL_ONLY,
|
||||
result_sensitivity=ResultSensitivity.UNKNOWN,
|
||||
migration_state=MigrationState.LEGACY_SHADOW,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_TOOL_POLICY_REGISTRY = ToolPolicyRegistry()
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BUILTIN_LEGACY_SHADOW_INVENTORY",
|
||||
"DEFAULT_TOOL_POLICY_REGISTRY",
|
||||
"SAFE_READ_TOOL_NAMES",
|
||||
"ToolPolicyRegistry",
|
||||
]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,69 +0,0 @@
|
||||
"""Agent 设置工具与宿主回执共用的敏感字段身份判定。"""
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
|
||||
_MAX_FIELD_NAME_CHARS = 256
|
||||
_ACRONYM_BOUNDARY_PATTERN = re.compile(r"(?<=[A-Z])(?=[A-Z][a-z])")
|
||||
_CAMEL_CASE_BOUNDARY_PATTERN = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
|
||||
_SECRET_FIELD_NAMES = frozenset(
|
||||
{
|
||||
"access_token",
|
||||
"api_key",
|
||||
"apikey",
|
||||
"api_token",
|
||||
"auth_header",
|
||||
"authorization",
|
||||
"client_secret",
|
||||
"cookie",
|
||||
"passkey",
|
||||
"passwd",
|
||||
"password",
|
||||
"private_key",
|
||||
"pwd",
|
||||
"refresh_token",
|
||||
"secret",
|
||||
"secret_access_key",
|
||||
"secret_key",
|
||||
"token",
|
||||
}
|
||||
)
|
||||
_SECRET_FIELD_ENDINGS = tuple(f"_{name}" for name in _SECRET_FIELD_NAMES)
|
||||
_SECRET_SETTING_NAMES = frozenset(
|
||||
{
|
||||
# CookieCloud 的用户 key 没有类型后缀，但与密码共同构成端到端加密凭据。
|
||||
"cookiecloud_key",
|
||||
}
|
||||
)
|
||||
_SECRET_SETTING_ENDINGS = (
|
||||
"_encrypt_key",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_field_name(value: Any) -> str:
|
||||
"""将短字段名规范化为 snake_case，非字符串不参与身份推导。"""
|
||||
if type(value) is not str:
|
||||
return ""
|
||||
text = value.strip()
|
||||
if len(text) > _MAX_FIELD_NAME_CHARS:
|
||||
text = text[-_MAX_FIELD_NAME_CHARS:]
|
||||
text = _ACRONYM_BOUNDARY_PATTERN.sub("_", text)
|
||||
text = _CAMEL_CASE_BOUNDARY_PATTERN.sub("_", text)
|
||||
return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_")
|
||||
|
||||
|
||||
def is_secret_setting_key(key: Any) -> bool:
|
||||
"""按完整字段或类型后缀识别凭据，避免误伤 token 统计与过期配置。"""
|
||||
normalized = _normalize_field_name(key)
|
||||
if not normalized:
|
||||
return False
|
||||
return (
|
||||
normalized in _SECRET_FIELD_NAMES
|
||||
or normalized in _SECRET_SETTING_NAMES
|
||||
or normalized.endswith(_SECRET_FIELD_ENDINGS)
|
||||
or normalized.endswith(_SECRET_SETTING_ENDINGS)
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["is_secret_setting_key"]
|
||||
@@ -21,7 +21,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
|
||||
<confirmation_policy>
|
||||
- Do not stop for approval on read-only operations.
|
||||
- Raw secret reads are protected operations rather than ordinary read-only queries. When a user explicitly asks for a raw credential or another unredacted sensitive setting, call `query_system_settings` with `show_secrets=true`; do not refuse the request solely because the value is sensitive. The host verifies administrator authority, obtains any required confirmation, and delivers the result through a protected channel. Never expose or repeat the secret in an ordinary assistant response, tool narration, or follow-up model context.
|
||||
- If the user has not explicitly requested an operation that changes system behavior, ask for confirmation before proceeding. This includes modifying system settings, updating plugin configuration, reloading plugins, running restart/stop/start commands, or triggering slash commands such as `/restart`.
|
||||
- Always get explicit consent before destructive or high-impact actions such as starting downloads, deleting subscriptions, deleting download tasks or files, removing history, installing/uninstalling plugins, changing site authentication, changing scheduler or workflow execution state, restarting services, or stopping services.
|
||||
- When the user explicitly asks for delayed, recurring, reminder, or monitoring work, use `create_agent_task` instead of promising to remember it or writing a JOB.md file. Use a `date` trigger with `delay_minutes` for requests such as "in 30 minutes", an exact `date` trigger for other single future runs, and a five-field `cron` trigger for recurring work. Manage existing autonomous tasks with `query_agent_tasks`, `update_agent_task`, `run_agent_task`, and `delete_agent_task`; these tools use integer `task_id` values. Use `query_schedulers` and `run_scheduler` only for MoviePilot system, plugin, or workflow runtime services, whose string `job_id` values must never be passed to autonomous-task tools.
|
||||
@@ -31,7 +30,7 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
|
||||
<moviepilot_domain_model>
|
||||
- Treat sites as a first-class system capability, not background detail. In MoviePilot, sites are the upstream source for search, account status, authentication, and many download or subscription decisions.
|
||||
- Understand the platform's core workflow as: site availability and configuration -> media search -> media recognition/metadata confirmation -> manual download or subscription -> transfer and library organization -> metadata scraping (including configured music lyrics) -> status/history confirmation.
|
||||
- Understand the platform's core workflow as: site availability and configuration -> media search -> media recognition/metadata confirmation -> manual download or subscription -> transfer and library organization -> status/history confirmation.
|
||||
- Treat manual download and subscription automation as two execution modes of the same acquisition pipeline. Manual download is user-triggered immediate acquisition; subscription is persistent site-driven monitoring and acquisition.
|
||||
- Keep the user anchored to the operational step that matters now: site, search, recognition, download, subscription, transfer, or status/history.
|
||||
- Users may attach images from supported channels; analyze them together with the text when relevant.
|
||||
@@ -49,7 +48,7 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
|
||||
<core_workflow>
|
||||
1. Site and Context Check: Determine whether site status, site scope, library state, existing subscriptions, or prior download/transfer history can affect the task.
|
||||
2. Media Identity Resolution: Confirm an exact source-native identity and pass it only as the fixed `media_source` enum plus `media_id`. Video and music share this pair; music also uses `media_type=music` and `music_type=recording|album|artist`. Use `search_media`, `query_media_detail`, or `recognize_media` as needed.
|
||||
2. Media Identity Resolution: Confirm exact media identity such as TMDB ID, title, year, type, season, or episode using `search_media`, `query_media_detail`, or `recognize_media` as needed.
|
||||
3. Resource Discovery: Use the appropriate search path for the task. For manual acquisition, search site resources and inspect result quality. For automation, prepare subscription conditions that will search sites continuously.
|
||||
4. Action Execution: Perform the requested task, typically one of: test/query site, search torrents, add download, add or modify subscription, or transfer and organize files.
|
||||
5. Final Confirmation: State the outcome briefly, including the key media facts, chosen site or resource scope when relevant, and the next blocker if the task could not be completed.
|
||||
@@ -67,7 +66,7 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- If torrent search yields no useful result, check site scope, site health, and recognition quality before concluding that the resource is unavailable.
|
||||
- Reuse the latest torrent search cache for `get_search_results` and `add_download_tasks` instead of re-running the same search unnecessarily.
|
||||
- For administrator code discovery across local files, use `execute_command(action="run")` with `rg` and narrow globs or paths; large searches may be split with narrower globs, paths, or `rg --files` filters. Use `list_directory` to inspect one known directory or a supported remote storage backend; request its `limit`/`offset` page fields when more than the first page is needed, and use `read_file` when the exact local file is known. If `read_file` reports truncation, continue with smaller `start_line` and `end_line` ranges instead of assuming the file ended.
|
||||
- Read the relevant file before changing it, then pick the editing tool by scope. Use `apply_patch` when one logical change spans multiple files, adds new files, or deletes files: submit a single patch wrapped in `*** Begin Patch` / `*** End Patch` with `*** Add File:`, `*** Update File:`, and `*** Delete File:` sections; every context and removed line must match the current content exactly, and the whole patch is validated before any file is written. Use `edit_file` for a single localized exact replacement within one already-read file; make `old_text` unique with enough surrounding context, and use `replace_all=true` only when every match must change. Use `write_file` for one standalone new file; set `overwrite=true` only for an intentional full rewrite, and use `read_file(include_metadata=true)` plus `expected_sha256` when preserving the previously read version matters.
|
||||
- Read the relevant file before changing it. Use `edit_file` for localized exact replacements; make `old_text` unique with enough surrounding context, and use `replace_all=true` only when every match must change. Use `write_file` for new files; set `overwrite=true` only for an intentional full rewrite, and use `read_file(include_metadata=true)` plus `expected_sha256` when preserving the previously read version matters.
|
||||
- When implementation depends on a Python or Node.js API, first identify the installed or locked dependency version from environment metadata, requirements, package manifests, lockfiles, local source, and type declarations. Use `rg` against the relevant package directory, `.venv`, or `node_modules` instead of scanning the entire project without bounds. If local evidence is insufficient, use `search_web` and then `browse_webpage` to read the matching version of the official documentation. Do not guess signatures from memory, mix examples from incompatible versions, or install a package only to inspect its API.
|
||||
- Use structured file tools for source edits because they enforce file access boundaries and conflict checks. Never use shell redirection, inline scripts, or another tool to bypass a file-tool permission denial.
|
||||
- Use `execute_command` for administrator-only multi-file diagnostics, tests, Git, service operations, SSH, or an exact command the user requested. Use `action=run` for short bounded commands. Use `action=start` for long-running or interactive commands, including SSH; then continue with `read`, `wait`, `write`, or `kill` using the returned `session_id`. Do not start a background session for a short command that can finish within `action=run`.
|
||||
@@ -82,13 +81,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
6. Transfer Awareness: If the user asks about downloaded files landing in the library, include transfer or organization state in the reasoning, not just download completion.
|
||||
7. Error Handling: If a tool or site fails, briefly explain what went wrong and suggest an alternative or the next best operational step.
|
||||
8. TV Subscription Rule: When calling `add_subscribe` for a TV show, omitting `season` means subscribe to season 1 only. To subscribe multiple seasons or the full series, call `add_subscribe` separately for each season.
|
||||
9. Music Entity Rule: A recording is one track, an album is a collection of tracks, and an artist is browse-only. Never subscribe, search torrents for, download, transfer, or library-check an artist entity.
|
||||
10. Music Identity Rule: Reuse the exact `media_source`, `media_id`, and `music_type` returned by music search/detail. Do not substitute a same-name track, album, or artist, and do not send TV-only season/episode arguments for music.
|
||||
11. Recording Rule: A recording subscription or download targets one track. Use the recording ID and organize/scrape one audio file unless the user explicitly selected a different recording.
|
||||
12. Album Rule: An album subscription or download targets the complete album, similar to a TV season pack. Use the album ID, prefer one album resource/directory, require the resource file list to cover the expected `total_tracks`, and do not mark the subscription complete or report the album as present when only some tracks exist.
|
||||
13. Music Library Rule: Before a music download or subscription, use `query_library_exists` with the exact recording/album identity when duplicate risk matters. For albums, a negative result can mean either absent or incomplete and should remain eligible for acquisition.
|
||||
14. Music Organization Rule: Organize one recording as one audio file with `music_type=recording`. Organize a complete album by passing its album directory once with `media_type=music`, `music_type=album`, and the album source-native identity; do not repeatedly identify every track as an unrelated media item.
|
||||
15. Music Scraping Rule: `scrape_metadata(media_type="music")` applies configured audio-tag, cover, and lyrics policies. For an album directory, pass the album identity when known; without one, allow per-file tag recognition. Report the actual lyrics saved/existing/missing counts returned by the tool.
|
||||
</media_rules>
|
||||
</agent_core>
|
||||
|
||||
@@ -105,7 +97,6 @@ You act as a proactive agent. Your goal is to fully resolve the user's media-rel
|
||||
- Channel-aware formatting: Follow the capability rules below for Markdown, plain text, buttons, and voice replies.
|
||||
{button_choice_spec}
|
||||
- Voice replies: {voice_reply_spec}
|
||||
{rich_message_spec}
|
||||
- If the current channel supports image sending and an image would materially help, you may use the `send_message` tool with `image_url` to send it.
|
||||
- If the current channel supports file sending and you need to return a local image or file for the user to download, use `send_local_file`.
|
||||
</communication_runtime>
|
||||
|
||||
@@ -40,13 +40,13 @@ task_types:
|
||||
- "Analyze the error message to determine the best retry strategy."
|
||||
- "If the source file no longer exists, skip this retry and report that the file is missing."
|
||||
- "Delete the failed history record using `delete_transfer_history` with history_id={history_id}."
|
||||
- "Re-identify the media using `recognize_media` with the source file path. For audio files, set media_type='music' and preserve artist/title/album context."
|
||||
- "If recognition fails, try `search_media` with keywords from the filename. For music, distinguish recording, album, and browse-only artist results."
|
||||
- "Re-transfer using `transfer_file` with the source path and exact identity fields. Reuse media_source + media_id for every media type, plus media_type + music_type for music."
|
||||
- "Re-identify the media using `recognize_media` with the source file path."
|
||||
- "If recognition fails, try `search_media` with keywords from the filename."
|
||||
- "Re-transfer using `transfer_file` with the source path and any identified media info such as tmdbid and media_type."
|
||||
- "Report the final result."
|
||||
batch_transfer_failed_retry:
|
||||
header: "[System Task - Batch Transfer Failed Retry]"
|
||||
objective: "Multiple file transfers have failed. Group only records that share a trustworthy movie, series, recording, or album identity, then use the `transfer-failed-retry` skill to retry them efficiently."
|
||||
objective: "Multiple file transfers from the same source have failed. These files likely belong to the same media. Please use the `transfer-failed-retry` skill to retry them efficiently."
|
||||
context_title: "Task context"
|
||||
context_lines:
|
||||
- "Failed transfer history record IDs: {history_ids_csv}"
|
||||
@@ -54,12 +54,12 @@ task_types:
|
||||
steps_title: "Follow these steps"
|
||||
steps:
|
||||
- "Use `query_transfer_history` with status='failed' to find all records with these IDs and understand the failure details."
|
||||
- "Group records by exact media identity and source directory before retrying. Do not assume all selected files belong to one media."
|
||||
- "If the error is about media recognition, identify each group once using `recognize_media` or `search_media`, then reuse that result inside the group. Album tracks should normally be retried from the shared album directory with the album identity."
|
||||
- "Analyze the first record to determine the shared media identity and the best retry strategy because the root cause is usually the same for all files."
|
||||
- "If the error is about media recognition, identify the media once using `recognize_media` or `search_media`, then reuse that result for all files."
|
||||
- "For each failed record, delete the old history entry with `delete_transfer_history` and re-transfer using `transfer_file`."
|
||||
- "Report how many retries succeeded and how many still failed."
|
||||
task_rules:
|
||||
- "Within one verified group, do NOT call `recognize_media` or `search_media` repeatedly for each file. A music recording is one track; a music album is one multi-track directory; an artist is never a transfer target."
|
||||
- "These files share the same media identity. Do NOT call `recognize_media` or `search_media` repeatedly for each file."
|
||||
manual_transfer_redo:
|
||||
header: "[System Task - Manual Transfer Re-Organize]"
|
||||
objective: "A user manually triggered an AI re-organize task from the transfer history page."
|
||||
@@ -77,10 +77,12 @@ task_types:
|
||||
- "- Destination path: {destination_path}"
|
||||
- "- Destination storage: {destination_storage}"
|
||||
- "- Transfer mode: {transfer_mode}"
|
||||
- "- Current TMDB ID: {tmdbid}"
|
||||
- "- Current Douban ID: {doubanid}"
|
||||
- "- Current Bangumi ID: {bangumiid}"
|
||||
- "- Current AniList ID: {anilistid}"
|
||||
- "- Current media source: {media_source}"
|
||||
- "- Current source-native ID: {media_id}"
|
||||
- "- Music entity type: {music_type}"
|
||||
- "- Expected album tracks: {total_tracks}"
|
||||
- "- Error message: {error_message}"
|
||||
steps_title: "Required workflow"
|
||||
steps:
|
||||
@@ -92,7 +94,7 @@ task_types:
|
||||
- "Only continue when you have high confidence in the target media."
|
||||
- "Before re-organizing, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file."
|
||||
- "Then use `transfer_file` to organize the source path directly."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. For an album, retry the album directory once with the album identity when the records share that directory."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, all known media IDs, media_source, media_id, and media_type."
|
||||
- "If this record is already correct and no re-organize is needed, do not perform destructive actions; simply report that no change is necessary."
|
||||
task_rules:
|
||||
- "Do NOT rely on previous chat context. Work only from the record above."
|
||||
@@ -114,11 +116,11 @@ task_types:
|
||||
- "Review the selected records below first and group them by likely shared media identity, source directory, or retry strategy when possible."
|
||||
- "Use the provided record context as the primary source of truth. Call `query_transfer_history` only when you need extra confirmation."
|
||||
- "For each group, decide whether the current recognition is trustworthy."
|
||||
- "If multiple records clearly belong to the same movie, series, or music album, identify the media once with `recognize_media` or `search_media`, then reuse that result for the related records. A recording remains a single-track target, and an artist is browse-only."
|
||||
- "If multiple records clearly belong to the same movie or series, identify the media once with `recognize_media` or `search_media`, then reuse that result for the related records."
|
||||
- "If a source file no longer exists or cannot be safely processed, skip that record and note the reason."
|
||||
- "Before re-organizing a record, delete the old transfer history record with `delete_transfer_history` so the system will not skip the source file."
|
||||
- "Then use `transfer_file` to organize the source path directly."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, media_source, media_id, media_type, and music_type. Prefer one directory transfer for a verified complete album instead of treating each track as an unrelated media item."
|
||||
- "When calling `transfer_file`, reuse known context when appropriate: source storage, target path, target storage, transfer mode, season, all known media IDs, media_source, media_id, and media_type."
|
||||
- "If a record is already correct and no re-organize is needed, do not perform destructive actions; simply mark it as skipped."
|
||||
- "Report only the aggregate outcome, including how many records succeeded, skipped, and failed."
|
||||
task_rules:
|
||||
|
||||
@@ -10,14 +10,15 @@ from typing import Any, Dict, Optional
|
||||
import yaml
|
||||
|
||||
from app.agent.llm.capability import AgentCapabilityManager
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.notification import ChannelCapability
|
||||
from app.schemas.notification import ChannelCapabilities
|
||||
from app.schemas.notification import NotificationChannel
|
||||
from app.schemas.notification import ChannelCapabilityManager
|
||||
from app.adapters.system.host import SystemUtils
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
from app.schemas import (
|
||||
ChannelCapability,
|
||||
ChannelCapabilities,
|
||||
MessageChannel,
|
||||
ChannelCapabilityManager,
|
||||
)
|
||||
from app.utils.system import SystemUtils
|
||||
|
||||
SYSTEM_TASKS_FILE = "System Tasks.yaml"
|
||||
SYSTEM_TASKS_SCHEMA_VERSION = 2
|
||||
@@ -127,7 +128,7 @@ class PromptManager:
|
||||
markdown_spec = ""
|
||||
msg_channel = (
|
||||
next(
|
||||
(c for c in NotificationChannel if c.value.lower() == channel.lower()), None
|
||||
(c for c in MessageChannel if c.value.lower() == channel.lower()), None
|
||||
)
|
||||
if channel
|
||||
else None
|
||||
@@ -138,7 +139,6 @@ class PromptManager:
|
||||
if caps:
|
||||
markdown_spec = self._generate_formatting_instructions(caps)
|
||||
button_choice_spec = self._generate_button_choice_instructions(msg_channel)
|
||||
rich_message_spec = self._generate_rich_message_instructions(msg_channel)
|
||||
|
||||
# MoviePilot系统信息
|
||||
moviepilot_info = self._get_moviepilot_info()
|
||||
@@ -150,7 +150,6 @@ class PromptManager:
|
||||
moviepilot_info=moviepilot_info,
|
||||
voice_reply_spec=voice_reply_spec,
|
||||
button_choice_spec=button_choice_spec,
|
||||
rich_message_spec=rich_message_spec,
|
||||
)
|
||||
|
||||
return base_prompt
|
||||
@@ -300,9 +299,9 @@ class PromptManager:
|
||||
def _get_runtime_path_lines() -> list[str]:
|
||||
"""返回基础系统提示词需要常驻注入的全局运行路径。"""
|
||||
paths = {
|
||||
"项目根目录": get_runtime_setting('ROOT_PATH'),
|
||||
"配置目录": get_runtime_setting('CONFIG_PATH'),
|
||||
"临时目录": get_runtime_setting('TEMP_PATH'),
|
||||
"项目根目录": settings.ROOT_PATH,
|
||||
"配置目录": settings.CONFIG_PATH,
|
||||
"临时目录": settings.TEMP_PATH,
|
||||
}
|
||||
return [f" - {label}: `{path}`" for label, path in paths.items()]
|
||||
|
||||
@@ -355,25 +354,9 @@ class PromptManager:
|
||||
"content as a text fallback and still completes the reply."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _generate_rich_message_instructions(
|
||||
channel: NotificationChannel = None,
|
||||
) -> str:
|
||||
"""根据渠道生成 Telegram Rich Message 回复提示。"""
|
||||
if channel != NotificationChannel.Telegram:
|
||||
return ""
|
||||
return (
|
||||
"- Telegram final replies: Prefer the `send_message` tool with its "
|
||||
"`rich_message` argument. Put the complete reply in that argument using "
|
||||
"GitHub-style Markdown; headings, lists, tables, blockquotes, code blocks, "
|
||||
"and links are converted to Telegram Rich Message blocks. Do not also set "
|
||||
"`message`, `title`, or `image_url` for the same reply. Use a normal plain "
|
||||
"reply only when the response is very short and has no useful structure."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _generate_button_choice_instructions(
|
||||
channel: NotificationChannel = None,
|
||||
channel: MessageChannel = None,
|
||||
) -> str:
|
||||
if (
|
||||
channel
|
||||
|
||||
@@ -17,7 +17,6 @@ def build_manual_redo_template_context(history: Any) -> dict[str, int | str]:
|
||||
source_storage = history.dest_storage or "local"
|
||||
source_path = source_path or history.src or ""
|
||||
season_episode = f"{history.seasons or ''}{history.episodes or ''}".strip()
|
||||
is_music = str(history.type or "") in {"music", "音乐"}
|
||||
return {
|
||||
"history_id": history.id,
|
||||
"current_status": "success" if history.status else "failed",
|
||||
@@ -31,14 +30,12 @@ def build_manual_redo_template_context(history: Any) -> dict[str, int | str]:
|
||||
"destination_path": history.dest or "unknown",
|
||||
"destination_storage": history.dest_storage or "unknown",
|
||||
"transfer_mode": history.mode or "unknown",
|
||||
"tmdbid": history.tmdbid or "none",
|
||||
"doubanid": history.doubanid or "none",
|
||||
"bangumiid": history.bangumiid or "none",
|
||||
"anilistid": history.anilistid or "none",
|
||||
"media_source": history.media_source or "none",
|
||||
"media_id": history.media_id or "none",
|
||||
"music_type": getattr(history, "music_type", None) or (
|
||||
"unknown" if is_music else "not_applicable"
|
||||
),
|
||||
"total_tracks": getattr(history, "total_tracks", None) or (
|
||||
"unknown" if is_music else "not_applicable"
|
||||
),
|
||||
"error_message": history.errmsg or "none",
|
||||
}
|
||||
|
||||
@@ -60,10 +57,12 @@ def format_manual_redo_record_context(history: Any) -> str:
|
||||
f"- Destination path: {context['destination_path']}",
|
||||
f"- Destination storage: {context['destination_storage']}",
|
||||
f"- Transfer mode: {context['transfer_mode']}",
|
||||
f"- Current TMDB ID: {context['tmdbid']}",
|
||||
f"- Current Douban ID: {context['doubanid']}",
|
||||
f"- Current Bangumi ID: {context['bangumiid']}",
|
||||
f"- Current AniList ID: {context['anilistid']}",
|
||||
f"- Current media source: {context['media_source']}",
|
||||
f"- Current source-native ID: {context['media_id']}",
|
||||
f"- Music entity type: {context['music_type']}",
|
||||
f"- Expected album tracks: {context['total_tracks']}",
|
||||
f"- Error message: {context['error_message']}",
|
||||
]
|
||||
)
|
||||
|
||||
+3
-14
@@ -3,7 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import importlib
|
||||
import shutil
|
||||
import threading
|
||||
import time
|
||||
@@ -13,8 +12,8 @@ from typing import Any, Iterable, Optional
|
||||
|
||||
import yaml
|
||||
|
||||
from app.application.configuration import get_runtime_settings
|
||||
from app.runtime.log import logger
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
CURRENT_PERSONA_FILE = "CURRENT_PERSONA.md"
|
||||
SYSTEM_RUNTIME_DIR = "runtime"
|
||||
@@ -32,16 +31,6 @@ SUBAGENT_SCHEMA_VERSION = 1
|
||||
DEFAULT_PERSONA_ID = "default"
|
||||
PERSONA_ID_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$")
|
||||
|
||||
|
||||
def _default_agent_root_dir() -> Path:
|
||||
"""从组合根设置服务取得 Agent 目录，导入早期保留旧设置回退。"""
|
||||
try:
|
||||
config_path = get_runtime_settings().get("CONFIG_PATH")
|
||||
except RuntimeError:
|
||||
legacy_settings = importlib.import_module("app.runtime.config").settings
|
||||
config_path = legacy_settings.CONFIG_PATH
|
||||
return Path(config_path) / "agent"
|
||||
|
||||
ROOT_LEVEL_RUNTIME_FILES = {
|
||||
CURRENT_PERSONA_FILE,
|
||||
}
|
||||
@@ -242,7 +231,7 @@ class AgentRuntimeManager:
|
||||
agent_root_dir: Optional[Path] = None,
|
||||
bundled_defaults_dir: Optional[Path] = None,
|
||||
) -> None:
|
||||
self.agent_root_dir = agent_root_dir or _default_agent_root_dir()
|
||||
self.agent_root_dir = agent_root_dir or (settings.CONFIG_PATH / "agent")
|
||||
self.runtime_dir = self.agent_root_dir / SYSTEM_RUNTIME_DIR
|
||||
self.memory_dir = self.agent_root_dir / MEMORY_DIR
|
||||
self.skills_dir = self.agent_root_dir / SKILLS_DIR
|
||||
|
||||
@@ -1,150 +0,0 @@
|
||||
"""Agent 重量级 canonical 对象的轻量首用入口。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import threading
|
||||
from typing import Any
|
||||
|
||||
from app.agent.capabilities import (
|
||||
AGENT_ENTRYPOINT_KIND,
|
||||
AGENT_MANAGER_CAPABILITY_ID,
|
||||
AGENT_SERVICE_CAPABILITY_ID,
|
||||
AGENT_SERVICE_KIND,
|
||||
MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID,
|
||||
TOOL_FACTORY_CAPABILITY_ID,
|
||||
)
|
||||
from app.agent.capabilities.adapter import (
|
||||
AgentEntrypointAdapter,
|
||||
AgentServiceAdapter,
|
||||
build_agent_capability_registry,
|
||||
should_run_agent_service,
|
||||
)
|
||||
from app.runtime.capabilities.model import CapabilityMaterializationState
|
||||
from app.runtime.capabilities.runtime import CapabilityRuntime
|
||||
|
||||
|
||||
_runtime_lock = threading.RLock()
|
||||
_agent_runtime: CapabilityRuntime | None = None
|
||||
|
||||
|
||||
def _build_agent_runtime() -> CapabilityRuntime:
|
||||
"""装配 Agent Runtime；构建阶段只解析 manifests。"""
|
||||
return CapabilityRuntime(
|
||||
build_agent_capability_registry(),
|
||||
adapters={
|
||||
AGENT_ENTRYPOINT_KIND: AgentEntrypointAdapter(),
|
||||
AGENT_SERVICE_KIND: AgentServiceAdapter(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _ensure_runtime() -> CapabilityRuntime:
|
||||
"""返回进程唯一 Runtime，同进程关闭后不重新创建。"""
|
||||
global _agent_runtime
|
||||
with _runtime_lock:
|
||||
if _agent_runtime is None:
|
||||
_agent_runtime = _build_agent_runtime()
|
||||
return _agent_runtime
|
||||
|
||||
|
||||
def _materialize_entrypoint(capability_id: str) -> Any:
|
||||
"""通过通用 Runtime 完成并发 single-flight 物化，不声明资源运行态。"""
|
||||
return _ensure_runtime().materialize(
|
||||
capability_id,
|
||||
reason="agent_entrypoint_first_use",
|
||||
)
|
||||
|
||||
|
||||
def get_agent_manager() -> Any:
|
||||
"""返回 canonical Agent Manager；关闭门禁生效后稳定拒绝首用。"""
|
||||
return _materialize_entrypoint(AGENT_MANAGER_CAPABILITY_ID)
|
||||
|
||||
|
||||
async def reconcile_agent_service(
|
||||
*,
|
||||
reason: str,
|
||||
changed_keys: set[str] | None = None,
|
||||
retry: bool = False,
|
||||
) -> Any | None:
|
||||
"""按 manifest watch/selector 协调唯一 Agent Service 生命周期。"""
|
||||
runtime = _ensure_runtime()
|
||||
spec = runtime.get_spec(AGENT_SERVICE_CAPABILITY_ID)
|
||||
if spec is None:
|
||||
raise RuntimeError("缺少 agent.service capability")
|
||||
if changed_keys is not None and not changed_keys.intersection(spec.watch):
|
||||
return runtime.get_running(AGENT_SERVICE_CAPABILITY_ID)
|
||||
if not should_run_agent_service(spec):
|
||||
# stop_async 会等待并发首启后再撤销实例；未物化能力则保持零导入。
|
||||
await runtime.stop_async(
|
||||
AGENT_SERVICE_CAPABILITY_ID,
|
||||
reason=reason,
|
||||
)
|
||||
return None
|
||||
return await runtime.activate_async(
|
||||
AGENT_SERVICE_CAPABILITY_ID,
|
||||
reason=reason,
|
||||
retry=retry,
|
||||
)
|
||||
|
||||
|
||||
async def activate_agent_service(*, retry: bool = False) -> Any | None:
|
||||
"""执行启动期协调；selector 未启用时保持 service 未物化。"""
|
||||
return await reconcile_agent_service(
|
||||
reason="agent_service_startup_reconcile",
|
||||
retry=retry,
|
||||
)
|
||||
|
||||
|
||||
def get_running_agent_manager() -> Any | None:
|
||||
"""只读返回 RUNNING Agent Service；未构建 Runtime 时不触发声明发现。"""
|
||||
with _runtime_lock:
|
||||
runtime = _agent_runtime
|
||||
if runtime is None:
|
||||
return None
|
||||
return runtime.get_running(AGENT_SERVICE_CAPABILITY_ID)
|
||||
|
||||
|
||||
def get_moviepilot_agent_type() -> type:
|
||||
"""返回 canonical MoviePilotAgent 类型。"""
|
||||
agent_type = _materialize_entrypoint(MOVIEPILOT_AGENT_TYPE_CAPABILITY_ID)
|
||||
if not isinstance(agent_type, type):
|
||||
raise TypeError("MoviePilot Agent entrypoint 必须是类型")
|
||||
return agent_type
|
||||
|
||||
|
||||
def get_tool_factory() -> type:
|
||||
"""返回 canonical 工具工厂类型。"""
|
||||
factory_type = _materialize_entrypoint(TOOL_FACTORY_CAPABILITY_ID)
|
||||
if not isinstance(factory_type, type):
|
||||
raise TypeError("Agent Tool Factory entrypoint 必须是类型")
|
||||
return factory_type
|
||||
|
||||
|
||||
def is_tool_factory_materialized() -> bool:
|
||||
"""只读判断工具工厂是否已解析；未建 Runtime 时不触发发现或导入。"""
|
||||
with _runtime_lock:
|
||||
runtime = _agent_runtime
|
||||
if runtime is None:
|
||||
return False
|
||||
return (
|
||||
runtime.snapshot(TOOL_FACTORY_CAPABILITY_ID).materialization
|
||||
is CapabilityMaterializationState.RESOLVED
|
||||
)
|
||||
|
||||
|
||||
async def close_materialized_terminal_sessions() -> None:
|
||||
"""关闭已物化的终端会话管理器，不触发新的 Agent 工具导入。"""
|
||||
module = sys.modules.get("app.agent.tools.impl._terminal_session")
|
||||
manager = getattr(module, "terminal_session_manager", None) if module else None
|
||||
close = getattr(manager, "close", None)
|
||||
if callable(close):
|
||||
await close()
|
||||
|
||||
|
||||
async def begin_agent_shutdown() -> bool:
|
||||
"""不可逆关闭首用闸门，并返回全部 Agent 能力是否真实收敛。"""
|
||||
runtime = _ensure_runtime()
|
||||
return await runtime.shutdown_async(
|
||||
reason="application_shutdown",
|
||||
)
|
||||
@@ -1 +0,0 @@
|
||||
"""Agent Skill 元数据、市场和本地生命周期能力。"""
|
||||
@@ -1,132 +0,0 @@
|
||||
import logging
|
||||
import re
|
||||
from typing import TypedDict
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 磁盘读取上限属于 Skill 文档格式约束，市场扫描和 Agent 加载必须共用。
|
||||
MAX_SKILL_FILE_SIZE = 1 * 1024 * 1024
|
||||
MAX_SKILL_NAME_LENGTH = 64
|
||||
MAX_SKILL_DESCRIPTION_LENGTH = 1024
|
||||
MAX_SKILL_COMPATIBILITY_LENGTH = 500
|
||||
|
||||
|
||||
class SkillMetadata(TypedDict):
|
||||
"""描述符合 Agent Skills 规范的已校验元数据。"""
|
||||
|
||||
path: str
|
||||
id: str
|
||||
name: str
|
||||
version: int
|
||||
description: str
|
||||
license: str | None
|
||||
compatibility: str | None
|
||||
metadata: dict[str, str]
|
||||
allowed_tools: list[str]
|
||||
|
||||
|
||||
def _validate_metadata(raw: object, skill_path: str) -> dict[str, str]:
|
||||
"""将 YAML metadata 字段规范化为字符串键值映射。"""
|
||||
if not isinstance(raw, dict):
|
||||
if raw:
|
||||
logger.warning(
|
||||
"Ignoring non-dict metadata in %s (got %s)",
|
||||
skill_path,
|
||||
type(raw).__name__,
|
||||
)
|
||||
return {}
|
||||
return {str(key): str(value) for key, value in raw.items()}
|
||||
|
||||
|
||||
def parse_skill_metadata( # noqa: C901
|
||||
content: str,
|
||||
skill_path: str,
|
||||
skill_id: str,
|
||||
) -> SkillMetadata | None:
|
||||
"""解析并校验一个 SKILL.md 的 YAML 前言。"""
|
||||
if len(content) > MAX_SKILL_FILE_SIZE:
|
||||
logger.warning(
|
||||
"Skipping %s: content too large (%d bytes)", skill_path, len(content)
|
||||
)
|
||||
return None
|
||||
|
||||
match = re.match(r"^---\s*\n(.*?)\n---\s*\n", content, re.DOTALL)
|
||||
if not match:
|
||||
logger.warning("Skipping %s: no valid YAML frontmatter found", skill_path)
|
||||
return None
|
||||
|
||||
try:
|
||||
frontmatter_data = yaml.safe_load(match.group(1))
|
||||
except yaml.YAMLError as err:
|
||||
logger.warning("Invalid YAML in %s: %s", skill_path, err)
|
||||
return None
|
||||
if not isinstance(frontmatter_data, dict):
|
||||
logger.warning("Skipping %s: frontmatter is not a mapping", skill_path)
|
||||
return None
|
||||
|
||||
name = str(frontmatter_data.get("name", "")).strip()
|
||||
description = str(frontmatter_data.get("description", "")).strip()
|
||||
if not name or not description:
|
||||
logger.warning(
|
||||
"Skipping %s: missing required 'name' or 'description'", skill_path
|
||||
)
|
||||
return None
|
||||
if len(description) > MAX_SKILL_DESCRIPTION_LENGTH:
|
||||
logger.warning(
|
||||
"Description exceeds %d characters in %s, truncating",
|
||||
MAX_SKILL_DESCRIPTION_LENGTH,
|
||||
skill_path,
|
||||
)
|
||||
description = description[:MAX_SKILL_DESCRIPTION_LENGTH]
|
||||
|
||||
raw_tools = frontmatter_data.get("allowed-tools")
|
||||
if isinstance(raw_tools, str):
|
||||
allowed_tools = [
|
||||
tool.strip(",")
|
||||
for tool in raw_tools.split()
|
||||
if tool.strip(",")
|
||||
]
|
||||
else:
|
||||
if raw_tools is not None:
|
||||
logger.warning(
|
||||
"Ignoring non-string 'allowed-tools' in %s (got %s)",
|
||||
skill_path,
|
||||
type(raw_tools).__name__,
|
||||
)
|
||||
allowed_tools = []
|
||||
|
||||
compatibility = str(frontmatter_data.get("compatibility", "")).strip() or None
|
||||
if compatibility and len(compatibility) > MAX_SKILL_COMPATIBILITY_LENGTH:
|
||||
logger.warning(
|
||||
"Compatibility exceeds %d characters in %s, truncating",
|
||||
MAX_SKILL_COMPATIBILITY_LENGTH,
|
||||
skill_path,
|
||||
)
|
||||
compatibility = compatibility[:MAX_SKILL_COMPATIBILITY_LENGTH]
|
||||
|
||||
raw_version = frontmatter_data.get("version")
|
||||
version = 0
|
||||
if raw_version is not None:
|
||||
try:
|
||||
version = int(raw_version)
|
||||
except (ValueError, TypeError):
|
||||
logger.warning(
|
||||
"Invalid 'version' in %s (got %r), defaulting to 0",
|
||||
skill_path,
|
||||
raw_version,
|
||||
)
|
||||
|
||||
return SkillMetadata(
|
||||
id=skill_id,
|
||||
name=name,
|
||||
version=version,
|
||||
description=description,
|
||||
path=skill_path,
|
||||
metadata=_validate_metadata(frontmatter_data.get("metadata", {}), skill_path),
|
||||
license=str(frontmatter_data.get("license", "")).strip() or None,
|
||||
compatibility=compatibility,
|
||||
allowed_tools=allowed_tools,
|
||||
)
|
||||
+152
-273
@@ -1,79 +1,24 @@
|
||||
import asyncio
|
||||
import inspect
|
||||
import json
|
||||
import threading
|
||||
from abc import ABCMeta, abstractmethod
|
||||
from concurrent.futures import Future as ConcurrentFuture, ThreadPoolExecutor
|
||||
from contextvars import Context, copy_context
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Protocol
|
||||
from typing import Any, Callable, ClassVar, Optional
|
||||
|
||||
from langchain_core.tools import BaseTool
|
||||
from pydantic import PrivateAttr
|
||||
|
||||
from app.agent.policy.sanitizer import (
|
||||
summarize_error,
|
||||
summarize_input,
|
||||
summarize_result,
|
||||
)
|
||||
from app.agent import StreamingHandler
|
||||
from app.agent.tools.tags import ToolTag
|
||||
from app.chain import ChainBase
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
|
||||
from app.application.messaging.agent import matches_channel_admin
|
||||
from app.application.notification import get_notification_configs
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.message import Message
|
||||
from app.schemas.types import NotificationChannel, MessageType
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.agent.callback import StreamingHandler as _StreamingHandlerProtocol
|
||||
else:
|
||||
class _StreamingHandlerProtocol(Protocol):
|
||||
"""工具执行仅依赖的流式缓冲合同。"""
|
||||
|
||||
@property
|
||||
def is_streaming(self) -> bool:
|
||||
"""是否正在收集流式输出。"""
|
||||
...
|
||||
|
||||
@property
|
||||
def is_auto_flushing(self) -> bool:
|
||||
"""是否由渠道编辑能力自动刷新缓冲。"""
|
||||
...
|
||||
|
||||
@property
|
||||
def last_buffer_char(self) -> str:
|
||||
"""返回缓冲区最后一个字符。"""
|
||||
...
|
||||
|
||||
def emit(self, token: str) -> str:
|
||||
"""追加流式文本并返回实际追加内容。"""
|
||||
...
|
||||
|
||||
async def take(self) -> str:
|
||||
"""取出并清空当前缓冲内容。"""
|
||||
...
|
||||
|
||||
def record_tool_call(
|
||||
self,
|
||||
tool_name: str,
|
||||
tool_message: Optional[str] = None,
|
||||
tool_kwargs: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
"""记录一次待汇总的工具调用。"""
|
||||
...
|
||||
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""显式访问历史 StreamingHandler 符号时返回 canonical 实现。"""
|
||||
if name == "StreamingHandler":
|
||||
from app.agent.callback import StreamingHandler
|
||||
|
||||
return StreamingHandler
|
||||
raise AttributeError(f"module 'app.agent.tools.base' has no attribute {name!r}")
|
||||
from app.core.config import settings
|
||||
from app.db.user_oper import UserOper
|
||||
from app.helper.service import ServiceConfigHelper
|
||||
from app.log import logger
|
||||
from app.schemas import Notification
|
||||
from app.schemas.types import MessageChannel, NotificationType
|
||||
|
||||
|
||||
class ToolChain(ChainBase):
|
||||
@@ -94,9 +39,7 @@ def serialize_tool_result_for_agent(result: Any) -> str:
|
||||
try:
|
||||
return json.dumps(result, ensure_ascii=False, indent=2, default=str)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"工具结果转换为JSON失败: {summarize_error(e)}, 使用字符串表示"
|
||||
)
|
||||
logger.warning(f"工具结果转换为JSON失败: {e}, 使用字符串表示")
|
||||
return str(result)
|
||||
|
||||
|
||||
@@ -167,130 +110,33 @@ _blocking_semaphores = {
|
||||
for bucket, limit in _BLOCKING_BUCKET_LIMITS.items()
|
||||
}
|
||||
_blocking_executors: dict[str, ThreadPoolExecutor] = {}
|
||||
_blocking_retiring_executors: set[ThreadPoolExecutor] = set()
|
||||
_blocking_futures: dict[ConcurrentFuture[Any], ThreadPoolExecutor] = {}
|
||||
_blocking_executor_lock = threading.RLock()
|
||||
_blocking_executor_accepting = True
|
||||
_blocking_executor_lock = threading.Lock()
|
||||
|
||||
|
||||
def _discard_blocking_future(future: ConcurrentFuture[Any]) -> None:
|
||||
"""在同步调用到达终态后撤销 Future 与 retiring executor owner。"""
|
||||
def _get_blocking_executor(bucket: str) -> ThreadPoolExecutor:
|
||||
"""按桶懒加载线程池，避免在导入阶段创建过多 worker。"""
|
||||
with _blocking_executor_lock:
|
||||
executor = _blocking_futures.pop(future, None)
|
||||
if executor is None or executor not in _blocking_retiring_executors:
|
||||
return
|
||||
if executor not in _blocking_futures.values():
|
||||
_blocking_retiring_executors.discard(executor)
|
||||
|
||||
|
||||
def _submit_blocking_call(
|
||||
bucket: str,
|
||||
bound_call: Callable[[], Any],
|
||||
) -> ConcurrentFuture[Any]:
|
||||
"""在提交门禁内原子取得 executor、提交调用并登记 Future owner。"""
|
||||
context = copy_context()
|
||||
with _blocking_executor_lock:
|
||||
if not _blocking_executor_accepting:
|
||||
raise RuntimeError("Agent 工具阻塞执行器正在关闭，不能再提交新任务")
|
||||
executor = _blocking_executors.get(bucket)
|
||||
if executor is None:
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
# 长期 worker 保持空底层上下文，每个任务只在自己的调用快照内运行。
|
||||
future = Context().run(executor.submit, context.run, bound_call)
|
||||
_blocking_futures[future] = executor
|
||||
future.add_done_callback(_discard_blocking_future)
|
||||
return future
|
||||
if executor:
|
||||
return executor
|
||||
|
||||
limit = _BLOCKING_BUCKET_LIMITS[bucket]
|
||||
executor = ThreadPoolExecutor(
|
||||
max_workers=limit,
|
||||
thread_name_prefix=f"agent-tool-{bucket}",
|
||||
)
|
||||
_blocking_executors[bucket] = executor
|
||||
return executor
|
||||
|
||||
|
||||
def _retire_blocking_executors(*, cancel_futures: bool) -> tuple[ThreadPoolExecutor, ...]:
|
||||
"""撤销活动 executor 的提交资格，并保留其运行 Future 对应的 owner。"""
|
||||
def shutdown_blocking_executors(*, wait: bool = True, cancel_futures: bool = False) -> None:
|
||||
"""关闭 Agent 工具阻塞线程池，释放长期运行进程或测试环境中的 worker。"""
|
||||
with _blocking_executor_lock:
|
||||
executors = tuple(_blocking_executors.values())
|
||||
executors = list(_blocking_executors.values())
|
||||
_blocking_executors.clear()
|
||||
_blocking_retiring_executors.update(executors)
|
||||
|
||||
for executor in executors:
|
||||
executor.shutdown(wait=False, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
owned_executors = set(_blocking_futures.values())
|
||||
_blocking_retiring_executors.intersection_update(owned_executors)
|
||||
return executors
|
||||
|
||||
|
||||
def begin_blocking_executor_shutdown(*, cancel_futures: bool = True) -> None:
|
||||
"""原子封住新阻塞工具提交，并请求取消尚未开始的同步调用。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
_blocking_executor_accepting = False
|
||||
_retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
|
||||
|
||||
def reopen_blocking_executors() -> bool:
|
||||
"""仅在旧 Future 和 executor 全部收敛后重新开放测试生命周期。"""
|
||||
global _blocking_executor_accepting
|
||||
with _blocking_executor_lock:
|
||||
if _blocking_futures or _blocking_retiring_executors:
|
||||
return False
|
||||
_blocking_executor_accepting = True
|
||||
return True
|
||||
|
||||
|
||||
async def close_blocking_executors(
|
||||
*,
|
||||
timeout_seconds: float,
|
||||
cancel_futures: bool = True,
|
||||
) -> bool:
|
||||
"""有限等待全部阻塞工具 Future，超时保留 Future 与 executor owner。"""
|
||||
begin_blocking_executor_shutdown(cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
futures = tuple(_blocking_futures)
|
||||
wrapped_futures = tuple(asyncio.wrap_future(future) for future in futures)
|
||||
if wrapped_futures:
|
||||
done, pending = await asyncio.wait(
|
||||
wrapped_futures,
|
||||
timeout=max(0.0, timeout_seconds),
|
||||
)
|
||||
if done:
|
||||
await asyncio.gather(*done, return_exceptions=True)
|
||||
for pending_future in pending:
|
||||
pending_future.add_done_callback(
|
||||
lambda completed: completed.exception()
|
||||
if not completed.cancelled()
|
||||
else None
|
||||
)
|
||||
|
||||
with _blocking_executor_lock:
|
||||
unfinished = tuple(
|
||||
future for future in _blocking_futures if not future.done()
|
||||
)
|
||||
retiring_count = len(_blocking_retiring_executors)
|
||||
if unfinished:
|
||||
logger.error(
|
||||
"Agent 阻塞工具未在 %.1f 秒内收敛：futures=%d，executors=%d",
|
||||
max(0.0, timeout_seconds),
|
||||
len(unfinished),
|
||||
retiring_count,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def shutdown_blocking_executors(
|
||||
*,
|
||||
wait: bool = True,
|
||||
cancel_futures: bool = False,
|
||||
) -> bool:
|
||||
"""同步清理测试 owner；非等待模式下保留尚未收敛的 executor 句柄。"""
|
||||
executors = _retire_blocking_executors(cancel_futures=cancel_futures)
|
||||
for executor in executors:
|
||||
if wait:
|
||||
executor.shutdown(wait=True, cancel_futures=cancel_futures)
|
||||
with _blocking_executor_lock:
|
||||
return not _blocking_futures and not _blocking_retiring_executors
|
||||
executor.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
|
||||
|
||||
class ToolExecutionTimeoutError(TimeoutError):
|
||||
@@ -300,7 +146,7 @@ class ToolExecutionTimeoutError(TimeoutError):
|
||||
def _get_tool_timeout_seconds() -> Optional[float]:
|
||||
"""读取工具执行超时时间，配置为 0 或负数时表示不限制。"""
|
||||
try:
|
||||
timeout = float(get_runtime_setting('LLM_TOOL_TIMEOUT') or 0)
|
||||
timeout = float(settings.LLM_TOOL_TIMEOUT or 0)
|
||||
except (TypeError, ValueError):
|
||||
timeout = 0
|
||||
return timeout if timeout > 0 else None
|
||||
@@ -322,7 +168,7 @@ async def run_agent_blocking(
|
||||
|
||||
await semaphore.acquire()
|
||||
try:
|
||||
future = _submit_blocking_call(bucket_name, bound_call)
|
||||
future = _get_blocking_executor(bucket_name).submit(bound_call)
|
||||
except Exception:
|
||||
semaphore.release()
|
||||
raise
|
||||
@@ -353,7 +199,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
_channel: Optional[str] = PrivateAttr(default=None)
|
||||
_source: Optional[str] = PrivateAttr(default=None)
|
||||
_username: Optional[str] = PrivateAttr(default=None)
|
||||
_stream_handler: Optional[_StreamingHandlerProtocol] = PrivateAttr(default=None)
|
||||
_stream_handler: Optional[StreamingHandler] = PrivateAttr(default=None)
|
||||
_require_admin: bool = PrivateAttr(default=False)
|
||||
_agent_context: dict = PrivateAttr(default_factory=dict)
|
||||
|
||||
@@ -402,10 +248,6 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
|
||||
permission_result = await self._check_permission()
|
||||
if permission_result:
|
||||
# 工具被权限门禁拦截时，模型在调用前可能已输出一段引导文本，且这里
|
||||
# 不会产生工具消息或统计摘要；补一个换行分隔符，避免随后的失败说明
|
||||
# 与引导文本直接连在一起。
|
||||
self._ensure_tool_boundary_separator()
|
||||
return permission_result
|
||||
|
||||
# 获取工具执行提示消息
|
||||
@@ -413,7 +255,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
|
||||
# 发送工具执行过程消息（流式传输且非最后终结工具时）
|
||||
if self._stream_handler and self._stream_handler.is_streaming and not self.return_direct:
|
||||
if get_runtime_setting('AI_AGENT_VERBOSE'):
|
||||
if settings.AI_AGENT_VERBOSE:
|
||||
if self._stream_handler.is_auto_flushing:
|
||||
# 渠道支持编辑：工具消息追加到 buffer，由定时刷新推送
|
||||
if tool_message:
|
||||
@@ -461,26 +303,27 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
# 未启用流式传输，不发送任何工具消息内容
|
||||
pass
|
||||
|
||||
logger.debug(
|
||||
f"Executing tool {self.name} with input summary: {summarize_input(kwargs)}"
|
||||
)
|
||||
logger.debug(f"Executing tool {self.name} with args: {kwargs}")
|
||||
|
||||
# 执行具体工具逻辑
|
||||
try:
|
||||
result = await self.run_with_timeout(**kwargs)
|
||||
|
||||
logger.info(
|
||||
f"Agent工具 {self.name} 执行完成，"
|
||||
f"结果摘要: {summarize_result(result)}"
|
||||
)
|
||||
# 记录工具执行结果摘要日志
|
||||
str_result = serialize_tool_result_for_agent(result)
|
||||
if len(str_result) > 500:
|
||||
summary = str_result[:500] + f"...(已截断，总长度: {len(str_result)})"
|
||||
else:
|
||||
summary = str_result
|
||||
logger.info(f"Agent工具 {self.name} 执行完成，结果摘要: {summary}")
|
||||
|
||||
except ToolExecutionTimeoutError as e:
|
||||
error_message = summarize_error(e)
|
||||
error_message = str(e)
|
||||
logger.warning(error_message)
|
||||
raise
|
||||
result = error_message
|
||||
except Exception as e:
|
||||
error_message = f"工具执行异常: {summarize_error(e)}"
|
||||
logger.error(f"Tool {self.name} execution failed: {summarize_error(e)}")
|
||||
error_message = f"工具执行异常 ({type(e).__name__}): {str(e)}"
|
||||
logger.error(f"Tool {self.name} execution failed: {e}", exc_info=True)
|
||||
result = error_message
|
||||
|
||||
return format_tool_result_for_agent(
|
||||
@@ -515,7 +358,6 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
except asyncio.TimeoutError as err:
|
||||
raise ToolExecutionTimeoutError(
|
||||
f"工具 {self.name} 执行超时（超过 {timeout:g} 秒），已停止等待结果。"
|
||||
"若工具包含外部写操作，操作可能仍在继续，请先确认实际状态再重试。"
|
||||
) from err
|
||||
|
||||
@staticmethod
|
||||
@@ -535,9 +377,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
self._source = source
|
||||
self._username = username
|
||||
|
||||
def set_stream_handler(
|
||||
self, stream_handler: Optional[_StreamingHandlerProtocol]
|
||||
) -> None:
|
||||
def set_stream_handler(self, stream_handler: StreamingHandler):
|
||||
"""
|
||||
设置回调处理器
|
||||
"""
|
||||
@@ -551,29 +391,14 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
# 独立的新 dict，跨工具状态（例如质量门槛拒绝标记）无法传播。
|
||||
self._agent_context = {} if agent_context is None else agent_context
|
||||
|
||||
def _ensure_tool_boundary_separator(self) -> None:
|
||||
"""
|
||||
在流式缓冲中为工具边界补一个换行分隔符。
|
||||
|
||||
工具被权限门禁等前置检查拦截时不产生工具消息或统计摘要，若模型在调用前
|
||||
已输出文本，后续内容会直接粘在前文后面；这里保证缓冲以换行结尾，让工具
|
||||
前后的内容分行展示。缓冲为空或已以换行结尾时无需处理。
|
||||
"""
|
||||
if (
|
||||
self._stream_handler
|
||||
and self._stream_handler.is_streaming
|
||||
and self._stream_handler.last_buffer_char not in ("", "\n")
|
||||
):
|
||||
self._stream_handler.emit("\n")
|
||||
|
||||
async def is_admin_user(self) -> bool:
|
||||
"""
|
||||
判断当前工具调用者是否拥有管理员级权限。
|
||||
|
||||
:return: 当前调用者是系统管理员、渠道管理员或显式管理员上下文时返回 True
|
||||
"""
|
||||
if "is_admin" in self._agent_context:
|
||||
return self._agent_context.get("is_admin") is True
|
||||
if bool(self._agent_context.get("is_admin")):
|
||||
return True
|
||||
|
||||
if not self._channel or not self._source:
|
||||
return False
|
||||
@@ -613,7 +438,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
:return: 普通用户允许读写的本地目录列表
|
||||
"""
|
||||
roots = [
|
||||
get_runtime_setting('CONFIG_PATH') / "agent",
|
||||
settings.CONFIG_PATH / "agent",
|
||||
]
|
||||
resolved_roots = []
|
||||
for root in roots:
|
||||
@@ -678,10 +503,12 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
|
||||
async def _check_permission(self) -> Optional[str]:
|
||||
"""
|
||||
检查管理员工具权限。
|
||||
|
||||
Agent 共享上下文中的显式管理员事实优先；没有该事实的旧调用才按渠道
|
||||
管理员名单回查，并保留无消息渠道内部调用的兼容行为。
|
||||
检查用户权限：
|
||||
1. 首先检查工具是否需要管理员权限
|
||||
2. 如果需要管理员权限，则检查用户是否是渠道管理员
|
||||
3. 如果渠道没有设置管理员名单，则检查用户是否是系统管理员
|
||||
4. 如果都不是系统管理员，检查用户ID是否等于渠道配置的用户ID
|
||||
5. 如果都不是，返回权限拒绝消息
|
||||
"""
|
||||
if not self._require_admin:
|
||||
return None
|
||||
@@ -689,9 +516,7 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
if await self.is_admin_user():
|
||||
return None
|
||||
|
||||
if "is_admin" not in self._agent_context and (
|
||||
not self._channel or not self._source
|
||||
):
|
||||
if not self._channel or not self._source:
|
||||
return None
|
||||
|
||||
return (
|
||||
@@ -705,75 +530,136 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
"""
|
||||
检查当前消息渠道身份是否具备管理员权限。
|
||||
|
||||
:return: 当前渠道稳定用户 ID 位于显式管理员名单或等于渠道主ID时返回 True
|
||||
:return: 当前渠道用户是渠道管理员、系统管理员或默认接收人时返回 True
|
||||
"""
|
||||
if not self._channel or not self._source:
|
||||
return False
|
||||
|
||||
# 渠道配置来自 SystemConfigOper 内存缓存，可以直接读取；
|
||||
# 只有用户信息需要走异步数据库查询。
|
||||
user_id_str = str(self._user_id) if self._user_id else None
|
||||
|
||||
try:
|
||||
channel = NotificationChannel(self._channel)
|
||||
except ValueError:
|
||||
channel_type_map = {
|
||||
MessageChannel.Telegram: "telegram",
|
||||
MessageChannel.Discord: "discord",
|
||||
MessageChannel.Wechat: "wechat",
|
||||
MessageChannel.Feishu: "feishu",
|
||||
MessageChannel.WechatClawBot: "wechatclawbot",
|
||||
MessageChannel.Slack: "slack",
|
||||
MessageChannel.VoceChat: "vocechat",
|
||||
MessageChannel.SynologyChat: "synologychat",
|
||||
MessageChannel.QQ: "qqbot",
|
||||
}
|
||||
|
||||
channel_type = None
|
||||
for key, value in channel_type_map.items():
|
||||
if self._channel == key.value:
|
||||
channel_type = value
|
||||
break
|
||||
|
||||
if not channel_type:
|
||||
return False
|
||||
|
||||
admin_key_map = {
|
||||
"telegram": "TELEGRAM_ADMINS",
|
||||
"discord": "DISCORD_ADMINS",
|
||||
"wechat": "WECHAT_ADMINS",
|
||||
"feishu": "FEISHU_ADMINS",
|
||||
"wechatclawbot": "WECHATCLAWBOT_ADMINS",
|
||||
"slack": "SLACK_ADMINS",
|
||||
"vocechat": "VOCECHAT_ADMINS",
|
||||
"synologychat": "SYNOLOGYCHAT_ADMINS",
|
||||
"qqbot": "QQBOT_ADMINS",
|
||||
}
|
||||
|
||||
user_id_key_map = {
|
||||
"telegram": "TELEGRAM_CHAT_ID",
|
||||
"vocechat": "VOCECHAT_CHANNEL_ID",
|
||||
"wechat": "WECHAT_BOT_CHAT_ID",
|
||||
"feishu": "FEISHU_OPEN_ID",
|
||||
"wechatclawbot": "WECHATCLAWBOT_DEFAULT_TARGET",
|
||||
"discord": "DISCORD_CHANNEL_ID",
|
||||
"slack": "SLACK_CHANNEL",
|
||||
"qqbot": "QQ_OPENID",
|
||||
}
|
||||
|
||||
admin_key = admin_key_map.get(channel_type)
|
||||
user_id_key = user_id_key_map.get(channel_type)
|
||||
|
||||
try:
|
||||
configs = get_notification_configs(include_disabled=True)
|
||||
configs = ServiceConfigHelper.get_notification_configs()
|
||||
for config in configs:
|
||||
if config.name == self._source and config.config:
|
||||
return matches_channel_admin(
|
||||
channel,
|
||||
config.config,
|
||||
user_id_str,
|
||||
)
|
||||
channel_admins = config.config.get(admin_key) if admin_key else None
|
||||
if channel_admins:
|
||||
admin_list = [
|
||||
aid.strip()
|
||||
for aid in str(channel_admins).split(",")
|
||||
if aid.strip()
|
||||
]
|
||||
if user_id_str and user_id_str in admin_list:
|
||||
return True
|
||||
|
||||
user = (
|
||||
await UserOper().async_get_by_name(self._username)
|
||||
if self._username
|
||||
else None
|
||||
)
|
||||
if user and user.is_superuser:
|
||||
return True
|
||||
|
||||
return False
|
||||
else:
|
||||
user = (
|
||||
await UserOper().async_get_by_name(self._username)
|
||||
if self._username
|
||||
else None
|
||||
)
|
||||
if user and user.is_superuser:
|
||||
return True
|
||||
|
||||
if user_id_key:
|
||||
config_user_id = config.config.get(user_id_key)
|
||||
if config_user_id and str(config_user_id) == user_id_str:
|
||||
return True
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"检查权限失败: {summarize_error(e)}")
|
||||
logger.error(f"检查权限失败: {e}")
|
||||
|
||||
return False
|
||||
|
||||
async def send_message(self, message: Message) -> None:
|
||||
async def send_notification_message(self, notification: Notification) -> None:
|
||||
"""
|
||||
发送工具消息。
|
||||
发送工具通知消息。
|
||||
|
||||
WebAgent 渠道没有后端模块实例，前端流式面板通过 Agent 上下文中的
|
||||
回调直接接收消息；无渠道的后台任务清空渠道侧定位信息后交由消息链广播，
|
||||
回调直接接收通知；无渠道的后台任务清空渠道侧定位信息后交由消息链广播，
|
||||
其它渠道继续走统一消息链。
|
||||
"""
|
||||
callback = self._agent_context.get("message_callback")
|
||||
callback = self._agent_context.get("notification_callback")
|
||||
if (
|
||||
self._channel == NotificationChannel.WebAgent.value
|
||||
self._channel == MessageChannel.WebAgent.value
|
||||
and callable(callback)
|
||||
):
|
||||
callback_result = callback(message)
|
||||
if inspect.isawaitable(callback_result):
|
||||
await callback_result
|
||||
callback(notification)
|
||||
return
|
||||
|
||||
if not self._channel or not self._source:
|
||||
message = message.model_copy(
|
||||
notification = notification.model_copy(
|
||||
update={
|
||||
"channel": None,
|
||||
"source": None,
|
||||
"userid": None,
|
||||
"username": message.username
|
||||
"username": notification.username
|
||||
or self._username
|
||||
or get_runtime_setting('SUPERUSER'),
|
||||
or settings.SUPERUSER,
|
||||
"original_message_id": None,
|
||||
"original_chat_id": None,
|
||||
}
|
||||
)
|
||||
elif not message.original_chat_id:
|
||||
# 工具回调消息默认回填当前会话的原会话 ID，
|
||||
# 保证群聊 @ 机器人时按钮选择、消息发送等交互消息回复到原群，而不是私聊窗口。
|
||||
original_chat_id = str(
|
||||
self._agent_context.get("original_chat_id") or ""
|
||||
).strip() or None
|
||||
if original_chat_id:
|
||||
message = message.model_copy(
|
||||
update={"original_chat_id": original_chat_id}
|
||||
)
|
||||
|
||||
await ToolChain().async_post_message(message)
|
||||
await ToolChain().async_post_message(notification)
|
||||
|
||||
async def send_tool_message(
|
||||
self, message: str, title: str = "", image: Optional[str] = None
|
||||
@@ -781,11 +667,11 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
"""
|
||||
发送工具消息
|
||||
"""
|
||||
await self.send_message(
|
||||
Message(
|
||||
await self.send_notification_message(
|
||||
Notification(
|
||||
channel=self._channel,
|
||||
source=self._source,
|
||||
mtype=MessageType.Agent,
|
||||
mtype=NotificationType.Agent,
|
||||
userid=self._user_id,
|
||||
username=self._username,
|
||||
title=title,
|
||||
@@ -794,10 +680,3 @@ class MoviePilotTool(BaseTool, metaclass=ABCMeta):
|
||||
save_history=False,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# 普通导入保持 callback 冷态；显式导入或历史星号导入仍解析真实类。
|
||||
__all__ = sorted(
|
||||
{name for name in globals() if not name.startswith("_")}
|
||||
| {"StreamingHandler"}
|
||||
)
|
||||
|
||||
@@ -1,189 +0,0 @@
|
||||
"""Agent 本地工具目录的不可变快照与严格解析。"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.agent.policy.contracts import ToolRevision
|
||||
|
||||
|
||||
class ToolCatalogError(RuntimeError):
|
||||
"""工具目录无法建立可信当前视图时的稳定失败。"""
|
||||
|
||||
|
||||
class ToolIdentityAmbiguousError(ToolCatalogError):
|
||||
"""同一工具名对应多个实现，无法进行严格解析。"""
|
||||
|
||||
|
||||
def _stable_json(value: Any) -> str:
|
||||
"""生成工具身份摘要使用的稳定 JSON。"""
|
||||
return json.dumps(
|
||||
value,
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _schema_digest(tool: Any) -> str:
|
||||
"""计算工具当前 Pydantic 参数契约摘要。"""
|
||||
args_schema = getattr(tool, "args_schema", None)
|
||||
if isinstance(args_schema, type) and issubclass(args_schema, BaseModel):
|
||||
schema = args_schema.model_json_schema()
|
||||
elif isinstance(args_schema, Mapping):
|
||||
schema = dict(args_schema)
|
||||
else:
|
||||
schema = {"type": "object", "properties": {}}
|
||||
return hashlib.sha256(_stable_json(schema).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _implementation_identity(tool: Any) -> str:
|
||||
"""返回不依赖对象地址、可区分动态绑定的工具实现身份。"""
|
||||
tool_class = type(tool)
|
||||
implementation = f"{tool_class.__module__}.{tool_class.__qualname__}"
|
||||
binding = str(getattr(tool, "_agent_tool_binding", "") or "")
|
||||
return f"{implementation}:{binding}" if binding else implementation
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolCatalogEntry:
|
||||
"""绑定一次目录构造中精确工具实例的身份记录。"""
|
||||
|
||||
name: str
|
||||
source: str
|
||||
identity: str
|
||||
description_digest: str
|
||||
schema_digest: str
|
||||
revision: ToolRevision
|
||||
tool: Any = field(repr=False, compare=False)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolCatalogSnapshot:
|
||||
"""图构造、缓存签名和身份碰撞审计共享的本地工具事实源。"""
|
||||
|
||||
entries: tuple[ToolCatalogEntry, ...]
|
||||
plugin_revision: int
|
||||
factory_revision: str
|
||||
_by_name: Mapping[str, tuple[ToolCatalogEntry, ...]] = field(
|
||||
init=False,
|
||||
repr=False,
|
||||
compare=False,
|
||||
)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""建立只读名称索引并保留所有冲突项。"""
|
||||
by_name: dict[str, list[ToolCatalogEntry]] = {}
|
||||
for entry in self.entries:
|
||||
by_name.setdefault(entry.name, []).append(entry)
|
||||
object.__setattr__(
|
||||
self,
|
||||
"_by_name",
|
||||
MappingProxyType(
|
||||
{name: tuple(matches) for name, matches in by_name.items()}
|
||||
),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_tools(
|
||||
cls,
|
||||
tools: list[Any],
|
||||
*,
|
||||
plugin_revision: int,
|
||||
factory_revision: str,
|
||||
) -> "ToolCatalogSnapshot":
|
||||
"""从已完成上下文注入的精确实例建立不可变目录。"""
|
||||
entries = []
|
||||
for tool in tools:
|
||||
name = str(getattr(tool, "name", "") or "")
|
||||
if not name:
|
||||
raise ToolCatalogError("工具缺少稳定名称")
|
||||
source = str(getattr(tool, "_agent_tool_source", "builtin"))
|
||||
schema_digest = _schema_digest(tool)
|
||||
description_digest = hashlib.sha256(
|
||||
str(getattr(tool, "description", "") or "").encode("utf-8")
|
||||
).hexdigest()
|
||||
implementation = _implementation_identity(tool)
|
||||
revision = ToolRevision(
|
||||
implementation=implementation,
|
||||
factory=factory_revision,
|
||||
plugin=str(plugin_revision),
|
||||
)
|
||||
entries.append(
|
||||
ToolCatalogEntry(
|
||||
name=name,
|
||||
source=source,
|
||||
identity=f"{source}:{implementation}:{schema_digest}",
|
||||
description_digest=description_digest,
|
||||
schema_digest=schema_digest,
|
||||
revision=revision,
|
||||
tool=tool,
|
||||
)
|
||||
)
|
||||
return cls(
|
||||
entries=tuple(entries),
|
||||
plugin_revision=plugin_revision,
|
||||
factory_revision=factory_revision,
|
||||
)
|
||||
|
||||
@property
|
||||
def tools(self) -> list[Any]:
|
||||
"""按目录顺序返回精确工具实例。"""
|
||||
return [entry.tool for entry in self.entries]
|
||||
|
||||
@property
|
||||
def collisions(self) -> Mapping[str, tuple[ToolCatalogEntry, ...]]:
|
||||
"""返回所有同名工具，不按注册顺序隐式选胜者。"""
|
||||
return MappingProxyType(
|
||||
{name: entries for name, entries in self._by_name.items() if len(entries) > 1}
|
||||
)
|
||||
|
||||
@property
|
||||
def signature(self) -> tuple[Any, ...]:
|
||||
"""返回可参与 Agent 图缓存的完整目录签名。"""
|
||||
return (
|
||||
self.factory_revision,
|
||||
self.plugin_revision,
|
||||
tuple(
|
||||
(
|
||||
entry.name,
|
||||
entry.identity,
|
||||
entry.description_digest,
|
||||
entry.schema_digest,
|
||||
)
|
||||
for entry in self.entries
|
||||
),
|
||||
)
|
||||
|
||||
def resolve_unique(self, name: str) -> Optional[ToolCatalogEntry]:
|
||||
"""严格解析当前唯一实现；重名时拒绝继承 first/last-wins。"""
|
||||
entries = self._by_name.get(name, ())
|
||||
if len(entries) > 1:
|
||||
raise ToolIdentityAmbiguousError("TOOL_IDENTITY_AMBIGUOUS")
|
||||
return entries[0] if entries else None
|
||||
|
||||
def select(self, tools: list[Any]) -> "ToolCatalogSnapshot":
|
||||
"""为子图保留所选工具名的全部候选身份与 revision 语义。"""
|
||||
selected_names = {
|
||||
str(getattr(tool, "name", "") or "") for tool in tools
|
||||
}
|
||||
return ToolCatalogSnapshot(
|
||||
entries=tuple(
|
||||
entry for entry in self.entries if entry.name in selected_names
|
||||
),
|
||||
plugin_revision=self.plugin_revision,
|
||||
factory_revision=self.factory_revision,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ToolCatalogEntry",
|
||||
"ToolCatalogError",
|
||||
"ToolCatalogSnapshot",
|
||||
"ToolIdentityAmbiguousError",
|
||||
]
|
||||
@@ -1,5 +1,3 @@
|
||||
import hashlib
|
||||
|
||||
from typing import Callable, List, Optional, Type
|
||||
|
||||
from app.agent.tools.impl.add_download_tasks import AddDownloadTasksTool
|
||||
@@ -66,7 +64,6 @@ from app.agent.tools.impl.list_directory import ListDirectoryTool
|
||||
from app.agent.tools.impl.query_transfer_history import QueryTransferHistoryTool
|
||||
from app.agent.tools.impl.transfer_file import TransferFileTool
|
||||
from app.agent.tools.impl.execute_command import ExecuteCommandTool
|
||||
from app.agent.tools.impl.apply_patch import ApplyPatchTool
|
||||
from app.agent.tools.impl.edit_file import EditFileTool
|
||||
from app.agent.tools.impl.write_file import WriteFileTool
|
||||
from app.agent.tools.impl.read_file import ReadFileTool
|
||||
@@ -88,24 +85,11 @@ from app.agent.tools.impl.update_custom_identifiers import UpdateCustomIdentifie
|
||||
from app.agent.tools.impl.query_system_settings import QuerySystemSettingsTool
|
||||
from app.agent.tools.impl.update_system_settings import UpdateSystemSettingsTool
|
||||
from app.agent.llm.capability import AgentCapabilityManager
|
||||
from app.application.plugin.runtime import get_plugin_manager
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.notification import ChannelCapabilityManager
|
||||
from app.schemas.types import NotificationChannel
|
||||
from app.core.plugin import PluginManager
|
||||
from app.log import logger
|
||||
from app.schemas.message import ChannelCapabilityManager
|
||||
from app.schemas.types import MessageChannel
|
||||
from .base import MoviePilotTool
|
||||
from .catalog import ToolCatalogError, ToolCatalogSnapshot
|
||||
|
||||
|
||||
def _get_plugin_agent_tools() -> list[dict]:
|
||||
"""读取当前插件工具投影，隔离 Agent 工具工厂与 Runtime 管理器。"""
|
||||
try:
|
||||
return get_plugin_manager().get_plugin_agent_tools()
|
||||
except RuntimeError as error:
|
||||
# 纯工具目录探针可以在启动组合根之前运行；此时只跳过可选插件工具，
|
||||
# 不隐式创建 PluginManager，避免冷导入重新引入 Runtime 定位器。
|
||||
if "尚未由启动组合根装配" not in str(error):
|
||||
raise
|
||||
return []
|
||||
|
||||
|
||||
class MoviePilotToolFactory:
|
||||
@@ -176,7 +160,6 @@ class MoviePilotToolFactory:
|
||||
UpdatePersonaDefinitionTool,
|
||||
ExecuteCommandTool,
|
||||
EditFileTool,
|
||||
ApplyPatchTool,
|
||||
WriteFileTool,
|
||||
ReadFileTool,
|
||||
BrowseWebpageTool,
|
||||
@@ -206,30 +189,18 @@ class MoviePilotToolFactory:
|
||||
"write_file",
|
||||
"read_file",
|
||||
"edit_file",
|
||||
"apply_patch",
|
||||
"execute_command",
|
||||
"ask_user_choice",
|
||||
"create_agent_task",
|
||||
"query_agent_tasks",
|
||||
)
|
||||
|
||||
CATALOG_BUILD_MAX_ATTEMPTS = 3
|
||||
|
||||
@classmethod
|
||||
def catalog_factory_revision(cls) -> str:
|
||||
"""返回当前内置工具工厂定义的稳定摘要。"""
|
||||
identities = (
|
||||
f"{tool_class.__module__}.{tool_class.__qualname__}"
|
||||
for tool_class in cls.BUILTIN_TOOL_CLASSES
|
||||
)
|
||||
return hashlib.sha256("\n".join(identities).encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _should_enable_choice_tool(channel: Optional[str] = None) -> bool:
|
||||
if not channel:
|
||||
return False
|
||||
try:
|
||||
message_channel = NotificationChannel(channel)
|
||||
message_channel = MessageChannel(channel)
|
||||
except ValueError:
|
||||
return False
|
||||
return ChannelCapabilityManager.supports_buttons(
|
||||
@@ -295,12 +266,11 @@ class MoviePilotToolFactory:
|
||||
tool.set_message_attr(channel=channel, source=source, username=username)
|
||||
tool.set_stream_handler(stream_handler=stream_handler)
|
||||
tool.set_agent_context(agent_context=agent_context)
|
||||
object.__setattr__(tool, "_agent_tool_source", "builtin")
|
||||
tools.append(tool)
|
||||
|
||||
# 加载插件提供的工具
|
||||
plugin_tools_count = 0
|
||||
plugin_tools_info = _get_plugin_agent_tools()
|
||||
plugin_tools_info = PluginManager().get_plugin_agent_tools()
|
||||
for plugin_info in plugin_tools_info:
|
||||
plugin_id = plugin_info.get("plugin_id")
|
||||
plugin_name = plugin_info.get("plugin_name")
|
||||
@@ -322,11 +292,6 @@ class MoviePilotToolFactory:
|
||||
)
|
||||
tool.set_stream_handler(stream_handler=stream_handler)
|
||||
tool.set_agent_context(agent_context=agent_context)
|
||||
object.__setattr__(
|
||||
tool,
|
||||
"_agent_tool_source",
|
||||
f"plugin:{plugin_id or 'unknown'}",
|
||||
)
|
||||
tools.append(tool)
|
||||
plugin_tools_count += 1
|
||||
logger.debug(
|
||||
@@ -345,30 +310,3 @@ class MoviePilotToolFactory:
|
||||
else:
|
||||
logger.debug(f"成功创建 {len(tools)} 个MoviePilot工具")
|
||||
return tools
|
||||
|
||||
@classmethod
|
||||
def create_catalog(cls, **tool_kwargs) -> ToolCatalogSnapshot:
|
||||
"""在插件目录稳定窗口内构造一份完整本地工具快照。"""
|
||||
try:
|
||||
plugin_manager = get_plugin_manager()
|
||||
except RuntimeError as error:
|
||||
# 没有启动上下文时仍允许构造内置工具目录；插件工具会在正式启动
|
||||
# 后由组合根提供的 Runtime 中重新物化。
|
||||
if "尚未由启动组合根装配" not in str(error):
|
||||
raise
|
||||
return ToolCatalogSnapshot.from_tools(
|
||||
cls.create_tools(**tool_kwargs),
|
||||
plugin_revision=0,
|
||||
factory_revision=cls.catalog_factory_revision(),
|
||||
)
|
||||
for _attempt in range(cls.CATALOG_BUILD_MAX_ATTEMPTS):
|
||||
before_revision = plugin_manager.get_plugin_agent_tools_revision()
|
||||
tools = cls.create_tools(**tool_kwargs)
|
||||
after_revision = plugin_manager.get_plugin_agent_tools_revision()
|
||||
if before_revision == after_revision:
|
||||
return ToolCatalogSnapshot.from_tools(
|
||||
tools,
|
||||
plugin_revision=after_revision,
|
||||
factory_revision=cls.catalog_factory_revision(),
|
||||
)
|
||||
raise ToolCatalogError("插件工具目录持续变化，无法建立当前快照")
|
||||
|
||||
@@ -2,20 +2,16 @@
|
||||
|
||||
import copy
|
||||
import re
|
||||
from typing import Any, Callable, Dict, Iterable, Optional
|
||||
from typing import Any, Dict, Iterable, Optional
|
||||
|
||||
from app.application.agentdata import get_agent_subscribe_port
|
||||
from app.application.configuration import get_configured_system_config
|
||||
from app.application.rules import (
|
||||
BUILTIN_RULE_SET,
|
||||
RuleHelper,
|
||||
RuleParser,
|
||||
replace_group_name_in_list,
|
||||
)
|
||||
from app.runtime.events import eventmanager
|
||||
from app.core.event import eventmanager
|
||||
from app.db.subscribe_oper import SubscribeOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.helper.rule import RuleHelper
|
||||
from app.modules.filter.RuleParser import RuleParser
|
||||
from app.modules.filter.builtin_rules import BUILTIN_RULE_SET
|
||||
from app.schemas import CustomRule, FilterRuleGroup
|
||||
from app.schemas.event import ConfigChangeEventData
|
||||
from app.schemas.rule import CustomRule
|
||||
from app.schemas.system import FilterRuleGroup
|
||||
from app.schemas.types import EventType, SystemConfigKey
|
||||
|
||||
RULE_ID_PATTERN = re.compile(r"^[A-Za-z0-9]+$")
|
||||
@@ -30,10 +26,8 @@ MEDIA_TYPE_ALIASES = {
|
||||
"tv": "电视剧",
|
||||
"series": "电视剧",
|
||||
"show": "电视剧",
|
||||
"music": "音乐",
|
||||
"电影": "电影",
|
||||
"电视剧": "电视剧",
|
||||
"音乐": "音乐",
|
||||
}
|
||||
|
||||
RULE_STRING_SYNTAX = {
|
||||
@@ -81,9 +75,9 @@ def normalize_media_type(value: Optional[str]) -> Optional[str]:
|
||||
if not value:
|
||||
return None
|
||||
normalized = MEDIA_TYPE_ALIASES.get(value.lower(), value)
|
||||
if normalized not in {"电影", "电视剧", "音乐"}:
|
||||
if normalized not in {"电影", "电视剧"}:
|
||||
raise ValueError(
|
||||
"media_type 仅支持 '电影'、'电视剧'、'音乐'、'movie'、'tv' 或 'music'"
|
||||
"media_type 仅支持 '电影'、'电视剧'、'movie' 或 'tv'"
|
||||
)
|
||||
return normalized
|
||||
|
||||
@@ -257,13 +251,13 @@ async def collect_rule_group_usages(
|
||||
"""收集规则组在全局配置和订阅上的引用情况。"""
|
||||
target_names = set(group_names or [])
|
||||
search_groups = set(
|
||||
get_configured_system_config().get(SystemConfigKey.SearchFilterRuleGroups) or []
|
||||
SystemConfigOper().get(SystemConfigKey.SearchFilterRuleGroups) or []
|
||||
)
|
||||
subscribe_groups = set(
|
||||
get_configured_system_config().get(SystemConfigKey.SubscribeFilterRuleGroups) or []
|
||||
SystemConfigOper().get(SystemConfigKey.SubscribeFilterRuleGroups) or []
|
||||
)
|
||||
best_version_groups = set(
|
||||
get_configured_system_config().get(SystemConfigKey.BestVersionFilterRuleGroups) or []
|
||||
SystemConfigOper().get(SystemConfigKey.BestVersionFilterRuleGroups) or []
|
||||
)
|
||||
|
||||
usage_map = {
|
||||
@@ -289,7 +283,7 @@ async def collect_rule_group_usages(
|
||||
continue
|
||||
ensure_usage(name)["used_in_global_best_version"] = True
|
||||
|
||||
subscribes = await get_agent_subscribe_port().async_list()
|
||||
subscribes = await SubscribeOper().async_list()
|
||||
for subscribe in subscribes:
|
||||
filter_groups = subscribe.filter_groups or []
|
||||
for name in filter_groups:
|
||||
@@ -433,70 +427,67 @@ async def save_system_config(
|
||||
]
|
||||
normalized_value = normalized_value or None
|
||||
|
||||
success = await get_configured_system_config().async_set(key, normalized_value)
|
||||
success = await SystemConfigOper().async_set(key, normalized_value)
|
||||
if success:
|
||||
await _publish_rule_config_changed(key, normalized_value)
|
||||
await eventmanager.async_send_event(
|
||||
etype=EventType.ConfigChanged,
|
||||
data=ConfigChangeEventData(
|
||||
key=key,
|
||||
value=normalized_value,
|
||||
change_type="update",
|
||||
),
|
||||
)
|
||||
return success
|
||||
|
||||
|
||||
async def _publish_rule_config_changed(
|
||||
key: SystemConfigKey,
|
||||
value: Any,
|
||||
) -> None:
|
||||
"""广播一项已经提交的规则配置变更。"""
|
||||
await eventmanager.async_send_event(
|
||||
etype=EventType.ConfigChanged,
|
||||
data=ConfigChangeEventData(
|
||||
key=key,
|
||||
value=value,
|
||||
change_type="update",
|
||||
),
|
||||
def replace_rule_id_in_rule_string(
|
||||
rule_string: str, old_rule_id: str, new_rule_id: str
|
||||
) -> str:
|
||||
"""只替换完整 token，避免误伤其他规则名。"""
|
||||
pattern = re.compile(
|
||||
rf"(?<![A-Za-z0-9]){re.escape(old_rule_id)}(?![A-Za-z0-9])"
|
||||
)
|
||||
return pattern.sub(new_rule_id, rule_string)
|
||||
|
||||
|
||||
async def _rewrite_rule_group_references(
|
||||
map_names: Callable[[Iterable[str]], list[str]],
|
||||
) -> dict:
|
||||
"""按名称映射器更新全局、默认订阅配置和已有订阅引用。"""
|
||||
def replace_group_name_in_list(
|
||||
values: Optional[Iterable[str]], old_name: str, new_name: str
|
||||
) -> list[str]:
|
||||
"""更新配置里的规则组名引用，并顺手去重。"""
|
||||
result = []
|
||||
for value in values or []:
|
||||
mapped = new_name if value == old_name else value
|
||||
if mapped not in result:
|
||||
result.append(mapped)
|
||||
return result
|
||||
|
||||
|
||||
async def rename_rule_group_references(old_name: str, new_name: str) -> dict:
|
||||
"""规则组改名后，联动更新全局设置和订阅引用。"""
|
||||
changed = {
|
||||
"global_settings": {},
|
||||
"subscribes": [],
|
||||
}
|
||||
system_config = get_configured_system_config()
|
||||
|
||||
for config_key in (
|
||||
SystemConfigKey.SearchFilterRuleGroups,
|
||||
SystemConfigKey.SubscribeFilterRuleGroups,
|
||||
SystemConfigKey.BestVersionFilterRuleGroups,
|
||||
):
|
||||
original = system_config.get(config_key) or []
|
||||
updated = map_names(original)
|
||||
original = SystemConfigOper().get(config_key) or []
|
||||
updated = replace_group_name_in_list(original, old_name, new_name)
|
||||
if updated != original:
|
||||
await save_system_config(config_key, updated)
|
||||
changed["global_settings"][config_key.value] = updated
|
||||
|
||||
for config_key in (
|
||||
SystemConfigKey.DefaultMovieSubscribeConfig,
|
||||
SystemConfigKey.DefaultTvSubscribeConfig,
|
||||
SystemConfigKey.DefaultMusicSubscribeConfig,
|
||||
):
|
||||
original = system_config.get(config_key) or {}
|
||||
original_groups = original.get("filter_groups") or []
|
||||
updated_groups = map_names(original_groups)
|
||||
if updated_groups == original_groups:
|
||||
continue
|
||||
updated = copy.deepcopy(original)
|
||||
updated["filter_groups"] = updated_groups
|
||||
await save_system_config(config_key, updated)
|
||||
changed["global_settings"][config_key.value] = updated
|
||||
|
||||
subscribe_port = get_agent_subscribe_port()
|
||||
subscribes = await subscribe_port.async_list()
|
||||
subscribe_oper = SubscribeOper()
|
||||
subscribes = await subscribe_oper.async_list()
|
||||
for subscribe in subscribes:
|
||||
original = subscribe.filter_groups or []
|
||||
updated = map_names(original)
|
||||
updated = replace_group_name_in_list(original, old_name, new_name)
|
||||
if updated == original:
|
||||
continue
|
||||
await subscribe_port.async_update_filter_groups(subscribe.id, updated)
|
||||
await subscribe_oper.async_update_filter_groups(subscribe.id, updated)
|
||||
changed["subscribes"].append(
|
||||
{
|
||||
"subscribe_id": subscribe.id,
|
||||
@@ -509,25 +500,39 @@ async def _rewrite_rule_group_references(
|
||||
return changed
|
||||
|
||||
|
||||
async def rename_rule_group_references(old_name: str, new_name: str) -> dict:
|
||||
"""规则组改名后，联动更新全部配置和已有订阅引用。"""
|
||||
return await _rewrite_rule_group_references(
|
||||
lambda values: replace_group_name_in_list(values, old_name, new_name)
|
||||
)
|
||||
|
||||
|
||||
async def remove_rule_group_references(group_name: str) -> dict:
|
||||
"""删除规则组后，清理全部配置和已有订阅中的悬空引用。"""
|
||||
return await _rewrite_rule_group_references(
|
||||
lambda values: [value for value in values or [] if value != group_name]
|
||||
)
|
||||
"""删除规则组后，清理全局设置和订阅里的悬空引用。"""
|
||||
changed = {
|
||||
"global_settings": {},
|
||||
"subscribes": [],
|
||||
}
|
||||
|
||||
for config_key in (
|
||||
SystemConfigKey.SearchFilterRuleGroups,
|
||||
SystemConfigKey.SubscribeFilterRuleGroups,
|
||||
SystemConfigKey.BestVersionFilterRuleGroups,
|
||||
):
|
||||
original = SystemConfigOper().get(config_key) or []
|
||||
updated = [value for value in original if value != group_name]
|
||||
if updated != original:
|
||||
await save_system_config(config_key, updated)
|
||||
changed["global_settings"][config_key.value] = updated
|
||||
|
||||
def replace_rule_id_in_rule_string(
|
||||
rule_string: str, old_rule_id: str, new_rule_id: str
|
||||
) -> str:
|
||||
"""只替换完整 token，避免误伤其他规则名。"""
|
||||
pattern = re.compile(
|
||||
rf"(?<![A-Za-z0-9]){re.escape(old_rule_id)}(?![A-Za-z0-9])"
|
||||
)
|
||||
return pattern.sub(new_rule_id, rule_string)
|
||||
subscribe_oper = SubscribeOper()
|
||||
subscribes = await subscribe_oper.async_list()
|
||||
for subscribe in subscribes:
|
||||
original = subscribe.filter_groups or []
|
||||
updated = [value for value in original if value != group_name]
|
||||
if updated == original:
|
||||
continue
|
||||
await subscribe_oper.async_update_filter_groups(subscribe.id, updated)
|
||||
changed["subscribes"].append(
|
||||
{
|
||||
"subscribe_id": subscribe.id,
|
||||
"name": subscribe.name,
|
||||
"season": subscribe.season,
|
||||
"filter_groups": updated,
|
||||
}
|
||||
)
|
||||
|
||||
return changed
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
"""Agent 音乐工具共享的实体校验与结果精简函数。"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.domain.context import (
|
||||
MusicAlbumInfo,
|
||||
MusicArtistInfo,
|
||||
MusicInfo,
|
||||
)
|
||||
from app.schemas.types import media_type_to_agent
|
||||
|
||||
|
||||
MUSIC_TRACK_PREVIEW_LIMIT = 100
|
||||
MUSIC_RELEASE_PREVIEW_LIMIT = 20
|
||||
|
||||
|
||||
def simplify_music_info(info: MusicInfo) -> dict[str, Any]:
|
||||
"""精简音乐列表项，同时保留订阅和下载所需的稳定身份。"""
|
||||
payload = {
|
||||
"title": info.title,
|
||||
"year": info.year,
|
||||
"type": media_type_to_agent(info.type),
|
||||
"music_type": info.music_type,
|
||||
"artists": list(info.artists or []),
|
||||
"artist_ids": list(info.artist_ids or []),
|
||||
"artist": info.artist,
|
||||
"album": info.album,
|
||||
"album_id": info.album_id,
|
||||
"album_type": info.album_type,
|
||||
"release_date": info.release_date,
|
||||
"disc_number": info.disc_number,
|
||||
"track_number": info.track_number,
|
||||
"total_tracks": info.total_tracks,
|
||||
"duration": info.duration,
|
||||
"isrc": info.isrc,
|
||||
"version": info.version,
|
||||
"genres": list(info.genres or []),
|
||||
"category": info.category,
|
||||
"listen_count": info.listen_count,
|
||||
"media_source": info.media_source,
|
||||
"media_id": info.media_id,
|
||||
"poster_path": info.poster_path,
|
||||
"detail_link": info.detail_link,
|
||||
"overview": info.overview,
|
||||
}
|
||||
return {key: value for key, value in payload.items() if value not in (None, "", [])}
|
||||
|
||||
|
||||
def simplify_music_album(
|
||||
info: MusicAlbumInfo,
|
||||
*,
|
||||
track_limit: int = MUSIC_TRACK_PREVIEW_LIMIT,
|
||||
) -> dict[str, Any]:
|
||||
"""精简专辑详情并限制曲目和发行版本预览，避免撑大 Agent 上下文。"""
|
||||
normalized_track_limit = max(1, min(track_limit, MUSIC_TRACK_PREVIEW_LIMIT))
|
||||
tracks = list(info.tracks or [])
|
||||
releases = list(info.releases or [])
|
||||
payload = simplify_music_info(info.to_music_info())
|
||||
payload.update({
|
||||
"release_date": info.release_date,
|
||||
"secondary_types": list(info.secondary_types or []),
|
||||
"tags": list(info.tags or []),
|
||||
"rating": info.rating,
|
||||
"rating_votes": info.rating_votes,
|
||||
"tracks": [
|
||||
simplify_music_info(track)
|
||||
for track in tracks[:normalized_track_limit]
|
||||
],
|
||||
"tracks_total": len(tracks),
|
||||
"tracks_truncated": len(tracks) > normalized_track_limit,
|
||||
"releases": [release.to_dict() for release in releases[:MUSIC_RELEASE_PREVIEW_LIMIT]],
|
||||
"releases_total": len(releases),
|
||||
"releases_truncated": len(releases) > MUSIC_RELEASE_PREVIEW_LIMIT,
|
||||
})
|
||||
return payload
|
||||
|
||||
|
||||
def simplify_music_artist(info: MusicArtistInfo) -> dict[str, Any]:
|
||||
"""精简艺术家详情，明确其仅用于浏览而非订阅或下载。"""
|
||||
payload = info.to_dict()
|
||||
payload.pop("raw_data", None)
|
||||
payload.pop("mediaid_prefix", None)
|
||||
payload["type"] = media_type_to_agent(info.type)
|
||||
payload["media_source"] = info.media_source
|
||||
payload["media_id"] = info.media_id
|
||||
payload["subscribable"] = False
|
||||
return {key: value for key, value in payload.items() if value not in (None, "", [])}
|
||||
@@ -2,18 +2,15 @@
|
||||
|
||||
import json
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.adapters.external.market import PluginHelper
|
||||
from app.application.configuration import get_configured_system_config
|
||||
from app.application.plugin.gateway import get_plugin_install_service
|
||||
from app.application.plugin.runtime import get_plugin_manager
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.schemas.plugin import PluginRuntimeStatus
|
||||
from app.core.config import settings
|
||||
from app.core.plugin import PluginManager
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.helper.server import MoviePilotServerHelper
|
||||
from app.helper.plugin import PluginHelper
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
# 默认只向智能体返回一个可读预览，避免超大插件数据挤爆上下文窗口。
|
||||
DEFAULT_PLUGIN_DATA_PREVIEW_CHARS = 12_000
|
||||
MAX_PLUGIN_DATA_PREVIEW_CHARS = 50_000
|
||||
@@ -23,22 +20,11 @@ DEFAULT_PLUGIN_CANDIDATE_LIMIT = 50
|
||||
MAX_PLUGIN_CANDIDATE_LIMIT = 200
|
||||
|
||||
|
||||
def _remove_plugin_directory(path: Path) -> bool:
|
||||
"""删除插件目录并返回是否完成，供受控线程执行。"""
|
||||
if not path.exists():
|
||||
return False
|
||||
try:
|
||||
shutil.rmtree(path)
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def get_plugin_snapshot(plugin_id: str) -> Optional[dict[str, Any]]:
|
||||
"""
|
||||
获取已安装插件的基础信息快照。
|
||||
"""
|
||||
plugin_manager = get_plugin_manager()
|
||||
plugin_manager = PluginManager()
|
||||
for plugin in plugin_manager.get_local_plugins():
|
||||
if plugin.id == plugin_id:
|
||||
return {
|
||||
@@ -79,27 +65,22 @@ def build_preview_payload(value: Any, max_chars: Optional[int]) -> tuple[bool, i
|
||||
return True, len(serialized), len(preview), preview
|
||||
|
||||
|
||||
def refresh_plugin_registrations(plugin_id: str) -> None:
|
||||
"""重新注册插件的定时任务、命令和动态 API 路由。"""
|
||||
def reload_plugin_runtime(plugin_id: str) -> None:
|
||||
"""
|
||||
重载插件并重新注册其命令、定时任务和 API。
|
||||
"""
|
||||
# 这些依赖只在真正执行重载时才导入，避免普通查询工具引入不必要的初始化开销。
|
||||
from app.application.plugin.routes import register_plugin_api
|
||||
from app.application.commands import init_commands
|
||||
from app.application.scheduling import update_plugin_job
|
||||
from app.api.endpoints.plugin import register_plugin_api
|
||||
from app.command import Command
|
||||
from app.scheduler import Scheduler
|
||||
|
||||
update_plugin_job(plugin_id)
|
||||
init_commands(plugin_id)
|
||||
plugin_manager = PluginManager()
|
||||
plugin_manager.reload_plugin(plugin_id)
|
||||
Scheduler().update_plugin_job(plugin_id)
|
||||
Command().init_commands(plugin_id)
|
||||
register_plugin_api(plugin_id)
|
||||
|
||||
|
||||
def reload_plugin_runtime(plugin_id: str) -> PluginRuntimeStatus:
|
||||
"""重载插件实例并重新注册其命令、定时任务和 API。"""
|
||||
plugin_manager = get_plugin_manager()
|
||||
with plugin_manager.mutation(f"重载插件 {plugin_id}"):
|
||||
runtime_status = plugin_manager.reload_plugin(plugin_id)
|
||||
refresh_plugin_registrations(plugin_id)
|
||||
return runtime_status
|
||||
|
||||
|
||||
def summarize_plugin(plugin: Any) -> dict[str, Any]:
|
||||
"""
|
||||
提取插件对象中对 Agent 有价值的摘要字段。
|
||||
@@ -172,7 +153,7 @@ async def enrich_installed_plugin_sources(
|
||||
if not missing_source_plugins:
|
||||
return installed_plugins
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
plugin_manager = PluginManager()
|
||||
local_repo_map = _map_plugins_by_id(plugin_manager.get_local_repo_plugins())
|
||||
for plugin in missing_source_plugins:
|
||||
source_plugin = local_repo_map.get(getattr(plugin, "id", None))
|
||||
@@ -199,7 +180,7 @@ async def load_market_plugins(force_refresh: bool = False) -> list[Any]:
|
||||
"""
|
||||
聚合插件市场与本地插件仓库中的候选插件。
|
||||
"""
|
||||
plugin_manager = get_plugin_manager()
|
||||
plugin_manager = PluginManager()
|
||||
online_plugins = await plugin_manager.async_get_online_plugins(force=force_refresh)
|
||||
local_repo_plugins = plugin_manager.get_local_repo_plugins()
|
||||
if not online_plugins and not local_repo_plugins:
|
||||
@@ -211,7 +192,7 @@ def list_installed_plugins() -> list[Any]:
|
||||
"""
|
||||
返回当前已安装插件列表。
|
||||
"""
|
||||
plugin_manager = get_plugin_manager()
|
||||
plugin_manager = PluginManager()
|
||||
return [plugin for plugin in plugin_manager.get_local_plugins() if plugin.installed]
|
||||
|
||||
|
||||
@@ -310,106 +291,81 @@ def summarize_candidates(matches: list[dict[str, Any]], limit: int = DEFAULT_PLU
|
||||
|
||||
|
||||
async def install_plugin_runtime(
|
||||
plugin_id: str,
|
||||
repo_url: Optional[str],
|
||||
force: bool = False,
|
||||
*,
|
||||
explicit_source: bool = False,
|
||||
plugin_id: str, repo_url: Optional[str], force: bool = False
|
||||
) -> tuple[bool, str, bool]:
|
||||
"""
|
||||
按现有插件接口的行为安装插件，并刷新运行态注册信息。
|
||||
"""
|
||||
result = await get_plugin_install_service().install(
|
||||
plugin_id=plugin_id,
|
||||
repo_url=repo_url or None,
|
||||
force=force,
|
||||
explicit_source=explicit_source,
|
||||
)
|
||||
return result.success, result.message, result.refreshed_only
|
||||
install_plugins = SystemConfigOper().get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
plugin_manager = PluginManager()
|
||||
plugin_helper = PluginHelper()
|
||||
|
||||
refreshed_only = False
|
||||
if not force and plugin_id in plugin_manager.get_plugin_ids():
|
||||
refreshed_only = True
|
||||
await MoviePilotServerHelper.async_install_plugin_reg(plugin_id=plugin_id, repo_url=repo_url)
|
||||
message = "插件已存在，已刷新加载"
|
||||
else:
|
||||
if not repo_url:
|
||||
return False, "没有传入仓库地址，无法正确安装插件，请检查配置", False
|
||||
state, message = await plugin_helper.async_install(
|
||||
pid=plugin_id,
|
||||
repo_url=repo_url,
|
||||
force_install=force,
|
||||
)
|
||||
if not state:
|
||||
return False, message, False
|
||||
await MoviePilotServerHelper.async_install_plugin_reg(plugin_id=plugin_id, repo_url=repo_url)
|
||||
|
||||
async def inspect_plugin_sources(
|
||||
plugin_id: str,
|
||||
*,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""返回 Agent 可展示的脱敏来源候选与当前准入状态。"""
|
||||
inspection = await get_plugin_install_service().inspect_source(
|
||||
plugin_id=plugin_id,
|
||||
force=force,
|
||||
)
|
||||
candidates = [
|
||||
candidate.public_dict()
|
||||
for candidate in inspection.online_candidates
|
||||
]
|
||||
if inspection.local_candidate is not None:
|
||||
candidates.append(inspection.local_candidate.public_dict())
|
||||
return {
|
||||
"selection_status": inspection.selection.status.value,
|
||||
"selection_reason": inspection.selection.reason,
|
||||
"inventory_complete": inspection.inventory_complete,
|
||||
"candidates": candidates,
|
||||
}
|
||||
if plugin_id not in install_plugins:
|
||||
install_plugins.append(plugin_id)
|
||||
await SystemConfigOper().async_set(
|
||||
SystemConfigKey.UserInstalledPlugins, install_plugins
|
||||
)
|
||||
|
||||
from app.agent.tools.base import run_agent_blocking
|
||||
|
||||
await run_agent_blocking("plugin", reload_plugin_runtime, plugin_id)
|
||||
return True, message or "插件安装成功", refreshed_only
|
||||
|
||||
|
||||
async def uninstall_plugin_runtime(plugin_id: str) -> dict[str, Any]:
|
||||
"""
|
||||
按现有卸载逻辑移除插件，并清理运行态注册与分组信息。
|
||||
"""
|
||||
from app.application.plugin.folders import remove_plugin_from_folders
|
||||
from app.application.plugin.routes import remove_plugin_api
|
||||
from app.application.scheduling import remove_plugin_job
|
||||
from app.agent.tools.base import run_agent_blocking
|
||||
from app.api.endpoints.plugin import _remove_plugin_from_folders, remove_plugin_api
|
||||
from app.scheduler import Scheduler
|
||||
|
||||
plugin_manager = get_plugin_manager()
|
||||
with plugin_manager.mutation(f"卸载插件 {plugin_id}"):
|
||||
virtual_instance = plugin_manager.get_plugin_instance(plugin_id)
|
||||
source_instances = plugin_manager.get_plugin_source_instances(plugin_id)
|
||||
if not virtual_instance and source_instances:
|
||||
instance_ids = "、".join(item.instance_id for item in source_instances)
|
||||
raise ValueError(f"请先卸载该插件的分身：{instance_ids}")
|
||||
config_oper = SystemConfigOper()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [plugin for plugin in install_plugins if plugin != plugin_id]
|
||||
await config_oper.async_set(SystemConfigKey.UserInstalledPlugins, install_plugins)
|
||||
|
||||
config_oper = get_configured_system_config()
|
||||
install_plugins = config_oper.get(SystemConfigKey.UserInstalledPlugins) or []
|
||||
if plugin_id in install_plugins:
|
||||
install_plugins = [
|
||||
plugin for plugin in install_plugins if plugin != plugin_id
|
||||
]
|
||||
await config_oper.async_set(
|
||||
SystemConfigKey.UserInstalledPlugins,
|
||||
install_plugins,
|
||||
)
|
||||
remove_plugin_api(plugin_id)
|
||||
Scheduler().remove_plugin_job(plugin_id)
|
||||
|
||||
remove_plugin_api(plugin_id)
|
||||
remove_plugin_job(plugin_id)
|
||||
plugin_manager = PluginManager()
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
plugin_class = plugin_manager.plugins.get(plugin_id)
|
||||
was_clone = bool(getattr(plugin_class, "is_clone", False))
|
||||
clone_files_removed = False
|
||||
|
||||
if virtual_instance:
|
||||
plugin_manager.delete_plugin_config(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_data(plugin_id, force=True)
|
||||
plugin_manager.delete_plugin_instance(plugin_id)
|
||||
elif was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = get_runtime_setting('ROOT_PATH') / "app" / "plugins" / plugin_id.lower()
|
||||
if was_clone:
|
||||
plugin_manager.delete_plugin_config(plugin_id)
|
||||
plugin_manager.delete_plugin_data(plugin_id)
|
||||
plugin_base_dir = settings.ROOT_PATH / "app" / "plugins" / plugin_id.lower()
|
||||
if plugin_base_dir.exists():
|
||||
try:
|
||||
clone_files_removed = await run_agent_blocking(
|
||||
"plugin",
|
||||
_remove_plugin_directory,
|
||||
plugin_base_dir,
|
||||
)
|
||||
if clone_files_removed:
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
shutil.rmtree(plugin_base_dir)
|
||||
plugin_manager.plugins.pop(plugin_id, None)
|
||||
clone_files_removed = True
|
||||
except Exception:
|
||||
clone_files_removed = False
|
||||
|
||||
remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
_remove_plugin_from_folders(plugin_id)
|
||||
plugin_manager.remove_plugin(plugin_id)
|
||||
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
return {
|
||||
"was_clone": was_clone,
|
||||
"clone_files_removed": clone_files_removed,
|
||||
}
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.agent.policy.secret_fields import is_secret_setting_key
|
||||
from app.runtime.config import Settings
|
||||
from app.core.config import Settings
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
@@ -120,10 +119,6 @@ SYSTEMCONFIG_SETTING_METADATA = {
|
||||
"group": "subscribe_defaults",
|
||||
"label": "默认电视剧订阅规则",
|
||||
},
|
||||
SystemConfigKey.DefaultMusicSubscribeConfig.value: {
|
||||
"group": "subscribe_defaults",
|
||||
"label": "默认音乐订阅规则",
|
||||
},
|
||||
SystemConfigKey.UserInstalledPlugins.value: {
|
||||
"group": "plugins",
|
||||
"label": "已安装插件列表",
|
||||
@@ -227,36 +222,13 @@ GROUP_ALIASES = {
|
||||
}
|
||||
|
||||
|
||||
# 这些前缀共同组成可启动、推理和扩展 AI Agent 的同一业务配置域。
|
||||
AI_AGENT_CORE_SETTING_PREFIXES = (
|
||||
"AI_AGENT_",
|
||||
"LLM_",
|
||||
"AUDIO_INPUT_",
|
||||
"AUDIO_OUTPUT_",
|
||||
"AI_RECOMMEND_",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_token(value: str) -> str:
|
||||
return str(value).strip().lower().replace("-", "_")
|
||||
|
||||
|
||||
def _resolve_core_setting_group(key: str) -> str:
|
||||
"""根据基础设置的业务归属返回 Agent 可查询的分类。"""
|
||||
|
||||
if key.startswith(AI_AGENT_CORE_SETTING_PREFIXES):
|
||||
return "ai_agent"
|
||||
return "settings"
|
||||
|
||||
|
||||
def _build_specs() -> tuple[dict[str, SettingSpec], dict[str, SettingSpec]]:
|
||||
core_specs = {
|
||||
key: SettingSpec(
|
||||
key=key,
|
||||
source="settings",
|
||||
group=_resolve_core_setting_group(key),
|
||||
label=key,
|
||||
)
|
||||
key: SettingSpec(key=key, source="settings", group="settings", label=key)
|
||||
for key in Settings.model_fields.keys()
|
||||
}
|
||||
system_specs = {}
|
||||
@@ -287,7 +259,7 @@ SINGLE_KEY_GROUP_ALIASES = {
|
||||
_normalize_token(alias): next(
|
||||
(
|
||||
spec.key
|
||||
for spec in ALL_SETTING_SPECS.values()
|
||||
for spec in SYSTEMCONFIG_SETTING_SPECS.values()
|
||||
if spec.group == canonical_group
|
||||
),
|
||||
None,
|
||||
@@ -297,7 +269,7 @@ SINGLE_KEY_GROUP_ALIASES = {
|
||||
and len(
|
||||
[
|
||||
spec.key
|
||||
for spec in ALL_SETTING_SPECS.values()
|
||||
for spec in SYSTEMCONFIG_SETTING_SPECS.values()
|
||||
if spec.group == canonical_group
|
||||
]
|
||||
)
|
||||
@@ -348,7 +320,7 @@ def list_setting_specs(
|
||||
else:
|
||||
specs = [
|
||||
spec
|
||||
for spec in ALL_SETTING_SPECS.values()
|
||||
for spec in SYSTEMCONFIG_SETTING_SPECS.values()
|
||||
if spec.group == normalized_group
|
||||
]
|
||||
|
||||
@@ -369,6 +341,26 @@ def get_default_list_match_field(setting_key: str) -> Optional[str]:
|
||||
return LIST_ITEM_MATCH_FIELD_DEFAULTS.get(setting_key)
|
||||
|
||||
|
||||
SECRET_KEYWORDS = (
|
||||
"api_key",
|
||||
"apikey",
|
||||
"token",
|
||||
"secret",
|
||||
"password",
|
||||
"passwd",
|
||||
"cookie",
|
||||
"authorization",
|
||||
"refresh_token",
|
||||
"access_token",
|
||||
)
|
||||
|
||||
|
||||
def is_secret_setting_key(key: str) -> bool:
|
||||
"""判断设置键名是否疑似敏感字段。"""
|
||||
normalized = _normalize_token(key)
|
||||
return any(keyword in normalized for keyword in SECRET_KEYWORDS)
|
||||
|
||||
|
||||
def redact_secret_value(value: Any, *, redact_scalar: bool = False) -> Any:
|
||||
"""递归脱敏配置值中的密钥、Cookie、Token 等敏感字段。"""
|
||||
if isinstance(value, dict):
|
||||
|
||||
@@ -14,8 +14,8 @@ from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
from app.agent.tools.impl._command_safety import validate_command_safety
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.settings import get_runtime_setting
|
||||
from app.core.config import settings
|
||||
from app.log import logger
|
||||
|
||||
if os.name == "posix":
|
||||
import fcntl as _fcntl
|
||||
@@ -128,11 +128,6 @@ class _TerminalSessionManager:
|
||||
"""初始化会话表和并发保护锁。"""
|
||||
self._sessions: dict[str, _TerminalSession] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
self._close_lock = asyncio.Lock()
|
||||
self._closed = False
|
||||
self._starting = 0
|
||||
self._starts_idle = asyncio.Event()
|
||||
self._starts_idle.set()
|
||||
|
||||
@staticmethod
|
||||
def _normalize_bool(value: Any, default: bool = True) -> bool:
|
||||
@@ -149,10 +144,10 @@ class _TerminalSessionManager:
|
||||
def _normalize_cwd(cwd: Optional[str]) -> str:
|
||||
"""解析工作目录，未传入时默认使用 MoviePilot 项目根目录。"""
|
||||
if not cwd:
|
||||
return str(get_runtime_setting('ROOT_PATH'))
|
||||
return str(settings.ROOT_PATH)
|
||||
path = Path(cwd).expanduser()
|
||||
if not path.is_absolute():
|
||||
path = (get_runtime_setting('ROOT_PATH') / path).resolve()
|
||||
path = (settings.ROOT_PATH / path).resolve()
|
||||
else:
|
||||
path = path.resolve()
|
||||
if not path.exists():
|
||||
@@ -216,55 +211,20 @@ class _TerminalSessionManager:
|
||||
should_use_pty = self._normalize_bool(use_pty, default=True) and os.name == "posix"
|
||||
|
||||
async with self._lock:
|
||||
if self._closed:
|
||||
raise RuntimeError("终端会话管理器已关闭")
|
||||
self._cleanup_finished_sessions_locked()
|
||||
if (
|
||||
self._active_session_count_locked() + self._starting
|
||||
>= TERMINAL_CONCURRENCY_LIMIT
|
||||
):
|
||||
if self._active_session_count_locked() >= TERMINAL_CONCURRENCY_LIMIT:
|
||||
raise RuntimeError(
|
||||
f"后台终端会话数已达到上限 {TERMINAL_CONCURRENCY_LIMIT}"
|
||||
)
|
||||
self._starting += 1
|
||||
self._starts_idle.clear()
|
||||
|
||||
session: Optional[_TerminalSession] = None
|
||||
reject_session = False
|
||||
session_registered = False
|
||||
session_released = False
|
||||
try:
|
||||
session = (
|
||||
await self._start_pty_session(command, normalized_cwd, normalized_env)
|
||||
if should_use_pty
|
||||
else await self._start_pipe_session(
|
||||
command, normalized_cwd, normalized_env
|
||||
)
|
||||
)
|
||||
session = (
|
||||
await self._start_pty_session(command, normalized_cwd, normalized_env)
|
||||
if should_use_pty
|
||||
else await self._start_pipe_session(command, normalized_cwd, normalized_env)
|
||||
)
|
||||
|
||||
async with self._lock:
|
||||
reject_session = self._closed
|
||||
if not reject_session:
|
||||
self._sessions[session.session_id] = session
|
||||
session_registered = True
|
||||
|
||||
if reject_session:
|
||||
await self._terminate_session(session)
|
||||
session_released = True
|
||||
raise RuntimeError("终端会话管理器已关闭")
|
||||
except BaseException:
|
||||
if session is not None and not session_registered and not session_released:
|
||||
cleanup_task = asyncio.create_task(self._terminate_session(session))
|
||||
try:
|
||||
await asyncio.shield(cleanup_task)
|
||||
except asyncio.CancelledError:
|
||||
await cleanup_task
|
||||
raise
|
||||
finally:
|
||||
async with self._lock:
|
||||
self._starting -= 1
|
||||
if self._starting == 0:
|
||||
self._starts_idle.set()
|
||||
async with self._lock:
|
||||
self._sessions[session.session_id] = session
|
||||
|
||||
logger.info(
|
||||
"启动后台终端会话: session_id=%s, pid=%s, use_pty=%s, command=%s",
|
||||
@@ -513,62 +473,6 @@ class _TerminalSessionManager:
|
||||
|
||||
return self._session_payload(session, output="", output_truncated=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""停止所有后台终端会话并释放 PTY、读取任务和会话记录。"""
|
||||
async with self._close_lock:
|
||||
async with self._lock:
|
||||
self._closed = True
|
||||
|
||||
await self._starts_idle.wait()
|
||||
|
||||
async with self._lock:
|
||||
sessions = list(self._sessions.values())
|
||||
|
||||
await asyncio.gather(
|
||||
*(self._terminate_session(session) for session in sessions),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
async with self._lock:
|
||||
for session in sessions:
|
||||
session.close_pty()
|
||||
self._sessions.clear()
|
||||
|
||||
async def _terminate_session(self, session: _TerminalSession) -> None:
|
||||
"""以有限等待停止进程，并在必要时升级为 SIGKILL。"""
|
||||
if session.status == "running":
|
||||
session.kill_requested = True
|
||||
self._send_signal(session, signal.SIGTERM)
|
||||
|
||||
wait_task = session.wait_task
|
||||
if wait_task and not wait_task.done():
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(wait_task),
|
||||
timeout=TERMINAL_KILL_GRACE_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
force_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
|
||||
self._send_signal(session, force_signal)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(wait_task),
|
||||
timeout=TERMINAL_KILL_GRACE_SECONDS,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(
|
||||
"终端会话关闭超时: session_id=%s, pid=%s",
|
||||
session.session_id,
|
||||
session.pid,
|
||||
)
|
||||
|
||||
for task in session.reader_tasks:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
if session.reader_tasks:
|
||||
await asyncio.gather(*session.reader_tasks, return_exceptions=True)
|
||||
session.close_pty()
|
||||
|
||||
def get_session(self, session_id: str) -> _TerminalSession:
|
||||
"""按 ID 获取会话，不存在时抛出清晰错误。"""
|
||||
session = self._sessions.get(session_id)
|
||||
@@ -721,11 +625,3 @@ class _TerminalSessionManager:
|
||||
|
||||
|
||||
terminal_session_manager = _TerminalSessionManager()
|
||||
|
||||
|
||||
def get_terminal_session_manager() -> _TerminalSessionManager:
|
||||
"""返回当前进程的终端会话管理器，避免复用已完成关停的实例。"""
|
||||
global terminal_session_manager
|
||||
if terminal_session_manager._closed:
|
||||
terminal_session_manager = _TerminalSessionManager()
|
||||
return terminal_session_manager
|
||||
|
||||
@@ -3,11 +3,9 @@
|
||||
import re
|
||||
from typing import List, Optional
|
||||
|
||||
from app.domain.context import Context
|
||||
from app.schemas.types import MediaType, media_type_to_agent
|
||||
from app.foundation.crypto import HashUtils
|
||||
from app.foundation import size as size_tools
|
||||
from ._music_utils import simplify_music_info
|
||||
from app.core.context import Context
|
||||
from app.utils.crypto import HashUtils
|
||||
from app.utils.string import StringUtils
|
||||
|
||||
SEARCH_RESULT_CACHE_FILE = "__search_result__"
|
||||
TORRENT_RESULT_LIMIT = 50
|
||||
@@ -152,7 +150,7 @@ def simplify_search_result(
|
||||
if torrent_info:
|
||||
simplified["torrent_info"] = {
|
||||
"title": torrent_info.title,
|
||||
"size": size_tools.format_size(torrent_info.size),
|
||||
"size": StringUtils.format_size(torrent_info.size),
|
||||
"seeders": torrent_info.seeders,
|
||||
"peers": torrent_info.peers,
|
||||
"site_name": torrent_info.site_name,
|
||||
@@ -168,44 +166,28 @@ def simplify_search_result(
|
||||
simplified["torrent_info"]["labels"] = torrent_info.labels or []
|
||||
|
||||
if media_info:
|
||||
if getattr(media_info, "type", None) == MediaType.MUSIC:
|
||||
simplified["media_info"] = simplify_music_info(media_info)
|
||||
else:
|
||||
simplified["media_info"] = {
|
||||
"title": getattr(media_info, "title", None),
|
||||
"en_title": getattr(media_info, "en_title", None),
|
||||
"year": getattr(media_info, "year", None),
|
||||
"type": media_type_to_agent(getattr(media_info, "type", None)),
|
||||
"season": getattr(media_info, "season", None),
|
||||
"tmdb_id": getattr(media_info, "tmdb_id", None),
|
||||
}
|
||||
simplified["media_info"] = {
|
||||
"title": media_info.title,
|
||||
"en_title": media_info.en_title,
|
||||
"year": media_info.year,
|
||||
"type": media_info.type.value if media_info.type else None,
|
||||
"season": media_info.season,
|
||||
"tmdb_id": media_info.tmdb_id,
|
||||
}
|
||||
|
||||
if meta_info:
|
||||
if getattr(meta_info, "type", None) == MediaType.MUSIC:
|
||||
simplified["meta_info"] = {
|
||||
key: getattr(meta_info, key, None)
|
||||
for key in (
|
||||
"title", "artists", "album", "album_artist", "year",
|
||||
"disc_number", "track_number", "total_tracks", "version",
|
||||
"audio_format", "bit_depth", "sample_rate", "bitrate",
|
||||
"duration", "isrc", "media_source", "media_id",
|
||||
)
|
||||
if getattr(meta_info, key, None) not in (None, "", [])
|
||||
}
|
||||
simplified["meta_info"]["type"] = "music"
|
||||
else:
|
||||
simplified["meta_info"] = {
|
||||
"name": getattr(meta_info, "name", None),
|
||||
"cn_name": getattr(meta_info, "cn_name", None),
|
||||
"en_name": getattr(meta_info, "en_name", None),
|
||||
"year": getattr(meta_info, "year", None),
|
||||
"type": media_type_to_agent(getattr(meta_info, "type", None)),
|
||||
"begin_season": getattr(meta_info, "begin_season", None),
|
||||
"season_episode": getattr(meta_info, "season_episode", None),
|
||||
"resource_team": getattr(meta_info, "resource_team", None),
|
||||
"video_encode": getattr(meta_info, "video_encode", None),
|
||||
"edition": getattr(meta_info, "edition", None),
|
||||
"resource_pix": getattr(meta_info, "resource_pix", None),
|
||||
}
|
||||
simplified["meta_info"] = {
|
||||
"name": meta_info.name,
|
||||
"cn_name": meta_info.cn_name,
|
||||
"en_name": meta_info.en_name,
|
||||
"year": meta_info.year,
|
||||
"type": meta_info.type.value if meta_info.type else None,
|
||||
"begin_season": meta_info.begin_season,
|
||||
"season_episode": meta_info.season_episode,
|
||||
"resource_team": meta_info.resource_team,
|
||||
"video_encode": meta_info.video_encode,
|
||||
"edition": meta_info.edition,
|
||||
"resource_pix": meta_info.resource_pix,
|
||||
}
|
||||
|
||||
return simplified
|
||||
|
||||
@@ -13,7 +13,7 @@ from app.agent.tools.impl._filter_rule_utils import (
|
||||
save_system_config,
|
||||
serialize_custom_rule,
|
||||
)
|
||||
from app.runtime.log import logger
|
||||
from app.log import logger
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user