Compare commits
84 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
16e14fc371 | ||
|
|
b4a5dee236 | ||
|
|
2b3e6f9547 | ||
|
|
f8b93d2c76 | ||
|
|
94462ca706 | ||
|
|
9cbefc26ab | ||
|
|
2bfd69c15e | ||
|
|
4765d6f50a | ||
|
|
bf306dfec3 | ||
|
|
a6425f11a2 | ||
|
|
395ef0013a | ||
|
|
ab0533210f | ||
|
|
3eb2f0b14d | ||
|
|
42272b1294 | ||
|
|
d1168f35f3 | ||
|
|
bc27778366 | ||
|
|
9c5f3452e9 | ||
|
|
d3b4559b2d | ||
|
|
59305c0bb4 | ||
|
|
32214d5d5f | ||
|
|
315ad13703 | ||
|
|
e12a9cda95 | ||
|
|
c995b3bf72 | ||
|
|
1467c262a1 | ||
|
|
7251802202 | ||
|
|
e1285ff49a | ||
|
|
e01a22136e | ||
|
|
eba69ff82a | ||
|
|
5af6fe5e6e | ||
|
|
9d8e398cbe | ||
|
|
7097b2a6b9 | ||
|
|
acf7359d56 | ||
|
|
7c514b2dcc | ||
|
|
2c4fa441e7 | ||
|
|
51672e8607 | ||
|
|
cc7f773300 | ||
|
|
802565e4f6 | ||
|
|
4984026017 | ||
|
|
2a98359085 | ||
|
|
979294bb94 | ||
|
|
40cf22a7fa | ||
|
|
9e5a8b0573 | ||
|
|
7c220f0d2b | ||
|
|
aa88f97eff | ||
|
|
b4177d4ffc | ||
|
|
b888db6a61 | ||
|
|
6ae87364b4 | ||
|
|
18c966a0f9 | ||
|
|
ab84a8dad1 | ||
|
|
1a32e38dc3 | ||
|
|
0f25923c52 | ||
|
|
cdc30e1b32 | ||
|
|
c10c14c125 | ||
|
|
60604aeb33 | ||
|
|
276fb5b3e4 | ||
|
|
e05f58b8a1 | ||
|
|
8dfc96e1dc | ||
|
|
cdc639cf75 | ||
|
|
847c3115cd | ||
|
|
7dc049ffe5 | ||
|
|
265fe630dd | ||
|
|
f31900e6c7 | ||
|
|
54b46c150e | ||
|
|
7d9999d6aa | ||
|
|
05aa30119e | ||
|
|
368b9ef735 | ||
|
|
0113bf704d | ||
|
|
66a7b1394e | ||
|
|
ae05cad22f | ||
|
|
be3abab13f | ||
|
|
c432a282a7 | ||
|
|
e9e20ace93 | ||
|
|
6187827e1b | ||
|
|
8a4a95e343 | ||
|
|
401fcdc630 | ||
|
|
b2d22253c5 | ||
|
|
29bfc2efce | ||
|
|
75de39dfbb | ||
|
|
8f37fdf841 | ||
|
|
20e3ac2129 | ||
|
|
3a8f33d273 | ||
|
|
d46881aea6 | ||
|
|
e25339c53c | ||
|
|
5102999676 |
@@ -1,24 +1,52 @@
|
||||
name: Build Binary And Release
|
||||
name: Build Binary
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
workflow_call:
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
name: Build frontend
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: web
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Setup bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: latest
|
||||
- name: Install dependencies
|
||||
run: bun install --frozen-lockfile
|
||||
- name: Cache dependencies
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.bun/install/cache
|
||||
key: ${{ runner.os }}-bun-${{ hashFiles('docs/bun.lockb') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-bun-
|
||||
- name: Build Frontend
|
||||
run: bun run build
|
||||
- name: Upload Web Build Artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: web-build
|
||||
path: web/build
|
||||
build:
|
||||
name: Release for ${{ matrix.platform.release_for }}
|
||||
name: Build bili-sync-rs for ${{ matrix.platform.release_for }}
|
||||
needs: build-frontend
|
||||
runs-on: ${{ matrix.platform.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
platform:
|
||||
- release_for: Linux-x86_64
|
||||
os: ubuntu-20.04
|
||||
os: ubuntu-24.04
|
||||
target: x86_64-unknown-linux-musl
|
||||
bin: bili-sync-rs
|
||||
name: bili-sync-rs-Linux-x86_64-musl.tar.gz
|
||||
- release_for: Linux-aarch64
|
||||
os: ubuntu-20.04
|
||||
os: ubuntu-24.04
|
||||
target: aarch64-unknown-linux-musl
|
||||
bin: bili-sync-rs
|
||||
name: bili-sync-rs-Linux-aarch64-musl.tar.gz
|
||||
@@ -37,10 +65,16 @@ jobs:
|
||||
target: x86_64-pc-windows-msvc
|
||||
bin: bili-sync-rs.exe
|
||||
name: bili-sync-rs-Windows-x86_64.zip
|
||||
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Download Web Build Artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: web-build
|
||||
path: web/build
|
||||
- name: Cache dependencies
|
||||
uses: Swatinem/rust-cache@v2
|
||||
- name: Install musl-tools
|
||||
@@ -57,7 +91,6 @@ jobs:
|
||||
- name: Package as archive
|
||||
shell: bash
|
||||
run: |
|
||||
cp target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin }} ${{ matrix.platform.release_for }}-${{ matrix.platform.bin }}
|
||||
cd target/${{ matrix.platform.target }}/release
|
||||
if [[ "${{ matrix.platform.target }}" == "x86_64-pc-windows-msvc" ]]; then
|
||||
7z a ../../../${{ matrix.platform.name }} ${{ matrix.platform.bin }}
|
||||
@@ -68,62 +101,5 @@ jobs:
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bili-sync-rs-${{ matrix.platform.release_for }}
|
||||
# contains raw binary and compressed archive
|
||||
path: |
|
||||
${{ github.workspace }}/${{ matrix.platform.release_for }}-${{ matrix.platform.bin }}
|
||||
${{ github.workspace }}/${{ matrix.platform.name }}
|
||||
release:
|
||||
name: Create GitHub Release & Docker Image
|
||||
needs: build
|
||||
runs-on: ubuntu-20.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Download release artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
merge-multiple: true
|
||||
- name: Publish GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: bili-sync-rs*
|
||||
tag_name: ${{ github.ref_name }}
|
||||
draft: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKERHUB_USERNAME }}/bili-sync-rs
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=raw,value=${{ github.ref_name }}
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha, scope=${{ github.workflow }}
|
||||
cache-to: type=gha, scope=${{ github.workflow }}
|
||||
- name: Update DockerHub description
|
||||
uses: peter-evans/dockerhub-description@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
repository: ${{ secrets.DOCKERHUB_USERNAME }}/bili-sync-rs
|
||||
@@ -1,15 +1,16 @@
|
||||
name: Build Docs
|
||||
name: Build Main Docs
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
paths:
|
||||
- 'docs/**'
|
||||
|
||||
jobs:
|
||||
doc:
|
||||
if: ${{ github.ref == 'refs/heads/main' }}
|
||||
name: Build documentation
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
defaults:
|
||||
run:
|
||||
working-directory: docs
|
||||
10
.github/workflows/commit-build.yaml
vendored
Normal file
@@ -0,0 +1,10 @@
|
||||
name: Build Main Binary
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
jobs:
|
||||
build-binary:
|
||||
uses: amtoaer/bili-sync/.github/workflows/build-binary.yaml@main
|
||||
@@ -20,13 +20,13 @@ env:
|
||||
jobs:
|
||||
tests:
|
||||
name: Run Clippy and tests
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- run: rustup default nightly-2024-04-30 && rustup component add rustfmt clippy
|
||||
- run: rustup default nightly && rustup component add rustfmt clippy
|
||||
|
||||
- name: Cache dependencies
|
||||
uses: swatinem/rust-cache@v2
|
||||
78
.github/workflows/release-build.yaml
vendored
Normal file
@@ -0,0 +1,78 @@
|
||||
name: Build Main Binary And Release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
|
||||
jobs:
|
||||
build-binary:
|
||||
uses: amtoaer/bili-sync/.github/workflows/build-binary.yaml@main
|
||||
github-release:
|
||||
name: Create GitHub Release
|
||||
needs: build-binary
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Download release artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
merge-multiple: true
|
||||
- name: Publish GitHub release
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: bili-sync-rs*
|
||||
tag_name: ${{ github.ref_name }}
|
||||
draft: true
|
||||
docker-release:
|
||||
name: Create Docker Image
|
||||
needs: build-binary
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repo
|
||||
uses: actions/checkout@v4
|
||||
- name: Download release artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
merge-multiple: true
|
||||
- name: Docker Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ secrets.DOCKERHUB_USERNAME }}/bili-sync-rs
|
||||
tags: |
|
||||
type=raw,value=latest
|
||||
type=raw,value=${{ github.ref_name }}
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: Dockerfile
|
||||
platforms: |
|
||||
linux/amd64
|
||||
linux/arm64
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha, scope=${{ github.workflow }}
|
||||
cache-to: type=gha, scope=${{ github.workflow }}
|
||||
- name: Update DockerHub description
|
||||
uses: peter-evans/dockerhub-description@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
repository: ${{ secrets.DOCKERHUB_USERNAME }}/bili-sync-rs
|
||||
1539
Cargo.lock
generated
91
Cargo.toml
@@ -4,63 +4,74 @@ default-members = ["crates/bili_sync"]
|
||||
resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
version = "2.1.2"
|
||||
version = "2.4.1"
|
||||
authors = ["amtoaer <amtoaer@gmail.com>"]
|
||||
license = "MIT"
|
||||
description = "由 Rust & Tokio 驱动的哔哩哔哩同步工具"
|
||||
edition = "2021"
|
||||
edition = "2024"
|
||||
publish = false
|
||||
|
||||
[workspace.dependencies]
|
||||
bili_sync_entity = { path = "crates/bili_sync_entity" }
|
||||
bili_sync_migration = { path = "crates/bili_sync_migration" }
|
||||
|
||||
anyhow = { version = "1.0.86", features = ["backtrace"] }
|
||||
anyhow = { version = "1.0.96", features = ["backtrace"] }
|
||||
arc-swap = { version = "1.7.1", features = ["serde"] }
|
||||
async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
|
||||
async-stream = "0.3.5"
|
||||
async-trait = "0.1.81"
|
||||
chrono = { version = "0.4.38", features = ["serde"] }
|
||||
clap = { version = "4.5.9", features = ["env"] }
|
||||
assert_matches = "1.5.0"
|
||||
async-std = { version = "1.13.0", features = ["attributes", "tokio1"] }
|
||||
async-stream = "0.3.6"
|
||||
async-trait = "0.1.86"
|
||||
axum = { version = "0.8.1", features = ["macros"] }
|
||||
built = { version = "0.7.7", features = ["git2", "chrono"] }
|
||||
chrono = { version = "0.4.39", features = ["serde"] }
|
||||
clap = { version = "4.5.30", features = ["env", "string"] }
|
||||
cookie = "0.18.1"
|
||||
dirs = "5.0.1"
|
||||
filenamify = "0.1.1"
|
||||
cow-utils = "0.1.3"
|
||||
dirs = "6.0.0"
|
||||
enum_dispatch = "0.3.13"
|
||||
float-ord = "0.3.2"
|
||||
futures = "0.3.30"
|
||||
handlebars = "6.0.0"
|
||||
futures = "0.3.31"
|
||||
handlebars = "6.3.1"
|
||||
hex = "0.4.3"
|
||||
leaky-bucket = "1.1.2"
|
||||
md5 = "0.7.0"
|
||||
memchr = "2.7.4"
|
||||
once_cell = "1.19.0"
|
||||
prost = "0.13.1"
|
||||
quick-xml = { version = "0.36.0", features = ["async-tokio"] }
|
||||
mime_guess = "2.0.5"
|
||||
once_cell = "1.20.3"
|
||||
prost = "0.13.5"
|
||||
quick-xml = { version = "0.37.2", features = ["async-tokio"] }
|
||||
rand = "0.8.5"
|
||||
regex = "1.10.5"
|
||||
reqwest = { version = "0.12.5", features = [
|
||||
"charset",
|
||||
"cookies",
|
||||
"gzip",
|
||||
"http2",
|
||||
"json",
|
||||
"rustls-tls",
|
||||
"stream",
|
||||
regex = "1.11.1"
|
||||
reqwest = { version = "0.12.12", features = [
|
||||
"charset",
|
||||
"cookies",
|
||||
"gzip",
|
||||
"http2",
|
||||
"json",
|
||||
"rustls-tls",
|
||||
"stream",
|
||||
], default-features = false }
|
||||
rsa = { version = "0.9.6", features = ["sha2"] }
|
||||
sea-orm = { version = "0.12.15", features = [
|
||||
"macros",
|
||||
"runtime-tokio-rustls",
|
||||
"sqlx-sqlite",
|
||||
rsa = { version = "0.9.7", features = ["sha2"] }
|
||||
rust-embed = "8.5.0"
|
||||
sea-orm = { version = "1.1.5", features = [
|
||||
"macros",
|
||||
"runtime-tokio-rustls",
|
||||
"sqlx-sqlite",
|
||||
] }
|
||||
sea-orm-migration = { version = "0.12.15", features = [] }
|
||||
serde = { version = "1.0.204", features = ["derive"] }
|
||||
serde_json = "1.0.120"
|
||||
sea-orm-migration = { version = "1.1.5", features = [] }
|
||||
serde = { version = "1.0.218", features = ["derive"] }
|
||||
serde_json = "1.0.139"
|
||||
serde_urlencoded = "0.7.1"
|
||||
strum = { version = "0.26.3", features = ["derive"] }
|
||||
thiserror = "1.0.63"
|
||||
tokio = { version = "1.38.1", features = ["full"] }
|
||||
toml = "0.8.15"
|
||||
tracing = "0.1.40"
|
||||
tracing-subscriber = { version = "0.3.18", features = ["chrono"] }
|
||||
strum = { version = "0.27.1", features = ["derive"] }
|
||||
thiserror = "2.0.11"
|
||||
tokio = { version = "1.43.0", features = ["full"] }
|
||||
tokio-util = { version = "0.7.13", features = ["io", "rt"] }
|
||||
toml = "0.8.20"
|
||||
tower = "0.5.2"
|
||||
tracing = "0.1.41"
|
||||
tracing-subscriber = { version = "0.3.19", features = ["chrono"] }
|
||||
utoipa = { version = "5.3.1", features = ["axum_extras"] }
|
||||
utoipa-swagger-ui = { version = "9.0.0", features = ["axum", "vendored"] }
|
||||
|
||||
[workspace.metadata.release]
|
||||
release = false
|
||||
@@ -69,8 +80,8 @@ tag-prefix = ""
|
||||
pre-release-commit-message = "chore: 发布 bili-sync {{version}}"
|
||||
publish = false
|
||||
pre-release-replacements = [
|
||||
{ file = "../../docs/.vitepress/config.mts", search = "\"v[0-9\\.]+\"", replace = "\"v{{version}}\"", exactly = 1 },
|
||||
{ file = "../../docs/introduction.md", search = " v[0-9\\.]+,", replace = " v{{version}},", exactly = 1 },
|
||||
{ file = "../../docs/.vitepress/config.mts", search = "\"v[0-9\\.]+\"", replace = "\"v{{version}}\"", exactly = 1 },
|
||||
{ file = "../../docs/introduction.md", search = " v[0-9\\.]+,", replace = " v{{version}},", exactly = 1 },
|
||||
]
|
||||
|
||||
[profile.release]
|
||||
|
||||
@@ -9,12 +9,12 @@ RUN apk update && apk add --no-cache \
|
||||
tzdata \
|
||||
ffmpeg
|
||||
|
||||
COPY ./*-bili-sync-rs ./targets/
|
||||
COPY ./bili-sync-rs-Linux-*.tar.gz ./targets/
|
||||
|
||||
RUN if [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
mv ./targets/Linux-x86_64-bili-sync-rs ./bili-sync-rs; \
|
||||
tar xzvf ./targets/bili-sync-rs-Linux-x86_64-musl.tar.gz -C ./; \
|
||||
else \
|
||||
mv ./targets/Linux-aarch64-bili-sync-rs ./bili-sync-rs; \
|
||||
tar xzvf ./targets/bili-sync-rs-Linux-aarch64-musl.tar.gz -C ./; \
|
||||
fi
|
||||
|
||||
RUN rm -rf ./targets && chmod +x ./bili-sync-rs
|
||||
|
||||
25
Justfile
@@ -1,21 +1,24 @@
|
||||
clean:
|
||||
rm -rf ./*-bili-sync-rs
|
||||
rm -rf ./bili-sync-rs-Linux*.tar.gz
|
||||
|
||||
build:
|
||||
build-frontend:
|
||||
cd ./web && bun run build && cd ..
|
||||
|
||||
build: build-frontend
|
||||
cargo build --target x86_64-unknown-linux-musl --release
|
||||
|
||||
build-debug: build-frontend
|
||||
cargo build --target x86_64-unknown-linux-musl
|
||||
|
||||
build-docker: build
|
||||
cp target/x86_64-unknown-linux-musl/release/bili-sync-rs ./Linux-x86_64-bili-sync-rs
|
||||
tar czvf ./bili-sync-rs-Linux-x86_64-musl.tar.gz -C ./target/x86_64-unknown-linux-musl/release/ ./bili-sync-rs
|
||||
docker build . -t bili-sync-rs-local --build-arg="TARGETPLATFORM=linux/amd64"
|
||||
just clean
|
||||
|
||||
copy-config:
|
||||
rm -rf ~/.config/bili-sync
|
||||
cp -r ~/.config/nas/bili-sync-rs ~/.config/bili-sync
|
||||
sed -i -e 's/\/Bilibilis/\/Test_Bilibilis/g' -e 's/.config\/nas/.config\/test_nas/g' ~/.config/bili-sync/config.toml
|
||||
build-docker-debug: build-debug
|
||||
tar czvf ./bili-sync-rs-Linux-x86_64-musl.tar.gz -C ./target/x86_64-unknown-linux-musl/debug/ ./bili-sync-rs
|
||||
docker build . -t bili-sync-rs-local --build-arg="TARGETPLATFORM=linux/amd64"
|
||||
just clean
|
||||
|
||||
run:
|
||||
debug: build-frontend
|
||||
cargo run
|
||||
|
||||
debug: copy-config
|
||||
just run
|
||||
11
README.md
@@ -10,13 +10,13 @@ bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具,由 Rust
|
||||
## 效果演示
|
||||
|
||||
### 概览
|
||||

|
||||

|
||||
### 详情
|
||||

|
||||

|
||||
### 播放(使用 infuse)
|
||||

|
||||

|
||||
### 文件排布
|
||||

|
||||

|
||||
|
||||
|
||||
## 功能与路线图
|
||||
@@ -31,7 +31,8 @@ bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具,由 Rust
|
||||
- [x] 打印日志,并在请求出现风控时自动终止,等待下一轮执行
|
||||
- [x] 提供多平台的二进制可执行文件,为 Linux 平台提供了立即可用的 Docker 镜像
|
||||
- [x] 支持对“稍后再看”内视频的自动扫描与下载
|
||||
- [ ] 支持对 UP 主投稿视频的自动扫描与下载
|
||||
- [x] 支持对 UP 主投稿视频的自动扫描与下载
|
||||
- [x] 支持限制任务的并行度和接口请求频率
|
||||
- [ ] 下载单个文件时支持断点续传与并发下载
|
||||
|
||||
|
||||
|
||||
|
Before Width: | Height: | Size: 6.1 MiB |
BIN
assets/detail.webp
Normal file
|
After Width: | Height: | Size: 342 KiB |
BIN
assets/dir.png
|
Before Width: | Height: | Size: 1015 KiB |
BIN
assets/dir.webp
Normal file
|
After Width: | Height: | Size: 130 KiB |
|
Before Width: | Height: | Size: 4.6 MiB |
BIN
assets/overview.webp
Normal file
|
After Width: | Height: | Size: 270 KiB |
BIN
assets/play.png
|
Before Width: | Height: | Size: 2.4 MiB |
BIN
assets/play.webp
Normal file
|
After Width: | Height: | Size: 216 KiB |
@@ -7,25 +7,29 @@ license = { workspace = true }
|
||||
description = { workspace = true }
|
||||
publish = { workspace = true }
|
||||
readme = "../../README.md"
|
||||
build = "build.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = { workspace = true }
|
||||
arc-swap = { workspace = true }
|
||||
async-stream = { workspace = true }
|
||||
async-trait = { workspace = true }
|
||||
axum = { workspace = true }
|
||||
bili_sync_entity = { workspace = true }
|
||||
bili_sync_migration = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
cookie = { workspace = true }
|
||||
cow-utils = { workspace = true }
|
||||
dirs = { workspace = true }
|
||||
filenamify = { workspace = true }
|
||||
enum_dispatch = { workspace = true }
|
||||
float-ord = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
handlebars = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
leaky-bucket = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
memchr = { workspace = true }
|
||||
mime_guess = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
quick-xml = { workspace = true }
|
||||
@@ -33,6 +37,7 @@ rand = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
rsa = { workspace = true }
|
||||
rust-embed = { workspace = true }
|
||||
sea-orm = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
@@ -40,9 +45,19 @@ serde_urlencoded = { workspace = true }
|
||||
strum = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tokio-util = { workspace = true }
|
||||
toml = { workspace = true }
|
||||
tower = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tracing-subscriber = { workspace = true }
|
||||
utoipa = { workspace = true }
|
||||
utoipa-swagger-ui = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
assert_matches = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
built = { workspace = true }
|
||||
|
||||
[package.metadata.release]
|
||||
release = true
|
||||
|
||||
3
crates/bili_sync/build.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
fn main() {
|
||||
built::write_built_file().expect("Failed to acquire build-time information");
|
||||
}
|
||||
@@ -1,29 +1,85 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bili_sync_entity::*;
|
||||
use bili_sync_migration::OnConflict;
|
||||
use filenamify::filenamify;
|
||||
use futures::Stream;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::sea_query::{OnConflict, SimpleExpr};
|
||||
use sea_orm::{DatabaseConnection, Unchanged};
|
||||
|
||||
use crate::adapter::VideoListModel;
|
||||
use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo};
|
||||
use crate::config::TEMPLATE;
|
||||
use crate::utils::id_time_key;
|
||||
use crate::utils::model::create_video_pages;
|
||||
use crate::utils::status::Status;
|
||||
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
||||
use crate::bilibili::{BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
|
||||
|
||||
pub async fn collection_from<'a>(
|
||||
impl VideoSource for collection::Model {
|
||||
fn filter_expr(&self) -> SimpleExpr {
|
||||
video::Column::CollectionId.eq(self.id)
|
||||
}
|
||||
|
||||
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
|
||||
video_model.collection_id = Set(Some(self.id));
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
Path::new(self.path.as_str())
|
||||
}
|
||||
|
||||
fn get_latest_row_at(&self) -> DateTime {
|
||||
self.latest_row_at
|
||||
}
|
||||
|
||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
|
||||
_ActiveModel::Collection(collection::ActiveModel {
|
||||
id: Unchanged(self.id),
|
||||
latest_row_at: Set(datetime),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描{}「{}」..", CollectionType::from(self.r#type), self.name);
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, count: usize) {
|
||||
info!(
|
||||
"扫描{}「{}」完成,获取到 {} 条新视频",
|
||||
CollectionType::from(self.r#type),
|
||||
self.name,
|
||||
count,
|
||||
);
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!(
|
||||
"开始填充{}「{}」视频详情..",
|
||||
CollectionType::from(self.r#type),
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("填充{}「{}」视频详情完成", CollectionType::from(self.r#type), self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载{}「{}」视频..", CollectionType::from(self.r#type), self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载{}「{}」视频完成", CollectionType::from(self.r#type), self.name);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn collection_from<'a>(
|
||||
collection_item: &'a CollectionItem,
|
||||
path: &Path,
|
||||
bili_client: &'a BiliClient,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
|
||||
) -> Result<(
|
||||
VideoSourceEnum,
|
||||
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
)> {
|
||||
let collection = Collection::new(bili_client, collection_item);
|
||||
let collection_info = collection.get_info().await?;
|
||||
collection::Entity::insert(collection::ActiveModel {
|
||||
@@ -46,196 +102,17 @@ pub async fn collection_from<'a>(
|
||||
.exec(connection)
|
||||
.await?;
|
||||
Ok((
|
||||
Box::new(
|
||||
collection::Entity::find()
|
||||
.filter(
|
||||
collection::Column::SId
|
||||
.eq(collection_item.sid.clone())
|
||||
.and(collection::Column::MId.eq(collection_item.mid.clone()))
|
||||
.and(collection::Column::Type.eq(Into::<i32>::into(collection_item.collection_type.clone()))),
|
||||
)
|
||||
.one(connection)
|
||||
.await?
|
||||
.unwrap(),
|
||||
),
|
||||
Box::pin(collection.into_simple_video_stream()),
|
||||
collection::Entity::find()
|
||||
.filter(
|
||||
collection::Column::SId
|
||||
.eq(collection_item.sid.clone())
|
||||
.and(collection::Column::MId.eq(collection_item.mid.clone()))
|
||||
.and(collection::Column::Type.eq(Into::<i32>::into(collection_item.collection_type.clone()))),
|
||||
)
|
||||
.one(connection)
|
||||
.await?
|
||||
.context("collection not found")?
|
||||
.into(),
|
||||
Box::pin(collection.into_video_stream()),
|
||||
))
|
||||
}
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
impl VideoListModel for collection::Model {
|
||||
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(video::Column::CollectionId.eq(self.id))
|
||||
.count(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::CollectionId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.eq(0))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_null()),
|
||||
)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unhandled_video_pages(
|
||||
&self,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::CollectionId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.lt(Status::handled()))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_not_null()),
|
||||
)
|
||||
.find_with_related(page::Entity)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn exist_labels(
|
||||
&self,
|
||||
videos_info: &[VideoInfo],
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<HashSet<String>> {
|
||||
let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::<Vec<_>>();
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::CollectionId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Bvid.is_in(bvids)),
|
||||
)
|
||||
.select_only()
|
||||
.columns([video::Column::Bvid, video::Column::Pubtime])
|
||||
.into_tuple()
|
||||
.all(connection)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(bvid, time)| id_time_key(&bvid, &time))
|
||||
.collect::<HashSet<_>>())
|
||||
}
|
||||
|
||||
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
|
||||
let mut video_model = video_info.to_model(base_model);
|
||||
video_model.collection_id = Set(Some(self.id));
|
||||
if let Some(fmt_args) = &video_info.to_fmt_args() {
|
||||
video_model.path = Set(Path::new(&self.path)
|
||||
.join(filenamify(
|
||||
TEMPLATE
|
||||
.render("video", fmt_args)
|
||||
.unwrap_or_else(|_| video_info.bvid().to_string()),
|
||||
))
|
||||
.to_string_lossy()
|
||||
.to_string());
|
||||
}
|
||||
video_model
|
||||
}
|
||||
|
||||
async fn fetch_videos_detail(
|
||||
&self,
|
||||
bili_clent: &BiliClient,
|
||||
videos_model: Vec<video::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
for video_model in videos_model {
|
||||
let video = Video::new(bili_clent, video_model.bvid.clone());
|
||||
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
|
||||
match info {
|
||||
Ok((tags, view_info)) => {
|
||||
let VideoInfo::View { pages, .. } = &view_info else {
|
||||
unreachable!("view_info must be VideoInfo::View")
|
||||
};
|
||||
let txn = connection.begin().await?;
|
||||
// 将分页信息写入数据库
|
||||
create_video_pages(pages, &video_model, &txn).await?;
|
||||
// 将页标记和 tag 写入数据库
|
||||
let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model));
|
||||
video_active_model.single_page = Set(Some(pages.len() == 1));
|
||||
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
|
||||
video_active_model.save(&txn).await?;
|
||||
txn.commit().await?;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"获取视频 {} - {} 的详细信息失败,错误为:{}",
|
||||
&video_model.bvid, &video_model.name, e
|
||||
);
|
||||
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.valid = Set(false);
|
||||
video_active_model.save(connection).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!(
|
||||
"开始获取{} {} - {} 的视频与分页信息...",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!(
|
||||
"获取{} {} - {} 的视频与分页信息完成",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!(
|
||||
"开始下载{}: {} - {} 中所有未处理过的视频...",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!(
|
||||
"下载{}: {} - {} 中未处理过的视频完成",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!(
|
||||
"开始扫描{}: {} - {} 的新视频...",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name
|
||||
);
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
|
||||
info!(
|
||||
"扫描{}: {} - {} 的新视频完成,获取了 {} 条新视频,其中有 {} 条新视频",
|
||||
CollectionType::from(self.r#type),
|
||||
self.s_id,
|
||||
self.name,
|
||||
got_count,
|
||||
new_count,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +1,76 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bili_sync_entity::*;
|
||||
use bili_sync_migration::OnConflict;
|
||||
use filenamify::filenamify;
|
||||
use futures::Stream;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::sea_query::{OnConflict, SimpleExpr};
|
||||
use sea_orm::{DatabaseConnection, Unchanged};
|
||||
|
||||
use crate::adapter::VideoListModel;
|
||||
use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo};
|
||||
use crate::config::TEMPLATE;
|
||||
use crate::utils::id_time_key;
|
||||
use crate::utils::model::create_video_pages;
|
||||
use crate::utils::status::Status;
|
||||
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
||||
use crate::bilibili::{BiliClient, FavoriteList, VideoInfo};
|
||||
|
||||
pub async fn favorite_from<'a>(
|
||||
impl VideoSource for favorite::Model {
|
||||
fn filter_expr(&self) -> SimpleExpr {
|
||||
video::Column::FavoriteId.eq(self.id)
|
||||
}
|
||||
|
||||
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
|
||||
video_model.favorite_id = Set(Some(self.id));
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
Path::new(self.path.as_str())
|
||||
}
|
||||
|
||||
fn get_latest_row_at(&self) -> DateTime {
|
||||
self.latest_row_at
|
||||
}
|
||||
|
||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
|
||||
_ActiveModel::Favorite(favorite::ActiveModel {
|
||||
id: Unchanged(self.id),
|
||||
latest_row_at: Set(datetime),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描收藏夹「{}」..", self.name);
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, count: usize) {
|
||||
info!("扫描收藏夹「{}」完成,获取到 {} 条新视频", self.name, count);
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!("开始填充收藏夹「{}」视频详情..", self.name);
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("填充收藏夹「{}」视频详情完成", self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载收藏夹「{}」视频..", self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载收藏夹「{}」视频完成", self.name);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn favorite_from<'a>(
|
||||
fid: &str,
|
||||
path: &Path,
|
||||
bili_client: &'a BiliClient,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
|
||||
) -> Result<(
|
||||
VideoSourceEnum,
|
||||
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
)> {
|
||||
let favorite = FavoriteList::new(bili_client, fid.to_owned());
|
||||
let favorite_info = favorite.get_info().await?;
|
||||
favorite::Entity::insert(favorite::ActiveModel {
|
||||
@@ -40,160 +87,12 @@ pub async fn favorite_from<'a>(
|
||||
.exec(connection)
|
||||
.await?;
|
||||
Ok((
|
||||
Box::new(
|
||||
favorite::Entity::find()
|
||||
.filter(favorite::Column::FId.eq(favorite_info.id))
|
||||
.one(connection)
|
||||
.await?
|
||||
.unwrap(),
|
||||
),
|
||||
favorite::Entity::find()
|
||||
.filter(favorite::Column::FId.eq(favorite_info.id))
|
||||
.one(connection)
|
||||
.await?
|
||||
.context("favorite not found")?
|
||||
.into(),
|
||||
Box::pin(favorite.into_video_stream()),
|
||||
))
|
||||
}
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
impl VideoListModel for favorite::Model {
|
||||
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(video::Column::FavoriteId.eq(self.id))
|
||||
.count(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::FavoriteId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.eq(0))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_null()),
|
||||
)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unhandled_video_pages(
|
||||
&self,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::FavoriteId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.lt(Status::handled()))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_not_null()),
|
||||
)
|
||||
.find_with_related(page::Entity)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn exist_labels(
|
||||
&self,
|
||||
videos_info: &[VideoInfo],
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<HashSet<String>> {
|
||||
let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::<Vec<_>>();
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::FavoriteId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Bvid.is_in(bvids)),
|
||||
)
|
||||
.select_only()
|
||||
.columns([video::Column::Bvid, video::Column::Favtime])
|
||||
.into_tuple()
|
||||
.all(connection)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(bvid, time)| id_time_key(&bvid, &time))
|
||||
.collect::<HashSet<_>>())
|
||||
}
|
||||
|
||||
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
|
||||
let mut video_model = video_info.to_model(base_model);
|
||||
video_model.favorite_id = Set(Some(self.id));
|
||||
if let Some(fmt_args) = &video_info.to_fmt_args() {
|
||||
video_model.path = Set(Path::new(&self.path)
|
||||
.join(filenamify(
|
||||
TEMPLATE
|
||||
.render("video", fmt_args)
|
||||
.unwrap_or_else(|_| video_info.bvid().to_string()),
|
||||
))
|
||||
.to_string_lossy()
|
||||
.to_string());
|
||||
}
|
||||
video_model
|
||||
}
|
||||
|
||||
async fn fetch_videos_detail(
|
||||
&self,
|
||||
bili_clent: &BiliClient,
|
||||
videos_model: Vec<video::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
for video_model in videos_model {
|
||||
let video = Video::new(bili_clent, video_model.bvid.clone());
|
||||
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await;
|
||||
match info {
|
||||
Ok((tags, pages_info)) => {
|
||||
let txn = connection.begin().await?;
|
||||
// 将分页信息写入数据库
|
||||
create_video_pages(&pages_info, &video_model, &txn).await?;
|
||||
// 将页标记和 tag 写入数据库
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.single_page = Set(Some(pages_info.len() == 1));
|
||||
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
|
||||
video_active_model.save(&txn).await?;
|
||||
txn.commit().await?;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"获取视频 {} - {} 的详细信息失败,错误为:{}",
|
||||
&video_model.bvid, &video_model.name, e
|
||||
);
|
||||
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.valid = Set(false);
|
||||
video_active_model.save(connection).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!("开始获取收藏夹 {} - {} 的视频与分页信息...", self.f_id, self.name);
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("获取收藏夹 {} - {} 的视频与分页信息完成", self.f_id, self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载收藏夹: {} - {} 中所有未处理过的视频...", self.f_id, self.name);
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载收藏夹: {} - {} 中未处理过的视频完成", self.f_id, self.name);
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描收藏夹: {} - {} 的新视频...", self.f_id, self.name);
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
|
||||
info!(
|
||||
"扫描收藏夹: {} - {} 的新视频完成,获取了 {} 条新视频,其中有 {} 条新视频",
|
||||
self.f_id, self.name, got_count, new_count
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,85 +1,124 @@
|
||||
mod collection;
|
||||
mod favorite;
|
||||
mod submission;
|
||||
mod watch_later;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::Result;
|
||||
use async_trait::async_trait;
|
||||
pub use collection::collection_from;
|
||||
pub use favorite::favorite_from;
|
||||
use enum_dispatch::enum_dispatch;
|
||||
use futures::Stream;
|
||||
use sea_orm::DatabaseConnection;
|
||||
use watch_later::watch_later_from;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::sea_query::SimpleExpr;
|
||||
|
||||
#[rustfmt::skip]
|
||||
use bili_sync_entity::collection::Model as Collection;
|
||||
use bili_sync_entity::favorite::Model as Favorite;
|
||||
use bili_sync_entity::submission::Model as Submission;
|
||||
use bili_sync_entity::watch_later::Model as WatchLater;
|
||||
|
||||
use crate::adapter::collection::collection_from;
|
||||
use crate::adapter::favorite::favorite_from;
|
||||
use crate::adapter::submission::submission_from;
|
||||
use crate::adapter::watch_later::watch_later_from;
|
||||
use crate::bilibili::{BiliClient, CollectionItem, VideoInfo};
|
||||
|
||||
#[enum_dispatch]
|
||||
pub enum VideoSourceEnum {
|
||||
Favorite,
|
||||
Collection,
|
||||
Submission,
|
||||
WatchLater,
|
||||
}
|
||||
|
||||
#[enum_dispatch(VideoSourceEnum)]
|
||||
pub trait VideoSource {
|
||||
/// 获取特定视频列表的筛选条件
|
||||
fn filter_expr(&self) -> SimpleExpr;
|
||||
|
||||
// 为 video_model 设置该视频列表的关联 id
|
||||
fn set_relation_id(&self, video_model: &mut bili_sync_entity::video::ActiveModel);
|
||||
|
||||
// 获取视频列表的保存路径
|
||||
fn path(&self) -> &Path;
|
||||
|
||||
/// 获取视频 model 中记录的最新时间
|
||||
fn get_latest_row_at(&self) -> DateTime;
|
||||
|
||||
/// 更新视频 model 中记录的最新时间,此处返回需要更新的 ActiveModel,接着调用 save 方法执行保存
|
||||
/// 不同 VideoSource 返回的类型不同,为了 VideoSource 的 object safety 不能使用 impl Trait
|
||||
/// Box<dyn ActiveModelTrait> 又提示 ActiveModelTrait 没有 object safety,因此手写一个 Enum 静态分发
|
||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel;
|
||||
|
||||
/// 开始刷新视频
|
||||
fn log_refresh_video_start(&self);
|
||||
|
||||
/// 结束刷新视频
|
||||
fn log_refresh_video_end(&self, count: usize);
|
||||
|
||||
/// 开始填充视频
|
||||
fn log_fetch_video_start(&self);
|
||||
|
||||
/// 结束填充视频
|
||||
fn log_fetch_video_end(&self);
|
||||
|
||||
/// 开始下载视频
|
||||
fn log_download_video_start(&self);
|
||||
|
||||
/// 结束下载视频
|
||||
fn log_download_video_end(&self);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub enum Args<'a> {
|
||||
Favorite { fid: &'a str },
|
||||
Collection { collection_item: &'a CollectionItem },
|
||||
WatchLater,
|
||||
Submission { upper_id: &'a str },
|
||||
}
|
||||
|
||||
pub async fn video_list_from<'a>(
|
||||
pub async fn video_source_from<'a>(
|
||||
args: Args<'a>,
|
||||
path: &Path,
|
||||
bili_client: &'a BiliClient,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
|
||||
) -> Result<(
|
||||
VideoSourceEnum,
|
||||
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
)> {
|
||||
match args {
|
||||
Args::Favorite { fid } => favorite_from(fid, path, bili_client, connection).await,
|
||||
Args::Collection { collection_item } => collection_from(collection_item, path, bili_client, connection).await,
|
||||
Args::WatchLater => watch_later_from(path, bili_client, connection).await,
|
||||
Args::Submission { upper_id } => submission_from(upper_id, path, bili_client, connection).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub trait VideoListModel {
|
||||
/* 逻辑相关 */
|
||||
|
||||
/// 获取与视频列表关联的视频总数
|
||||
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64>;
|
||||
|
||||
/// 获取未填充的视频
|
||||
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<bili_sync_entity::video::Model>>;
|
||||
|
||||
/// 获取未处理的视频和分页
|
||||
async fn unhandled_video_pages(
|
||||
&self,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Vec<(bili_sync_entity::video::Model, Vec<bili_sync_entity::page::Model>)>>;
|
||||
|
||||
/// 获取该批次视频的存在标记
|
||||
async fn exist_labels(&self, videos_info: &[VideoInfo], connection: &DatabaseConnection)
|
||||
-> Result<HashSet<String>>;
|
||||
|
||||
/// 获取视频信息对应的视频 model
|
||||
fn video_model_by_info(
|
||||
&self,
|
||||
video_info: &VideoInfo,
|
||||
base_model: Option<bili_sync_entity::video::Model>,
|
||||
) -> bili_sync_entity::video::ActiveModel;
|
||||
|
||||
/// 获取视频 model 中缺失的信息
|
||||
async fn fetch_videos_detail(
|
||||
&self,
|
||||
bili_client: &BiliClient,
|
||||
videos_model: Vec<bili_sync_entity::video::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()>;
|
||||
|
||||
/* 日志相关 */
|
||||
fn log_fetch_video_start(&self);
|
||||
|
||||
fn log_fetch_video_end(&self);
|
||||
|
||||
fn log_download_video_start(&self);
|
||||
|
||||
fn log_download_video_end(&self);
|
||||
|
||||
fn log_refresh_video_start(&self);
|
||||
|
||||
fn log_refresh_video_end(&self, got_count: usize, new_count: u64);
|
||||
pub enum _ActiveModel {
|
||||
Favorite(bili_sync_entity::favorite::ActiveModel),
|
||||
Collection(bili_sync_entity::collection::ActiveModel),
|
||||
Submission(bili_sync_entity::submission::ActiveModel),
|
||||
WatchLater(bili_sync_entity::watch_later::ActiveModel),
|
||||
}
|
||||
|
||||
impl _ActiveModel {
|
||||
pub async fn save(self, connection: &DatabaseConnection) -> Result<()> {
|
||||
match self {
|
||||
_ActiveModel::Favorite(model) => {
|
||||
model.save(connection).await?;
|
||||
}
|
||||
_ActiveModel::Collection(model) => {
|
||||
model.save(connection).await?;
|
||||
}
|
||||
_ActiveModel::Submission(model) => {
|
||||
model.save(connection).await?;
|
||||
}
|
||||
_ActiveModel::WatchLater(model) => {
|
||||
model.save(connection).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
98
crates/bili_sync/src/adapter/submission.rs
Normal file
@@ -0,0 +1,98 @@
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use bili_sync_entity::*;
|
||||
use futures::Stream;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::sea_query::{OnConflict, SimpleExpr};
|
||||
use sea_orm::{DatabaseConnection, Unchanged};
|
||||
|
||||
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
||||
use crate::bilibili::{BiliClient, Submission, VideoInfo};
|
||||
|
||||
impl VideoSource for submission::Model {
|
||||
fn filter_expr(&self) -> SimpleExpr {
|
||||
video::Column::SubmissionId.eq(self.id)
|
||||
}
|
||||
|
||||
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
|
||||
video_model.submission_id = Set(Some(self.id));
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
Path::new(self.path.as_str())
|
||||
}
|
||||
|
||||
fn get_latest_row_at(&self) -> DateTime {
|
||||
self.latest_row_at
|
||||
}
|
||||
|
||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
|
||||
_ActiveModel::Submission(submission::ActiveModel {
|
||||
id: Unchanged(self.id),
|
||||
latest_row_at: Set(datetime),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描「{}」投稿..", self.upper_name);
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, count: usize) {
|
||||
info!("扫描「{}」投稿完成,获取到 {} 条新视频", self.upper_name, count,);
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!("开始填充「{}」投稿视频详情..", self.upper_name);
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("填充「{}」投稿视频详情完成", self.upper_name);
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载「{}」投稿视频..", self.upper_name);
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载「{}」投稿视频完成", self.upper_name);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn submission_from<'a>(
|
||||
upper_id: &str,
|
||||
path: &Path,
|
||||
bili_client: &'a BiliClient,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<(
|
||||
VideoSourceEnum,
|
||||
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
)> {
|
||||
let submission = Submission::new(bili_client, upper_id.to_owned());
|
||||
let upper = submission.get_info().await?;
|
||||
submission::Entity::insert(submission::ActiveModel {
|
||||
upper_id: Set(upper.mid.parse()?),
|
||||
upper_name: Set(upper.name),
|
||||
path: Set(path.to_string_lossy().to_string()),
|
||||
..Default::default()
|
||||
})
|
||||
.on_conflict(
|
||||
OnConflict::column(submission::Column::UpperId)
|
||||
.update_columns([submission::Column::UpperName, submission::Column::Path])
|
||||
.to_owned(),
|
||||
)
|
||||
.exec(connection)
|
||||
.await?;
|
||||
Ok((
|
||||
submission::Entity::find()
|
||||
.filter(submission::Column::UpperId.eq(upper.mid))
|
||||
.one(connection)
|
||||
.await?
|
||||
.context("submission not found")?
|
||||
.into(),
|
||||
Box::pin(submission.into_video_stream()),
|
||||
))
|
||||
}
|
||||
@@ -1,28 +1,75 @@
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bili_sync_entity::*;
|
||||
use bili_sync_migration::OnConflict;
|
||||
use filenamify::filenamify;
|
||||
use futures::Stream;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::sea_query::{OnConflict, SimpleExpr};
|
||||
use sea_orm::{DatabaseConnection, Unchanged};
|
||||
|
||||
use crate::adapter::VideoListModel;
|
||||
use crate::bilibili::{BiliClient, BiliError, Video, VideoInfo, WatchLater};
|
||||
use crate::config::TEMPLATE;
|
||||
use crate::utils::id_time_key;
|
||||
use crate::utils::model::create_video_pages;
|
||||
use crate::utils::status::Status;
|
||||
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
||||
use crate::bilibili::{BiliClient, VideoInfo, WatchLater};
|
||||
|
||||
pub async fn watch_later_from<'a>(
|
||||
impl VideoSource for watch_later::Model {
|
||||
fn filter_expr(&self) -> SimpleExpr {
|
||||
video::Column::WatchLaterId.eq(self.id)
|
||||
}
|
||||
|
||||
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
|
||||
video_model.watch_later_id = Set(Some(self.id));
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
Path::new(self.path.as_str())
|
||||
}
|
||||
|
||||
fn get_latest_row_at(&self) -> DateTime {
|
||||
self.latest_row_at
|
||||
}
|
||||
|
||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
|
||||
_ActiveModel::WatchLater(watch_later::ActiveModel {
|
||||
id: Unchanged(self.id),
|
||||
latest_row_at: Set(datetime),
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描稍后再看..");
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, count: usize) {
|
||||
info!("扫描稍后再看完成,获取到 {} 条新视频", count);
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!("开始填充稍后再看视频详情..");
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("填充稍后再看视频详情完成");
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载稍后再看视频..");
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载稍后再看视频完成");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn watch_later_from<'a>(
|
||||
path: &Path,
|
||||
bili_client: &'a BiliClient,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<(Box<dyn VideoListModel>, Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>)> {
|
||||
) -> Result<(
|
||||
VideoSourceEnum,
|
||||
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
)> {
|
||||
let watch_later = WatchLater::new(bili_client);
|
||||
watch_later::Entity::insert(watch_later::ActiveModel {
|
||||
id: Set(1),
|
||||
@@ -37,159 +84,12 @@ pub async fn watch_later_from<'a>(
|
||||
.exec(connection)
|
||||
.await?;
|
||||
Ok((
|
||||
Box::new(
|
||||
watch_later::Entity::find()
|
||||
.filter(watch_later::Column::Id.eq(1))
|
||||
.one(connection)
|
||||
.await?
|
||||
.unwrap(),
|
||||
),
|
||||
watch_later::Entity::find()
|
||||
.filter(watch_later::Column::Id.eq(1))
|
||||
.one(connection)
|
||||
.await?
|
||||
.context("watch_later not found")?
|
||||
.into(),
|
||||
Box::pin(watch_later.into_video_stream()),
|
||||
))
|
||||
}
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
impl VideoListModel for watch_later::Model {
|
||||
async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(video::Column::WatchLaterId.eq(self.id))
|
||||
.count(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::WatchLaterId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.eq(0))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_null()),
|
||||
)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn unhandled_video_pages(
|
||||
&self,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::WatchLaterId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Valid.eq(true))
|
||||
.and(video::Column::DownloadStatus.lt(Status::handled()))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_not_null()),
|
||||
)
|
||||
.find_with_related(page::Entity)
|
||||
.all(connection)
|
||||
.await?)
|
||||
}
|
||||
|
||||
async fn exist_labels(
|
||||
&self,
|
||||
videos_info: &[VideoInfo],
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<HashSet<String>> {
|
||||
let bvids = videos_info.iter().map(|v| v.bvid().to_string()).collect::<Vec<_>>();
|
||||
Ok(video::Entity::find()
|
||||
.filter(
|
||||
video::Column::WatchLaterId
|
||||
.eq(self.id)
|
||||
.and(video::Column::Bvid.is_in(bvids)),
|
||||
)
|
||||
.select_only()
|
||||
.columns([video::Column::Bvid, video::Column::Favtime])
|
||||
.into_tuple()
|
||||
.all(connection)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(|(bvid, time)| id_time_key(&bvid, &time))
|
||||
.collect::<HashSet<_>>())
|
||||
}
|
||||
|
||||
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
|
||||
let mut video_model = video_info.to_model(base_model);
|
||||
video_model.watch_later_id = Set(Some(self.id));
|
||||
if let Some(fmt_args) = &video_info.to_fmt_args() {
|
||||
video_model.path = Set(Path::new(&self.path)
|
||||
.join(filenamify(
|
||||
TEMPLATE
|
||||
.render("video", fmt_args)
|
||||
.unwrap_or_else(|_| video_info.bvid().to_string()),
|
||||
))
|
||||
.to_string_lossy()
|
||||
.to_string());
|
||||
}
|
||||
video_model
|
||||
}
|
||||
|
||||
async fn fetch_videos_detail(
|
||||
&self,
|
||||
bili_clent: &BiliClient,
|
||||
videos_model: Vec<video::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
for video_model in videos_model {
|
||||
let video = Video::new(bili_clent, video_model.bvid.clone());
|
||||
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await;
|
||||
match info {
|
||||
Ok((tags, pages_info)) => {
|
||||
let txn = connection.begin().await?;
|
||||
// 将分页信息写入数据库
|
||||
create_video_pages(&pages_info, &video_model, &txn).await?;
|
||||
// 将页标记和 tag 写入数据库
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.single_page = Set(Some(pages_info.len() == 1));
|
||||
video_active_model.tags = Set(Some(serde_json::to_value(tags).unwrap()));
|
||||
video_active_model.save(&txn).await?;
|
||||
txn.commit().await?;
|
||||
}
|
||||
Err(e) => {
|
||||
error!(
|
||||
"获取视频 {} - {} 的详细信息失败,错误为:{}",
|
||||
&video_model.bvid, &video_model.name, e
|
||||
);
|
||||
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.valid = Set(false);
|
||||
video_active_model.save(connection).await?;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn log_fetch_video_start(&self) {
|
||||
info!("开始获取稍后再看的视频与分页信息...");
|
||||
}
|
||||
|
||||
fn log_fetch_video_end(&self) {
|
||||
info!("获取稍后再看的视频与分页信息完成");
|
||||
}
|
||||
|
||||
fn log_download_video_start(&self) {
|
||||
info!("开始下载稍后再看中所有未处理过的视频...");
|
||||
}
|
||||
|
||||
fn log_download_video_end(&self) {
|
||||
info!("下载稍后再看中未处理过的视频完成");
|
||||
}
|
||||
|
||||
fn log_refresh_video_start(&self) {
|
||||
info!("开始扫描稍后再看的新视频...");
|
||||
}
|
||||
|
||||
fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
|
||||
info!(
|
||||
"扫描稍后再看的新视频完成,获取了 {} 条新视频,其中有 {} 条新视频",
|
||||
got_count, new_count,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
40
crates/bili_sync/src/api/auth.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
use axum::extract::Request;
|
||||
use axum::http::HeaderMap;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use reqwest::StatusCode;
|
||||
use utoipa::Modify;
|
||||
use utoipa::openapi::security::{ApiKey, ApiKeyValue, SecurityScheme};
|
||||
|
||||
use crate::api::wrapper::ApiResponse;
|
||||
use crate::config::CONFIG;
|
||||
|
||||
pub async fn auth(headers: HeaderMap, request: Request, next: Next) -> Result<Response, StatusCode> {
|
||||
if request.uri().path().starts_with("/api/") && get_token(&headers) != CONFIG.auth_token {
|
||||
return Ok(ApiResponse::unauthorized(()).into_response());
|
||||
}
|
||||
Ok(next.run(request).await)
|
||||
}
|
||||
|
||||
fn get_token(headers: &HeaderMap) -> Option<String> {
|
||||
headers
|
||||
.get("Authorization")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(Into::into)
|
||||
}
|
||||
|
||||
pub(super) struct OpenAPIAuth;
|
||||
|
||||
impl Modify for OpenAPIAuth {
|
||||
fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {
|
||||
if let Some(schema) = openapi.components.as_mut() {
|
||||
schema.add_security_scheme(
|
||||
"Token",
|
||||
SecurityScheme::ApiKey(ApiKey::Header(ApiKeyValue::with_description(
|
||||
"Authorization",
|
||||
"与配置文件中的 auth_token 相同",
|
||||
))),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
7
crates/bili_sync/src/api/error.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum InnerApiError {
|
||||
#[error("Primary key not found: {0}")]
|
||||
NotFound(i32),
|
||||
}
|
||||
252
crates/bili_sync/src/api/handler.rs
Normal file
@@ -0,0 +1,252 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Result, anyhow};
|
||||
use axum::extract::{Extension, Path, Query};
|
||||
use bili_sync_entity::*;
|
||||
use bili_sync_migration::{Expr, OnConflict};
|
||||
use sea_orm::{
|
||||
ColumnTrait, DatabaseConnection, EntityTrait, IntoActiveModel, PaginatorTrait, QueryFilter, QueryOrder,
|
||||
QuerySelect, Set, TransactionTrait, Unchanged,
|
||||
};
|
||||
use utoipa::OpenApi;
|
||||
|
||||
use crate::api::auth::OpenAPIAuth;
|
||||
use crate::api::error::InnerApiError;
|
||||
use crate::api::request::VideosRequest;
|
||||
use crate::api::response::{
|
||||
PageInfo, ResetVideoResponse, VideoInfo, VideoResponse, VideoSource, VideoSourcesResponse, VideosResponse,
|
||||
};
|
||||
use crate::api::wrapper::{ApiError, ApiResponse};
|
||||
use crate::utils::status::{PageStatus, VideoStatus};
|
||||
|
||||
#[derive(OpenApi)]
|
||||
#[openapi(
|
||||
paths(get_video_sources, get_videos, get_video, reset_video),
|
||||
modifiers(&OpenAPIAuth),
|
||||
security(
|
||||
("Token" = []),
|
||||
)
|
||||
)]
|
||||
pub struct ApiDoc;
|
||||
|
||||
/// 列出所有视频来源
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/video-sources",
|
||||
responses(
|
||||
(status = 200, body = ApiResponse<VideoSourcesResponse>),
|
||||
)
|
||||
)]
|
||||
pub async fn get_video_sources(
|
||||
Extension(db): Extension<Arc<DatabaseConnection>>,
|
||||
) -> Result<ApiResponse<VideoSourcesResponse>, ApiError> {
|
||||
Ok(ApiResponse::ok(VideoSourcesResponse {
|
||||
collection: collection::Entity::find()
|
||||
.select_only()
|
||||
.columns([collection::Column::Id, collection::Column::Name])
|
||||
.into_model::<VideoSource>()
|
||||
.all(db.as_ref())
|
||||
.await?,
|
||||
favorite: favorite::Entity::find()
|
||||
.select_only()
|
||||
.columns([favorite::Column::Id, favorite::Column::Name])
|
||||
.into_model::<VideoSource>()
|
||||
.all(db.as_ref())
|
||||
.await?,
|
||||
submission: submission::Entity::find()
|
||||
.select_only()
|
||||
.column(submission::Column::Id)
|
||||
.column_as(submission::Column::UpperName, "name")
|
||||
.into_model::<VideoSource>()
|
||||
.all(db.as_ref())
|
||||
.await?,
|
||||
watch_later: watch_later::Entity::find()
|
||||
.select_only()
|
||||
.column(watch_later::Column::Id)
|
||||
.column_as(Expr::value("稍后再看"), "name")
|
||||
.into_model::<VideoSource>()
|
||||
.all(db.as_ref())
|
||||
.await?,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 列出视频的基本信息,支持根据视频来源筛选、名称查找和分页
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/videos",
|
||||
params(
|
||||
VideosRequest,
|
||||
),
|
||||
responses(
|
||||
(status = 200, body = ApiResponse<VideosResponse>),
|
||||
)
|
||||
)]
|
||||
pub async fn get_videos(
|
||||
Extension(db): Extension<Arc<DatabaseConnection>>,
|
||||
Query(params): Query<VideosRequest>,
|
||||
) -> Result<ApiResponse<VideosResponse>, ApiError> {
|
||||
let mut query = video::Entity::find();
|
||||
for (field, column) in [
|
||||
(params.collection, video::Column::CollectionId),
|
||||
(params.favorite, video::Column::FavoriteId),
|
||||
(params.submission, video::Column::SubmissionId),
|
||||
(params.watch_later, video::Column::WatchLaterId),
|
||||
] {
|
||||
if let Some(id) = field {
|
||||
query = query.filter(column.eq(id));
|
||||
}
|
||||
}
|
||||
if let Some(query_word) = params.query {
|
||||
query = query.filter(video::Column::Name.contains(query_word));
|
||||
}
|
||||
let total_count = query.clone().count(db.as_ref()).await?;
|
||||
let (page, page_size) = if let (Some(page), Some(page_size)) = (params.page, params.page_size) {
|
||||
(page, page_size)
|
||||
} else {
|
||||
(1, 10)
|
||||
};
|
||||
Ok(ApiResponse::ok(VideosResponse {
|
||||
videos: query
|
||||
.order_by_desc(video::Column::Id)
|
||||
.select_only()
|
||||
.columns([
|
||||
video::Column::Id,
|
||||
video::Column::Name,
|
||||
video::Column::UpperName,
|
||||
video::Column::DownloadStatus,
|
||||
])
|
||||
.into_tuple::<(i32, String, String, u32)>()
|
||||
.paginate(db.as_ref(), page_size)
|
||||
.fetch_page(page)
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(VideoInfo::from)
|
||||
.collect(),
|
||||
total_count,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 获取视频详细信息,包括关联的所有 page
|
||||
#[utoipa::path(
|
||||
get,
|
||||
path = "/api/videos/{id}",
|
||||
responses(
|
||||
(status = 200, body = ApiResponse<VideoResponse>),
|
||||
)
|
||||
)]
|
||||
pub async fn get_video(
|
||||
Path(id): Path<i32>,
|
||||
Extension(db): Extension<Arc<DatabaseConnection>>,
|
||||
) -> Result<ApiResponse<VideoResponse>, ApiError> {
|
||||
let video_info = video::Entity::find_by_id(id)
|
||||
.select_only()
|
||||
.columns([
|
||||
video::Column::Id,
|
||||
video::Column::Name,
|
||||
video::Column::UpperName,
|
||||
video::Column::DownloadStatus,
|
||||
])
|
||||
.into_tuple::<(i32, String, String, u32)>()
|
||||
.one(db.as_ref())
|
||||
.await?
|
||||
.map(VideoInfo::from);
|
||||
let Some(video_info) = video_info else {
|
||||
return Err(InnerApiError::NotFound(id).into());
|
||||
};
|
||||
let pages = page::Entity::find()
|
||||
.filter(page::Column::VideoId.eq(id))
|
||||
.order_by_asc(page::Column::Pid)
|
||||
.select_only()
|
||||
.columns([
|
||||
page::Column::Id,
|
||||
page::Column::Pid,
|
||||
page::Column::Name,
|
||||
page::Column::DownloadStatus,
|
||||
])
|
||||
.into_tuple::<(i32, i32, String, u32)>()
|
||||
.all(db.as_ref())
|
||||
.await?
|
||||
.into_iter()
|
||||
.map(PageInfo::from)
|
||||
.collect();
|
||||
Ok(ApiResponse::ok(VideoResponse {
|
||||
video: video_info,
|
||||
pages,
|
||||
}))
|
||||
}
|
||||
|
||||
/// 将某个视频与其所有分页的失败状态清空为未下载状态,这样在下次下载任务中会触发重试
|
||||
#[utoipa::path(
|
||||
post,
|
||||
path = "/api/videos/{id}/reset",
|
||||
responses(
|
||||
(status = 200, body = ApiResponse<ResetVideoResponse> ),
|
||||
)
|
||||
)]
|
||||
pub async fn reset_video(
|
||||
Path(id): Path<i32>,
|
||||
Extension(db): Extension<Arc<DatabaseConnection>>,
|
||||
) -> Result<ApiResponse<ResetVideoResponse>, ApiError> {
|
||||
let txn = db.begin().await?;
|
||||
let video_status: Option<u32> = video::Entity::find_by_id(id)
|
||||
.select_only()
|
||||
.column(video::Column::DownloadStatus)
|
||||
.into_tuple()
|
||||
.one(&txn)
|
||||
.await?;
|
||||
let Some(video_status) = video_status else {
|
||||
return Err(anyhow!(InnerApiError::NotFound(id)).into());
|
||||
};
|
||||
let resetted_pages_model: Vec<_> = page::Entity::find()
|
||||
.filter(page::Column::VideoId.eq(id))
|
||||
.all(&txn)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter_map(|mut model| {
|
||||
let mut page_status = PageStatus::from(model.download_status);
|
||||
if page_status.reset_failed() {
|
||||
model.download_status = page_status.into();
|
||||
Some(model)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let mut video_status = VideoStatus::from(video_status);
|
||||
let mut should_update_video = video_status.reset_failed();
|
||||
if !resetted_pages_model.is_empty() {
|
||||
// 视频状态标志的第 5 位表示是否有分 P 下载失败,如果有需要重置的分页,需要同时重置视频的该状态
|
||||
video_status.set(4, 0);
|
||||
should_update_video = true;
|
||||
}
|
||||
if should_update_video {
|
||||
video::Entity::update(video::ActiveModel {
|
||||
id: Unchanged(id),
|
||||
download_status: Set(video_status.into()),
|
||||
..Default::default()
|
||||
})
|
||||
.exec(&txn)
|
||||
.await?;
|
||||
}
|
||||
let resetted_pages_id: Vec<_> = resetted_pages_model.iter().map(|model| model.id).collect();
|
||||
let resetted_pages_model: Vec<page::ActiveModel> = resetted_pages_model
|
||||
.into_iter()
|
||||
.map(|model| model.into_active_model())
|
||||
.collect();
|
||||
for page_trunk in resetted_pages_model.chunks(50) {
|
||||
page::Entity::insert_many(page_trunk.to_vec())
|
||||
.on_conflict(
|
||||
OnConflict::column(page::Column::Id)
|
||||
.update_column(page::Column::DownloadStatus)
|
||||
.to_owned(),
|
||||
)
|
||||
.exec(&txn)
|
||||
.await?;
|
||||
}
|
||||
txn.commit().await?;
|
||||
Ok(ApiResponse::ok(ResetVideoResponse {
|
||||
resetted: should_update_video,
|
||||
video: id,
|
||||
pages: resetted_pages_id,
|
||||
}))
|
||||
}
|
||||
7
crates/bili_sync/src/api/mod.rs
Normal file
@@ -0,0 +1,7 @@
|
||||
pub mod auth;
|
||||
pub mod handler;
|
||||
|
||||
mod error;
|
||||
mod request;
|
||||
mod response;
|
||||
mod wrapper;
|
||||
13
crates/bili_sync/src/api/request.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
use serde::Deserialize;
|
||||
use utoipa::IntoParams;
|
||||
|
||||
#[derive(Deserialize, IntoParams)]
|
||||
pub struct VideosRequest {
|
||||
pub collection: Option<i32>,
|
||||
pub favorite: Option<i32>,
|
||||
pub submission: Option<i32>,
|
||||
pub watch_later: Option<i32>,
|
||||
pub query: Option<String>,
|
||||
pub page: Option<u64>,
|
||||
pub page_size: Option<u64>,
|
||||
}
|
||||
76
crates/bili_sync/src/api/response.rs
Normal file
@@ -0,0 +1,76 @@
|
||||
use sea_orm::FromQueryResult;
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::utils::status::{PageStatus, VideoStatus};
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct VideoSourcesResponse {
|
||||
pub collection: Vec<VideoSource>,
|
||||
pub favorite: Vec<VideoSource>,
|
||||
pub submission: Vec<VideoSource>,
|
||||
pub watch_later: Vec<VideoSource>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct VideosResponse {
|
||||
pub videos: Vec<VideoInfo>,
|
||||
pub total_count: u64,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct VideoResponse {
|
||||
pub video: VideoInfo,
|
||||
pub pages: Vec<PageInfo>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct ResetVideoResponse {
|
||||
pub resetted: bool,
|
||||
pub video: i32,
|
||||
pub pages: Vec<i32>,
|
||||
}
|
||||
|
||||
#[derive(FromQueryResult, Serialize, ToSchema)]
|
||||
pub struct VideoSource {
|
||||
id: i32,
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct PageInfo {
|
||||
pub id: i32,
|
||||
pub pid: i32,
|
||||
pub name: String,
|
||||
pub download_status: [u32; 5],
|
||||
}
|
||||
|
||||
impl From<(i32, i32, String, u32)> for PageInfo {
|
||||
fn from((id, pid, name, download_status): (i32, i32, String, u32)) -> Self {
|
||||
Self {
|
||||
id,
|
||||
pid,
|
||||
name,
|
||||
download_status: PageStatus::from(download_status).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, ToSchema)]
|
||||
pub struct VideoInfo {
|
||||
pub id: i32,
|
||||
pub name: String,
|
||||
pub upper_name: String,
|
||||
pub download_status: [u32; 5],
|
||||
}
|
||||
|
||||
impl From<(i32, String, String, u32)> for VideoInfo {
|
||||
fn from((id, name, upper_name, download_status): (i32, String, String, u32)) -> Self {
|
||||
Self {
|
||||
id,
|
||||
name,
|
||||
upper_name,
|
||||
download_status: VideoStatus::from(download_status).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
64
crates/bili_sync/src/api/wrapper.rs
Normal file
@@ -0,0 +1,64 @@
|
||||
use anyhow::Error;
|
||||
use axum::Json;
|
||||
use axum::response::IntoResponse;
|
||||
use reqwest::StatusCode;
|
||||
use serde::Serialize;
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use crate::api::error::InnerApiError;
|
||||
|
||||
#[derive(ToSchema, Serialize)]
|
||||
pub struct ApiResponse<T: Serialize> {
|
||||
status_code: u16,
|
||||
data: T,
|
||||
}
|
||||
|
||||
impl<T: Serialize> ApiResponse<T> {
|
||||
pub fn ok(data: T) -> Self {
|
||||
Self { status_code: 200, data }
|
||||
}
|
||||
|
||||
pub fn unauthorized(data: T) -> Self {
|
||||
Self { status_code: 401, data }
|
||||
}
|
||||
|
||||
pub fn not_found(data: T) -> Self {
|
||||
Self { status_code: 404, data }
|
||||
}
|
||||
|
||||
pub fn internal_server_error(data: T) -> Self {
|
||||
Self { status_code: 500, data }
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Serialize> IntoResponse for ApiResponse<T> {
|
||||
fn into_response(self) -> axum::response::Response {
|
||||
(
|
||||
StatusCode::from_u16(self.status_code).expect("invalid Http Status Code"),
|
||||
Json(self),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ApiError(Error);
|
||||
|
||||
impl<E> From<E> for ApiError
|
||||
where
|
||||
E: Into<anyhow::Error>,
|
||||
{
|
||||
fn from(value: E) -> Self {
|
||||
Self(value.into())
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ApiError {
|
||||
fn into_response(self) -> axum::response::Response {
|
||||
if let Some(inner_error) = self.0.downcast_ref::<InnerApiError>() {
|
||||
match inner_error {
|
||||
InnerApiError::NotFound(_) => return ApiResponse::not_found(self.0.to_string()).into_response(),
|
||||
}
|
||||
}
|
||||
ApiResponse::internal_server_error(self.0.to_string()).into_response()
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use anyhow::{Context, Result, bail};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::bilibili::error::BiliError;
|
||||
@@ -7,7 +7,7 @@ pub struct PageAnalyzer {
|
||||
info: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug, strum::FromRepr, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
#[derive(Debug, strum::FromRepr, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
pub enum VideoQuality {
|
||||
Quality360p = 16,
|
||||
Quality480p = 32,
|
||||
@@ -20,7 +20,8 @@ pub enum VideoQuality {
|
||||
QualityDolby = 126,
|
||||
Quality8k = 127,
|
||||
}
|
||||
#[derive(Debug, strum::FromRepr, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
|
||||
#[derive(Debug, Clone, Copy, strum::FromRepr, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum AudioQuality {
|
||||
Quality64k = 30216,
|
||||
Quality132k = 30232,
|
||||
@@ -29,8 +30,30 @@ pub enum AudioQuality {
|
||||
Quality192k = 30280,
|
||||
}
|
||||
|
||||
impl Ord for AudioQuality {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.as_sort_key().cmp(&other.as_sort_key())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for AudioQuality {
|
||||
fn partial_cmp(&self, other: &AudioQuality) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl AudioQuality {
|
||||
pub fn as_sort_key(&self) -> isize {
|
||||
match self {
|
||||
// 这可以让 Dolby 和 Hi-RES 排在 192k 之后,且 Dolby 和 Hi-RES 之间的顺序不变
|
||||
Self::QualityHiRES | Self::QualityDolby => (*self as isize) + 40,
|
||||
_ => *self as isize,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::upper_case_acronyms)]
|
||||
#[derive(Debug, strum::EnumString, strum::Display, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
#[derive(Debug, strum::EnumString, strum::Display, strum::AsRefStr, PartialEq, PartialOrd, Serialize, Deserialize)]
|
||||
pub enum VideoCodecs {
|
||||
#[strum(serialize = "hev")]
|
||||
HEV,
|
||||
@@ -75,7 +98,7 @@ impl Default for FilterOption {
|
||||
pub enum Stream {
|
||||
Flv(String),
|
||||
Html5Mp4(String),
|
||||
EpositeTryMp4(String),
|
||||
EpisodeTryMp4(String),
|
||||
DashVideo {
|
||||
url: String,
|
||||
quality: VideoQuality,
|
||||
@@ -93,7 +116,7 @@ impl Stream {
|
||||
match self {
|
||||
Self::Flv(url) => url,
|
||||
Self::Html5Mp4(url) => url,
|
||||
Self::EpositeTryMp4(url) => url,
|
||||
Self::EpisodeTryMp4(url) => url,
|
||||
Self::DashVideo { url, .. } => url,
|
||||
Self::DashAudio { url, .. } => url,
|
||||
}
|
||||
@@ -115,32 +138,28 @@ impl PageAnalyzer {
|
||||
}
|
||||
|
||||
fn is_flv_stream(&self) -> bool {
|
||||
self.info.get("durl").is_some()
|
||||
&& self.info["format"].is_string()
|
||||
&& self.info["format"].as_str().unwrap().starts_with("flv")
|
||||
self.info.get("durl").is_some() && self.info["format"].as_str().is_some_and(|f| f.starts_with("flv"))
|
||||
}
|
||||
|
||||
fn is_html5_mp4_stream(&self) -> bool {
|
||||
self.info.get("durl").is_some()
|
||||
&& self.info["format"].is_string()
|
||||
&& self.info["format"].as_str().unwrap().starts_with("mp4")
|
||||
&& self.info["is_html5"].is_boolean()
|
||||
&& self.info["is_html5"].as_bool().unwrap()
|
||||
&& self.info["format"].as_str().is_some_and(|f| f.starts_with("mp4"))
|
||||
&& self.info["is_html5"].as_bool().is_some_and(|b| b)
|
||||
}
|
||||
|
||||
fn is_episode_try_mp4_stream(&self) -> bool {
|
||||
self.info.get("durl").is_some()
|
||||
&& self.info["format"].is_string()
|
||||
&& self.info["format"].as_str().unwrap().starts_with("mp4")
|
||||
&& !(self.info["is_html5"].is_boolean() && self.info["is_html5"].as_bool().unwrap())
|
||||
&& self.info["format"].as_str().is_some_and(|f| f.starts_with("mp4"))
|
||||
&& self.info["is_html5"].as_bool().is_none_or(|b| !b)
|
||||
}
|
||||
|
||||
/// 获取所有的视频、音频流,并根据条件筛选
|
||||
fn streams(&mut self, filter_option: &FilterOption) -> Result<Vec<Stream>> {
|
||||
if self.is_flv_stream() {
|
||||
return Ok(vec![Stream::Flv(
|
||||
self.info["durl"][0]["url"]
|
||||
.as_str()
|
||||
.ok_or(anyhow!("invalid flv stream"))?
|
||||
.context("invalid flv stream")?
|
||||
.to_string(),
|
||||
)]);
|
||||
}
|
||||
@@ -148,98 +167,92 @@ impl PageAnalyzer {
|
||||
return Ok(vec![Stream::Html5Mp4(
|
||||
self.info["durl"][0]["url"]
|
||||
.as_str()
|
||||
.ok_or(anyhow!("invalid html5 mp4 stream"))?
|
||||
.context("invalid html5 mp4 stream")?
|
||||
.to_string(),
|
||||
)]);
|
||||
}
|
||||
if self.is_episode_try_mp4_stream() {
|
||||
return Ok(vec![Stream::EpositeTryMp4(
|
||||
return Ok(vec![Stream::EpisodeTryMp4(
|
||||
self.info["durl"][0]["url"]
|
||||
.as_str()
|
||||
.ok_or(anyhow!("invalid episode try mp4 stream"))?
|
||||
.context("invalid episode try mp4 stream")?
|
||||
.to_string(),
|
||||
)]);
|
||||
}
|
||||
let mut streams: Vec<Stream> = Vec::new();
|
||||
let videos_data = self.info["dash"]["video"].take();
|
||||
let audios_data = self.info["dash"]["audio"].take();
|
||||
let flac_data = self.info["dash"]["flac"].take();
|
||||
let dolby_data = self.info["dash"]["dolby"].take();
|
||||
for video_data in videos_data.as_array().ok_or(BiliError::RiskControlOccurred)?.iter() {
|
||||
let video_stream_url = video_data["baseUrl"].as_str().unwrap().to_string();
|
||||
let video_stream_quality = VideoQuality::from_repr(video_data["id"].as_u64().unwrap() as usize)
|
||||
.ok_or(anyhow!("invalid video stream quality"))?;
|
||||
if (video_stream_quality == VideoQuality::QualityHdr && filter_option.no_hdr)
|
||||
|| (video_stream_quality == VideoQuality::QualityDolby && filter_option.no_dolby_video)
|
||||
|| (video_stream_quality != VideoQuality::QualityDolby
|
||||
&& video_stream_quality != VideoQuality::QualityHdr
|
||||
&& (video_stream_quality < filter_option.video_min_quality
|
||||
|| video_stream_quality > filter_option.video_max_quality))
|
||||
// 此处过滤包含三种情况:
|
||||
// 1. HDR 视频,但指定不需要 HDR
|
||||
// 2. 杜比视界视频,但指定不需要杜比视界
|
||||
// 3. 视频质量不在指定范围内
|
||||
for video in self.info["dash"]["video"]
|
||||
.as_array()
|
||||
.ok_or(BiliError::RiskControlOccurred)?
|
||||
.iter()
|
||||
{
|
||||
let (Some(url), Some(quality), Some(codecs)) = (
|
||||
video["baseUrl"].as_str(),
|
||||
video["id"].as_u64(),
|
||||
video["codecs"].as_str(),
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
let quality = VideoQuality::from_repr(quality as usize).context("invalid video stream quality")?;
|
||||
// 从视频流的 codecs 字段中获取编码格式,此处并非精确匹配而是判断包含,比如 codecs 是 av1.42c01e,需要匹配为 av1
|
||||
let Some(codecs) = [VideoCodecs::HEV, VideoCodecs::AVC, VideoCodecs::AV1]
|
||||
.into_iter()
|
||||
.find(|c| codecs.contains(c.as_ref()))
|
||||
else {
|
||||
// 少数情况会走到此处,如 codecs 为 dvh1.08.09、hvc1.2.4.L123.90 等,直接跳过,不影响流程
|
||||
continue;
|
||||
};
|
||||
if !filter_option.codecs.contains(&codecs)
|
||||
|| quality < filter_option.video_min_quality
|
||||
|| quality > filter_option.video_max_quality
|
||||
|| (quality == VideoQuality::QualityHdr && filter_option.no_hdr)
|
||||
|| (quality == VideoQuality::QualityDolby && filter_option.no_dolby_video)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let video_codecs = video_data["codecs"].as_str().unwrap();
|
||||
// 从视频流的 codecs 字段中获取编码格式,此处并非精确匹配而是判断包含,比如 codecs 是 av1.42c01e,需要匹配为 av1
|
||||
let video_codecs = vec![VideoCodecs::HEV, VideoCodecs::AVC, VideoCodecs::AV1]
|
||||
.into_iter()
|
||||
.find(|c| video_codecs.contains(c.to_string().as_str()));
|
||||
|
||||
let Some(video_codecs) = video_codecs else {
|
||||
continue;
|
||||
};
|
||||
if !filter_option.codecs.contains(&video_codecs) {
|
||||
continue;
|
||||
}
|
||||
streams.push(Stream::DashVideo {
|
||||
url: video_stream_url,
|
||||
quality: video_stream_quality,
|
||||
codecs: video_codecs,
|
||||
url: url.to_string(),
|
||||
quality,
|
||||
codecs,
|
||||
});
|
||||
}
|
||||
if audios_data.is_array() {
|
||||
for audio_data in audios_data.as_array().unwrap().iter() {
|
||||
let audio_stream_url = audio_data["baseUrl"].as_str().unwrap().to_string();
|
||||
let audio_stream_quality = AudioQuality::from_repr(audio_data["id"].as_u64().unwrap() as usize);
|
||||
let Some(audio_stream_quality) = audio_stream_quality else {
|
||||
if let Some(audios) = self.info["dash"]["audio"].as_array() {
|
||||
for audio in audios.iter() {
|
||||
let (Some(url), Some(quality)) = (audio["baseUrl"].as_str(), audio["id"].as_u64()) else {
|
||||
continue;
|
||||
};
|
||||
if audio_stream_quality > filter_option.audio_max_quality
|
||||
|| audio_stream_quality < filter_option.audio_min_quality
|
||||
{
|
||||
let quality = AudioQuality::from_repr(quality as usize).context("invalid audio stream quality")?;
|
||||
if quality < filter_option.audio_min_quality || quality > filter_option.audio_max_quality {
|
||||
continue;
|
||||
}
|
||||
streams.push(Stream::DashAudio {
|
||||
url: audio_stream_url,
|
||||
quality: audio_stream_quality,
|
||||
url: url.to_string(),
|
||||
quality,
|
||||
});
|
||||
}
|
||||
}
|
||||
if !(filter_option.no_hires || flac_data["audio"].is_null()) {
|
||||
// 允许 hires 且存在 flac 音频流才会进来
|
||||
let flac_stream_url = flac_data["audio"]["baseUrl"].as_str().unwrap().to_string();
|
||||
let flac_stream_quality =
|
||||
AudioQuality::from_repr(flac_data["audio"]["id"].as_u64().unwrap() as usize).unwrap();
|
||||
streams.push(Stream::DashAudio {
|
||||
url: flac_stream_url,
|
||||
quality: flac_stream_quality,
|
||||
});
|
||||
}
|
||||
if !(filter_option.no_dolby_audio || dolby_data["audio"].is_null()) {
|
||||
// 同理,允许杜比音频且存在杜比音频流才会进来
|
||||
let dolby_stream_data = dolby_data["audio"].as_array().and_then(|v| v.first());
|
||||
if dolby_stream_data.is_some() {
|
||||
let dolby_stream_data = dolby_stream_data.unwrap();
|
||||
let dolby_stream_url = dolby_stream_data["baseUrl"].as_str().unwrap().to_string();
|
||||
let dolby_stream_quality =
|
||||
AudioQuality::from_repr(dolby_stream_data["id"].as_u64().unwrap() as usize).unwrap();
|
||||
let flac = &self.info["dash"]["flac"]["audio"];
|
||||
if !(filter_option.no_hires || flac.is_null()) {
|
||||
let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else {
|
||||
bail!("invalid flac stream");
|
||||
};
|
||||
let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?;
|
||||
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
|
||||
streams.push(Stream::DashAudio {
|
||||
url: dolby_stream_url,
|
||||
quality: dolby_stream_quality,
|
||||
url: url.to_string(),
|
||||
quality,
|
||||
});
|
||||
}
|
||||
}
|
||||
let dolby_audio = &self.info["dash"]["dolby"]["audio"][0];
|
||||
if !(filter_option.no_dolby_audio || dolby_audio.is_null()) {
|
||||
let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else {
|
||||
bail!("invalid dolby audio stream");
|
||||
};
|
||||
let quality = AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?;
|
||||
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
|
||||
streams.push(Stream::DashAudio {
|
||||
url: url.to_string(),
|
||||
quality,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -250,68 +263,130 @@ impl PageAnalyzer {
|
||||
let streams = self.streams(filter_option)?;
|
||||
if self.is_flv_stream() || self.is_html5_mp4_stream() || self.is_episode_try_mp4_stream() {
|
||||
// 按照 streams 中的假设,符合这三种情况的流只有一个,直接取
|
||||
return Ok(BestStream::Mixed(streams.into_iter().next().unwrap()));
|
||||
return Ok(BestStream::Mixed(
|
||||
streams.into_iter().next().context("no stream found")?,
|
||||
));
|
||||
}
|
||||
// 将视频流和音频流拆分,分别做排序
|
||||
let (mut video_streams, mut audio_streams): (Vec<_>, Vec<_>) =
|
||||
let (videos, audios): (Vec<Stream>, Vec<Stream>) =
|
||||
streams.into_iter().partition(|s| matches!(s, Stream::DashVideo { .. }));
|
||||
// 因为该处的排序与筛选选项有关,因此不能在外面实现 PartialOrd trait,只能在这里写闭包
|
||||
video_streams.sort_by(|a, b| match (a, b) {
|
||||
(
|
||||
Stream::DashVideo {
|
||||
quality: a_quality,
|
||||
codecs: a_codecs,
|
||||
..
|
||||
},
|
||||
Stream::DashVideo {
|
||||
quality: b_quality,
|
||||
codecs: b_codecs,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
if a_quality == &VideoQuality::QualityDolby && !filter_option.no_dolby_video {
|
||||
return std::cmp::Ordering::Greater;
|
||||
}
|
||||
if b_quality == &VideoQuality::QualityDolby && !filter_option.no_dolby_video {
|
||||
return std::cmp::Ordering::Less;
|
||||
}
|
||||
if a_quality == &VideoQuality::QualityHdr && !filter_option.no_hdr {
|
||||
return std::cmp::Ordering::Greater;
|
||||
}
|
||||
if b_quality == &VideoQuality::QualityHdr && !filter_option.no_hdr {
|
||||
return std::cmp::Ordering::Less;
|
||||
}
|
||||
if a_quality != b_quality {
|
||||
return a_quality.partial_cmp(b_quality).unwrap();
|
||||
}
|
||||
// 如果视频质量相同,按照偏好的编码优先级排序
|
||||
filter_option
|
||||
.codecs
|
||||
.iter()
|
||||
.position(|c| c == b_codecs)
|
||||
.cmp(&filter_option.codecs.iter().position(|c| c == a_codecs))
|
||||
}
|
||||
_ => unreachable!(),
|
||||
});
|
||||
audio_streams.sort_by(|a, b| match (a, b) {
|
||||
(Stream::DashAudio { quality: a_quality, .. }, Stream::DashAudio { quality: b_quality, .. }) => {
|
||||
if a_quality == &AudioQuality::QualityDolby && !filter_option.no_dolby_audio {
|
||||
return std::cmp::Ordering::Greater;
|
||||
}
|
||||
if b_quality == &AudioQuality::QualityDolby && !filter_option.no_dolby_audio {
|
||||
return std::cmp::Ordering::Less;
|
||||
}
|
||||
a_quality.partial_cmp(b_quality).unwrap()
|
||||
}
|
||||
_ => unreachable!(),
|
||||
});
|
||||
if video_streams.is_empty() {
|
||||
bail!("no video stream found");
|
||||
}
|
||||
Ok(BestStream::VideoAudio {
|
||||
video: video_streams.remove(video_streams.len() - 1),
|
||||
// 音频流可能为空,因此直接使用 pop 返回 Option
|
||||
audio: audio_streams.pop(),
|
||||
video: Iterator::max_by(videos.into_iter(), |a, b| match (a, b) {
|
||||
(
|
||||
Stream::DashVideo {
|
||||
quality: a_quality,
|
||||
codecs: a_codecs,
|
||||
..
|
||||
},
|
||||
Stream::DashVideo {
|
||||
quality: b_quality,
|
||||
codecs: b_codecs,
|
||||
..
|
||||
},
|
||||
) => {
|
||||
if a_quality != b_quality {
|
||||
return a_quality.cmp(b_quality);
|
||||
};
|
||||
filter_option
|
||||
.codecs
|
||||
.iter()
|
||||
.position(|c| c == b_codecs)
|
||||
.cmp(&filter_option.codecs.iter().position(|c| c == a_codecs))
|
||||
}
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.context("no video stream found")?,
|
||||
audio: Iterator::max_by(audios.into_iter(), |a, b| match (a, b) {
|
||||
(Stream::DashAudio { quality: a_quality, .. }, Stream::DashAudio { quality: b_quality, .. }) => {
|
||||
a_quality.cmp(b_quality)
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bilibili::{BiliClient, Video};
|
||||
use crate::config::CONFIG;
|
||||
|
||||
#[test]
|
||||
fn test_quality_order() {
|
||||
assert!(
|
||||
[
|
||||
VideoQuality::Quality360p,
|
||||
VideoQuality::Quality480p,
|
||||
VideoQuality::Quality720p,
|
||||
VideoQuality::Quality1080p,
|
||||
VideoQuality::Quality1080pPLUS,
|
||||
VideoQuality::Quality1080p60,
|
||||
VideoQuality::Quality4k,
|
||||
VideoQuality::QualityHdr,
|
||||
VideoQuality::QualityDolby,
|
||||
VideoQuality::Quality8k
|
||||
]
|
||||
.is_sorted()
|
||||
);
|
||||
assert!(
|
||||
[
|
||||
AudioQuality::Quality64k,
|
||||
AudioQuality::Quality132k,
|
||||
AudioQuality::Quality192k,
|
||||
AudioQuality::QualityDolby,
|
||||
AudioQuality::QualityHiRES,
|
||||
]
|
||||
.is_sorted()
|
||||
);
|
||||
}
|
||||
|
||||
#[ignore = "only for manual test"]
|
||||
#[tokio::test]
|
||||
async fn test_best_stream() {
|
||||
let testcases = [
|
||||
// 随便一个 8k + hires 视频
|
||||
(
|
||||
"BV1xRChYUE2R",
|
||||
VideoQuality::Quality8k,
|
||||
Some(AudioQuality::QualityHiRES),
|
||||
),
|
||||
// 一个没有声音的纯视频
|
||||
("BV1J7411H7KQ", VideoQuality::Quality720p, None),
|
||||
// 一个杜比全景声的演示片
|
||||
(
|
||||
"BV1Mm4y1P7JV",
|
||||
VideoQuality::Quality4k,
|
||||
Some(AudioQuality::QualityDolby),
|
||||
),
|
||||
];
|
||||
for (bvid, video_quality, audio_quality) in testcases.into_iter() {
|
||||
let client = BiliClient::new();
|
||||
let video = Video::new(&client, bvid.to_owned());
|
||||
let pages = video.get_pages().await.expect("failed to get pages");
|
||||
let first_page = pages.into_iter().next().expect("no page found");
|
||||
let best_stream = video
|
||||
.get_page_analyzer(&first_page)
|
||||
.await
|
||||
.expect("failed to get page analyzer")
|
||||
.best_stream(&CONFIG.filter_option)
|
||||
.expect("failed to get best stream");
|
||||
dbg!(bvid, &best_stream);
|
||||
match best_stream {
|
||||
BestStream::VideoAudio {
|
||||
video: Stream::DashVideo { quality, .. },
|
||||
audio,
|
||||
} => {
|
||||
assert_eq!(quality, video_quality);
|
||||
assert_eq!(
|
||||
audio.map(|audio_stream| match audio_stream {
|
||||
Stream::DashAudio { quality, .. } => quality,
|
||||
_ => unreachable!(),
|
||||
}),
|
||||
audio_quality,
|
||||
);
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use reqwest::{header, Method};
|
||||
use anyhow::{Context, Result};
|
||||
use leaky_bucket::RateLimiter;
|
||||
use reqwest::{Method, header};
|
||||
|
||||
use crate::bilibili::credential::WbiImg;
|
||||
use crate::bilibili::Credential;
|
||||
use crate::config::CONFIG;
|
||||
use crate::bilibili::credential::WbiImg;
|
||||
use crate::config::{CONFIG, RateLimit};
|
||||
|
||||
// 一个对 reqwest::Client 的简单封装,用于 Bilibili 请求
|
||||
#[derive(Clone)]
|
||||
@@ -32,7 +34,7 @@ impl Client {
|
||||
.connect_timeout(std::time::Duration::from_secs(10))
|
||||
.read_timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap(),
|
||||
.expect("failed to build reqwest client"),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -61,15 +63,32 @@ impl Default for Client {
|
||||
|
||||
pub struct BiliClient {
|
||||
pub client: Client,
|
||||
limiter: Option<RateLimiter>,
|
||||
}
|
||||
|
||||
impl BiliClient {
|
||||
pub fn new() -> Self {
|
||||
let client = Client::new();
|
||||
Self { client }
|
||||
let limiter = CONFIG
|
||||
.concurrent_limit
|
||||
.rate_limit
|
||||
.as_ref()
|
||||
.map(|RateLimit { limit, duration }| {
|
||||
RateLimiter::builder()
|
||||
.initial(*limit)
|
||||
.refill(*limit)
|
||||
.max(*limit)
|
||||
.interval(Duration::from_millis(*duration))
|
||||
.build()
|
||||
});
|
||||
Self { client, limiter }
|
||||
}
|
||||
|
||||
pub fn request(&self, method: Method, url: &str) -> reqwest::RequestBuilder {
|
||||
/// 获取一个预构建的请求,通过该方法获取请求时会检查并等待速率限制
|
||||
pub async fn request(&self, method: Method, url: &str) -> reqwest::RequestBuilder {
|
||||
if let Some(limiter) = &self.limiter {
|
||||
limiter.acquire_one().await;
|
||||
}
|
||||
let credential = CONFIG.credential.load();
|
||||
self.client.request(method, url, credential.as_deref())
|
||||
}
|
||||
@@ -90,9 +109,7 @@ impl BiliClient {
|
||||
/// 获取 wbi img,用于生成请求签名
|
||||
pub async fn wbi_img(&self) -> Result<WbiImg> {
|
||||
let credential = CONFIG.credential.load();
|
||||
let Some(credential) = credential.as_deref() else {
|
||||
bail!("no credential found");
|
||||
};
|
||||
let credential = credential.as_deref().context("no credential found")?;
|
||||
credential.wbi_img(&self.client).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,14 @@
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::fmt::{Display, Formatter};
|
||||
|
||||
use anyhow::Result;
|
||||
use async_stream::stream;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_stream::try_stream;
|
||||
use futures::Stream;
|
||||
use reqwest::Method;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::bilibili::credential::encoded_query;
|
||||
use crate::bilibili::{BiliClient, Validate, VideoInfo, MIXIN_KEY};
|
||||
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
|
||||
|
||||
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
|
||||
pub enum CollectionType {
|
||||
@@ -40,8 +38,8 @@ impl From<i32> for CollectionType {
|
||||
impl Display for CollectionType {
|
||||
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
|
||||
let s = match self {
|
||||
CollectionType::Series => "视频列表",
|
||||
CollectionType::Season => "视频合集",
|
||||
CollectionType::Series => "列表",
|
||||
CollectionType::Season => "合集",
|
||||
};
|
||||
write!(f, "{}", s)
|
||||
}
|
||||
@@ -111,6 +109,7 @@ impl<'a> Collection<'a> {
|
||||
async fn get_series_info(&self) -> Result<Value> {
|
||||
self.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/series/series")
|
||||
.await
|
||||
.query(&[("series_id", self.collection.sid.as_str())])
|
||||
.send()
|
||||
.await?
|
||||
@@ -134,7 +133,7 @@ impl<'a> Collection<'a> {
|
||||
("pn", page.as_str()),
|
||||
("ps", "30"),
|
||||
],
|
||||
MIXIN_KEY.load().as_ref().unwrap(),
|
||||
MIXIN_KEY.load().as_deref(),
|
||||
),
|
||||
),
|
||||
CollectionType::Season => (
|
||||
@@ -147,12 +146,13 @@ impl<'a> Collection<'a> {
|
||||
("page_num", page.as_str()),
|
||||
("page_size", "30"),
|
||||
],
|
||||
MIXIN_KEY.load().as_ref().unwrap(),
|
||||
MIXIN_KEY.load().as_deref(),
|
||||
),
|
||||
),
|
||||
};
|
||||
self.client
|
||||
.request(Method::GET, url)
|
||||
.await
|
||||
.query(&query)
|
||||
.send()
|
||||
.await?
|
||||
@@ -162,44 +162,57 @@ impl<'a> Collection<'a> {
|
||||
.validate()
|
||||
}
|
||||
|
||||
pub fn into_simple_video_stream(self) -> impl Stream<Item = VideoInfo> + 'a {
|
||||
stream! {
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
|
||||
try_stream! {
|
||||
let mut page = 1;
|
||||
loop {
|
||||
let mut videos = match self.get_videos(page).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("failed to get videos of collection {:?} page {}: {}", self.collection, page, e);
|
||||
break;
|
||||
},
|
||||
};
|
||||
if !videos["data"]["archives"].is_array() {
|
||||
warn!("no videos found in collection {:?} page {}", self.collection, page);
|
||||
break;
|
||||
let mut videos = self.get_videos(page).await.with_context(|| {
|
||||
format!(
|
||||
"failed to get videos of collection {:?} page {}",
|
||||
self.collection, page
|
||||
)
|
||||
})?;
|
||||
let archives = &mut videos["data"]["archives"];
|
||||
if archives.as_array().is_none_or(|v| v.is_empty()) {
|
||||
Err(anyhow!(
|
||||
"no videos found in collection {:?} page {}",
|
||||
self.collection,
|
||||
page
|
||||
))?;
|
||||
}
|
||||
let videos_info = match serde_json::from_value::<Vec<VideoInfo>>(videos["data"]["archives"].take()) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("failed to parse videos of collection {:?} page {}: {}", self.collection, page, e);
|
||||
break;
|
||||
},
|
||||
};
|
||||
for video_info in videos_info{
|
||||
let videos_info: Vec<VideoInfo> = serde_json::from_value(archives.take()).with_context(|| {
|
||||
format!(
|
||||
"failed to parse videos of collection {:?} page {}",
|
||||
self.collection, page
|
||||
)
|
||||
})?;
|
||||
for video_info in videos_info {
|
||||
yield video_info;
|
||||
}
|
||||
let fields = match self.collection.collection_type{
|
||||
let page_info = &videos["data"]["page"];
|
||||
let fields = match self.collection.collection_type {
|
||||
CollectionType::Series => ["num", "size", "total"],
|
||||
CollectionType::Season => ["page_num", "page_size", "total"],
|
||||
};
|
||||
let fields = fields.into_iter().map(|f| videos["data"]["page"][f].as_i64()).collect::<Option<Vec<i64>>>().map(|v| (v[0], v[1], v[2]));
|
||||
let Some((num, size, total)) = fields else {
|
||||
error!("failed to get pages of collection {:?} page {}: {:?}", self.collection, page, fields);
|
||||
break;
|
||||
};
|
||||
if num * size >= total {
|
||||
break;
|
||||
let values = fields
|
||||
.iter()
|
||||
.map(|f| page_info[f].as_i64())
|
||||
.collect::<Vec<Option<i64>>>();
|
||||
if let [Some(num), Some(size), Some(total)] = values[..] {
|
||||
if num * size < total {
|
||||
page += 1;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
Err(anyhow!(
|
||||
"invalid page info of collection {:?} page {}: read {:?} from {}",
|
||||
self.collection,
|
||||
page,
|
||||
fields,
|
||||
page_info
|
||||
))?;
|
||||
}
|
||||
page += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashSet;
|
||||
|
||||
use anyhow::{anyhow, bail, Result};
|
||||
use anyhow::{Context, Result, bail, ensure};
|
||||
use cookie::Cookie;
|
||||
use cow_utils::CowUtils;
|
||||
use regex::Regex;
|
||||
use reqwest::{header, Method};
|
||||
use reqwest::{Method, header};
|
||||
use rsa::pkcs8::DecodePublicKey;
|
||||
use rsa::sha2::Sha256;
|
||||
use rsa::{Oaep, RsaPublicKey};
|
||||
@@ -32,9 +34,18 @@ pub struct WbiImg {
|
||||
sub_url: String,
|
||||
}
|
||||
|
||||
impl WbiImg {
|
||||
pub fn into_mixin_key(self) -> Option<String> {
|
||||
get_mixin_key(self)
|
||||
impl From<WbiImg> for Option<String> {
|
||||
/// 尝试将 WbiImg 转换成 mixin_key
|
||||
fn from(value: WbiImg) -> Self {
|
||||
let key = match (
|
||||
get_filename(value.img_url.as_str()),
|
||||
get_filename(value.sub_url.as_str()),
|
||||
) {
|
||||
(Some(img_key), Some(sub_key)) => img_key.to_string() + sub_key,
|
||||
_ => return None,
|
||||
};
|
||||
let key = key.as_bytes();
|
||||
Some(MIXIN_KEY_ENC_TAB.iter().take(32).map(|&x| key[x] as char).collect())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +55,7 @@ impl Credential {
|
||||
.request(Method::GET, "https://api.bilibili.com/x/web-interface/nav", Some(self))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json::<serde_json::Value>()
|
||||
.await?
|
||||
.validate()?;
|
||||
@@ -64,7 +76,7 @@ impl Credential {
|
||||
.json::<serde_json::Value>()
|
||||
.await?
|
||||
.validate()?;
|
||||
res["data"]["refresh"].as_bool().ok_or(anyhow!("check refresh failed"))
|
||||
res["data"]["refresh"].as_bool().context("check refresh failed")
|
||||
}
|
||||
|
||||
pub async fn refresh(&self, client: &Client) -> Result<Self> {
|
||||
@@ -85,11 +97,13 @@ nzPjfdTcqMz7djHum0qSZA0AyCBDABUqCrfNgCiJ00Ra7GmRj+YCK1NJEuewlb40
|
||||
JNrRuoEUXpabUzGB8QIDAQAB
|
||||
-----END PUBLIC KEY-----",
|
||||
)
|
||||
.unwrap();
|
||||
.expect("fail to decode public key");
|
||||
let ts = chrono::Local::now().timestamp_millis();
|
||||
let data = format!("refresh_{}", ts).into_bytes();
|
||||
let mut rng = rand::rngs::OsRng;
|
||||
let encrypted = key.encrypt(&mut rng, Oaep::new::<Sha256>(), &data).unwrap();
|
||||
let encrypted = key
|
||||
.encrypt(&mut rng, Oaep::new::<Sha256>(), &data)
|
||||
.expect("fail to encrypt");
|
||||
hex::encode(encrypted)
|
||||
}
|
||||
|
||||
@@ -140,9 +154,10 @@ JNrRuoEUXpabUzGB8QIDAQAB
|
||||
.filter_map(|x| Cookie::parse(x).ok())
|
||||
.filter(|x| required_cookies.contains(x.name()))
|
||||
.collect();
|
||||
if cookies.len() != required_cookies.len() {
|
||||
bail!("not all required cookies found");
|
||||
}
|
||||
ensure!(
|
||||
cookies.len() == required_cookies.len(),
|
||||
"not all required cookies found"
|
||||
);
|
||||
for cookie in cookies {
|
||||
match cookie.name() {
|
||||
"SESSDATA" => credential.sessdata = cookie.value().to_string(),
|
||||
@@ -151,10 +166,10 @@ JNrRuoEUXpabUzGB8QIDAQAB
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
if !res["data"]["refresh_token"].is_string() {
|
||||
bail!("refresh_token not found");
|
||||
match res["data"]["refresh_token"].as_str() {
|
||||
Some(token) => credential.ac_time_value = token.to_string(),
|
||||
None => bail!("refresh_token not found"),
|
||||
}
|
||||
credential.ac_time_value = res["data"]["refresh_token"].as_str().unwrap().to_string();
|
||||
Ok(credential)
|
||||
}
|
||||
|
||||
@@ -185,9 +200,9 @@ fn regex_find(pattern: &str, doc: &str) -> Result<String> {
|
||||
let re = Regex::new(pattern)?;
|
||||
Ok(re
|
||||
.captures(doc)
|
||||
.ok_or(anyhow!("pattern not match"))?
|
||||
.context("no match found")?
|
||||
.get(1)
|
||||
.unwrap()
|
||||
.context("no capture found")?
|
||||
.as_str()
|
||||
.to_string())
|
||||
}
|
||||
@@ -198,37 +213,47 @@ fn get_filename(url: &str) -> Option<&str> {
|
||||
.map(|(s, _)| s)
|
||||
}
|
||||
|
||||
fn get_mixin_key(wbi_img: WbiImg) -> Option<String> {
|
||||
let key = match (
|
||||
get_filename(wbi_img.img_url.as_str()),
|
||||
get_filename(wbi_img.sub_url.as_str()),
|
||||
) {
|
||||
(Some(img_key), Some(sub_key)) => img_key.to_string() + sub_key,
|
||||
_ => return None,
|
||||
};
|
||||
let key = key.as_bytes();
|
||||
Some(MIXIN_KEY_ENC_TAB.iter().take(32).map(|&x| key[x] as char).collect())
|
||||
pub fn encoded_query<'a>(
|
||||
params: Vec<(&'a str, impl Into<Cow<'a, str>>)>,
|
||||
mixin_key: Option<impl AsRef<str>>,
|
||||
) -> Vec<(&'a str, Cow<'a, str>)> {
|
||||
match mixin_key {
|
||||
Some(key) => _encoded_query(params, key.as_ref(), chrono::Local::now().timestamp().to_string()),
|
||||
None => params.into_iter().map(|(k, v)| (k, v.into())).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encoded_query<'a>(params: Vec<(&'a str, impl Into<String>)>, mixin_key: &str) -> Vec<(&'a str, String)> {
|
||||
let params = params.into_iter().map(|(k, v)| (k, v.into())).collect();
|
||||
_encoded_query(params, mixin_key, chrono::Local::now().timestamp().to_string())
|
||||
}
|
||||
|
||||
fn _encoded_query<'a>(params: Vec<(&'a str, String)>, mixin_key: &str, timestamp: String) -> Vec<(&'a str, String)> {
|
||||
let mut params: Vec<(&'a str, String)> = params
|
||||
fn _encoded_query<'a>(
|
||||
params: Vec<(&'a str, impl Into<Cow<'a, str>>)>,
|
||||
mixin_key: &str,
|
||||
timestamp: String,
|
||||
) -> Vec<(&'a str, Cow<'a, str>)> {
|
||||
let disallowed = ['!', '\'', '(', ')', '*'];
|
||||
let mut params: Vec<(&'a str, Cow<'a, str>)> = params
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, v.chars().filter(|&x| !"!'()*".contains(x)).collect::<String>()))
|
||||
.map(|(k, v)| {
|
||||
(
|
||||
k,
|
||||
match Into::<Cow<'a, str>>::into(v) {
|
||||
Cow::Borrowed(v) => v.cow_replace(&disallowed[..], ""),
|
||||
Cow::Owned(v) => v.replace(&disallowed[..], "").into(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
params.push(("wts", timestamp));
|
||||
params.push(("wts", timestamp.into()));
|
||||
params.sort_by(|a, b| a.0.cmp(b.0));
|
||||
let query = serde_urlencoded::to_string(¶ms).unwrap().replace('+', "%20");
|
||||
params.push(("w_rid", format!("{:x}", md5::compute(query.clone() + mixin_key))));
|
||||
let query = serde_urlencoded::to_string(¶ms)
|
||||
.expect("fail to encode query")
|
||||
.replace('+', "%20");
|
||||
params.push(("w_rid", format!("{:x}", md5::compute(query.clone() + mixin_key)).into()));
|
||||
params
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use assert_matches::assert_matches;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
@@ -265,25 +290,49 @@ mod tests {
|
||||
img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(),
|
||||
sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(),
|
||||
};
|
||||
let mixin_key = get_mixin_key(key);
|
||||
assert_eq!(mixin_key, Some("ea1db124af3c7062474693fa704f4ff8".to_string()));
|
||||
assert_eq!(
|
||||
_encoded_query(
|
||||
vec![
|
||||
("foo", "114".to_string()),
|
||||
("bar", "514".to_string()),
|
||||
("zab", "1919810".to_string())
|
||||
],
|
||||
&mixin_key.unwrap(),
|
||||
let key = Option::<String>::from(key).expect("fail to convert key");
|
||||
assert_eq!(key.as_str(), "ea1db124af3c7062474693fa704f4ff8");
|
||||
// 没有特殊字符
|
||||
assert_matches!(
|
||||
&_encoded_query(
|
||||
vec![("foo", "114"), ("bar", "514"), ("zab", "1919810")],
|
||||
key.as_str(),
|
||||
"1702204169".to_string(),
|
||||
),
|
||||
vec![
|
||||
("bar", "514".to_string()),
|
||||
("foo", "114".to_string()),
|
||||
("wts", "1702204169".to_string()),
|
||||
("zab", "1919810".to_string()),
|
||||
("w_rid", "8f6f2b5b3d485fe1886cec6a0be8c5d4".to_string()),
|
||||
]
|
||||
)
|
||||
)[..],
|
||||
[
|
||||
("bar", Cow::Borrowed(a)),
|
||||
("foo", Cow::Borrowed(b)),
|
||||
("wts", Cow::Owned(c)),
|
||||
("zab", Cow::Borrowed(d)),
|
||||
("w_rid", Cow::Owned(e)),
|
||||
] => {
|
||||
assert_eq!(*a, "514");
|
||||
assert_eq!(*b, "114");
|
||||
assert_eq!(c, "1702204169");
|
||||
assert_eq!(*d, "1919810");
|
||||
assert_eq!(e, "8f6f2b5b3d485fe1886cec6a0be8c5d4");
|
||||
}
|
||||
);
|
||||
// 有特殊字符
|
||||
assert_matches!(
|
||||
&_encoded_query(
|
||||
vec![("foo", "'1(1)4'"), ("bar", "!5*1!14"), ("zab", "1919810")],
|
||||
key.as_str(),
|
||||
"1702204169".to_string(),
|
||||
)[..],
|
||||
[
|
||||
("bar", Cow::Owned(a)),
|
||||
("foo", Cow::Owned(b)),
|
||||
("wts", Cow::Owned(c)),
|
||||
("zab", Cow::Borrowed(d)),
|
||||
("w_rid", Cow::Owned(e)),
|
||||
] => {
|
||||
assert_eq!(a, "5114");
|
||||
assert_eq!(b, "114");
|
||||
assert_eq!(c, "1702204169");
|
||||
assert_eq!(*d, "1919810");
|
||||
assert_eq!(e, "6a2c86c4b0648ce062ba0dac2de91a85");
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::bilibili::danmaku::canvas::CanvasConfig;
|
||||
use crate::bilibili::danmaku::Danmu;
|
||||
use crate::bilibili::danmaku::canvas::CanvasConfig;
|
||||
|
||||
pub enum Collision {
|
||||
// 会越来越远
|
||||
|
||||
@@ -5,10 +5,10 @@ use anyhow::Result;
|
||||
use float_ord::FloatOrd;
|
||||
use lane::Lane;
|
||||
|
||||
use crate::bilibili::PageInfo;
|
||||
use crate::bilibili::danmaku::canvas::lane::Collision;
|
||||
use crate::bilibili::danmaku::danmu::DanmuType;
|
||||
use crate::bilibili::danmaku::{Danmu, DrawEffect, Drawable};
|
||||
use crate::bilibili::PageInfo;
|
||||
|
||||
#[derive(Debug, serde::Deserialize, serde::Serialize)]
|
||||
pub struct DanmakuOption {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! 一个弹幕实例,但是没有位置信息
|
||||
use anyhow::{bail, Result};
|
||||
use anyhow::{Result, bail};
|
||||
|
||||
use crate::bilibili::danmaku::canvas::CanvasConfig;
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ use std::path::PathBuf;
|
||||
use anyhow::Result;
|
||||
use tokio::fs::{self, File};
|
||||
|
||||
use crate::bilibili::PageInfo;
|
||||
use crate::bilibili::danmaku::canvas::CanvasConfig;
|
||||
use crate::bilibili::danmaku::{AssWriter, Danmu};
|
||||
use crate::bilibili::PageInfo;
|
||||
use crate::config::CONFIG;
|
||||
|
||||
pub struct DanmakuWriter<'a> {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use async_stream::stream;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_stream::try_stream;
|
||||
use futures::Stream;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -16,8 +16,8 @@ pub struct FavoriteListInfo {
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct Upper {
|
||||
pub mid: i64,
|
||||
pub struct Upper<T> {
|
||||
pub mid: T,
|
||||
pub name: String,
|
||||
pub face: String,
|
||||
}
|
||||
@@ -30,6 +30,7 @@ impl<'a> FavoriteList<'a> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v3/fav/folder/info")
|
||||
.await
|
||||
.query(&[("media_id", &self.fid)])
|
||||
.send()
|
||||
.await?
|
||||
@@ -43,9 +44,10 @@ impl<'a> FavoriteList<'a> {
|
||||
async fn get_videos(&self, page: u32) -> Result<Value> {
|
||||
self.client
|
||||
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v3/fav/resource/list")
|
||||
.await
|
||||
.query(&[
|
||||
("media_id", self.fid.as_str()),
|
||||
("pn", &page.to_string()),
|
||||
("pn", page.to_string().as_str()),
|
||||
("ps", "20"),
|
||||
("order", "mtime"),
|
||||
("type", "0"),
|
||||
@@ -60,34 +62,31 @@ impl<'a> FavoriteList<'a> {
|
||||
}
|
||||
|
||||
// 拿到收藏夹的所有权,返回一个收藏夹下的视频流
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = VideoInfo> + 'a {
|
||||
stream! {
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
|
||||
try_stream! {
|
||||
let mut page = 1;
|
||||
loop {
|
||||
let mut videos = match self.get_videos(page).await {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("failed to get videos of favorite {} page {}: {}", self.fid, page, e);
|
||||
break;
|
||||
},
|
||||
};
|
||||
if !videos["data"]["medias"].is_array() {
|
||||
warn!("no medias found in favorite {} page {}", self.fid, page);
|
||||
break;
|
||||
let mut videos = self
|
||||
.get_videos(page)
|
||||
.await
|
||||
.with_context(|| format!("failed to get videos of favorite {} page {}", self.fid, page))?;
|
||||
let medias = &mut videos["data"]["medias"];
|
||||
if medias.as_array().is_none_or(|v| v.is_empty()) {
|
||||
Err(anyhow!("no medias found in favorite {} page {}", self.fid, page))?;
|
||||
}
|
||||
let videos_info = match serde_json::from_value::<Vec<VideoInfo>>(videos["data"]["medias"].take()) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("failed to parse videos of favorite {} page {}: {}", self.fid, page, e);
|
||||
break;
|
||||
},
|
||||
};
|
||||
for video_info in videos_info{
|
||||
let videos_info: Vec<VideoInfo> = serde_json::from_value(medias.take())
|
||||
.with_context(|| format!("failed to parse videos of favorite {} page {}", self.fid, page))?;
|
||||
for video_info in videos_info {
|
||||
yield video_info;
|
||||
}
|
||||
if videos["data"]["has_more"].is_boolean() && videos["data"]["has_more"].as_bool().unwrap(){
|
||||
page += 1;
|
||||
continue;
|
||||
let has_more = &videos["data"]["has_more"];
|
||||
if let Some(v) = has_more.as_bool() {
|
||||
if v {
|
||||
page += 1;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
Err(anyhow!("has_more is not a bool"))?;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
pub use analyzer::{BestStream, FilterOption};
|
||||
use anyhow::{bail, Result};
|
||||
use anyhow::{Result, bail, ensure};
|
||||
use arc_swap::ArcSwapOption;
|
||||
use chrono::serde::ts_seconds;
|
||||
use chrono::{DateTime, Utc};
|
||||
@@ -13,6 +13,7 @@ pub use error::BiliError;
|
||||
pub use favorite_list::FavoriteList;
|
||||
use favorite_list::Upper;
|
||||
use once_cell::sync::Lazy;
|
||||
pub use submission::Submission;
|
||||
pub use video::{Dimension, PageInfo, Video};
|
||||
pub use watch_later::WatchLater;
|
||||
|
||||
@@ -23,6 +24,8 @@ mod credential;
|
||||
mod danmaku;
|
||||
mod error;
|
||||
mod favorite_list;
|
||||
mod submission;
|
||||
mod subtitle;
|
||||
mod video;
|
||||
mod watch_later;
|
||||
|
||||
@@ -46,9 +49,7 @@ impl Validate for serde_json::Value {
|
||||
(Some(code), Some(msg)) => (code, msg),
|
||||
_ => bail!("no code or message found"),
|
||||
};
|
||||
if code != 0 {
|
||||
bail!(BiliError::RequestFailed(code, msg.to_owned()));
|
||||
}
|
||||
ensure!(code == 0, BiliError::RequestFailed(code, msg.to_owned()));
|
||||
Ok(self)
|
||||
}
|
||||
}
|
||||
@@ -60,7 +61,7 @@ impl Validate for serde_json::Value {
|
||||
/// > Serde will try to match the data against each variant in order and the first one that deserializes successfully is the one returned.
|
||||
pub enum VideoInfo {
|
||||
/// 从视频详情接口获取的视频信息
|
||||
View {
|
||||
Detail {
|
||||
title: String,
|
||||
bvid: String,
|
||||
#[serde(rename = "desc")]
|
||||
@@ -68,7 +69,7 @@ pub enum VideoInfo {
|
||||
#[serde(rename = "pic")]
|
||||
cover: String,
|
||||
#[serde(rename = "owner")]
|
||||
upper: Upper,
|
||||
upper: Upper<i64>,
|
||||
#[serde(with = "ts_seconds")]
|
||||
ctime: DateTime<Utc>,
|
||||
#[serde(rename = "pubdate", with = "ts_seconds")]
|
||||
@@ -76,15 +77,15 @@ pub enum VideoInfo {
|
||||
pages: Vec<PageInfo>,
|
||||
state: i32,
|
||||
},
|
||||
/// 从收藏夹中获取的视频信息
|
||||
Detail {
|
||||
/// 从收藏夹接口获取的视频信息
|
||||
Favorite {
|
||||
title: String,
|
||||
#[serde(rename = "type")]
|
||||
vtype: i32,
|
||||
bvid: String,
|
||||
intro: String,
|
||||
cover: String,
|
||||
upper: Upper,
|
||||
upper: Upper<i64>,
|
||||
#[serde(with = "ts_seconds")]
|
||||
ctime: DateTime<Utc>,
|
||||
#[serde(with = "ts_seconds")]
|
||||
@@ -93,7 +94,7 @@ pub enum VideoInfo {
|
||||
pubtime: DateTime<Utc>,
|
||||
attr: i32,
|
||||
},
|
||||
/// 从稍后再看中获取的视频信息
|
||||
/// 从稍后再看接口获取的视频信息
|
||||
WatchLater {
|
||||
title: String,
|
||||
bvid: String,
|
||||
@@ -102,7 +103,7 @@ pub enum VideoInfo {
|
||||
#[serde(rename = "pic")]
|
||||
cover: String,
|
||||
#[serde(rename = "owner")]
|
||||
upper: Upper,
|
||||
upper: Upper<i64>,
|
||||
#[serde(with = "ts_seconds")]
|
||||
ctime: DateTime<Utc>,
|
||||
#[serde(rename = "add_at", with = "ts_seconds")]
|
||||
@@ -111,8 +112,8 @@ pub enum VideoInfo {
|
||||
pubtime: DateTime<Utc>,
|
||||
state: i32,
|
||||
},
|
||||
/// 从视频列表中获取的视频信息
|
||||
Simple {
|
||||
/// 从视频合集/视频列表接口获取的视频信息
|
||||
Collection {
|
||||
bvid: String,
|
||||
#[serde(rename = "pic")]
|
||||
cover: String,
|
||||
@@ -121,44 +122,102 @@ pub enum VideoInfo {
|
||||
#[serde(rename = "pubdate", with = "ts_seconds")]
|
||||
pubtime: DateTime<Utc>,
|
||||
},
|
||||
// 从用户投稿接口获取的视频信息
|
||||
Submission {
|
||||
title: String,
|
||||
bvid: String,
|
||||
#[serde(rename = "description")]
|
||||
intro: String,
|
||||
#[serde(rename = "pic")]
|
||||
cover: String,
|
||||
#[serde(rename = "created", with = "ts_seconds")]
|
||||
ctime: DateTime<Utc>,
|
||||
},
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use futures::{pin_mut, StreamExt};
|
||||
use futures::StreamExt;
|
||||
|
||||
use super::*;
|
||||
use crate::utils::init_logger;
|
||||
|
||||
#[ignore = "only for manual test"]
|
||||
#[tokio::test]
|
||||
async fn test_video_info_type() {
|
||||
init_logger("None,bili_sync=debug");
|
||||
let bili_client = BiliClient::new();
|
||||
set_global_mixin_key(
|
||||
bili_client
|
||||
.wbi_img()
|
||||
.await
|
||||
.map(|x| x.into_mixin_key())
|
||||
.unwrap()
|
||||
.unwrap(),
|
||||
);
|
||||
let video = Video::new(&bili_client, "BV1Z54y1C7ZB".to_string());
|
||||
assert!(matches!(video.get_view_info().await, Ok(VideoInfo::View { .. })));
|
||||
// 请求 UP 主视频必须要获取 mixin key,使用 key 计算请求参数的签名,否则直接提示权限不足返回空
|
||||
let Ok(Some(mixin_key)) = bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) else {
|
||||
panic!("获取 mixin key 失败");
|
||||
};
|
||||
set_global_mixin_key(mixin_key);
|
||||
// 测试视频合集
|
||||
let collection_item = CollectionItem {
|
||||
mid: "521722088".to_string(),
|
||||
sid: "387214".to_string(),
|
||||
collection_type: CollectionType::Series,
|
||||
sid: "4523".to_string(),
|
||||
collection_type: CollectionType::Season,
|
||||
};
|
||||
let collection = Collection::new(&bili_client, &collection_item);
|
||||
let stream = collection.into_simple_video_stream();
|
||||
pin_mut!(stream);
|
||||
assert!(matches!(stream.next().await, Some(VideoInfo::Simple { .. })));
|
||||
let favorite = FavoriteList::new(&bili_client, "3084505258".to_string());
|
||||
let stream = favorite.into_video_stream();
|
||||
pin_mut!(stream);
|
||||
assert!(matches!(stream.next().await, Some(VideoInfo::Detail { .. })));
|
||||
let videos = collection
|
||||
.into_video_stream()
|
||||
.take(20)
|
||||
.filter_map(|v| futures::future::ready(v.ok()))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Collection { .. })));
|
||||
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||
// 测试收藏夹
|
||||
let favorite = FavoriteList::new(&bili_client, "3144336058".to_string());
|
||||
let videos = favorite
|
||||
.into_video_stream()
|
||||
.take(20)
|
||||
.filter_map(|v| futures::future::ready(v.ok()))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Favorite { .. })));
|
||||
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||
// 测试稍后再看
|
||||
let watch_later = WatchLater::new(&bili_client);
|
||||
let stream = watch_later.into_video_stream();
|
||||
pin_mut!(stream);
|
||||
assert!(matches!(stream.next().await, Some(VideoInfo::WatchLater { .. })));
|
||||
let videos = watch_later
|
||||
.into_video_stream()
|
||||
.take(20)
|
||||
.filter_map(|v| futures::future::ready(v.ok()))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
assert!(videos.iter().all(|v| matches!(v, VideoInfo::WatchLater { .. })));
|
||||
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||
// 测试投稿
|
||||
let submission = Submission::new(&bili_client, "956761".to_string());
|
||||
let videos = submission
|
||||
.into_video_stream()
|
||||
.take(20)
|
||||
.filter_map(|v| futures::future::ready(v.ok()))
|
||||
.collect::<Vec<_>>()
|
||||
.await;
|
||||
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Submission { .. })));
|
||||
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||
}
|
||||
|
||||
#[ignore = "only for manual test"]
|
||||
#[tokio::test]
|
||||
async fn test_subtitle_parse() -> Result<()> {
|
||||
let bili_client = BiliClient::new();
|
||||
let Ok(Some(mixin_key)) = bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) else {
|
||||
panic!("获取 mixin key 失败");
|
||||
};
|
||||
set_global_mixin_key(mixin_key);
|
||||
let video = Video::new(&bili_client, "BV1gLfnY8E6D".to_string());
|
||||
let pages = video.get_pages().await?;
|
||||
println!("pages: {:?}", pages);
|
||||
let subtitles = video.get_subtitles(&pages[0]).await?;
|
||||
for subtitle in subtitles {
|
||||
println!(
|
||||
"{}: {}",
|
||||
subtitle.lan,
|
||||
subtitle.body.to_string().chars().take(200).collect::<String>()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
90
crates/bili_sync/src/bilibili/submission.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use arc_swap::access::Access;
|
||||
use async_stream::try_stream;
|
||||
use futures::Stream;
|
||||
use reqwest::Method;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::bilibili::credential::encoded_query;
|
||||
use crate::bilibili::favorite_list::Upper;
|
||||
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
|
||||
pub struct Submission<'a> {
|
||||
client: &'a BiliClient,
|
||||
upper_id: String,
|
||||
}
|
||||
|
||||
impl<'a> Submission<'a> {
|
||||
pub fn new(client: &'a BiliClient, upper_id: String) -> Self {
|
||||
Self { client, upper_id }
|
||||
}
|
||||
|
||||
pub async fn get_info(&self) -> Result<Upper<String>> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/web-interface/card")
|
||||
.await
|
||||
.query(&[("mid", self.upper_id.as_str())])
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json::<serde_json::Value>()
|
||||
.await?
|
||||
.validate()?;
|
||||
Ok(serde_json::from_value(res["data"]["card"].take())?)
|
||||
}
|
||||
|
||||
async fn get_videos(&self, page: i32) -> Result<Value> {
|
||||
self.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/space/wbi/arc/search")
|
||||
.await
|
||||
.query(&encoded_query(
|
||||
vec![
|
||||
("mid", self.upper_id.as_str()),
|
||||
("order", "pubdate"),
|
||||
("order_avoided", "true"),
|
||||
("platform", "web"),
|
||||
("web_location", "1550101"),
|
||||
("pn", page.to_string().as_str()),
|
||||
("ps", "30"),
|
||||
],
|
||||
MIXIN_KEY.load().as_deref(),
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json::<serde_json::Value>()
|
||||
.await?
|
||||
.validate()
|
||||
}
|
||||
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
|
||||
try_stream! {
|
||||
let mut page = 1;
|
||||
loop {
|
||||
let mut videos = self
|
||||
.get_videos(page)
|
||||
.await
|
||||
.with_context(|| format!("failed to get videos of upper {} page {}", self.upper_id, page))?;
|
||||
let vlist = &mut videos["data"]["list"]["vlist"];
|
||||
if vlist.as_array().is_none_or(|v| v.is_empty()) {
|
||||
Err(anyhow!("no medias found in upper {} page {}", self.upper_id, page))?;
|
||||
}
|
||||
let videos_info: Vec<VideoInfo> = serde_json::from_value(vlist.take())
|
||||
.with_context(|| format!("failed to parse videos of upper {} page {}", self.upper_id, page))?;
|
||||
for video_info in videos_info {
|
||||
yield video_info;
|
||||
}
|
||||
let count = &videos["data"]["page"]["count"];
|
||||
if let Some(v) = count.as_i64() {
|
||||
if v > (page * 30) as i64 {
|
||||
page += 1;
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
Err(anyhow!("count is not an i64"))?;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
75
crates/bili_sync/src/bilibili/subtitle.rs
Normal file
@@ -0,0 +1,75 @@
|
||||
use std::fmt::Display;
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct SubTitlesInfo {
|
||||
pub subtitles: Vec<SubTitleInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct SubTitleInfo {
|
||||
pub lan: String,
|
||||
pub subtitle_url: String,
|
||||
}
|
||||
|
||||
pub struct SubTitle {
|
||||
pub lan: String,
|
||||
pub body: SubTitleBody,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct SubTitleBody(pub Vec<SubTitleItem>);
|
||||
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
pub struct SubTitleItem {
|
||||
from: f64,
|
||||
to: f64,
|
||||
content: String,
|
||||
}
|
||||
|
||||
impl SubTitleInfo {
|
||||
pub fn is_ai_sub(&self) -> bool {
|
||||
// ai: aisubtitle.hdslb.com/bfs/ai_subtitle/xxxx
|
||||
// 非 ai: aisubtitle.hdslb.com/bfs/subtitle/xxxx
|
||||
self.subtitle_url.contains("ai_subtitle")
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for SubTitleBody {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
for (idx, item) in self.0.iter().enumerate() {
|
||||
writeln!(f, "{}", idx)?;
|
||||
writeln!(f, "{} --> {}", format_time(item.from), format_time(item.to))?;
|
||||
writeln!(f, "{}", item.content)?;
|
||||
writeln!(f)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn format_time(time: f64) -> String {
|
||||
let (second, millisecond) = (time.trunc(), (time.fract() * 1e3) as u32);
|
||||
let (hour, minute, second) = (
|
||||
(second / 3600.0) as u32,
|
||||
((second % 3600.0) / 60.0) as u32,
|
||||
(second % 60.0) as u32,
|
||||
);
|
||||
format!("{:02}:{:02}:{:02},{:03}", hour, minute, second, millisecond)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn test_format_time() {
|
||||
// float 解析会有精度问题,但误差几毫秒应该不太关键
|
||||
// 想再健壮一点就得手写 serde_json 解析拆分秒和毫秒,然后分别处理了
|
||||
let testcases = [
|
||||
(0.0, "00:00:00,000"),
|
||||
(1.5, "00:00:01,500"),
|
||||
(206.45, "00:03:26,449"),
|
||||
(360001.23, "100:00:01,229"),
|
||||
];
|
||||
for (time, expect) in testcases.iter() {
|
||||
assert_eq!(super::format_time(*time), *expect);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
use anyhow::{bail, Result};
|
||||
use futures::stream::FuturesUnordered;
|
||||
use anyhow::{Result, ensure};
|
||||
use futures::TryStreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use prost::Message;
|
||||
use reqwest::Method;
|
||||
|
||||
@@ -8,7 +8,8 @@ use crate::bilibili::analyzer::PageAnalyzer;
|
||||
use crate::bilibili::client::BiliClient;
|
||||
use crate::bilibili::credential::encoded_query;
|
||||
use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply};
|
||||
use crate::bilibili::{Validate, VideoInfo, MIXIN_KEY};
|
||||
use crate::bilibili::subtitle::{SubTitle, SubTitleBody, SubTitleInfo, SubTitlesInfo};
|
||||
use crate::bilibili::{MIXIN_KEY, Validate, VideoInfo};
|
||||
|
||||
static MASK_CODE: u64 = 2251799813685247;
|
||||
static XOR_CODE: u64 = 23442827791579;
|
||||
@@ -62,11 +63,12 @@ impl<'a> Video<'a> {
|
||||
Self { client, aid, bvid }
|
||||
}
|
||||
|
||||
/// 直接调用视频信息接口获取详细的视频信息
|
||||
/// 直接调用视频信息接口获取详细的视频信息,视频信息中包含了视频的分页信息
|
||||
pub async fn get_view_info(&self) -> Result<VideoInfo> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view")
|
||||
.await
|
||||
.query(&[("aid", &self.aid), ("bvid", &self.bvid)])
|
||||
.send()
|
||||
.await?
|
||||
@@ -77,10 +79,12 @@ impl<'a> Video<'a> {
|
||||
Ok(serde_json::from_value(res["data"].take())?)
|
||||
}
|
||||
|
||||
#[allow(unused)]
|
||||
pub async fn get_pages(&self) -> Result<Vec<PageInfo>> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/player/pagelist")
|
||||
.await
|
||||
.query(&[("aid", &self.aid), ("bvid", &self.bvid)])
|
||||
.send()
|
||||
.await?
|
||||
@@ -95,6 +99,7 @@ impl<'a> Video<'a> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view/detail/tag")
|
||||
.await
|
||||
.query(&[("aid", &self.aid), ("bvid", &self.bvid)])
|
||||
.send()
|
||||
.await?
|
||||
@@ -120,19 +125,19 @@ impl<'a> Video<'a> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "http://api.bilibili.com/x/v2/dm/web/seg.so")
|
||||
.await
|
||||
.query(&[("type", 1), ("oid", page.cid), ("segment_index", segment_idx)])
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
let headers = std::mem::take(res.headers_mut());
|
||||
let content_type = headers.get("content-type");
|
||||
if !content_type.is_some_and(|v| v == "application/octet-stream") {
|
||||
bail!(
|
||||
"unexpected content type: {:?}, body: {:?}",
|
||||
content_type,
|
||||
res.text().await
|
||||
);
|
||||
}
|
||||
ensure!(
|
||||
content_type.is_some_and(|v| v == "application/octet-stream"),
|
||||
"unexpected content type: {:?}, body: {:?}",
|
||||
content_type,
|
||||
res.text().await
|
||||
);
|
||||
Ok(DmSegMobileReply::decode(res.bytes().await?)?.elems)
|
||||
}
|
||||
|
||||
@@ -140,6 +145,7 @@ impl<'a> Video<'a> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/playurl")
|
||||
.await
|
||||
.query(&encoded_query(
|
||||
vec![
|
||||
("avid", self.aid.as_str()),
|
||||
@@ -149,7 +155,7 @@ impl<'a> Video<'a> {
|
||||
("fnval", "4048"),
|
||||
("fourk", "1"),
|
||||
],
|
||||
MIXIN_KEY.load().as_ref().unwrap(),
|
||||
MIXIN_KEY.load().as_deref(),
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
@@ -159,6 +165,46 @@ impl<'a> Video<'a> {
|
||||
.validate()?;
|
||||
Ok(PageAnalyzer::new(res["data"].take()))
|
||||
}
|
||||
|
||||
pub async fn get_subtitles(&self, page: &PageInfo) -> Result<Vec<SubTitle>> {
|
||||
let mut res = self
|
||||
.client
|
||||
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/v2")
|
||||
.await
|
||||
.query(&encoded_query(
|
||||
vec![("cid", &page.cid.to_string()), ("bvid", &self.bvid), ("aid", &self.aid)],
|
||||
MIXIN_KEY.load().as_deref(),
|
||||
))
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json::<serde_json::Value>()
|
||||
.await?
|
||||
.validate()?;
|
||||
// 接口返回的信息,包含了一系列的字幕,每个字幕包含了字幕的语言和 json 下载地址
|
||||
let subtitles_info: SubTitlesInfo = serde_json::from_value(res["data"]["subtitle"].take())?;
|
||||
let tasks = subtitles_info
|
||||
.subtitles
|
||||
.into_iter()
|
||||
.filter(|v| !v.is_ai_sub())
|
||||
.map(|v| self.get_subtitle(v))
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
tasks.try_collect().await
|
||||
}
|
||||
|
||||
async fn get_subtitle(&self, info: SubTitleInfo) -> Result<SubTitle> {
|
||||
let mut res = self
|
||||
.client
|
||||
.client // 这里可以直接使用 inner_client,因为该请求不需要鉴权
|
||||
.request(Method::GET, format!("https:{}", &info.subtitle_url).as_str(), None)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
.json::<serde_json::Value>()
|
||||
.await?;
|
||||
let body: SubTitleBody = serde_json::from_value(res["body"].take())?;
|
||||
Ok(SubTitle { lan: info.lan, body })
|
||||
}
|
||||
}
|
||||
|
||||
fn bvid_to_aid(bvid: &str) -> u64 {
|
||||
@@ -167,7 +213,7 @@ fn bvid_to_aid(bvid: &str) -> u64 {
|
||||
(bvid[4], bvid[7]) = (bvid[7], bvid[4]);
|
||||
let mut tmp = 0u64;
|
||||
for char in bvid.into_iter().skip(3) {
|
||||
let idx = DATA.iter().position(|&x| x == char).unwrap();
|
||||
let idx = DATA.iter().position(|&x| x == char).expect("invalid bvid");
|
||||
tmp = tmp * BASE + idx as u64;
|
||||
}
|
||||
(tmp & MASK_CODE) ^ XOR_CODE
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use anyhow::Result;
|
||||
use async_stream::stream;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use async_stream::try_stream;
|
||||
use futures::Stream;
|
||||
use serde_json::Value;
|
||||
|
||||
@@ -16,6 +16,7 @@ impl<'a> WatchLater<'a> {
|
||||
async fn get_videos(&self) -> Result<Value> {
|
||||
self.client
|
||||
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v2/history/toview")
|
||||
.await
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?
|
||||
@@ -24,24 +25,20 @@ impl<'a> WatchLater<'a> {
|
||||
.validate()
|
||||
}
|
||||
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = VideoInfo> + 'a {
|
||||
stream! {
|
||||
let Ok(mut videos) = self.get_videos().await else {
|
||||
error!("Failed to get watch later list");
|
||||
return;
|
||||
};
|
||||
if !videos["data"]["list"].is_array() {
|
||||
error!("Watch later list is not an array");
|
||||
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
|
||||
try_stream! {
|
||||
let mut videos = self
|
||||
.get_videos()
|
||||
.await
|
||||
.with_context(|| "Failed to get watch later list")?;
|
||||
let list = &mut videos["data"]["list"];
|
||||
if list.as_array().is_none_or(|v| v.is_empty()) {
|
||||
Err(anyhow!("No videos found in watch later list"))?;
|
||||
}
|
||||
let videos_info = match serde_json::from_value::<Vec<VideoInfo>>(videos["data"]["list"].take()) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
error!("Failed to parse watch later list: {}", e);
|
||||
return;
|
||||
}
|
||||
};
|
||||
for video in videos_info {
|
||||
yield video;
|
||||
let videos_info: Vec<VideoInfo> =
|
||||
serde_json::from_value(list.take()).with_context(|| "Failed to parse watch later list")?;
|
||||
for video_info in videos_info {
|
||||
yield video_info;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,265 +0,0 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use arc_swap::ArcSwapOption;
|
||||
use handlebars::handlebars_helper;
|
||||
use once_cell::sync::Lazy;
|
||||
use serde::de::{Deserializer, MapAccess, Visitor};
|
||||
use serde::ser::SerializeMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::bilibili::{CollectionItem, CollectionType, Credential, DanmakuOption, FilterOption};
|
||||
|
||||
pub static TEMPLATE: Lazy<handlebars::Handlebars> = Lazy::new(|| {
|
||||
let mut handlebars = handlebars::Handlebars::new();
|
||||
handlebars_helper!(truncate: |s: String, len: usize| {
|
||||
if s.chars().count() > len {
|
||||
s.chars().take(len).collect::<String>()
|
||||
} else {
|
||||
s.to_string()
|
||||
}
|
||||
});
|
||||
handlebars.register_helper("truncate", Box::new(truncate));
|
||||
handlebars
|
||||
.register_template_string("video", &CONFIG.video_name)
|
||||
.unwrap();
|
||||
handlebars.register_template_string("page", &CONFIG.page_name).unwrap();
|
||||
handlebars
|
||||
});
|
||||
|
||||
pub static CONFIG: Lazy<Config> = Lazy::new(|| {
|
||||
let config = Config::load().unwrap_or_else(|err| {
|
||||
if err
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.map_or(true, |e| e.kind() != std::io::ErrorKind::NotFound)
|
||||
{
|
||||
panic!("加载配置文件失败,错误为: {err}");
|
||||
}
|
||||
warn!("配置文件不存在,使用默认配置...");
|
||||
Config::default()
|
||||
});
|
||||
// 放到外面,确保新的配置项被保存
|
||||
info!("配置加载完毕,覆盖刷新原有配置");
|
||||
config.save().unwrap();
|
||||
// 检查配置文件内容
|
||||
info!("校验配置文件内容...");
|
||||
config.check();
|
||||
config
|
||||
});
|
||||
|
||||
pub static ARGS: Lazy<Args> = Lazy::new(Args::parse);
|
||||
|
||||
pub static CONFIG_DIR: Lazy<PathBuf> =
|
||||
Lazy::new(|| dirs::config_dir().expect("No config path found").join("bili-sync"));
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Config {
|
||||
pub credential: ArcSwapOption<Credential>,
|
||||
pub filter_option: FilterOption,
|
||||
#[serde(default)]
|
||||
pub danmaku_option: DanmakuOption,
|
||||
pub favorite_list: HashMap<String, PathBuf>,
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_collection_list",
|
||||
deserialize_with = "deserialize_collection_list"
|
||||
)]
|
||||
pub collection_list: HashMap<CollectionItem, PathBuf>,
|
||||
#[serde(default)]
|
||||
pub watch_later: WatchLaterConfig,
|
||||
pub video_name: Cow<'static, str>,
|
||||
pub page_name: Cow<'static, str>,
|
||||
pub interval: u64,
|
||||
pub upper_path: PathBuf,
|
||||
#[serde(default)]
|
||||
pub nfo_time_type: NFOTimeType,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
pub struct WatchLaterConfig {
|
||||
pub enabled: bool,
|
||||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum NFOTimeType {
|
||||
#[default]
|
||||
FavTime,
|
||||
PubTime,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
credential: ArcSwapOption::from(Some(Arc::new(Credential::default()))),
|
||||
filter_option: FilterOption::default(),
|
||||
danmaku_option: DanmakuOption::default(),
|
||||
favorite_list: HashMap::new(),
|
||||
collection_list: HashMap::new(),
|
||||
watch_later: Default::default(),
|
||||
video_name: Cow::Borrowed("{{title}}"),
|
||||
page_name: Cow::Borrowed("{{bvid}}"),
|
||||
interval: 1200,
|
||||
upper_path: CONFIG_DIR.join("upper_face"),
|
||||
nfo_time_type: NFOTimeType::FavTime,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// 简单的预检查
|
||||
pub fn check(&self) {
|
||||
let mut ok = true;
|
||||
if self.favorite_list.is_empty() && self.collection_list.is_empty() && !self.watch_later.enabled {
|
||||
ok = false;
|
||||
error!("没有配置任何需要扫描的内容,程序空转没有意义");
|
||||
}
|
||||
if self.watch_later.enabled && !self.watch_later.path.is_absolute() {
|
||||
error!(
|
||||
"稍后再看保存的路径应为绝对路径,检测到:{}",
|
||||
self.watch_later.path.display()
|
||||
);
|
||||
}
|
||||
for path in self.favorite_list.values() {
|
||||
if !path.is_absolute() {
|
||||
ok = false;
|
||||
error!("收藏夹保存的路径应为绝对路径,检测到: {}", path.display());
|
||||
}
|
||||
}
|
||||
if !self.upper_path.is_absolute() {
|
||||
ok = false;
|
||||
error!("up 主头像保存的路径应为绝对路径");
|
||||
}
|
||||
if self.video_name.is_empty() {
|
||||
ok = false;
|
||||
error!("未设置 video_name 模板");
|
||||
}
|
||||
if self.page_name.is_empty() {
|
||||
ok = false;
|
||||
error!("未设置 page_name 模板");
|
||||
}
|
||||
let credential = self.credential.load();
|
||||
match credential.as_deref() {
|
||||
Some(credential) => {
|
||||
if credential.sessdata.is_empty()
|
||||
|| credential.bili_jct.is_empty()
|
||||
|| credential.buvid3.is_empty()
|
||||
|| credential.dedeuserid.is_empty()
|
||||
|| credential.ac_time_value.is_empty()
|
||||
{
|
||||
ok = false;
|
||||
error!("Credential 信息不完整,请确保填写完整");
|
||||
}
|
||||
}
|
||||
None => {
|
||||
ok = false;
|
||||
error!("未设置 Credential 信息");
|
||||
}
|
||||
}
|
||||
|
||||
if !ok {
|
||||
panic!(
|
||||
"位于 {} 的配置文件不合法,请参考提示信息修复后继续运行",
|
||||
CONFIG_DIR.join("config.toml").display()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn load() -> Result<Self> {
|
||||
let config_path = CONFIG_DIR.join("config.toml");
|
||||
let config_content = std::fs::read_to_string(config_path)?;
|
||||
Ok(toml::from_str(&config_content)?)
|
||||
}
|
||||
|
||||
pub fn save(&self) -> Result<()> {
|
||||
let config_path = CONFIG_DIR.join("config.toml");
|
||||
std::fs::create_dir_all(&*CONFIG_DIR)?;
|
||||
std::fs::write(config_path, toml::to_string_pretty(self)?)?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn serialize_collection_list<S>(
|
||||
collection_list: &HashMap<CollectionItem, PathBuf>,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let mut map = serializer.serialize_map(Some(collection_list.len()))?;
|
||||
for (k, v) in collection_list {
|
||||
let prefix = match k.collection_type {
|
||||
CollectionType::Series => "series",
|
||||
CollectionType::Season => "season",
|
||||
};
|
||||
map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
|
||||
fn deserialize_collection_list<'de, D>(deserializer: D) -> Result<HashMap<CollectionItem, PathBuf>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct CollectionListVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for CollectionListVisitor {
|
||||
type Value = HashMap<CollectionItem, PathBuf>;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("a map of collection list")
|
||||
}
|
||||
|
||||
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
||||
where
|
||||
A: MapAccess<'de>,
|
||||
{
|
||||
let mut collection_list = HashMap::new();
|
||||
while let Some((key, value)) = map.next_entry::<String, PathBuf>()? {
|
||||
let collection_item = match key.split(':').collect::<Vec<&str>>().as_slice() {
|
||||
[prefix, mid, sid] => {
|
||||
let collection_type = match *prefix {
|
||||
"series" => CollectionType::Series,
|
||||
"season" => CollectionType::Season,
|
||||
_ => {
|
||||
return Err(serde::de::Error::custom(
|
||||
"invalid collection type, should be series or season",
|
||||
))
|
||||
}
|
||||
};
|
||||
CollectionItem {
|
||||
mid: mid.to_string(),
|
||||
sid: sid.to_string(),
|
||||
collection_type,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(serde::de::Error::custom(
|
||||
"invalid collection key, should be series:mid:sid or season:mid:sid",
|
||||
))
|
||||
}
|
||||
};
|
||||
collection_list.insert(collection_item, value);
|
||||
}
|
||||
Ok(collection_list)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_map(CollectionListVisitor)
|
||||
}
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(version, about, long_about = None)]
|
||||
pub struct Args {
|
||||
#[arg(short, long, env = "SCAN_ONLY")]
|
||||
pub scan_only: bool,
|
||||
|
||||
#[arg(short, long, default_value = "None,bili_sync=info", env = "RUST_LOG")]
|
||||
pub log_level: String,
|
||||
}
|
||||
41
crates/bili_sync/src/config/clap.rs
Normal file
@@ -0,0 +1,41 @@
|
||||
use std::borrow::Cow;
|
||||
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "Bili-Sync", version = detail_version(), about, long_about = None)]
|
||||
pub struct Args {
|
||||
#[arg(short, long, env = "SCAN_ONLY")]
|
||||
pub scan_only: bool,
|
||||
|
||||
#[arg(short, long, default_value = "None,bili_sync=info", env = "RUST_LOG")]
|
||||
pub log_level: String,
|
||||
}
|
||||
|
||||
mod built_info {
|
||||
include!(concat!(env!("OUT_DIR"), "/built.rs"));
|
||||
}
|
||||
|
||||
pub fn version() -> Cow<'static, str> {
|
||||
if let (Some(git_version), Some(git_dirty)) = (built_info::GIT_VERSION, built_info::GIT_DIRTY) {
|
||||
Cow::Owned(format!("{}{}", git_version, if git_dirty { "-dirty" } else { "" }))
|
||||
} else {
|
||||
Cow::Borrowed(built_info::PKG_VERSION)
|
||||
}
|
||||
}
|
||||
|
||||
fn detail_version() -> String {
|
||||
format!(
|
||||
"{}
|
||||
Architecture: {}-{}
|
||||
Author: {}
|
||||
Built Time: {}
|
||||
Rustc Version: {}",
|
||||
version(),
|
||||
built_info::CFG_OS,
|
||||
built_info::CFG_TARGET_ARCH,
|
||||
built_info::PKG_AUTHORS,
|
||||
built_info::BUILT_TIME_UTC,
|
||||
built_info::RUSTC_VERSION,
|
||||
)
|
||||
}
|
||||
86
crates/bili_sync/src/config/global.rs
Normal file
@@ -0,0 +1,86 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use clap::Parser;
|
||||
use handlebars::handlebars_helper;
|
||||
use once_cell::sync::Lazy;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::config::clap::Args;
|
||||
use crate::config::item::PathSafeTemplate;
|
||||
|
||||
/// 全局的 CONFIG,可以从中读取配置信息
|
||||
pub static CONFIG: Lazy<Config> = Lazy::new(load_config);
|
||||
|
||||
/// 全局的 TEMPLATE,用来渲染 video_name 和 page_name 模板
|
||||
pub static TEMPLATE: Lazy<handlebars::Handlebars> = Lazy::new(|| {
|
||||
let mut handlebars = handlebars::Handlebars::new();
|
||||
handlebars_helper!(truncate: |s: String, len: usize| {
|
||||
if s.chars().count() > len {
|
||||
s.chars().take(len).collect::<String>()
|
||||
} else {
|
||||
s.to_string()
|
||||
}
|
||||
});
|
||||
handlebars.register_helper("truncate", Box::new(truncate));
|
||||
handlebars
|
||||
.path_safe_register("video", &CONFIG.video_name)
|
||||
.expect("failed to register video template");
|
||||
handlebars
|
||||
.path_safe_register("page", &CONFIG.page_name)
|
||||
.expect("failed to register page template");
|
||||
handlebars
|
||||
});
|
||||
|
||||
/// 全局的 ARGS,用来解析命令行参数
|
||||
pub static ARGS: Lazy<Args> = Lazy::new(Args::parse);
|
||||
|
||||
/// 全局的 CONFIG_DIR,表示配置文件夹的路径
|
||||
pub static CONFIG_DIR: Lazy<PathBuf> =
|
||||
Lazy::new(|| dirs::config_dir().expect("No config path found").join("bili-sync"));
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn load_config() -> Config {
|
||||
info!("开始加载配置文件..");
|
||||
let config = Config::load().unwrap_or_else(|err| {
|
||||
if err
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.is_none_or(|e| e.kind() != std::io::ErrorKind::NotFound)
|
||||
{
|
||||
panic!("加载配置文件失败,错误为: {err}");
|
||||
}
|
||||
warn!("配置文件不存在,使用默认配置..");
|
||||
Config::default()
|
||||
});
|
||||
info!("配置文件加载完毕,覆盖刷新原有配置");
|
||||
config.save().expect("保存默认配置时遇到错误");
|
||||
info!("检查配置文件..");
|
||||
config.check();
|
||||
info!("配置文件检查通过");
|
||||
config
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn load_config() -> Config {
|
||||
let credential = match (
|
||||
std::env::var("TEST_SESSDATA"),
|
||||
std::env::var("TEST_BILI_JCT"),
|
||||
std::env::var("TEST_BUVID3"),
|
||||
std::env::var("TEST_DEDEUSERID"),
|
||||
std::env::var("TEST_AC_TIME_VALUE"),
|
||||
) {
|
||||
(Ok(sessdata), Ok(bili_jct), Ok(buvid3), Ok(dedeuserid), Ok(ac_time_value)) => {
|
||||
Some(std::sync::Arc::new(crate::bilibili::Credential {
|
||||
sessdata,
|
||||
bili_jct,
|
||||
buvid3,
|
||||
dedeuserid,
|
||||
ac_time_value,
|
||||
}))
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Config {
|
||||
credential: arc_swap::ArcSwapOption::from(credential),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
139
crates/bili_sync/src/config/item.rs
Normal file
@@ -0,0 +1,139 @@
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Result;
|
||||
use serde::de::{Deserializer, MapAccess, Visitor};
|
||||
use serde::ser::SerializeMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::bilibili::{CollectionItem, CollectionType};
|
||||
use crate::utils::filenamify::filenamify;
|
||||
|
||||
/// 稍后再看的配置
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
pub struct WatchLaterConfig {
|
||||
pub enabled: bool,
|
||||
pub path: PathBuf,
|
||||
}
|
||||
|
||||
/// NFO 文件使用的时间类型
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum NFOTimeType {
|
||||
#[default]
|
||||
FavTime,
|
||||
PubTime,
|
||||
}
|
||||
|
||||
/// 并发下载相关的配置
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct ConcurrentLimit {
|
||||
pub video: usize,
|
||||
pub page: usize,
|
||||
pub rate_limit: Option<RateLimit>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct RateLimit {
|
||||
pub limit: usize,
|
||||
pub duration: u64,
|
||||
}
|
||||
|
||||
impl Default for ConcurrentLimit {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
video: 3,
|
||||
page: 2,
|
||||
// 默认的限速配置,每 250ms 允许请求 4 次
|
||||
rate_limit: Some(RateLimit {
|
||||
limit: 4,
|
||||
duration: 250,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait PathSafeTemplate {
|
||||
fn path_safe_register(&mut self, name: &'static str, template: &'static str) -> Result<()>;
|
||||
fn path_safe_render(&self, name: &'static str, data: &serde_json::Value) -> Result<String>;
|
||||
}
|
||||
|
||||
/// 通过将模板字符串中的分隔符替换为自定义的字符串,使得模板字符串中的分隔符得以保留
|
||||
impl PathSafeTemplate for handlebars::Handlebars<'_> {
|
||||
fn path_safe_register(&mut self, name: &'static str, template: &'static str) -> Result<()> {
|
||||
Ok(self.register_template_string(name, template.replace(std::path::MAIN_SEPARATOR_STR, "__SEP__"))?)
|
||||
}
|
||||
|
||||
fn path_safe_render(&self, name: &'static str, data: &serde_json::Value) -> Result<String> {
|
||||
Ok(filenamify(&self.render(name, data)?).replace("__SEP__", std::path::MAIN_SEPARATOR_STR))
|
||||
}
|
||||
}
|
||||
/* 后面是用于自定义 Collection 的序列化、反序列化的样板代码 */
|
||||
pub(super) fn serialize_collection_list<S>(
|
||||
collection_list: &HashMap<CollectionItem, PathBuf>,
|
||||
serializer: S,
|
||||
) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let mut map = serializer.serialize_map(Some(collection_list.len()))?;
|
||||
for (k, v) in collection_list {
|
||||
let prefix = match k.collection_type {
|
||||
CollectionType::Series => "series",
|
||||
CollectionType::Season => "season",
|
||||
};
|
||||
map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?;
|
||||
}
|
||||
map.end()
|
||||
}
|
||||
|
||||
pub(super) fn deserialize_collection_list<'de, D>(deserializer: D) -> Result<HashMap<CollectionItem, PathBuf>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
struct CollectionListVisitor;
|
||||
|
||||
impl<'de> Visitor<'de> for CollectionListVisitor {
|
||||
type Value = HashMap<CollectionItem, PathBuf>;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("a map of collection list")
|
||||
}
|
||||
|
||||
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
|
||||
where
|
||||
A: MapAccess<'de>,
|
||||
{
|
||||
let mut collection_list = HashMap::new();
|
||||
while let Some((key, value)) = map.next_entry::<String, PathBuf>()? {
|
||||
let collection_item = match key.split(':').collect::<Vec<&str>>().as_slice() {
|
||||
[prefix, mid, sid] => {
|
||||
let collection_type = match *prefix {
|
||||
"series" => CollectionType::Series,
|
||||
"season" => CollectionType::Season,
|
||||
_ => {
|
||||
return Err(serde::de::Error::custom(
|
||||
"invalid collection type, should be series or season",
|
||||
));
|
||||
}
|
||||
};
|
||||
CollectionItem {
|
||||
mid: mid.to_string(),
|
||||
sid: sid.to_string(),
|
||||
collection_type,
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(serde::de::Error::custom(
|
||||
"invalid collection key, should be series:mid:sid or season:mid:sid",
|
||||
));
|
||||
}
|
||||
};
|
||||
collection_list.insert(collection_item, value);
|
||||
}
|
||||
Ok(collection_list)
|
||||
}
|
||||
}
|
||||
|
||||
deserializer.deserialize_map(CollectionListVisitor)
|
||||
}
|
||||
184
crates/bili_sync/src/config/mod.rs
Normal file
@@ -0,0 +1,184 @@
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::Result;
|
||||
use arc_swap::ArcSwapOption;
|
||||
use rand::seq::SliceRandom;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
mod clap;
|
||||
mod global;
|
||||
mod item;
|
||||
|
||||
use crate::adapter::Args;
|
||||
use crate::bilibili::{CollectionItem, Credential, DanmakuOption, FilterOption};
|
||||
pub use crate::config::clap::version;
|
||||
pub use crate::config::global::{ARGS, CONFIG, CONFIG_DIR, TEMPLATE};
|
||||
use crate::config::item::{ConcurrentLimit, deserialize_collection_list, serialize_collection_list};
|
||||
pub use crate::config::item::{NFOTimeType, PathSafeTemplate, RateLimit, WatchLaterConfig};
|
||||
|
||||
fn default_time_format() -> String {
|
||||
"%Y-%m-%d".to_string()
|
||||
}
|
||||
|
||||
/// 默认的 auth_token 实现,生成随机 16 位字符串
|
||||
fn default_auth_token() -> Option<String> {
|
||||
let byte_choices = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()_+-=";
|
||||
let mut rng = rand::thread_rng();
|
||||
Some(
|
||||
(0..16)
|
||||
.map(|_| *(byte_choices.choose(&mut rng).expect("choose byte failed")) as char)
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
fn default_bind_address() -> String {
|
||||
"0.0.0.0:12345".to_string()
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct Config {
|
||||
#[serde(default = "default_auth_token")]
|
||||
pub auth_token: Option<String>,
|
||||
#[serde(default = "default_bind_address")]
|
||||
pub bind_address: String,
|
||||
pub credential: ArcSwapOption<Credential>,
|
||||
pub filter_option: FilterOption,
|
||||
#[serde(default)]
|
||||
pub danmaku_option: DanmakuOption,
|
||||
pub favorite_list: HashMap<String, PathBuf>,
|
||||
#[serde(
|
||||
default,
|
||||
serialize_with = "serialize_collection_list",
|
||||
deserialize_with = "deserialize_collection_list"
|
||||
)]
|
||||
pub collection_list: HashMap<CollectionItem, PathBuf>,
|
||||
#[serde(default)]
|
||||
pub submission_list: HashMap<String, PathBuf>,
|
||||
#[serde(default)]
|
||||
pub watch_later: WatchLaterConfig,
|
||||
pub video_name: Cow<'static, str>,
|
||||
pub page_name: Cow<'static, str>,
|
||||
pub interval: u64,
|
||||
pub upper_path: PathBuf,
|
||||
#[serde(default)]
|
||||
pub nfo_time_type: NFOTimeType,
|
||||
#[serde(default)]
|
||||
pub concurrent_limit: ConcurrentLimit,
|
||||
#[serde(default = "default_time_format")]
|
||||
pub time_format: String,
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
auth_token: default_auth_token(),
|
||||
bind_address: default_bind_address(),
|
||||
credential: ArcSwapOption::from(Some(Arc::new(Credential::default()))),
|
||||
filter_option: FilterOption::default(),
|
||||
danmaku_option: DanmakuOption::default(),
|
||||
favorite_list: HashMap::new(),
|
||||
collection_list: HashMap::new(),
|
||||
submission_list: HashMap::new(),
|
||||
watch_later: Default::default(),
|
||||
video_name: Cow::Borrowed("{{title}}"),
|
||||
page_name: Cow::Borrowed("{{bvid}}"),
|
||||
interval: 1200,
|
||||
upper_path: CONFIG_DIR.join("upper_face"),
|
||||
nfo_time_type: NFOTimeType::FavTime,
|
||||
concurrent_limit: ConcurrentLimit::default(),
|
||||
time_format: default_time_format(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub fn save(&self) -> Result<()> {
|
||||
let config_path = CONFIG_DIR.join("config.toml");
|
||||
std::fs::create_dir_all(&*CONFIG_DIR)?;
|
||||
std::fs::write(config_path, toml::to_string_pretty(self)?)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
fn load() -> Result<Self> {
|
||||
let config_path = CONFIG_DIR.join("config.toml");
|
||||
let config_content = std::fs::read_to_string(config_path)?;
|
||||
Ok(toml::from_str(&config_content)?)
|
||||
}
|
||||
|
||||
pub fn as_video_sources(&self) -> Vec<(Args<'_>, &PathBuf)> {
|
||||
let mut params = Vec::new();
|
||||
self.favorite_list
|
||||
.iter()
|
||||
.for_each(|(fid, path)| params.push((Args::Favorite { fid }, path)));
|
||||
self.collection_list
|
||||
.iter()
|
||||
.for_each(|(collection_item, path)| params.push((Args::Collection { collection_item }, path)));
|
||||
if self.watch_later.enabled {
|
||||
params.push((Args::WatchLater, &self.watch_later.path));
|
||||
}
|
||||
self.submission_list
|
||||
.iter()
|
||||
.for_each(|(upper_id, path)| params.push((Args::Submission { upper_id }, path)));
|
||||
params
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
pub fn check(&self) {
|
||||
let mut ok = true;
|
||||
let video_sources = self.as_video_sources();
|
||||
if video_sources.is_empty() {
|
||||
ok = false;
|
||||
error!("没有配置任何需要扫描的内容,程序空转没有意义");
|
||||
}
|
||||
for (args, path) in video_sources {
|
||||
if !path.is_absolute() {
|
||||
ok = false;
|
||||
error!("{:?} 保存的路径应为绝对路径,检测到: {}", args, path.display());
|
||||
}
|
||||
}
|
||||
if !self.upper_path.is_absolute() {
|
||||
ok = false;
|
||||
error!("up 主头像保存的路径应为绝对路径");
|
||||
}
|
||||
if self.video_name.is_empty() {
|
||||
ok = false;
|
||||
error!("未设置 video_name 模板");
|
||||
}
|
||||
if self.page_name.is_empty() {
|
||||
ok = false;
|
||||
error!("未设置 page_name 模板");
|
||||
}
|
||||
let credential = self.credential.load();
|
||||
match credential.as_deref() {
|
||||
Some(credential) => {
|
||||
if credential.sessdata.is_empty()
|
||||
|| credential.bili_jct.is_empty()
|
||||
|| credential.buvid3.is_empty()
|
||||
|| credential.dedeuserid.is_empty()
|
||||
|| credential.ac_time_value.is_empty()
|
||||
{
|
||||
ok = false;
|
||||
error!("Credential 信息不完整,请确保填写完整");
|
||||
}
|
||||
}
|
||||
None => {
|
||||
ok = false;
|
||||
error!("未设置 Credential 信息");
|
||||
}
|
||||
}
|
||||
if !(self.concurrent_limit.video > 0 && self.concurrent_limit.page > 0) {
|
||||
ok = false;
|
||||
error!("video 和 page 允许的并发数必须大于 0");
|
||||
}
|
||||
if !ok {
|
||||
panic!(
|
||||
"位于 {} 的配置文件不合法,请参考提示信息修复后继续运行",
|
||||
CONFIG_DIR.join("config.toml").display()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ fn database_url() -> String {
|
||||
format!("sqlite://{}?mode=rwc", CONFIG_DIR.join("data.sqlite").to_string_lossy())
|
||||
}
|
||||
|
||||
pub async fn database_connection() -> Result<DatabaseConnection> {
|
||||
async fn database_connection() -> Result<DatabaseConnection> {
|
||||
let mut option = ConnectOptions::new(database_url());
|
||||
option
|
||||
.max_connections(100)
|
||||
@@ -17,9 +17,15 @@ pub async fn database_connection() -> Result<DatabaseConnection> {
|
||||
Ok(Database::connect(option).await?)
|
||||
}
|
||||
|
||||
pub async fn migrate_database() -> Result<()> {
|
||||
async fn migrate_database() -> Result<()> {
|
||||
// 注意此处使用内部构造的 DatabaseConnection,而不是通过 database_connection() 获取
|
||||
// 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题,而使用默认的连接选项不会
|
||||
let connection = Database::connect(database_url()).await?;
|
||||
Ok(Migrator::up(&connection, None).await?)
|
||||
}
|
||||
|
||||
/// 进行数据库迁移并获取数据库连接,供外部使用
|
||||
pub async fn setup_database() -> DatabaseConnection {
|
||||
migrate_database().await.expect("数据库迁移失败");
|
||||
database_connection().await.expect("获取数据库连接失败")
|
||||
}
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
use core::str;
|
||||
use std::path::Path;
|
||||
|
||||
use anyhow::{anyhow, Result};
|
||||
use futures::StreamExt;
|
||||
use anyhow::{Result, bail, ensure};
|
||||
use futures::TryStreamExt;
|
||||
use reqwest::Method;
|
||||
use tokio::fs::{self, File};
|
||||
use tokio::io;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
use crate::bilibili::Client;
|
||||
pub struct Downloader {
|
||||
@@ -24,10 +26,22 @@ impl Downloader {
|
||||
fs::create_dir_all(parent).await?;
|
||||
}
|
||||
let mut file = File::create(path).await?;
|
||||
let mut res = self.client.request(Method::GET, url, None).send().await?.bytes_stream();
|
||||
while let Some(item) = res.next().await {
|
||||
io::copy(&mut item?.as_ref(), &mut file).await?;
|
||||
}
|
||||
let resp = self
|
||||
.client
|
||||
.request(Method::GET, url, None)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
let expected = resp.content_length().unwrap_or_default();
|
||||
let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
|
||||
let received = tokio::io::copy(&mut stream_reader, &mut file).await?;
|
||||
file.flush().await?;
|
||||
ensure!(
|
||||
received >= expected,
|
||||
"received {} bytes, expected {} bytes",
|
||||
received,
|
||||
expected
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -35,24 +49,19 @@ impl Downloader {
|
||||
let output = tokio::process::Command::new("ffmpeg")
|
||||
.args([
|
||||
"-i",
|
||||
video_path.to_str().unwrap(),
|
||||
video_path.to_string_lossy().as_ref(),
|
||||
"-i",
|
||||
audio_path.to_str().unwrap(),
|
||||
audio_path.to_string_lossy().as_ref(),
|
||||
"-c",
|
||||
"copy",
|
||||
"-y",
|
||||
output_path.to_str().unwrap(),
|
||||
output_path.to_string_lossy().as_ref(),
|
||||
])
|
||||
.output()
|
||||
.await?;
|
||||
if !output.status.success() {
|
||||
return match String::from_utf8(output.stderr) {
|
||||
Ok(err) => Err(anyhow!(err)),
|
||||
_ => Err(anyhow!("ffmpeg error")),
|
||||
};
|
||||
bail!("ffmpeg error: {}", str::from_utf8(&output.stderr).unwrap_or("unknown"));
|
||||
}
|
||||
let _ = fs::remove_file(video_path).await;
|
||||
let _ = fs::remove_file(audio_path).await;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
use std::io;
|
||||
|
||||
use anyhow::Result;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
@@ -7,3 +10,41 @@ pub struct DownloadAbortError();
|
||||
#[derive(Error, Debug)]
|
||||
#[error("Process page error")]
|
||||
pub struct ProcessPageError();
|
||||
|
||||
pub enum ExecutionStatus {
|
||||
Skipped,
|
||||
Succeeded,
|
||||
Ignored(anyhow::Error),
|
||||
Failed(anyhow::Error),
|
||||
// 任务可以返回该状态固定自己的 status
|
||||
FixedFailed(u32, anyhow::Error),
|
||||
}
|
||||
|
||||
// 目前 stable rust 似乎不支持自定义类型使用 ? 运算符,只能先在返回值使用 Result,再这样套层娃
|
||||
impl From<Result<ExecutionStatus>> for ExecutionStatus {
|
||||
fn from(res: Result<ExecutionStatus>) -> Self {
|
||||
match res {
|
||||
Ok(status) => status,
|
||||
Err(err) => {
|
||||
if let Some(error) = err.downcast_ref::<io::Error>() {
|
||||
let error_kind = error.kind();
|
||||
if error_kind == io::ErrorKind::PermissionDenied
|
||||
|| (error_kind == io::ErrorKind::Other
|
||||
&& error.get_ref().is_some_and(|e| {
|
||||
e.downcast_ref::<reqwest::Error>()
|
||||
.is_some_and(|e| e.is_decode() || e.is_body() || e.is_timeout())
|
||||
}))
|
||||
{
|
||||
return ExecutionStatus::Ignored(err);
|
||||
}
|
||||
}
|
||||
if let Some(error) = err.downcast_ref::<reqwest::Error>() {
|
||||
if error.is_decode() || error.is_body() || error.is_timeout() {
|
||||
return ExecutionStatus::Ignored(err);
|
||||
}
|
||||
}
|
||||
ExecutionStatus::Failed(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,78 +2,81 @@
|
||||
extern crate tracing;
|
||||
|
||||
mod adapter;
|
||||
mod api;
|
||||
mod bilibili;
|
||||
mod config;
|
||||
mod database;
|
||||
mod downloader;
|
||||
mod error;
|
||||
mod task;
|
||||
mod utils;
|
||||
mod workflow;
|
||||
use std::time::Duration;
|
||||
|
||||
use std::fmt::Debug;
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
|
||||
use once_cell::sync::Lazy;
|
||||
use tokio::time;
|
||||
use task::{http_server, video_downloader};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::TaskTracker;
|
||||
|
||||
use crate::adapter::Args;
|
||||
use crate::bilibili::BiliClient;
|
||||
use crate::config::{ARGS, CONFIG};
|
||||
use crate::database::{database_connection, migrate_database};
|
||||
use crate::database::setup_database;
|
||||
use crate::utils::init_logger;
|
||||
use crate::workflow::process_video_list;
|
||||
use crate::utils::signal::terminate;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
init_logger(&ARGS.log_level);
|
||||
Lazy::force(&CONFIG);
|
||||
migrate_database().await.expect("数据库迁移失败");
|
||||
let connection = database_connection().await.expect("获取数据库连接失败");
|
||||
let mut anchor = chrono::Local::now().date_naive();
|
||||
let bili_client = BiliClient::new();
|
||||
let watch_later_config = &CONFIG.watch_later;
|
||||
loop {
|
||||
'inner: {
|
||||
match bili_client.wbi_img().await.map(|wbi_img| wbi_img.into_mixin_key()) {
|
||||
Ok(Some(mixin_key)) => bilibili::set_global_mixin_key(mixin_key),
|
||||
Ok(_) => {
|
||||
error!("获取 mixin key 失败,无法进行 wbi 签名,等待下一轮执行");
|
||||
break 'inner;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("获取 mixin key 时遇到错误:{e},等待下一轮执行");
|
||||
break 'inner;
|
||||
}
|
||||
};
|
||||
if anchor != chrono::Local::now().date_naive() {
|
||||
if let Err(e) = bili_client.check_refresh().await {
|
||||
error!("检查刷新 Credential 遇到错误:{e},等待下一轮执行");
|
||||
break 'inner;
|
||||
}
|
||||
anchor = chrono::Local::now().date_naive();
|
||||
init();
|
||||
let connection = Arc::new(setup_database().await);
|
||||
let token = CancellationToken::new();
|
||||
let tracker = TaskTracker::new();
|
||||
|
||||
spawn_task("HTTP 服务", http_server(connection.clone()), &tracker, token.clone());
|
||||
spawn_task("定时下载", video_downloader(connection), &tracker, token.clone());
|
||||
|
||||
tracker.close();
|
||||
handle_shutdown(tracker, token).await
|
||||
}
|
||||
|
||||
fn spawn_task(
|
||||
task_name: &'static str,
|
||||
task: impl Future<Output = impl Debug> + Send + 'static,
|
||||
tracker: &TaskTracker,
|
||||
token: CancellationToken,
|
||||
) {
|
||||
tracker.spawn(async move {
|
||||
tokio::select! {
|
||||
res = task => {
|
||||
error!("「{}」异常结束,返回结果为:「{:?}」,取消其它仍在执行的任务..", task_name, res);
|
||||
token.cancel();
|
||||
},
|
||||
_ = token.cancelled() => {
|
||||
info!("「{}」接收到取消信号,终止运行..", task_name);
|
||||
}
|
||||
for (fid, path) in &CONFIG.favorite_list {
|
||||
if let Err(e) = process_video_list(Args::Favorite { fid }, &bili_client, path, &connection).await {
|
||||
error!("处理收藏夹 {fid} 时遇到非预期的错误:{e}");
|
||||
}
|
||||
}
|
||||
info!("所有收藏夹处理完毕");
|
||||
for (collection_item, path) in &CONFIG.collection_list {
|
||||
if let Err(e) =
|
||||
process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await
|
||||
{
|
||||
error!("处理合集 {collection_item:?} 时遇到非预期的错误:{e}");
|
||||
}
|
||||
}
|
||||
info!("所有合集处理完毕");
|
||||
if watch_later_config.enabled {
|
||||
if let Err(e) =
|
||||
process_video_list(Args::WatchLater, &bili_client, &watch_later_config.path, &connection).await
|
||||
{
|
||||
error!("处理稍后再看时遇到非预期的错误:{e}");
|
||||
}
|
||||
}
|
||||
info!("稍后再看处理完毕");
|
||||
info!("本轮任务执行完毕,等待下一轮执行");
|
||||
}
|
||||
time::sleep(Duration::from_secs(CONFIG.interval)).await;
|
||||
});
|
||||
}
|
||||
|
||||
/// 初始化日志系统,打印欢迎信息,加载配置文件
|
||||
fn init() {
|
||||
init_logger(&ARGS.log_level);
|
||||
info!("欢迎使用 Bili-Sync,当前程序版本:{}", config::version());
|
||||
info!("项目地址:https://github.com/amtoaer/bili-sync");
|
||||
Lazy::force(&CONFIG);
|
||||
}
|
||||
|
||||
async fn handle_shutdown(tracker: TaskTracker, token: CancellationToken) {
|
||||
tokio::select! {
|
||||
_ = tracker.wait() => {
|
||||
error!("所有任务均已终止,程序退出")
|
||||
}
|
||||
_ = terminate() => {
|
||||
info!("接收到终止信号,正在终止任务..");
|
||||
token.cancel();
|
||||
tracker.wait().await;
|
||||
info!("所有任务均已终止,程序退出");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
61
crates/bili_sync/src/task/http_server.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use axum::extract::Request;
|
||||
use axum::http::{Uri, header};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Extension, Router, ServiceExt, middleware};
|
||||
use reqwest::StatusCode;
|
||||
use rust_embed::Embed;
|
||||
use sea_orm::DatabaseConnection;
|
||||
use utoipa::OpenApi;
|
||||
use utoipa_swagger_ui::{Config, SwaggerUi};
|
||||
|
||||
use crate::api::auth;
|
||||
use crate::api::handler::{ApiDoc, get_video, get_video_sources, get_videos, reset_video};
|
||||
use crate::config::CONFIG;
|
||||
|
||||
#[derive(Embed)]
|
||||
#[folder = "../../web/build"]
|
||||
struct Asset;
|
||||
|
||||
pub async fn http_server(database_connection: Arc<DatabaseConnection>) -> Result<()> {
|
||||
let app = Router::new()
|
||||
.route("/api/video-sources", get(get_video_sources))
|
||||
.route("/api/videos", get(get_videos))
|
||||
.route("/api/videos/{id}", get(get_video))
|
||||
.route("/api/videos/{id}/reset", post(reset_video))
|
||||
.merge(
|
||||
SwaggerUi::new("/swagger-ui/")
|
||||
.url("/api-docs/openapi.json", ApiDoc::openapi())
|
||||
.config(
|
||||
Config::default()
|
||||
.try_it_out_enabled(true)
|
||||
.persist_authorization(true)
|
||||
.validator_url("none"),
|
||||
),
|
||||
)
|
||||
.fallback_service(get(frontend_files))
|
||||
.layer(Extension(database_connection))
|
||||
.layer(middleware::from_fn(auth::auth));
|
||||
let listener = tokio::net::TcpListener::bind(&CONFIG.bind_address)
|
||||
.await
|
||||
.context("bind address failed")?;
|
||||
info!("开始运行管理页: http://{}", CONFIG.bind_address);
|
||||
Ok(axum::serve(listener, ServiceExt::<Request>::into_make_service(app)).await?)
|
||||
}
|
||||
|
||||
async fn frontend_files(uri: Uri) -> impl IntoResponse {
|
||||
let mut path = uri.path().trim_start_matches('/');
|
||||
if path.is_empty() {
|
||||
path = "index.html";
|
||||
}
|
||||
match Asset::get(path) {
|
||||
Some(content) => {
|
||||
let mime = mime_guess::from_path(path).first_or_octet_stream();
|
||||
([(header::CONTENT_TYPE, mime.as_ref())], content.data).into_response()
|
||||
}
|
||||
None => (StatusCode::NOT_FOUND, "404 Not Found").into_response(),
|
||||
}
|
||||
}
|
||||
5
crates/bili_sync/src/task/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
mod http_server;
|
||||
mod video_downloader;
|
||||
|
||||
pub use http_server::http_server;
|
||||
pub use video_downloader::video_downloader;
|
||||
45
crates/bili_sync/src/task/video_downloader.rs
Normal file
@@ -0,0 +1,45 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use sea_orm::DatabaseConnection;
|
||||
use tokio::time;
|
||||
|
||||
use crate::bilibili::{self, BiliClient};
|
||||
use crate::config::CONFIG;
|
||||
use crate::workflow::process_video_source;
|
||||
|
||||
/// 启动周期下载视频的任务
|
||||
pub async fn video_downloader(connection: Arc<DatabaseConnection>) {
|
||||
let mut anchor = chrono::Local::now().date_naive();
|
||||
let bili_client = BiliClient::new();
|
||||
let video_sources = CONFIG.as_video_sources();
|
||||
loop {
|
||||
info!("开始执行本轮视频下载任务..");
|
||||
'inner: {
|
||||
match bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) {
|
||||
Ok(Some(mixin_key)) => bilibili::set_global_mixin_key(mixin_key),
|
||||
Ok(_) => {
|
||||
error!("解析 mixin key 失败,等待下一轮执行");
|
||||
break 'inner;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("获取 mixin key 遇到错误:{:#},等待下一轮执行", e);
|
||||
break 'inner;
|
||||
}
|
||||
};
|
||||
if anchor != chrono::Local::now().date_naive() {
|
||||
if let Err(e) = bili_client.check_refresh().await {
|
||||
error!("检查刷新 Credential 遇到错误:{:#},等待下一轮执行", e);
|
||||
break 'inner;
|
||||
}
|
||||
anchor = chrono::Local::now().date_naive();
|
||||
}
|
||||
for (args, path) in &video_sources {
|
||||
if let Err(e) = process_video_source(*args, &bili_client, path, &connection).await {
|
||||
error!("处理过程遇到错误:{:#}", e);
|
||||
}
|
||||
}
|
||||
info!("本轮任务执行完毕,等待下一轮执行");
|
||||
}
|
||||
time::sleep(time::Duration::from_secs(CONFIG.interval)).await;
|
||||
}
|
||||
}
|
||||
@@ -1,39 +1,34 @@
|
||||
use sea_orm::ActiveValue::NotSet;
|
||||
use sea_orm::{IntoActiveModel, Set};
|
||||
use serde_json::json;
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use sea_orm::ActiveValue::{NotSet, Set};
|
||||
use sea_orm::IntoActiveModel;
|
||||
|
||||
use crate::bilibili::VideoInfo;
|
||||
use crate::utils::id_time_key;
|
||||
use crate::bilibili::{PageInfo, VideoInfo};
|
||||
|
||||
impl VideoInfo {
|
||||
/// 将 VideoInfo 转换为 ActiveModel
|
||||
pub fn to_model(&self, base_model: Option<bili_sync_entity::video::Model>) -> bili_sync_entity::video::ActiveModel {
|
||||
let base_model = match base_model {
|
||||
Some(base_model) => base_model.into_active_model(),
|
||||
None => {
|
||||
let mut tmp_model = bili_sync_entity::video::Model::default().into_active_model();
|
||||
// 注意此处要把 id 和 created_at 设置为 NotSet,方便在 sql 中忽略这些字段,交由数据库自动生成
|
||||
tmp_model.id = NotSet;
|
||||
tmp_model.created_at = NotSet;
|
||||
tmp_model
|
||||
}
|
||||
/// 在检测视频更新时,通过该方法将 VideoInfo 转换为简单的 ActiveModel,此处仅填充一些简单信息,后续会使用详情覆盖
|
||||
pub fn into_simple_model(self) -> bili_sync_entity::video::ActiveModel {
|
||||
let default = bili_sync_entity::video::ActiveModel {
|
||||
id: NotSet,
|
||||
created_at: NotSet,
|
||||
// 此处不使用 ActiveModel::default() 是为了让其它字段有默认值
|
||||
..bili_sync_entity::video::Model::default().into_active_model()
|
||||
};
|
||||
match self {
|
||||
VideoInfo::Simple {
|
||||
VideoInfo::Collection {
|
||||
bvid,
|
||||
cover,
|
||||
ctime,
|
||||
pubtime,
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid.clone()),
|
||||
cover: Set(cover.clone()),
|
||||
bvid: Set(bvid),
|
||||
cover: Set(cover),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
pubtime: Set(pubtime.naive_utc()),
|
||||
category: Set(2), // 视频合集里的内容类型肯定是视频
|
||||
valid: Set(true),
|
||||
..base_model
|
||||
..default
|
||||
},
|
||||
VideoInfo::Detail {
|
||||
VideoInfo::Favorite {
|
||||
title,
|
||||
vtype,
|
||||
bvid,
|
||||
@@ -45,50 +40,20 @@ impl VideoInfo {
|
||||
pubtime,
|
||||
attr,
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid.clone()),
|
||||
name: Set(title.clone()),
|
||||
category: Set(*vtype),
|
||||
intro: Set(intro.clone()),
|
||||
cover: Set(cover.clone()),
|
||||
bvid: Set(bvid),
|
||||
name: Set(title),
|
||||
category: Set(vtype),
|
||||
intro: Set(intro),
|
||||
cover: Set(cover),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
pubtime: Set(pubtime.naive_utc()),
|
||||
favtime: Set(fav_time.naive_utc()),
|
||||
download_status: Set(0),
|
||||
valid: Set(*attr == 0),
|
||||
tags: Set(None),
|
||||
single_page: Set(None),
|
||||
valid: Set(attr == 0),
|
||||
upper_id: Set(upper.mid),
|
||||
upper_name: Set(upper.name.clone()),
|
||||
upper_face: Set(upper.face.clone()),
|
||||
..base_model
|
||||
},
|
||||
VideoInfo::View {
|
||||
title,
|
||||
bvid,
|
||||
intro,
|
||||
cover,
|
||||
upper,
|
||||
ctime,
|
||||
pubtime,
|
||||
state,
|
||||
..
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid.clone()),
|
||||
name: Set(title.clone()),
|
||||
category: Set(2), // 视频合集里的内容类型肯定是视频
|
||||
intro: Set(intro.clone()),
|
||||
cover: Set(cover.clone()),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
pubtime: Set(pubtime.naive_utc()),
|
||||
favtime: Set(pubtime.naive_utc()), // 合集不包括 fav_time,使用发布时间代替
|
||||
download_status: Set(0),
|
||||
valid: Set(*state == 0),
|
||||
tags: Set(None),
|
||||
single_page: Set(None),
|
||||
upper_id: Set(upper.mid),
|
||||
upper_name: Set(upper.name.clone()),
|
||||
upper_face: Set(upper.face.clone()),
|
||||
..base_model
|
||||
upper_name: Set(upper.name),
|
||||
upper_face: Set(upper.face),
|
||||
..default
|
||||
},
|
||||
VideoInfo::WatchLater {
|
||||
title,
|
||||
@@ -101,68 +66,117 @@ impl VideoInfo {
|
||||
pubtime,
|
||||
state,
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid.clone()),
|
||||
name: Set(title.clone()),
|
||||
bvid: Set(bvid),
|
||||
name: Set(title),
|
||||
category: Set(2), // 稍后再看里的内容类型肯定是视频
|
||||
intro: Set(intro.clone()),
|
||||
cover: Set(cover.clone()),
|
||||
intro: Set(intro),
|
||||
cover: Set(cover),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
pubtime: Set(pubtime.naive_utc()),
|
||||
favtime: Set(fav_time.naive_utc()),
|
||||
download_status: Set(0),
|
||||
valid: Set(*state == 0),
|
||||
tags: Set(None),
|
||||
single_page: Set(None),
|
||||
valid: Set(state == 0),
|
||||
upper_id: Set(upper.mid),
|
||||
upper_name: Set(upper.name.clone()),
|
||||
upper_face: Set(upper.face.clone()),
|
||||
..base_model
|
||||
upper_name: Set(upper.name),
|
||||
upper_face: Set(upper.face),
|
||||
..default
|
||||
},
|
||||
VideoInfo::Submission {
|
||||
title,
|
||||
bvid,
|
||||
intro,
|
||||
cover,
|
||||
ctime,
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid),
|
||||
name: Set(title),
|
||||
intro: Set(intro),
|
||||
cover: Set(cover),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
category: Set(2), // 投稿视频的内容类型肯定是视频
|
||||
valid: Set(true),
|
||||
..default
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_fmt_args(&self) -> Option<serde_json::Value> {
|
||||
match self {
|
||||
VideoInfo::Simple { .. } => None, // 不能从简单的视频信息中构造格式化参数
|
||||
VideoInfo::Detail { title, bvid, upper, .. } => Some(json!({
|
||||
"bvid": &bvid,
|
||||
"title": &title,
|
||||
"upper_name": &upper.name,
|
||||
"upper_mid": &upper.mid,
|
||||
})),
|
||||
VideoInfo::View { title, bvid, upper, .. } => Some(json!({
|
||||
"bvid": &bvid,
|
||||
"title": &title,
|
||||
"upper_name": &upper.name,
|
||||
"upper_mid": &upper.mid,
|
||||
})),
|
||||
VideoInfo::WatchLater { title, bvid, upper, .. } => Some(json!({
|
||||
"bvid": &bvid,
|
||||
"title": &title,
|
||||
"upper_name": &upper.name,
|
||||
"upper_mid": &upper.mid,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn video_key(&self) -> String {
|
||||
match self {
|
||||
// 对于合集没有 fav_time,只能用 pubtime 代替
|
||||
VideoInfo::Simple { bvid, pubtime, .. } => id_time_key(bvid, pubtime),
|
||||
VideoInfo::Detail { bvid, fav_time, .. } => id_time_key(bvid, fav_time),
|
||||
VideoInfo::WatchLater { bvid, fav_time, .. } => id_time_key(bvid, fav_time),
|
||||
// 详情接口返回的数据仅用于填充详情,不会被作为 video_key
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bvid(&self) -> &str {
|
||||
/// 填充视频详情时调用,该方法会将视频详情附加到原有的 Model 上
|
||||
/// 特殊地,如果在检测视频更新时记录了 favtime,那么 favtime 会维持原样,否则会使用 pubtime 填充
|
||||
pub fn into_detail_model(self, base_model: bili_sync_entity::video::Model) -> bili_sync_entity::video::ActiveModel {
|
||||
match self {
|
||||
VideoInfo::Simple { bvid, .. } => bvid,
|
||||
VideoInfo::Detail { bvid, .. } => bvid,
|
||||
VideoInfo::WatchLater { bvid, .. } => bvid,
|
||||
// 同上
|
||||
VideoInfo::Detail {
|
||||
title,
|
||||
bvid,
|
||||
intro,
|
||||
cover,
|
||||
upper,
|
||||
ctime,
|
||||
pubtime,
|
||||
state,
|
||||
..
|
||||
} => bili_sync_entity::video::ActiveModel {
|
||||
bvid: Set(bvid),
|
||||
name: Set(title),
|
||||
category: Set(2),
|
||||
intro: Set(intro),
|
||||
cover: Set(cover),
|
||||
ctime: Set(ctime.naive_utc()),
|
||||
pubtime: Set(pubtime.naive_utc()),
|
||||
favtime: if base_model.favtime != NaiveDateTime::default() {
|
||||
NotSet // 之前设置了 favtime,不覆盖
|
||||
} else {
|
||||
Set(pubtime.naive_utc()) // 未设置过 favtime,使用 pubtime 填充
|
||||
},
|
||||
download_status: Set(0),
|
||||
valid: Set(state == 0),
|
||||
upper_id: Set(upper.mid),
|
||||
upper_name: Set(upper.name),
|
||||
upper_face: Set(upper.face),
|
||||
..base_model.into_active_model()
|
||||
},
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
/// 获取视频的发布时间,用于对时间做筛选检查新视频
|
||||
pub fn release_datetime(&self) -> &DateTime<Utc> {
|
||||
match self {
|
||||
VideoInfo::Collection { pubtime: time, .. }
|
||||
| VideoInfo::Favorite { fav_time: time, .. }
|
||||
| VideoInfo::WatchLater { fav_time: time, .. }
|
||||
| VideoInfo::Submission { ctime: time, .. } => time,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PageInfo {
|
||||
pub fn into_active_model(
|
||||
self,
|
||||
video_model: &bili_sync_entity::video::Model,
|
||||
) -> bili_sync_entity::page::ActiveModel {
|
||||
let (width, height) = match &self.dimension {
|
||||
Some(d) => {
|
||||
if d.rotate == 0 {
|
||||
(Some(d.width), Some(d.height))
|
||||
} else {
|
||||
(Some(d.height), Some(d.width))
|
||||
}
|
||||
}
|
||||
None => (None, None),
|
||||
};
|
||||
bili_sync_entity::page::ActiveModel {
|
||||
video_id: Set(video_model.id),
|
||||
cid: Set(self.cid),
|
||||
pid: Set(self.page),
|
||||
name: Set(self.name),
|
||||
width: Set(width),
|
||||
height: Set(height),
|
||||
duration: Set(self.duration),
|
||||
image: Set(self.first_frame),
|
||||
download_status: Set(0),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
61
crates/bili_sync/src/utils/filenamify.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
macro_rules! regex {
|
||||
($re:literal $(,)?) => {{
|
||||
static RE: once_cell::sync::OnceCell<regex::Regex> = once_cell::sync::OnceCell::new();
|
||||
RE.get_or_init(|| regex::Regex::new($re).expect("invalid regex"))
|
||||
}};
|
||||
}
|
||||
|
||||
pub fn filenamify<S: AsRef<str>>(input: S) -> String {
|
||||
let reserved = regex!("[<>:\"/\\\\|?*\u{0000}-\u{001F}\u{007F}\u{0080}-\u{009F}]+");
|
||||
let windows_reserved = regex!("^(con|prn|aux|nul|com\\d|lpt\\d)$");
|
||||
let outer_periods = regex!("^\\.+|\\.+$");
|
||||
|
||||
let replacement = "_";
|
||||
|
||||
let input = reserved.replace_all(input.as_ref(), replacement);
|
||||
let input = outer_periods.replace_all(input.as_ref(), replacement);
|
||||
|
||||
let mut result = input.into_owned();
|
||||
if windows_reserved.is_match(result.as_str()) {
|
||||
result.push_str(replacement);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::filenamify;
|
||||
|
||||
#[test]
|
||||
fn test_filenamify() {
|
||||
assert_eq!(filenamify("foo/bar"), "foo_bar");
|
||||
assert_eq!(filenamify("foo//bar"), "foo_bar");
|
||||
assert_eq!(filenamify("//foo//bar//"), "_foo_bar_");
|
||||
assert_eq!(filenamify("foo\\bar"), "foo_bar");
|
||||
assert_eq!(filenamify("foo\\\\\\bar"), "foo_bar");
|
||||
assert_eq!(filenamify(r"foo\\bar"), "foo_bar");
|
||||
assert_eq!(filenamify(r"foo\\\\\\bar"), "foo_bar");
|
||||
assert_eq!(filenamify("////foo////bar////"), "_foo_bar_");
|
||||
assert_eq!(filenamify("foo\u{0000}bar"), "foo_bar");
|
||||
assert_eq!(filenamify("\"foo<>bar*"), "_foo_bar_");
|
||||
assert_eq!(filenamify("."), "_");
|
||||
assert_eq!(filenamify(".."), "_");
|
||||
assert_eq!(filenamify("./"), "__");
|
||||
assert_eq!(filenamify("../"), "__");
|
||||
assert_eq!(filenamify("../../foo/bar"), "__.._foo_bar");
|
||||
assert_eq!(filenamify("foo.bar."), "foo.bar_");
|
||||
assert_eq!(filenamify("foo.bar.."), "foo.bar_");
|
||||
assert_eq!(filenamify("foo.bar..."), "foo.bar_");
|
||||
assert_eq!(filenamify("con"), "con_");
|
||||
assert_eq!(filenamify("com1"), "com1_");
|
||||
assert_eq!(filenamify(":nul|"), "_nul_");
|
||||
assert_eq!(filenamify("foo/bar/nul"), "foo_bar_nul");
|
||||
assert_eq!(filenamify("file:///file.tar.gz"), "file_file.tar.gz");
|
||||
assert_eq!(filenamify("http://www.google.com"), "http_www.google.com");
|
||||
assert_eq!(
|
||||
filenamify("https://www.youtube.com/watch?v=dQw4w9WgXcQ"),
|
||||
"https_www.youtube.com_watch_v=dQw4w9WgXcQ"
|
||||
);
|
||||
}
|
||||
}
|
||||
30
crates/bili_sync/src/utils/format_arg.rs
Normal file
@@ -0,0 +1,30 @@
|
||||
use serde_json::json;
|
||||
|
||||
use crate::config::CONFIG;
|
||||
|
||||
pub fn video_format_args(video_model: &bili_sync_entity::video::Model) -> serde_json::Value {
|
||||
json!({
|
||||
"bvid": &video_model.bvid,
|
||||
"title": &video_model.name,
|
||||
"upper_name": &video_model.upper_name,
|
||||
"upper_mid": &video_model.upper_id,
|
||||
"pubtime": &video_model.pubtime.and_utc().format(&CONFIG.time_format).to_string(),
|
||||
"fav_time": &video_model.favtime.and_utc().format(&CONFIG.time_format).to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn page_format_args(
|
||||
video_model: &bili_sync_entity::video::Model,
|
||||
page_model: &bili_sync_entity::page::Model,
|
||||
) -> serde_json::Value {
|
||||
json!({
|
||||
"bvid": &video_model.bvid,
|
||||
"title": &video_model.name,
|
||||
"upper_name": &video_model.upper_name,
|
||||
"upper_mid": &video_model.upper_id,
|
||||
"ptitle": &page_model.name,
|
||||
"pid": page_model.pid,
|
||||
"pubtime": video_model.pubtime.and_utc().format(&CONFIG.time_format).to_string(),
|
||||
"fav_time": video_model.favtime.and_utc().format(&CONFIG.time_format).to_string(),
|
||||
})
|
||||
}
|
||||
@@ -1,23 +1,22 @@
|
||||
pub mod convert;
|
||||
pub mod filenamify;
|
||||
pub mod format_arg;
|
||||
pub mod model;
|
||||
pub mod nfo;
|
||||
pub mod signal;
|
||||
pub mod status;
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use tracing_subscriber::util::SubscriberInitExt;
|
||||
|
||||
pub fn init_logger(log_level: &str) {
|
||||
tracing_subscriber::fmt::Subscriber::builder()
|
||||
.compact()
|
||||
.with_env_filter(tracing_subscriber::EnvFilter::builder().parse_lossy(log_level))
|
||||
.with_target(false)
|
||||
.with_timer(tracing_subscriber::fmt::time::ChronoLocal::new(
|
||||
"%Y-%m-%d %H:%M:%S%.3f".to_owned(),
|
||||
"%b %d %H:%M:%S".to_owned(),
|
||||
))
|
||||
.finish()
|
||||
.try_init()
|
||||
.expect("初始化日志失败");
|
||||
}
|
||||
|
||||
/// 生成视频的唯一标记,均由 bvid 和时间戳构成
|
||||
pub fn id_time_key(bvid: &String, time: &DateTime<Utc>) -> String {
|
||||
format!("{}-{}", bvid, time.timestamp())
|
||||
}
|
||||
|
||||
@@ -1,21 +1,65 @@
|
||||
use anyhow::Result;
|
||||
use anyhow::{Context, Result};
|
||||
use bili_sync_entity::*;
|
||||
use bili_sync_migration::OnConflict;
|
||||
use sea_orm::DatabaseTransaction;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::sea_query::{OnConflict, SimpleExpr};
|
||||
|
||||
use crate::adapter::VideoListModel;
|
||||
use crate::adapter::{VideoSource, VideoSourceEnum};
|
||||
use crate::bilibili::{PageInfo, VideoInfo};
|
||||
use crate::utils::status::STATUS_COMPLETED;
|
||||
|
||||
/// 筛选未填充的视频
|
||||
pub async fn filter_unfilled_videos(
|
||||
additional_expr: SimpleExpr,
|
||||
conn: &DatabaseConnection,
|
||||
) -> Result<Vec<video::Model>> {
|
||||
video::Entity::find()
|
||||
.filter(
|
||||
video::Column::Valid
|
||||
.eq(true)
|
||||
.and(video::Column::DownloadStatus.eq(0))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_null())
|
||||
.and(additional_expr),
|
||||
)
|
||||
.all(conn)
|
||||
.await
|
||||
.context("filter unfilled videos failed")
|
||||
}
|
||||
|
||||
/// 筛选未处理完成的视频和视频页
|
||||
pub async fn filter_unhandled_video_pages(
|
||||
additional_expr: SimpleExpr,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
|
||||
video::Entity::find()
|
||||
.filter(
|
||||
video::Column::Valid
|
||||
.eq(true)
|
||||
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
|
||||
.and(video::Column::Category.eq(2))
|
||||
.and(video::Column::SinglePage.is_not_null())
|
||||
.and(additional_expr),
|
||||
)
|
||||
.find_with_related(page::Entity)
|
||||
.all(connection)
|
||||
.await
|
||||
.context("filter unhandled video pages failed")
|
||||
}
|
||||
|
||||
/// 尝试创建 Video Model,如果发生冲突则忽略
|
||||
pub async fn create_videos(
|
||||
videos_info: &[VideoInfo],
|
||||
video_list_model: &dyn VideoListModel,
|
||||
videos_info: Vec<VideoInfo>,
|
||||
video_source: &VideoSourceEnum,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
let video_models = videos_info
|
||||
.iter()
|
||||
.map(|v| video_list_model.video_model_by_info(v, None))
|
||||
.into_iter()
|
||||
.map(|v| {
|
||||
let mut model = v.into_simple_model();
|
||||
video_source.set_relation_id(&mut model);
|
||||
model
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
video::Entity::insert_many(video_models)
|
||||
// 这里想表达的是 on 索引名,但 sea-orm 的 api 似乎只支持列名而不支持索引名,好在留空可以达到相同的目的
|
||||
@@ -26,48 +70,27 @@ pub async fn create_videos(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 创建视频的所有分 P
|
||||
pub async fn create_video_pages(
|
||||
pages_info: &[PageInfo],
|
||||
video_model: &video::Model,
|
||||
connection: &impl ConnectionTrait,
|
||||
/// 尝试创建 Page Model,如果发生冲突则忽略
|
||||
pub async fn create_pages(
|
||||
pages_info: Vec<PageInfo>,
|
||||
video_model: &bili_sync_entity::video::Model,
|
||||
connection: &DatabaseTransaction,
|
||||
) -> Result<()> {
|
||||
let page_models = pages_info
|
||||
.iter()
|
||||
.map(move |p| {
|
||||
let (width, height) = match &p.dimension {
|
||||
Some(d) => {
|
||||
if d.rotate == 0 {
|
||||
(Some(d.width), Some(d.height))
|
||||
} else {
|
||||
(Some(d.height), Some(d.width))
|
||||
}
|
||||
}
|
||||
None => (None, None),
|
||||
};
|
||||
page::ActiveModel {
|
||||
video_id: Set(video_model.id),
|
||||
cid: Set(p.cid),
|
||||
pid: Set(p.page),
|
||||
name: Set(p.name.clone()),
|
||||
width: Set(width),
|
||||
height: Set(height),
|
||||
duration: Set(p.duration),
|
||||
image: Set(p.first_frame.clone()),
|
||||
download_status: Set(0),
|
||||
..Default::default()
|
||||
}
|
||||
})
|
||||
.into_iter()
|
||||
.map(|p| p.into_active_model(video_model))
|
||||
.collect::<Vec<page::ActiveModel>>();
|
||||
page::Entity::insert_many(page_models)
|
||||
.on_conflict(
|
||||
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
|
||||
.do_nothing()
|
||||
.to_owned(),
|
||||
)
|
||||
.do_nothing()
|
||||
.exec(connection)
|
||||
.await?;
|
||||
for page_chunk in page_models.chunks(50) {
|
||||
page::Entity::insert_many(page_chunk.to_vec())
|
||||
.on_conflict(
|
||||
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
|
||||
.do_nothing()
|
||||
.to_owned(),
|
||||
)
|
||||
.do_nothing()
|
||||
.exec(connection)
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -76,7 +99,7 @@ pub async fn update_videos_model(videos: Vec<video::ActiveModel>, connection: &D
|
||||
video::Entity::insert_many(videos)
|
||||
.on_conflict(
|
||||
OnConflict::column(video::Column::Id)
|
||||
.update_column(video::Column::DownloadStatus)
|
||||
.update_columns([video::Column::DownloadStatus, video::Column::Path])
|
||||
.to_owned(),
|
||||
)
|
||||
.exec(connection)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use anyhow::Result;
|
||||
use bili_sync_entity::*;
|
||||
use quick_xml::Error;
|
||||
use quick_xml::events::{BytesCData, BytesText};
|
||||
use quick_xml::writer::Writer;
|
||||
use quick_xml::Error;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use crate::config::NFOTimeType;
|
||||
@@ -24,7 +24,7 @@ pub struct NFOSerializer<'a>(pub ModelWrapper<'a>, pub NFOMode);
|
||||
|
||||
/// serde xml 似乎不太好用,先这么裸着写
|
||||
/// (真是又臭又长啊
|
||||
impl<'a> NFOSerializer<'a> {
|
||||
impl NFOSerializer<'_> {
|
||||
pub async fn generate_nfo(self, nfo_time_type: &NFOTimeType) -> Result<String> {
|
||||
let mut buffer = r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
|
||||
"#
|
||||
@@ -43,62 +43,52 @@ impl<'a> NFOSerializer<'a> {
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer
|
||||
.create_element("plot")
|
||||
.write_cdata_content_async(BytesCData::new(&v.intro))
|
||||
.await
|
||||
.unwrap();
|
||||
writer.create_element("outline").write_empty_async().await.unwrap();
|
||||
.write_cdata_content_async(BytesCData::new(Self::format_plot(v)))
|
||||
.await?;
|
||||
writer.create_element("outline").write_empty_async().await?;
|
||||
writer
|
||||
.create_element("title")
|
||||
.write_text_content_async(BytesText::new(&v.name))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("actor")
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer
|
||||
.create_element("name")
|
||||
.write_text_content_async(BytesText::new(&v.upper_id.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("role")
|
||||
.write_text_content_async(BytesText::new(&v.upper_name))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("year")
|
||||
.write_text_content_async(BytesText::new(&nfo_time.format("%Y").to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
if let Some(tags) = &v.tags {
|
||||
let tags: Vec<String> = serde_json::from_value(tags.clone()).unwrap();
|
||||
let tags: Vec<String> = serde_json::from_value(tags.clone()).unwrap_or_default();
|
||||
for tag in tags {
|
||||
writer
|
||||
.create_element("genre")
|
||||
.write_text_content_async(BytesText::new(&tag))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
writer
|
||||
.create_element("uniqueid")
|
||||
.with_attribute(("type", "bilibili"))
|
||||
.write_text_content_async(BytesText::new(&v.bvid))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("aired")
|
||||
.write_text_content_async(BytesText::new(&nfo_time.format("%Y-%m-%d").to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
NFOSerializer(ModelWrapper::Video(v), NFOMode::TVSHOW) => {
|
||||
let nfo_time = match nfo_time_type {
|
||||
@@ -110,126 +100,115 @@ impl<'a> NFOSerializer<'a> {
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer
|
||||
.create_element("plot")
|
||||
.write_cdata_content_async(BytesCData::new(&v.intro))
|
||||
.await
|
||||
.unwrap();
|
||||
writer.create_element("outline").write_empty_async().await.unwrap();
|
||||
.write_cdata_content_async(BytesCData::new(Self::format_plot(v)))
|
||||
.await?;
|
||||
writer.create_element("outline").write_empty_async().await?;
|
||||
writer
|
||||
.create_element("title")
|
||||
.write_text_content_async(BytesText::new(&v.name))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("actor")
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer
|
||||
.create_element("name")
|
||||
.write_text_content_async(BytesText::new(&v.upper_id.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("role")
|
||||
.write_text_content_async(BytesText::new(&v.upper_name))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("year")
|
||||
.write_text_content_async(BytesText::new(&nfo_time.format("%Y").to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
if let Some(tags) = &v.tags {
|
||||
let tags: Vec<String> = serde_json::from_value(tags.clone()).unwrap();
|
||||
let tags: Vec<String> = serde_json::from_value(tags.clone()).unwrap_or_default();
|
||||
for tag in tags {
|
||||
writer
|
||||
.create_element("genre")
|
||||
.write_text_content_async(BytesText::new(&tag))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
writer
|
||||
.create_element("uniqueid")
|
||||
.with_attribute(("type", "bilibili"))
|
||||
.write_text_content_async(BytesText::new(&v.bvid))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("aired")
|
||||
.write_text_content_async(BytesText::new(&nfo_time.format("%Y-%m-%d").to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
NFOSerializer(ModelWrapper::Video(v), NFOMode::UPPER) => {
|
||||
writer
|
||||
.create_element("person")
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer.create_element("plot").write_empty_async().await.unwrap();
|
||||
writer.create_element("outline").write_empty_async().await.unwrap();
|
||||
writer.create_element("plot").write_empty_async().await?;
|
||||
writer.create_element("outline").write_empty_async().await?;
|
||||
writer
|
||||
.create_element("lockdata")
|
||||
.write_text_content_async(BytesText::new("false"))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("dateadded")
|
||||
.write_text_content_async(BytesText::new(
|
||||
&v.pubtime.format("%Y-%m-%d %H:%M:%S").to_string(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("title")
|
||||
.write_text_content_async(BytesText::new(&v.upper_id.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("sorttitle")
|
||||
.write_text_content_async(BytesText::new(&v.upper_id.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
NFOSerializer(ModelWrapper::Page(p), NFOMode::EPOSODE) => {
|
||||
writer
|
||||
.create_element("episodedetails")
|
||||
.write_inner_content_async::<_, _, Error>(|writer| async move {
|
||||
writer.create_element("plot").write_empty_async().await.unwrap();
|
||||
writer.create_element("outline").write_empty_async().await.unwrap();
|
||||
writer.create_element("plot").write_empty_async().await?;
|
||||
writer.create_element("outline").write_empty_async().await?;
|
||||
writer
|
||||
.create_element("title")
|
||||
.write_text_content_async(BytesText::new(&p.name))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("season")
|
||||
.write_text_content_async(BytesText::new("1"))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
writer
|
||||
.create_element("episode")
|
||||
.write_text_content_async(BytesText::new(&p.pid.to_string()))
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
Ok(writer)
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
tokio_buffer.flush().await?;
|
||||
Ok(std::str::from_utf8(&buffer).unwrap().to_owned())
|
||||
Ok(String::from_utf8(buffer)?)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn format_plot(model: &video::Model) -> String {
|
||||
format!(
|
||||
r#"原始视频:<a href="https://www.bilibili.com/video/{}/">{}</a><br/><br/>{}"#,
|
||||
model.bvid, model.bvid, model.intro
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,7 +231,7 @@ mod tests {
|
||||
chrono::NaiveDate::from_ymd_opt(2033, 3, 3).unwrap(),
|
||||
chrono::NaiveTime::from_hms_opt(3, 3, 3).unwrap(),
|
||||
),
|
||||
bvid: "bvid".to_string(),
|
||||
bvid: "BV1nWcSeeEkV".to_string(),
|
||||
tags: Some(serde_json::json!(["tag1", "tag2"])),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -263,7 +242,7 @@ mod tests {
|
||||
.unwrap(),
|
||||
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
|
||||
<movie>
|
||||
<plot><![CDATA[intro]]></plot>
|
||||
<plot><![CDATA[原始视频:<a href="https://www.bilibili.com/video/BV1nWcSeeEkV/">BV1nWcSeeEkV</a><br/><br/>intro]]></plot>
|
||||
<outline/>
|
||||
<title>name</title>
|
||||
<actor>
|
||||
@@ -273,7 +252,7 @@ mod tests {
|
||||
<year>2033</year>
|
||||
<genre>tag1</genre>
|
||||
<genre>tag2</genre>
|
||||
<uniqueid type="bilibili">bvid</uniqueid>
|
||||
<uniqueid type="bilibili">BV1nWcSeeEkV</uniqueid>
|
||||
<aired>2033-03-03</aired>
|
||||
</movie>"#,
|
||||
);
|
||||
@@ -284,7 +263,7 @@ mod tests {
|
||||
.unwrap(),
|
||||
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
|
||||
<tvshow>
|
||||
<plot><![CDATA[intro]]></plot>
|
||||
<plot><![CDATA[原始视频:<a href="https://www.bilibili.com/video/BV1nWcSeeEkV/">BV1nWcSeeEkV</a><br/><br/>intro]]></plot>
|
||||
<outline/>
|
||||
<title>name</title>
|
||||
<actor>
|
||||
@@ -294,7 +273,7 @@ mod tests {
|
||||
<year>2022</year>
|
||||
<genre>tag1</genre>
|
||||
<genre>tag2</genre>
|
||||
<uniqueid type="bilibili">bvid</uniqueid>
|
||||
<uniqueid type="bilibili">BV1nWcSeeEkV</uniqueid>
|
||||
<aired>2022-02-02</aired>
|
||||
</tvshow>"#,
|
||||
);
|
||||
|
||||
21
crates/bili_sync/src/utils/signal.rs
Normal file
@@ -0,0 +1,21 @@
|
||||
use std::io;
|
||||
|
||||
use tokio::signal;
|
||||
|
||||
#[cfg(target_family = "windows")]
|
||||
pub async fn terminate() -> io::Result<()> {
|
||||
signal::ctrl_c().await
|
||||
}
|
||||
|
||||
/// ctrl + c 发送的是 SIGINT 信号,docker stop 发送的是 SIGTERM 信号,都需要处理
|
||||
#[cfg(target_family = "unix")]
|
||||
pub async fn terminate() -> io::Result<()> {
|
||||
use tokio::select;
|
||||
|
||||
let mut term = signal::unix::signal(signal::unix::SignalKind::terminate())?;
|
||||
let mut int = signal::unix::signal(signal::unix::SignalKind::interrupt())?;
|
||||
select! {
|
||||
_ = term.recv() => Ok(()),
|
||||
_ = int.recv() => Ok(()),
|
||||
}
|
||||
}
|
||||
@@ -1,136 +1,170 @@
|
||||
use anyhow::Result;
|
||||
use crate::error::ExecutionStatus;
|
||||
|
||||
static STATUS_MAX_RETRY: u32 = 0b100;
|
||||
static STATUS_OK: u32 = 0b111;
|
||||
pub(super) static STATUS_MAX_RETRY: u32 = 0b100;
|
||||
pub static STATUS_OK: u32 = 0b111;
|
||||
pub static STATUS_COMPLETED: u32 = 1 << 31;
|
||||
|
||||
/// 用来表示下载的状态,不想写太多列了,所以仅使用一个 u32 表示。
|
||||
/// 从低位开始,固定每三位表示一种数据的状态,从 0b000 开始,每失败一次加一,最多 0b100(即重试 4 次),
|
||||
/// 如果成功,将对应的三位设置为 0b111。
|
||||
/// 当所有任务都成功或者由于尝试次数过多失败,为 status 最高位打上标记 1,将来不再继续尝试。
|
||||
#[derive(Clone)]
|
||||
pub struct Status(u32);
|
||||
/// 从低位开始,固定每三位表示一种子任务的状态。
|
||||
/// 子任务状态从 0b000 开始,每执行失败一次将状态加一,最多 0b100(即允许重试 4 次),该值定义为 STATUS_MAX_RETRY。
|
||||
/// 如果子任务执行成功,将状态设置为 0b111,该值定义为 STATUS_OK。
|
||||
/// 子任务达到最大失败次数或者执行成功时,认为该子任务已经完成。
|
||||
/// 当所有子任务都已经完成时,为最高位打上标记 1,表示整个下载任务已经完成。
|
||||
#[derive(Clone, Copy, Default)]
|
||||
pub struct Status<const N: usize>(u32);
|
||||
|
||||
impl Status {
|
||||
/// 如果 status 整体大于等于 1 << 31,则表示任务已经被处理过,不再需要重试。
|
||||
/// 数据库可以使用 status < Status::handled() 来筛选需要处理的内容。
|
||||
pub const fn handled() -> u32 {
|
||||
1 << 31
|
||||
impl<const N: usize> Status<N> {
|
||||
// 获取最高位的完成标记
|
||||
pub fn get_completed(&self) -> bool {
|
||||
self.0 >> 31 == 1
|
||||
}
|
||||
|
||||
fn new(status: u32) -> Self {
|
||||
Self(status)
|
||||
/// 依次检查所有子任务是否还应该继续执行,返回一个 bool 数组
|
||||
pub fn should_run(&self) -> [bool; N] {
|
||||
let mut result = [false; N];
|
||||
for (i, item) in result.iter_mut().enumerate() {
|
||||
*item = self.check_continue(i);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// 一般仅需要被内部调用,用来设置最高位的标记
|
||||
fn set_flag(&mut self, handled: bool) {
|
||||
if handled {
|
||||
/// 重置所有失败的状态,将状态设置为 0b000,返回值表示 status 是否发生了变化
|
||||
pub fn reset_failed(&mut self) -> bool {
|
||||
let mut changed = false;
|
||||
for i in 0..N {
|
||||
let status = self.get_status(i);
|
||||
if !(status < STATUS_MAX_RETRY || status == STATUS_OK) {
|
||||
self.set_status(i, 0);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
// 理论上 changed 可以直接从上面的循环中得到,因为 completed 标志位的改变是由子任务状态的改变引起的,子任务没有改变则 completed 也不会改变
|
||||
// 但考虑特殊情况,新版本引入了一个新的子任务项,此时会出现明明有子任务未执行,但 completed 标记位仍然为 true 的情况
|
||||
// 当然可以在新版本迁移文件中全局重置 completed 标记位,但这样影响范围太大感觉不太好
|
||||
// 在后面进行这部分额外判断可以兼容这种情况,在由用户手动触发的 reset_failed 调用中修正 completed 标记位
|
||||
if self.should_run().into_iter().any(|x| x) {
|
||||
changed |= self.get_completed();
|
||||
self.set_completed(false);
|
||||
}
|
||||
changed
|
||||
}
|
||||
|
||||
/// 覆盖某个子任务的状态
|
||||
pub fn set(&mut self, offset: usize, status: u32) {
|
||||
assert!(status < 0b1000, "status should be less than 0b1000");
|
||||
self.set_status(offset, status);
|
||||
if self.should_run().into_iter().all(|x| !x) {
|
||||
self.set_completed(true);
|
||||
} else {
|
||||
self.set_completed(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// 根据任务结果更新状态,任务结果是一个 Result 数组,需要与子任务一一对应
|
||||
/// 如果所有子任务都已经完成,那么打上最高位的完成标记
|
||||
pub fn update_status(&mut self, result: &[ExecutionStatus]) {
|
||||
assert!(result.len() == N, "result length should be equal to N");
|
||||
for (i, res) in result.iter().enumerate() {
|
||||
self.set_result(res, i);
|
||||
}
|
||||
if self.should_run().into_iter().all(|x| !x) {
|
||||
self.set_completed(true);
|
||||
} else {
|
||||
self.set_completed(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// 设置最高位的完成标记
|
||||
fn set_completed(&mut self, completed: bool) {
|
||||
if completed {
|
||||
self.0 |= 1 << 31;
|
||||
} else {
|
||||
self.0 &= !(1 << 31);
|
||||
}
|
||||
}
|
||||
|
||||
/// 从低到高检查状态,如果该位置的任务应该继续尝试执行,则返回 true,否则返回 false
|
||||
fn should_run(&self, size: usize) -> Vec<bool> {
|
||||
(0..size).map(|x| self.check_continue(x)).collect()
|
||||
/// 获取某个子任务的状态
|
||||
fn get_status(&self, offset: usize) -> u32 {
|
||||
(self.0 >> (offset * 3)) & 0b111
|
||||
}
|
||||
|
||||
/// 如果任务的执行次数小于 STATUS_MAX_RETRY,说明可以继续运行
|
||||
fn check_continue(&self, offset: usize) -> bool {
|
||||
self.get_status(offset) < STATUS_MAX_RETRY
|
||||
}
|
||||
|
||||
/// 根据任务结果更新状态,如果任务成功,设置为 STATUS_OK,否则加一
|
||||
fn update_status(&mut self, result: &[Result<()>]) {
|
||||
for (i, res) in result.iter().enumerate() {
|
||||
self.set_result(res, i);
|
||||
}
|
||||
if self.should_run(result.len()).iter().all(|x| !x) {
|
||||
// 所有任务都成功或者由于尝试次数过多失败,为 status 最高位打上标记,将来不再重试
|
||||
self.set_flag(true)
|
||||
}
|
||||
}
|
||||
|
||||
fn set_result(&mut self, result: &Result<()>, offset: usize) {
|
||||
if result.is_ok() {
|
||||
// 如果任务已经执行到最大次数,那么此时 Result 也是 Ok,此时不应该更新状态
|
||||
if self.get_status(offset) < STATUS_MAX_RETRY {
|
||||
self.set_ok(offset);
|
||||
}
|
||||
} else {
|
||||
self.plus_one(offset);
|
||||
}
|
||||
/// 设置某个子任务的状态
|
||||
fn set_status(&mut self, offset: usize, status: u32) {
|
||||
self.0 = (self.0 & !(0b111 << (offset * 3))) | (status << (offset * 3));
|
||||
}
|
||||
|
||||
// 将某个子任务的状态加一(在任务失败时使用)
|
||||
fn plus_one(&mut self, offset: usize) {
|
||||
self.0 += 1 << (3 * offset);
|
||||
}
|
||||
|
||||
// 设置某个子任务的状态为 STATUS_OK(在任务成功时使用)
|
||||
fn set_ok(&mut self, offset: usize) {
|
||||
self.0 |= STATUS_OK << (3 * offset);
|
||||
}
|
||||
|
||||
fn get_status(&self, offset: usize) -> u32 {
|
||||
let helper = !0u32;
|
||||
(self.0 & (helper << (offset * 3)) & (helper >> (32 - 3 * offset - 3))) >> (offset * 3)
|
||||
/// 检查某个子任务是否还应该继续执行,实际是检查该子任务的状态是否小于 STATUS_MAX_RETRY
|
||||
fn check_continue(&self, offset: usize) -> bool {
|
||||
self.get_status(offset) < STATUS_MAX_RETRY
|
||||
}
|
||||
|
||||
/// 根据子任务执行结果更新子任务的状态
|
||||
fn set_result(&mut self, result: &ExecutionStatus, offset: usize) {
|
||||
// 如果任务返回 FixedFailed 状态,那么无论之前的状态如何,都将状态设置为 FixedFailed 的状态
|
||||
if let ExecutionStatus::FixedFailed(status, _) = result {
|
||||
assert!(*status < 0b1000, "status should be less than 0b1000");
|
||||
self.set_status(offset, *status);
|
||||
} else if self.get_status(offset) < STATUS_MAX_RETRY {
|
||||
match result {
|
||||
ExecutionStatus::Succeeded | ExecutionStatus::Skipped => self.set_ok(offset),
|
||||
ExecutionStatus::Failed(_) => self.plus_one(offset),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Status> for u32 {
|
||||
fn from(status: Status) -> Self {
|
||||
impl<const N: usize> From<u32> for Status<N> {
|
||||
fn from(status: u32) -> Self {
|
||||
Status(status)
|
||||
}
|
||||
}
|
||||
|
||||
impl<const N: usize> From<Status<N>> for u32 {
|
||||
fn from(status: Status<N>) -> Self {
|
||||
status.0
|
||||
}
|
||||
}
|
||||
|
||||
/// 从前到后分别表示:视频封面、视频信息、Up 主头像、Up 主信息、分 P 下载
|
||||
#[derive(Clone)]
|
||||
pub struct VideoStatus(Status);
|
||||
|
||||
impl VideoStatus {
|
||||
pub fn new(status: u32) -> Self {
|
||||
Self(Status::new(status))
|
||||
}
|
||||
|
||||
pub fn should_run(&self) -> Vec<bool> {
|
||||
self.0.should_run(5)
|
||||
}
|
||||
|
||||
pub fn update_status(&mut self, result: &[Result<()>]) {
|
||||
assert!(result.len() == 5, "VideoStatus should have 5 status");
|
||||
self.0.update_status(result)
|
||||
impl<const N: usize> From<Status<N>> for [u32; N] {
|
||||
fn from(status: Status<N>) -> Self {
|
||||
let mut result = [0; N];
|
||||
for (i, item) in result.iter_mut().enumerate() {
|
||||
*item = status.get_status(i);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl From<VideoStatus> for u32 {
|
||||
fn from(status: VideoStatus) -> Self {
|
||||
status.0.into()
|
||||
impl<const N: usize> From<[u32; N]> for Status<N> {
|
||||
fn from(status: [u32; N]) -> Self {
|
||||
let mut result = Status::<N>::default();
|
||||
for (i, item) in status.iter().enumerate() {
|
||||
assert!(*item < 0b1000, "status should be less than 0b1000");
|
||||
result.set_status(i, *item);
|
||||
}
|
||||
if result.should_run().iter().all(|x| !x) {
|
||||
result.set_completed(true);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// 从前到后分别表示:视频封面、视频内容、视频信息
|
||||
#[derive(Clone)]
|
||||
pub struct PageStatus(Status);
|
||||
/// 包含五个子任务,从前到后依次是:视频封面、视频信息、Up 主头像、Up 主信息、分 P 下载
|
||||
pub type VideoStatus = Status<5>;
|
||||
|
||||
impl PageStatus {
|
||||
pub fn new(status: u32) -> Self {
|
||||
Self(Status::new(status))
|
||||
}
|
||||
|
||||
pub fn should_run(&self) -> Vec<bool> {
|
||||
self.0.should_run(4)
|
||||
}
|
||||
|
||||
pub fn update_status(&mut self, result: &[Result<()>]) {
|
||||
assert!(result.len() == 4, "PageStatus should have 4 status");
|
||||
self.0.update_status(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<PageStatus> for u32 {
|
||||
fn from(status: PageStatus) -> Self {
|
||||
status.0.into()
|
||||
}
|
||||
}
|
||||
/// 包含五个子任务,从前到后分别是:视频封面、视频内容、视频信息、视频弹幕、视频字幕
|
||||
pub type PageStatus = Status<5>;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
@@ -139,16 +173,90 @@ mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_status() {
|
||||
let mut status = Status::new(0);
|
||||
assert_eq!(status.should_run(3), vec![true, true, true]);
|
||||
for count in 1..=3 {
|
||||
status.update_status(&[Err(anyhow!("")), Ok(()), Ok(())]);
|
||||
assert_eq!(status.should_run(3), vec![true, false, false]);
|
||||
assert_eq!(u32::from(status.clone()), 0b111_111_000 + count);
|
||||
fn test_status_update() {
|
||||
let mut status = Status::<3>::default();
|
||||
assert_eq!(status.should_run(), [true, true, true]);
|
||||
for _ in 0..3 {
|
||||
status.update_status(&[
|
||||
ExecutionStatus::Failed(anyhow!("")),
|
||||
ExecutionStatus::Succeeded,
|
||||
ExecutionStatus::Succeeded,
|
||||
]);
|
||||
assert_eq!(status.should_run(), [true, false, false]);
|
||||
}
|
||||
status.update_status(&[Err(anyhow!("")), Ok(()), Ok(())]);
|
||||
assert_eq!(status.should_run(3), vec![false, false, false]);
|
||||
assert_eq!(u32::from(status), 0b111_111_100 | Status::handled());
|
||||
status.update_status(&[
|
||||
ExecutionStatus::Failed(anyhow!("")),
|
||||
ExecutionStatus::Succeeded,
|
||||
ExecutionStatus::Succeeded,
|
||||
]);
|
||||
assert_eq!(status.should_run(), [false, false, false]);
|
||||
assert!(status.get_completed());
|
||||
status.update_status(&[
|
||||
ExecutionStatus::FixedFailed(1, anyhow!("")),
|
||||
ExecutionStatus::FixedFailed(4, anyhow!("")),
|
||||
ExecutionStatus::FixedFailed(7, anyhow!("")),
|
||||
]);
|
||||
assert_eq!(status.should_run(), [true, false, false]);
|
||||
assert!(!status.get_completed());
|
||||
assert_eq!(<[u32; 3]>::from(status), [1, 4, 7]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_status_convert() {
|
||||
let testcases = [[0, 0, 1], [1, 2, 3], [3, 1, 2], [3, 0, 7]];
|
||||
for testcase in testcases.iter() {
|
||||
let status = Status::<3>::from(testcase.clone());
|
||||
assert_eq!(<[u32; 3]>::from(status), *testcase);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_status_convert_and_update() {
|
||||
let testcases = [([0, 0, 1], [1, 7, 7]), ([3, 4, 3], [4, 4, 7]), ([3, 1, 7], [4, 7, 7])];
|
||||
for (before, after) in testcases.iter() {
|
||||
let mut status = Status::<3>::from(before.clone());
|
||||
status.update_status(&[
|
||||
ExecutionStatus::Failed(anyhow!("")),
|
||||
ExecutionStatus::Succeeded,
|
||||
ExecutionStatus::Succeeded,
|
||||
]);
|
||||
assert_eq!(<[u32; 3]>::from(status), *after);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_status_reset_failed() {
|
||||
// 重置一个已经失败的任务
|
||||
let mut status = Status::<3>::from([3, 4, 7]);
|
||||
assert!(!status.get_completed());
|
||||
assert!(status.reset_failed());
|
||||
assert!(!status.get_completed());
|
||||
assert_eq!(<[u32; 3]>::from(status), [3, 0, 7]);
|
||||
// 没有内容需要重置,但 completed 标记位是错误的(模拟新增一个子任务状态的情况),此时 reset_failed 会修正 completed 标记位
|
||||
status.set_completed(true);
|
||||
assert!(status.get_completed());
|
||||
assert!(status.reset_failed());
|
||||
assert!(!status.get_completed());
|
||||
// 重置一个已经成功的任务,没有改变状态,也不会修改标记位
|
||||
let mut status = Status::<3>::from([7, 7, 7]);
|
||||
assert!(status.get_completed());
|
||||
assert!(!status.reset_failed());
|
||||
assert!(status.get_completed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_status_set() {
|
||||
// 设置子状态,从 completed 到 uncompleted
|
||||
let mut status = Status::<5>::from([7, 7, 7, 7, 7]);
|
||||
assert!(status.get_completed());
|
||||
status.set(4, 0);
|
||||
assert!(!status.get_completed());
|
||||
assert_eq!(<[u32; 5]>::from(status), [7, 7, 7, 7, 0]);
|
||||
// 设置子状态,从 uncompleted 到 completed
|
||||
let mut status = Status::<5>::from([4, 7, 7, 7, 0]);
|
||||
assert!(!status.get_completed());
|
||||
status.set(4, 7);
|
||||
assert!(status.get_completed());
|
||||
assert_eq!(<[u32; 5]>::from(status), [4, 7, 7, 7, 7]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,171 +1,228 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use bili_sync_entity::{page, video};
|
||||
use filenamify::filenamify;
|
||||
use anyhow::{Context, Result, anyhow, bail};
|
||||
use bili_sync_entity::*;
|
||||
use futures::stream::{FuturesOrdered, FuturesUnordered};
|
||||
use futures::{Future, Stream, StreamExt};
|
||||
use sea_orm::entity::prelude::*;
|
||||
use futures::{Future, Stream, StreamExt, TryStreamExt};
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use serde_json::json;
|
||||
use sea_orm::TransactionTrait;
|
||||
use sea_orm::entity::prelude::*;
|
||||
use tokio::fs;
|
||||
use tokio::sync::{Mutex, Semaphore};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
use crate::adapter::{video_list_from, Args, VideoListModel};
|
||||
use crate::adapter::{Args, VideoSource, VideoSourceEnum, video_source_from};
|
||||
use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Video, VideoInfo};
|
||||
use crate::config::{ARGS, CONFIG, TEMPLATE};
|
||||
use crate::config::{ARGS, CONFIG, PathSafeTemplate, TEMPLATE};
|
||||
use crate::downloader::Downloader;
|
||||
use crate::error::{DownloadAbortError, ProcessPageError};
|
||||
use crate::utils::model::{create_videos, update_pages_model, update_videos_model};
|
||||
use crate::error::{DownloadAbortError, ExecutionStatus, ProcessPageError};
|
||||
use crate::utils::format_arg::{page_format_args, video_format_args};
|
||||
use crate::utils::model::{
|
||||
create_pages, create_videos, filter_unfilled_videos, filter_unhandled_video_pages, update_pages_model,
|
||||
update_videos_model,
|
||||
};
|
||||
use crate::utils::nfo::{ModelWrapper, NFOMode, NFOSerializer};
|
||||
use crate::utils::status::{PageStatus, VideoStatus};
|
||||
use crate::utils::status::{PageStatus, STATUS_OK, VideoStatus};
|
||||
|
||||
pub async fn process_video_list(
|
||||
/// 完整地处理某个视频来源
|
||||
pub async fn process_video_source(
|
||||
args: Args<'_>,
|
||||
bili_client: &BiliClient,
|
||||
path: &Path,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
let (video_list_model, video_streams) = video_list_from(args, path, bili_client, connection).await?;
|
||||
let video_list_model = refresh_video_list(video_list_model, video_streams, connection).await?;
|
||||
let video_list_model = fetch_video_details(bili_client, video_list_model, connection).await?;
|
||||
// 从参数中获取视频列表的 Model 与视频流
|
||||
let (video_source, video_streams) = video_source_from(args, path, bili_client, connection).await?;
|
||||
// 从视频流中获取新视频的简要信息,写入数据库
|
||||
refresh_video_source(&video_source, video_streams, connection).await?;
|
||||
// 单独请求视频详情接口,获取视频的详情信息与所有的分页,写入数据库
|
||||
fetch_video_details(bili_client, &video_source, connection).await?;
|
||||
if ARGS.scan_only {
|
||||
warn!("已开启仅扫描模式,跳过视频下载...");
|
||||
return Ok(());
|
||||
warn!("已开启仅扫描模式,跳过视频下载..");
|
||||
} else {
|
||||
// 从数据库中查找所有未下载的视频与分页,下载并处理
|
||||
download_unprocessed_videos(bili_client, &video_source, connection).await?;
|
||||
}
|
||||
download_unprocessed_videos(bili_client, video_list_model, connection).await
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 请求接口,获取视频列表中所有新添加的视频信息,将其写入数据库
|
||||
pub async fn refresh_video_list<'a>(
|
||||
video_list_model: Box<dyn VideoListModel>,
|
||||
video_streams: Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>,
|
||||
pub async fn refresh_video_source<'a>(
|
||||
video_source: &VideoSourceEnum,
|
||||
video_streams: Pin<Box<dyn Stream<Item = Result<VideoInfo>> + 'a + Send>>,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Box<dyn VideoListModel>> {
|
||||
video_list_model.log_refresh_video_start();
|
||||
let mut video_streams = video_streams.chunks(10);
|
||||
let mut got_count = 0;
|
||||
let mut new_count = video_list_model.video_count(connection).await?;
|
||||
) -> Result<()> {
|
||||
video_source.log_refresh_video_start();
|
||||
let latest_row_at = video_source.get_latest_row_at().and_utc();
|
||||
let mut max_datetime = latest_row_at;
|
||||
let mut error = Ok(());
|
||||
let mut video_streams = video_streams
|
||||
.take_while(|res| {
|
||||
match res {
|
||||
Err(e) => {
|
||||
error = Err(anyhow!(e.to_string()));
|
||||
futures::future::ready(false)
|
||||
}
|
||||
Ok(v) => {
|
||||
// 虽然 video_streams 是从新到旧的,但由于此处是分页请求,极端情况下可能发生访问完第一页时插入了两整页视频的情况
|
||||
// 此时获取到的第二页视频比第一页的还要新,因此为了确保正确,理应对每一页的第一个视频进行时间比较
|
||||
// 但在 streams 的抽象下,无法判断具体是在哪里分页的,所以暂且对每个视频都进行比较,应该不会有太大性能损失
|
||||
let release_datetime = v.release_datetime();
|
||||
if release_datetime > &max_datetime {
|
||||
max_datetime = *release_datetime;
|
||||
}
|
||||
futures::future::ready(release_datetime > &latest_row_at)
|
||||
}
|
||||
}
|
||||
})
|
||||
.filter_map(|res| futures::future::ready(res.ok()))
|
||||
.chunks(10);
|
||||
let mut count = 0;
|
||||
while let Some(videos_info) = video_streams.next().await {
|
||||
got_count += videos_info.len();
|
||||
let exist_labels = video_list_model.exist_labels(&videos_info, connection).await?;
|
||||
// 如果发现有视频的收藏时间和 bvid 和数据库中重合,说明到达了上次处理到的地方,可以直接退出
|
||||
let should_break = videos_info.iter().any(|v| exist_labels.contains(&v.video_key()));
|
||||
// 将视频信息写入数据库
|
||||
create_videos(&videos_info, video_list_model.as_ref(), connection).await?;
|
||||
if should_break {
|
||||
info!("到达上一次处理的位置,提前中止");
|
||||
break;
|
||||
}
|
||||
count += videos_info.len();
|
||||
create_videos(videos_info, video_source, connection).await?;
|
||||
}
|
||||
new_count = video_list_model.video_count(connection).await? - new_count;
|
||||
video_list_model.log_refresh_video_end(got_count, new_count);
|
||||
Ok(video_list_model)
|
||||
// 如果获取视频分页过程中发生了错误,直接在此处返回,不更新 latest_row_at
|
||||
error?;
|
||||
if max_datetime != latest_row_at {
|
||||
video_source
|
||||
.update_latest_row_at(max_datetime.naive_utc())
|
||||
.save(connection)
|
||||
.await?;
|
||||
}
|
||||
video_source.log_refresh_video_end(count);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 筛选出所有未获取到全部信息的视频,尝试补充其详细信息
|
||||
pub async fn fetch_video_details(
|
||||
bili_client: &BiliClient,
|
||||
video_list_model: Box<dyn VideoListModel>,
|
||||
video_source: &VideoSourceEnum,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<Box<dyn VideoListModel>> {
|
||||
video_list_model.log_fetch_video_start();
|
||||
let videos_model = video_list_model.unfilled_videos(connection).await?;
|
||||
video_list_model
|
||||
.fetch_videos_detail(bili_client, videos_model, connection)
|
||||
.await?;
|
||||
video_list_model.log_fetch_video_end();
|
||||
Ok(video_list_model)
|
||||
) -> Result<()> {
|
||||
video_source.log_fetch_video_start();
|
||||
let videos_model = filter_unfilled_videos(video_source.filter_expr(), connection).await?;
|
||||
for video_model in videos_model {
|
||||
let video = Video::new(bili_client, video_model.bvid.clone());
|
||||
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
|
||||
match info {
|
||||
Err(e) => {
|
||||
error!(
|
||||
"获取视频 {} - {} 的详细信息失败,错误为:{:#}",
|
||||
&video_model.bvid, &video_model.name, e
|
||||
);
|
||||
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
|
||||
let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into();
|
||||
video_active_model.valid = Set(false);
|
||||
video_active_model.save(connection).await?;
|
||||
}
|
||||
}
|
||||
Ok((tags, mut view_info)) => {
|
||||
let VideoInfo::Detail { pages, .. } = &mut view_info else {
|
||||
unreachable!()
|
||||
};
|
||||
let pages = std::mem::take(pages);
|
||||
let pages_len = pages.len();
|
||||
let txn = connection.begin().await?;
|
||||
// 将分页信息写入数据库
|
||||
create_pages(pages, &video_model, &txn).await?;
|
||||
let mut video_active_model = view_info.into_detail_model(video_model);
|
||||
video_source.set_relation_id(&mut video_active_model);
|
||||
video_active_model.single_page = Set(Some(pages_len == 1));
|
||||
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
|
||||
video_active_model.save(&txn).await?;
|
||||
txn.commit().await?;
|
||||
}
|
||||
};
|
||||
}
|
||||
video_source.log_fetch_video_end();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 下载所有未处理成功的视频
|
||||
pub async fn download_unprocessed_videos(
|
||||
bili_client: &BiliClient,
|
||||
video_list_model: Box<dyn VideoListModel>,
|
||||
video_source: &VideoSourceEnum,
|
||||
connection: &DatabaseConnection,
|
||||
) -> Result<()> {
|
||||
video_list_model.log_download_video_start();
|
||||
let unhandled_videos_pages = video_list_model.unhandled_video_pages(connection).await?;
|
||||
// 对于视频,允许三个同时下载(视频内还有分页、不同分页还有多种下载任务)
|
||||
let semaphore = Semaphore::new(3);
|
||||
video_source.log_download_video_start();
|
||||
let semaphore = Semaphore::new(CONFIG.concurrent_limit.video);
|
||||
let downloader = Downloader::new(bili_client.client.clone());
|
||||
let mut uppers_mutex: HashMap<i64, (Mutex<()>, Mutex<()>)> = HashMap::new();
|
||||
for (video_model, _) in &unhandled_videos_pages {
|
||||
uppers_mutex.insert(video_model.upper_id, (Mutex::new(()), Mutex::new(())));
|
||||
}
|
||||
let mut tasks = unhandled_videos_pages
|
||||
let unhandled_videos_pages = filter_unhandled_video_pages(video_source.filter_expr(), connection).await?;
|
||||
let mut assigned_upper = HashSet::new();
|
||||
let tasks = unhandled_videos_pages
|
||||
.into_iter()
|
||||
.map(|(video_model, pages_model)| {
|
||||
let upper_mutex = uppers_mutex.get(&video_model.upper_id).unwrap();
|
||||
let should_download_upper = !assigned_upper.contains(&video_model.upper_id);
|
||||
assigned_upper.insert(video_model.upper_id);
|
||||
download_video_pages(
|
||||
bili_client,
|
||||
video_source,
|
||||
video_model,
|
||||
pages_model,
|
||||
connection,
|
||||
&semaphore,
|
||||
&downloader,
|
||||
&CONFIG.upper_path,
|
||||
upper_mutex,
|
||||
should_download_upper,
|
||||
)
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
let mut models = Vec::with_capacity(10);
|
||||
while let Some(res) = tasks.next().await {
|
||||
match res {
|
||||
Ok(model) => {
|
||||
models.push(model);
|
||||
let mut download_aborted = false;
|
||||
let mut stream = tasks
|
||||
// 触发风控时设置 download_aborted 标记并终止流
|
||||
.take_while(|res| {
|
||||
if res
|
||||
.as_ref()
|
||||
.is_err_and(|e| e.downcast_ref::<DownloadAbortError>().is_some())
|
||||
{
|
||||
download_aborted = true;
|
||||
}
|
||||
Err(e) => {
|
||||
if e.downcast_ref::<DownloadAbortError>().is_some() {
|
||||
error!("下载视频时触发风控,将终止收藏夹下所有下载任务,等待下一轮执行");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// 满十个就写入数据库
|
||||
if models.len() == 10 {
|
||||
update_videos_model(std::mem::replace(&mut models, Vec::with_capacity(10)), connection).await?;
|
||||
}
|
||||
}
|
||||
if !models.is_empty() {
|
||||
futures::future::ready(!download_aborted)
|
||||
})
|
||||
// 过滤掉没有触发风控的普通 Err,只保留正确返回的 Model
|
||||
.filter_map(|res| futures::future::ready(res.ok()))
|
||||
// 将成功返回的 Model 按十个一组合并
|
||||
.chunks(10);
|
||||
while let Some(models) = stream.next().await {
|
||||
update_videos_model(models, connection).await?;
|
||||
}
|
||||
video_list_model.log_download_video_end();
|
||||
if download_aborted {
|
||||
error!("下载触发风控,已终止所有任务,等待下一轮执行");
|
||||
}
|
||||
video_source.log_download_video_end();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 暂时这样做,后面提取成上下文
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn download_video_pages(
|
||||
bili_client: &BiliClient,
|
||||
video_source: &VideoSourceEnum,
|
||||
video_model: video::Model,
|
||||
pages: Vec<page::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
semaphore: &Semaphore,
|
||||
downloader: &Downloader,
|
||||
upper_path: &Path,
|
||||
upper_mutex: &(Mutex<()>, Mutex<()>),
|
||||
should_download_upper: bool,
|
||||
) -> Result<video::ActiveModel> {
|
||||
let permit = semaphore.acquire().await;
|
||||
if let Err(e) = permit {
|
||||
bail!(e);
|
||||
}
|
||||
let mut status = VideoStatus::new(video_model.download_status);
|
||||
let seprate_status = status.should_run();
|
||||
let base_path = Path::new(&video_model.path);
|
||||
let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
|
||||
let mut status = VideoStatus::from(video_model.download_status);
|
||||
let separate_status = status.should_run();
|
||||
let base_path = video_source
|
||||
.path()
|
||||
.join(TEMPLATE.path_safe_render("video", &video_format_args(&video_model))?);
|
||||
let upper_id = video_model.upper_id.to_string();
|
||||
let base_upper_path = upper_path
|
||||
.join(upper_id.chars().next().unwrap().to_string())
|
||||
let base_upper_path = &CONFIG
|
||||
.upper_path
|
||||
.join(upper_id.chars().next().context("upper_id is empty")?.to_string())
|
||||
.join(upper_id);
|
||||
let is_single_page = video_model.single_page.unwrap();
|
||||
let is_single_page = video_model.single_page.context("single_page is null")?;
|
||||
// 对于单页视频,page 的下载已经足够
|
||||
// 对于多页视频,page 下载仅包含了分集内容,需要额外补上视频的 poster 的 tvshow.nfo
|
||||
let tasks: Vec<Pin<Box<dyn Future<Output = Result<()>>>>> = vec![
|
||||
let tasks: Vec<Pin<Box<dyn Future<Output = Result<ExecutionStatus>> + Send>>> = vec![
|
||||
// 下载视频封面
|
||||
Box::pin(fetch_video_poster(
|
||||
seprate_status[0] && !is_single_page,
|
||||
separate_status[0] && !is_single_page,
|
||||
&video_model,
|
||||
downloader,
|
||||
base_path.join("poster.jpg"),
|
||||
@@ -173,62 +230,66 @@ pub async fn download_video_pages(
|
||||
)),
|
||||
// 生成视频信息的 nfo
|
||||
Box::pin(generate_video_nfo(
|
||||
seprate_status[1] && !is_single_page,
|
||||
separate_status[1] && !is_single_page,
|
||||
&video_model,
|
||||
base_path.join("tvshow.nfo"),
|
||||
)),
|
||||
// 下载 Up 主头像
|
||||
Box::pin(fetch_upper_face(
|
||||
seprate_status[2],
|
||||
separate_status[2] && should_download_upper,
|
||||
&video_model,
|
||||
downloader,
|
||||
&upper_mutex.0,
|
||||
base_upper_path.join("folder.jpg"),
|
||||
)),
|
||||
// 生成 Up 主信息的 nfo
|
||||
Box::pin(generate_upper_nfo(
|
||||
seprate_status[3],
|
||||
separate_status[3] && should_download_upper,
|
||||
&video_model,
|
||||
&upper_mutex.1,
|
||||
base_upper_path.join("person.nfo"),
|
||||
)),
|
||||
// 分发并执行分 P 下载的任务
|
||||
Box::pin(dispatch_download_page(
|
||||
seprate_status[4],
|
||||
separate_status[4],
|
||||
bili_client,
|
||||
&video_model,
|
||||
pages,
|
||||
connection,
|
||||
downloader,
|
||||
&base_path,
|
||||
)),
|
||||
];
|
||||
let tasks: FuturesOrdered<_> = tasks.into_iter().collect();
|
||||
let results: Vec<Result<()>> = tasks.collect().await;
|
||||
let results: Vec<ExecutionStatus> = tasks.collect::<Vec<_>>().await.into_iter().map(Into::into).collect();
|
||||
status.update_status(&results);
|
||||
results
|
||||
.iter()
|
||||
.take(4)
|
||||
.zip(["封面", "视频 nfo", "up 主头像", "up 主 nfo"])
|
||||
.zip(["封面", "详情", "作者头像", "作者详情"])
|
||||
.for_each(|(res, task_name)| match res {
|
||||
Ok(_) => info!(
|
||||
"处理视频 {} - {} 的 {} 成功",
|
||||
&video_model.bvid, &video_model.name, task_name
|
||||
),
|
||||
Err(e) => error!(
|
||||
"处理视频 {} - {} 的 {} 失败: {}",
|
||||
&video_model.bvid, &video_model.name, task_name, e
|
||||
),
|
||||
ExecutionStatus::Skipped => info!("处理视频「{}」{}已成功过,跳过", &video_model.name, task_name),
|
||||
ExecutionStatus::Succeeded => info!("处理视频「{}」{}成功", &video_model.name, task_name),
|
||||
ExecutionStatus::Ignored(e) => {
|
||||
error!(
|
||||
"处理视频「{}」{}出现常见错误,已忽略: {:#}",
|
||||
&video_model.name, task_name, e
|
||||
)
|
||||
}
|
||||
ExecutionStatus::Failed(e) | ExecutionStatus::FixedFailed(_, e) => {
|
||||
error!("处理视频「{}」{}失败: {:#}", &video_model.name, task_name, e)
|
||||
}
|
||||
});
|
||||
if let Err(e) = results.into_iter().nth(4).unwrap() {
|
||||
if let ExecutionStatus::Failed(e) = results.into_iter().nth(4).context("page download result not found")? {
|
||||
if e.downcast_ref::<DownloadAbortError>().is_some() {
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
let mut video_active_model: video::ActiveModel = video_model.into();
|
||||
video_active_model.download_status = Set(status.into());
|
||||
video_active_model.path = Set(base_path.to_string_lossy().to_string());
|
||||
Ok(video_active_model)
|
||||
}
|
||||
|
||||
/// 分发并执行分页下载任务,当且仅当所有分页成功下载或达到最大重试次数时返回 Ok,否则根据失败原因返回对应的错误
|
||||
pub async fn dispatch_download_page(
|
||||
should_run: bool,
|
||||
bili_client: &BiliClient,
|
||||
@@ -236,96 +297,86 @@ pub async fn dispatch_download_page(
|
||||
pages: Vec<page::Model>,
|
||||
connection: &DatabaseConnection,
|
||||
downloader: &Downloader,
|
||||
) -> Result<()> {
|
||||
base_path: &Path,
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
// 对于视频的分页,允许两个同时下载(绝大部分是单页视频)
|
||||
let child_semaphore = Semaphore::new(2);
|
||||
let mut tasks = pages
|
||||
let child_semaphore = Semaphore::new(CONFIG.concurrent_limit.page);
|
||||
let tasks = pages
|
||||
.into_iter()
|
||||
.map(|page_model| download_page(bili_client, video_model, page_model, &child_semaphore, downloader))
|
||||
.map(|page_model| {
|
||||
download_page(
|
||||
bili_client,
|
||||
video_model,
|
||||
page_model,
|
||||
&child_semaphore,
|
||||
downloader,
|
||||
base_path,
|
||||
)
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
let mut models = Vec::with_capacity(10);
|
||||
let (mut should_error, mut is_break) = (false, false);
|
||||
while let Some(res) = tasks.next().await {
|
||||
match res {
|
||||
Ok(model) => {
|
||||
if let Set(status) = model.download_status {
|
||||
let status = PageStatus::new(status);
|
||||
if status.should_run().iter().any(|v| *v) {
|
||||
// 有一个分页没变成终止状态(即下载成功或者重试次数达到限制),就应该向上层传递 Error
|
||||
should_error = true;
|
||||
let (mut download_aborted, mut target_status) = (false, STATUS_OK);
|
||||
let mut stream = tasks
|
||||
.take_while(|res| {
|
||||
match res {
|
||||
Ok(model) => {
|
||||
// 该视频的所有分页的下载状态都会在此返回,需要根据这些状态确认视频层“分 P 下载”子任务的状态
|
||||
// 在过去的实现中,此处仅仅根据 page_download_status 的最高标志位来判断,如果最高标志位是 true 则认为完成
|
||||
// 这样会导致即使分页中有失败到 MAX_RETRY 的情况,视频层的分 P 下载状态也会被认为是 Succeeded,不够准确
|
||||
// 新版本实现会将此处取值为所有子任务状态的最小值,这样只有所有分页的子任务全部成功时才会认为视频层的分 P 下载状态是 Succeeded
|
||||
let page_download_status = model.download_status.try_as_ref().expect("download_status must be set");
|
||||
let separate_status: [u32; 5] = PageStatus::from(*page_download_status).into();
|
||||
for status in separate_status {
|
||||
target_status = target_status.min(status);
|
||||
}
|
||||
}
|
||||
models.push(model);
|
||||
}
|
||||
Err(e) => {
|
||||
if e.downcast_ref::<DownloadAbortError>().is_some() {
|
||||
should_error = true;
|
||||
is_break = true;
|
||||
break;
|
||||
Err(e) => {
|
||||
if e.downcast_ref::<DownloadAbortError>().is_some() {
|
||||
download_aborted = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if models.len() == 10 {
|
||||
update_pages_model(std::mem::replace(&mut models, Vec::with_capacity(10)), connection).await?;
|
||||
}
|
||||
}
|
||||
if !models.is_empty() {
|
||||
// 仅在发生风控时终止流,其它情况继续执行
|
||||
futures::future::ready(!download_aborted)
|
||||
})
|
||||
.filter_map(|res| futures::future::ready(res.ok()))
|
||||
.chunks(10);
|
||||
while let Some(models) = stream.next().await {
|
||||
update_pages_model(models, connection).await?;
|
||||
}
|
||||
if should_error {
|
||||
if is_break {
|
||||
error!(
|
||||
"下载视频 {} - {} 的分页时触发风控,将异常向上传递...",
|
||||
&video_model.bvid, &video_model.name
|
||||
);
|
||||
bail!(DownloadAbortError());
|
||||
} else {
|
||||
error!(
|
||||
"下载视频 {} - {} 的分页时出现了错误,将在下一轮尝试重新处理",
|
||||
&video_model.bvid, &video_model.name
|
||||
);
|
||||
bail!(ProcessPageError());
|
||||
}
|
||||
if download_aborted {
|
||||
error!("下载视频「{}」的分页时触发风控,将异常向上传递..", &video_model.name);
|
||||
bail!(DownloadAbortError());
|
||||
}
|
||||
Ok(())
|
||||
if target_status != STATUS_OK {
|
||||
return Ok(ExecutionStatus::FixedFailed(target_status, ProcessPageError().into()));
|
||||
}
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
/// 下载某个分页,未发生风控且正常运行时返回 Ok(Page::ActiveModel),其中 status 字段存储了新的下载状态,发生风控时返回 DownloadAbortError
|
||||
pub async fn download_page(
|
||||
bili_client: &BiliClient,
|
||||
video_model: &video::Model,
|
||||
page_model: page::Model,
|
||||
semaphore: &Semaphore,
|
||||
downloader: &Downloader,
|
||||
base_path: &Path,
|
||||
) -> Result<page::ActiveModel> {
|
||||
let permit = semaphore.acquire().await;
|
||||
if let Err(e) = permit {
|
||||
return Err(e.into());
|
||||
}
|
||||
let mut status = PageStatus::new(page_model.download_status);
|
||||
let seprate_status = status.should_run();
|
||||
let is_single_page = video_model.single_page.unwrap();
|
||||
let base_path = Path::new(&video_model.path);
|
||||
let base_name = filenamify(TEMPLATE.render(
|
||||
"page",
|
||||
&json!({
|
||||
"bvid": &video_model.bvid,
|
||||
"title": &video_model.name,
|
||||
"upper_name": &video_model.upper_name,
|
||||
"upper_mid": &video_model.upper_id,
|
||||
"ptitle": &page_model.name,
|
||||
"pid": page_model.pid,
|
||||
}),
|
||||
)?);
|
||||
let (poster_path, video_path, nfo_path, danmaku_path, fanart_path) = if is_single_page {
|
||||
let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
|
||||
let mut status = PageStatus::from(page_model.download_status);
|
||||
let separate_status = status.should_run();
|
||||
let is_single_page = video_model.single_page.context("single_page is null")?;
|
||||
let base_name = TEMPLATE.path_safe_render("page", &page_format_args(video_model, &page_model))?;
|
||||
let (poster_path, video_path, nfo_path, danmaku_path, fanart_path, subtitle_path) = if is_single_page {
|
||||
(
|
||||
base_path.join(format!("{}-poster.jpg", &base_name)),
|
||||
base_path.join(format!("{}.mp4", &base_name)),
|
||||
base_path.join(format!("{}.nfo", &base_name)),
|
||||
base_path.join(format!("{}.zh-CN.default.ass", &base_name)),
|
||||
Some(base_path.join(format!("{}-fanart.jpg", &base_name))),
|
||||
base_path.join(format!("{}.srt", &base_name)),
|
||||
)
|
||||
} else {
|
||||
(
|
||||
@@ -343,16 +394,18 @@ pub async fn download_page(
|
||||
.join(format!("{} - S01E{:0>2}.zh-CN.default.ass", &base_name, page_model.pid)),
|
||||
// 对于多页视频,会在上一步 fetch_video_poster 中获取剧集的 fanart,无需在此处下载单集的
|
||||
None,
|
||||
base_path
|
||||
.join("Season 1")
|
||||
.join(format!("{} - S01E{:0>2}.srt", &base_name, page_model.pid)),
|
||||
)
|
||||
};
|
||||
let dimension = if page_model.width.is_some() && page_model.height.is_some() {
|
||||
Some(Dimension {
|
||||
width: page_model.width.unwrap(),
|
||||
height: page_model.height.unwrap(),
|
||||
let dimension = match (page_model.width, page_model.height) {
|
||||
(Some(width), Some(height)) => Some(Dimension {
|
||||
width,
|
||||
height,
|
||||
rotate: 0,
|
||||
})
|
||||
} else {
|
||||
None
|
||||
}),
|
||||
_ => None,
|
||||
};
|
||||
let page_info = PageInfo {
|
||||
cid: page_model.cid,
|
||||
@@ -360,9 +413,9 @@ pub async fn download_page(
|
||||
dimension,
|
||||
..Default::default()
|
||||
};
|
||||
let tasks: Vec<Pin<Box<dyn Future<Output = Result<()>>>>> = vec![
|
||||
let tasks: Vec<Pin<Box<dyn Future<Output = Result<ExecutionStatus>> + Send>>> = vec![
|
||||
Box::pin(fetch_page_poster(
|
||||
seprate_status[0],
|
||||
separate_status[0],
|
||||
video_model,
|
||||
&page_model,
|
||||
downloader,
|
||||
@@ -370,47 +423,69 @@ pub async fn download_page(
|
||||
fanart_path,
|
||||
)),
|
||||
Box::pin(fetch_page_video(
|
||||
seprate_status[1],
|
||||
separate_status[1],
|
||||
bili_client,
|
||||
video_model,
|
||||
downloader,
|
||||
&page_info,
|
||||
video_path.clone(),
|
||||
&video_path,
|
||||
)),
|
||||
Box::pin(generate_page_nfo(
|
||||
separate_status[2],
|
||||
video_model,
|
||||
&page_model,
|
||||
nfo_path,
|
||||
)),
|
||||
Box::pin(generate_page_nfo(seprate_status[2], video_model, &page_model, nfo_path)),
|
||||
Box::pin(fetch_page_danmaku(
|
||||
seprate_status[3],
|
||||
separate_status[3],
|
||||
bili_client,
|
||||
video_model,
|
||||
&page_info,
|
||||
danmaku_path,
|
||||
)),
|
||||
Box::pin(fetch_page_subtitle(
|
||||
separate_status[4],
|
||||
bili_client,
|
||||
video_model,
|
||||
&page_info,
|
||||
&subtitle_path,
|
||||
)),
|
||||
];
|
||||
let tasks: FuturesOrdered<_> = tasks.into_iter().collect();
|
||||
let results: Vec<Result<()>> = tasks.collect().await;
|
||||
let results: Vec<ExecutionStatus> = tasks.collect::<Vec<_>>().await.into_iter().map(Into::into).collect();
|
||||
status.update_status(&results);
|
||||
results
|
||||
.iter()
|
||||
.zip(["封面", "视频", "视频 nfo", "弹幕"])
|
||||
.zip(["封面", "视频", "详情", "弹幕", "字幕"])
|
||||
.for_each(|(res, task_name)| match res {
|
||||
Ok(_) => info!(
|
||||
"处理视频 {} - {} 第 {} 页的 {} 成功",
|
||||
&video_model.bvid, &video_model.name, page_model.pid, task_name
|
||||
ExecutionStatus::Skipped => info!(
|
||||
"处理视频「{}」第 {} 页{}已成功过,跳过",
|
||||
&video_model.name, page_model.pid, task_name
|
||||
),
|
||||
Err(e) => error!(
|
||||
"处理视频 {} - {} 第 {} 页的 {} 失败: {}",
|
||||
&video_model.bvid, &video_model.name, page_model.pid, task_name, e
|
||||
ExecutionStatus::Succeeded => info!(
|
||||
"处理视频「{}」第 {} 页{}成功",
|
||||
&video_model.name, page_model.pid, task_name
|
||||
),
|
||||
ExecutionStatus::Ignored(e) => {
|
||||
error!(
|
||||
"处理视频「{}」第 {} 页{}出现常见错误,已忽略: {:#}",
|
||||
&video_model.name, page_model.pid, task_name, e
|
||||
)
|
||||
}
|
||||
ExecutionStatus::Failed(e) | ExecutionStatus::FixedFailed(_, e) => error!(
|
||||
"处理视频「{}」第 {} 页{}失败: {:#}",
|
||||
&video_model.name, page_model.pid, task_name, e
|
||||
),
|
||||
});
|
||||
// 查看下载视频的状态,该状态会影响上层是否 break
|
||||
if let Err(e) = results.into_iter().nth(1).unwrap() {
|
||||
// 如果下载视频时触发风控,直接返回 DownloadAbortError
|
||||
if let ExecutionStatus::Failed(e) = results.into_iter().nth(1).context("video download result not found")? {
|
||||
if let Ok(BiliError::RiskControlOccurred) = e.downcast::<BiliError>() {
|
||||
bail!(DownloadAbortError());
|
||||
}
|
||||
}
|
||||
let mut page_active_model: page::ActiveModel = page_model.into();
|
||||
page_active_model.download_status = Set(status.into());
|
||||
page_active_model.path = Set(Some(video_path.to_str().unwrap().to_string()));
|
||||
page_active_model.path = Set(Some(video_path.to_string_lossy().to_string()));
|
||||
Ok(page_active_model)
|
||||
}
|
||||
|
||||
@@ -421,11 +496,11 @@ pub async fn fetch_page_poster(
|
||||
downloader: &Downloader,
|
||||
poster_path: PathBuf,
|
||||
fanart_path: Option<PathBuf>,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let single_page = video_model.single_page.unwrap();
|
||||
let single_page = video_model.single_page.context("single_page is null")?;
|
||||
let url = if single_page {
|
||||
// 单页视频直接用视频的封面
|
||||
video_model.cover.as_str()
|
||||
@@ -440,7 +515,7 @@ pub async fn fetch_page_poster(
|
||||
if let Some(fanart_path) = fanart_path {
|
||||
fs::copy(&poster_path, &fanart_path).await?;
|
||||
}
|
||||
Ok(())
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn fetch_page_video(
|
||||
@@ -449,10 +524,10 @@ pub async fn fetch_page_video(
|
||||
video_model: &video::Model,
|
||||
downloader: &Downloader,
|
||||
page_info: &PageInfo,
|
||||
page_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
page_path: &Path,
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let bili_video = Video::new(bili_client, video_model.bvid.clone());
|
||||
let streams = bili_video
|
||||
@@ -460,15 +535,11 @@ pub async fn fetch_page_video(
|
||||
.await?
|
||||
.best_stream(&CONFIG.filter_option)?;
|
||||
match streams {
|
||||
BestStream::Mixed(mix_stream) => {
|
||||
downloader.fetch(mix_stream.url(), &page_path).await?;
|
||||
}
|
||||
BestStream::Mixed(mix_stream) => downloader.fetch(mix_stream.url(), page_path).await?,
|
||||
BestStream::VideoAudio {
|
||||
video: video_stream,
|
||||
audio: None,
|
||||
} => {
|
||||
downloader.fetch(video_stream.url(), &page_path).await?;
|
||||
}
|
||||
} => downloader.fetch(video_stream.url(), page_path).await?,
|
||||
BestStream::VideoAudio {
|
||||
video: video_stream,
|
||||
audio: Some(audio_stream),
|
||||
@@ -477,12 +548,18 @@ pub async fn fetch_page_video(
|
||||
page_path.with_extension("tmp_video"),
|
||||
page_path.with_extension("tmp_audio"),
|
||||
);
|
||||
downloader.fetch(video_stream.url(), &tmp_video_path).await?;
|
||||
downloader.fetch(audio_stream.url(), &tmp_audio_path).await?;
|
||||
downloader.merge(&tmp_video_path, &tmp_audio_path, &page_path).await?;
|
||||
let res = async {
|
||||
downloader.fetch(video_stream.url(), &tmp_video_path).await?;
|
||||
downloader.fetch(audio_stream.url(), &tmp_audio_path).await?;
|
||||
downloader.merge(&tmp_video_path, &tmp_audio_path, page_path).await
|
||||
}
|
||||
.await;
|
||||
let _ = fs::remove_file(tmp_video_path).await;
|
||||
let _ = fs::remove_file(tmp_audio_path).await;
|
||||
res?
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn fetch_page_danmaku(
|
||||
@@ -491,9 +568,9 @@ pub async fn fetch_page_danmaku(
|
||||
video_model: &video::Model,
|
||||
page_info: &PageInfo,
|
||||
danmaku_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let bili_video = Video::new(bili_client, video_model.bvid.clone());
|
||||
bili_video
|
||||
@@ -501,7 +578,30 @@ pub async fn fetch_page_danmaku(
|
||||
.await?
|
||||
.write(danmaku_path)
|
||||
.await?;
|
||||
Ok(())
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn fetch_page_subtitle(
|
||||
should_run: bool,
|
||||
bili_client: &BiliClient,
|
||||
video_model: &video::Model,
|
||||
page_info: &PageInfo,
|
||||
subtitle_path: &Path,
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let bili_video = Video::new(bili_client, video_model.bvid.clone());
|
||||
let subtitles = bili_video.get_subtitles(page_info).await?;
|
||||
let tasks = subtitles
|
||||
.into_iter()
|
||||
.map(|subtitle| async move {
|
||||
let path = subtitle_path.with_extension(format!("{}.srt", subtitle.lan));
|
||||
tokio::fs::write(path, subtitle.body.to_string()).await
|
||||
})
|
||||
.collect::<FuturesUnordered<_>>();
|
||||
tasks.try_collect::<Vec<()>>().await?;
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn generate_page_nfo(
|
||||
@@ -509,17 +609,18 @@ pub async fn generate_page_nfo(
|
||||
video_model: &video::Model,
|
||||
page_model: &page::Model,
|
||||
nfo_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let single_page = video_model.single_page.unwrap();
|
||||
let single_page = video_model.single_page.context("single_page is null")?;
|
||||
let nfo_serializer = if single_page {
|
||||
NFOSerializer(ModelWrapper::Video(video_model), NFOMode::MOVIE)
|
||||
} else {
|
||||
NFOSerializer(ModelWrapper::Page(page_model), NFOMode::EPOSODE)
|
||||
};
|
||||
generate_nfo(nfo_serializer, nfo_path).await
|
||||
generate_nfo(nfo_serializer, nfo_path).await?;
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn fetch_video_poster(
|
||||
@@ -528,56 +629,52 @@ pub async fn fetch_video_poster(
|
||||
downloader: &Downloader,
|
||||
poster_path: PathBuf,
|
||||
fanart_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
downloader.fetch(&video_model.cover, &poster_path).await?;
|
||||
fs::copy(&poster_path, &fanart_path).await?;
|
||||
Ok(())
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn fetch_upper_face(
|
||||
should_run: bool,
|
||||
video_model: &video::Model,
|
||||
downloader: &Downloader,
|
||||
upper_face_mutex: &Mutex<()>,
|
||||
upper_face_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
// 这个锁只是为了避免多个视频同时下载同一个 up 主的头像,不携带实际内容
|
||||
let _ = upper_face_mutex.lock().await;
|
||||
if !upper_face_path.exists() {
|
||||
return downloader.fetch(&video_model.upper_face, &upper_face_path).await;
|
||||
}
|
||||
Ok(())
|
||||
downloader.fetch(&video_model.upper_face, &upper_face_path).await?;
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn generate_upper_nfo(
|
||||
should_run: bool,
|
||||
video_model: &video::Model,
|
||||
upper_nfo_mutex: &Mutex<()>,
|
||||
nfo_path: PathBuf,
|
||||
) -> Result<()> {
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let _ = upper_nfo_mutex.lock().await;
|
||||
if !nfo_path.exists() {
|
||||
let nfo_serializer = NFOSerializer(ModelWrapper::Video(video_model), NFOMode::UPPER);
|
||||
return generate_nfo(nfo_serializer, nfo_path).await;
|
||||
}
|
||||
Ok(())
|
||||
let nfo_serializer = NFOSerializer(ModelWrapper::Video(video_model), NFOMode::UPPER);
|
||||
generate_nfo(nfo_serializer, nfo_path).await?;
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
pub async fn generate_video_nfo(should_run: bool, video_model: &video::Model, nfo_path: PathBuf) -> Result<()> {
|
||||
pub async fn generate_video_nfo(
|
||||
should_run: bool,
|
||||
video_model: &video::Model,
|
||||
nfo_path: PathBuf,
|
||||
) -> Result<ExecutionStatus> {
|
||||
if !should_run {
|
||||
return Ok(());
|
||||
return Ok(ExecutionStatus::Skipped);
|
||||
}
|
||||
let nfo_serializer = NFOSerializer(ModelWrapper::Video(video_model), NFOMode::TVSHOW);
|
||||
generate_nfo(nfo_serializer, nfo_path).await
|
||||
generate_nfo(nfo_serializer, nfo_path).await?;
|
||||
Ok(ExecutionStatus::Succeeded)
|
||||
}
|
||||
|
||||
/// 创建 nfo_path 的父目录,然后写入 nfo 文件
|
||||
@@ -596,6 +693,7 @@ async fn generate_nfo(serializer: NFOSerializer<'_>, nfo_path: PathBuf) -> Resul
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use handlebars::handlebars_helper;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -610,15 +708,49 @@ mod tests {
|
||||
}
|
||||
});
|
||||
template.register_helper("truncate", Box::new(truncate));
|
||||
let _ = template.register_template_string("video", "test{{bvid}}test");
|
||||
let _ = template.register_template_string("test_truncate", "哈哈,{{ truncate title 30 }}");
|
||||
let _ = template.path_safe_register("video", "test{{bvid}}test");
|
||||
let _ = template.path_safe_register("test_truncate", "哈哈,{{ truncate title 30 }}");
|
||||
let _ = template.path_safe_register("test_path_unix", "{{ truncate title 7 }}/test/a");
|
||||
let _ = template.path_safe_register("test_path_windows", r"{{ truncate title 7 }}\\test\\a");
|
||||
#[cfg(not(windows))]
|
||||
{
|
||||
assert_eq!(
|
||||
template
|
||||
.path_safe_render("test_path_unix", &json!({"title": "关注/永雏塔菲喵"}))
|
||||
.unwrap(),
|
||||
"关注_永雏塔菲/test/a"
|
||||
);
|
||||
assert_eq!(
|
||||
template
|
||||
.path_safe_render("test_path_windows", &json!({"title": "关注/永雏塔菲喵"}))
|
||||
.unwrap(),
|
||||
"关注_永雏塔菲_test_a"
|
||||
);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
assert_eq!(
|
||||
template
|
||||
.path_safe_render("test_path_unix", &json!({"title": "关注/永雏塔菲喵"}))
|
||||
.unwrap(),
|
||||
"关注_永雏塔菲_test_a"
|
||||
);
|
||||
assert_eq!(
|
||||
template
|
||||
.path_safe_render("test_path_windows", &json!({"title": "关注/永雏塔菲喵"}))
|
||||
.unwrap(),
|
||||
r"关注_永雏塔菲\\test\\a"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
template.render("video", &json!({"bvid": "BV1b5411h7g7"})).unwrap(),
|
||||
template
|
||||
.path_safe_render("video", &json!({"bvid": "BV1b5411h7g7"}))
|
||||
.unwrap(),
|
||||
"testBV1b5411h7g7test"
|
||||
);
|
||||
assert_eq!(
|
||||
template
|
||||
.render(
|
||||
.path_safe_render(
|
||||
"test_truncate",
|
||||
&json!({"title": "你说得对,但是 Rust 是由 Mozilla 自主研发的一款全新的编译期格斗游戏。\
|
||||
编译将发生在一个被称作「Cargo」的构建系统中。在这里,被引用的指针将被授予「生命周期」之力,导引对象安全。\
|
||||
|
||||
@@ -13,6 +13,7 @@ pub struct Model {
|
||||
pub r#type: i32,
|
||||
pub path: String,
|
||||
pub created_at: String,
|
||||
pub latest_row_at: DateTime,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
|
||||
@@ -12,6 +12,7 @@ pub struct Model {
|
||||
pub name: String,
|
||||
pub path: String,
|
||||
pub created_at: String,
|
||||
pub latest_row_at: DateTime,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
|
||||
@@ -5,5 +5,6 @@ pub mod prelude;
|
||||
pub mod collection;
|
||||
pub mod favorite;
|
||||
pub mod page;
|
||||
pub mod submission;
|
||||
pub mod video;
|
||||
pub mod watch_later;
|
||||
|
||||
20
crates/bili_sync_entity/src/entities/submission.rs
Normal file
@@ -0,0 +1,20 @@
|
||||
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15
|
||||
|
||||
use sea_orm::entity::prelude::*;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
|
||||
#[sea_orm(table_name = "submission")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key)]
|
||||
pub id: i32,
|
||||
pub upper_id: i64,
|
||||
pub upper_name: String,
|
||||
pub path: String,
|
||||
pub created_at: String,
|
||||
pub latest_row_at: DateTime,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
@@ -10,6 +10,7 @@ pub struct Model {
|
||||
pub collection_id: Option<i32>,
|
||||
pub favorite_id: Option<i32>,
|
||||
pub watch_later_id: Option<i32>,
|
||||
pub submission_id: Option<i32>,
|
||||
pub upper_id: i64,
|
||||
pub upper_name: String,
|
||||
pub upper_face: String,
|
||||
|
||||
@@ -9,6 +9,7 @@ pub struct Model {
|
||||
pub id: i32,
|
||||
pub path: String,
|
||||
pub created_at: String,
|
||||
pub latest_row_at: DateTime,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
|
||||
@@ -3,6 +3,8 @@ pub use sea_orm_migration::prelude::*;
|
||||
mod m20240322_000001_create_table;
|
||||
mod m20240505_130850_add_collection;
|
||||
mod m20240709_130914_watch_later;
|
||||
mod m20240724_161008_submission;
|
||||
mod m20250122_062926_add_latest_row_at;
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
@@ -13,6 +15,8 @@ impl MigratorTrait for Migrator {
|
||||
Box::new(m20240322_000001_create_table::Migration),
|
||||
Box::new(m20240505_130850_add_collection::Migration),
|
||||
Box::new(m20240709_130914_watch_later::Migration),
|
||||
Box::new(m20240724_161008_submission::Migration),
|
||||
Box::new(m20250122_062926_add_latest_row_at::Migration),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
use sea_orm_migration::prelude::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
let db = manager.get_connection();
|
||||
manager
|
||||
.create_table(
|
||||
Table::create()
|
||||
.table(Submission::Table)
|
||||
.if_not_exists()
|
||||
.col(
|
||||
ColumnDef::new(Submission::Id)
|
||||
.unsigned()
|
||||
.not_null()
|
||||
.auto_increment()
|
||||
.primary_key(),
|
||||
)
|
||||
.col(ColumnDef::new(Submission::UpperId).unique_key().unsigned().not_null())
|
||||
.col(ColumnDef::new(Submission::UpperName).string().not_null())
|
||||
.col(ColumnDef::new(Submission::Path).string().not_null())
|
||||
.col(
|
||||
ColumnDef::new(Submission::CreatedAt)
|
||||
.timestamp()
|
||||
.default(Expr::current_timestamp())
|
||||
.not_null(),
|
||||
)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.drop_index(Index::drop().table(Video::Table).name("idx_video_unique").to_owned())
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Video::Table)
|
||||
.add_column(ColumnDef::new(Video::SubmissionId).unsigned().null())
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
db.execute_unprepared("CREATE UNIQUE INDEX `idx_video_unique` ON `video` (ifnull(`collection_id`, -1), ifnull(`favorite_id`, -1), ifnull(`watch_later_id`, -1), ifnull(`submission_id`, -1), `bvid`)")
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
let db = manager.get_connection();
|
||||
manager
|
||||
.drop_index(Index::drop().table(Video::Table).name("idx_video_unique").to_owned())
|
||||
.await?;
|
||||
db.execute_unprepared("DELETE FROM video WHERE submission_id IS NOT NULL")
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Video::Table)
|
||||
.drop_column(Video::SubmissionId)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
db.execute_unprepared("CREATE UNIQUE INDEX `idx_video_unique` ON `video` (ifnull(`collection_id`, -1), ifnull(`favorite_id`, -1), ifnull(`watch_later_id`, -1), `bvid`)")
|
||||
.await?;
|
||||
manager
|
||||
.drop_table(Table::drop().table(Submission::Table).to_owned())
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum Submission {
|
||||
Table,
|
||||
Id,
|
||||
UpperId,
|
||||
UpperName,
|
||||
Path,
|
||||
CreatedAt,
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum Video {
|
||||
Table,
|
||||
SubmissionId,
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
use sea_orm_migration::prelude::*;
|
||||
use sea_orm_migration::schema::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
// 为四张 video source 表添加 latest_row_at 字段,表示该列表处理到的最新时间
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Favorite::Table)
|
||||
.add_column(timestamp(Favorite::LatestRowAt).default("1970-01-01 00:00:00"))
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Collection::Table)
|
||||
.add_column(timestamp(Collection::LatestRowAt).default("1970-01-01 00:00:00"))
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(WatchLater::Table)
|
||||
.add_column(timestamp(WatchLater::LatestRowAt).default("1970-01-01 00:00:00"))
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Submission::Table)
|
||||
.add_column(timestamp(Submission::LatestRowAt).default("1970-01-01 00:00:00"))
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
// 手动写 SQL 更新这四张表的 latest 字段到当前取值
|
||||
let db = manager.get_connection();
|
||||
db.execute_unprepared(
|
||||
"UPDATE favorite SET latest_row_at = (SELECT IFNULL(MAX(favtime), '1970-01-01 00:00:00') FROM video WHERE favorite_id = favorite.id)",
|
||||
)
|
||||
.await?;
|
||||
db.execute_unprepared(
|
||||
"UPDATE collection SET latest_row_at = (SELECT IFNULL(MAX(pubtime), '1970-01-01 00:00:00') FROM video WHERE collection_id = collection.id)",
|
||||
)
|
||||
.await?;
|
||||
db.execute_unprepared(
|
||||
"UPDATE watch_later SET latest_row_at = (SELECT IFNULL(MAX(favtime), '1970-01-01 00:00:00') FROM video WHERE watch_later_id = watch_later.id)",
|
||||
)
|
||||
.await?;
|
||||
db.execute_unprepared(
|
||||
"UPDATE submission SET latest_row_at = (SELECT IFNULL(MAX(ctime), '1970-01-01 00:00:00') FROM video WHERE submission_id = submission.id)",
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Favorite::Table)
|
||||
.drop_column(Favorite::LatestRowAt)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Collection::Table)
|
||||
.drop_column(Collection::LatestRowAt)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(WatchLater::Table)
|
||||
.drop_column(WatchLater::LatestRowAt)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.alter_table(
|
||||
Table::alter()
|
||||
.table(Submission::Table)
|
||||
.drop_column(Submission::LatestRowAt)
|
||||
.to_owned(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum Favorite {
|
||||
Table,
|
||||
LatestRowAt,
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum Collection {
|
||||
Table,
|
||||
LatestRowAt,
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum WatchLater {
|
||||
Table,
|
||||
LatestRowAt,
|
||||
}
|
||||
|
||||
#[derive(DeriveIden)]
|
||||
enum Submission {
|
||||
Table,
|
||||
LatestRowAt,
|
||||
}
|
||||
@@ -21,7 +21,7 @@ export default defineConfig({
|
||||
nav: [
|
||||
{ text: "主页", link: "/" },
|
||||
{
|
||||
text: "v2.1.2",
|
||||
text: "v2.4.1",
|
||||
items: [
|
||||
{
|
||||
text: "程序更新",
|
||||
@@ -58,8 +58,16 @@ export default defineConfig({
|
||||
text: "获取视频合集/视频列表信息",
|
||||
link: "/collection",
|
||||
},
|
||||
{ text: "获取投稿信息", link: "/submission" },
|
||||
],
|
||||
},
|
||||
{
|
||||
text: "其它",
|
||||
items: [
|
||||
{ text: "常见问题", link: "/question" },
|
||||
{ text: "管理页", link: "/frontend" },
|
||||
],
|
||||
}
|
||||
],
|
||||
socialLinks: [
|
||||
{ icon: "github", link: "https://github.com/amtoaer/bili-sync" },
|
||||
|
||||
7
docs/.vitepress/theme/index.css
Normal file
@@ -0,0 +1,7 @@
|
||||
.medium-zoom-overlay {
|
||||
z-index: 30;
|
||||
}
|
||||
|
||||
.medium-zoom-image--opened {
|
||||
z-index: 31;
|
||||
}
|
||||
22
docs/.vitepress/theme/index.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import DefaultTheme from "vitepress/theme";
|
||||
import { onMounted, watch, nextTick } from "vue";
|
||||
import { useRoute } from "vitepress";
|
||||
import mediumZoom from "medium-zoom";
|
||||
import "./index.css";
|
||||
|
||||
export default {
|
||||
...DefaultTheme,
|
||||
setup() {
|
||||
const route = useRoute();
|
||||
const initZoom = () => {
|
||||
mediumZoom(".main img", { background: "var(--vp-c-bg)" });
|
||||
};
|
||||
onMounted(() => {
|
||||
initZoom();
|
||||
});
|
||||
watch(
|
||||
() => route.path,
|
||||
() => nextTick(() => initZoom()),
|
||||
);
|
||||
},
|
||||
};
|
||||
|
Before Width: | Height: | Size: 173 KiB |
BIN
docs/assets/bili_collection.webp
Normal file
|
After Width: | Height: | Size: 33 KiB |
|
Before Width: | Height: | Size: 172 KiB |
BIN
docs/assets/bili_video.webp
Normal file
|
After Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 2.9 MiB |
|
Before Width: | Height: | Size: 6.1 MiB |
BIN
docs/assets/detail.webp
Normal file
|
After Width: | Height: | Size: 342 KiB |
|
Before Width: | Height: | Size: 1015 KiB |
BIN
docs/assets/dir.webp
Normal file
|
After Width: | Height: | Size: 130 KiB |
|
Before Width: | Height: | Size: 3.4 MiB |
BIN
docs/assets/favorite.webp
Normal file
|
After Width: | Height: | Size: 258 KiB |
BIN
docs/assets/frontend.webp
Normal file
|
After Width: | Height: | Size: 167 KiB |
|
Before Width: | Height: | Size: 3.1 MiB |
BIN
docs/assets/multi_page.webp
Normal file
|
After Width: | Height: | Size: 167 KiB |
|
Before Width: | Height: | Size: 3.1 MiB |
BIN
docs/assets/multi_page_detail.webp
Normal file
|
After Width: | Height: | Size: 152 KiB |
|
Before Width: | Height: | Size: 4.6 MiB |
BIN
docs/assets/overview.webp
Normal file
|
After Width: | Height: | Size: 270 KiB |
|
Before Width: | Height: | Size: 2.4 MiB |
BIN
docs/assets/play.webp
Normal file
|
After Width: | Height: | Size: 216 KiB |
|
Before Width: | Height: | Size: 3.2 MiB |