Compare commits

...

44 Commits

Author SHA1 Message Date
amtoaer
e76673d076 chore: 发布 bili-sync 2.8.0 2025-12-01 23:15:07 +08:00
Naomi
f3822dd536 feat: 完善nfo时间字段、演员缩略图 (#542) 2025-11-29 01:22:26 +08:00
amtoaer
688c8cec6a feat: 凭据刷新部分添加一些 context 方便调试 2025-11-21 10:50:52 +08:00
amtoaer
c854e4e889 fix: 尝试修复执行速度过快导致的时间戳问题 2025-11-20 15:04:39 +08:00
amtoaer
645e686822 fix: 确保流中出现的错误类型能够正确保留 2025-11-11 14:42:05 +08:00
ᴀᴍᴛᴏᴀᴇʀ
670f21a725 refactor: 整理重构下载任务调度部分的代码,增强可读性和鲁棒性 (#531) 2025-11-11 01:29:52 +08:00
amtoaer
8931cb5d2a feat: 滚动条不再导致布局抖动,优化图表配色 2025-11-09 21:48:05 +08:00
amtoaer
66996a77c6 chore: flac 流解析错误时打印错误的流信息,方便后续修复 2025-11-09 19:11:46 +08:00
ᴀᴍᴛᴏᴀᴇʀ
170bd14fe3 feat: 重构视频下载任务的触发逻辑,由简单的 tokio::sleep 迁移至调度器调度 (#529) 2025-11-09 01:11:42 +08:00
ᴀᴍᴛᴏᴀᴇʀ
c69a88f1da feat: 优化风控相关的细节处理 (#527) 2025-11-08 00:41:07 +08:00
ᴀᴍᴛᴏᴀᴇʀ
8ac6829e61 feat: 支持配置通知器,在视频源处理或整个下载任务出现错误时会触发消息通知 (#526) 2025-11-07 20:37:09 +08:00
ᴀᴍᴛᴏᴀᴇʀ
a871db655f feat: 支持删除视频源 (#525) 2025-11-07 15:15:03 +08:00
ᴀᴍᴛᴏᴀᴇʀ
854d39cf88 feat: 优化对全局配置的处理,调整下载路径填充逻辑 (#523) 2025-11-06 17:25:26 +08:00
amtoaer
b6cba69e11 chore: 处理视频流出错时报出具体错误信息 2025-11-02 00:43:07 +08:00
ᴀᴍᴛᴏᴀᴇʀ
ff6db0ad97 feat: 更换部分 API,重构 wbi 签名实现,增加额外风控检测 (#503) 2025-10-15 02:01:41 +08:00
ᴀᴍᴛᴏᴀᴇʀ
84d353365a feat: 支持设置快捷订阅的路径默认值 (#502) 2025-10-14 18:44:33 +08:00
ᴀᴍᴛᴏᴀᴇʀ
c7e0d31811 chore: 移除旧版配置文件的迁移逻辑 (#501) 2025-10-14 16:32:40 +08:00
ᴀᴍᴛᴏᴀᴇʀ
2fff5134cf fix: 修复 sysinfo 初始值偶尔异常偏高的问题 (#499) 2025-10-14 01:38:26 +08:00
ᴀᴍᴛᴏᴀᴇʀ
8a1569d085 refactor: 重构 WebSocket 处理部分,整理逻辑并优化性能 (#498) 2025-10-13 20:15:30 +08:00
ᴀᴍᴛᴏᴀᴇʀ
de702435af feat: 重构下载模块,将文件下载到临时目录再最终移动至目标路径 (#495) 2025-10-13 01:59:50 +08:00
ᴀᴍᴛᴏᴀᴇʀ
eb2606f120 feat: 加入充电视频和番剧、影视判断,同时修复 category 被错误覆盖的问题 (#494) 2025-10-12 03:01:04 +08:00
ᴀᴍᴛᴏᴀᴇʀ
02c42861ab feat: 支持跳过视频的某些处理部分 (#492) 2025-10-11 20:45:44 +08:00
ᴀᴍᴛᴏᴀᴇʀ
ed54ca13b8 feat: 支持使用动态 api 获取投稿,该 api 会返回动态视频 (#485) 2025-10-10 18:52:07 +08:00
ᴀᴍᴛᴏᴀᴇʀ
4d6669a48a refactor: 使用 returning 简化逻辑 (#488) 2025-10-10 13:57:53 +08:00
ᴀᴍᴛᴏᴀᴇʀ
eadb464363 chore: 更新 rust 依赖 (#486) 2025-10-10 12:49:11 +08:00
amtoaer
2b046362d7 chore: 发布 bili-sync 2.7.0 2025-09-25 00:51:59 +08:00
ᴀᴍᴛᴏᴀᴇʀ
61c9e7de88 chore: 前端小修改,ua 随机范围添加 windows (#470) 2025-09-25 00:50:17 +08:00
ᴀᴍᴛᴏᴀᴇʀ
3d25c6b321 chore: 跑一遍 auto-correct (#468) 2025-09-24 18:50:47 +08:00
ᴀᴍᴛᴏᴀᴇʀ
d35858790b chore: clippy 应该拒绝 warning (#466) 2025-09-24 17:58:04 +08:00
ᴀᴍᴛᴏᴀᴇʀ
b441f04cdf chore: 修复新的 clippy warnings (#467) 2025-09-24 17:36:20 +08:00
ᴀᴍᴛᴏᴀᴇʀ
4db7e6763a feat: 支持重新评估历史视频,前端显示视频的规则评估状态 (#465) 2025-09-24 17:08:04 +08:00
ᴀᴍᴛᴏᴀᴇʀ
bbbb7d0c5b feat: 利用 etag 节省内容传输,显式写明生命周期 (#464) 2025-09-24 02:03:06 +08:00
ᴀᴍᴛᴏᴀᴇʀ
210c94398a feat: 实现视频的筛选规则 (#457) 2025-09-24 00:42:27 +08:00
ᴀᴍᴛᴏᴀᴇʀ
6c7d295fe6 fix: 修复字幕风控的报错 (#463) 2025-09-23 08:27:14 +08:00
ᴀᴍᴛᴏᴀᴇʀ
71519af2f3 chore: 移除不必要的 image-proxy (#451) 2025-08-28 18:51:23 +08:00
Thomas Yang
8ed2fbae24 feat: 请求中header的User-Agent使用随机值 (#447) 2025-08-27 10:27:23 +08:00
amtoaer
fd90bc8b73 chore: 下载失败时不再打印一大串 URL 2025-08-08 20:23:40 +08:00
amtoaer
66bd3d6a41 chore: ffmpeg 执行失败时添加一条说明 2025-08-07 15:11:29 +08:00
amtoaer
5ef23a678f chore: 发布 bili-sync 2.6.3 2025-08-07 12:41:48 +08:00
ᴀᴍᴛᴏᴀᴇʀ
66079f3adc feat: sqlite 开启 Wal,移除不必要的 Arc,妥善释放数据库 (#421) 2025-08-06 17:20:06 +08:00
ᴀᴍᴛᴏᴀᴇʀ
4f780faf64 fix: 增加 busy_timeout、最小化事务块、增加每批处理 page 量 (#420) 2025-08-06 14:08:07 +08:00
ᴀᴍᴛᴏᴀᴇʀ
dbcb1fa78b fix: 为填充视频详情添加并发限制,避免数据库竞争 (#419) 2025-08-06 10:37:06 +08:00
amtoaer
386dac7735 chore: 格式化后端代码 2025-08-05 23:11:55 +08:00
Xinyu Bao
5537c621be Add error messages for the case in which the database initialization fails (#415) 2025-08-05 23:11:11 +08:00
111 changed files with 5139 additions and 2441 deletions

View File

@@ -37,7 +37,7 @@ jobs:
run: cargo +nightly fmt --check
- name: cargo clippy
run: cargo clippy
run: cargo clippy -- -D warnings
- name: cargo test
run: cargo test

2
.gitignore vendored
View File

@@ -1,6 +1,6 @@
**/target
auth_data
*.sqlite
*.sqlite*
debug*
node_modules
docs/.vitepress/cache

850
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -4,7 +4,7 @@ default-members = ["crates/bili_sync"]
resolver = "2"
[workspace.package]
version = "2.6.2"
version = "2.8.0"
authors = ["amtoaer <amtoaer@gmail.com>"]
license = "MIT"
description = "由 Rust & Tokio 驱动的哔哩哔哩同步工具"
@@ -15,20 +15,20 @@ publish = false
bili_sync_entity = { path = "crates/bili_sync_entity" }
bili_sync_migration = { path = "crates/bili_sync_migration" }
anyhow = { version = "1.0.98", features = ["backtrace"] }
anyhow = { version = "1.0.100", features = ["backtrace"] }
arc-swap = { version = "1.7.1", features = ["serde"] }
assert_matches = "1.5.0"
async-std = { version = "1.13.1", features = ["attributes", "tokio1"] }
async-stream = "0.3.6"
async-trait = "0.1.88"
axum = { version = "0.8.4", features = ["macros", "ws"] }
async-tempfile = { version = "0.7.0", features = ["uuid"] }
async-trait = "0.1.89"
axum = { version = "0.8.6", features = ["macros", "ws"] }
base64 = "0.22.1"
built = { version = "0.7.7", features = ["git2", "chrono"] }
chrono = { version = "0.4.41", features = ["serde"] }
clap = { version = "4.5.41", features = ["env", "string"] }
chrono = { version = "0.4.42", features = ["serde"] }
clap = { version = "4.5.48", features = ["env", "string"] }
cookie = "0.18.1"
cow-utils = "0.1.3"
croner = "3.0.1"
dashmap = "6.1.0"
derivative = "2.2.0"
dirs = "6.0.0"
enum_dispatch = "0.3.13"
float-ord = "0.3.2"
@@ -38,14 +38,14 @@ handlebars = "6.3.2"
hex = "0.4.3"
leaky-bucket = "1.1.2"
md5 = "0.8.0"
memchr = "2.7.5"
memchr = "2.7.6"
once_cell = "1.21.3"
parking_lot = "0.12.4"
parking_lot = "0.12.5"
prost = "0.14.1"
quick-xml = { version = "0.38.0", features = ["async-tokio"] }
rand = "0.9.1"
regex = "1.11.1"
reqwest = { version = "0.12.22", features = [
quick-xml = { version = "0.38.3", features = ["async-tokio"] }
rand = "0.9.2"
regex = "1.11.3"
reqwest = { version = "0.12.23", features = [
"charset",
"cookies",
"gzip",
@@ -54,28 +54,31 @@ reqwest = { version = "0.12.22", features = [
"rustls-tls",
"stream",
], default-features = false }
rsa = { version = "0.10.0-rc.3", features = ["sha2"] }
rsa = { version = "0.10.0-rc.9", features = ["sha2"] }
rust-embed-for-web = { git = "https://github.com/amtoaer/rust-embed-for-web", tag = "v1.0.0" }
sea-orm = { version = "1.1.13", features = [
sea-orm = { version = "1.1.17", features = [
"macros",
"runtime-tokio-rustls",
"sqlx-sqlite",
"sqlite-use-returning-for-3_35",
] }
sea-orm-migration = { version = "1.1.13", features = [] }
serde = { version = "1.0.219", features = ["derive"] }
serde_json = "1.0.140"
sea-orm-migration = { version = "1.1.17", features = [] }
serde = { version = "1.0.228", features = ["derive"] }
serde_json = "1.0.145"
serde_urlencoded = "0.7.1"
strum = { version = "0.27.1", features = ["derive"] }
sysinfo = "0.36.0"
thiserror = "2.0.12"
tokio = { version = "1.46.1", features = ["full"] }
strum = { version = "0.27.2", features = ["derive"] }
sysinfo = "0.37.2"
thiserror = "2.0.17"
tokio = { version = "1.47.1", features = ["full"] }
tokio-cron-scheduler = "0.15.1"
tokio-stream = { version = "0.1.17", features = ["sync"] }
tokio-util = { version = "0.7.15", features = ["io", "rt"] }
toml = "0.9.1"
tokio-util = { version = "0.7.16", features = ["io", "rt"] }
toml = "0.9.7"
tower = "0.5.2"
tracing = "0.1.41"
tracing-subscriber = { version = "0.3.19", features = ["chrono", "json"] }
uuid = { version = "1.17.0", features = ["v4"] }
tracing-subscriber = { version = "0.3.20", features = ["chrono", "json"] }
ua_generator = "0.5.31"
uuid = { version = "1.18.1", features = ["v4"] }
validator = { version = "0.20.0", features = ["derive"] }
[workspace.metadata.release]

View File

@@ -13,6 +13,7 @@ build = "build.rs"
anyhow = { workspace = true }
arc-swap = { workspace = true }
async-stream = { workspace = true }
async-tempfile = { workspace = true }
axum = { workspace = true }
base64 = { workspace = true }
bili_sync_entity = { workspace = true }
@@ -20,7 +21,7 @@ bili_sync_migration = { workspace = true }
chrono = { workspace = true }
clap = { workspace = true }
cookie = { workspace = true }
cow-utils = { workspace = true }
croner = { workspace = true }
dashmap = { workspace = true }
dirs = { workspace = true }
enum_dispatch = { workspace = true }
@@ -48,18 +49,17 @@ strum = { workspace = true }
sysinfo = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio-cron-scheduler = { workspace = true }
tokio-stream = { workspace = true }
tokio-util = { workspace = true }
toml = { workspace = true }
tower = { workspace = true }
tracing = { workspace = true }
tracing-subscriber = { workspace = true }
ua_generator = { workspace = true }
uuid = { workspace = true }
validator = { workspace = true }
[dev-dependencies]
assert_matches = { workspace = true }
[build-dependencies]
built = { workspace = true }
git2 = { workspace = true }

View File

@@ -2,7 +2,8 @@ use std::borrow::Cow;
use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use anyhow::{Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use chrono::Utc;
use futures::Stream;
@@ -12,7 +13,7 @@ use sea_orm::sea_query::SimpleExpr;
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
use crate::bilibili::{BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
use crate::bilibili::{BiliClient, Collection, CollectionItem, CollectionType, Credential, VideoInfo};
impl VideoSource for collection::Model {
fn display_name(&self) -> Cow<'static, str> {
@@ -43,7 +44,12 @@ impl VideoSource for collection::Model {
})
}
fn should_take(&self, _release_datetime: &chrono::DateTime<Utc>, _latest_row_at: &chrono::DateTime<Utc>) -> bool {
fn should_take(
&self,
_idx: usize,
_release_datetime: &chrono::DateTime<Utc>,
_latest_row_at: &chrono::DateTime<Utc>,
) -> bool {
// collection视频合集/视频列表)返回的内容似乎并非严格按照时间排序,并且不同 collection 的排序方式也不同
// 为了保证程序正确性collection 不根据时间提前 break而是每次都全量拉取
true
@@ -51,21 +57,27 @@ impl VideoSource for collection::Model {
fn should_filter(
&self,
_idx: usize,
video_info: Result<VideoInfo, anyhow::Error>,
latest_row_at: &chrono::DateTime<Utc>,
) -> Option<VideoInfo> {
// 由于 collection 的视频无固定时间顺序should_take 无法提前中断拉取,因此 should_filter 环节需要进行额外过滤
if let Ok(video_info) = video_info {
if video_info.release_datetime() > latest_row_at {
return Some(video_info);
}
if let Ok(video_info) = video_info
&& video_info.release_datetime() > latest_row_at
{
return Some(video_info);
}
None
}
fn rule(&self) -> &Option<Rule> {
&self.rule
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
credential: &'a Credential,
connection: &'a DatabaseConnection,
) -> Result<(
VideoSourceEnum,
@@ -78,6 +90,7 @@ impl VideoSource for collection::Model {
mid: self.m_id.to_string(),
collection_type: CollectionType::from_expected(self.r#type),
},
credential,
);
let collection_info = collection.get_info().await?;
ensure!(
@@ -88,21 +101,18 @@ impl VideoSource for collection::Model {
collection_info,
collection.collection
);
collection::ActiveModel {
let updated_model = collection::ActiveModel {
id: Unchanged(self.id),
name: Set(collection_info.name.clone()),
name: Set(collection_info.name),
..Default::default()
}
.save(connection)
.update(connection)
.await?;
Ok((
collection::Entity::find()
.filter(collection::Column::Id.eq(self.id))
.one(connection)
.await?
.context("collection not found")?
.into(),
Box::pin(collection.into_video_stream()),
))
Ok((updated_model.into(), Box::pin(collection.into_video_stream())))
}
async fn delete_from_db(self, conn: &impl ConnectionTrait) -> Result<()> {
self.delete(conn).await?;
Ok(())
}
}

View File

@@ -2,7 +2,8 @@ use std::borrow::Cow;
use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use anyhow::{Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -11,7 +12,7 @@ use sea_orm::sea_query::SimpleExpr;
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
use crate::bilibili::{BiliClient, FavoriteList, VideoInfo};
use crate::bilibili::{BiliClient, Credential, FavoriteList, VideoInfo};
impl VideoSource for favorite::Model {
fn display_name(&self) -> Cow<'static, str> {
@@ -42,15 +43,20 @@ impl VideoSource for favorite::Model {
})
}
fn rule(&self) -> &Option<Rule> {
&self.rule
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
credential: &'a Credential,
connection: &'a DatabaseConnection,
) -> Result<(
VideoSourceEnum,
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + Send + 'a>>,
)> {
let favorite = FavoriteList::new(bili_client, self.f_id.to_string());
let favorite = FavoriteList::new(bili_client, self.f_id.to_string(), credential);
let favorite_info = favorite.get_info().await?;
ensure!(
favorite_info.id == self.f_id,
@@ -58,21 +64,18 @@ impl VideoSource for favorite::Model {
favorite_info.id,
self.f_id
);
favorite::ActiveModel {
let updated_model = favorite::ActiveModel {
id: Unchanged(self.id),
name: Set(favorite_info.title.clone()),
name: Set(favorite_info.title),
..Default::default()
}
.save(connection)
.update(connection)
.await?;
Ok((
favorite::Entity::find()
.filter(favorite::Column::Id.eq(self.id))
.one(connection)
.await?
.context("favorite not found")?
.into(),
Box::pin(favorite.into_video_stream()),
))
Ok((updated_model.into(), Box::pin(favorite.into_video_stream())))
}
async fn delete_from_db(self, conn: &impl ConnectionTrait) -> Result<()> {
self.delete(conn).await?;
Ok(())
}
}

View File

@@ -7,7 +7,7 @@ use std::borrow::Cow;
use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use anyhow::{Context, Result};
use chrono::Utc;
use enum_dispatch::enum_dispatch;
use futures::Stream;
@@ -19,10 +19,11 @@ use sea_orm::sea_query::SimpleExpr;
#[rustfmt::skip]
use bili_sync_entity::collection::Model as Collection;
use bili_sync_entity::favorite::Model as Favorite;
use bili_sync_entity::rule::Rule;
use bili_sync_entity::submission::Model as Submission;
use bili_sync_entity::watch_later::Model as WatchLater;
use crate::bilibili::{BiliClient, VideoInfo};
use crate::bilibili::{BiliClient, Credential, VideoInfo};
#[enum_dispatch]
pub enum VideoSourceEnum {
@@ -55,12 +56,18 @@ pub trait VideoSource {
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel;
// 判断是否应该继续拉取视频
fn should_take(&self, release_datetime: &chrono::DateTime<Utc>, latest_row_at: &chrono::DateTime<Utc>) -> bool {
fn should_take(
&self,
_idx: usize,
release_datetime: &chrono::DateTime<Utc>,
latest_row_at: &chrono::DateTime<Utc>,
) -> bool {
release_datetime > latest_row_at
}
fn should_filter(
&self,
_idx: usize,
video_info: Result<VideoInfo, anyhow::Error>,
_latest_row_at: &chrono::DateTime<Utc>,
) -> Option<VideoInfo> {
@@ -68,6 +75,8 @@ pub trait VideoSource {
video_info.ok()
}
fn rule(&self) -> &Option<Rule>;
fn log_refresh_video_start(&self) {
info!("开始扫描{}..", self.display_name());
}
@@ -95,11 +104,25 @@ pub trait VideoSource {
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
credential: &'a Credential,
connection: &'a DatabaseConnection,
) -> Result<(
VideoSourceEnum,
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + Send + 'a>>,
)>;
async fn create_dir_all(&self) -> Result<()> {
let video_source_path = self.path();
tokio::fs::create_dir_all(video_source_path).await.with_context(|| {
format!(
"failed to create video source directory {}",
video_source_path.display()
)
})?;
Ok(())
}
async fn delete_from_db(self, conn: &impl ConnectionTrait) -> Result<()>;
}
pub enum _ActiveModel {

View File

@@ -1,7 +1,8 @@
use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use anyhow::{Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -10,7 +11,7 @@ use sea_orm::sea_query::SimpleExpr;
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
use crate::bilibili::{BiliClient, Submission, VideoInfo};
use crate::bilibili::{BiliClient, Credential, Dynamic, Submission, VideoInfo};
impl VideoSource for submission::Model {
fn display_name(&self) -> std::borrow::Cow<'static, str> {
@@ -41,15 +42,56 @@ impl VideoSource for submission::Model {
})
}
fn should_take(
&self,
idx: usize,
release_datetime: &chrono::DateTime<chrono::Utc>,
latest_row_at: &chrono::DateTime<chrono::Utc>,
) -> bool {
// 如果使用动态 API那么可能出现用户置顶了一个很久以前的视频在动态顶部的情况
// 这种情况应该继续拉取下去,不能因为第一条不满足条件就停止
// 后续的非置顶内容是正常由新到旧排序的,可以继续使用常规方式处理
if idx == 0 && self.use_dynamic_api {
return true;
}
release_datetime > latest_row_at
}
fn should_filter(
&self,
idx: usize,
video_info: Result<VideoInfo, anyhow::Error>,
latest_row_at: &chrono::DateTime<chrono::Utc>,
) -> Option<VideoInfo> {
if idx == 0 && self.use_dynamic_api {
// 同理,动态 API 的第一条内容可能是置顶的老视频,单独做个过滤
// 其实不过滤也不影响逻辑正确性,因为后续 insert 发生冲突仍然会忽略掉
// 此处主要是出于性能考虑,减少不必要的数据库操作
if let Ok(video_info) = video_info
&& video_info.release_datetime() > latest_row_at
{
return Some(video_info);
}
None
} else {
video_info.ok()
}
}
fn rule(&self) -> &Option<Rule> {
&self.rule
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
credential: &'a Credential,
connection: &'a DatabaseConnection,
) -> Result<(
VideoSourceEnum,
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + Send + 'a>>,
)> {
let submission = Submission::new(bili_client, self.upper_id.to_string());
let submission = Submission::new(bili_client, self.upper_id.to_string(), credential);
let upper = submission.get_info().await?;
ensure!(
upper.mid == submission.upper_id,
@@ -57,21 +99,24 @@ impl VideoSource for submission::Model {
upper.mid,
submission.upper_id
);
submission::ActiveModel {
let updated_model = submission::ActiveModel {
id: Unchanged(self.id),
upper_name: Set(upper.name),
..Default::default()
}
.save(connection)
.update(connection)
.await?;
Ok((
submission::Entity::find()
.filter(submission::Column::Id.eq(self.id))
.one(connection)
.await?
.context("submission not found")?
.into(),
Box::pin(submission.into_video_stream()),
))
let video_stream = if self.use_dynamic_api {
// 必须显式写出 dyn否则 rust 会自动推导到 impl 从而认为 if else 返回类型不一致
Box::pin(Dynamic::from(submission).into_video_stream()) as Pin<Box<dyn Stream<Item = _> + Send + 'a>>
} else {
Box::pin(submission.into_video_stream())
};
Ok((updated_model.into(), video_stream))
}
async fn delete_from_db(self, conn: &impl ConnectionTrait) -> Result<()> {
self.delete(conn).await?;
Ok(())
}
}

View File

@@ -2,6 +2,7 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -10,7 +11,7 @@ use sea_orm::sea_query::SimpleExpr;
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
use crate::bilibili::{BiliClient, VideoInfo, WatchLater};
use crate::bilibili::{BiliClient, Credential, VideoInfo, WatchLater};
impl VideoSource for watch_later::Model {
fn display_name(&self) -> std::borrow::Cow<'static, str> {
@@ -41,15 +42,25 @@ impl VideoSource for watch_later::Model {
})
}
fn rule(&self) -> &Option<Rule> {
&self.rule
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
credential: &'a Credential,
_connection: &'a DatabaseConnection,
) -> Result<(
VideoSourceEnum,
Pin<Box<dyn Stream<Item = Result<VideoInfo>> + Send + 'a>>,
)> {
let watch_later = WatchLater::new(bili_client);
let watch_later = WatchLater::new(bili_client, credential);
Ok((self.into(), Box::pin(watch_later.into_video_stream())))
}
async fn delete_from_db(self, conn: &impl ConnectionTrait) -> Result<()> {
self.delete(conn).await?;
Ok(())
}
}

View File

@@ -1,4 +1,5 @@
use serde::Deserialize;
use bili_sync_entity::rule::Rule;
use serde::{Deserialize, Serialize};
use validator::Validate;
use crate::bilibili::CollectionType;
@@ -81,14 +82,17 @@ pub struct InsertSubmissionRequest {
pub path: String,
}
#[derive(Deserialize)]
pub struct ImageProxyParams {
pub url: String,
}
#[derive(Deserialize, Validate)]
#[serde(rename_all = "camelCase")]
pub struct UpdateVideoSourceRequest {
#[validate(custom(function = "crate::utils::validation::validate_path"))]
pub path: String,
pub enabled: bool,
pub rule: Option<Rule>,
pub use_dynamic_api: Option<bool>,
}
#[derive(Serialize, Deserialize)]
pub struct DefaultPathRequest {
pub name: String,
}

View File

@@ -1,3 +1,4 @@
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use sea_orm::{DerivePartialModel, FromQueryResult};
use serde::Serialize;
@@ -58,6 +59,7 @@ pub struct VideoInfo {
pub bvid: String,
pub name: String,
pub upper_name: String,
pub should_download: bool,
#[serde(serialize_with = "serde_video_download_status")]
pub download_status: u32,
}
@@ -157,7 +159,7 @@ pub struct DashBoardResponse {
pub videos_by_day: Vec<DayCountPair>,
}
#[derive(Serialize)]
#[derive(Serialize, Clone, Copy)]
pub struct SysInfo {
pub total_memory: u64,
pub used_memory: u64,
@@ -169,9 +171,21 @@ pub struct SysInfo {
}
#[derive(Serialize, FromQueryResult)]
#[serde(rename_all = "camelCase")]
pub struct VideoSourceDetail {
pub id: i32,
pub name: String,
pub path: String,
pub rule: Option<Rule>,
#[serde(default)]
pub rule_display: Option<String>,
#[serde(default)]
pub use_dynamic_api: Option<bool>,
pub enabled: bool,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UpdateVideoSourceResponse {
pub rule_display: Option<String>,
}

View File

@@ -6,10 +6,8 @@ use axum::extract::Extension;
use axum::routing::get;
use sea_orm::DatabaseConnection;
use crate::api::error::InnerApiError;
use crate::api::wrapper::{ApiError, ApiResponse, ValidatedJson};
use crate::config::{Config, VersionedConfig};
use crate::utils::task_notifier::TASK_STATUS_NOTIFIER;
pub(super) fn router() -> Router {
Router::new().route("/config", get(get_config).put(update_config))
@@ -17,20 +15,15 @@ pub(super) fn router() -> Router {
/// 获取全局配置
pub async fn get_config() -> Result<ApiResponse<Arc<Config>>, ApiError> {
Ok(ApiResponse::ok(VersionedConfig::get().load_full()))
Ok(ApiResponse::ok(VersionedConfig::get().snapshot()))
}
/// 更新全局配置
pub async fn update_config(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
ValidatedJson(config): ValidatedJson<Config>,
) -> Result<ApiResponse<Arc<Config>>, ApiError> {
let Some(_lock) = TASK_STATUS_NOTIFIER.detect_running() else {
// 简单避免一下可能的不一致现象
return Err(InnerApiError::BadRequest("下载任务正在运行,无法修改配置".to_string()).into());
};
config.check()?;
let new_config = VersionedConfig::get().update(config, db.as_ref()).await?;
drop(_lock);
let new_config = VersionedConfig::get().update(config, &db).await?;
Ok(ApiResponse::ok(new_config))
}

View File

@@ -1,5 +1,3 @@
use std::sync::Arc;
use axum::routing::get;
use axum::{Extension, Router};
use bili_sync_entity::*;
@@ -14,21 +12,21 @@ pub(super) fn router() -> Router {
}
async fn get_dashboard(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<DashBoardResponse>, ApiError> {
let (enabled_favorites, enabled_collections, enabled_submissions, enabled_watch_later, videos_by_day) = tokio::try_join!(
favorite::Entity::find()
.filter(favorite::Column::Enabled.eq(true))
.count(db.as_ref()),
.count(&db),
collection::Entity::find()
.filter(collection::Column::Enabled.eq(true))
.count(db.as_ref()),
.count(&db),
submission::Entity::find()
.filter(submission::Column::Enabled.eq(true))
.count(db.as_ref()),
.count(&db),
watch_later::Entity::find()
.filter(watch_later::Column::Enabled.eq(true))
.count(db.as_ref()),
.count(&db),
DayCountPair::find_by_statement(Statement::from_string(
db.get_database_backend(),
// 用 SeaORM 太复杂了,直接写个裸 SQL
@@ -55,13 +53,13 @@ ORDER BY
dates.day;
"
))
.all(db.as_ref()),
.all(&db),
)?;
return Ok(ApiResponse::ok(DashBoardResponse {
Ok(ApiResponse::ok(DashBoardResponse {
enabled_favorites,
enabled_collections,
enabled_submissions,
enable_watch_later: enabled_watch_later > 0,
videos_by_day,
}));
}))
}

View File

@@ -15,6 +15,7 @@ use crate::api::response::{
};
use crate::api::wrapper::{ApiError, ApiResponse};
use crate::bilibili::{BiliClient, Me};
use crate::config::VersionedConfig;
pub(super) fn router() -> Router {
Router::new()
@@ -25,14 +26,15 @@ pub(super) fn router() -> Router {
/// 获取当前用户创建的收藏夹
pub async fn get_created_favorites(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
) -> Result<ApiResponse<FavoritesResponse>, ApiError> {
let me = Me::new(bili_client.as_ref());
let credential = &VersionedConfig::get().read().credential;
let me = Me::new(bili_client.as_ref(), credential);
let bili_favorites = me.get_created_favorites().await?;
let favorites = if let Some(bili_favorites) = bili_favorites {
// b 站收藏夹相关接口使用的所谓 “fid” 其实是该处的 id即 fid + mid 后两位
// b 站收藏夹相关接口使用的所谓“fid”其实是该处的 id即 fid + mid 后两位
let bili_fids: Vec<_> = bili_favorites.iter().map(|fav| fav.id).collect();
let subscribed_fids: Vec<i64> = favorite::Entity::find()
@@ -40,7 +42,7 @@ pub async fn get_created_favorites(
.column(favorite::Column::FId)
.filter(favorite::Column::FId.is_in(bili_fids))
.into_tuple()
.all(db.as_ref())
.all(&db)
.await?;
let subscribed_set: HashSet<i64> = subscribed_fids.into_iter().collect();
@@ -64,11 +66,12 @@ pub async fn get_created_favorites(
/// 获取当前用户收藏的合集
pub async fn get_followed_collections(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
Query(params): Query<FollowedCollectionsRequest>,
) -> Result<ApiResponse<CollectionsResponse>, ApiError> {
let me = Me::new(bili_client.as_ref());
let credential = &VersionedConfig::get().read().credential;
let me = Me::new(bili_client.as_ref(), credential);
let (page_num, page_size) = (params.page_num.unwrap_or(1), params.page_size.unwrap_or(50));
let bili_collections = me.get_followed_collections(page_num, page_size).await?;
@@ -80,7 +83,7 @@ pub async fn get_followed_collections(
.column(collection::Column::SId)
.filter(collection::Column::SId.is_in(bili_sids))
.into_tuple()
.all(db.as_ref())
.all(&db)
.await?;
let subscribed_set: HashSet<i64> = subscribed_ids.into_iter().collect();
@@ -106,11 +109,12 @@ pub async fn get_followed_collections(
/// 获取当前用户关注的 UP 主
pub async fn get_followed_uppers(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
Query(params): Query<FollowedUppersRequest>,
) -> Result<ApiResponse<UppersResponse>, ApiError> {
let me = Me::new(bili_client.as_ref());
let credential = &VersionedConfig::get().read().credential;
let me = Me::new(bili_client.as_ref(), credential);
let (page_num, page_size) = (params.page_num.unwrap_or(1), params.page_size.unwrap_or(20));
let bili_uppers = me.get_followed_uppers(page_num, page_size).await?;
@@ -121,7 +125,7 @@ pub async fn get_followed_uppers(
.column(submission::Column::UpperId)
.filter(submission::Column::UpperId.is_in(bili_uid))
.into_tuple()
.all(db.as_ref())
.all(&db)
.await?;
let subscribed_set: HashSet<i64> = subscribed_ids.into_iter().collect();

View File

@@ -1,25 +1,19 @@
use std::collections::HashSet;
use std::sync::Arc;
use axum::body::Body;
use axum::extract::{Extension, Query, Request};
use axum::extract::Request;
use axum::http::HeaderMap;
use axum::middleware::Next;
use axum::response::{IntoResponse, Response};
use axum::routing::get;
use axum::{Router, middleware};
use base64::Engine;
use base64::prelude::BASE64_URL_SAFE_NO_PAD;
use reqwest::{Method, StatusCode, header};
use reqwest::StatusCode;
use super::request::ImageProxyParams;
use crate::api::wrapper::ApiResponse;
use crate::bilibili::BiliClient;
use crate::config::VersionedConfig;
mod config;
mod dashboard;
mod me;
mod task;
mod video_sources;
mod videos;
mod ws;
@@ -27,7 +21,7 @@ mod ws;
pub use ws::{LogHelper, MAX_HISTORY_LOGS};
pub fn router() -> Router {
Router::new().route("/image-proxy", get(image_proxy)).nest(
Router::new().nest(
"/api",
config::router()
.merge(me::router())
@@ -35,13 +29,14 @@ pub fn router() -> Router {
.merge(videos::router())
.merge(dashboard::router())
.merge(ws::router())
.merge(task::router())
.layer(middleware::from_fn(auth)),
)
}
/// 中间件:使用 auth token 对请求进行身份验证
pub async fn auth(mut headers: HeaderMap, request: Request, next: Next) -> Result<Response, StatusCode> {
let config = VersionedConfig::get().load();
let config = VersionedConfig::get().read();
let token = config.auth_token.as_str();
if headers
.get("Authorization")
@@ -50,57 +45,16 @@ pub async fn auth(mut headers: HeaderMap, request: Request, next: Next) -> Resul
{
return Ok(next.run(request).await);
}
if let Some(protocol) = headers.remove("Sec-WebSocket-Protocol") {
if protocol
if let Some(protocol) = headers.remove("Sec-WebSocket-Protocol")
&& protocol
.to_str()
.ok()
.and_then(|s| BASE64_URL_SAFE_NO_PAD.decode(s).ok())
.is_some_and(|s| s == token.as_bytes())
{
let mut resp = next.run(request).await;
resp.headers_mut().insert("Sec-WebSocket-Protocol", protocol);
return Ok(resp);
}
{
let mut resp = next.run(request).await;
resp.headers_mut().insert("Sec-WebSocket-Protocol", protocol);
return Ok(resp);
}
Ok(ApiResponse::<()>::unauthorized("auth token does not match").into_response())
}
/// B 站的图片会检查 referer需要做个转发伪造一下否则直接返回 403
pub async fn image_proxy(
Extension(bili_client): Extension<Arc<BiliClient>>,
Query(params): Query<ImageProxyParams>,
) -> Response {
let resp = bili_client.client.request(Method::GET, &params.url, None).send().await;
let whitelist = [
header::CONTENT_TYPE,
header::CONTENT_LENGTH,
header::CACHE_CONTROL,
header::EXPIRES,
header::LAST_MODIFIED,
header::ETAG,
header::CONTENT_DISPOSITION,
header::CONTENT_ENCODING,
header::ACCEPT_RANGES,
header::ACCESS_CONTROL_ALLOW_ORIGIN,
]
.into_iter()
.collect::<HashSet<_>>();
let builder = Response::builder();
let response = match resp {
Err(e) => builder.status(StatusCode::BAD_GATEWAY).body(Body::new(e.to_string())),
Ok(res) => {
let mut response = builder.status(res.status());
for (k, v) in res.headers() {
if whitelist.contains(k) {
response = response.header(k, v);
}
}
let streams = res.bytes_stream();
response.body(Body::from_stream(streams))
}
};
//safety: all previously configured headers are taken from a valid response, ensuring the response is safe to use
response.unwrap()
}

View File

@@ -0,0 +1,15 @@
use anyhow::Result;
use axum::Router;
use axum::routing::post;
use crate::api::wrapper::{ApiError, ApiResponse};
use crate::task::DownloadTaskManager;
pub(super) fn router() -> Router {
Router::new().route("/task/download", post(new_download_task))
}
pub async fn new_download_task() -> Result<ApiResponse<bool>, ApiError> {
DownloadTaskManager::get().download_once().await?;
Ok(ApiResponse::ok(true))
}

View File

@@ -2,27 +2,42 @@ use std::sync::Arc;
use anyhow::Result;
use axum::Router;
use axum::extract::{Extension, Path};
use axum::extract::{Extension, Path, Query};
use axum::routing::{get, post, put};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use bili_sync_migration::Expr;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, EntityTrait, QuerySelect};
use sea_orm::entity::prelude::*;
use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QuerySelect, QueryTrait, TransactionTrait};
use crate::adapter::_ActiveModel;
use crate::adapter::{_ActiveModel, VideoSource as _, VideoSourceEnum};
use crate::api::error::InnerApiError;
use crate::api::request::{
InsertCollectionRequest, InsertFavoriteRequest, InsertSubmissionRequest, UpdateVideoSourceRequest,
DefaultPathRequest, InsertCollectionRequest, InsertFavoriteRequest, InsertSubmissionRequest,
UpdateVideoSourceRequest,
};
use crate::api::response::{
UpdateVideoSourceResponse, VideoSource, VideoSourceDetail, VideoSourcesDetailsResponse, VideoSourcesResponse,
};
use crate::api::response::{VideoSource, VideoSourceDetail, VideoSourcesDetailsResponse, VideoSourcesResponse};
use crate::api::wrapper::{ApiError, ApiResponse, ValidatedJson};
use crate::bilibili::{BiliClient, Collection, CollectionItem, FavoriteList, Submission};
use crate::config::{PathSafeTemplate, TEMPLATE, VersionedConfig};
use crate::utils::rule::FieldEvaluatable;
pub(super) fn router() -> Router {
Router::new()
.route("/video-sources", get(get_video_sources))
.route("/video-sources/details", get(get_video_sources_details))
.route("/video-sources/{type}/{id}", put(update_video_source))
.route(
"/video-sources/{type}/default-path",
get(get_video_sources_default_path),
) // 仅用于前端获取默认路径
.route(
"/video-sources/{type}/{id}",
put(update_video_source).delete(remove_video_source),
)
.route("/video-sources/{type}/{id}/evaluate", post(evaluate_video_source))
.route("/video-sources/favorites", post(insert_favorite))
.route("/video-sources/collections", post(insert_collection))
.route("/video-sources/submissions", post(insert_submission))
@@ -30,31 +45,31 @@ pub(super) fn router() -> Router {
/// 列出所有视频来源
pub async fn get_video_sources(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<VideoSourcesResponse>, ApiError> {
let (collection, favorite, submission, mut watch_later) = tokio::try_join!(
collection::Entity::find()
.select_only()
.columns([collection::Column::Id, collection::Column::Name])
.into_model::<VideoSource>()
.all(db.as_ref()),
.all(&db),
favorite::Entity::find()
.select_only()
.columns([favorite::Column::Id, favorite::Column::Name])
.into_model::<VideoSource>()
.all(db.as_ref()),
.all(&db),
submission::Entity::find()
.select_only()
.column(submission::Column::Id)
.column_as(submission::Column::UpperName, "name")
.into_model::<VideoSource>()
.all(db.as_ref()),
.all(&db),
watch_later::Entity::find()
.select_only()
.column(watch_later::Column::Id)
.column_as(Expr::value("稍后再看"), "name")
.into_model::<VideoSource>()
.all(db.as_ref())
.all(&db)
)?;
// watch_later 是一个特殊的视频来源,如果不存在则添加一个默认项
if watch_later.is_empty() {
@@ -73,52 +88,73 @@ pub async fn get_video_sources(
/// 获取视频来源详情
pub async fn get_video_sources_details(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<VideoSourcesDetailsResponse>, ApiError> {
let (collections, favorites, submissions, mut watch_later) = tokio::try_join!(
let (mut collections, mut favorites, mut submissions, mut watch_later) = tokio::try_join!(
collection::Entity::find()
.select_only()
.columns([
collection::Column::Id,
collection::Column::Name,
collection::Column::Path,
collection::Column::Rule,
collection::Column::Enabled
])
.into_model::<VideoSourceDetail>()
.all(db.as_ref()),
.all(&db),
favorite::Entity::find()
.select_only()
.columns([
favorite::Column::Id,
favorite::Column::Name,
favorite::Column::Path,
favorite::Column::Rule,
favorite::Column::Enabled
])
.into_model::<VideoSourceDetail>()
.all(db.as_ref()),
.all(&db),
submission::Entity::find()
.select_only()
.column(submission::Column::Id)
.column_as(submission::Column::UpperName, "name")
.columns([submission::Column::Path, submission::Column::Enabled])
.columns([
submission::Column::Id,
submission::Column::Path,
submission::Column::Enabled,
submission::Column::Rule,
submission::Column::UseDynamicApi
])
.into_model::<VideoSourceDetail>()
.all(db.as_ref()),
.all(&db),
watch_later::Entity::find()
.select_only()
.column(watch_later::Column::Id)
.column_as(Expr::value("稍后再看"), "name")
.columns([watch_later::Column::Path, watch_later::Column::Enabled])
.columns([
watch_later::Column::Id,
watch_later::Column::Path,
watch_later::Column::Enabled,
watch_later::Column::Rule
])
.into_model::<VideoSourceDetail>()
.all(db.as_ref())
.all(&db)
)?;
if watch_later.is_empty() {
watch_later.push(VideoSourceDetail {
id: 1,
name: "稍后再看".to_string(),
path: String::new(),
rule: None,
rule_display: None,
use_dynamic_api: None,
enabled: false,
})
}
for sources in [&mut collections, &mut favorites, &mut submissions, &mut watch_later] {
sources.iter_mut().for_each(|item| {
if let Some(rule) = &item.rule {
item.rule_display = Some(rule.to_string());
}
});
}
Ok(ApiResponse::ok(VideoSourcesDetailsResponse {
collections,
favorites,
@@ -127,32 +163,55 @@ pub async fn get_video_sources_details(
}))
}
pub async fn get_video_sources_default_path(
Path(source_type): Path<String>,
Query(params): Query<DefaultPathRequest>,
) -> Result<ApiResponse<String>, ApiError> {
let template_name = match source_type.as_str() {
"favorites" => "favorite_default_path",
"collections" => "collection_default_path",
"submissions" => "submission_default_path",
_ => return Err(InnerApiError::BadRequest("Invalid video source type".to_string()).into()),
};
let template = TEMPLATE.read();
Ok(ApiResponse::ok(
template.path_safe_render(template_name, &serde_json::to_value(params)?)?,
))
}
/// 更新视频来源
pub async fn update_video_source(
Path((source_type, id)): Path<(String, i32)>,
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
ValidatedJson(request): ValidatedJson<UpdateVideoSourceRequest>,
) -> Result<ApiResponse<bool>, ApiError> {
) -> Result<ApiResponse<UpdateVideoSourceResponse>, ApiError> {
let rule_display = request.rule.as_ref().map(|rule| rule.to_string());
let active_model = match source_type.as_str() {
"collections" => collection::Entity::find_by_id(id).one(db.as_ref()).await?.map(|model| {
"collections" => collection::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: collection::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
_ActiveModel::Collection(active_model)
}),
"favorites" => favorite::Entity::find_by_id(id).one(db.as_ref()).await?.map(|model| {
"favorites" => favorite::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: favorite::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
_ActiveModel::Favorite(active_model)
}),
"submissions" => submission::Entity::find_by_id(id).one(db.as_ref()).await?.map(|model| {
"submissions" => submission::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: submission::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
if let Some(use_dynamic_api) = request.use_dynamic_api {
active_model.use_dynamic_api = Set(use_dynamic_api);
}
_ActiveModel::Submission(active_model)
}),
"watch_later" => match watch_later::Entity::find_by_id(id).one(db.as_ref()).await? {
"watch_later" => match watch_later::Entity::find_by_id(id).one(&db).await? {
// 稍后再看需要做特殊处理get 时如果稍后再看不存在返回的是 id 为 1 的假记录
// 因此此处可能是更新也可能是插入,做个额外的处理
Some(model) => {
@@ -160,6 +219,7 @@ pub async fn update_video_source(
let mut active_model: watch_later::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
Some(_ActiveModel::WatchLater(active_model))
}
None => {
@@ -170,6 +230,7 @@ pub async fn update_video_source(
Some(_ActiveModel::WatchLater(watch_later::ActiveModel {
path: Set(request.path),
enabled: Set(request.enabled),
rule: Set(request.rule),
..Default::default()
}))
}
@@ -180,36 +241,152 @@ pub async fn update_video_source(
let Some(active_model) = active_model else {
return Err(InnerApiError::NotFound(id).into());
};
active_model.save(db.as_ref()).await?;
active_model.save(&db).await?;
Ok(ApiResponse::ok(UpdateVideoSourceResponse { rule_display }))
}
pub async fn remove_video_source(
Path((source_type, id)): Path<(String, i32)>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<bool>, ApiError> {
// 不允许删除稍后再看
let video_source: Option<VideoSourceEnum> = match source_type.as_str() {
"collections" => collection::Entity::find_by_id(id).one(&db).await?.map(Into::into),
"favorites" => favorite::Entity::find_by_id(id).one(&db).await?.map(Into::into),
"submissions" => submission::Entity::find_by_id(id).one(&db).await?.map(Into::into),
_ => return Err(InnerApiError::BadRequest("Invalid video source type".to_string()).into()),
};
let Some(video_source) = video_source else {
return Err(InnerApiError::NotFound(id).into());
};
let txn = db.begin().await?;
page::Entity::delete_many()
.filter(
page::Column::VideoId.in_subquery(
video::Entity::find()
.filter(video_source.filter_expr())
.select_only()
.column(video::Column::Id)
.as_query()
.to_owned(),
),
)
.exec(&txn)
.await?;
video::Entity::delete_many()
.filter(video_source.filter_expr())
.exec(&txn)
.await?;
video_source.delete_from_db(&txn).await?;
txn.commit().await?;
Ok(ApiResponse::ok(true))
}
pub async fn evaluate_video_source(
Path((source_type, id)): Path<(String, i32)>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<bool>, ApiError> {
// 找出对应 source 的规则与 video 筛选条件
let (rule, filter_condition) = match source_type.as_str() {
"collections" => (
collection::Entity::find_by_id(id)
.select_only()
.column(collection::Column::Rule)
.into_tuple::<Option<Rule>>()
.one(&db)
.await?
.and_then(|r| r),
video::Column::CollectionId.eq(id),
),
"favorites" => (
favorite::Entity::find_by_id(id)
.select_only()
.column(favorite::Column::Rule)
.into_tuple::<Option<Rule>>()
.one(&db)
.await?
.and_then(|r| r),
video::Column::FavoriteId.eq(id),
),
"submissions" => (
submission::Entity::find_by_id(id)
.select_only()
.column(submission::Column::Rule)
.into_tuple::<Option<Rule>>()
.one(&db)
.await?
.and_then(|r| r),
video::Column::SubmissionId.eq(id),
),
"watch_later" => (
watch_later::Entity::find_by_id(id)
.select_only()
.column(watch_later::Column::Rule)
.into_tuple::<Option<Rule>>()
.one(&db)
.await?
.and_then(|r| r),
video::Column::WatchLaterId.eq(id),
),
_ => return Err(InnerApiError::BadRequest("Invalid video source type".to_string()).into()),
};
let videos: Vec<(video::Model, Vec<page::Model>)> = video::Entity::find()
.filter(filter_condition)
.find_with_related(page::Entity)
.all(&db)
.await?;
let video_should_download_pairs = videos
.into_iter()
.map(|(video, pages)| (video.id, rule.evaluate_model(&video, &pages)))
.collect::<Vec<(i32, bool)>>();
let txn = db.begin().await?;
for chunk in video_should_download_pairs.chunks(500) {
let sql = format!(
"WITH tempdata(id, should_download) AS (VALUES {}) \
UPDATE video \
SET should_download = tempdata.should_download \
FROM tempdata \
WHERE video.id = tempdata.id",
chunk
.iter()
.map(|item| format!("({}, {})", item.0, item.1))
.collect::<Vec<_>>()
.join(", ")
);
txn.execute_unprepared(&sql).await?;
}
txn.commit().await?;
Ok(ApiResponse::ok(true))
}
/// 新增收藏夹订阅
pub async fn insert_favorite(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
ValidatedJson(request): ValidatedJson<InsertFavoriteRequest>,
) -> Result<ApiResponse<bool>, ApiError> {
let favorite = FavoriteList::new(bili_client.as_ref(), request.fid.to_string());
let credential = &VersionedConfig::get().read().credential;
let favorite = FavoriteList::new(bili_client.as_ref(), request.fid.to_string(), credential);
let favorite_info = favorite.get_info().await?;
favorite::Entity::insert(favorite::ActiveModel {
f_id: Set(favorite_info.id),
name: Set(favorite_info.title.clone()),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(db.as_ref())
.exec(&db)
.await?;
Ok(ApiResponse::ok(true))
}
/// 新增合集/列表订阅
pub async fn insert_collection(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
ValidatedJson(request): ValidatedJson<InsertCollectionRequest>,
) -> Result<ApiResponse<bool>, ApiError> {
let credential = &VersionedConfig::get().read().credential;
let collection = Collection::new(
bili_client.as_ref(),
CollectionItem {
@@ -217,6 +394,7 @@ pub async fn insert_collection(
mid: request.mid.to_string(),
collection_type: request.collection_type,
},
credential,
);
let collection_info = collection.get_info().await?;
collection::Entity::insert(collection::ActiveModel {
@@ -225,10 +403,10 @@ pub async fn insert_collection(
r#type: Set(collection_info.collection_type.into()),
name: Set(collection_info.name.clone()),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(db.as_ref())
.exec(&db)
.await?;
Ok(ApiResponse::ok(true))
@@ -236,20 +414,21 @@ pub async fn insert_collection(
/// 新增投稿订阅
pub async fn insert_submission(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Extension(bili_client): Extension<Arc<BiliClient>>,
ValidatedJson(request): ValidatedJson<InsertSubmissionRequest>,
) -> Result<ApiResponse<bool>, ApiError> {
let submission = Submission::new(bili_client.as_ref(), request.upper_id.to_string());
let credential = &VersionedConfig::get().read().credential;
let submission = Submission::new(bili_client.as_ref(), request.upper_id.to_string(), credential);
let upper = submission.get_info().await?;
submission::Entity::insert(submission::ActiveModel {
upper_id: Set(upper.mid.parse()?),
upper_name: Set(upper.name),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(db.as_ref())
.exec(&db)
.await?;
Ok(ApiResponse::ok(true))
}

View File

@@ -1,5 +1,4 @@
use std::collections::HashSet;
use std::sync::Arc;
use anyhow::Result;
use axum::extract::{Extension, Path, Query};
@@ -31,7 +30,7 @@ pub(super) fn router() -> Router {
/// 列出视频的基本信息,支持根据视频来源筛选、名称查找和分页
pub async fn get_videos(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Query(params): Query<VideosRequest>,
) -> Result<ApiResponse<VideosResponse>, ApiError> {
let mut query = video::Entity::find();
@@ -48,7 +47,7 @@ pub async fn get_videos(
if let Some(query_word) = params.query {
query = query.filter(video::Column::Name.contains(query_word));
}
let total_count = query.clone().count(db.as_ref()).await?;
let total_count = query.clone().count(&db).await?;
let (page, page_size) = if let (Some(page), Some(page_size)) = (params.page, params.page_size) {
(page, page_size)
} else {
@@ -58,7 +57,7 @@ pub async fn get_videos(
videos: query
.order_by_desc(video::Column::Id)
.into_partial_model::<VideoInfo>()
.paginate(db.as_ref(), page_size)
.paginate(&db, page_size)
.fetch_page(page)
.await?,
total_count,
@@ -67,17 +66,15 @@ pub async fn get_videos(
pub async fn get_video(
Path(id): Path<i32>,
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<VideoResponse>, ApiError> {
let (video_info, pages_info) = tokio::try_join!(
video::Entity::find_by_id(id)
.into_partial_model::<VideoInfo>()
.one(db.as_ref()),
video::Entity::find_by_id(id).into_partial_model::<VideoInfo>().one(&db),
page::Entity::find()
.filter(page::Column::VideoId.eq(id))
.order_by_asc(page::Column::Cid)
.into_partial_model::<PageInfo>()
.all(db.as_ref())
.all(&db)
)?;
let Some(video_info) = video_info else {
return Err(InnerApiError::NotFound(id).into());
@@ -90,18 +87,16 @@ pub async fn get_video(
pub async fn reset_video(
Path(id): Path<i32>,
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Json(request): Json<ResetRequest>,
) -> Result<ApiResponse<ResetVideoResponse>, ApiError> {
let (video_info, pages_info) = tokio::try_join!(
video::Entity::find_by_id(id)
.into_partial_model::<VideoInfo>()
.one(db.as_ref()),
video::Entity::find_by_id(id).into_partial_model::<VideoInfo>().one(&db),
page::Entity::find()
.filter(page::Column::VideoId.eq(id))
.order_by_asc(page::Column::Cid)
.into_partial_model::<PageInfo>()
.all(db.as_ref())
.all(&db)
)?;
let Some(mut video_info) = video_info else {
return Err(InnerApiError::NotFound(id).into());
@@ -150,13 +145,13 @@ pub async fn reset_video(
}
pub async fn reset_all_videos(
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
Json(request): Json<ResetRequest>,
) -> Result<ApiResponse<ResetAllVideosResponse>, ApiError> {
// 先查询所有视频和页面数据
let (all_videos, all_pages) = tokio::try_join!(
video::Entity::find().into_partial_model::<VideoInfo>().all(db.as_ref()),
page::Entity::find().into_partial_model::<PageInfo>().all(db.as_ref())
video::Entity::find().into_partial_model::<VideoInfo>().all(&db),
page::Entity::find().into_partial_model::<PageInfo>().all(&db)
)?;
let resetted_pages_info = all_pages
.into_iter()
@@ -210,18 +205,16 @@ pub async fn reset_all_videos(
pub async fn update_video_status(
Path(id): Path<i32>,
Extension(db): Extension<Arc<DatabaseConnection>>,
Extension(db): Extension<DatabaseConnection>,
ValidatedJson(request): ValidatedJson<UpdateVideoStatusRequest>,
) -> Result<ApiResponse<UpdateVideoStatusResponse>, ApiError> {
let (video_info, mut pages_info) = tokio::try_join!(
video::Entity::find_by_id(id)
.into_partial_model::<VideoInfo>()
.one(db.as_ref()),
video::Entity::find_by_id(id).into_partial_model::<VideoInfo>().one(&db),
page::Entity::find()
.filter(page::Column::VideoId.eq(id))
.order_by_asc(page::Column::Cid)
.into_partial_model::<PageInfo>()
.all(db.as_ref())
.all(&db)
)?;
let Some(mut video_info) = video_info else {
return Err(InnerApiError::NotFound(id).into());

View File

@@ -1,7 +1,7 @@
use std::collections::VecDeque;
use std::sync::Arc;
use parking_lot::Mutex;
use parking_lot::RwLock;
use tokio::sync::broadcast;
use tracing_subscriber::fmt::MakeWriter;
@@ -10,11 +10,11 @@ pub const MAX_HISTORY_LOGS: usize = 30;
/// LogHelper 维护了日志发送器和一个日志历史记录的缓冲区
pub struct LogHelper {
pub sender: broadcast::Sender<String>,
pub log_history: Arc<Mutex<VecDeque<String>>>,
pub log_history: Arc<RwLock<VecDeque<String>>>,
}
impl LogHelper {
pub fn new(sender: broadcast::Sender<String>, log_history: Arc<Mutex<VecDeque<String>>>) -> Self {
pub fn new(sender: broadcast::Sender<String>, log_history: Arc<RwLock<VecDeque<String>>>) -> Self {
LogHelper { sender, log_history }
}
}
@@ -31,7 +31,7 @@ impl std::io::Write for LogHelper {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let log_message = String::from_utf8_lossy(buf).to_string();
let _ = self.sender.send(log_message.clone());
let mut history = self.log_history.lock();
let mut history = self.log_history.write();
history.push_back(log_message);
if history.len() > MAX_HISTORY_LOGS {
history.pop_front();

View File

@@ -15,15 +15,18 @@ pub use log_helper::{LogHelper, MAX_HISTORY_LOGS};
use parking_lot::RwLock;
use serde::{Deserialize, Serialize};
use sysinfo::{
CpuRefreshKind, DiskRefreshKind, Disks, MemoryRefreshKind, ProcessRefreshKind, RefreshKind, System, get_current_pid,
CpuRefreshKind, DiskRefreshKind, Disks, MemoryRefreshKind, Pid, ProcessRefreshKind, ProcessesToUpdate, System,
get_current_pid,
};
use tokio::pin;
use tokio::task::JoinHandle;
use tokio_stream::wrappers::{BroadcastStream, IntervalStream, WatchStream};
use tokio::sync::mpsc;
use tokio::{pin, select};
use tokio_stream::wrappers::{BroadcastStream, WatchStream};
use tokio_util::future::FutureExt;
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
use crate::api::response::SysInfo;
use crate::utils::task_notifier::{TASK_STATUS_NOTIFIER, TaskStatus};
use crate::task::{DownloadTaskManager, TaskStatus};
static WEBSOCKET_HANDLER: LazyLock<WebSocketHandler> = LazyLock::new(WebSocketHandler::new);
@@ -55,191 +58,235 @@ enum ClientEvent {
#[serde(rename_all = "camelCase")]
enum ServerEvent {
Logs(String),
Tasks(Arc<TaskStatus>),
SysInfo(Arc<SysInfo>),
Tasks(TaskStatus),
SysInfo(SysInfo),
}
struct WebSocketHandler {
sysinfo_subscribers: Arc<DashMap<Uuid, tokio::sync::mpsc::Sender<ServerEvent>>>,
sysinfo_handles: RwLock<Option<JoinHandle<()>>>,
sysinfo_subscribers: Arc<DashMap<Uuid, mpsc::Sender<ServerEvent>>>,
sysinfo_cancel: RwLock<Option<CancellationToken>>,
}
impl WebSocketHandler {
fn new() -> Self {
Self {
sysinfo_subscribers: Arc::new(DashMap::new()),
sysinfo_handles: RwLock::new(None),
sysinfo_cancel: RwLock::new(None),
}
}
async fn handle_sender(
&self,
mut sender: SplitSink<WebSocket, Message>,
mut rx: tokio::sync::mpsc::Receiver<ServerEvent>,
) {
/// 向客户端推送信息
async fn handle_sender(&self, mut sender: SplitSink<WebSocket, Message>, mut rx: mpsc::Receiver<ServerEvent>) {
while let Some(event) = rx.recv().await {
match serde_json::to_string(&event) {
Ok(text) => {
if let Err(e) = sender.send(Message::Text(text.into())).await {
error!("Failed to send message: {:?}", e);
break;
}
}
let text = match serde_json::to_string(&event) {
Ok(text) => text,
Err(e) => {
error!("Failed to serialize event: {:?}", e);
continue;
}
};
if let Err(e) = sender.send(Message::Text(text.into())).await {
error!("Failed to send message: {:?}", e);
break;
}
}
}
/// 从客户端接收信息
async fn handle_receiver(
&self,
mut receiver: SplitStream<WebSocket>,
tx: tokio::sync::mpsc::Sender<ServerEvent>,
tx: mpsc::Sender<ServerEvent>,
uuid: Uuid,
log_writer: LogHelper,
) {
// 日志和任务状态的处理本身就是由 stream 驱动的,可以直接为每个 ws 连接维护独立的任务处理器
// 系统信息是服务端轮询然后推送的,如果单独维护会导致每个连接都独立轮询系统信息,造成不必要的浪费
// 因此采用了全局的订阅者管理,所有连接共享同一个系统信息轮询任务
let (mut log_handle, mut task_handle) = (None, None);
let (mut log_cancel, mut task_cancel) = (None, None);
while let Some(Ok(msg)) = receiver.next().await {
if let Message::Text(text) = msg {
match serde_json::from_str::<ClientEvent>(&text) {
Ok(ClientEvent::Subscribe(event_type)) => match event_type {
EventType::Logs => {
if log_handle.as_ref().is_none_or(|h: &JoinHandle<()>| h.is_finished()) {
let log_writer_clone = log_writer.clone();
let tx_clone = tx.clone();
let history = log_writer_clone.log_history.lock();
let history_logs: Vec<String> = history.iter().cloned().collect();
drop(history);
log_handle = Some(tokio::spawn(async move {
let rx = log_writer_clone.sender.subscribe();
let log_stream = futures::stream::iter(history_logs.into_iter())
.chain(BroadcastStream::new(rx).filter_map(async |msg| msg.ok()))
.map(|msg| ServerEvent::Logs(msg));
pin!(log_stream);
while let Some(event) = log_stream.next().await {
if let Err(e) = tx_clone.send(event).await {
error!("Failed to send log event: {:?}", e);
break;
}
}
}));
}
}
EventType::Tasks => {
if task_handle.as_ref().is_none_or(|h: &JoinHandle<()>| h.is_finished()) {
let tx_clone = tx.clone();
task_handle = Some(tokio::spawn(async move {
let mut stream = WatchStream::new(TASK_STATUS_NOTIFIER.subscribe())
.map(|status| ServerEvent::Tasks(status));
while let Some(event) = stream.next().await {
if let Err(e) = tx_clone.send(event).await {
error!("Failed to send task status: {:?}", e);
break;
}
}
}));
}
}
EventType::SysInfo => self.add_sysinfo_subscriber(uuid, tx.clone()).await,
},
Ok(ClientEvent::Unsubscribe(event_type)) => match event_type {
EventType::Logs => {
if let Some(handle) = log_handle.take() {
handle.abort();
}
}
EventType::Tasks => {
if let Some(handle) = task_handle.take() {
handle.abort();
}
}
EventType::SysInfo => {
self.remove_sysinfo_subscriber(uuid).await;
}
},
Err(e) => {
error!("Failed to parse client message: {:?}", e);
let Message::Text(text) = msg else {
continue;
};
let client_event = match serde_json::from_str::<ClientEvent>(&text) {
Ok(event) => event,
Err(e) => {
error!("Failed to parse client message: {:?}, error: {:?}", text, e);
continue;
}
};
match client_event {
ClientEvent::Subscribe(EventType::Logs) => {
if log_cancel.is_none() {
log_cancel = Some(self.new_log_handler(tx.clone(), &log_writer));
}
}
ClientEvent::Unsubscribe(EventType::Logs) => {
if let Some(cancel) = log_cancel.take() {
cancel.cancel();
}
}
ClientEvent::Subscribe(EventType::Tasks) => {
if task_cancel.is_none() {
task_cancel = Some(self.new_task_handler(tx.clone()));
}
}
ClientEvent::Unsubscribe(EventType::Tasks) => {
if let Some(cancel) = task_cancel.take() {
cancel.cancel();
}
}
ClientEvent::Subscribe(EventType::SysInfo) => {
self.add_sysinfo_subscriber(uuid, tx.clone());
}
ClientEvent::Unsubscribe(EventType::SysInfo) => {
self.remove_sysinfo_subscriber(uuid);
}
}
}
// 连接关闭,清除仍然残留的任务
if let Some(cancel) = log_cancel {
cancel.cancel();
}
if let Some(cancel) = task_cancel {
cancel.cancel();
}
self.remove_sysinfo_subscriber(uuid);
}
/// 添加全局系统信息订阅者
fn add_sysinfo_subscriber(&self, uuid: Uuid, sender: mpsc::Sender<ServerEvent>) {
self.sysinfo_subscribers.insert(uuid, sender);
if self.sysinfo_cancel.read().is_none() {
let mut sys_info_cancel = self.sysinfo_cancel.write();
if sys_info_cancel.is_some() {
return;
}
*sys_info_cancel = Some(self.new_sysinfo_handler(self.sysinfo_subscribers.clone()));
}
}
/// 移除全局系统信息订阅者
fn remove_sysinfo_subscriber(&self, uuid: Uuid) {
self.sysinfo_subscribers.remove(&uuid);
if self.sysinfo_subscribers.is_empty()
&& let Some(token) = self.sysinfo_cancel.write().take()
{
token.cancel();
}
}
/// 创建异步日志推送任务,返回任务的取消令牌
fn new_log_handler(&self, tx: mpsc::Sender<ServerEvent>, log_writer: &LogHelper) -> CancellationToken {
let cancel_token = CancellationToken::new();
// 读取历史日志
let history = log_writer.log_history.read();
let history_logs = history.iter().cloned().collect::<Vec<String>>();
drop(history);
// 获取日志广播接收器
let log_rx = log_writer.sender.subscribe();
tokio::spawn(
async move {
// 合并历史日志和实时日志流
let log_stream = futures::stream::iter(history_logs)
.chain(BroadcastStream::new(log_rx).filter_map(async |msg| msg.ok()))
.map(ServerEvent::Logs);
pin!(log_stream);
while let Some(event) = log_stream.next().await {
if let Err(e) = tx.send(event).await {
error!("Failed to send log event: {:?}", e);
break;
}
}
}
}
if let Some(handle) = log_handle {
handle.abort();
}
if let Some(handle) = task_handle {
handle.abort();
}
self.remove_sysinfo_subscriber(uuid).await;
.with_cancellation_token_owned(cancel_token.clone()),
);
cancel_token
}
// 添加订阅者
async fn add_sysinfo_subscriber(&self, uuid: Uuid, sender: tokio::sync::mpsc::Sender<ServerEvent>) {
self.sysinfo_subscribers.insert(uuid, sender);
if self.sysinfo_subscribers.len() > 0
&& self
.sysinfo_handles
.read()
.as_ref()
.is_none_or(|h: &JoinHandle<()>| h.is_finished())
{
let sysinfo_subscribers = self.sysinfo_subscribers.clone();
let mut write_guard = self.sysinfo_handles.write();
if write_guard.as_ref().is_some_and(|h: &JoinHandle<()>| !h.is_finished()) {
return;
/// 创建异步任务状态推送任务,返回任务的取消令牌
fn new_task_handler(&self, tx: mpsc::Sender<ServerEvent>) -> CancellationToken {
let cancel_token = CancellationToken::new();
tokio::spawn(
async move {
let mut stream = WatchStream::new(DownloadTaskManager::get().subscribe()).map(ServerEvent::Tasks);
while let Some(event) = stream.next().await {
if let Err(e) = tx.send(event).await {
error!("Failed to send task status: {:?}", e);
break;
}
}
}
*write_guard = Some(tokio::spawn(async move {
let mut system = System::new();
let mut disks = Disks::new();
let sys_refresh_kind = sys_refresh_kind();
let disk_refresh_kind = disk_refresh_kind();
.with_cancellation_token_owned(cancel_token.clone()),
);
cancel_token
}
/// 创建异步系统信息推送任务,返回任务的取消令牌
fn new_sysinfo_handler(
&self,
sysinfo_subscribers: Arc<DashMap<Uuid, mpsc::Sender<ServerEvent>>>,
) -> CancellationToken {
let cancel_token = CancellationToken::new();
let cancel_token_clone = cancel_token.clone();
tokio::spawn(async move {
let (tx, mut rx) = mpsc::channel(10);
let (tick_tx, mut tick_rx) = mpsc::channel(3);
// 在阻塞线程中轮询系统信息,防止阻塞异步运行时
tokio::task::spawn_blocking(move || {
// 对于 linux/mac/windows 平台,该方法永远返回 Some(pid)expect 基本是安全的
let self_pid = get_current_pid().expect("Unsupported platform");
let mut stream =
IntervalStream::new(tokio::time::interval(Duration::from_secs(2))).filter_map(move |_| {
system.refresh_specifics(sys_refresh_kind);
disks.refresh_specifics(true, disk_refresh_kind);
let process = match system.process(self_pid) {
Some(p) => p,
None => return futures::future::ready(None),
};
futures::future::ready(Some(SysInfo {
total_memory: system.total_memory(),
used_memory: system.used_memory(),
process_memory: process.memory(),
used_cpu: system.global_cpu_usage(),
process_cpu: process.cpu_usage() / system.cpus().len() as f32,
total_disk: disks.iter().map(|d| d.total_space()).sum(),
available_disk: disks.iter().map(|d| d.available_space()).sum(),
}))
});
while let Some(sys_info) = stream.next().await {
let sys_info = Arc::new(sys_info);
future::join_all(sysinfo_subscribers.iter().map(async |subscriber| {
if let Err(e) = subscriber.send(ServerEvent::SysInfo(sys_info.clone())).await {
error!(
"Failed to send sysinfo event to subscriber {}: {:?}",
subscriber.key(),
e
);
}
}))
.await;
let mut system = System::new();
let mut disks = Disks::new();
while tick_rx.blocking_recv().is_some() {
system.refresh_needed(self_pid);
disks.refresh_needed(self_pid);
let process = match system.process(self_pid) {
Some(p) => p,
None => continue,
};
let sys_info = SysInfo {
total_memory: system.total_memory(),
used_memory: system.used_memory(),
process_memory: process.memory(),
used_cpu: system.global_cpu_usage(),
process_cpu: process.cpu_usage() / system.cpus().len() as f32,
total_disk: disks.iter().map(|d| d.total_space()).sum(),
available_disk: disks.iter().map(|d| d.available_space()).sum(),
};
if tx.blocking_send(sys_info).is_err() {
break;
}
}
});
// 异步部分负责获取由阻塞线程发送过来的系统信息,并推送给所有订阅者
// 收到取消信号时,设置标志位,确保阻塞线程正常退出
let mut interval = tokio::time::interval(Duration::from_secs(2));
loop {
select! {
_ = cancel_token_clone.cancelled() => {
drop(tick_tx);
break;
}
_ = interval.tick() => {
let _ = tick_tx.send(()).await;
}
Some(sys_info) = rx.recv() => {
future::join_all(sysinfo_subscribers.iter().map(async |subscriber| {
if let Err(e) = subscriber.send(ServerEvent::SysInfo(sys_info)).await {
error!(
"Failed to send sysinfo event to subscriber {}: {:?}",
subscriber.key(),
e
);
}
}))
.await;
}
}
}));
}
}
async fn remove_sysinfo_subscriber(&self, uuid: Uuid) {
self.sysinfo_subscribers.remove(&uuid);
if self.sysinfo_subscribers.is_empty() {
if let Some(handle) = self.sysinfo_handles.write().take() {
handle.abort();
}
}
});
cancel_token
}
}
@@ -251,13 +298,24 @@ async fn handle_socket(socket: WebSocket, log_writer: LogHelper) {
tokio::spawn(WEBSOCKET_HANDLER.handle_receiver(ws_receiver, tx, uuid, log_writer));
}
fn sys_refresh_kind() -> RefreshKind {
RefreshKind::nothing()
.with_cpu(CpuRefreshKind::nothing().with_cpu_usage())
.with_memory(MemoryRefreshKind::nothing().with_ram())
.with_processes(ProcessRefreshKind::nothing().with_cpu().with_memory())
trait SysInfoExt {
fn refresh_needed(&mut self, self_pid: Pid);
}
fn disk_refresh_kind() -> DiskRefreshKind {
DiskRefreshKind::nothing().with_storage()
impl SysInfoExt for System {
fn refresh_needed(&mut self, self_pid: Pid) {
self.refresh_memory_specifics(MemoryRefreshKind::nothing().with_ram());
self.refresh_cpu_specifics(CpuRefreshKind::nothing().with_cpu_usage());
self.refresh_processes_specifics(
ProcessesToUpdate::Some(&[self_pid]),
true,
ProcessRefreshKind::nothing().with_cpu().with_memory(),
);
}
}
impl SysInfoExt for Disks {
fn refresh_needed(&mut self, _self_pid: Pid) {
self.refresh_specifics(false, DiskRefreshKind::nothing().with_storage());
}
}

View File

@@ -2,7 +2,6 @@ use anyhow::{Context, Result, bail};
use serde::{Deserialize, Serialize};
use crate::bilibili::error::BiliError;
use crate::config::VersionedConfig;
pub struct PageAnalyzer {
info: serde_json::Value,
@@ -131,14 +130,14 @@ pub enum Stream {
// 通用的获取流链接的方法,交由 Downloader 使用
impl Stream {
pub fn urls(&self) -> Vec<&str> {
pub fn urls(&self, enable_cdn_sorting: bool) -> Vec<&str> {
match self {
Self::Flv(url) | Self::Html5Mp4(url) | Self::EpisodeTryMp4(url) => vec![url],
Self::DashVideo { url, backup_url, .. } | Self::DashAudio { url, backup_url, .. } => {
let mut urls = std::iter::once(url.as_str())
.chain(backup_url.iter().map(|s| s.as_str()))
.collect::<Vec<_>>();
if VersionedConfig::get().load().cdn_sorting {
if enable_cdn_sorting {
urls.sort_by_key(|u| {
if u.contains("upos-") {
0 // 服务商 cdn
@@ -218,7 +217,7 @@ impl PageAnalyzer {
.info
.pointer_mut("/dash/video")
.and_then(|v| v.as_array_mut())
.ok_or(BiliError::RiskControlOccurred)?
.ok_or(BiliError::VideoStreamsEmpty)?
.iter_mut()
{
let (Some(url), Some(quality), Some(codecs_id)) = (
@@ -263,39 +262,37 @@ impl PageAnalyzer {
});
}
}
if !filter_option.no_hires {
if let Some(flac) = self.info.pointer_mut("/dash/flac/audio") {
let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else {
bail!("invalid flac stream");
};
let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?;
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
streams.push(Stream::DashAudio {
url: url.to_string(),
backup_url: serde_json::from_value(flac["backupUrl"].take()).unwrap_or_default(),
quality,
});
}
if !filter_option.no_hires
&& let Some(flac) = self.info.pointer_mut("/dash/flac/audio")
{
let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else {
bail!("invalid flac stream, flac content: {}", flac);
};
let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?;
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
streams.push(Stream::DashAudio {
url: url.to_string(),
backup_url: serde_json::from_value(flac["backupUrl"].take()).unwrap_or_default(),
quality,
});
}
}
if !filter_option.no_dolby_audio {
if let Some(dolby_audio) = self
if !filter_option.no_dolby_audio
&& let Some(dolby_audio) = self
.info
.pointer_mut("/dash/dolby/audio/0")
.and_then(|a| a.as_object_mut())
{
let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else {
bail!("invalid dolby audio stream");
};
let quality =
AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?;
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
streams.push(Stream::DashAudio {
url: url.to_string(),
backup_url: serde_json::from_value(dolby_audio["backupUrl"].take()).unwrap_or_default(),
quality,
});
}
{
let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else {
bail!("invalid dolby audio stream");
};
let quality = AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?;
if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
streams.push(Stream::DashAudio {
url: url.to_string(),
backup_url: serde_json::from_value(dolby_audio["backupUrl"].take()).unwrap_or_default(),
quality,
});
}
}
Ok(streams)
@@ -426,16 +423,17 @@ mod tests {
Some(AudioQuality::Quality192k),
),
];
let config = VersionedConfig::get().read();
for (bvid, video_quality, video_codec, audio_quality) in testcases.into_iter() {
let client = BiliClient::new();
let video = Video::new(&client, bvid.to_owned());
let video = Video::new(&client, bvid.to_owned(), &config.credential);
let pages = video.get_pages().await.expect("failed to get pages");
let first_page = pages.into_iter().next().expect("no page found");
let best_stream = video
.get_page_analyzer(&first_page)
.await
.expect("failed to get page analyzer")
.best_stream(&VersionedConfig::get().load().filter_option)
.best_stream(&config.filter_option)
.expect("failed to get best stream");
dbg!(bvid, &best_stream);
match best_stream {
@@ -471,7 +469,7 @@ mod tests {
codecs: VideoCodecs::AVC,
};
assert_eq!(
stream.urls(),
stream.urls(true),
vec![
"https://upos-sz-mirrorcos.bilivideo.com",
"https://cn-tj-cu-01-11.bilivideo.com",

View File

@@ -1,13 +1,14 @@
use std::sync::Arc;
use std::time::Duration;
use anyhow::Result;
use anyhow::{Result, bail};
use leaky_bucket::RateLimiter;
use reqwest::{Method, header};
use sea_orm::DatabaseConnection;
use ua_generator::ua;
use crate::bilibili::Credential;
use crate::bilibili::credential::WbiImg;
use crate::config::{RateLimit, VersionedCache, VersionedConfig};
use crate::config::{RateLimit, VersionedCache};
// 一个对 reqwest::Client 的简单封装,用于 Bilibili 请求
#[derive(Clone)]
@@ -19,9 +20,7 @@ impl Client {
let mut headers = header::HeaderMap::new();
headers.insert(
header::USER_AGENT,
header::HeaderValue::from_static(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
),
header::HeaderValue::from_static(ua::spoof_chrome_ua()),
);
headers.insert(
header::REFERER,
@@ -61,56 +60,82 @@ impl Default for Client {
}
}
enum Limiter {
Latest(VersionedCache<Option<RateLimiter>>),
Snapshot(Arc<Option<RateLimiter>>),
}
pub struct BiliClient {
pub client: Client,
limiter: VersionedCache<Option<RateLimiter>>,
limiter: Limiter,
}
impl BiliClient {
pub fn new() -> Self {
let client = Client::new();
let limiter = VersionedCache::new(|config| {
Ok(config
.concurrent_limit
.rate_limit
.as_ref()
.map(|RateLimit { limit, duration }| {
RateLimiter::builder()
.initial(*limit)
.refill(*limit)
.max(*limit)
.interval(Duration::from_millis(*duration))
.build()
}))
})
.expect("failed to create rate limiter");
let limiter = Limiter::Latest(
VersionedCache::new(|config| {
Ok(config
.concurrent_limit
.rate_limit
.as_ref()
.map(|RateLimit { limit, duration }| {
RateLimiter::builder()
.initial(*limit)
.refill(*limit)
.max(*limit)
.interval(Duration::from_millis(*duration))
.build()
}))
})
.expect("failed to create rate limiter"),
);
Self { client, limiter }
}
/// 获取当前 BiliClient 的快照,快照中的限流器固定不变
pub fn snapshot(&self) -> Result<Self> {
let Limiter::Latest(inner) = &self.limiter else {
// 语法上没问题,但语义上不允许对快照进行快照
bail!("cannot snapshot a snapshot BiliClient");
};
Ok(Self {
client: self.client.clone(),
limiter: Limiter::Snapshot(inner.snapshot()),
})
}
/// 获取一个预构建的请求,通过该方法获取请求时会检查并等待速率限制
pub async fn request(&self, method: Method, url: &str) -> reqwest::RequestBuilder {
if let Some(limiter) = self.limiter.load().as_ref() {
limiter.acquire_one().await;
pub async fn request(&self, method: Method, url: &str, credential: &Credential) -> reqwest::RequestBuilder {
match &self.limiter {
Limiter::Latest(inner) => {
if let Some(limiter) = inner.read().as_ref() {
limiter.acquire_one().await;
}
}
Limiter::Snapshot(inner) => {
if let Some(limiter) = inner.as_ref() {
limiter.acquire_one().await;
}
}
}
let credential = &VersionedConfig::get().load().credential;
self.client.request(method, url, Some(credential))
}
pub async fn check_refresh(&self, connection: &DatabaseConnection) -> Result<()> {
let credential = &VersionedConfig::get().load().credential;
/// 检查并刷新 Credential不需要刷新返回 Ok(None),需要刷新返回 Ok(Some(new_credential))
pub async fn check_refresh(&self, credential: &Credential) -> Result<Option<Credential>> {
if !credential.need_refresh(&self.client).await? {
return Ok(());
return Ok(None);
}
let new_credential = credential.refresh(&self.client).await?;
VersionedConfig::get()
.update_credential(new_credential, connection)
.await?;
Ok(())
Ok(Some(credential.refresh(&self.client).await?))
}
/// 获取 wbi img用于生成请求签名
pub async fn wbi_img(&self) -> Result<WbiImg> {
let credential = &VersionedConfig::get().load().credential;
pub async fn wbi_img(&self, credential: &Credential) -> Result<WbiImg> {
credential.wbi_img(&self.client).await
}
pub fn inner_client(&self) -> &reqwest::Client {
&self.client.0
}
}

View File

@@ -7,8 +7,7 @@ use reqwest::Method;
use serde::Deserialize;
use serde_json::Value;
use crate::bilibili::credential::encoded_query;
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
#[derive(PartialEq, Eq, Hash, Clone, Debug, Default, Copy)]
pub enum CollectionType {
@@ -74,6 +73,7 @@ pub struct CollectionItem {
pub struct Collection<'a> {
client: &'a BiliClient,
pub collection: CollectionItem,
credential: &'a Credential,
}
#[derive(Debug, PartialEq)]
@@ -112,8 +112,12 @@ impl<'de> Deserialize<'de> for CollectionInfo {
}
impl<'a> Collection<'a> {
pub fn new(client: &'a BiliClient, collection: CollectionItem) -> Self {
Self { client, collection }
pub fn new(client: &'a BiliClient, collection: CollectionItem, credential: &'a Credential) -> Self {
Self {
client,
collection,
credential,
}
}
pub async fn get_info(&self) -> Result<CollectionInfo> {
@@ -127,7 +131,7 @@ impl<'a> Collection<'a> {
async fn get_series_info(&self) -> Result<Value> {
self.client
.request(Method::GET, "https://api.bilibili.com/x/series/series")
.request(Method::GET, "https://api.bilibili.com/x/series/series", self.credential)
.await
.query(&[("series_id", self.collection.sid.as_str())])
.send()
@@ -139,46 +143,40 @@ impl<'a> Collection<'a> {
}
async fn get_videos(&self, page: i32) -> Result<Value> {
let page = page.to_string();
let (url, query) = match self.collection.collection_type {
CollectionType::Series => (
"https://api.bilibili.com/x/series/archives",
encoded_query(
vec![
("mid", self.collection.mid.as_str()),
("series_id", self.collection.sid.as_str()),
("only_normal", "true"),
("sort", "desc"),
("pn", page.as_str()),
("ps", "30"),
],
MIXIN_KEY.load().as_deref(),
),
),
CollectionType::Season => (
"https://api.bilibili.com/x/polymer/web-space/seasons_archives_list",
encoded_query(
vec![
("mid", self.collection.mid.as_str()),
("season_id", self.collection.sid.as_str()),
("sort_reverse", "true"),
("page_num", page.as_str()),
("page_size", "30"),
],
MIXIN_KEY.load().as_deref(),
),
),
let req = match self.collection.collection_type {
CollectionType::Series => self
.client
.request(
Method::GET,
"https://api.bilibili.com/x/series/archives",
self.credential,
)
.await
.query(&[("pn", page)])
.query(&[
("mid", self.collection.mid.as_str()),
("series_id", self.collection.sid.as_str()),
("only_normal", "true"),
("sort", "desc"),
("ps", "30"),
]),
CollectionType::Season => self
.client
.request(
Method::GET,
"https://api.bilibili.com/x/polymer/web-space/seasons_archives_list",
self.credential,
)
.await
.query(&[("page_num", page)])
.query(&[
("mid", self.collection.mid.as_str()),
("season_id", self.collection.sid.as_str()),
("sort_reverse", "true"),
("page_size", "30"),
]),
};
self.client
.request(Method::GET, url)
.await
.query(&query)
.send()
.await?
.error_for_status()?
.json::<Value>()
.await?
.validate()
req.send().await?.error_for_status()?.json::<Value>().await?.validate()
}
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {

View File

@@ -1,9 +1,7 @@
use std::borrow::Cow;
use std::collections::HashSet;
use anyhow::{Context, Result, bail, ensure};
use cookie::Cookie;
use cow_utils::CowUtils;
use regex::Regex;
use reqwest::{Method, header};
use rsa::pkcs8::DecodePublicKey;
@@ -30,17 +28,13 @@ pub struct Credential {
#[derive(Debug, Deserialize)]
pub struct WbiImg {
img_url: String,
sub_url: String,
pub(crate) img_url: String,
pub(crate) sub_url: String,
}
impl From<WbiImg> for Option<String> {
/// 尝试将 WbiImg 转换成 mixin_key
fn from(value: WbiImg) -> Self {
let key = match (
get_filename(value.img_url.as_str()),
get_filename(value.sub_url.as_str()),
) {
impl WbiImg {
pub fn into_mixin_key(self) -> Option<String> {
let key = match (get_filename(self.img_url.as_str()), get_filename(self.sub_url.as_str())) {
(Some(img_key), Some(sub_key)) => img_key.to_string() + sub_key,
_ => return None,
};
@@ -81,9 +75,17 @@ impl Credential {
pub async fn refresh(&self, client: &Client) -> Result<Self> {
let correspond_path = Self::get_correspond_path();
let csrf = self.get_refresh_csrf(client, correspond_path).await?;
let new_credential = self.get_new_credential(client, &csrf).await?;
self.confirm_refresh(client, &new_credential).await?;
let csrf = self
.get_refresh_csrf(client, correspond_path)
.await
.context("获取 refresh_csrf 失败")?;
let new_credential = self
.get_new_credential(client, &csrf)
.await
.context("刷新 Credential 失败")?;
self.confirm_refresh(client, &new_credential)
.await
.context("确认更新 Credential 失败")?;
Ok(new_credential)
}
@@ -98,11 +100,11 @@ JNrRuoEUXpabUzGB8QIDAQAB
-----END PUBLIC KEY-----",
)
.expect("fail to decode public key");
let ts = chrono::Local::now().timestamp_millis();
// 精确到毫秒的时间戳可能出现时间比服务器快的情况,提前 20s 以防万一
let ts = chrono::Local::now().timestamp_millis() - 20000;
let data = format!("refresh_{}", ts).into_bytes();
let mut rng = rand::rng();
let encrypted = key
.encrypt(&mut rng, Oaep::new::<Sha256>(), &data)
.encrypt(&mut rand::rng(), Oaep::new::<Sha256>(), &data)
.expect("fail to encrypt");
hex::encode(encrypted)
}
@@ -213,47 +215,8 @@ fn get_filename(url: &str) -> Option<&str> {
.map(|(s, _)| s)
}
pub fn encoded_query<'a>(
params: Vec<(&'a str, impl Into<Cow<'a, str>>)>,
mixin_key: Option<impl AsRef<str>>,
) -> Vec<(&'a str, Cow<'a, str>)> {
match mixin_key {
Some(key) => _encoded_query(params, key.as_ref(), chrono::Local::now().timestamp().to_string()),
None => params.into_iter().map(|(k, v)| (k, v.into())).collect(),
}
}
fn _encoded_query<'a>(
params: Vec<(&'a str, impl Into<Cow<'a, str>>)>,
mixin_key: &str,
timestamp: String,
) -> Vec<(&'a str, Cow<'a, str>)> {
let disallowed = ['!', '\'', '(', ')', '*'];
let mut params: Vec<(&'a str, Cow<'a, str>)> = params
.into_iter()
.map(|(k, v)| {
(
k,
match Into::<Cow<'a, str>>::into(v) {
Cow::Borrowed(v) => v.cow_replace(&disallowed[..], ""),
Cow::Owned(v) => v.replace(&disallowed[..], "").into(),
},
)
})
.collect();
params.push(("wts", timestamp.into()));
params.sort_by(|a, b| a.0.cmp(b.0));
let query = serde_urlencoded::to_string(&params)
.expect("fail to encode query")
.replace('+', "%20");
params.push(("w_rid", format!("{:x}", md5::compute(query.clone() + mixin_key)).into()));
params
}
#[cfg(test)]
mod tests {
use assert_matches::assert_matches;
use super::*;
#[test]
@@ -283,56 +246,4 @@ mod tests {
"bar=%E4%BA%94%E4%B8%80%E5%9B%9B&baz=1919810&foo=one%20one%20four"
);
}
#[test]
fn test_wbi_key() {
let key = WbiImg {
img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(),
sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(),
};
let key = Option::<String>::from(key).expect("fail to convert key");
assert_eq!(key.as_str(), "ea1db124af3c7062474693fa704f4ff8");
// 没有特殊字符
assert_matches!(
&_encoded_query(
vec![("foo", "114"), ("bar", "514"), ("zab", "1919810")],
key.as_str(),
"1702204169".to_string(),
)[..],
[
("bar", Cow::Borrowed(a)),
("foo", Cow::Borrowed(b)),
("wts", Cow::Owned(c)),
("zab", Cow::Borrowed(d)),
("w_rid", Cow::Owned(e)),
] => {
assert_eq!(*a, "514");
assert_eq!(*b, "114");
assert_eq!(c, "1702204169");
assert_eq!(*d, "1919810");
assert_eq!(e, "8f6f2b5b3d485fe1886cec6a0be8c5d4");
}
);
// 有特殊字符
assert_matches!(
&_encoded_query(
vec![("foo", "'1(1)4'"), ("bar", "!5*1!14"), ("zab", "1919810")],
key.as_str(),
"1702204169".to_string(),
)[..],
[
("bar", Cow::Owned(a)),
("foo", Cow::Owned(b)),
("wts", Cow::Owned(c)),
("zab", Cow::Borrowed(d)),
("w_rid", Cow::Owned(e)),
] => {
assert_eq!(a, "5114");
assert_eq!(b, "114");
assert_eq!(c, "1702204169");
assert_eq!(*d, "1919810");
assert_eq!(e, "6a2c86c4b0648ce062ba0dac2de91a85");
}
);
}
}

View File

@@ -184,7 +184,7 @@ impl<'a, W: AsyncWrite> AssWriter<'a, W> {
}
}
fn escape_text(text: &str) -> Cow<str> {
fn escape_text(text: &'_ str) -> Cow<'_, str> {
let text = text.trim();
if memchr::memchr(b'\n', text.as_bytes()).is_some() {
Cow::from(text.replace('\n', "\\N"))

View File

@@ -26,7 +26,7 @@ pub struct DanmakuOption {
pub bottom_percentage: f64,
/// 透明度0-255
pub opacity: u8,
/// 是否加粗1代表是0代表否
/// 是否加粗1 代表是0 代表否
pub bold: bool,
/// 描边
pub outline: f64,

View File

@@ -39,7 +39,7 @@ pub struct Danmu {
impl Danmu {
/// 计算弹幕的“像素长度”,会乘上一个缩放因子
///
/// 汉字算一个全宽英文算2/3宽
/// 汉字算一个全宽,英文算 2/3
pub fn length(&self, config: &CanvasConfig<'_>) -> f64 {
let pts = config.danmaku_option.font_size
* self

View File

@@ -3,10 +3,9 @@ use std::path::PathBuf;
use anyhow::Result;
use tokio::fs::{self, File};
use crate::bilibili::PageInfo;
use crate::bilibili::danmaku::canvas::CanvasConfig;
use crate::bilibili::danmaku::{AssWriter, Danmu};
use crate::config::VersionedConfig;
use crate::bilibili::{DanmakuOption, PageInfo};
pub struct DanmakuWriter<'a> {
page: &'a PageInfo,
@@ -18,12 +17,11 @@ impl<'a> DanmakuWriter<'a> {
DanmakuWriter { page, danmaku }
}
pub async fn write(self, path: PathBuf) -> Result<()> {
pub async fn write(self, path: PathBuf, danmaku_option: &DanmakuOption) -> Result<()> {
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let config = VersionedConfig::get().load_full();
let canvas_config = CanvasConfig::new(&config.danmaku_option, self.page);
let canvas_config = CanvasConfig::new(danmaku_option, self.page);
let mut writer =
AssWriter::construct(File::create(path).await?, self.page.name.clone(), canvas_config.clone()).await?;
let mut canvas = canvas_config.canvas();

View File

@@ -0,0 +1,92 @@
use anyhow::{Context, Result, anyhow};
use async_stream::try_stream;
use chrono::serde::ts_seconds;
use chrono::{DateTime, Utc};
use futures::Stream;
use reqwest::Method;
use serde_json::Value;
use crate::bilibili::{BiliClient, Credential, MIXIN_KEY, Validate, VideoInfo, WbiSign};
pub struct Dynamic<'a> {
client: &'a BiliClient,
pub upper_id: String,
credential: &'a Credential,
}
#[derive(Debug, serde::Deserialize)]
pub struct DynamicItemPublished {
#[serde(with = "ts_seconds")]
pub_ts: DateTime<Utc>,
}
impl<'a> Dynamic<'a> {
pub fn new(client: &'a BiliClient, upper_id: String, credential: &'a Credential) -> Self {
Self {
client,
upper_id,
credential,
}
}
pub async fn get_dynamics(&self, offset: Option<String>) -> Result<Value> {
self.client
.request(
Method::GET,
"https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/space",
self.credential,
)
.await
.query(&[
("host_mid", self.upper_id.as_str()),
("offset", offset.as_deref().unwrap_or("")),
("type", "video"),
])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?
.json::<serde_json::Value>()
.await?
.validate()
}
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
try_stream! {
let mut offset = None;
loop {
let mut res = self
.get_dynamics(offset.take())
.await
.with_context(|| "failed to get dynamics")?;
let items = res["data"]["items"].as_array_mut().context("items not exist")?;
for item in items.iter_mut() {
if item["type"].as_str().is_none_or(|t| t != "DYNAMIC_TYPE_AV") {
continue;
}
let published: DynamicItemPublished = serde_json::from_value(item["modules"]["module_author"].take())
.with_context(|| "failed to parse published time")?;
let mut video_info: VideoInfo =
serde_json::from_value(item["modules"]["module_dynamic"]["major"]["archive"].take())?;
// 这些地方不使用 let else 是因为 try_stream! 宏不支持
if let VideoInfo::Dynamic { ref mut pubtime, .. } = video_info {
*pubtime = published.pub_ts;
yield video_info;
} else {
Err(anyhow!("video info is not dynamic"))?;
}
}
if let (Some(has_more), Some(new_offset)) =
(res["data"]["has_more"].as_bool(), res["data"]["offset"].as_str())
{
if !has_more {
break;
}
offset = Some(new_offset.to_string());
} else {
Err(anyhow!("no has_more or offset found"))?;
}
}
}
}
}

View File

@@ -1,9 +1,19 @@
use thiserror::Error;
#[derive(Error, Debug)]
#[derive(Error, Debug, Clone)]
pub enum BiliError {
#[error("risk control occurred")]
RiskControlOccurred,
#[error("request failed, status code: {0}, message: {1}")]
RequestFailed(i64, String),
#[error("response missing 'code' or 'message' field, full response: {0}")]
InvalidResponse(String),
#[error("API returned error code {0}, message: {1}, full response: {2}")]
ErrorResponse(i64, String, String),
#[error("risk control triggered by server, full response: {0}")]
RiskControlOccurred(String),
#[error("no video streams available (may indicate risk control)")]
VideoStreamsEmpty,
}
impl BiliError {
pub fn is_risk_control_related(&self) -> bool {
matches!(self, BiliError::RiskControlOccurred(_) | BiliError::VideoStreamsEmpty)
}
}

View File

@@ -3,10 +3,11 @@ use async_stream::try_stream;
use futures::Stream;
use serde_json::Value;
use crate::bilibili::{BiliClient, Validate, VideoInfo};
use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
pub struct FavoriteList<'a> {
client: &'a BiliClient,
fid: String,
credential: &'a Credential,
}
#[derive(Debug, serde::Deserialize)]
@@ -22,14 +23,22 @@ pub struct Upper<T> {
pub face: String,
}
impl<'a> FavoriteList<'a> {
pub fn new(client: &'a BiliClient, fid: String) -> Self {
Self { client, fid }
pub fn new(client: &'a BiliClient, fid: String, credential: &'a Credential) -> Self {
Self {
client,
fid,
credential,
}
}
pub async fn get_info(&self) -> Result<FavoriteListInfo> {
let mut res = self
.client
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v3/fav/folder/info")
.request(
reqwest::Method::GET,
"https://api.bilibili.com/x/v3/fav/folder/info",
self.credential,
)
.await
.query(&[("media_id", &self.fid)])
.send()
@@ -43,7 +52,11 @@ impl<'a> FavoriteList<'a> {
async fn get_videos(&self, page: u32) -> Result<Value> {
self.client
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v3/fav/resource/list")
.request(
reqwest::Method::GET,
"https://api.bilibili.com/x/v3/fav/resource/list",
self.credential,
)
.await
.query(&[
("media_id", self.fid.as_str()),

View File

@@ -1,28 +1,32 @@
use anyhow::{Result, ensure};
use reqwest::Method;
use crate::bilibili::{BiliClient, Validate};
use crate::config::VersionedConfig;
use crate::bilibili::{BiliClient, Credential, Validate};
pub struct Me<'a> {
client: &'a BiliClient,
mid: String,
credential: &'a Credential,
}
impl<'a> Me<'a> {
pub fn new(client: &'a BiliClient) -> Self {
Self {
client,
mid: Self::my_id(),
}
pub fn new(client: &'a BiliClient, credential: &'a Credential) -> Self {
Self { client, credential }
}
pub async fn get_created_favorites(&self) -> Result<Option<Vec<FavoriteItem>>> {
ensure!(!self.mid.is_empty(), "未获取到用户 ID请确保填写设置中的 B 站认证信息");
ensure!(
!self.mid().is_empty(),
"未获取到用户 ID请确保填写设置中的 B 站认证信息"
);
let mut resp = self
.client
.request(Method::GET, "https://api.bilibili.com/x/v3/fav/folder/created/list-all")
.request(
Method::GET,
"https://api.bilibili.com/x/v3/fav/folder/created/list-all",
self.credential,
)
.await
.query(&[("up_mid", &self.mid)])
.query(&[("up_mid", &self.mid())])
.send()
.await?
.error_for_status()?
@@ -33,17 +37,20 @@ impl<'a> Me<'a> {
}
pub async fn get_followed_collections(&self, page_num: i32, page_size: i32) -> Result<Collections> {
ensure!(!self.mid.is_empty(), "未获取到用户 ID请确保填写设置中的 B 站认证信息");
ensure!(
!self.mid().is_empty(),
"未获取到用户 ID请确保填写设置中的 B 站认证信息"
);
let mut resp = self
.client
.request(Method::GET, "https://api.bilibili.com/x/v3/fav/folder/collected/list")
.request(
Method::GET,
"https://api.bilibili.com/x/v3/fav/folder/collected/list",
self.credential,
)
.await
.query(&[
("up_mid", self.mid.as_str()),
("pn", page_num.to_string().as_str()),
("ps", page_size.to_string().as_str()),
("platform", "web"),
])
.query(&[("up_mid", self.mid()), ("platform", "web")])
.query(&[("pn", page_num), ("ps", page_size)])
.send()
.await?
.error_for_status()?
@@ -54,16 +61,20 @@ impl<'a> Me<'a> {
}
pub async fn get_followed_uppers(&self, page_num: i32, page_size: i32) -> Result<FollowedUppers> {
ensure!(!self.mid.is_empty(), "未获取到用户 ID请确保填写设置中的 B 站认证信息");
ensure!(
!self.mid().is_empty(),
"未获取到用户 ID请确保填写设置中的 B 站认证信息"
);
let mut resp = self
.client
.request(Method::GET, "https://api.bilibili.com/x/relation/followings")
.request(
Method::GET,
"https://api.bilibili.com/x/relation/followings",
self.credential,
)
.await
.query(&[
("vmid", self.mid.as_str()),
("pn", page_num.to_string().as_str()),
("ps", page_size.to_string().as_str()),
])
.query(&[("vmid", self.mid())])
.query(&[("pn", page_num), ("ps", page_size)])
.send()
.await?
.error_for_status()?
@@ -73,8 +84,8 @@ impl<'a> Me<'a> {
Ok(serde_json::from_value(resp["data"].take())?)
}
fn my_id() -> String {
VersionedConfig::get().load().credential.dedeuserid.clone()
fn mid(&self) -> &str {
&self.credential.dedeuserid
}
}

View File

@@ -1,3 +1,4 @@
use std::borrow::Cow;
use std::sync::Arc;
pub use analyzer::{BestStream, FilterOption};
@@ -9,11 +10,13 @@ pub use client::{BiliClient, Client};
pub use collection::{Collection, CollectionItem, CollectionType};
pub use credential::Credential;
pub use danmaku::DanmakuOption;
pub use dynamic::Dynamic;
pub use error::BiliError;
pub use favorite_list::FavoriteList;
use favorite_list::Upper;
pub use me::Me;
use once_cell::sync::Lazy;
use reqwest::RequestBuilder;
pub use submission::Submission;
pub use video::{Dimension, PageInfo, Video};
pub use watch_later::WatchLater;
@@ -23,6 +26,7 @@ mod client;
mod collection;
mod credential;
mod danmaku;
mod dynamic;
mod error;
mod favorite_list;
mod me;
@@ -49,13 +53,55 @@ impl Validate for serde_json::Value {
fn validate(self) -> Result<Self::Output> {
let (code, msg) = match (self["code"].as_i64(), self["message"].as_str()) {
(Some(code), Some(msg)) => (code, msg),
_ => bail!("no code or message found"),
_ => bail!(BiliError::InvalidResponse(self.to_string())),
};
ensure!(code == 0, BiliError::RequestFailed(code, msg.to_owned()));
if code == -352 || !self["data"]["v_voucher"].is_null() {
bail!(BiliError::RiskControlOccurred(self.to_string()));
}
ensure!(
code == 0,
BiliError::ErrorResponse(code, msg.to_owned(), self.to_string())
);
Ok(self)
}
}
pub(crate) trait WbiSign {
type Output;
fn wbi_sign(self, mixin_key: Option<impl AsRef<str>>) -> Result<Self::Output>;
}
impl WbiSign for RequestBuilder {
type Output = RequestBuilder;
fn wbi_sign(self, mixin_key: Option<impl AsRef<str>>) -> Result<Self::Output> {
let Some(mixin_key) = mixin_key else {
return Ok(self);
};
let (client, req) = self.build_split();
let mut req = match req {
Ok(req) => req,
Err(e) => return Err(e.into()),
};
sign_request(&mut req, mixin_key.as_ref(), chrono::Utc::now().timestamp())?;
Ok(RequestBuilder::from_parts(client, req))
}
}
fn sign_request(req: &mut reqwest::Request, mixin_key: &str, timestamp: i64) -> Result<()> {
let mut query_pairs = req.url().query_pairs().collect::<Vec<_>>();
let timestamp = timestamp.to_string();
query_pairs.push(("wts".into(), Cow::Borrowed(timestamp.as_str())));
query_pairs.sort_by(|a, b| a.0.cmp(&b.0));
let query_str = serde_urlencoded::to_string(query_pairs)?.replace('+', "%20");
let w_rid = format!("{:x}", md5::compute(query_str + mixin_key));
req.url_mut()
.query_pairs_mut()
.extend_pairs([("w_rid", w_rid), ("wts", timestamp)]);
Ok(())
}
#[derive(Debug, serde::Deserialize)]
#[serde(untagged)]
/// 注意此处的顺序是有要求的,因为对于 untagged 的 enum 来说serde 会按照顺序匹配
@@ -76,6 +122,9 @@ pub enum VideoInfo {
ctime: DateTime<Utc>,
#[serde(rename = "pubdate", with = "ts_seconds")]
pubtime: DateTime<Utc>,
is_upower_exclusive: bool,
is_upower_play: bool,
redirect_url: Option<String>,
pages: Vec<PageInfo>,
state: i32,
},
@@ -135,24 +184,44 @@ pub enum VideoInfo {
#[serde(rename = "created", with = "ts_seconds")]
ctime: DateTime<Utc>,
},
// 从动态获取的视频信息(此处 pubtime 未在结构中,因此使用 default + 手动赋值)
Dynamic {
title: String,
bvid: String,
desc: String,
cover: String,
#[serde(default)]
pubtime: DateTime<Utc>,
},
}
#[cfg(test)]
mod tests {
use std::path::Path;
use anyhow::Context;
use futures::StreamExt;
use reqwest::Method;
use super::*;
use crate::bilibili::credential::WbiImg;
use crate::config::VersionedConfig;
use crate::database::setup_database;
use crate::utils::init_logger;
#[ignore = "only for manual test"]
#[tokio::test]
async fn test_video_info_type() {
async fn test_video_info_type() -> Result<()> {
VersionedConfig::init_for_test(&setup_database(Path::new("./test.sqlite")).await?).await?;
let credential = &VersionedConfig::get().read().credential;
init_logger("None,bili_sync=debug", None);
let bili_client = BiliClient::new();
// 请求 UP 主视频必须要获取 mixin key使用 key 计算请求参数的签名,否则直接提示权限不足返回空
let Ok(Some(mixin_key)) = bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) else {
panic!("获取 mixin key 失败");
};
let mixin_key = bili_client
.wbi_img(credential)
.await?
.into_mixin_key()
.context("no mixin key")?;
set_global_mixin_key(mixin_key);
let collection = Collection::new(
&bili_client,
@@ -161,6 +230,7 @@ mod tests {
sid: "4523".to_string(),
collection_type: CollectionType::Season,
},
&credential,
);
let videos = collection
.into_video_stream()
@@ -171,7 +241,7 @@ mod tests {
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Collection { .. })));
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
// 测试收藏夹
let favorite = FavoriteList::new(&bili_client, "3144336058".to_string());
let favorite = FavoriteList::new(&bili_client, "3144336058".to_string(), &credential);
let videos = favorite
.into_video_stream()
.take(20)
@@ -181,7 +251,7 @@ mod tests {
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Favorite { .. })));
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
// 测试稍后再看
let watch_later = WatchLater::new(&bili_client);
let watch_later = WatchLater::new(&bili_client, &credential);
let videos = watch_later
.into_video_stream()
.take(20)
@@ -191,7 +261,7 @@ mod tests {
assert!(videos.iter().all(|v| matches!(v, VideoInfo::WatchLater { .. })));
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
// 测试投稿
let submission = Submission::new(&bili_client, "956761".to_string());
let submission = Submission::new(&bili_client, "956761".to_string(), &credential);
let videos = submission
.into_video_stream()
.take(20)
@@ -200,17 +270,32 @@ mod tests {
.await;
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Submission { .. })));
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
// 测试动态
let dynamic = Dynamic::new(&bili_client, "659898".to_string(), &credential);
let videos = dynamic
.into_video_stream()
.take(20)
.filter_map(|v| futures::future::ready(v.ok()))
.collect::<Vec<_>>()
.await;
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Dynamic { .. })));
assert!(videos.iter().skip(1).rev().is_sorted_by_key(|v| v.release_datetime()));
Ok(())
}
#[ignore = "only for manual test"]
#[tokio::test]
async fn test_subtitle_parse() -> Result<()> {
VersionedConfig::init_for_test(&setup_database(Path::new("./test.sqlite")).await?).await?;
let credential = &VersionedConfig::get().read().credential;
let bili_client = BiliClient::new();
let Ok(Some(mixin_key)) = bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) else {
panic!("获取 mixin key 失败");
};
let mixin_key = bili_client
.wbi_img(credential)
.await?
.into_mixin_key()
.context("no mixin key")?;
set_global_mixin_key(mixin_key);
let video = Video::new(&bili_client, "BV1gLfnY8E6D".to_string());
let video = Video::new(&bili_client, "BV1gLfnY8E6D".to_string(), &credential);
let pages = video.get_pages().await?;
println!("pages: {:?}", pages);
let subtitles = video.get_subtitles(&pages[0]).await?;
@@ -223,4 +308,116 @@ mod tests {
}
Ok(())
}
#[ignore = "only for manual test"]
#[tokio::test]
async fn test_upower_parse() -> Result<()> {
VersionedConfig::init_for_test(&setup_database(Path::new("./test.sqlite")).await?).await?;
let credential = &VersionedConfig::get().read().credential;
let bili_client = BiliClient::new();
let mixin_key = bili_client
.wbi_img(credential)
.await?
.into_mixin_key()
.context("no mixin key")?;
set_global_mixin_key(mixin_key);
for (bvid, (upower_exclusive, upower_play)) in [
("BV1HxXwYEEqt", (true, false)), // 充电专享且无权观看
("BV16w41187fx", (true, true)), // 充电专享但有权观看
("BV1n34jzPEYq", (false, false)), // 普通视频
] {
let video = Video::new(&bili_client, bvid.to_string(), credential);
let info = video.get_view_info().await?;
let VideoInfo::Detail {
is_upower_exclusive,
is_upower_play,
..
} = info
else {
unreachable!();
};
assert_eq!(is_upower_exclusive, upower_exclusive, "bvid: {}", bvid);
assert_eq!(is_upower_play, upower_play, "bvid: {}", bvid);
}
Ok(())
}
#[ignore = "only for manual test"]
#[tokio::test]
async fn test_ep_parse() -> Result<()> {
VersionedConfig::init_for_test(&setup_database(Path::new("./test.sqlite")).await?).await?;
let credential = &VersionedConfig::get().read().credential;
let bili_client = BiliClient::new();
let mixin_key = bili_client
.wbi_img(credential)
.await?
.into_mixin_key()
.context("no mixin key")?;
set_global_mixin_key(mixin_key);
for (bvid, redirect_is_none) in [
("BV1SF411g796", false), // EP
("BV13xtnzPEye", false), // 番剧
("BV1kT4NzTEZj", true), // 普通视频
] {
let video = Video::new(&bili_client, bvid.to_string(), credential);
let info = video.get_view_info().await?;
let VideoInfo::Detail { redirect_url, .. } = info else {
unreachable!();
};
assert_eq!(redirect_url.is_none(), redirect_is_none, "bvid: {}", bvid);
}
Ok(())
}
#[test]
fn test_wbi_key() -> Result<()> {
let key = WbiImg {
img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(),
sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(),
};
let key = key.into_mixin_key().context("no mixin key")?;
assert_eq!(key.as_str(), "ea1db124af3c7062474693fa704f4ff8");
let client = Client::new();
let mut req = client
.request(Method::GET, "https://www.baidu.com/", None)
.query(&[("foo", "114"), ("bar", "514")])
.query(&[("zab", "1919810")])
.build()?;
sign_request(&mut req, key.as_str(), 1702204169).unwrap();
let query: Vec<_> = req.url().query_pairs().collect();
assert_eq!(
query,
vec![
("foo".into(), "114".into()),
("bar".into(), "514".into()),
("zab".into(), "1919810".into()),
("w_rid".into(), "8f6f2b5b3d485fe1886cec6a0be8c5d4".into()),
("wts".into(), "1702204169".into()),
]
);
let key = WbiImg {
img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(),
sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(),
};
let key = key.into_mixin_key().context("no mixin key")?;
let mut req = client
.request(Method::GET, "https://www.baidu.com/", None)
.query(&[("mid", "11997177"), ("token", "")])
.query(&[("platform", "web"), ("web_location", "1550101")])
.build()?;
sign_request(&mut req, key.as_str(), 1703513649).unwrap();
let query: Vec<_> = req.url().query_pairs().collect();
assert_eq!(
query,
vec![
("mid".into(), "11997177".into()),
("token".into(), "".into()),
("platform".into(), "web".into()),
("web_location".into(), "1550101".into()),
("w_rid".into(), "7d4428b3f2f9ee2811e116ec6fd41a4f".into()),
("wts".into(), "1703513649".into()),
]
);
Ok(())
}
}

View File

@@ -4,23 +4,37 @@ use futures::Stream;
use reqwest::Method;
use serde_json::Value;
use crate::bilibili::credential::encoded_query;
use crate::bilibili::favorite_list::Upper;
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
use crate::bilibili::{BiliClient, Credential, Dynamic, MIXIN_KEY, Validate, VideoInfo, WbiSign};
pub struct Submission<'a> {
client: &'a BiliClient,
pub upper_id: String,
credential: &'a Credential,
}
impl<'a> From<Submission<'a>> for Dynamic<'a> {
fn from(submission: Submission<'a>) -> Self {
Dynamic::new(submission.client, submission.upper_id, submission.credential)
}
}
impl<'a> Submission<'a> {
pub fn new(client: &'a BiliClient, upper_id: String) -> Self {
Self { client, upper_id }
pub fn new(client: &'a BiliClient, upper_id: String, credential: &'a Credential) -> Self {
Self {
client,
upper_id,
credential,
}
}
pub async fn get_info(&self) -> Result<Upper<String>> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/card")
.request(
Method::GET,
"https://api.bilibili.com/x/web-interface/card",
self.credential,
)
.await
.query(&[("mid", self.upper_id.as_str())])
.send()
@@ -34,20 +48,22 @@ impl<'a> Submission<'a> {
async fn get_videos(&self, page: i32) -> Result<Value> {
self.client
.request(Method::GET, "https://api.bilibili.com/x/space/wbi/arc/search")
.request(
Method::GET,
"https://api.bilibili.com/x/space/wbi/arc/search",
self.credential,
)
.await
.query(&encoded_query(
vec![
("mid", self.upper_id.as_str()),
("order", "pubdate"),
("order_avoided", "true"),
("platform", "web"),
("web_location", "1550101"),
("pn", page.to_string().as_str()),
("ps", "30"),
],
MIXIN_KEY.load().as_deref(),
))
.query(&[
("mid", self.upper_id.as_str()),
("order", "pubdate"),
("order_avoided", "true"),
("platform", "web"),
("web_location", "1550101"),
("ps", "30"),
])
.query(&[("pn", page)])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?

View File

@@ -29,7 +29,7 @@ pub struct SubTitleItem {
impl SubTitleInfo {
pub fn is_ai_sub(&self) -> bool {
// ai aisubtitle.hdslb.com/bfs/ai_subtitle/xxxx
// 非 ai aisubtitle.hdslb.com/bfs/subtitle/xxxx
// 非 aiaisubtitle.hdslb.com/bfs/subtitle/xxxx
self.subtitle_url.contains("ai_subtitle")
}
}

View File

@@ -1,4 +1,4 @@
use anyhow::{Result, ensure};
use anyhow::{Context, Result, ensure};
use futures::TryStreamExt;
use futures::stream::FuturesUnordered;
use prost::Message;
@@ -6,29 +6,16 @@ use reqwest::Method;
use crate::bilibili::analyzer::PageAnalyzer;
use crate::bilibili::client::BiliClient;
use crate::bilibili::credential::encoded_query;
use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply};
use crate::bilibili::subtitle::{SubTitle, SubTitleBody, SubTitleInfo, SubTitlesInfo};
use crate::bilibili::{MIXIN_KEY, Validate, VideoInfo};
use crate::bilibili::{Credential, MIXIN_KEY, Validate, VideoInfo, WbiSign};
pub struct Video<'a> {
client: &'a BiliClient,
pub bvid: String,
credential: &'a Credential,
}
#[derive(Debug, serde::Deserialize)]
pub struct Tag {
pub tag_name: String,
}
impl serde::Serialize for Tag {
fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.tag_name)
}
}
#[derive(Debug, serde::Deserialize, Default)]
pub struct PageInfo {
pub cid: i64,
@@ -48,17 +35,26 @@ pub struct Dimension {
}
impl<'a> Video<'a> {
pub fn new(client: &'a BiliClient, bvid: String) -> Self {
Self { client, bvid }
pub fn new(client: &'a BiliClient, bvid: String, credential: &'a Credential) -> Self {
Self {
client,
bvid,
credential,
}
}
/// 直接调用视频信息接口获取详细的视频信息,视频信息中包含了视频的分页信息
pub async fn get_view_info(&self) -> Result<VideoInfo> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view")
.request(
Method::GET,
"https://api.bilibili.com/x/web-interface/wbi/view",
self.credential,
)
.await
.query(&[("bvid", &self.bvid)])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?
@@ -68,11 +64,15 @@ impl<'a> Video<'a> {
Ok(serde_json::from_value(res["data"].take())?)
}
#[allow(dead_code)]
#[cfg(test)]
pub async fn get_pages(&self) -> Result<Vec<PageInfo>> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/player/pagelist")
.request(
Method::GET,
"https://api.bilibili.com/x/player/pagelist",
self.credential,
)
.await
.query(&[("bvid", &self.bvid)])
.send()
@@ -84,10 +84,14 @@ impl<'a> Video<'a> {
Ok(serde_json::from_value(res["data"].take())?)
}
pub async fn get_tags(&self) -> Result<Vec<Tag>> {
let mut res = self
pub async fn get_tags(&self) -> Result<Vec<String>> {
let res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view/detail/tag")
.request(
Method::GET,
"https://api.bilibili.com/x/web-interface/view/detail/tag",
self.credential,
)
.await
.query(&[("bvid", &self.bvid)])
.send()
@@ -96,10 +100,15 @@ impl<'a> Video<'a> {
.json::<serde_json::Value>()
.await?
.validate()?;
Ok(serde_json::from_value(res["data"].take())?)
Ok(res["data"]
.as_array()
.context("tags is not an array")?
.iter()
.filter_map(|v| v["tag_name"].as_str().map(String::from))
.collect())
}
pub async fn get_danmaku_writer(&self, page: &'a PageInfo) -> Result<DanmakuWriter> {
pub async fn get_danmaku_writer(&self, page: &'a PageInfo) -> Result<DanmakuWriter<'a>> {
let tasks = FuturesUnordered::new();
for i in 1..=page.duration.div_ceil(360) {
tasks.push(self.get_danmaku_segment(page, i as i64));
@@ -113,9 +122,14 @@ impl<'a> Video<'a> {
async fn get_danmaku_segment(&self, page: &PageInfo, segment_idx: i64) -> Result<Vec<DanmakuElem>> {
let mut res = self
.client
.request(Method::GET, "http://api.bilibili.com/x/v2/dm/web/seg.so")
.request(
Method::GET,
"https://api.bilibili.com/x/v2/dm/wbi/web/seg.so",
self.credential,
)
.await
.query(&[("type", 1), ("oid", page.cid), ("segment_index", segment_idx)])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?;
@@ -133,19 +147,21 @@ impl<'a> Video<'a> {
pub async fn get_page_analyzer(&self, page: &PageInfo) -> Result<PageAnalyzer> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/playurl")
.request(
Method::GET,
"https://api.bilibili.com/x/player/wbi/playurl",
self.credential,
)
.await
.query(&encoded_query(
vec![
("bvid", self.bvid.as_str()),
("cid", page.cid.to_string().as_str()),
("qn", "127"),
("otype", "json"),
("fnval", "4048"),
("fourk", "1"),
],
MIXIN_KEY.load().as_deref(),
))
.query(&[
("bvid", self.bvid.as_str()),
("qn", "127"),
("otype", "json"),
("fnval", "4048"),
("fourk", "1"),
])
.query(&[("cid", page.cid)])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?
@@ -158,12 +174,11 @@ impl<'a> Video<'a> {
pub async fn get_subtitles(&self, page: &PageInfo) -> Result<Vec<SubTitle>> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/v2")
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/v2", self.credential)
.await
.query(&encoded_query(
vec![("cid", &page.cid.to_string()), ("bvid", &self.bvid)],
MIXIN_KEY.load().as_deref(),
))
.query(&[("bvid", self.bvid.as_str())])
.query(&[("cid", page.cid)])
.wbi_sign(MIXIN_KEY.load().as_deref())?
.send()
.await?
.error_for_status()?
@@ -171,14 +186,18 @@ impl<'a> Video<'a> {
.await?
.validate()?;
// 接口返回的信息,包含了一系列的字幕,每个字幕包含了字幕的语言和 json 下载地址
let subtitles_info: SubTitlesInfo = serde_json::from_value(res["data"]["subtitle"].take())?;
let tasks = subtitles_info
.subtitles
.into_iter()
.filter(|v| !v.is_ai_sub())
.map(|v| self.get_subtitle(v))
.collect::<FuturesUnordered<_>>();
tasks.try_collect().await
match serde_json::from_value::<Option<SubTitlesInfo>>(res["data"]["subtitle"].take())? {
Some(subtitles_info) => {
let tasks = subtitles_info
.subtitles
.into_iter()
.filter(|v| !v.is_ai_sub())
.map(|v| self.get_subtitle(v))
.collect::<FuturesUnordered<_>>();
tasks.try_collect().await
}
None => Ok(vec![]),
}
}
async fn get_subtitle(&self, info: SubTitleInfo) -> Result<SubTitle> {

View File

@@ -3,19 +3,24 @@ use async_stream::try_stream;
use futures::Stream;
use serde_json::Value;
use crate::bilibili::{BiliClient, Validate, VideoInfo};
use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
pub struct WatchLater<'a> {
client: &'a BiliClient,
credential: &'a Credential,
}
impl<'a> WatchLater<'a> {
pub fn new(client: &'a BiliClient) -> Self {
Self { client }
pub fn new(client: &'a BiliClient, credential: &'a Credential) -> Self {
Self { client, credential }
}
async fn get_videos(&self) -> Result<Value> {
self.client
.request(reqwest::Method::GET, "https://api.bilibili.com/x/v2/history/toview")
.request(
reqwest::Method::GET,
"https://api.bilibili.com/x/v2/history/toview",
self.credential,
)
.await
.send()
.await?

View File

@@ -1,15 +1,19 @@
use std::path::PathBuf;
use std::sync::LazyLock;
use std::sync::{Arc, LazyLock};
use anyhow::{Result, bail};
use croner::parser::CronParser;
use sea_orm::DatabaseConnection;
use serde::{Deserialize, Serialize};
use validator::Validate;
use crate::bilibili::{Credential, DanmakuOption, FilterOption};
use crate::config::LegacyConfig;
use crate::config::default::{default_auth_token, default_bind_address, default_time_format};
use crate::config::item::{ConcurrentLimit, NFOTimeType};
use crate::config::item::{
ConcurrentLimit, NFOTimeType, SkipOption, Trigger, default_collection_path, default_favorite_path,
default_submission_path,
};
use crate::notifier::Notifier;
use crate::utils::model::{load_db_config, save_db_config};
pub static CONFIG_DIR: LazyLock<PathBuf> =
@@ -22,9 +26,19 @@ pub struct Config {
pub credential: Credential,
pub filter_option: FilterOption,
pub danmaku_option: DanmakuOption,
#[serde(default)]
pub skip_option: SkipOption,
pub video_name: String,
pub page_name: String,
pub interval: u64,
#[serde(default)]
pub notifiers: Option<Arc<Vec<Notifier>>>,
#[serde(default = "default_favorite_path")]
pub favorite_default_path: String,
#[serde(default = "default_collection_path")]
pub collection_default_path: String,
#[serde(default = "default_submission_path")]
pub submission_default_path: String,
pub interval: Trigger,
pub upper_path: PathBuf,
pub nfo_time_type: NFOTimeType,
pub concurrent_limit: ConcurrentLimit,
@@ -65,6 +79,24 @@ impl Config {
if !(self.concurrent_limit.video > 0 && self.concurrent_limit.page > 0) {
errors.push("video 和 page 允许的并发数必须大于 0");
}
match &self.interval {
Trigger::Interval(secs) => {
if *secs <= 60 {
errors.push("下载任务执行间隔时间必须大于 60 秒");
}
}
Trigger::Cron(cron) => {
if CronParser::builder()
.seconds(croner::parser::Seconds::Required)
.dom_and_dow(true)
.build()
.parse(cron)
.is_err()
{
errors.push("Cron 表达式无效,正确格式为“秒 分 时 日 月 周”");
}
}
};
if !errors.is_empty() {
bail!(
errors
@@ -76,14 +108,6 @@ impl Config {
}
Ok(())
}
#[cfg(test)]
pub(super) fn test_default() -> Self {
Self {
cdn_sorting: true,
..Default::default()
}
}
}
impl Default for Config {
@@ -94,9 +118,14 @@ impl Default for Config {
credential: Credential::default(),
filter_option: FilterOption::default(),
danmaku_option: DanmakuOption::default(),
skip_option: SkipOption::default(),
video_name: "{{title}}".to_owned(),
page_name: "{{bvid}}".to_owned(),
interval: 1200,
notifiers: None,
favorite_default_path: default_favorite_path(),
collection_default_path: default_collection_path(),
submission_default_path: default_submission_path(),
interval: Trigger::default(),
upper_path: CONFIG_DIR.join("upper_face"),
nfo_time_type: NFOTimeType::FavTime,
concurrent_limit: ConcurrentLimit::default(),
@@ -106,24 +135,3 @@ impl Default for Config {
}
}
}
impl From<LegacyConfig> for Config {
fn from(legacy: LegacyConfig) -> Self {
Self {
auth_token: legacy.auth_token,
bind_address: legacy.bind_address,
credential: legacy.credential,
filter_option: legacy.filter_option,
danmaku_option: legacy.danmaku_option,
video_name: legacy.video_name,
page_name: legacy.page_name,
interval: legacy.interval,
upper_path: legacy.upper_path,
nfo_time_type: legacy.nfo_time_type,
concurrent_limit: legacy.concurrent_limit,
time_format: legacy.time_format,
cdn_sorting: legacy.cdn_sorting,
version: 0,
}
}
}

View File

@@ -12,8 +12,11 @@ pub static TEMPLATE: LazyLock<VersionedCache<handlebars::Handlebars<'static>>> =
fn create_template(config: &Config) -> Result<handlebars::Handlebars<'static>> {
let mut handlebars = handlebars::Handlebars::new();
handlebars.register_helper("truncate", Box::new(truncate));
handlebars.path_safe_register("video", config.video_name.to_owned())?;
handlebars.path_safe_register("page", config.page_name.to_owned())?;
handlebars.path_safe_register("video", config.video_name.clone())?;
handlebars.path_safe_register("page", config.page_name.clone())?;
handlebars.path_safe_register("favorite_default_path", config.favorite_default_path.clone())?;
handlebars.path_safe_register("collection_default_path", config.collection_default_path.clone())?;
handlebars.path_safe_register("submission_default_path", config.submission_default_path.clone())?;
Ok(handlebars)
}
@@ -81,7 +84,7 @@ mod tests {
"test_truncate",
&json!({"title": "你说得对,但是 Rust 是由 Mozilla 自主研发的一款全新的编译期格斗游戏。\
编译将发生在一个被称作「Cargo」的构建系统中。在这里被引用的指针将被授予「生命周期」之力导引对象安全。\
你将扮演一位名为「Rustacean」的神秘角色, 在与「Rustc」的搏斗中邂逅各种骨骼惊奇的傲娇报错。\
你将扮演一位名为「Rustacean」的神秘角色在与「Rustc」的搏斗中邂逅各种骨骼惊奇的傲娇报错。\
征服她们、通过编译同时逐步发掘「C++」程序崩溃的真相。"})
)
.unwrap(),

View File

@@ -1,19 +1,10 @@
use std::path::PathBuf;
use anyhow::Result;
use serde::{Deserialize, Serialize};
use crate::utils::filenamify::filenamify;
/// 稍后再看的配置
#[derive(Serialize, Deserialize, Default)]
pub struct WatchLaterConfig {
pub enabled: bool,
pub path: PathBuf,
}
/// NFO 文件使用的时间类型
#[derive(Serialize, Deserialize, Default, Clone)]
#[derive(Serialize, Deserialize, Default, Clone, Copy)]
#[serde(rename_all = "lowercase")]
pub enum NFOTimeType {
#[default]
@@ -69,6 +60,28 @@ impl Default for ConcurrentLimit {
}
}
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct SkipOption {
pub no_poster: bool,
pub no_video_nfo: bool,
pub no_upper: bool,
pub no_danmaku: bool,
pub no_subtitle: bool,
}
#[derive(Serialize, Deserialize, Clone)]
#[serde(untagged)]
pub enum Trigger {
Interval(u64),
Cron(String),
}
impl Default for Trigger {
fn default() -> Self {
Trigger::Interval(1200)
}
}
pub trait PathSafeTemplate {
fn path_safe_register(&mut self, name: &'static str, template: impl Into<String>) -> Result<()>;
fn path_safe_render(&self, name: &'static str, data: &serde_json::Value) -> Result<String>;
@@ -85,3 +98,15 @@ impl PathSafeTemplate for handlebars::Handlebars<'_> {
Ok(filenamify(&self.render(name, data)?).replace("__SEP__", std::path::MAIN_SEPARATOR_STR))
}
}
pub fn default_favorite_path() -> String {
"收藏夹/{{name}}".to_owned()
}
pub fn default_collection_path() -> String {
"合集/{{name}}".to_owned()
}
pub fn default_submission_path() -> String {
"投稿/{{name}}".to_owned()
}

View File

@@ -1,134 +0,0 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use anyhow::Result;
use sea_orm::DatabaseConnection;
use serde::de::{Deserializer, MapAccess, Visitor};
use serde::ser::SerializeMap;
use serde::{Deserialize, Serialize};
use crate::bilibili::{CollectionItem, CollectionType, Credential, DanmakuOption, FilterOption};
use crate::config::Config;
use crate::config::default::{default_auth_token, default_bind_address, default_time_format};
use crate::config::item::{ConcurrentLimit, NFOTimeType, WatchLaterConfig};
use crate::utils::model::migrate_legacy_config;
#[derive(Serialize, Deserialize)]
pub struct LegacyConfig {
#[serde(default = "default_auth_token")]
pub auth_token: String,
#[serde(default = "default_bind_address")]
pub bind_address: String,
pub credential: Credential,
pub filter_option: FilterOption,
#[serde(default)]
pub danmaku_option: DanmakuOption,
pub favorite_list: HashMap<String, PathBuf>,
#[serde(
default,
serialize_with = "serialize_collection_list",
deserialize_with = "deserialize_collection_list"
)]
pub collection_list: HashMap<CollectionItem, PathBuf>,
#[serde(default)]
pub submission_list: HashMap<String, PathBuf>,
#[serde(default)]
pub watch_later: WatchLaterConfig,
pub video_name: String,
pub page_name: String,
pub interval: u64,
pub upper_path: PathBuf,
#[serde(default)]
pub nfo_time_type: NFOTimeType,
#[serde(default)]
pub concurrent_limit: ConcurrentLimit,
#[serde(default = "default_time_format")]
pub time_format: String,
#[serde(default)]
pub cdn_sorting: bool,
}
impl LegacyConfig {
async fn load_from_file(path: &Path) -> Result<Self> {
let legacy_config_str = tokio::fs::read_to_string(path).await?;
Ok(toml::from_str(&legacy_config_str)?)
}
pub async fn migrate_from_file(path: &Path, connection: &DatabaseConnection) -> Result<Config> {
let legacy_config = Self::load_from_file(path).await?;
migrate_legacy_config(&legacy_config, connection).await?;
Ok(legacy_config.into())
}
}
/*
后面是用于自定义 Collection 的序列化、反序列化的样板代码
*/
pub(super) fn serialize_collection_list<S>(
collection_list: &HashMap<CollectionItem, PathBuf>,
serializer: S,
) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut map = serializer.serialize_map(Some(collection_list.len()))?;
for (k, v) in collection_list {
let prefix = match k.collection_type {
CollectionType::Series => "series",
CollectionType::Season => "season",
};
map.serialize_entry(&[prefix, &k.mid, &k.sid].join(":"), v)?;
}
map.end()
}
pub(super) fn deserialize_collection_list<'de, D>(deserializer: D) -> Result<HashMap<CollectionItem, PathBuf>, D::Error>
where
D: Deserializer<'de>,
{
struct CollectionListVisitor;
impl<'de> Visitor<'de> for CollectionListVisitor {
type Value = HashMap<CollectionItem, PathBuf>;
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
formatter.write_str("a map of collection list")
}
fn visit_map<A>(self, mut map: A) -> Result<Self::Value, A::Error>
where
A: MapAccess<'de>,
{
let mut collection_list = HashMap::new();
while let Some((key, value)) = map.next_entry::<String, PathBuf>()? {
let collection_item = match key.split(':').collect::<Vec<&str>>().as_slice() {
[prefix, mid, sid] => {
let collection_type = match *prefix {
"series" => CollectionType::Series,
"season" => CollectionType::Season,
_ => {
return Err(serde::de::Error::custom(
"invalid collection type, should be series or season",
));
}
};
CollectionItem {
mid: mid.to_string(),
sid: sid.to_string(),
collection_type,
}
}
_ => {
return Err(serde::de::Error::custom(
"invalid collection key, should be series:mid:sid or season:mid:sid",
));
}
};
collection_list.insert(collection_item, value);
}
Ok(collection_list)
}
}
deserializer.deserialize_map(CollectionListVisitor)
}

View File

@@ -3,14 +3,12 @@ mod current;
mod default;
mod handlebar;
mod item;
mod legacy;
mod versioned_cache;
mod versioned_config;
pub use crate::config::args::{ARGS, version};
pub use crate::config::current::{CONFIG_DIR, Config};
pub use crate::config::handlebar::TEMPLATE;
pub use crate::config::item::{NFOTimeType, PathSafeTemplate, RateLimit};
pub use crate::config::legacy::LegacyConfig;
pub use crate::config::item::{ConcurrentDownloadLimit, NFOTimeType, PathSafeTemplate, RateLimit, Trigger};
pub use crate::config::versioned_cache::VersionedCache;
pub use crate::config::versioned_config::VersionedConfig;

View File

@@ -1,54 +1,56 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use anyhow::Result;
use arc_swap::{ArcSwap, Guard};
use tokio_util::future::FutureExt;
use tokio_util::sync::CancellationToken;
use crate::config::{Config, VersionedConfig};
pub struct VersionedCache<T> {
inner: ArcSwap<T>,
version: AtomicU64,
builder: fn(&Config) -> Result<T>,
mutex: parking_lot::Mutex<()>,
inner: Arc<ArcSwap<T>>,
cancel_token: CancellationToken,
}
impl<T> VersionedCache<T> {
/// 一个跟随全局配置变化自动更新的缓存
impl<T: Send + Sync + 'static> VersionedCache<T> {
pub fn new(builder: fn(&Config) -> Result<T>) -> Result<Self> {
let current_config = VersionedConfig::get().load();
let current_version = current_config.version;
let initial_value = builder(&current_config)?;
Ok(Self {
inner: ArcSwap::from_pointee(initial_value),
version: AtomicU64::new(current_version),
builder,
mutex: parking_lot::Mutex::new(()),
})
let mut rx = VersionedConfig::get().subscribe();
let initial_value = builder(&rx.borrow_and_update())?;
let cancel_token = CancellationToken::new();
let inner = Arc::new(ArcSwap::from_pointee(initial_value));
let inner_clone = inner.clone();
tokio::spawn(
async move {
while rx.changed().await.is_ok() {
match builder(&rx.borrow()) {
Ok(new_value) => {
inner_clone.store(Arc::new(new_value));
}
Err(e) => {
error!("Failed to update versioned cache: {:?}", e);
}
}
}
}
.with_cancellation_token_owned(cancel_token.clone()),
);
Ok(Self { inner, cancel_token })
}
pub fn load(&self) -> Guard<Arc<T>> {
self.reload_if_needed();
/// 获取一个临时的只读引用
pub fn read(&self) -> Guard<Arc<T>> {
self.inner.load()
}
fn reload_if_needed(&self) {
let current_config = VersionedConfig::get().load();
let current_version = current_config.version;
let version = self.version.load(Ordering::Relaxed);
if version < current_version {
let _lock = self.mutex.lock();
if self.version.load(Ordering::Relaxed) >= current_version {
return;
}
match (self.builder)(&current_config) {
Err(e) => {
error!("Failed to rebuild versioned cache: {:?}", e);
}
Ok(new_value) => {
self.inner.store(Arc::new(new_value));
self.version.store(current_version, Ordering::Relaxed);
}
}
}
/// 获取当前缓存的完整快照
pub fn snapshot(&self) -> Arc<T> {
self.inner.load_full()
}
}
impl<T> Drop for VersionedCache<T> {
fn drop(&mut self) {
self.cancel_token.cancel();
}
}

View File

@@ -1,63 +1,66 @@
use std::sync::Arc;
use anyhow::{Result, anyhow, bail};
use anyhow::{Result, bail};
use arc_swap::{ArcSwap, Guard};
use sea_orm::DatabaseConnection;
use tokio::sync::OnceCell;
use tokio::sync::{OnceCell, watch};
use crate::bilibili::Credential;
use crate::config::{CONFIG_DIR, Config, LegacyConfig};
use crate::config::{CONFIG_DIR, Config};
pub static VERSIONED_CONFIG: OnceCell<VersionedConfig> = OnceCell::const_new();
static VERSIONED_CONFIG: OnceCell<VersionedConfig> = OnceCell::const_new();
pub struct VersionedConfig {
inner: ArcSwap<Config>,
update_lock: tokio::sync::Mutex<()>,
tx: watch::Sender<Arc<Config>>,
rx: watch::Receiver<Arc<Config>>,
}
impl VersionedConfig {
/// 初始化全局的 `VersionedConfig`,初始化失败或者已初始化过则返回错误
pub async fn init(connection: &DatabaseConnection) -> Result<()> {
let mut config = match Config::load_from_database(connection).await? {
Some(Ok(config)) => config,
Some(Err(e)) => bail!("解析数据库配置失败: {}", e),
None => {
let config = match LegacyConfig::migrate_from_file(&CONFIG_DIR.join("config.toml"), connection).await {
Ok(config) => config,
Err(e) => {
if e.downcast_ref::<std::io::Error>()
.is_none_or(|e| e.kind() != std::io::ErrorKind::NotFound)
{
bail!("未成功读取并迁移旧版本配置:{:#}", e);
} else {
let config = Config::default();
warn!(
"生成 auth_token{},可使用该 token 登录 web UI该信息仅在首次运行时打印",
config.auth_token
pub async fn init(connection: &DatabaseConnection) -> Result<&'static VersionedConfig> {
VERSIONED_CONFIG
.get_or_try_init(|| async move {
let mut config = match Config::load_from_database(connection).await? {
Some(Ok(config)) => config,
Some(Err(e)) => bail!("解析数据库配置失败: {}", e),
None => {
if CONFIG_DIR.join("config.toml").exists() {
// 数据库中没有配置,但旧版配置文件存在,说明是从 2.6.0 之前的版本直接升级的
bail!(
"当前版本已移除配置文件的迁移逻辑,不再支持从配置文件加载配置。\n\
如果你正在运行 2.6.0 之前的版本,请先升级至 2.6.x 或 2.7.x\n\
启动时会自动将配置文件迁移至数据库,然后再升级至最新版本。"
);
config
}
let config = Config::default();
warn!(
"生成 auth_token{},可使用该 token 登录 web UI该信息仅在首次运行时打印",
config.auth_token
);
config.save_to_database(connection).await?;
config
}
};
config.save_to_database(connection).await?;
config
}
};
// version 本身不具有实际意义,仅用于并发更新时的版本控制,在初始化时可以直接清空
config.version = 0;
let versioned_config = VersionedConfig::new(config);
VERSIONED_CONFIG
.set(versioned_config)
.map_err(|e| anyhow!("VERSIONED_CONFIG has already been initialized: {}", e))?;
Ok(())
// version 本身不具有实际意义,仅用于并发更新时的版本控制,在初始化时可以直接清空
config.version = 0;
Ok(VersionedConfig::new(config))
})
.await
}
#[cfg(test)]
/// 单元测试直接使用测试专用的配置即可
pub fn get() -> &'static VersionedConfig {
use std::sync::LazyLock;
static TEST_CONFIG: LazyLock<VersionedConfig> = LazyLock::new(|| VersionedConfig::new(Config::test_default()));
return &TEST_CONFIG;
/// 仅在测试环境使用,该方法会尝试从测试数据库中加载配置并写入到全局的 VERSIONED_CONFIG
pub async fn init_for_test(connection: &DatabaseConnection) -> Result<&'static VersionedConfig> {
VERSIONED_CONFIG
.get_or_try_init(|| async move {
let Some(Ok(config)) = Config::load_from_database(&connection).await? else {
bail!("no config found in test database");
};
Ok(VersionedConfig::new(config))
})
.await
}
#[cfg(not(test))]
@@ -66,37 +69,52 @@ impl VersionedConfig {
VERSIONED_CONFIG.get().expect("VERSIONED_CONFIG is not initialized")
}
pub fn new(config: Config) -> Self {
#[cfg(test)]
/// 尝试获取全局的 `VersionedConfig`,如果未初始化则退回默认配置
pub fn get() -> &'static VersionedConfig {
use std::sync::LazyLock;
static FALLBACK_CONFIG: LazyLock<VersionedConfig> = LazyLock::new(|| VersionedConfig::new(Config::default()));
// 优先从全局变量获取,未初始化则退回默认配置
return VERSIONED_CONFIG.get().unwrap_or_else(|| &FALLBACK_CONFIG);
}
fn new(config: Config) -> Self {
let inner = ArcSwap::from_pointee(config);
let (tx, rx) = watch::channel(inner.load_full());
Self {
inner: ArcSwap::from_pointee(config),
inner,
update_lock: tokio::sync::Mutex::new(()),
tx,
rx,
}
}
pub fn load(&self) -> Guard<Arc<Config>> {
pub fn read(&self) -> Guard<Arc<Config>> {
self.inner.load()
}
pub fn load_full(&self) -> Arc<Config> {
pub fn snapshot(&self) -> Arc<Config> {
self.inner.load_full()
}
pub async fn update_credential(&self, new_credential: Credential, connection: &DatabaseConnection) -> Result<()> {
// 确保更新内容与写入数据库的操作是原子性的
pub fn subscribe(&self) -> watch::Receiver<Arc<Config>> {
self.rx.clone()
}
pub async fn update_credential(
&self,
new_credential: Credential,
connection: &DatabaseConnection,
) -> Result<Arc<Config>> {
let _lock = self.update_lock.lock().await;
loop {
let old_config = self.inner.load();
let mut new_config = old_config.as_ref().clone();
new_config.credential = new_credential.clone();
new_config.version += 1;
if Arc::ptr_eq(
&old_config,
&self.inner.compare_and_swap(&old_config, Arc::new(new_config)),
) {
break;
}
}
self.inner.load().save_to_database(connection).await
let mut new_config = self.inner.load().as_ref().clone();
new_config.credential = new_credential;
new_config.version += 1;
new_config.save_to_database(connection).await?;
let new_config = Arc::new(new_config);
self.inner.store(new_config.clone());
self.tx.send(new_config.clone())?;
Ok(new_config)
}
/// 外部 API 会调用这个方法,如果更新失败直接返回错误
@@ -107,14 +125,10 @@ impl VersionedConfig {
bail!("配置版本不匹配,请刷新页面修改后重新提交");
}
new_config.version += 1;
let new_config = Arc::new(new_config);
if !Arc::ptr_eq(
&old_config,
&self.inner.compare_and_swap(&old_config, new_config.clone()),
) {
bail!("配置版本不匹配,请刷新页面修改后重新提交");
}
new_config.save_to_database(connection).await?;
let new_config = Arc::new(new_config);
self.inner.store(new_config.clone());
self.tx.send(new_config.clone())?;
Ok(new_config)
}
}

View File

@@ -1,34 +1,58 @@
use std::path::Path;
use std::time::Duration;
use anyhow::{Context, Result};
use bili_sync_migration::{Migrator, MigratorTrait};
use sea_orm::{ConnectOptions, Database, DatabaseConnection};
use sea_orm::sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqliteSynchronous};
use sea_orm::sqlx::{ConnectOptions as SqlxConnectOptions, Sqlite};
use sea_orm::{ConnectOptions, Database, DatabaseConnection, SqlxSqliteConnector};
use crate::config::CONFIG_DIR;
fn database_url() -> String {
format!("sqlite://{}?mode=rwc", CONFIG_DIR.join("data.sqlite").to_string_lossy())
fn database_url(path: &Path) -> String {
format!("sqlite://{}?mode=rwc", path.to_string_lossy())
}
async fn database_connection() -> Result<DatabaseConnection> {
let mut option = ConnectOptions::new(database_url());
async fn database_connection(database_url: &str) -> Result<DatabaseConnection> {
let mut option = ConnectOptions::new(database_url);
option
.max_connections(100)
.max_connections(50)
.min_connections(5)
.acquire_timeout(std::time::Duration::from_secs(90));
Ok(Database::connect(option).await?)
.acquire_timeout(Duration::from_secs(90));
let connect_option = option
.get_url()
.parse::<SqliteConnectOptions>()
.context("Failed to parse database URL")?
.disable_statement_logging()
.busy_timeout(Duration::from_secs(90))
.journal_mode(SqliteJournalMode::Wal)
.synchronous(SqliteSynchronous::Normal)
.optimize_on_close(true, None);
Ok(SqlxSqliteConnector::from_sqlx_sqlite_pool(
option
.sqlx_pool_options::<Sqlite>()
.connect_with(connect_option)
.await?,
))
}
async fn migrate_database() -> Result<()> {
async fn migrate_database(database_url: &str) -> Result<()> {
// 注意此处使用内部构造的 DatabaseConnection而不是通过 database_connection() 获取
// 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题,而使用默认的连接选项不会
let connection = Database::connect(database_url()).await?;
let connection = Database::connect(database_url).await?;
Ok(Migrator::up(&connection, None).await?)
}
/// 进行数据库迁移并获取数据库连接,供外部使用
pub async fn setup_database() -> Result<DatabaseConnection> {
tokio::fs::create_dir_all(CONFIG_DIR.as_path())
pub async fn setup_database(path: &Path) -> Result<DatabaseConnection> {
if let Some(parent) = path.parent() {
tokio::fs::create_dir_all(parent).await.context(
"Failed to create config directory. Please check if you have granted necessary permissions to your folder.",
)?;
}
let database_url = database_url(path);
migrate_database(&database_url)
.await
.context("Failed to create config directory")?;
migrate_database().await.context("Failed to migrate database")?;
database_connection().await.context("Failed to connect to database")
.context("Failed to migrate database")?;
database_connection(&database_url)
.await
.context("Failed to connect to database")
}

View File

@@ -4,15 +4,18 @@ use std::path::Path;
use std::sync::Arc;
use anyhow::{Context, Result, bail, ensure};
use async_tempfile::TempFile;
use futures::TryStreamExt;
use reqwest::{Method, header};
use tokio::fs::{self, File, OpenOptions};
use tokio::fs::{self};
use tokio::io::{AsyncSeekExt, AsyncWriteExt};
use tokio::process::Command;
use tokio::task::JoinSet;
use tokio_util::io::StreamReader;
use crate::bilibili::Client;
use crate::config::VersionedConfig;
use crate::config::ConcurrentDownloadLimit;
pub struct Downloader {
client: Client,
}
@@ -25,15 +28,119 @@ impl Downloader {
Self { client }
}
pub async fn fetch(&self, url: &str, path: &Path) -> Result<()> {
if VersionedConfig::get().load().concurrent_limit.download.enable {
self.fetch_parallel(url, path).await
pub async fn fetch(&self, url: &str, path: &Path, concurrent_download: &ConcurrentDownloadLimit) -> Result<()> {
let mut temp_file = TempFile::new().await?;
self.fetch_internal(url, &mut temp_file, concurrent_download).await?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
fs::copy(temp_file.file_path(), path).await?;
// temp_file 的 drop 需要 std::fs::remove_file
// 如果交由 rust 自动执行虽然逻辑正确但会略微阻塞异步上下文
// 尽量主动调用,保证正常执行的情况下文件清除操作由 spawn_blocking 在专门线程中完成
temp_file.drop_async().await;
Ok(())
}
pub async fn multi_fetch(
&self,
urls: &[&str],
path: &Path,
concurrent_download: &ConcurrentDownloadLimit,
) -> Result<()> {
let temp_file = self.multi_fetch_internal(urls, concurrent_download).await?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
fs::copy(temp_file.file_path(), path).await?;
temp_file.drop_async().await;
Ok(())
}
pub async fn multi_fetch_and_merge(
&self,
video_urls: &[&str],
audio_urls: &[&str],
path: &Path,
concurrent_download: &ConcurrentDownloadLimit,
) -> Result<()> {
let (video_temp_file, audio_temp_file) = tokio::try_join!(
self.multi_fetch_internal(video_urls, concurrent_download),
self.multi_fetch_internal(audio_urls, concurrent_download)
)?;
let final_temp_file = TempFile::new().await?;
let output = Command::new("ffmpeg")
.args([
"-i",
video_temp_file.file_path().to_string_lossy().as_ref(),
"-i",
audio_temp_file.file_path().to_string_lossy().as_ref(),
"-c",
"copy",
"-strict",
"unofficial",
"-f",
"mp4",
"-y",
final_temp_file.file_path().to_string_lossy().as_ref(),
])
.output()
.await
.context("failed to run ffmpeg")?;
if !output.status.success() {
bail!("ffmpeg error: {}", str::from_utf8(&output.stderr).unwrap_or("unknown"));
}
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
fs::copy(final_temp_file.file_path(), path).await?;
tokio::join!(
video_temp_file.drop_async(),
audio_temp_file.drop_async(),
final_temp_file.drop_async()
);
Ok(())
}
async fn multi_fetch_internal(
&self,
urls: &[&str],
concurrent_download: &ConcurrentDownloadLimit,
) -> Result<TempFile> {
if urls.is_empty() {
bail!("no urls provided");
}
let mut temp_file = TempFile::new().await?;
for (idx, url) in urls.iter().enumerate() {
match self.fetch_internal(url, &mut temp_file, concurrent_download).await {
Ok(_) => return Ok(temp_file),
Err(e) => {
if idx == urls.len() - 1 {
temp_file.drop_async().await;
return Err(e).with_context(|| format!("failed to download file from all {} urls", urls.len()));
}
temp_file.set_len(0).await?;
temp_file.rewind().await?;
}
}
}
unreachable!()
}
async fn fetch_internal(
&self,
url: &str,
file: &mut TempFile,
concurrent_download: &ConcurrentDownloadLimit,
) -> Result<()> {
if concurrent_download.enable {
self.fetch_parallel(url, file, concurrent_download).await
} else {
self.fetch_serial(url, path).await
self.fetch_serial(url, file).await
}
}
async fn fetch_serial(&self, url: &str, path: &Path) -> Result<()> {
async fn fetch_serial(&self, url: &str, file: &mut TempFile) -> Result<()> {
let resp = self
.client
.request(Method::GET, url, None)
@@ -41,12 +148,8 @@ impl Downloader {
.await?
.error_for_status()?;
let expected = resp.header_content_length();
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let mut file = File::create(path).await?;
let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
let received = tokio::io::copy(&mut stream_reader, &mut file).await?;
let received = tokio::io::copy(&mut stream_reader, file).await?;
file.flush().await?;
if let Some(expected) = expected {
ensure!(
@@ -59,14 +162,13 @@ impl Downloader {
Ok(())
}
async fn fetch_parallel(&self, url: &str, path: &Path) -> Result<()> {
let (concurrency, threshold) = {
let config = VersionedConfig::get().load();
(
config.concurrent_limit.download.concurrency,
config.concurrent_limit.download.threshold,
)
};
async fn fetch_parallel(
&self,
url: &str,
file: &mut TempFile,
concurrent_download: &ConcurrentDownloadLimit,
) -> Result<()> {
let (concurrency, threshold) = (concurrent_download.concurrency, concurrent_download.threshold);
let resp = self
.client
.request(Method::HEAD, url, None)
@@ -81,17 +183,11 @@ impl Downloader {
.is_none_or(|v| v.to_str().unwrap_or_default() == "none") // https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Accept-Ranges#none
|| chunk_size < threshold
{
return self.fetch_serial(url, path).await;
return self.fetch_serial(url, file).await;
}
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let file = File::create(path).await?;
file.set_len(file_size).await?;
drop(file);
let mut tasks = JoinSet::new();
let url = Arc::new(url.to_string());
let path = Arc::new(path.to_path_buf());
for i in 0..concurrency {
let start = i as u64 * chunk_size;
let end = if i == concurrency - 1 {
@@ -99,10 +195,10 @@ impl Downloader {
} else {
start + chunk_size
} - 1;
let (url_clone, path_clone, client_clone) = (url.clone(), path.clone(), self.client.clone());
let (url_clone, client_clone) = (url.clone(), self.client.clone());
let mut file_clone = file.open_rw().await?;
tasks.spawn(async move {
let mut file = OpenOptions::new().write(true).open(path_clone.as_ref()).await?;
file.seek(SeekFrom::Start(start)).await?;
file_clone.seek(SeekFrom::Start(start)).await?;
let range_header = format!("bytes={}-{}", start, end);
let resp = client_clone
.request(Method::GET, &url_clone, None)
@@ -119,8 +215,8 @@ impl Downloader {
);
}
let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
let received = tokio::io::copy(&mut stream_reader, &mut file).await?;
file.flush().await?;
let received = tokio::io::copy(&mut stream_reader, &mut file_clone).await?;
file_clone.flush().await?;
ensure!(
received == end - start + 1,
"downloaded bytes mismatch: expected {}, got {}",
@@ -135,44 +231,6 @@ impl Downloader {
}
Ok(())
}
pub async fn fetch_with_fallback(&self, urls: &[&str], path: &Path) -> Result<()> {
if urls.is_empty() {
bail!("no urls provided");
}
let mut res = Ok(());
for url in urls {
match self.fetch(url, path).await {
Ok(_) => return Ok(()),
Err(err) => {
res = Err(err);
}
}
}
res.with_context(|| format!("failed to download from {:?}", urls))
}
pub async fn merge(&self, video_path: &Path, audio_path: &Path, output_path: &Path) -> Result<()> {
let output = tokio::process::Command::new("ffmpeg")
.args([
"-i",
video_path.to_string_lossy().as_ref(),
"-i",
audio_path.to_string_lossy().as_ref(),
"-c",
"copy",
"-strict",
"unofficial",
"-y",
output_path.to_string_lossy().as_ref(),
])
.output()
.await?;
if !output.status.success() {
bail!("ffmpeg error: {}", str::from_utf8(&output.stderr).unwrap_or("unknown"));
}
Ok(())
}
}
/// reqwest.content_length() 居然指的是 body_size 而非 content-length header没办法自己实现一下

View File

@@ -1,15 +1,6 @@
use std::io;
use anyhow::Result;
use thiserror::Error;
#[derive(Error, Debug)]
#[error("Request too frequently")]
pub struct DownloadAbortError();
#[derive(Error, Debug)]
#[error("Process page error")]
pub struct ProcessPageError();
pub enum ExecutionStatus {
Skipped,
@@ -17,7 +8,7 @@ pub enum ExecutionStatus {
Ignored(anyhow::Error),
Failed(anyhow::Error),
// 任务可以返回该状态固定自己的 status
FixedFailed(u32, anyhow::Error),
Fixed(u32),
}
// 目前 stable rust 似乎不支持自定义类型使用 ? 运算符,只能先在返回值使用 Result再这样套层娃
@@ -42,10 +33,10 @@ impl From<Result<ExecutionStatus>> for ExecutionStatus {
}
}
// 未包裹的 reqwest::Error
if let Some(error) = cause.downcast_ref::<reqwest::Error>() {
if is_ignored_reqwest_error(error) {
return ExecutionStatus::Ignored(err);
}
if let Some(error) = cause.downcast_ref::<reqwest::Error>()
&& is_ignored_reqwest_error(error)
{
return ExecutionStatus::Ignored(err);
}
}
ExecutionStatus::Failed(err)

View File

@@ -8,6 +8,7 @@ mod config;
mod database;
mod downloader;
mod error;
mod notifier;
mod task;
mod utils;
mod workflow;
@@ -18,14 +19,14 @@ use std::future::Future;
use std::sync::Arc;
use bilibili::BiliClient;
use parking_lot::Mutex;
use parking_lot::RwLock;
use sea_orm::DatabaseConnection;
use task::{http_server, video_downloader};
use tokio_util::sync::CancellationToken;
use tokio_util::task::TaskTracker;
use crate::api::{LogHelper, MAX_HISTORY_LOGS};
use crate::config::{ARGS, VersionedConfig};
use crate::config::{ARGS, CONFIG_DIR, VersionedConfig};
use crate::database::setup_database;
use crate::utils::init_logger;
use crate::utils::signal::terminate;
@@ -44,17 +45,16 @@ async fn main() {
&tracker,
token.clone(),
);
if !cfg!(debug_assertions) {
spawn_task(
"定时下载",
video_downloader(connection, bili_client),
&tracker,
token.clone(),
);
}
spawn_task(
"定时下载",
video_downloader(connection.clone(), bili_client),
&tracker,
token.clone(),
);
tracker.close();
handle_shutdown(tracker, token).await
handle_shutdown(connection, tracker, token).await
}
fn spawn_task(
@@ -77,15 +77,17 @@ fn spawn_task(
}
/// 初始化日志系统、打印欢迎信息,初始化数据库连接和全局配置
async fn init() -> (Arc<DatabaseConnection>, LogHelper) {
async fn init() -> (DatabaseConnection, LogHelper) {
let (tx, _rx) = tokio::sync::broadcast::channel(30);
let log_history = Arc::new(Mutex::new(VecDeque::with_capacity(MAX_HISTORY_LOGS + 1)));
let log_history = Arc::new(RwLock::new(VecDeque::with_capacity(MAX_HISTORY_LOGS + 1)));
let log_writer = LogHelper::new(tx, log_history.clone());
init_logger(&ARGS.log_level, Some(log_writer.clone()));
info!("欢迎使用 Bili-Sync当前程序版本{}", config::version());
info!("项目地址https://github.com/amtoaer/bili-sync");
let connection = Arc::new(setup_database().await.expect("数据库初始化失败"));
let connection = setup_database(&CONFIG_DIR.join("data.sqlite"))
.await
.expect("数据库初始化失败");
info!("数据库初始化完成");
VersionedConfig::init(&connection).await.expect("配置初始化失败");
info!("配置初始化完成");
@@ -93,16 +95,21 @@ async fn init() -> (Arc<DatabaseConnection>, LogHelper) {
(connection, log_writer)
}
async fn handle_shutdown(tracker: TaskTracker, token: CancellationToken) {
async fn handle_shutdown(connection: DatabaseConnection, tracker: TaskTracker, token: CancellationToken) {
tokio::select! {
_ = tracker.wait() => {
error!("所有任务均已终止,程序退出")
error!("所有任务均已终止..")
}
_ = terminate() => {
info!("接收到终止信号,正在终止任务..");
info!("接收到终止信号,开始终止任务..");
token.cancel();
tracker.wait().await;
info!("所有任务均已终止,程序退出");
info!("所有任务均已终止..");
}
}
info!("正在关闭数据库连接..");
match connection.close().await {
Ok(()) => info!("数据库连接已关闭,程序结束"),
Err(e) => error!("关闭数据库连接时遇到错误:{:#},程序异常结束", e),
}
}

View File

@@ -0,0 +1,43 @@
use anyhow::Result;
use futures::future;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(rename_all = "camelCase", tag = "type")]
pub enum Notifier {
Telegram { bot_token: String, chat_id: String },
Webhook { url: String },
}
#[derive(Serialize)]
struct WebhookPayload<'a> {
text: &'a str,
}
pub trait NotifierAllExt {
async fn notify_all(&self, client: &reqwest::Client, message: &str) -> Result<()>;
}
impl NotifierAllExt for Vec<Notifier> {
async fn notify_all(&self, client: &reqwest::Client, message: &str) -> Result<()> {
future::join_all(self.iter().map(|notifier| notifier.notify(client, message))).await;
Ok(())
}
}
impl Notifier {
pub async fn notify(&self, client: &reqwest::Client, message: &str) -> Result<()> {
match self {
Notifier::Telegram { bot_token, chat_id } => {
let url = format!("https://api.telegram.org/bot{}/sendMessage", bot_token);
let params = [("chat_id", chat_id.as_str()), ("text", message)];
client.post(&url).form(&params).send().await?;
}
Notifier::Webhook { url } => {
let payload = WebhookPayload { text: message };
client.post(url).json(&payload).send().await?;
}
}
Ok(())
}
}

View File

@@ -21,20 +21,20 @@ use crate::config::VersionedConfig;
struct Asset;
pub async fn http_server(
database_connection: Arc<DatabaseConnection>,
database_connection: DatabaseConnection,
bili_client: Arc<BiliClient>,
log_writer: LogHelper,
) -> Result<()> {
let app = router()
.fallback_service(get(frontend_files))
.fallback_service(get(frontend_files).head(frontend_files))
.layer(Extension(database_connection))
.layer(Extension(bili_client))
.layer(Extension(log_writer));
let config = VersionedConfig::get().load_full();
let listener = tokio::net::TcpListener::bind(&config.bind_address)
let bind_address = VersionedConfig::get().read().bind_address.to_owned();
let listener = tokio::net::TcpListener::bind(&bind_address)
.await
.context("bind address failed")?;
info!("开始运行管理页: http://{}", config.bind_address);
info!("开始运行管理页http://{}", bind_address);
Ok(axum::serve(listener, ServiceExt::<Request>::into_make_service(app)).await?)
}
@@ -48,34 +48,51 @@ async fn frontend_files(request: Request) -> impl IntoResponse {
};
let mime_type = content.mime_type();
let content_type = mime_type.as_deref().unwrap_or("application/octet-stream");
if cfg!(debug_assertions) {
(
[(header::CONTENT_TYPE, content_type)],
// safety: `RustEmbed` returns uncompressed files directly from the filesystem in debug mode
content.data().unwrap(),
)
.into_response()
} else {
let accepted_encodings = request
.headers()
.get(header::ACCEPT_ENCODING)
.and_then(|v| v.to_str().ok())
.map(|s| s.split(',').map(str::trim).collect::<HashSet<_>>())
.unwrap_or_default();
for (encoding, data) in [("br", content.data_br()), ("gzip", content.data_gzip())] {
if accepted_encodings.contains(encoding) {
if let Some(data) = data {
return (
[
(header::CONTENT_TYPE, content_type),
(header::CONTENT_ENCODING, encoding),
],
data,
)
.into_response();
}
}
}
"Unsupported Encoding".into_response()
let default_headers = [
(header::CONTENT_TYPE, content_type),
(header::CACHE_CONTROL, "no-cache"),
(header::ETAG, &content.hash()),
];
if let Some(if_none_match) = request.headers().get(header::IF_NONE_MATCH)
&& let Ok(client_etag) = if_none_match.to_str()
&& client_etag == content.hash()
{
return (StatusCode::NOT_MODIFIED, default_headers).into_response();
}
if request.method() == axum::http::Method::HEAD {
return (StatusCode::OK, default_headers).into_response();
}
if cfg!(debug_assertions) {
// safety: `RustEmbed` returns uncompressed files directly from the filesystem in debug mode
return (StatusCode::OK, default_headers, content.data().unwrap()).into_response();
}
let accepted_encodings = request
.headers()
.get(header::ACCEPT_ENCODING)
.and_then(|v| v.to_str().ok())
.map(|s| s.split(',').map(str::trim).collect::<HashSet<_>>())
.unwrap_or_default();
for (encoding, data) in [("br", content.data_br()), ("gzip", content.data_gzip())] {
if accepted_encodings.contains(encoding)
&& let Some(data) = data
{
return (
StatusCode::OK,
[
(header::CONTENT_TYPE, content_type),
(header::CACHE_CONTROL, "no-cache"),
(header::ETAG, &content.hash()),
(header::CONTENT_ENCODING, encoding),
],
data,
)
.into_response();
}
}
(
StatusCode::NOT_ACCEPTABLE,
"Client must support gzip or brotli compression",
)
.into_response()
}

View File

@@ -2,4 +2,4 @@ mod http_server;
mod video_downloader;
pub use http_server::http_server;
pub use video_downloader::video_downloader;
pub use video_downloader::{DownloadTaskManager, TaskStatus, video_downloader};

View File

@@ -1,62 +1,369 @@
use std::pin::Pin;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context, Result, bail};
use sea_orm::DatabaseConnection;
use tokio::time;
use serde::Serialize;
use tokio::sync::{OnceCell, watch};
use tokio_cron_scheduler::{Job, JobScheduler};
use crate::adapter::VideoSource;
use crate::bilibili::{self, BiliClient};
use crate::config::VersionedConfig;
use crate::bilibili::{self, BiliClient, BiliError};
use crate::config::{Config, TEMPLATE, Trigger, VersionedConfig};
use crate::utils::model::get_enabled_video_sources;
use crate::utils::task_notifier::TASK_STATUS_NOTIFIER;
use crate::utils::notify::error_and_notify;
use crate::workflow::process_video_source;
static INSTANCE: OnceCell<DownloadTaskManager> = OnceCell::const_new();
/// 启动周期下载视频的任务
pub async fn video_downloader(connection: Arc<DatabaseConnection>, bili_client: Arc<BiliClient>) {
let mut anchor = chrono::Local::now().date_naive();
loop {
info!("开始执行本轮视频下载任务..");
let _lock = TASK_STATUS_NOTIFIER.start_running().await;
let config = VersionedConfig::get().load_full();
'inner: {
if let Err(e) = config.check() {
error!("配置检查失败,跳过本轮执行:\n{:#}", e);
break 'inner;
}
match bili_client.wbi_img().await.map(|wbi_img| wbi_img.into()) {
Ok(Some(mixin_key)) => bilibili::set_global_mixin_key(mixin_key),
Ok(_) => {
error!("解析 mixin key 失败,等待下一轮执行");
break 'inner;
}
Err(e) => {
error!("获取 mixin key 遇到错误:{:#},等待下一轮执行", e);
break 'inner;
}
};
if anchor != chrono::Local::now().date_naive() {
if let Err(e) = bili_client.check_refresh(&connection).await {
error!("检查刷新 Credential 遇到错误:{:#},等待下一轮执行", e);
break 'inner;
}
anchor = chrono::Local::now().date_naive();
}
let Ok(video_sources) = get_enabled_video_sources(&connection).await else {
error!("获取视频源列表失败,等待下一轮执行");
break 'inner;
};
if video_sources.is_empty() {
info!("没有可用的视频源,等待下一轮执行");
break 'inner;
}
for video_source in video_sources {
let display_name = video_source.display_name();
if let Err(e) = process_video_source(video_source, &bili_client, &connection).await {
error!("处理 {} 时遇到错误:{:#},等待下一轮执行", display_name, e);
}
}
info!("本轮任务执行完毕,等待下一轮执行");
pub async fn video_downloader(connection: DatabaseConnection, bili_client: Arc<BiliClient>) -> Result<()> {
let task_manager = DownloadTaskManager::init(connection, bili_client).await?;
task_manager.start().await
}
pub struct DownloadTaskManager {
sched: Arc<tokio::sync::Mutex<JobScheduler>>,
cx: Arc<TaskContext>,
shutdown_rx: watch::Receiver<Result<()>>,
}
#[derive(Serialize, Default, Clone, Copy, Debug)]
pub struct TaskStatus {
is_running: bool,
last_run: Option<chrono::DateTime<chrono::Local>>,
last_finish: Option<chrono::DateTime<chrono::Local>>,
next_run: Option<chrono::DateTime<chrono::Local>>,
}
struct TaskContext {
connection: DatabaseConnection,
bili_client: Arc<BiliClient>,
running: tokio::sync::Mutex<()>,
status_tx: watch::Sender<TaskStatus>,
status_rx: watch::Receiver<TaskStatus>,
video_task_id: tokio::sync::Mutex<Option<uuid::Uuid>>, // 存储当前视频下载任务的 UUID
}
impl DownloadTaskManager {
/// 初始化 DownloadTaskManager 单例
pub async fn init(
connection: DatabaseConnection,
bili_client: Arc<BiliClient>,
) -> Result<&'static DownloadTaskManager> {
INSTANCE
.get_or_try_init(|| DownloadTaskManager::new(connection, bili_client))
.await
}
/// 获取 DownloadTaskManager 单例,未初始化时直接 panic
pub fn get() -> &'static DownloadTaskManager {
INSTANCE.get().expect("DownloadTaskManager is not initialized")
}
/// 订阅下载任务的状态更新
pub fn subscribe(&self) -> watch::Receiver<TaskStatus> {
self.cx.status_rx.clone()
}
/// 手动执行一次下载任务
pub async fn download_once(&self) -> Result<()> {
let _ = self
.sched
.lock()
.await
.add(Job::new_one_shot_async(
Duration::from_secs(0),
DownloadTaskManager::download_video_task(self.cx.clone()),
)?)
.await?;
Ok(())
}
/// 启动任务调度器
async fn start(&self) -> Result<()> {
self.sched.lock().await.start().await?;
let mut shutdown_rx = self.shutdown_rx.clone();
shutdown_rx.changed().await?;
self.sched.lock().await.shutdown().await.context("任务调度器关闭失败")?;
if let Err(e) = &*shutdown_rx.borrow() {
bail!("{:#}", e);
}
Ok(())
}
/// 私有的调度器构造函数
async fn new(connection: DatabaseConnection, bili_client: Arc<BiliClient>) -> Result<Self> {
let sched = Arc::new(tokio::sync::Mutex::new(JobScheduler::new().await?));
let (status_tx, status_rx) = watch::channel(TaskStatus::default());
let (running, video_task_id) = (tokio::sync::Mutex::new(()), tokio::sync::Mutex::new(None));
let cx = Arc::new(TaskContext {
connection,
bili_client,
running,
status_tx,
status_rx,
video_task_id,
});
// 读取初始配置
let mut rx = VersionedConfig::get().subscribe();
let initial_config = rx.borrow_and_update().clone();
// 初始化凭据检查与刷新任务,该任务必须成功,否则直接退出
sched
.lock()
.await
.add(Job::new_async_tz(
"0 0 1 * * *",
chrono::Local,
DownloadTaskManager::check_and_refresh_credential_task(cx.clone()),
)?)
.await?;
// 初始化并添加视频下载任务,将任务 ID 保存到 TaskManager 中
let video_task_id = async {
let job_run = DownloadTaskManager::download_video_task(cx.clone());
let job = match &initial_config.interval {
Trigger::Interval(interval) => Job::new_repeated_async(Duration::from_secs(*interval), job_run)?,
Trigger::Cron(cron) => Job::new_async_tz(cron, chrono::Local, job_run)?,
};
Result::<_, anyhow::Error>::Ok(sched.lock().await.add(job).await?)
}
.await;
let video_task_id = match video_task_id {
Ok(id) => Some(id),
Err(err) => {
error_and_notify(
&initial_config,
&cx.bili_client,
format!("初始化视频下载任务失败:{:#}", err),
);
None
}
};
*cx.video_task_id.lock().await = video_task_id;
// 发起一个一次性的任务,更新一下下次运行的时间
if let Some(video_task_id) = video_task_id {
sched
.lock()
.await
.add(Job::new_one_shot_async(
Duration::from_secs(0),
DownloadTaskManager::refresh_next_run(video_task_id, cx.clone()),
)?)
.await?;
}
// 发起一个新任务,用来监听配置变更,动态更新视频下载任务
let cx_clone = cx.clone();
let sched_clone = sched.clone();
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(Ok(()));
tokio::spawn(async move {
let update_task_result = async {
while rx.changed().await.is_ok() {
let new_config = rx.borrow().clone();
let cx = cx_clone.clone();
let mut video_task_id = cx.video_task_id.lock().await;
if let Some(old_video_task_id) = *video_task_id {
// 这里必须成功,不然后面会重复添加任务
sched_clone
.lock()
.await
.remove(&old_video_task_id)
.await
.context("移除旧的视频下载任务失败")?;
}
let new_video_task_id = async {
let job_run = DownloadTaskManager::download_video_task(cx.clone());
let job = match &new_config.interval {
Trigger::Interval(interval) => {
Job::new_repeated_async(Duration::from_secs(*interval), job_run)?
}
Trigger::Cron(cron) => Job::new_async_tz(cron, chrono::Local, job_run)?,
};
Result::<_, anyhow::Error>::Ok(sched_clone.lock().await.add(job).await?)
}
.await;
let new_video_task_id = match new_video_task_id {
Ok(id) => Some(id),
Err(err) => {
error_and_notify(
&initial_config,
&cx.bili_client,
format!("重载视频下载任务失败:{:#}", err),
);
None
}
};
*video_task_id = new_video_task_id;
if let Some(video_task_id) = new_video_task_id {
sched_clone
.lock()
.await
.add(Job::new_one_shot_async(
Duration::from_secs(0),
DownloadTaskManager::refresh_next_run(video_task_id, cx.clone()),
)?)
.await?;
}
}
Result::<(), anyhow::Error>::Ok(())
}
.await;
// 如果执行正常,上面应该是永远不会退出的
let _ = shutdown_tx.send(update_task_result);
});
Ok(Self { sched, cx, shutdown_rx })
}
fn check_and_refresh_credential_task(
cx: Arc<TaskContext>,
) -> impl FnMut(uuid::Uuid, JobScheduler) -> Pin<Box<dyn Future<Output = ()> + Send>> {
move |_uuid, _l| {
let cx = cx.clone();
Box::pin(async move {
let _lock = cx.running.lock().await;
let config = VersionedConfig::get().read();
info!("开始执行本轮凭据检查与刷新任务..");
match check_and_refresh_credential(&cx.connection, &cx.bili_client, &config).await {
Ok(_) => info!("本轮凭据检查与刷新任务执行完毕"),
Err(e) => {
error_and_notify(
&config,
&cx.bili_client,
format!("本轮凭据检查与刷新任务执行遇到错误:{:#}", e),
);
}
}
})
}
}
fn refresh_next_run(
video_task_id: uuid::Uuid,
cx: Arc<TaskContext>,
) -> impl FnMut(uuid::Uuid, JobScheduler) -> Pin<Box<dyn Future<Output = ()> + Send>> {
move |_uuid, mut l| {
let cx = cx.clone();
Box::pin(async move {
let old_status = *cx.status_rx.borrow();
let next_run = l
.next_tick_for_job(video_task_id)
.await
.ok()
.flatten()
.map(|dt| dt.with_timezone(&chrono::Local));
let _ = cx.status_tx.send(TaskStatus { next_run, ..old_status });
})
}
}
fn download_video_task(
cx: Arc<TaskContext>,
) -> impl FnMut(uuid::Uuid, JobScheduler) -> Pin<Box<dyn Future<Output = ()> + Send>> {
move |uuid, mut l| {
let cx = cx.clone();
Box::pin(async move {
let Ok(_lock) = cx.running.try_lock() else {
warn!("上一次视频下载任务尚未结束,跳过本次执行..");
return;
};
let _ = cx.status_tx.send(TaskStatus {
is_running: true,
last_run: Some(chrono::Local::now()),
last_finish: None,
next_run: None,
});
info!("开始执行本轮视频下载任务..");
let mut config = VersionedConfig::get().snapshot();
match download_video(&cx.connection, &cx.bili_client, &mut config).await {
Ok(_) => info!("本轮视频下载任务执行完毕"),
Err(e) => {
error_and_notify(
&config,
&cx.bili_client,
format!("本轮视频下载任务执行遇到错误:{:#}", e),
);
}
}
// 注意此处尽量从 updating 中读取 uuid因为当前任务可能是不存在 next_tick 的 oneshot 任务
let task_uuid = (*cx.video_task_id.lock().await).unwrap_or(uuid);
let next_run = l
.next_tick_for_job(task_uuid)
.await
.ok()
.flatten()
.map(|dt| dt.with_timezone(&chrono::Local));
let last_status = *cx.status_rx.borrow();
let _ = cx.status_tx.send(TaskStatus {
is_running: false,
last_run: last_status.last_run,
last_finish: Some(chrono::Local::now()),
next_run,
});
})
}
TASK_STATUS_NOTIFIER.finish_running(_lock);
time::sleep(time::Duration::from_secs(config.interval)).await;
}
}
async fn check_and_refresh_credential(
connection: &DatabaseConnection,
bili_client: &BiliClient,
config: &Config,
) -> Result<()> {
match bili_client
.check_refresh(&config.credential)
.await
.context("检查刷新 Credential 失败")?
{
None => {
info!("Credential 无需刷新");
}
Some(new_credential) => {
VersionedConfig::get()
.update_credential(new_credential, connection)
.await
.context("新 Credential 持久化失败")?;
info!("Credential 已刷新并保存");
}
}
Ok(())
}
async fn download_video(
connection: &DatabaseConnection,
bili_client: &BiliClient,
config: &mut Arc<Config>,
) -> Result<()> {
config.check().context("配置检查失败")?;
let mixin_key = bili_client
.wbi_img(&config.credential)
.await
.context("获取 wbi_img 失败")?
.into_mixin_key()
.context("解析 mixin key 失败")?;
bilibili::set_global_mixin_key(mixin_key);
let template = TEMPLATE.snapshot();
let bili_client = bili_client.snapshot()?;
let video_sources = get_enabled_video_sources(connection)
.await
.context("获取视频源列表失败")?;
if video_sources.is_empty() {
bail!("没有可用的视频源");
}
for video_source in video_sources {
let display_name = video_source.display_name();
if let Err(e) = process_video_source(video_source, &bili_client, connection, &template, config).await {
error_and_notify(
config,
&bili_client,
format!("处理 {} 时遇到错误:{:#},跳过该视频源", display_name, e),
);
if let Ok(e) = e.downcast::<BiliError>()
&& e.is_risk_control_related()
{
warn!("检测到风控,终止此轮视频下载任务..");
break;
}
}
}
Ok(())
}

View File

@@ -97,7 +97,23 @@ impl VideoInfo {
valid: Set(true),
..default
},
_ => unreachable!(),
VideoInfo::Dynamic {
title,
bvid,
desc,
cover,
pubtime,
} => bili_sync_entity::video::ActiveModel {
bvid: Set(bvid),
name: Set(title),
intro: Set(desc),
cover: Set(cover),
pubtime: Set(pubtime.naive_utc()),
category: Set(2), // 动态里的视频内容类型肯定是视频
valid: Set(true),
..default
},
VideoInfo::Detail { .. } => unreachable!(),
}
}
@@ -114,22 +130,30 @@ impl VideoInfo {
ctime,
pubtime,
state,
is_upower_exclusive,
is_upower_play,
redirect_url,
..
} => bili_sync_entity::video::ActiveModel {
bvid: Set(bvid),
name: Set(title),
category: Set(2),
intro: Set(intro),
cover: Set(cover),
ctime: Set(ctime.naive_utc()),
pubtime: Set(pubtime.naive_utc()),
favtime: if base_model.favtime != NaiveDateTime::default() {
NotSet // 之前设置了 favtime不覆盖
Set(base_model.favtime) // 之前设置了 favtime使用之前的值等价于 unset但设置上以支持后续的规则匹配
} else {
Set(pubtime.naive_utc()) // 未设置过 favtime使用 pubtime 填充
},
download_status: Set(0),
valid: Set(state == 0),
// state == 0 表示开放浏览
// is_upower_exclusive 和 is_upower_play 相等有两种情况:
// 1. 都为 true表示视频是充电专享但是已经充过电有权观看
// 2. 都为 false表示视频是非充电视频
// redirect_url 仅在视频为番剧、影视、纪录片等特殊视频时才会有值,如果为空说明是普通视频
// 仅在三种条件都满足时,才认为视频是可下载的
valid: Set(state == 0 && (is_upower_exclusive == is_upower_play) && redirect_url.is_none()),
upper_id: Set(upper.mid),
upper_name: Set(upper.name),
upper_face: Set(upper.face),
@@ -145,17 +169,15 @@ impl VideoInfo {
VideoInfo::Collection { pubtime: time, .. }
| VideoInfo::Favorite { fav_time: time, .. }
| VideoInfo::WatchLater { fav_time: time, .. }
| VideoInfo::Submission { ctime: time, .. } => time,
_ => unreachable!(),
| VideoInfo::Submission { ctime: time, .. }
| VideoInfo::Dynamic { pubtime: time, .. } => time,
VideoInfo::Detail { .. } => unreachable!(),
}
}
}
impl PageInfo {
pub fn into_active_model(
self,
video_model: &bili_sync_entity::video::Model,
) -> bili_sync_entity::page::ActiveModel {
pub fn into_active_model(self, video_model_id: i32) -> bili_sync_entity::page::ActiveModel {
let (width, height) = match &self.dimension {
Some(d) => {
if d.rotate == 0 {
@@ -167,7 +189,7 @@ impl PageInfo {
None => (None, None),
};
bili_sync_entity::page::ActiveModel {
video_id: Set(video_model.id),
video_id: Set(video_model_id),
cid: Set(self.cid),
pid: Set(self.page),
name: Set(self.name),

View File

@@ -0,0 +1,36 @@
use sea_orm::DatabaseConnection;
use crate::adapter::VideoSourceEnum;
use crate::bilibili::BiliClient;
use crate::config::Config;
use crate::downloader::Downloader;
#[derive(Clone, Copy)]
pub struct DownloadContext<'a> {
pub bili_client: &'a BiliClient,
pub video_source: &'a VideoSourceEnum,
pub template: &'a handlebars::Handlebars<'a>,
pub connection: &'a DatabaseConnection,
pub downloader: &'a Downloader,
pub config: &'a Config,
}
impl<'a> DownloadContext<'a> {
pub fn new(
bili_client: &'a BiliClient,
video_source: &'a VideoSourceEnum,
template: &'a handlebars::Handlebars<'a>,
connection: &'a DatabaseConnection,
downloader: &'a Downloader,
config: &'a Config,
) -> Self {
Self {
bili_client,
video_source,
template,
connection,
downloader,
config,
}
}
}

View File

@@ -1,24 +1,21 @@
use serde_json::json;
use crate::config::VersionedConfig;
pub fn video_format_args(video_model: &bili_sync_entity::video::Model) -> serde_json::Value {
let config = VersionedConfig::get().load();
pub fn video_format_args(video_model: &bili_sync_entity::video::Model, time_format: &str) -> serde_json::Value {
json!({
"bvid": &video_model.bvid,
"title": &video_model.name,
"upper_name": &video_model.upper_name,
"upper_mid": &video_model.upper_id,
"pubtime": &video_model.pubtime.and_utc().format(&config.time_format).to_string(),
"fav_time": &video_model.favtime.and_utc().format(&config.time_format).to_string(),
"pubtime": &video_model.pubtime.and_utc().format(time_format).to_string(),
"fav_time": &video_model.favtime.and_utc().format(time_format).to_string(),
})
}
pub fn page_format_args(
video_model: &bili_sync_entity::video::Model,
page_model: &bili_sync_entity::page::Model,
time_format: &str,
) -> serde_json::Value {
let config = VersionedConfig::get().load();
json!({
"bvid": &video_model.bvid,
"title": &video_model.name,
@@ -26,7 +23,7 @@ pub fn page_format_args(
"upper_mid": &video_model.upper_id,
"ptitle": &page_model.name,
"pid": page_model.pid,
"pubtime": video_model.pubtime.and_utc().format(&config.time_format).to_string(),
"fav_time": video_model.favtime.and_utc().format(&config.time_format).to_string(),
"pubtime": video_model.pubtime.and_utc().format(time_format).to_string(),
"fav_time": video_model.favtime.and_utc().format(time_format).to_string(),
})
}

View File

@@ -1,11 +1,13 @@
pub mod convert;
pub mod download_context;
pub mod filenamify;
pub mod format_arg;
pub mod model;
pub mod nfo;
pub mod notify;
pub mod rule;
pub mod signal;
pub mod status;
pub mod task_notifier;
pub mod validation;
use tracing_subscriber::fmt;
use tracing_subscriber::layer::SubscriberExt;

View File

@@ -1,13 +1,14 @@
use anyhow::{Context, Result, anyhow};
use bili_sync_entity::*;
use rand::seq::SliceRandom;
use sea_orm::ActiveValue::Set;
use sea_orm::DatabaseTransaction;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::{DatabaseTransaction, TransactionTrait};
use crate::adapter::{VideoSource, VideoSourceEnum};
use crate::bilibili::{PageInfo, VideoInfo};
use crate::config::{Config, LegacyConfig};
use crate::bilibili::VideoInfo;
use crate::config::Config;
use crate::utils::status::STATUS_COMPLETED;
/// 筛选未填充的视频
@@ -41,6 +42,7 @@ pub async fn filter_unhandled_video_pages(
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.and(video::Column::ShouldDownload.eq(true))
.and(additional_expr),
)
.find_with_related(page::Entity)
@@ -73,16 +75,8 @@ pub async fn create_videos(
}
/// 尝试创建 Page Model如果发生冲突则忽略
pub async fn create_pages(
pages_info: Vec<PageInfo>,
video_model: &bili_sync_entity::video::Model,
connection: &DatabaseTransaction,
) -> Result<()> {
let page_models = pages_info
.into_iter()
.map(|p| p.into_active_model(video_model))
.collect::<Vec<page::ActiveModel>>();
for page_chunk in page_models.chunks(50) {
pub async fn create_pages(pages_model: Vec<page::ActiveModel>, connection: &DatabaseTransaction) -> Result<()> {
for page_chunk in pages_model.chunks(200) {
page::Entity::insert_many(page_chunk.to_vec())
.on_conflict(
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
@@ -141,6 +135,8 @@ pub async fn get_enabled_video_sources(connection: &DatabaseConnection) -> Resul
sources.extend(watch_later.into_iter().map(VideoSourceEnum::from));
sources.extend(submission.into_iter().map(VideoSourceEnum::from));
sources.extend(collection.into_iter().map(VideoSourceEnum::from));
// 此处将视频源随机打乱顺序,从概率上确保每个视频源都有机会优先执行,避免后面视频源的长期饥饿问题
sources.shuffle(&mut rand::rng());
Ok(sources)
}
@@ -173,69 +169,3 @@ pub async fn save_db_config(config: &Config, connection: &DatabaseConnection) ->
.context("Failed to save config to database")?;
Ok(())
}
/// 迁移旧版本配置(即将所有相关联的内容设置为 enabled
pub async fn migrate_legacy_config(config: &LegacyConfig, connection: &DatabaseConnection) -> Result<()> {
let transaction = connection.begin().await.context("Failed to begin transaction")?;
tokio::try_join!(
migrate_favorite(config, &transaction),
migrate_watch_later(config, &transaction),
migrate_submission(config, &transaction),
migrate_collection(config, &transaction)
)?;
transaction.commit().await.context("Failed to commit transaction")?;
Ok(())
}
async fn migrate_favorite(config: &LegacyConfig, connection: &DatabaseTransaction) -> Result<()> {
favorite::Entity::update_many()
.filter(favorite::Column::FId.is_in(config.favorite_list.keys().collect::<Vec<_>>()))
.col_expr(favorite::Column::Enabled, Expr::value(true))
.exec(connection)
.await
.context("Failed to migrate favorite config")?;
Ok(())
}
async fn migrate_watch_later(config: &LegacyConfig, connection: &DatabaseTransaction) -> Result<()> {
if config.watch_later.enabled {
watch_later::Entity::update_many()
.col_expr(watch_later::Column::Enabled, Expr::value(true))
.exec(connection)
.await
.context("Failed to migrate watch later config")?;
}
Ok(())
}
async fn migrate_submission(config: &LegacyConfig, connection: &DatabaseTransaction) -> Result<()> {
submission::Entity::update_many()
.filter(submission::Column::UpperId.is_in(config.submission_list.keys().collect::<Vec<_>>()))
.col_expr(submission::Column::Enabled, Expr::value(true))
.exec(connection)
.await
.context("Failed to migrate submission config")?;
Ok(())
}
async fn migrate_collection(config: &LegacyConfig, connection: &DatabaseTransaction) -> Result<()> {
let tuples: Vec<(i64, i64, i32)> = config
.collection_list
.keys()
.filter_map(|key| Some((key.sid.parse().ok()?, key.mid.parse().ok()?, key.collection_type.into())))
.collect();
collection::Entity::update_many()
.filter(
Expr::tuple([
Expr::column(collection::Column::SId),
Expr::column(collection::Column::MId),
Expr::column(collection::Column::Type),
])
.in_tuples(tuples),
)
.col_expr(collection::Column::Enabled, Expr::value(true))
.exec(connection)
.await
.context("Failed to migrate collection config")?;
Ok(())
}

View File

@@ -6,7 +6,7 @@ use quick_xml::events::{BytesCData, BytesText};
use quick_xml::writer::Writer;
use tokio::io::{AsyncWriteExt, BufWriter};
use crate::config::{NFOTimeType, VersionedConfig};
use crate::config::NFOTimeType;
#[allow(clippy::upper_case_acronyms)]
pub enum NFO<'a> {
@@ -22,7 +22,8 @@ pub struct Movie<'a> {
pub bvid: &'a str,
pub upper_id: i64,
pub upper_name: &'a str,
pub aired: NaiveDateTime,
pub upper_thumb: &'a str,
pub premiered: NaiveDateTime,
pub tags: Option<Vec<String>>,
}
@@ -32,7 +33,8 @@ pub struct TVShow<'a> {
pub bvid: &'a str,
pub upper_id: i64,
pub upper_name: &'a str,
pub aired: NaiveDateTime,
pub upper_thumb: &'a str,
pub premiered: NaiveDateTime,
pub tags: Option<Vec<String>>,
}
@@ -96,12 +98,16 @@ impl NFO<'_> {
.create_element("role")
.write_text_content_async(BytesText::new(movie.upper_name))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(movie.upper_thumb))
.await?;
Ok(writer)
})
.await?;
writer
.create_element("year")
.write_text_content_async(BytesText::new(&movie.aired.format("%Y").to_string()))
.write_text_content_async(BytesText::new(&movie.premiered.format("%Y").to_string()))
.await?;
if let Some(tags) = movie.tags {
for tag in tags {
@@ -117,8 +123,8 @@ impl NFO<'_> {
.write_text_content_async(BytesText::new(movie.bvid))
.await?;
writer
.create_element("aired")
.write_text_content_async(BytesText::new(&movie.aired.format("%Y-%m-%d").to_string()))
.create_element("premiered")
.write_text_content_async(BytesText::new(&movie.premiered.format("%Y-%m-%d").to_string()))
.await?;
Ok(writer)
})
@@ -150,12 +156,16 @@ impl NFO<'_> {
.create_element("role")
.write_text_content_async(BytesText::new(tvshow.upper_name))
.await?;
writer
.create_element("thumb")
.write_text_content_async(BytesText::new(tvshow.upper_thumb))
.await?;
Ok(writer)
})
.await?;
writer
.create_element("year")
.write_text_content_async(BytesText::new(&tvshow.aired.format("%Y").to_string()))
.write_text_content_async(BytesText::new(&tvshow.premiered.format("%Y").to_string()))
.await?;
if let Some(tags) = tvshow.tags {
for tag in tags {
@@ -171,8 +181,8 @@ impl NFO<'_> {
.write_text_content_async(BytesText::new(tvshow.bvid))
.await?;
writer
.create_element("aired")
.write_text_content_async(BytesText::new(&tvshow.aired.format("%Y-%m-%d").to_string()))
.create_element("premiered")
.write_text_content_async(BytesText::new(&tvshow.premiered.format("%Y-%m-%d").to_string()))
.await?;
Ok(writer)
})
@@ -252,6 +262,7 @@ mod tests {
name: "name".to_string(),
upper_id: 1,
upper_name: "upper_name".to_string(),
upper_face: "https://i1.hdslb.com/bfs/face/72e8f33cadc72e022fc34624cc69e1b12ebb72c0.jpg".to_string(),
favtime: chrono::NaiveDateTime::new(
chrono::NaiveDate::from_ymd_opt(2022, 2, 2).unwrap(),
chrono::NaiveTime::from_hms_opt(2, 2, 2).unwrap(),
@@ -261,11 +272,14 @@ mod tests {
chrono::NaiveTime::from_hms_opt(3, 3, 3).unwrap(),
),
bvid: "BV1nWcSeeEkV".to_string(),
tags: Some(serde_json::json!(["tag1", "tag2"])),
tags: Some(vec!["tag1".to_owned(), "tag2".to_owned()].into()),
..Default::default()
};
assert_eq!(
NFO::Movie((&video).into()).generate_nfo().await.unwrap(),
NFO::Movie((&video).to_nfo(NFOTimeType::FavTime))
.generate_nfo()
.await
.unwrap(),
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<movie>
<plot><![CDATA[原始视频:<a href="https://www.bilibili.com/video/BV1nWcSeeEkV/">BV1nWcSeeEkV</a><br/><br/>intro]]></plot>
@@ -274,16 +288,20 @@ mod tests {
<actor>
<name>1</name>
<role>upper_name</role>
<thumb>https://i1.hdslb.com/bfs/face/72e8f33cadc72e022fc34624cc69e1b12ebb72c0.jpg</thumb>
</actor>
<year>2022</year>
<genre>tag1</genre>
<genre>tag2</genre>
<uniqueid type="bilibili">BV1nWcSeeEkV</uniqueid>
<aired>2022-02-02</aired>
<premiered>2022-02-02</premiered>
</movie>"#,
);
assert_eq!(
NFO::TVShow((&video).into()).generate_nfo().await.unwrap(),
NFO::TVShow((&video).to_nfo(NFOTimeType::FavTime))
.generate_nfo()
.await
.unwrap(),
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<tvshow>
<plot><![CDATA[原始视频:<a href="https://www.bilibili.com/video/BV1nWcSeeEkV/">BV1nWcSeeEkV</a><br/><br/>intro]]></plot>
@@ -292,16 +310,20 @@ mod tests {
<actor>
<name>1</name>
<role>upper_name</role>
<thumb>https://i1.hdslb.com/bfs/face/72e8f33cadc72e022fc34624cc69e1b12ebb72c0.jpg</thumb>
</actor>
<year>2022</year>
<genre>tag1</genre>
<genre>tag2</genre>
<uniqueid type="bilibili">BV1nWcSeeEkV</uniqueid>
<aired>2022-02-02</aired>
<premiered>2022-02-02</premiered>
</tvshow>"#,
);
assert_eq!(
NFO::Upper((&video).into()).generate_nfo().await.unwrap(),
NFO::Upper((&video).to_nfo(NFOTimeType::FavTime))
.generate_nfo()
.await
.unwrap(),
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<person>
<plot/>
@@ -318,7 +340,10 @@ mod tests {
..Default::default()
};
assert_eq!(
NFO::Episode((&page).into()).generate_nfo().await.unwrap(),
NFO::Episode((&page).to_nfo(NFOTimeType::FavTime))
.generate_nfo()
.await
.unwrap(),
r#"<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<episodedetails>
<plot/>
@@ -331,60 +356,60 @@ mod tests {
}
}
impl<'a> From<&'a video::Model> for Movie<'a> {
fn from(video: &'a video::Model) -> Self {
Self {
name: &video.name,
intro: &video.intro,
bvid: &video.bvid,
upper_id: video.upper_id,
upper_name: &video.upper_name,
aired: match VersionedConfig::get().load().nfo_time_type {
NFOTimeType::FavTime => video.favtime,
NFOTimeType::PubTime => video.pubtime,
pub trait ToNFO<'a, T> {
fn to_nfo(&'a self, nfo_time_type: NFOTimeType) -> T;
}
impl<'a> ToNFO<'a, Movie<'a>> for &'a video::Model {
fn to_nfo(&'a self, nfo_time_type: NFOTimeType) -> Movie<'a> {
Movie {
name: &self.name,
intro: &self.intro,
bvid: &self.bvid,
upper_id: self.upper_id,
upper_name: &self.upper_name,
upper_thumb: &self.upper_face,
premiered: match nfo_time_type {
NFOTimeType::FavTime => self.favtime,
NFOTimeType::PubTime => self.pubtime,
},
tags: video
.tags
.as_ref()
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
tags: self.tags.as_ref().map(|tags| tags.clone().into()),
}
}
}
impl<'a> From<&'a video::Model> for TVShow<'a> {
fn from(video: &'a video::Model) -> Self {
Self {
name: &video.name,
intro: &video.intro,
bvid: &video.bvid,
upper_id: video.upper_id,
upper_name: &video.upper_name,
aired: match VersionedConfig::get().load().nfo_time_type {
NFOTimeType::FavTime => video.favtime,
NFOTimeType::PubTime => video.pubtime,
impl<'a> ToNFO<'a, TVShow<'a>> for &'a video::Model {
fn to_nfo(&'a self, nfo_time_type: NFOTimeType) -> TVShow<'a> {
TVShow {
name: &self.name,
intro: &self.intro,
bvid: &self.bvid,
upper_id: self.upper_id,
upper_name: &self.upper_name,
upper_thumb: &self.upper_face,
premiered: match nfo_time_type {
NFOTimeType::FavTime => self.favtime,
NFOTimeType::PubTime => self.pubtime,
},
tags: video
.tags
.as_ref()
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
tags: self.tags.as_ref().map(|tags| tags.clone().into()),
}
}
}
impl<'a> From<&'a video::Model> for Upper {
fn from(video: &'a video::Model) -> Self {
Self {
upper_id: video.upper_id.to_string(),
pubtime: video.pubtime,
impl<'a> ToNFO<'a, Upper> for &'a video::Model {
fn to_nfo(&'a self, _nfo_time_type: NFOTimeType) -> Upper {
Upper {
upper_id: self.upper_id.to_string(),
pubtime: self.pubtime,
}
}
}
impl<'a> From<&'a page::Model> for Episode<'a> {
fn from(page: &'a page::Model) -> Self {
Self {
name: &page.name,
pid: page.pid.to_string(),
impl<'a> ToNFO<'a, Episode<'a>> for &'a page::Model {
fn to_nfo(&'a self, _nfo_time_type: NFOTimeType) -> Episode<'a> {
Episode {
name: &self.name,
pid: self.pid.to_string(),
}
}
}

View File

@@ -0,0 +1,13 @@
use crate::bilibili::BiliClient;
use crate::config::Config;
use crate::notifier::NotifierAllExt;
pub fn error_and_notify(config: &Config, bili_client: &BiliClient, msg: String) {
error!("{msg}");
if let Some(notifiers) = &config.notifiers
&& !notifiers.is_empty()
{
let (notifiers, inner_client) = (notifiers.clone(), bili_client.inner_client().clone());
tokio::spawn(async move { notifiers.notify_all(&inner_client, msg.as_str()).await });
}
}

View File

@@ -0,0 +1,267 @@
use bili_sync_entity::rule::{AndGroup, Condition, Rule, RuleTarget};
use bili_sync_entity::{page, video};
use chrono::{Local, NaiveDateTime};
pub(crate) trait Evaluatable<T> {
fn evaluate(&self, value: T) -> bool;
}
pub(crate) trait FieldEvaluatable {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool;
fn evaluate_model(&self, video: &video::Model, pages: &[page::Model]) -> bool;
}
impl Evaluatable<&str> for Condition<String> {
fn evaluate(&self, value: &str) -> bool {
match self {
Condition::Equals(expected) => expected == value,
Condition::Contains(substring) => value.contains(substring),
Condition::Prefix(prefix) => value.starts_with(prefix),
Condition::Suffix(suffix) => value.ends_with(suffix),
Condition::MatchesRegex(_, regex) => regex.is_match(value),
_ => false,
}
}
}
impl Evaluatable<usize> for Condition<usize> {
fn evaluate(&self, value: usize) -> bool {
match self {
Condition::Equals(expected) => *expected == value,
Condition::GreaterThan(threshold) => value > *threshold,
Condition::LessThan(threshold) => value < *threshold,
Condition::Between(start, end) => value > *start && value < *end,
_ => false,
}
}
}
impl Evaluatable<&NaiveDateTime> for Condition<NaiveDateTime> {
fn evaluate(&self, value: &NaiveDateTime) -> bool {
match self {
Condition::Equals(expected) => expected == value,
Condition::GreaterThan(threshold) => value > threshold,
Condition::LessThan(threshold) => value < threshold,
Condition::Between(start, end) => value > start && value < end,
_ => false,
}
}
}
impl FieldEvaluatable for RuleTarget {
/// 修改模型后进行评估,此时能访问的是未保存的 activeModel就地使用 activeModel 评估
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
match self {
RuleTarget::Title(cond) => video.name.try_as_ref().is_some_and(|title| cond.evaluate(title)),
// 目前的所有条件都是分别针对全体标签进行 any 评估的,例如 Prefix("a") && Suffix("b") 意味着 any(tag.Prefix("a")) && any(tag.Suffix("b")) 而非 any(tag.Prefix("a") && tag.Suffix("b"))
// 这可能不满足用户预期,但应该问题不大,如果真有很多人用复杂标签筛选再单独改
RuleTarget::Tags(cond) => video
.tags
.try_as_ref()
.and_then(|t| t.as_ref())
.is_some_and(|tags| tags.0.iter().any(|tag| cond.evaluate(tag))),
RuleTarget::FavTime(cond) => video
.favtime
.try_as_ref()
.map(|fav_time| fav_time.and_utc().with_timezone(&Local).naive_local()) // 数据库中保存的一律是 utc 时间,转换为 local 时间再比较
.is_some_and(|fav_time| cond.evaluate(&fav_time)),
RuleTarget::PubTime(cond) => video
.pubtime
.try_as_ref()
.map(|pub_time| pub_time.and_utc().with_timezone(&Local).naive_local())
.is_some_and(|pub_time| cond.evaluate(&pub_time)),
RuleTarget::PageCount(cond) => cond.evaluate(pages.len()),
RuleTarget::Not(inner) => !inner.evaluate(video, pages),
}
}
/// 手动触发对历史视频的评估,拿到的是原始 Model直接使用
fn evaluate_model(&self, video: &video::Model, pages: &[page::Model]) -> bool {
match self {
RuleTarget::Title(cond) => cond.evaluate(&video.name),
// 目前的所有条件都是分别针对全体标签进行 any 评估的,例如 Prefix("a") && Suffix("b") 意味着 any(tag.Prefix("a")) && any(tag.Suffix("b")) 而非 any(tag.Prefix("a") && tag.Suffix("b"))
// 这可能不满足用户预期,但应该问题不大,如果真有很多人用复杂标签筛选再单独改
RuleTarget::Tags(cond) => video
.tags
.as_ref()
.is_some_and(|tags| tags.0.iter().any(|tag| cond.evaluate(tag))),
RuleTarget::FavTime(cond) => cond.evaluate(&video.favtime.and_utc().with_timezone(&Local).naive_local()),
RuleTarget::PubTime(cond) => cond.evaluate(&video.pubtime.and_utc().with_timezone(&Local).naive_local()),
RuleTarget::PageCount(cond) => cond.evaluate(pages.len()),
RuleTarget::Not(inner) => !inner.evaluate_model(video, pages),
}
}
}
impl FieldEvaluatable for AndGroup {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
self.iter().all(|target| target.evaluate(video, pages))
}
fn evaluate_model(&self, video: &video::Model, pages: &[page::Model]) -> bool {
self.iter().all(|target| target.evaluate_model(video, pages))
}
}
impl FieldEvaluatable for Rule {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
if self.0.is_empty() {
return true;
}
self.0.iter().any(|group| group.evaluate(video, pages))
}
fn evaluate_model(&self, video: &video::Model, pages: &[page::Model]) -> bool {
if self.0.is_empty() {
return true;
}
self.0.iter().any(|group| group.evaluate_model(video, pages))
}
}
/// 对于 Option<Rule> 如果 rule 不存在应该被认为是通过评估
impl FieldEvaluatable for Option<Rule> {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
self.as_ref().is_none_or(|rule| rule.evaluate(video, pages))
}
fn evaluate_model(&self, video: &video::Model, pages: &[page::Model]) -> bool {
self.as_ref().is_none_or(|rule| rule.evaluate_model(video, pages))
}
}
#[cfg(test)]
mod tests {
use bili_sync_entity::page;
use chrono::NaiveDate;
use sea_orm::ActiveValue::Set;
use super::*;
#[test]
fn test_display() {
let test_cases = vec![
(
Rule(vec![vec![RuleTarget::Title(Condition::Contains("唐氏".to_string()))]]),
"「(标题包含“唐氏”)」",
),
(
Rule(vec![vec![
RuleTarget::Title(Condition::Prefix("街霸".to_string())),
RuleTarget::Tags(Condition::Contains("套路".to_string())),
]]),
"「(标题以“街霸”开头)且(标签包含“套路”)」",
),
(
Rule(vec![
vec![
RuleTarget::Title(Condition::Contains("Rust".to_string())),
RuleTarget::PageCount(Condition::GreaterThan(5)),
],
vec![
RuleTarget::Tags(Condition::Suffix("入门".to_string())),
RuleTarget::PubTime(Condition::GreaterThan(
NaiveDate::from_ymd_opt(2023, 1, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap(),
)),
],
]),
"标题包含“Rust”视频分页数量大于“5”」或「标签以“入门”结尾发布时间大于“2023-01-01 00:00:00”",
),
(
Rule(vec![vec![
RuleTarget::Not(Box::new(RuleTarget::Title(Condition::Contains("广告".to_string())))),
RuleTarget::PageCount(Condition::LessThan(10)),
]]),
"标题不包含“广告”视频分页数量小于“10”",
),
(
Rule(vec![vec![
RuleTarget::FavTime(Condition::Between(
NaiveDate::from_ymd_opt(2023, 6, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap(),
NaiveDate::from_ymd_opt(2023, 12, 31)
.unwrap()
.and_hms_opt(23, 59, 59)
.unwrap(),
)),
// autocorrect-disable
RuleTarget::Tags(Condition::MatchesRegex(
"技术|教程".to_string(),
regex::Regex::new("技术|教程").unwrap(),
)),
]]),
"收藏时间在“2023-06-01 00:00:00”和“2023-12-31 23:59:59”之间标签匹配“技术|教程”)」",
// autocorrect-enable
),
];
for (rule, expected) in test_cases {
assert_eq!(rule.to_string(), expected);
}
}
#[test]
fn test_evaluate() {
let test_cases = vec![
(
(
video::ActiveModel {
name: Set("骂谁唐氏呢!!!".to_string()),
..Default::default()
},
vec![],
),
Rule(vec![vec![RuleTarget::Title(Condition::Contains("唐氏".to_string()))]]),
true,
),
(
(
video::ActiveModel::default(),
vec![page::ActiveModel::default(); 2],
),
Rule(vec![vec![RuleTarget::PageCount(Condition::Equals(1))]]),
false,
),
(
(
video::ActiveModel{
tags: Set(Some(vec!["原神".to_owned(),"永雏塔菲".to_owned(),"虚拟主播".to_owned()].into())),
..Default::default()
},
vec![],
),
Rule (vec![vec![RuleTarget::Not(Box::new(RuleTarget::Tags(Condition::Equals(
"原神".to_string(),
))))]],
),
false,
),
(
(
video::ActiveModel {
name: Set(
"万字怒扒网易《归唐》底裤中国首款大厂买断制单机靠谱吗——全网最全官方非独家幕后关于《归唐》PV 的所有秘密~都在这里了~".to_owned(),
),
..Default::default()
},
vec![],
),
Rule(vec![vec![RuleTarget::Not(Box::new(RuleTarget::Title(Condition::MatchesRegex(
r"^\S+字(解析|怒扒|拆解)".to_owned(),
regex::Regex::new(r"^\S+字(解析|怒扒)").unwrap(),
))))]],
),
false,
),
];
for ((video, pages), rule, expected) in test_cases {
assert_eq!(rule.evaluate(&video, &pages), expected);
}
}
}

View File

@@ -119,8 +119,8 @@ impl<const N: usize> Status<N> {
/// 根据子任务执行结果更新子任务的状态
fn set_result(&mut self, result: &ExecutionStatus, offset: usize) {
// 如果任务返回 FixedFailed 状态,那么无论之前的状态如何,都将状态设置为 FixedFailed 的状态
if let ExecutionStatus::FixedFailed(status, _) = result {
// 如果任务返回 Fixed 状态,那么无论之前的状态如何,都将状态设置为 Fixed 的状态
if let ExecutionStatus::Fixed(status) = result {
assert!(*status < 0b1000, "status should be less than 0b1000");
self.set_status(offset, *status);
} else if self.get_status(offset) < STATUS_MAX_RETRY {
@@ -176,7 +176,7 @@ pub type VideoStatus = Status<5>;
pub type PageStatus = Status<5>;
#[cfg(test)]
mod test {
mod tests {
use anyhow::anyhow;
use super::*;
@@ -201,9 +201,9 @@ mod test {
assert_eq!(status.should_run(), [false, false, false]);
assert!(status.get_completed());
status.update_status(&[
ExecutionStatus::FixedFailed(1, anyhow!("")),
ExecutionStatus::FixedFailed(4, anyhow!("")),
ExecutionStatus::FixedFailed(7, anyhow!("")),
ExecutionStatus::Fixed(1),
ExecutionStatus::Fixed(4),
ExecutionStatus::Fixed(7),
]);
assert_eq!(status.should_run(), [true, false, false]);
assert!(!status.get_completed());

View File

@@ -1,79 +0,0 @@
use std::sync::{Arc, LazyLock};
use serde::Serialize;
use tokio::sync::MutexGuard;
use crate::config::VersionedConfig;
pub static TASK_STATUS_NOTIFIER: LazyLock<TaskStatusNotifier> = LazyLock::new(TaskStatusNotifier::new);
#[derive(Serialize)]
pub struct TaskStatus {
is_running: bool,
last_run: Option<chrono::DateTime<chrono::Local>>,
last_finish: Option<chrono::DateTime<chrono::Local>>,
next_run: Option<chrono::DateTime<chrono::Local>>,
}
pub struct TaskStatusNotifier {
mutex: tokio::sync::Mutex<()>,
tx: tokio::sync::watch::Sender<Arc<TaskStatus>>,
rx: tokio::sync::watch::Receiver<Arc<TaskStatus>>,
}
impl Default for TaskStatus {
fn default() -> Self {
Self {
is_running: false,
last_run: None,
last_finish: None,
next_run: None,
}
}
}
impl TaskStatusNotifier {
pub fn new() -> Self {
let (tx, rx) = tokio::sync::watch::channel(Arc::new(TaskStatus::default()));
Self {
mutex: tokio::sync::Mutex::const_new(()),
tx,
rx,
}
}
pub async fn start_running(&self) -> MutexGuard<()> {
let lock = self.mutex.lock().await;
let _ = self.tx.send(Arc::new(TaskStatus {
is_running: true,
last_run: Some(chrono::Local::now()),
last_finish: None,
next_run: None,
}));
lock
}
pub fn finish_running(&self, _lock: MutexGuard<()>) {
let last_status = self.tx.borrow();
let last_run = last_status.last_run.clone();
drop(last_status);
let config = VersionedConfig::get().load();
let now = chrono::Local::now();
let _ = self.tx.send(Arc::new(TaskStatus {
is_running: false,
last_run,
last_finish: Some(now),
next_run: now.checked_add_signed(chrono::Duration::seconds(config.interval as i64)),
}));
}
/// 精确探测任务执行状态,保证如果读取到“未运行”,那么在锁释放之前任务不会被执行
pub fn detect_running(&self) -> Option<MutexGuard<'_, ()>> {
self.mutex.try_lock().ok()
}
pub fn subscribe(&self) -> tokio::sync::watch::Receiver<Arc<TaskStatus>> {
self.rx.clone()
}
}

View File

@@ -14,15 +14,17 @@ use tokio::sync::Semaphore;
use crate::adapter::{VideoSource, VideoSourceEnum};
use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Video, VideoInfo};
use crate::config::{ARGS, PathSafeTemplate, TEMPLATE, VersionedConfig};
use crate::config::{ARGS, Config, PathSafeTemplate};
use crate::downloader::Downloader;
use crate::error::{DownloadAbortError, ExecutionStatus, ProcessPageError};
use crate::error::ExecutionStatus;
use crate::utils::download_context::DownloadContext;
use crate::utils::format_arg::{page_format_args, video_format_args};
use crate::utils::model::{
create_pages, create_videos, filter_unfilled_videos, filter_unhandled_video_pages, update_pages_model,
update_videos_model,
};
use crate::utils::nfo::NFO;
use crate::utils::nfo::{NFO, ToNFO};
use crate::utils::rule::FieldEvaluatable;
use crate::utils::status::{PageStatus, STATUS_OK, VideoStatus};
/// 完整地处理某个视频来源
@@ -30,18 +32,24 @@ pub async fn process_video_source(
video_source: VideoSourceEnum,
bili_client: &BiliClient,
connection: &DatabaseConnection,
template: &handlebars::Handlebars<'_>,
config: &Config,
) -> Result<()> {
// 预创建视频源目录,提前检测目录是否可写
video_source.create_dir_all().await?;
// 从参数中获取视频列表的 Model 与视频流
let (video_source, video_streams) = video_source.refresh(bili_client, connection).await?;
let (video_source, video_streams) = video_source
.refresh(bili_client, &config.credential, connection)
.await?;
// 从视频流中获取新视频的简要信息,写入数据库
refresh_video_source(&video_source, video_streams, connection).await?;
// 单独请求视频详情接口,获取视频的详情信息与所有的分页,写入数据库
fetch_video_details(bili_client, &video_source, connection).await?;
fetch_video_details(bili_client, &video_source, connection, config).await?;
if ARGS.scan_only {
warn!("已开启仅扫描模式,跳过视频下载..");
} else {
// 从数据库中查找所有未下载的视频与分页,下载并处理
download_unprocessed_videos(bili_client, &video_source, connection).await?;
download_unprocessed_videos(bili_client, &video_source, connection, template, config).await?;
}
Ok(())
}
@@ -57,10 +65,18 @@ pub async fn refresh_video_source<'a>(
let mut max_datetime = latest_row_at;
let mut error = Ok(());
let mut video_streams = video_streams
.take_while(|res| {
.enumerate()
.take_while(|(idx, res)| {
match res {
Err(e) => {
error = Err(anyhow!(e.to_string()));
// 这里拿到的 e 是引用,无法直接传递所有权
// 对于 BiliError我们需要克隆内部的错误并附带原来的上下文方便外部检查错误类型
// 对于其他错误只保留字符串信息用作提示
if let Some(inner) = e.downcast_ref::<BiliError>() {
error = Err(inner.clone()).context(e.to_string());
} else {
error = Err(anyhow!("{:#}", e));
}
futures::future::ready(false)
}
Ok(v) => {
@@ -71,11 +87,11 @@ pub async fn refresh_video_source<'a>(
if release_datetime > &max_datetime {
max_datetime = *release_datetime;
}
futures::future::ready(video_source.should_take(release_datetime, &latest_row_at))
futures::future::ready(video_source.should_take(*idx, release_datetime, &latest_row_at))
}
}
})
.filter_map(|res| futures::future::ready(video_source.should_filter(res, &latest_row_at)))
.filter_map(|(idx, res)| futures::future::ready(video_source.should_filter(idx, res, &latest_row_at)))
.chunks(10);
let mut count = 0;
while let Some(videos_info) = video_streams.next().await {
@@ -99,46 +115,53 @@ pub async fn fetch_video_details(
bili_client: &BiliClient,
video_source: &VideoSourceEnum,
connection: &DatabaseConnection,
config: &Config,
) -> Result<()> {
video_source.log_fetch_video_start();
let videos_model = filter_unfilled_videos(video_source.filter_expr(), connection).await?;
let semaphore = Semaphore::new(config.concurrent_limit.video);
let semaphore_ref = &semaphore;
let tasks = videos_model
.into_iter()
.map(|video_model| {
async move {
let video = Video::new(bili_client, video_model.bvid.clone());
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Err(e) => {
error!(
"获取视频 {} - {} 的详细信息失败,错误为:{:#}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
.map(|video_model| async move {
let _permit = semaphore_ref.acquire().await.context("acquire semaphore failed")?;
let video = Video::new(bili_client, video_model.bvid.clone(), &config.credential);
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Err(e) => {
error!(
"获取视频 {} - {} 的详细信息失败,错误为:{:#}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::ErrorResponse(-404, _, _)) = e.downcast_ref::<BiliError>() {
let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
Ok((tags, mut view_info)) => {
let VideoInfo::Detail { pages, .. } = &mut view_info else {
unreachable!()
};
let pages = std::mem::take(pages);
let pages_len = pages.len();
let txn = connection.begin().await?;
// 将分页信息写入数据库
create_pages(pages, &video_model, &txn).await?;
let mut video_active_model = view_info.into_detail_model(video_model);
video_source.set_relation_id(&mut video_active_model);
video_active_model.single_page = Set(Some(pages_len == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
};
Ok::<_, anyhow::Error>(())
}
}
Ok((tags, mut view_info)) => {
let VideoInfo::Detail { pages, .. } = &mut view_info else {
unreachable!()
};
// 构造 page model
let pages = std::mem::take(pages);
let pages = pages
.into_iter()
.map(|p| p.into_active_model(video_model.id))
.collect::<Vec<page::ActiveModel>>();
// 更新 video model 的各项有关属性
let mut video_active_model = view_info.into_detail_model(video_model);
video_source.set_relation_id(&mut video_active_model);
video_active_model.single_page = Set(Some(pages.len() == 1));
video_active_model.tags = Set(Some(tags.into()));
video_active_model.should_download = Set(video_source.rule().evaluate(&video_active_model, &pages));
let txn = connection.begin().await?;
create_pages(pages, &txn).await?;
video_active_model.save(&txn).await?;
txn.commit().await?;
}
};
Ok::<_, anyhow::Error>(())
})
.collect::<FuturesUnordered<_>>();
tasks.try_collect::<Vec<_>>().await?;
@@ -151,10 +174,13 @@ pub async fn download_unprocessed_videos(
bili_client: &BiliClient,
video_source: &VideoSourceEnum,
connection: &DatabaseConnection,
template: &handlebars::Handlebars<'_>,
config: &Config,
) -> Result<()> {
video_source.log_download_video_start();
let semaphore = Semaphore::new(VersionedConfig::get().load().concurrent_limit.video);
let semaphore = Semaphore::new(config.concurrent_limit.video);
let downloader = Downloader::new(bili_client.client.clone());
let cx = DownloadContext::new(bili_client, video_source, template, connection, &downloader, config);
let unhandled_videos_pages = filter_unhandled_video_pages(video_source.filter_expr(), connection).await?;
let mut assigned_upper = HashSet::new();
let tasks = unhandled_videos_pages
@@ -162,29 +188,20 @@ pub async fn download_unprocessed_videos(
.map(|(video_model, pages_model)| {
let should_download_upper = !assigned_upper.contains(&video_model.upper_id);
assigned_upper.insert(video_model.upper_id);
download_video_pages(
bili_client,
video_source,
video_model,
pages_model,
connection,
&semaphore,
&downloader,
should_download_upper,
)
download_video_pages(video_model, pages_model, &semaphore, should_download_upper, cx)
})
.collect::<FuturesUnordered<_>>();
let mut download_aborted = false;
let mut risk_control_related_error = None;
let mut stream = tasks
// 触发风控时设置 download_aborted 标记并终止流
.take_while(|res| {
if res
.as_ref()
.is_err_and(|e| e.downcast_ref::<DownloadAbortError>().is_some())
if let Err(e) = res
&& let Some(e) = e.downcast_ref::<BiliError>()
&& e.is_risk_control_related()
{
download_aborted = true;
risk_control_related_error = Some(e.clone());
}
futures::future::ready(!download_aborted)
futures::future::ready(risk_control_related_error.is_none())
})
// 过滤掉没有触发风控的普通 Err只保留正确返回的 Model
.filter_map(|res| futures::future::ready(res.ok()))
@@ -193,35 +210,35 @@ pub async fn download_unprocessed_videos(
while let Some(models) = stream.next().await {
update_videos_model(models, connection).await?;
}
if download_aborted {
error!("下载触发风控,已终止所有任务,等待下一轮执行");
if let Some(e) = risk_control_related_error {
bail!(e);
}
video_source.log_download_video_end();
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn download_video_pages(
bili_client: &BiliClient,
video_source: &VideoSourceEnum,
video_model: video::Model,
pages: Vec<page::Model>,
connection: &DatabaseConnection,
page_models: Vec<page::Model>,
semaphore: &Semaphore,
downloader: &Downloader,
should_download_upper: bool,
cx: DownloadContext<'_>,
) -> Result<video::ActiveModel> {
let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
let mut status = VideoStatus::from(video_model.download_status);
let separate_status = status.should_run();
let base_path = video_source.path().join(
TEMPLATE
.load()
.path_safe_render("video", &video_format_args(&video_model))?,
);
// 未记录路径时填充,已经填充过路径时使用现有的
let base_path = if !video_model.path.is_empty() {
PathBuf::from(&video_model.path)
} else {
cx.video_source.path().join(
cx.template
.path_safe_render("video", &video_format_args(&video_model, &cx.config.time_format))?,
)
};
let upper_id = video_model.upper_id.to_string();
let base_upper_path = VersionedConfig::get()
.load()
let base_upper_path = cx
.config
.upper_path
.join(upper_id.chars().next().context("upper_id is empty")?.to_string())
.join(upper_id);
@@ -231,46 +248,37 @@ pub async fn download_video_pages(
let (res_1, res_2, res_3, res_4, res_5) = tokio::join!(
// 下载视频封面
fetch_video_poster(
separate_status[0] && !is_single_page,
separate_status[0] && !is_single_page && !cx.config.skip_option.no_poster,
&video_model,
downloader,
base_path.join("poster.jpg"),
base_path.join("fanart.jpg"),
cx
),
// 生成视频信息的 nfo
generate_video_nfo(
separate_status[1] && !is_single_page,
separate_status[1] && !is_single_page && !cx.config.skip_option.no_video_nfo,
&video_model,
base_path.join("tvshow.nfo"),
cx
),
// 下载 Up 主头像
fetch_upper_face(
separate_status[2] && should_download_upper,
separate_status[2] && should_download_upper && !cx.config.skip_option.no_upper,
&video_model,
downloader,
base_upper_path.join("folder.jpg"),
cx
),
// 生成 Up 主信息的 nfo
generate_upper_nfo(
separate_status[3] && should_download_upper,
separate_status[3] && should_download_upper && !cx.config.skip_option.no_upper,
&video_model,
base_upper_path.join("person.nfo"),
cx,
),
// 分发并执行分页下载的任务
dispatch_download_page(
separate_status[4],
bili_client,
&video_model,
pages,
connection,
downloader,
&base_path
)
dispatch_download_page(separate_status[4], &video_model, page_models, &base_path, cx)
);
let results = [res_1, res_2, res_3, res_4, res_5]
.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let results = [res_1.into(), res_2.into(), res_3.into(), res_4.into(), res_5.into()];
status.update_status(&results);
results
.iter()
@@ -281,17 +289,21 @@ pub async fn download_video_pages(
ExecutionStatus::Succeeded => info!("处理视频「{}」{}成功", &video_model.name, task_name),
ExecutionStatus::Ignored(e) => {
error!(
"处理视频「{}」{}出现常见错误,已忽略: {:#}",
"处理视频「{}」{}出现常见错误,已忽略{:#}",
&video_model.name, task_name, e
)
}
ExecutionStatus::Failed(e) | ExecutionStatus::FixedFailed(_, e) => {
error!("处理视频「{}」{}失败: {:#}", &video_model.name, task_name, e)
ExecutionStatus::Failed(e) => {
error!("处理视频「{}」{}失败{:#}", &video_model.name, task_name, e)
}
ExecutionStatus::Fixed(_) => unreachable!(),
});
if let ExecutionStatus::Failed(e) = results.into_iter().nth(4).context("page download result not found")? {
if e.downcast_ref::<DownloadAbortError>().is_some() {
return Err(e);
for result in results {
if let ExecutionStatus::Failed(e) = result
&& let Ok(e) = e.downcast::<BiliError>()
&& e.is_risk_control_related()
{
bail!(e);
}
}
let mut video_active_model: video::ActiveModel = video_model.into();
@@ -303,31 +315,20 @@ pub async fn download_video_pages(
/// 分发并执行分页下载任务,当且仅当所有分页成功下载或达到最大重试次数时返回 Ok否则根据失败原因返回对应的错误
pub async fn dispatch_download_page(
should_run: bool,
bili_client: &BiliClient,
video_model: &video::Model,
pages: Vec<page::Model>,
connection: &DatabaseConnection,
downloader: &Downloader,
page_models: Vec<page::Model>,
base_path: &Path,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
let child_semaphore = Semaphore::new(VersionedConfig::get().load().concurrent_limit.page);
let tasks = pages
let child_semaphore = Semaphore::new(cx.config.concurrent_limit.page);
let tasks = page_models
.into_iter()
.map(|page_model| {
download_page(
bili_client,
video_model,
page_model,
&child_semaphore,
downloader,
base_path,
)
})
.map(|page_model| download_page(video_model, page_model, &child_semaphore, base_path, cx))
.collect::<FuturesUnordered<_>>();
let (mut download_aborted, mut target_status) = (false, STATUS_OK);
let (mut risk_control_related_error, mut target_status) = (None, STATUS_OK);
let mut stream = tasks
.take_while(|res| {
match res {
@@ -343,45 +344,78 @@ pub async fn dispatch_download_page(
}
}
Err(e) => {
if e.downcast_ref::<DownloadAbortError>().is_some() {
download_aborted = true;
if let Some(e) = e.downcast_ref::<BiliError>()
&& e.is_risk_control_related()
{
risk_control_related_error = Some(e.clone());
}
}
}
// 仅在发生风控时终止流,其它情况继续执行
futures::future::ready(!download_aborted)
futures::future::ready(risk_control_related_error.is_none())
})
.filter_map(|res| futures::future::ready(res.ok()))
.chunks(10);
while let Some(models) = stream.next().await {
update_pages_model(models, connection).await?;
update_pages_model(models, cx.connection).await?;
}
if download_aborted {
error!("下载视频「{}」的分页时触发风控,将异常向上传递..", &video_model.name);
bail!(DownloadAbortError());
if let Some(e) = risk_control_related_error {
bail!(e);
}
if target_status != STATUS_OK {
return Ok(ExecutionStatus::FixedFailed(target_status, ProcessPageError().into()));
}
Ok(ExecutionStatus::Succeeded)
// 视频中“分页下载”任务的状态始终与所有分页的最小状态一致
Ok(ExecutionStatus::Fixed(target_status))
}
/// 下载某个分页,未发生风控且正常运行时返回 Ok(Page::ActiveModel),其中 status 字段存储了新的下载状态,发生风控时返回 DownloadAbortError
pub async fn download_page(
bili_client: &BiliClient,
video_model: &video::Model,
page_model: page::Model,
semaphore: &Semaphore,
downloader: &Downloader,
base_path: &Path,
cx: DownloadContext<'_>,
) -> Result<page::ActiveModel> {
let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
let mut status = PageStatus::from(page_model.download_status);
let separate_status = status.should_run();
let is_single_page = video_model.single_page.context("single_page is null")?;
let base_name = TEMPLATE
.load()
.path_safe_render("page", &page_format_args(video_model, &page_model))?;
// 未记录路径时填充,已经填充过路径时使用现有的
let (base_path, base_name) = if let Some(old_video_path) = &page_model.path
&& !old_video_path.is_empty()
{
let old_video_path = Path::new(old_video_path);
let old_video_filename = old_video_path
.file_name()
.context("invalid page path format")?
.to_string_lossy();
if is_single_page {
// 单页下的路径是 {base_path}/{base_name}.mp4
(
old_video_path.parent().context("invalid page path format")?,
old_video_filename.trim_end_matches(".mp4").to_string(),
)
} else {
// 多页下的路径是 {base_path}/Season 1/{base_name} - S01Exx.mp4
(
old_video_path
.parent()
.and_then(|p| p.parent())
.context("invalid page path format")?,
old_video_filename
.rsplit_once(" - ")
.context("invalid page path format")?
.0
.to_string(),
)
}
} else {
(
base_path,
cx.template.path_safe_render(
"page",
&page_format_args(video_model, &page_model, &cx.config.time_format),
)?,
)
};
let (poster_path, video_path, nfo_path, danmaku_path, fanart_path, subtitle_path) = if is_single_page {
(
base_path.join(format!("{}-poster.jpg", &base_name)),
@@ -429,33 +463,41 @@ pub async fn download_page(
let (res_1, res_2, res_3, res_4, res_5) = tokio::join!(
// 下载分页封面
fetch_page_poster(
separate_status[0],
separate_status[0] && !cx.config.skip_option.no_poster,
video_model,
&page_model,
downloader,
poster_path,
fanart_path
fanart_path,
cx
),
// 下载分页视频
fetch_page_video(
separate_status[1],
bili_client,
video_model,
downloader,
&page_info,
&video_path
),
fetch_page_video(separate_status[1], video_model, &page_info, &video_path, cx),
// 生成分页视频信息的 nfo
generate_page_nfo(separate_status[2], video_model, &page_model, nfo_path),
generate_page_nfo(
separate_status[2] && !cx.config.skip_option.no_video_nfo,
video_model,
&page_model,
nfo_path,
cx,
),
// 下载分页弹幕
fetch_page_danmaku(separate_status[3], bili_client, video_model, &page_info, danmaku_path),
fetch_page_danmaku(
separate_status[3] && !cx.config.skip_option.no_danmaku,
video_model,
&page_info,
danmaku_path,
cx,
),
// 下载分页字幕
fetch_page_subtitle(separate_status[4], bili_client, video_model, &page_info, &subtitle_path)
fetch_page_subtitle(
separate_status[4] && !cx.config.skip_option.no_subtitle,
video_model,
&page_info,
&subtitle_path,
cx
)
);
let results = [res_1, res_2, res_3, res_4, res_5]
.into_iter()
.map(Into::into)
.collect::<Vec<_>>();
let results = [res_1.into(), res_2.into(), res_3.into(), res_4.into(), res_5.into()];
status.update_status(&results);
results
.iter()
@@ -471,19 +513,22 @@ pub async fn download_page(
),
ExecutionStatus::Ignored(e) => {
error!(
"处理视频「{}」第 {} 页{}出现常见错误,已忽略: {:#}",
"处理视频「{}」第 {} 页{}出现常见错误,已忽略{:#}",
&video_model.name, page_model.pid, task_name, e
)
}
ExecutionStatus::Failed(e) | ExecutionStatus::FixedFailed(_, e) => error!(
"处理视频「{}」第 {} 页{}失败: {:#}",
ExecutionStatus::Failed(e) => error!(
"处理视频「{}」第 {} 页{}失败{:#}",
&video_model.name, page_model.pid, task_name, e
),
ExecutionStatus::Fixed(_) => unreachable!(),
});
// 如果下载视频时触发风控,直接返回 DownloadAbortError
if let ExecutionStatus::Failed(e) = results.into_iter().nth(1).context("video download result not found")? {
if let Ok(BiliError::RiskControlOccurred) = e.downcast::<BiliError>() {
bail!(DownloadAbortError());
for result in results {
if let ExecutionStatus::Failed(e) = result
&& let Ok(e) = e.downcast::<BiliError>()
&& e.is_risk_control_related()
{
bail!(e);
}
}
let mut page_active_model: page::ActiveModel = page_model.into();
@@ -496,9 +541,9 @@ pub async fn fetch_page_poster(
should_run: bool,
video_model: &video::Model,
page_model: &page::Model,
downloader: &Downloader,
poster_path: PathBuf,
fanart_path: Option<PathBuf>,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
@@ -514,7 +559,9 @@ pub async fn fetch_page_poster(
None => video_model.cover.as_str(),
}
};
downloader.fetch(url, &poster_path).await?;
cx.downloader
.fetch(url, &poster_path, &cx.config.concurrent_limit.download)
.await?;
if let Some(fanart_path) = fanart_path {
fs::copy(&poster_path, &fanart_path).await?;
}
@@ -523,47 +570,53 @@ pub async fn fetch_page_poster(
pub async fn fetch_page_video(
should_run: bool,
bili_client: &BiliClient,
video_model: &video::Model,
downloader: &Downloader,
page_info: &PageInfo,
page_path: &Path,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
let bili_video = Video::new(bili_client, video_model.bvid.clone());
let bili_video = Video::new(cx.bili_client, video_model.bvid.clone(), &cx.config.credential);
let streams = bili_video
.get_page_analyzer(page_info)
.await?
.best_stream(&VersionedConfig::get().load().filter_option)?;
.best_stream(&cx.config.filter_option)?;
match streams {
BestStream::Mixed(mix_stream) => downloader.fetch_with_fallback(&mix_stream.urls(), page_path).await?,
BestStream::Mixed(mix_stream) => {
cx.downloader
.multi_fetch(
&mix_stream.urls(cx.config.cdn_sorting),
page_path,
&cx.config.concurrent_limit.download,
)
.await?
}
BestStream::VideoAudio {
video: video_stream,
audio: None,
} => downloader.fetch_with_fallback(&video_stream.urls(), page_path).await?,
} => {
cx.downloader
.multi_fetch(
&video_stream.urls(cx.config.cdn_sorting),
page_path,
&cx.config.concurrent_limit.download,
)
.await?
}
BestStream::VideoAudio {
video: video_stream,
audio: Some(audio_stream),
} => {
let (tmp_video_path, tmp_audio_path) = (
page_path.with_extension("tmp_video"),
page_path.with_extension("tmp_audio"),
);
let res = async {
downloader
.fetch_with_fallback(&video_stream.urls(), &tmp_video_path)
.await?;
downloader
.fetch_with_fallback(&audio_stream.urls(), &tmp_audio_path)
.await?;
downloader.merge(&tmp_video_path, &tmp_audio_path, page_path).await
}
.await;
let _ = fs::remove_file(tmp_video_path).await;
let _ = fs::remove_file(tmp_audio_path).await;
res?
cx.downloader
.multi_fetch_and_merge(
&video_stream.urls(cx.config.cdn_sorting),
&audio_stream.urls(cx.config.cdn_sorting),
page_path,
&cx.config.concurrent_limit.download,
)
.await?
}
}
Ok(ExecutionStatus::Succeeded)
@@ -571,34 +624,34 @@ pub async fn fetch_page_video(
pub async fn fetch_page_danmaku(
should_run: bool,
bili_client: &BiliClient,
video_model: &video::Model,
page_info: &PageInfo,
danmaku_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
let bili_video = Video::new(bili_client, video_model.bvid.clone());
let bili_video = Video::new(cx.bili_client, video_model.bvid.clone(), &cx.config.credential);
bili_video
.get_danmaku_writer(page_info)
.await?
.write(danmaku_path)
.write(danmaku_path, &cx.config.danmaku_option)
.await?;
Ok(ExecutionStatus::Succeeded)
}
pub async fn fetch_page_subtitle(
should_run: bool,
bili_client: &BiliClient,
video_model: &video::Model,
page_info: &PageInfo,
subtitle_path: &Path,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
let bili_video = Video::new(bili_client, video_model.bvid.clone());
let bili_video = Video::new(cx.bili_client, video_model.bvid.clone(), &cx.config.credential);
let subtitles = bili_video.get_subtitles(page_info).await?;
let tasks = subtitles
.into_iter()
@@ -616,15 +669,16 @@ pub async fn generate_page_nfo(
video_model: &video::Model,
page_model: &page::Model,
nfo_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
let single_page = video_model.single_page.context("single_page is null")?;
let nfo = if single_page {
NFO::Movie(video_model.into())
NFO::Movie(video_model.to_nfo(cx.config.nfo_time_type))
} else {
NFO::Episode(page_model.into())
NFO::Episode(page_model.to_nfo(cx.config.nfo_time_type))
};
generate_nfo(nfo, nfo_path).await?;
Ok(ExecutionStatus::Succeeded)
@@ -633,14 +687,16 @@ pub async fn generate_page_nfo(
pub async fn fetch_video_poster(
should_run: bool,
video_model: &video::Model,
downloader: &Downloader,
poster_path: PathBuf,
fanart_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
downloader.fetch(&video_model.cover, &poster_path).await?;
cx.downloader
.fetch(&video_model.cover, &poster_path, &cx.config.concurrent_limit.download)
.await?;
fs::copy(&poster_path, &fanart_path).await?;
Ok(ExecutionStatus::Succeeded)
}
@@ -648,13 +704,19 @@ pub async fn fetch_video_poster(
pub async fn fetch_upper_face(
should_run: bool,
video_model: &video::Model,
downloader: &Downloader,
upper_face_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
downloader.fetch(&video_model.upper_face, &upper_face_path).await?;
cx.downloader
.fetch(
&video_model.upper_face,
&upper_face_path,
&cx.config.concurrent_limit.download,
)
.await?;
Ok(ExecutionStatus::Succeeded)
}
@@ -662,11 +724,12 @@ pub async fn generate_upper_nfo(
should_run: bool,
video_model: &video::Model,
nfo_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
generate_nfo(NFO::Upper(video_model.into()), nfo_path).await?;
generate_nfo(NFO::Upper(video_model.to_nfo(cx.config.nfo_time_type)), nfo_path).await?;
Ok(ExecutionStatus::Succeeded)
}
@@ -674,11 +737,12 @@ pub async fn generate_video_nfo(
should_run: bool,
video_model: &video::Model,
nfo_path: PathBuf,
cx: DownloadContext<'_>,
) -> Result<ExecutionStatus> {
if !should_run {
return Ok(ExecutionStatus::Skipped);
}
generate_nfo(NFO::TVShow(video_model.into()), nfo_path).await?;
generate_nfo(NFO::TVShow(video_model.to_nfo(cx.config.nfo_time_type)), nfo_path).await?;
Ok(ExecutionStatus::Succeeded)
}

View File

@@ -5,5 +5,8 @@ edition = { workspace = true }
publish = { workspace = true }
[dependencies]
derivative = { workspace = true }
sea-orm = { workspace = true }
regex = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }

View File

@@ -0,0 +1,2 @@
pub mod rule;
pub mod string_vec;

View File

@@ -0,0 +1,120 @@
use std::fmt::Display;
use derivative::Derivative;
use sea_orm::FromJsonQueryResult;
use sea_orm::prelude::DateTime;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
#[derive(Clone, Debug, Serialize, Deserialize, Derivative)]
#[derivative(PartialEq, Eq)]
#[serde(rename_all = "camelCase", tag = "operator", content = "value")]
pub enum Condition<T: Serialize + Display> {
Equals(T),
Contains(T),
#[serde(deserialize_with = "deserialize_regex", serialize_with = "serialize_regex")]
MatchesRegex(String, #[derivative(PartialEq = "ignore")] regex::Regex),
Prefix(T),
Suffix(T),
GreaterThan(T),
LessThan(T),
Between(T, T),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)]
#[serde(rename_all = "camelCase", tag = "field", content = "rule")]
pub enum RuleTarget {
Title(Condition<String>),
Tags(Condition<String>),
FavTime(Condition<DateTime>),
PubTime(Condition<DateTime>),
PageCount(Condition<usize>),
Not(Box<RuleTarget>),
}
pub type AndGroup = Vec<RuleTarget>;
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)]
pub struct Rule(pub Vec<AndGroup>);
impl<T: Serialize + Display> Display for Condition<T> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Condition::Equals(v) => write!(f, "等于“{}”", v),
Condition::Contains(v) => write!(f, "包含“{}”", v),
Condition::MatchesRegex(pat, _) => write!(f, "匹配“{}”", pat),
Condition::Prefix(v) => write!(f, "以“{}”开头", v),
Condition::Suffix(v) => write!(f, "以“{}”结尾", v),
Condition::GreaterThan(v) => write!(f, "大于“{}”", v),
Condition::LessThan(v) => write!(f, "小于“{}”", v),
Condition::Between(start, end) => write!(f, "在“{}”和“{}”之间", start, end),
}
}
}
impl Display for RuleTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
fn get_field_name(rt: &RuleTarget, depth: usize) -> &'static str {
match rt {
RuleTarget::Title(_) => "标题",
RuleTarget::Tags(_) => "标签",
RuleTarget::FavTime(_) => "收藏时间",
RuleTarget::PubTime(_) => "发布时间",
RuleTarget::PageCount(_) => "视频分页数量",
RuleTarget::Not(inner) => {
if depth == 0 {
get_field_name(inner, depth + 1)
} else {
"格式化失败"
}
}
}
}
let field_name = get_field_name(self, 0);
match self {
RuleTarget::Not(inner) => match inner.as_ref() {
RuleTarget::Title(cond) | RuleTarget::Tags(cond) => write!(f, "{}不{}", field_name, cond),
RuleTarget::FavTime(cond) | RuleTarget::PubTime(cond) => {
write!(f, "{}不{}", field_name, cond)
}
RuleTarget::PageCount(cond) => write!(f, "{}不{}", field_name, cond),
RuleTarget::Not(_) => write!(f, "格式化失败"),
},
RuleTarget::Title(cond) | RuleTarget::Tags(cond) => write!(f, "{}{}", field_name, cond),
RuleTarget::FavTime(cond) | RuleTarget::PubTime(cond) => {
write!(f, "{}{}", field_name, cond)
}
RuleTarget::PageCount(cond) => write!(f, "{}{}", field_name, cond),
}
}
}
impl Display for Rule {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let groups: Vec<String> = self
.0
.iter()
.map(|group| {
let conditions: Vec<String> = group.iter().map(|target| format!("{}", target)).collect();
format!("{}", conditions.join(""))
})
.collect();
write!(f, "{}", groups.join(""))
}
}
fn deserialize_regex<'de, D>(deserializer: D) -> Result<(String, regex::Regex), D::Error>
where
D: Deserializer<'de>,
{
let pattern = String::deserialize(deserializer)?;
// 反序列化时预编译 regex优化性能
let regex = regex::Regex::new(&pattern).map_err(serde::de::Error::custom)?;
Ok((pattern, regex))
}
fn serialize_regex<S>(pattern: &str, _regex: &regex::Regex, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(pattern)
}

View File

@@ -0,0 +1,20 @@
use sea_orm::FromJsonQueryResult;
use serde::{Deserialize, Serialize};
// reference: https://www.sea-ql.org/SeaORM/docs/generate-entity/column-types/#json-column
// 在 entity 中使用裸 Vec 仅在 postgres 中支持sea-orm 会将其映射为 postgres array
// 如果需要实现跨数据库的 array必须将其包裹在 wrapper type 中
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, FromJsonQueryResult)]
pub struct StringVec(pub Vec<String>);
impl From<Vec<String>> for StringVec {
fn from(value: Vec<String>) -> Self {
Self(value)
}
}
impl From<StringVec> for Vec<String> {
fn from(value: StringVec) -> Self {
value.0
}
}

View File

@@ -2,6 +2,8 @@
use sea_orm::entity::prelude::*;
use crate::rule::Rule;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "collection")]
pub struct Model {
@@ -14,6 +16,7 @@ pub struct Model {
pub path: String,
pub created_at: String,
pub latest_row_at: DateTime,
pub rule: Option<Rule>,
pub enabled: bool,
}

View File

@@ -2,6 +2,8 @@
use sea_orm::entity::prelude::*;
use crate::rule::Rule;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "favorite")]
pub struct Model {
@@ -13,6 +15,7 @@ pub struct Model {
pub path: String,
pub created_at: String,
pub latest_row_at: DateTime,
pub rule: Option<Rule>,
pub enabled: bool,
}

View File

@@ -2,6 +2,8 @@
use sea_orm::entity::prelude::*;
use crate::rule::Rule;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "submission")]
pub struct Model {
@@ -11,7 +13,9 @@ pub struct Model {
pub upper_name: String,
pub path: String,
pub created_at: String,
pub use_dynamic_api: bool,
pub latest_row_at: DateTime,
pub rule: Option<Rule>,
pub enabled: bool,
}

View File

@@ -2,6 +2,8 @@
use sea_orm::entity::prelude::*;
use crate::string_vec::StringVec;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
#[sea_orm(table_name = "video")]
pub struct Model {
@@ -25,7 +27,8 @@ pub struct Model {
pub favtime: DateTime,
pub download_status: u32,
pub valid: bool,
pub tags: Option<serde_json::Value>,
pub should_download: bool,
pub tags: Option<StringVec>,
pub single_page: Option<bool>,
pub created_at: String,
}

View File

@@ -2,6 +2,8 @@
use sea_orm::entity::prelude::*;
use crate::rule::Rule;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "watch_later")]
pub struct Model {
@@ -10,6 +12,7 @@ pub struct Model {
pub path: String,
pub created_at: String,
pub latest_row_at: DateTime,
pub rule: Option<Rule>,
pub enabled: bool,
}

View File

@@ -1,2 +1,5 @@
mod custom_type;
mod entities;
pub use custom_type::*;
pub use entities::*;

View File

@@ -5,5 +5,4 @@ edition = { workspace = true }
publish = { workspace = true }
[dependencies]
async-std = { workspace = true }
sea-orm-migration = { workspace = true }

View File

@@ -8,6 +8,8 @@ mod m20250122_062926_add_latest_row_at;
mod m20250612_090826_add_enabled;
mod m20250613_043257_add_config;
mod m20250712_080013_add_video_created_at_index;
mod m20250903_094454_add_rule_and_should_download;
mod m20251009_123713_add_use_dynamic_api;
pub struct Migrator;
@@ -23,6 +25,8 @@ impl MigratorTrait for Migrator {
Box::new(m20250612_090826_add_enabled::Migration),
Box::new(m20250613_043257_add_config::Migration),
Box::new(m20250712_080013_add_video_created_at_index::Migration),
Box::new(m20250903_094454_add_rule_and_should_download::Migration),
Box::new(m20251009_123713_add_use_dynamic_api::Migration),
]
}
}

View File

@@ -0,0 +1,124 @@
use sea_orm_migration::prelude::*;
use sea_orm_migration::schema::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Video::Table)
.add_column(boolean(Video::ShouldDownload).default(true))
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(WatchLater::Table)
.add_column(text_null(WatchLater::Rule))
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Submission::Table)
.add_column(text_null(Submission::Rule))
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Favorite::Table)
.add_column(text_null(Favorite::Rule))
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Collection::Table)
.add_column(text_null(Collection::Rule))
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Video::Table)
.drop_column(Video::ShouldDownload)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(WatchLater::Table)
.drop_column(WatchLater::Rule)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Submission::Table)
.drop_column(Submission::Rule)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Favorite::Table)
.drop_column(Favorite::Rule)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Collection::Table)
.drop_column(Collection::Rule)
.to_owned(),
)
.await
}
}
#[derive(DeriveIden)]
enum Video {
Table,
ShouldDownload,
}
#[derive(DeriveIden)]
enum WatchLater {
Table,
Rule,
}
#[derive(DeriveIden)]
enum Submission {
Table,
Rule,
}
#[derive(DeriveIden)]
enum Favorite {
Table,
Rule,
}
#[derive(DeriveIden)]
enum Collection {
Table,
Rule,
}

View File

@@ -0,0 +1,36 @@
use sea_orm_migration::prelude::*;
use sea_orm_migration::schema::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Submission::Table)
.add_column(boolean(Submission::UseDynamicApi).default(false))
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Submission::Table)
.drop_column(Submission::UseDynamicApi)
.to_owned(),
)
.await
}
}
#[derive(DeriveIden)]
enum Submission {
Table,
UseDynamicApi,
}

View File

@@ -1,6 +0,0 @@
use sea_orm_migration::prelude::*;
#[async_std::main]
async fn main() {
cli::run_cli(bili_sync_migration::Migrator).await;
}

View File

@@ -21,7 +21,7 @@ export default defineConfig({
nav: [
{ text: "主页", link: "/" },
{
text: "v2.6.2",
text: "v2.8.0",
items: [
{
text: "程序更新",

View File

@@ -32,7 +32,7 @@
EMBY 的一般结构是: `媒体库 - 文件夹 - 电影/电视剧 - 分季/分集`,方便起见,我采用了如下的对应关系:
1. **文件夹**:对应 b 站的 video source
2. **电视剧** 对应 b 站的 video
2. **电视剧**:对应 b 站的 video
3. **第一季的所有分集**:对应 b 站的 page。
特别的,当 video 仅有一个 page 时为了避免过多的层级bili-sync 会将 page 展开到第二层级,变成与电视剧同级的电影。

View File

@@ -1,7 +1,7 @@
# bili-sync 是什么?
> [!TIP]
> 当前最新程序版本为 v2.6.2,文档将始终与最新程序版本保持一致。
> 当前最新程序版本为 v2.8.0,文档将始终与最新程序版本保持一致。
bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具。

View File

@@ -13,7 +13,7 @@
在[程序发布页](https://github.com/amtoaer/bili-sync/releases)选择最新版本中对应机器架构的压缩包,解压后会获取一个名为 `bili-sync-rs` 的可执行文件,直接双击执行。
### 其二: 使用 Docker Compose 运行
### 其二:使用 Docker Compose 运行
Linux/amd64 与 Linux/arm64 两个平台可直接使用 Docker 或 Docker Compose 运行,此处以 Compose 为例:
> 请注意其中的注释,有不清楚的地方可以先继续往下看。
@@ -88,9 +88,9 @@ Jul 12 16:11:10 INFO 开始运行管理页: http://0.0.0.0:12345
认证后会看到一系列的配置,除绑定地址外的选项**基本都会实时生效**。为避免意料外的情况,建议将配置文件一次修改完毕后再点击保存。
如无特殊需求一般仅需修改“B站认证”与“视频质量”两个标签下的配置。
如无特殊需求一般仅需修改“B 站认证”与“视频质量”两个标签下的配置。
其中“B站认证”在一次填写后即可忽略程序会在**每日第一次运行视频下载任务**时检查认证状态,并在有必要时自动刷新。
其中“B 站认证”在一次填写后即可忽略,程序会在**每日第一次运行视频下载任务**时检查认证状态,并在有必要时自动刷新。
对于这些设置项的含义,请参考[配置说明](./configuration.md),可善用右侧导航在不同配置项间跳转。
@@ -98,7 +98,7 @@ Jul 12 16:11:10 INFO 开始运行管理页: http://0.0.0.0:12345
配置完毕后,我们便可以随时添加视频源订阅。
用户在正确填写“B站认证”后可以在“快捷订阅”部分查看自己创建的收藏夹、关注的合集与 UP 主一键订阅,也可以在“视频源”页手动添加并管理。
用户在正确填写“B 站认证”后可以在“快捷订阅”部分查看自己创建的收藏夹、关注的合集与 UP 主一键订阅,也可以在“视频源”页手动添加并管理。
对于手动添加的视频源,可参考如下页面获取所需的参数:

View File

@@ -15,7 +15,7 @@ from pathlib import Path
def main():
if len(sys.argv) <= 1:
print("用法: python 2.0.3_add_fanart.py <path1> <path2> ...")
print("用法python 2.0.3_add_fanart.py <path1> <path2> ...")
exit(1)
paths = [Path(path) for path in sys.argv[1:]]
for path in paths:

View File

@@ -7,7 +7,7 @@
"@eslint/compat": "^1.2.5",
"@eslint/js": "^9.18.0",
"@internationalized/date": "^3.8.1",
"@lucide/svelte": "^0.525.0",
"@lucide/svelte": "^0.544.0",
"@sveltejs/adapter-static": "^3.0.8",
"@sveltejs/kit": "2.22.2",
"@sveltejs/vite-plugin-svelte": "^6.0.0",
@@ -16,7 +16,7 @@
"@tailwindcss/vite": "^4.0.0",
"@types/d3-scale": "^4.0.9",
"@types/d3-shape": "^3.1.7",
"bits-ui": "^2.8.6",
"bits-ui": "^2.11.0",
"clsx": "^2.1.1",
"eslint": "^9.18.0",
"eslint-config-prettier": "^10.0.1",
@@ -155,7 +155,7 @@
"@layerstack/utils": ["@layerstack/utils@2.0.0-next.12", "", { "dependencies": { "d3-array": "^3.2.4", "d3-time": "^3.1.0", "d3-time-format": "^4.1.0", "lodash-es": "^4.17.21" } }, "sha512-fhGZUlSr3N+D44BYm37WKMGSEFyZBW+dwIqtGU8Cl54mR4TLQ/UwyGhdpgIHyH/x/8q1abE0fP0Dn6ZsrDE3BA=="],
"@lucide/svelte": ["@lucide/svelte@0.525.0", "", { "peerDependencies": { "svelte": "^5" } }, "sha512-dyUxkXzepagLUzL8jHQNdeH286nC66ClLACsg+Neu/bjkRJWPWMzkT+H0DKlE70QdkicGCfs1ZGmXCc351hmZA=="],
"@lucide/svelte": ["@lucide/svelte@0.544.0", "", { "peerDependencies": { "svelte": "^5" } }, "sha512-9f9O6uxng2pLB01sxNySHduJN3HTl5p0HDu4H26VR51vhZfiMzyOMe9Mhof3XAk4l813eTtl+/DYRvGyoRR+yw=="],
"@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="],
@@ -301,7 +301,7 @@
"balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="],
"bits-ui": ["bits-ui@2.8.10", "", { "dependencies": { "@floating-ui/core": "^1.7.1", "@floating-ui/dom": "^1.7.1", "esm-env": "^1.1.2", "runed": "^0.29.1", "svelte-toolbelt": "^0.9.3", "tabbable": "^6.2.0" }, "peerDependencies": { "@internationalized/date": "^3.8.1", "svelte": "^5.33.0" } }, "sha512-MOobkqapDZNrpcNmeL2g664xFmH4tZBOKBTxFmsQYMZQuybSZHQnPXy+AjM5XZEXRmCFx5+XRmo6+fC3vHh1hQ=="],
"bits-ui": ["bits-ui@2.11.0", "", { "dependencies": { "@floating-ui/core": "^1.7.1", "@floating-ui/dom": "^1.7.1", "esm-env": "^1.1.2", "runed": "^0.31.1", "svelte-toolbelt": "^0.10.4", "tabbable": "^6.2.0" }, "peerDependencies": { "@internationalized/date": "^3.8.1", "svelte": "^5.33.0" } }, "sha512-j/lOFHz6ZDWwj9sOUb6zYSJQdvPc7kr1IRyAdPjln4wOw9UVvKCosbRFEyP4JEzvNFX7HksMG4naDrDHta5bSA=="],
"brace-expansion": ["brace-expansion@1.1.11", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA=="],
@@ -615,7 +615,7 @@
"run-parallel": ["run-parallel@1.2.0", "", { "dependencies": { "queue-microtask": "^1.2.2" } }, "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA=="],
"runed": ["runed@0.29.1", "", { "dependencies": { "esm-env": "^1.0.0" }, "peerDependencies": { "svelte": "^5.7.0" } }, "sha512-RGQEB8ZiWv4OvzBJhbMj2hMgRM8QrEptzTrDr7TDfkHaRePKjiUka4vJ9QHGY+8s87KymNvFoZAxFdQ4jtZNcA=="],
"runed": ["runed@0.31.1", "", { "dependencies": { "esm-env": "^1.0.0" }, "peerDependencies": { "svelte": "^5.7.0" } }, "sha512-v3czcTnO+EJjiPvD4dwIqfTdHLZ8oH0zJheKqAHh9QMViY7Qb29UlAMRpX7ZtHh7AFqV60KmfxaJ9QMy+L1igQ=="],
"rw": ["rw@1.3.3", "", {}, "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ=="],
@@ -649,7 +649,7 @@
"svelte-sonner": ["svelte-sonner@1.0.4", "", { "dependencies": { "runed": "^0.26.0" }, "peerDependencies": { "svelte": "^5.0.0" } }, "sha512-ctm9jeV0Rf3im2J6RU1emccrJFjRSdNSPsLlxaF62TLZw9bB1D40U/U7+wqEgohJY/X7FBdghdj0BFQF/IqKPQ=="],
"svelte-toolbelt": ["svelte-toolbelt@0.9.3", "", { "dependencies": { "clsx": "^2.1.1", "runed": "^0.29.0", "style-to-object": "^1.0.8" }, "peerDependencies": { "svelte": "^5.30.2" } }, "sha512-HCSWxCtVmv+c6g1ACb8LTwHVbDqLKJvHpo6J8TaqwUme2hj9ATJCpjCPNISR1OCq2Q4U1KT41if9ON0isINQZw=="],
"svelte-toolbelt": ["svelte-toolbelt@0.10.5", "", { "dependencies": { "clsx": "^2.1.1", "runed": "^0.29.0", "style-to-object": "^1.0.8" }, "peerDependencies": { "svelte": "^5.30.2" } }, "sha512-8e+eWTgxw1aiLxhDE8Rb1X6AoLitqpJz+WhAul2W7W58C8KoLoJQf1TgQdFPBiCPJ0Jg5y0Zi1uyua9em4VS0w=="],
"tabbable": ["tabbable@6.2.0", "", {}, "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="],
@@ -745,6 +745,8 @@
"svelte-sonner/runed": ["runed@0.26.0", "", { "dependencies": { "esm-env": "^1.0.0" }, "peerDependencies": { "svelte": "^5.7.0" } }, "sha512-qWFv0cvLVRd8pdl/AslqzvtQyEn5KaIugEernwg9G98uJVSZcs/ygvPBvF80LA46V8pwRvSKnaVLDI3+i2wubw=="],
"svelte-toolbelt/runed": ["runed@0.29.1", "", { "dependencies": { "esm-env": "^1.0.0" }, "peerDependencies": { "svelte": "^5.7.0" } }, "sha512-RGQEB8ZiWv4OvzBJhbMj2hMgRM8QrEptzTrDr7TDfkHaRePKjiUka4vJ9QHGY+8s87KymNvFoZAxFdQ4jtZNcA=="],
"tailwind-variants/tailwind-merge": ["tailwind-merge@3.0.2", "", {}, "sha512-l7z+OYZ7mu3DTqrL88RiKrKIqO3NcpEO8V/Od04bNpvk0kiIFndGEoqfuzvj4yuhRkHKjRkII2z+KS2HfPcSxw=="],
"vite/fdir": ["fdir@6.4.6", "", { "peerDependencies": { "picomatch": "^3 || ^4" }, "optionalPeers": ["picomatch"] }, "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w=="],

View File

@@ -1,11 +1,11 @@
{
"name": "bili-sync-web",
"version": "2.6.2",
"version": "2.8.0",
"devDependencies": {
"@eslint/compat": "^1.2.5",
"@eslint/js": "^9.18.0",
"@internationalized/date": "^3.8.1",
"@lucide/svelte": "^0.525.0",
"@lucide/svelte": "^0.544.0",
"@sveltejs/adapter-static": "^3.0.8",
"@sveltejs/kit": "2.22.2",
"@sveltejs/vite-plugin-svelte": "^6.0.0",
@@ -14,7 +14,7 @@
"@tailwindcss/vite": "^4.0.0",
"@types/d3-scale": "^4.0.9",
"@types/d3-shape": "^3.1.7",
"bits-ui": "^2.8.6",
"bits-ui": "^2.11.0",
"clsx": "^2.1.1",
"eslint": "^9.18.0",
"eslint-config-prettier": "^10.0.1",

View File

@@ -4,6 +4,16 @@
@custom-variant dark (&:is(.dark *));
@layer utilities {
.no-scrollbar::-webkit-scrollbar {
display: none;
}
.no-scrollbar {
-ms-overflow-style: none;
scrollbar-width: none;
}
}
html {
scroll-behavior: smooth !important;
}

View File

@@ -4,6 +4,7 @@
<meta charset="utf-8" />
<link rel="icon" href="%sveltekit.assets%/favicon.png" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="referrer" content="no-referrer" />
%sveltekit.head%
</head>
<body data-sveltekit-preload-data="hover">

View File

@@ -21,7 +21,8 @@ import type {
DashBoardResponse,
SysInfo,
TaskStatus,
ResetRequest
ResetRequest,
UpdateVideoSourceResponse
} from './types';
import { wsManager } from './ws';
@@ -59,7 +60,7 @@ class ApiClient {
clearAuthToken() {
delete this.defaultHeaders['Authorization'];
localStorage.removeItem('authToken');
// 断开WebSocket连接因为token已经无效
// 断开 WebSocket 连接,因为 token 已经无效
wsManager.disconnect();
}
@@ -212,8 +213,20 @@ class ApiClient {
type: string,
id: number,
request: UpdateVideoSourceRequest
): Promise<ApiResponse<boolean>> {
return this.put<boolean>(`/video-sources/${type}/${id}`, request);
): Promise<ApiResponse<UpdateVideoSourceResponse>> {
return this.put<UpdateVideoSourceResponse>(`/video-sources/${type}/${id}`, request);
}
async removeVideoSource(type: string, id: number): Promise<ApiResponse<boolean>> {
return this.request<boolean>(`/video-sources/${type}/${id}`, 'DELETE');
}
async evaluateVideoSourceRules(type: string, id: number): Promise<ApiResponse<boolean>> {
return this.post<boolean>(`/video-sources/${type}/${id}/evaluate`, null);
}
async getDefaultPath(type: string, name: string): Promise<ApiResponse<string>> {
return this.get<string>(`/video-sources/${type}/default-path`, { name });
}
async getConfig(): Promise<ApiResponse<Config>> {
@@ -227,6 +240,11 @@ class ApiClient {
async getDashboard(): Promise<ApiResponse<DashBoardResponse>> {
return this.get<DashBoardResponse>('/dashboard');
}
async triggerDownloadTask(): Promise<ApiResponse<boolean>> {
return this.post<boolean>('/task/download');
}
subscribeToLogs(onMessage: (data: string) => void) {
return wsManager.subscribeToLogs(onMessage);
}
@@ -261,9 +279,14 @@ const api = {
getVideoSourcesDetails: () => apiClient.getVideoSourcesDetails(),
updateVideoSource: (type: string, id: number, request: UpdateVideoSourceRequest) =>
apiClient.updateVideoSource(type, id, request),
removeVideoSource: (type: string, id: number) => apiClient.removeVideoSource(type, id),
evaluateVideoSourceRules: (type: string, id: number) =>
apiClient.evaluateVideoSourceRules(type, id),
getDefaultPath: (type: string, name: string) => apiClient.getDefaultPath(type, name),
getConfig: () => apiClient.getConfig(),
updateConfig: (config: Config) => apiClient.updateConfig(config),
getDashboard: () => apiClient.getDashboard(),
triggerDownloadTask: () => apiClient.triggerDownloadTask(),
subscribeToSysInfo: (onMessage: (data: SysInfo) => void) =>
apiClient.subscribeToSysInfo(onMessage),

View File

@@ -0,0 +1,479 @@
<script lang="ts">
import { Button } from '$lib/components/ui/button/index.js';
import { Input } from '$lib/components/ui/input/index.js';
import { Label } from '$lib/components/ui/label/index.js';
import { Checkbox } from '$lib/components/ui/checkbox/index.js';
import * as Card from '$lib/components/ui/card/index.js';
import { Badge } from '$lib/components/ui/badge/index.js';
import PlusIcon from '@lucide/svelte/icons/plus';
import MinusIcon from '@lucide/svelte/icons/minus';
import XIcon from '@lucide/svelte/icons/x';
import type { Rule, RuleTarget, Condition } from '$lib/types';
import { onMount } from 'svelte';
interface Props {
rule: Rule | null;
onRuleChange: (rule: Rule | null) => void;
}
let { rule, onRuleChange }: Props = $props();
const FIELD_OPTIONS = [
{ value: 'title', label: '标题' },
{ value: 'tags', label: '标签' },
{ value: 'favTime', label: '收藏时间' },
{ value: 'pubTime', label: '发布时间' },
{ value: 'pageCount', label: '视频分页数量' }
];
const getOperatorOptions = (field: string) => {
switch (field) {
case 'title':
case 'tags':
return [
{ value: 'equals', label: '等于' },
{ value: 'contains', label: '包含' },
{ value: 'prefix', label: '以...开头' },
{ value: 'suffix', label: '以...结尾' },
{ value: 'matchesRegex', label: '匹配正则' }
];
case 'pageCount':
return [
{ value: 'equals', label: '等于' },
{ value: 'greaterThan', label: '大于' },
{ value: 'lessThan', label: '小于' },
{ value: 'between', label: '范围' }
];
case 'favTime':
case 'pubTime':
return [
{ value: 'equals', label: '等于' },
{ value: 'greaterThan', label: '晚于' },
{ value: 'lessThan', label: '早于' },
{ value: 'between', label: '时间范围' }
];
default:
return [];
}
};
interface LocalCondition {
field: string;
operator: string;
value: string;
value2?: string;
isNot: boolean;
}
interface LocalAndGroup {
conditions: LocalCondition[];
}
let localRule: LocalAndGroup[] = $state([]);
onMount(() => {
if (rule && rule.length > 0) {
localRule = rule.map((andGroup) => ({
conditions: andGroup.map((target) => convertRuleTargetToLocal(target))
}));
} else {
localRule = [];
}
});
function convertRuleTargetToLocal(target: RuleTarget<string | number | Date>): LocalCondition {
if (typeof target.rule === 'object' && 'field' in target.rule) {
// 嵌套的 not
const innerCondition = convertRuleTargetToLocal(target.rule);
return {
...innerCondition,
isNot: true
};
}
const condition = target.rule as Condition<string | number | Date>;
let value = '';
let value2 = '';
if (Array.isArray(condition.value)) {
value = String(condition.value[0] || '');
value2 = String(condition.value[1] || '');
} else {
value = String(condition.value || '');
}
return {
field: target.field,
operator: condition.operator,
value,
value2,
isNot: false
};
}
function convertLocalToRule(): Rule | null {
if (localRule.length === 0) return null;
return localRule.map((andGroup) =>
andGroup.conditions.map((condition) => {
let value: string | number | Date | (string | number | Date)[];
if (condition.field === 'pageCount') {
if (condition.operator === 'between') {
value = [parseInt(condition.value) || 0, parseInt(condition.value2 || '0') || 0];
} else {
value = parseInt(condition.value) || 0;
}
} else if (condition.field === 'favTime' || condition.field === 'pubTime') {
if (condition.operator === 'between') {
value = [condition.value, condition.value2 || ''];
} else {
value = condition.value;
}
} else {
if (condition.operator === 'between') {
value = [condition.value, condition.value2 || ''];
} else {
value = condition.value;
}
}
const conditionObj: Condition<string | number | Date> = {
operator: condition.operator,
value
};
let target: RuleTarget<string | number | Date> = {
field: condition.field,
rule: conditionObj
};
if (condition.isNot) {
target = {
field: 'not',
rule: target
};
}
return target;
})
);
}
function addAndGroup() {
localRule.push({ conditions: [] });
onRuleChange?.(convertLocalToRule());
}
function removeAndGroup(index: number) {
localRule.splice(index, 1);
onRuleChange?.(convertLocalToRule());
}
function addCondition(groupIndex: number) {
localRule[groupIndex].conditions.push({
field: 'title',
operator: 'contains',
value: '',
isNot: false
});
onRuleChange?.(convertLocalToRule());
}
function removeCondition(groupIndex: number, conditionIndex: number) {
localRule[groupIndex].conditions.splice(conditionIndex, 1);
onRuleChange?.(convertLocalToRule());
}
function updateCondition(
groupIndex: number,
conditionIndex: number,
field: string,
value: string
) {
const condition = localRule[groupIndex].conditions[conditionIndex];
if (field === 'field') {
condition.field = value;
const operators = getOperatorOptions(value);
condition.operator = operators[0]?.value || 'equals';
condition.value = '';
condition.value2 = '';
} else if (field === 'operator') {
condition.operator = value;
// 如果切换到/从 between 操作符,重置值
if (value === 'between') {
condition.value2 = condition.value2 || '';
}
} else if (field === 'value') {
condition.value = value;
} else if (field === 'value2') {
condition.value2 = value;
} else if (field === 'isNot') {
condition.isNot = value === 'true';
}
onRuleChange?.(convertLocalToRule());
}
function clearRules() {
localRule = [];
onRuleChange?.(convertLocalToRule());
}
</script>
<div class="space-y-4">
<div class="flex items-center justify-between">
<Label class="text-sm font-medium">过滤规则</Label>
<div class="flex gap-2">
{#if localRule.length > 0}
<Button size="sm" variant="outline" onclick={clearRules}>清空规则</Button>
{/if}
<Button size="sm" onclick={addAndGroup}>
<PlusIcon class="mr-1 h-3 w-3" />
添加规则组
</Button>
</div>
</div>
{#if localRule.length === 0}
<div class="border-muted-foreground/25 rounded-lg border-2 border-dashed p-8 text-center">
<p class="text-muted-foreground mb-4 text-sm">暂无过滤规则,将下载所有视频</p>
<Button size="sm" onclick={addAndGroup}>
<PlusIcon class="mr-1 h-3 w-3" />
添加第一个规则组
</Button>
</div>
{:else}
<div class="space-y-4">
{#each localRule as andGroup, groupIndex (groupIndex)}
<Card.Root>
<Card.Header>
<div class="flex items-center justify-between">
<div class="flex items-center gap-2">
<Badge variant="secondary">规则组 {groupIndex + 1}</Badge>
</div>
<Button
size="sm"
variant="ghost"
onclick={() => removeAndGroup(groupIndex)}
class="h-7 w-7 p-0"
>
<XIcon class="h-3 w-3" />
</Button>
</div>
</Card.Header>
<Card.Content class="space-y-3">
{#each andGroup.conditions as condition, conditionIndex (conditionIndex)}
<div class="space-y-3 rounded-lg border p-4">
<div class="flex items-center justify-between">
<Badge variant="secondary">条件 {conditionIndex + 1}</Badge>
<Button
size="sm"
variant="ghost"
onclick={() => removeCondition(groupIndex, conditionIndex)}
class="h-7 w-7 p-0"
>
<MinusIcon class="h-3 w-3" />
</Button>
</div>
<!-- 取反选项 -->
<div class="flex items-center space-x-2">
<Checkbox
id={`not-${groupIndex}-${conditionIndex}`}
checked={condition.isNot}
onCheckedChange={(checked) =>
updateCondition(
groupIndex,
conditionIndex,
'isNot',
checked ? 'true' : 'false'
)}
/>
<Label for={`not-${groupIndex}-${conditionIndex}`} class="text-sm">
取反NOT
</Label>
</div>
<!-- 字段和操作符 -->
<div class="grid grid-cols-2 gap-3">
<!-- 字段选择 -->
<div>
<Label class="text-muted-foreground text-xs">字段</Label>
<select
class="border-input bg-background placeholder:text-muted-foreground focus-visible:ring-ring flex h-9 w-full rounded-md border px-3 py-1 text-sm shadow-sm transition-colors file:border-0 file:bg-transparent file:text-sm file:font-medium focus-visible:ring-1 focus-visible:outline-none disabled:cursor-not-allowed disabled:opacity-50"
value={condition.field}
onchange={(e) =>
updateCondition(groupIndex, conditionIndex, 'field', e.currentTarget.value)}
>
{#each FIELD_OPTIONS as option (option.value)}
<option value={option.value}>{option.label}</option>
{/each}
</select>
</div>
<!-- 操作符选择 -->
<div>
<Label class="text-muted-foreground text-xs">操作符</Label>
<select
class="border-input bg-background placeholder:text-muted-foreground focus-visible:ring-ring flex h-9 w-full rounded-md border px-3 py-1 text-sm shadow-sm transition-colors file:border-0 file:bg-transparent file:text-sm file:font-medium focus-visible:ring-1 focus-visible:outline-none disabled:cursor-not-allowed disabled:opacity-50"
value={condition.operator}
onchange={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'operator',
e.currentTarget.value
)}
>
{#each getOperatorOptions(condition.field) as option (option.value)}
<option value={option.value}>{option.label}</option>
{/each}
</select>
</div>
</div>
<!-- 值输入 -->
<div>
<Label class="text-muted-foreground text-xs"></Label>
{#if condition.operator === 'between'}
<div class="grid grid-cols-2 gap-2">
{#if condition.field === 'pageCount'}
<Input
type="number"
placeholder="最小值"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value',
e.currentTarget.value
)}
/>
<Input
type="number"
placeholder="最大值"
class="h-9"
value={condition.value2 || ''}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value2',
e.currentTarget.value
)}
/>
{:else if condition.field === 'favTime' || condition.field === 'pubTime'}
<Input
type="datetime-local"
placeholder="开始时间"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value',
e.currentTarget.value + ':00' // 前端选择器只能精确到分钟,此处附加额外的 :00 以满足后端传参条件
)}
/>
<Input
type="datetime-local"
placeholder="结束时间"
class="h-9"
value={condition.value2 || ''}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value2',
e.currentTarget.value + ':00'
)}
/>
{:else}
<Input
type="text"
placeholder="起始值"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value',
e.currentTarget.value
)}
/>
<Input
type="text"
placeholder="结束值"
class="h-9"
value={condition.value2 || ''}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value2',
e.currentTarget.value
)}
/>
{/if}
</div>
{:else if condition.field === 'pageCount'}
<Input
type="number"
placeholder="输入数值"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(groupIndex, conditionIndex, 'value', e.currentTarget.value)}
/>
{:else if condition.field === 'favTime' || condition.field === 'pubTime'}
<Input
type="datetime-local"
placeholder="选择时间"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(
groupIndex,
conditionIndex,
'value',
e.currentTarget.value + ':00'
)}
/>
{:else}
<Input
type="text"
placeholder="输入文本"
class="h-9"
value={condition.value}
oninput={(e) =>
updateCondition(groupIndex, conditionIndex, 'value', e.currentTarget.value)}
/>
{/if}
</div>
</div>
{/each}
<Button
size="sm"
variant="outline"
onclick={() => addCondition(groupIndex)}
class="w-full"
>
<PlusIcon class="mr-1 h-3 w-3" />
添加条件
</Button>
</Card.Content>
</Card.Root>
{/each}
</div>
{/if}
{#if localRule.length > 0}
<div class="text-muted-foreground bg-muted/50 rounded p-3 text-xs">
<p class="mb-1 font-medium">规则说明:</p>
<ul class="space-y-1">
<li>• 多个规则组之间是"或"的关系,同一规则组内的条件是"且"的关系</li>
<li>
• 规则内配置的时间不包含时区,在处理时会默认应用<strong>服务器时区</strong
>,不受浏览器影响
</li>
</ul>
</div>
{/if}
</div>

View File

@@ -19,7 +19,7 @@
export let onsubmit: (request: UpdateVideoStatusRequest) => void;
// 视频任务名称(与后端 VideoStatus 对应)
const videoTaskNames = ['视频封面', '视频信息', 'UP主头像', 'UP主信息', '分页下载'];
const videoTaskNames = ['视频封面', '视频信息', 'UP 主头像', 'UP 主信息', '分页下载'];
// 分页任务名称(与后端 PageStatus 对应)
const pageTaskNames = ['视频封面', '视频内容', '视频信息', '视频弹幕', '视频字幕'];

View File

@@ -20,18 +20,18 @@
| FavoriteWithSubscriptionStatus
| CollectionWithSubscriptionStatus
| UpperWithSubscriptionStatus;
export let type: 'favorite' | 'collection' | 'upper' = 'favorite';
export let type: 'favorites' | 'collections' | 'submissions' = 'favorites';
export let onSubscriptionSuccess: (() => void) | null = null;
let dialogOpen = false;
function getIcon() {
switch (type) {
case 'favorite':
case 'favorites':
return HeartIcon;
case 'collection':
case 'collections':
return FolderIcon;
case 'upper':
case 'submissions':
return UserIcon;
default:
return VideoIcon;
@@ -40,12 +40,12 @@
function getTypeLabel() {
switch (type) {
case 'favorite':
case 'favorites':
return '收藏夹';
case 'collection':
case 'collections':
return '合集';
case 'upper':
return 'UP主';
case 'submissions':
return 'UP 主';
default:
return '';
}
@@ -53,11 +53,11 @@
function getTitle(): string {
switch (type) {
case 'favorite':
case 'favorites':
return (item as FavoriteWithSubscriptionStatus).title;
case 'collection':
case 'collections':
return (item as CollectionWithSubscriptionStatus).title;
case 'upper':
case 'submissions':
return (item as UpperWithSubscriptionStatus).uname;
default:
return '';
@@ -66,12 +66,10 @@
function getSubtitle(): string {
switch (type) {
case 'favorite':
case 'favorites':
return `uid: ${(item as FavoriteWithSubscriptionStatus).mid}`;
case 'collection':
case 'collections':
return `uid: ${(item as CollectionWithSubscriptionStatus).mid}`;
case 'upper':
return '';
default:
return '';
}
@@ -79,7 +77,7 @@
function getDescription(): string {
switch (type) {
case 'upper':
case 'submissions':
return (item as UpperWithSubscriptionStatus).sign || '';
default:
return '';
@@ -88,9 +86,9 @@
function isDisabled(): boolean {
switch (type) {
case 'collection':
case 'collections':
return (item as CollectionWithSubscriptionStatus).invalid;
case 'upper': {
case 'submissions': {
return (item as UpperWithSubscriptionStatus).invalid;
}
default:
@@ -100,9 +98,9 @@
function getDisabledReason(): string {
switch (type) {
case 'collection':
case 'collections':
return '已失效';
case 'upper':
case 'submissions':
return '账号已注销';
default:
return '';
@@ -111,7 +109,7 @@
function getCount(): number | null {
switch (type) {
case 'favorite':
case 'favorites':
return (item as FavoriteWithSubscriptionStatus).media_count;
default:
return null;
@@ -124,8 +122,8 @@
function getAvatarUrl(): string {
switch (type) {
case 'upper':
return `/image-proxy?url=${(item as UpperWithSubscriptionStatus).face}`;
case 'submissions':
return (item as UpperWithSubscriptionStatus).face;
default:
return '';
}
@@ -171,7 +169,7 @@
? 'opacity-50'
: ''}"
>
{#if avatarUrl && type === 'upper'}
{#if avatarUrl && type === 'submissions'}
<img
src={avatarUrl}
alt={title}
@@ -265,5 +263,3 @@
<!-- 订阅对话框 -->
<SubscriptionDialog bind:open={dialogOpen} {item} {type} onSuccess={handleSubscriptionSuccess} />
<!-- 订阅对话框 -->
<SubscriptionDialog bind:open={dialogOpen} {item} {type} onSuccess={handleSubscriptionSuccess} />

View File

@@ -2,6 +2,7 @@
import { Button } from '$lib/components/ui/button/index.js';
import { Input } from '$lib/components/ui/input/index.js';
import { Label } from '$lib/components/ui/label/index.js';
import { toast } from 'svelte-sonner';
import {
Sheet,
SheetContent,
@@ -10,7 +11,6 @@
SheetHeader,
SheetTitle
} from '$lib/components/ui/sheet/index.js';
import { toast } from 'svelte-sonner';
import api from '$lib/api';
import type {
FavoriteWithSubscriptionStatus,
@@ -22,48 +22,41 @@
ApiError
} from '$lib/types';
export let open = false;
export let item:
| FavoriteWithSubscriptionStatus
| CollectionWithSubscriptionStatus
| UpperWithSubscriptionStatus
| null = null;
export let type: 'favorite' | 'collection' | 'upper' = 'favorite';
export let onSuccess: (() => void) | null = null;
interface Props {
open: boolean;
item:
| FavoriteWithSubscriptionStatus
| CollectionWithSubscriptionStatus
| UpperWithSubscriptionStatus
| null;
type: 'favorites' | 'collections' | 'submissions';
onSuccess: (() => void) | null;
}
let customPath = '';
let loading = false;
let {
open = $bindable(false),
item = null,
type = 'favorites',
onSuccess = null
}: Props = $props();
// 根据类型和item生成默认路径
function generateDefaultPath(): string {
if (!item) return '';
let customPath = $state('');
let loading = $state(false);
switch (type) {
case 'favorite': {
const favorite = item as FavoriteWithSubscriptionStatus;
return `收藏夹/${favorite.title}`;
}
case 'collection': {
const collection = item as CollectionWithSubscriptionStatus;
return `合集/${collection.title}`;
}
case 'upper': {
const upper = item as UpperWithSubscriptionStatus;
return `UP主/${upper.uname}`;
}
default:
return '';
}
// 根据类型和 item 生成默认路径
async function generateDefaultPath(): Promise<string> {
if (!itemTitle) return '';
return (await api.getDefaultPath(type, itemTitle)).data;
}
function getTypeLabel(): string {
switch (type) {
case 'favorite':
case 'favorites':
return '收藏夹';
case 'collection':
case 'collections':
return '合集';
case 'upper':
return 'UP主';
case 'submissions':
return 'UP 主';
default:
return '';
}
@@ -73,11 +66,11 @@
if (!item) return '';
switch (type) {
case 'favorite':
case 'favorites':
return (item as FavoriteWithSubscriptionStatus).title;
case 'collection':
case 'collections':
return (item as CollectionWithSubscriptionStatus).title;
case 'upper':
case 'submissions':
return (item as UpperWithSubscriptionStatus).uname;
default:
return '';
@@ -92,7 +85,7 @@
let response;
switch (type) {
case 'favorite': {
case 'favorites': {
const favorite = item as FavoriteWithSubscriptionStatus;
const request: InsertFavoriteRequest = {
fid: favorite.fid,
@@ -101,7 +94,7 @@
response = await api.insertFavorite(request);
break;
}
case 'collection': {
case 'collections': {
const collection = item as CollectionWithSubscriptionStatus;
const request: InsertCollectionRequest = {
sid: collection.sid,
@@ -111,7 +104,7 @@
response = await api.insertCollection(request);
break;
}
case 'upper': {
case 'submissions': {
const upper = item as UpperWithSubscriptionStatus;
const request: InsertSubmissionRequest = {
upper_id: upper.mid,
@@ -145,10 +138,20 @@
open = false;
}
// 当对话框打开时重置path
$: if (open && item) {
customPath = generateDefaultPath();
}
$effect(() => {
if (open && item) {
generateDefaultPath()
.then((path) => {
customPath = path;
})
.catch((error) => {
toast.error('获取默认路径失败', {
description: (error as ApiError).message
});
customPath = '';
});
}
});
const typeLabel = getTypeLabel();
const itemTitle = getItemTitle();
@@ -173,14 +176,14 @@
<span class="text-muted-foreground text-sm font-medium">{typeLabel}名称:</span>
<span class="text-sm">{itemTitle}</span>
</div>
{#if type === 'favorite'}
{#if type === 'favorites'}
{@const favorite = item as FavoriteWithSubscriptionStatus}
<div class="flex items-center gap-2">
<span class="text-muted-foreground text-sm font-medium">视频数量:</span>
<span class="text-sm">{favorite.media_count}</span>
</div>
{/if}
{#if type === 'upper'}
{#if type === 'submissions'}
{@const upper = item as UpperWithSubscriptionStatus}
{#if upper.sign}
<div class="flex items-start gap-2">

View File

@@ -0,0 +1,17 @@
import { Popover as PopoverPrimitive } from 'bits-ui';
import Content from './popover-content.svelte';
import Trigger from './popover-trigger.svelte';
const Root = PopoverPrimitive.Root;
const Close = PopoverPrimitive.Close;
export {
Root,
Content,
Trigger,
Close,
//
Root as Popover,
Content as PopoverContent,
Trigger as PopoverTrigger,
Close as PopoverClose
};

View File

@@ -0,0 +1,29 @@
<script lang="ts">
import { cn } from '$lib/utils.js';
import { Popover as PopoverPrimitive } from 'bits-ui';
let {
ref = $bindable(null),
class: className,
sideOffset = 4,
align = 'center',
portalProps,
...restProps
}: PopoverPrimitive.ContentProps & {
portalProps?: PopoverPrimitive.PortalProps;
} = $props();
</script>
<PopoverPrimitive.Portal {...portalProps}>
<PopoverPrimitive.Content
bind:ref
data-slot="popover-content"
{sideOffset}
{align}
class={cn(
'bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-72 origin-(--bits-popover-content-transform-origin) rounded-md border p-4 shadow-md outline-hidden',
className
)}
{...restProps}
/>
</PopoverPrimitive.Portal>

View File

@@ -0,0 +1,17 @@
<script lang="ts">
import { cn } from '$lib/utils.js';
import { Popover as PopoverPrimitive } from 'bits-ui';
let {
ref = $bindable(null),
class: className,
...restProps
}: PopoverPrimitive.TriggerProps = $props();
</script>
<PopoverPrimitive.Trigger
bind:ref
data-slot="popover-trigger"
class={cn('', className)}
{...restProps}
/>

View File

@@ -49,20 +49,30 @@
}
}
function getOverallStatus(downloadStatus: number[]): {
function getOverallStatus(
downloadStatus: number[],
shouldDownload: boolean
): {
text: string;
color: 'default' | 'secondary' | 'destructive' | 'outline';
style: string;
} {
if (!shouldDownload) {
// 被过滤规则排除,显示为“跳过”
return { text: '跳过', style: 'bg-gray-100 text-gray-700' };
}
const completed = downloadStatus.filter((status) => status === 7).length;
const total = downloadStatus.length;
const failed = downloadStatus.filter((status) => status !== 7 && status !== 0).length;
if (completed === total) {
return { text: '完成', color: 'outline' };
// 全部完成,显示为“完成”
return { text: '完成', style: 'bg-emerald-700 text-emerald-100' };
} else if (failed > 0) {
return { text: '失败', color: 'destructive' };
// 出现了失败,显示为“失败”
return { text: '失败', style: 'bg-rose-700 text-rose-100' };
} else {
return { text: '进行中', color: 'secondary' };
// 还未开始,显示为“等待”
return { text: '等待', style: 'bg-yellow-700 text-yellow-100' };
}
}
@@ -74,7 +84,7 @@
return defaultTaskNames[index] || `任务${index + 1}`;
}
$: overallStatus = getOverallStatus(video.download_status);
$: overallStatus = getOverallStatus(video.download_status, video.should_download);
$: completed = video.download_status.filter((status) => status === 7).length;
$: total = video.download_status.length;
@@ -112,7 +122,10 @@
>
{displayTitle}
</CardTitle>
<Badge variant={overallStatus.color} class="shrink-0 px-2 py-1 text-xs font-medium">
<Badge
variant="secondary"
class="shrink-0 px-2 py-1 text-xs font-medium {overallStatus.style} "
>
{overallStatus.text}
</Badge>
</div>

View File

@@ -36,6 +36,7 @@ export interface VideoInfo {
bvid: string;
name: string;
upper_name: string;
should_download: boolean;
download_status: [number, number, number, number, number];
}
@@ -136,7 +137,7 @@ export interface CollectionsResponse {
total: number;
}
// UP主相关类型
// UP 主相关类型
export interface UpperWithSubscriptionStatus {
mid: number;
uname: string;
@@ -168,11 +169,28 @@ export interface InsertSubmissionRequest {
path: string;
}
// Rule 相关类型
export interface Condition<T> {
operator: string;
value: T | T[];
}
export interface RuleTarget<T> {
field: string;
rule: Condition<T> | RuleTarget<T>;
}
export type AndGroup = RuleTarget<string | number | Date>[];
export type Rule = AndGroup[];
// 视频源详细信息类型
export interface VideoSourceDetail {
id: number;
name: string;
path: string;
rule: Rule | null;
ruleDisplay: string | null;
useDynamicApi: boolean | null;
enabled: boolean;
}
@@ -188,6 +206,8 @@ export interface VideoSourcesDetailsResponse {
export interface UpdateVideoSourceRequest {
path: string;
enabled: boolean;
rule?: Rule | null;
useDynamicApi?: boolean | null;
}
// 配置相关类型
@@ -226,6 +246,14 @@ export interface DanmakuOption {
time_offset: number;
}
export interface SkipOption {
no_poster: boolean;
no_video_nfo: boolean;
no_upper: boolean;
no_danmaku: boolean;
no_subtitle: boolean;
}
export interface RateLimit {
limit: number;
duration: number;
@@ -244,15 +272,36 @@ export interface ConcurrentLimit {
download: ConcurrentDownloadLimit;
}
// Notifier 相关类型
export interface TelegramNotifier {
type: 'telegram';
bot_token: string;
chat_id: string;
}
export interface WebhookNotifier {
type: 'webhook';
url: string;
}
export type Notifier = TelegramNotifier | WebhookNotifier;
export type Trigger = number | string;
export interface Config {
auth_token: string;
bind_address: string;
credential: Credential;
filter_option: FilterOption;
danmaku_option: DanmakuOption;
skip_option: SkipOption;
video_name: string;
page_name: string;
interval: number;
notifiers: Notifier[] | null;
favorite_default_path: string;
collection_default_path: string;
submission_default_path: string;
interval: Trigger;
upper_path: string;
nfo_time_type: string;
concurrent_limit: ConcurrentLimit;
@@ -295,3 +344,7 @@ export interface TaskStatus {
last_finish: Date | null;
next_run: Date | null;
}
export interface UpdateVideoSourceResponse {
ruleDisplay: string;
}

View File

@@ -122,7 +122,7 @@ export class WebSocketManager {
} catch (error) {
console.error('Failed to parse WebSocket message:', error, event.data);
toast.error('解析 WebSocket 消息失败', {
description: `消息内容: ${event.data}\n错误信息: ${error instanceof Error ? error.message : String(error)}`
description: `消息内容${event.data}\n错误信息${error instanceof Error ? error.message : String(error)}`
});
}
}
@@ -137,7 +137,7 @@ export class WebSocketManager {
} catch (error) {
console.error('Failed to send message:', error);
toast.error('发送 WebSocket 消息失败', {
description: `消息内容: ${JSON.stringify(message)}\n错误信息: ${error instanceof Error ? error.message : String(error)}`
description: `消息内容${JSON.stringify(message)}\n错误信息${error instanceof Error ? error.message : String(error)}`
});
}
}

View File

@@ -19,7 +19,11 @@
<BreadCrumb items={$breadcrumbStore} />
</div>
</header>
<div class="w-full overflow-y-auto px-6 py-2" style="scrollbar-width: thin;" id="main">
<div
class="w-full overflow-y-auto px-6 py-2"
style="scrollbar-width: thin; scrollbar-gutter: stable !important;"
id="main"
>
<slot />
</div>
</Sidebar.Inset>

Some files were not shown because too many files have changed in this diff Show More