GithubBackup/bili-sync
1
0
Fork 0
mirror of https://github.com/amtoaer/bili-sync.git synced 2026-09-05 15:39:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 实现视频的筛选规则 (#457)

Browse Source
This commit is contained in:
ᴀᴍᴛᴏᴀᴇʀ
2025-09-24 00:42:27 +08:00
committed by GitHub
parent 6c7d295fe6
commit 210c94398a
39 changed files with 1345 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bili_sync/src/adapter/collection.rs
+5
View File
@@ -3,6 +3,7 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use chrono::Utc;
use futures::Stream;
@@ -63,6 +64,10 @@ impl VideoSource for collection::Model {
None
}
fn rule(&self) -> Option<&Rule> {
self.rule.as_ref()
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
crates/bili_sync/src/adapter/favorite.rs
+5
View File
@@ -3,6 +3,7 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -42,6 +43,10 @@ impl VideoSource for favorite::Model {
})
}
fn rule(&self) -> Option<&Rule> {
self.rule.as_ref()
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
crates/bili_sync/src/adapter/mod.rs
+3
View File
@@ -19,6 +19,7 @@ use sea_orm::sea_query::SimpleExpr;
#[rustfmt::skip]
use bili_sync_entity::collection::Model as Collection;
use bili_sync_entity::favorite::Model as Favorite;
use bili_sync_entity::rule::Rule;
use bili_sync_entity::submission::Model as Submission;
use bili_sync_entity::watch_later::Model as WatchLater;
@@ -68,6 +69,8 @@ pub trait VideoSource {
video_info.ok()
}
fn rule(&self) -> Option<&Rule>;
fn log_refresh_video_start(&self) {
info!("开始扫描{}..", self.display_name());
}
crates/bili_sync/src/adapter/submission.rs
+5
View File
@@ -2,6 +2,7 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result, ensure};
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -41,6 +42,10 @@ impl VideoSource for submission::Model {
})
}
fn rule(&self) -> Option<&Rule> {
self.rule.as_ref()
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
crates/bili_sync/src/adapter/watch_later.rs
+5
View File
@@ -2,6 +2,7 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::ActiveValue::Set;
@@ -41,6 +42,10 @@ impl VideoSource for watch_later::Model {
})
}
fn rule(&self) -> Option<&Rule> {
self.rule.as_ref()
}
async fn refresh<'a>(
self,
bili_client: &'a BiliClient,
crates/bili_sync/src/api/request.rs
+2
View File
@@ -1,3 +1,4 @@
use bili_sync_entity::rule::Rule;
use serde::Deserialize;
use validator::Validate;
@@ -86,4 +87,5 @@ pub struct UpdateVideoSourceRequest {
#[validate(custom(function = "crate::utils::validation::validate_path"))]
pub path: String,
pub enabled: bool,
pub rule: Option<Rule>,
}
crates/bili_sync/src/api/response.rs
+11
View File
@@ -1,3 +1,4 @@
use bili_sync_entity::rule::Rule;
use bili_sync_entity::*;
use sea_orm::{DerivePartialModel, FromQueryResult};
use serde::Serialize;
@@ -169,9 +170,19 @@ pub struct SysInfo {
}
#[derive(Serialize, FromQueryResult)]
#[serde(rename_all = "camelCase")]
pub struct VideoSourceDetail {
pub id: i32,
pub name: String,
pub path: String,
pub rule: Option<Rule>,
#[serde(default)]
pub rule_display: Option<String>,
pub enabled: bool,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct UpdateVideoSourceResponse {
pub rule_display: Option<String>,
}
crates/bili_sync/src/api/routes/video_sources/mod.rs
+38 -11
View File
@@ -14,7 +14,9 @@ use crate::api::error::InnerApiError;
use crate::api::request::{
InsertCollectionRequest, InsertFavoriteRequest, InsertSubmissionRequest, UpdateVideoSourceRequest,
};
use crate::api::response::{VideoSource, VideoSourceDetail, VideoSourcesDetailsResponse, VideoSourcesResponse};
use crate::api::response::{
UpdateVideoSourceResponse, VideoSource, VideoSourceDetail, VideoSourcesDetailsResponse, VideoSourcesResponse,
};
use crate::api::wrapper::{ApiError, ApiResponse, ValidatedJson};
use crate::bilibili::{BiliClient, Collection, CollectionItem, FavoriteList, Submission};
@@ -75,13 +77,14 @@ pub async fn get_video_sources(
pub async fn get_video_sources_details(
Extension(db): Extension<DatabaseConnection>,
) -> Result<ApiResponse<VideoSourcesDetailsResponse>, ApiError> {
let (collections, favorites, submissions, mut watch_later) = tokio::try_join!(
let (mut collections, mut favorites, mut submissions, mut watch_later) = tokio::try_join!(
collection::Entity::find()
.select_only()
.columns([
collection::Column::Id,
collection::Column::Name,
collection::Column::Path,
collection::Column::Rule,
collection::Column::Enabled
])
.into_model::<VideoSourceDetail>()
@@ -92,22 +95,31 @@ pub async fn get_video_sources_details(
favorite::Column::Id,
favorite::Column::Name,
favorite::Column::Path,
favorite::Column::Rule,
favorite::Column::Enabled
])
.into_model::<VideoSourceDetail>()
.all(&db),
submission::Entity::find()
.select_only()
.column(submission::Column::Id)
.column_as(submission::Column::UpperName, "name")
.columns([submission::Column::Path, submission::Column::Enabled])
.columns([
submission::Column::Id,
submission::Column::Path,
submission::Column::Enabled,
submission::Column::Rule
])
.into_model::<VideoSourceDetail>()
.all(&db),
watch_later::Entity::find()
.select_only()
.column(watch_later::Column::Id)
.column_as(Expr::value("稍后再看"), "name")
.columns([watch_later::Column::Path, watch_later::Column::Enabled])
.columns([
watch_later::Column::Id,
watch_later::Column::Path,
watch_later::Column::Enabled,
watch_later::Column::Rule
])
.into_model::<VideoSourceDetail>()
.all(&db)
)?;
@@ -116,9 +128,18 @@ pub async fn get_video_sources_details(
id: 1,
name: "稍后再看".to_string(),
path: String::new(),
rule: None,
rule_display: None,
enabled: false,
})
}
for sources in [&mut collections, &mut favorites, &mut submissions, &mut watch_later] {
sources.iter_mut().for_each(|item| {
if let Some(rule) = &item.rule {
item.rule_display = Some(rule.to_string());
}
});
}
Ok(ApiResponse::ok(VideoSourcesDetailsResponse {
collections,
favorites,
@@ -132,24 +153,28 @@ pub async fn update_video_source(
Path((source_type, id)): Path<(String, i32)>,
Extension(db): Extension<DatabaseConnection>,
ValidatedJson(request): ValidatedJson<UpdateVideoSourceRequest>,
) -> Result<ApiResponse<bool>, ApiError> {
) -> Result<ApiResponse<UpdateVideoSourceResponse>, ApiError> {
let rule_display = request.rule.as_ref().map(|rule| rule.to_string());
let active_model = match source_type.as_str() {
"collections" => collection::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: collection::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
_ActiveModel::Collection(active_model)
}),
"favorites" => favorite::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: favorite::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
_ActiveModel::Favorite(active_model)
}),
"submissions" => submission::Entity::find_by_id(id).one(&db).await?.map(|model| {
let mut active_model: submission::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
_ActiveModel::Submission(active_model)
}),
"watch_later" => match watch_later::Entity::find_by_id(id).one(&db).await? {
@@ -160,6 +185,7 @@ pub async fn update_video_source(
let mut active_model: watch_later::ActiveModel = model.into();
active_model.path = Set(request.path);
active_model.enabled = Set(request.enabled);
active_model.rule = Set(request.rule);
Some(_ActiveModel::WatchLater(active_model))
}
None => {
@@ -170,6 +196,7 @@ pub async fn update_video_source(
Some(_ActiveModel::WatchLater(watch_later::ActiveModel {
path: Set(request.path),
enabled: Set(request.enabled),
rule: Set(request.rule),
..Default::default()
}))
}
@@ -181,7 +208,7 @@ pub async fn update_video_source(
return Err(InnerApiError::NotFound(id).into());
};
active_model.save(&db).await?;
Ok(ApiResponse::ok(true))
Ok(ApiResponse::ok(UpdateVideoSourceResponse { rule_display }))
}
/// 新增收藏夹订阅
@@ -196,7 +223,7 @@ pub async fn insert_favorite(
f_id: Set(favorite_info.id),
name: Set(favorite_info.title.clone()),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(&db)
@@ -225,7 +252,7 @@ pub async fn insert_collection(
r#type: Set(collection_info.collection_type.into()),
name: Set(collection_info.name.clone()),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(&db)
@@ -246,7 +273,7 @@ pub async fn insert_submission(
upper_id: Set(upper.mid.parse()?),
upper_name: Set(upper.name),
path: Set(request.path),
enabled: Set(true),
enabled: Set(false),
..Default::default()
})
.exec(&db)
crates/bili_sync/src/bilibili/video.rs
+9 -17
View File
@@ -1,4 +1,4 @@
use anyhow::{Result, ensure};
use anyhow::{Context, Result, ensure};
use futures::TryStreamExt;
use futures::stream::FuturesUnordered;
use prost::Message;
@@ -16,19 +16,6 @@ pub struct Video<'a> {
pub bvid: String,
}
#[derive(Debug, serde::Deserialize)]
pub struct Tag {
pub tag_name: String,
}
impl serde::Serialize for Tag {
fn serialize<S>(&self, serializer: S) -> core::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.tag_name)
}
}
#[derive(Debug, serde::Deserialize, Default)]
pub struct PageInfo {
pub cid: i64,
@@ -84,8 +71,8 @@ impl<'a> Video<'a> {
Ok(serde_json::from_value(res["data"].take())?)
}
pub async fn get_tags(&self) -> Result<Vec<Tag>> {
let mut res = self
pub async fn get_tags(&self) -> Result<Vec<String>> {
let res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/view/detail/tag")
.await
@@ -96,7 +83,12 @@ impl<'a> Video<'a> {
.json::<serde_json::Value>()
.await?
.validate()?;
Ok(serde_json::from_value(res["data"].take())?)
Ok(res["data"]
.as_array()
.context("tags is not an array")?
.iter()
.filter_map(|v| v["tag_name"].as_str().map(String::from))
.collect())
}
pub async fn get_danmaku_writer(&self, page: &'a PageInfo) -> Result<DanmakuWriter> {
crates/bili_sync/src/utils/convert.rs
+1 -1
View File
@@ -124,7 +124,7 @@ impl VideoInfo {
ctime: Set(ctime.naive_utc()),
pubtime: Set(pubtime.naive_utc()),
favtime: if base_model.favtime != NaiveDateTime::default() {
NotSet // 之前设置了 favtime，不覆盖
Set(base_model.favtime) // 之前设置了 favtime，使用之前的值（等价于 unset，但设置上以支持后续的规则匹配）
} else {
Set(pubtime.naive_utc()) // 未设置过 favtime，使用 pubtime 填充
},
crates/bili_sync/src/utils/mod.rs
+1
View File
@@ -3,6 +3,7 @@ pub mod filenamify;
pub mod format_arg;
pub mod model;
pub mod nfo;
pub mod rule;
pub mod signal;
pub mod status;
pub mod task_notifier;
crates/bili_sync/src/utils/model.rs
+1
View File
@@ -41,6 +41,7 @@ pub async fn filter_unhandled_video_pages(
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.and(video::Column::ShouldDownload.eq(true))
.and(additional_expr),
)
.find_with_related(page::Entity)
crates/bili_sync/src/utils/nfo.rs
+3 -9
View File
@@ -261,7 +261,7 @@ mod tests {
chrono::NaiveTime::from_hms_opt(3, 3, 3).unwrap(),
),
bvid: "BV1nWcSeeEkV".to_string(),
tags: Some(serde_json::json!(["tag1", "tag2"])),
tags: Some(vec!["tag1".to_owned(), "tag2".to_owned()].into()),
..Default::default()
};
assert_eq!(
@@ -343,10 +343,7 @@ impl<'a> From<&'a video::Model> for Movie<'a> {
NFOTimeType::FavTime => video.favtime,
NFOTimeType::PubTime => video.pubtime,
},
tags: video
.tags
.as_ref()
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
tags: video.tags.as_ref().map(|tags| tags.clone().into()),
}
}
}
@@ -363,10 +360,7 @@ impl<'a> From<&'a video::Model> for TVShow<'a> {
NFOTimeType::FavTime => video.favtime,
NFOTimeType::PubTime => video.pubtime,
},
tags: video
.tags
.as_ref()
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
tags: video.tags.as_ref().map(|tags| tags.clone().into()),
}
}
}
crates/bili_sync/src/utils/rule.rs
+224
View File
@@ -0,0 +1,224 @@
use bili_sync_entity::rule::{AndGroup, Condition, Rule, RuleTarget};
use bili_sync_entity::{page, video};
use chrono::{Local, NaiveDateTime};
pub(crate) trait Evaluatable<T> {
fn evaluate(&self, value: T) -> bool;
}
pub(crate) trait FieldEvaluatable {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool;
}
impl Evaluatable<&str> for Condition<String> {
fn evaluate(&self, value: &str) -> bool {
match self {
Condition::Equals(expected) => expected == value,
Condition::Contains(substring) => value.contains(substring),
Condition::Prefix(prefix) => value.starts_with(prefix),
Condition::Suffix(suffix) => value.ends_with(suffix),
Condition::MatchesRegex(_, regex) => regex.is_match(value),
_ => false,
}
}
}
impl Evaluatable<usize> for Condition<usize> {
fn evaluate(&self, value: usize) -> bool {
match self {
Condition::Equals(expected) => *expected == value,
Condition::GreaterThan(threshold) => value > *threshold,
Condition::LessThan(threshold) => value < *threshold,
Condition::Between(start, end) => value > *start && value < *end,
_ => false,
}
}
}
impl Evaluatable<&NaiveDateTime> for Condition<NaiveDateTime> {
fn evaluate(&self, value: &NaiveDateTime) -> bool {
match self {
Condition::Equals(expected) => expected == value,
Condition::GreaterThan(threshold) => value > threshold,
Condition::LessThan(threshold) => value < threshold,
Condition::Between(start, end) => value > start && value < end,
_ => false,
}
}
}
impl FieldEvaluatable for RuleTarget {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
match self {
RuleTarget::Title(cond) => video.name.try_as_ref().is_some_and(|title| cond.evaluate(&title)),
// 目前的所有条件都是分别针对全体标签进行 any 评估的，例如 Prefix("a") && Suffix("b") 意味着 any(tag.Prefix("a")) && any(tag.Suffix("b")) 而非 any(tag.Prefix("a") && tag.Suffix("b"))
// 这可能不满足用户预期，但应该问题不大，如果真有很多人用复杂标签筛选再单独改
RuleTarget::Tags(cond) => video
.tags
.try_as_ref()
.and_then(|t| t.as_ref())
.is_some_and(|tags| tags.0.iter().any(|tag| cond.evaluate(&tag))),
RuleTarget::FavTime(cond) => video
.favtime
.try_as_ref()
.map(|fav_time| fav_time.and_utc().with_timezone(&Local).naive_local()) // 数据库中保存的一律是 utc 时间，转换为 local 时间再比较
.is_some_and(|fav_time| cond.evaluate(&fav_time)),
RuleTarget::PubTime(cond) => video
.pubtime
.try_as_ref()
.map(|pub_time| pub_time.and_utc().with_timezone(&Local).naive_local())
.is_some_and(|pub_time| cond.evaluate(&pub_time)),
RuleTarget::PageCount(cond) => cond.evaluate(pages.len()),
RuleTarget::Not(inner) => !inner.evaluate(video, pages),
}
}
}
impl FieldEvaluatable for AndGroup {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
self.iter().all(|target| target.evaluate(video, pages))
}
}
impl FieldEvaluatable for Rule {
fn evaluate(&self, video: &video::ActiveModel, pages: &[page::ActiveModel]) -> bool {
if self.0.is_empty() {
return true;
}
self.0.iter().any(|group| group.evaluate(video, pages))
}
}
#[cfg(test)]
mod tests {
use bili_sync_entity::page;
use chrono::NaiveDate;
use sea_orm::ActiveValue::Set;
use super::*;
#[test]
fn test_display() {
let test_cases = vec![
(
Rule(vec![vec![RuleTarget::Title(Condition::Contains("唐氏".to_string()))]]),
"「(标题包含“唐氏”)」",
),
(
Rule(vec![vec![
RuleTarget::Title(Condition::Prefix("街霸".to_string())),
RuleTarget::Tags(Condition::Contains("套路".to_string())),
]]),
"「(标题以“街霸”开头)且(标签包含“套路”)」",
),
(
Rule(vec![
vec![
RuleTarget::Title(Condition::Contains("Rust".to_string())),
RuleTarget::PageCount(Condition::GreaterThan(5)),
],
vec![
RuleTarget::Tags(Condition::Suffix("入门".to_string())),
RuleTarget::PubTime(Condition::GreaterThan(
NaiveDate::from_ymd_opt(2023, 1, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap(),
)),
],
]),
"「(标题包含“Rust”)且(视频分页数量大于“5”)」或「(标签以“入门”结尾)且(发布时间大于“2023-01-01 00:00:00”)」",
),
(
Rule(vec![vec![
RuleTarget::Not(Box::new(RuleTarget::Title(Condition::Contains("广告".to_string())))),
RuleTarget::PageCount(Condition::LessThan(10)),
]]),
"「(标题不包含“广告”)且(视频分页数量小于“10”)」",
),
(
Rule(vec![vec![
RuleTarget::FavTime(Condition::Between(
NaiveDate::from_ymd_opt(2023, 6, 1)
.unwrap()
.and_hms_opt(0, 0, 0)
.unwrap(),
NaiveDate::from_ymd_opt(2023, 12, 31)
.unwrap()
.and_hms_opt(23, 59, 59)
.unwrap(),
)),
RuleTarget::Tags(Condition::MatchesRegex(
"技术|教程".to_string(),
regex::Regex::new("技术|教程").unwrap(),
)),
]]),
"「(收藏时间在“2023-06-01 00:00:00”和“2023-12-31 23:59:59”之间)且(标签匹配“技术|教程”)」",
),
];
for (rule, expected) in test_cases {
assert_eq!(rule.to_string(), expected);
}
}
#[test]
fn test_evaluate() {
let test_cases = vec![
(
(
video::ActiveModel {
name: Set("骂谁唐氏呢！！！".to_string()),
..Default::default()
},
vec![],
),
Rule(vec![vec![RuleTarget::Title(Condition::Contains("唐氏".to_string()))]]),
true,
),
(
(
video::ActiveModel::default(),
vec![page::ActiveModel::default(); 2],
),
Rule(vec![vec![RuleTarget::PageCount(Condition::Equals(1))]]),
false,
),
(
(
video::ActiveModel{
tags: Set(Some(vec!["原神".to_owned(),"永雏塔菲".to_owned(),"虚拟主播".to_owned()].into())),
..Default::default()
},
vec![],
),
Rule (vec![vec![RuleTarget::Not(Box::new(RuleTarget::Tags(Condition::Equals(
"原神".to_string(),
))))]],
),
false,
),
(
(
video::ActiveModel {
name: Set(
"万字怒扒网易《归唐》底裤！中国首款大厂买断制单机，靠谱吗？——全网最全！官方非独家幕后！关于《归唐》PV的所有秘密~都在这里了~".to_owned(),
),
..Default::default()
},
vec![],
),
Rule(vec![vec![RuleTarget::Not(Box::new(RuleTarget::Title(Condition::MatchesRegex(
r"^\S+字(解析|怒扒|拆解)".to_owned(),
regex::Regex::new(r"^\S+字(解析|怒扒)").unwrap(),
))))]],
),
false,
),
];
for ((video, pages), rule, expected) in test_cases {
assert_eq!(rule.evaluate(&video, &pages), expected);
}
}
}
crates/bili_sync/src/utils/status.rs
+1 -1
View File
@@ -176,7 +176,7 @@ pub type VideoStatus = Status<5>;
pub type PageStatus = Status<5>;
#[cfg(test)]
mod test {
mod tests {
use anyhow::anyhow;
use super::*;
crates/bili_sync/src/workflow.rs
+5 -1
View File
@@ -23,6 +23,7 @@ use crate::utils::model::{
update_videos_model,
};
use crate::utils::nfo::NFO;
use crate::utils::rule::FieldEvaluatable;
use crate::utils::status::{PageStatus, STATUS_OK, VideoStatus};
/// 完整地处理某个视频来源
@@ -136,7 +137,10 @@ pub async fn fetch_video_details(
let mut video_active_model = view_info.into_detail_model(video_model);
video_source.set_relation_id(&mut video_active_model);
video_active_model.single_page = Set(Some(pages.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.tags = Set(Some(tags.into()));
video_active_model.should_download = Set(video_source
.rule()
.is_none_or(|r| r.evaluate(&video_active_model, &pages)));
let txn = connection.begin().await?;
create_pages(pages, &txn).await?;
video_active_model.save(&txn).await?;