GithubBackup/bili-sync
1
0
Fork 0
mirror of https://github.com/amtoaer/bili-sync.git synced 2026-09-04 23:18:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 精简代码，统一逻辑 (#229)

Browse Source
This commit is contained in:
ᴀᴍᴛᴏᴀᴇʀ
2025-01-24 01:11:59 +08:00
committed by GitHub
parent aa88f97eff
commit 7c220f0d2b
18 changed files with 387 additions and 604 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bili_sync/src/adapter/collection.rs
+12 -72
View File
@@ -2,99 +2,39 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result};
use async_trait::async_trait;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
use crate::utils::status::STATUS_COMPLETED;
use crate::adapter::{VideoListModel, _ActiveModel};
use crate::bilibili::{BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
#[async_trait]
impl VideoListModel for collection::Model {
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::CollectionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null())
.into_condition(),
connection,
)
.await
fn filter_expr(&self) -> SimpleExpr {
video::Column::CollectionId.eq(self.id)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
helper::filter_videos_with_pages(
video::Column::CollectionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.into_condition(),
connection,
)
.await
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
video_model.collection_id = Set(Some(self.id));
helper::video_with_path(video_model, &self.path, video_info)
}
async fn fetch_videos_detail(
&self,
video: bilibili::Video<'_>,
video_model: video::Model,
connection: &DatabaseConnection,
) -> Result<()> {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Ok((tags, view_info)) => {
let VideoInfo::Detail { pages, .. } = &view_info else {
unreachable!("view_info must be VideoInfo::View")
};
let txn = connection.begin().await?;
// 将分页信息写入数据库
helper::create_video_pages(pages, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model));
video_active_model.single_page = Set(Some(pages.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
helper::error_fetch_video_detail(e, video_model, connection).await?;
}
};
Ok(())
fn path(&self) -> &Path {
Path::new(self.path.as_str())
}
fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
collection::ActiveModel {
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
_ActiveModel::Collection(collection::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
})
}
fn log_fetch_video_start(&self) {
crates/bili_sync/src/adapter/favorite.rs
+12 -69
View File
@@ -2,96 +2,39 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result};
use async_trait::async_trait;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, FavoriteList, VideoInfo};
use crate::utils::status::STATUS_COMPLETED;
use crate::adapter::{VideoListModel, _ActiveModel};
use crate::bilibili::{BiliClient, FavoriteList, VideoInfo};
#[async_trait]
impl VideoListModel for favorite::Model {
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::FavoriteId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null())
.into_condition(),
connection,
)
.await
fn filter_expr(&self) -> SimpleExpr {
video::Column::FavoriteId.eq(self.id)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
helper::filter_videos_with_pages(
video::Column::FavoriteId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.into_condition(),
connection,
)
.await
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
video_model.favorite_id = Set(Some(self.id));
helper::video_with_path(video_model, &self.path, video_info)
}
async fn fetch_videos_detail(
&self,
video: bilibili::Video<'_>,
video_model: video::Model,
connection: &DatabaseConnection,
) -> Result<()> {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await;
match info {
Ok((tags, pages_info)) => {
let txn = connection.begin().await?;
// 将分页信息写入数据库
helper::create_video_pages(&pages_info, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.single_page = Set(Some(pages_info.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
helper::error_fetch_video_detail(e, video_model, connection).await?;
}
};
Ok(())
fn path(&self) -> &Path {
Path::new(self.path.as_str())
}
fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
favorite::ActiveModel {
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
_ActiveModel::Favorite(favorite::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
})
}
fn log_fetch_video_start(&self) {
crates/bili_sync/src/adapter/helper/mod.rs
-112
View File
@@ -1,112 +0,0 @@
use std::path::Path;
use anyhow::Result;
use bili_sync_entity::*;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::OnConflict;
use sea_orm::ActiveValue::Set;
use sea_orm::{Condition, DatabaseTransaction};
use crate::bilibili::{BiliError, PageInfo, VideoInfo};
use crate::config::{PathSafeTemplate, TEMPLATE};
/// 使用 condition 筛选视频，返回视频列表
pub(super) async fn filter_videos(condition: Condition, conn: &DatabaseConnection) -> Result<Vec<video::Model>> {
Ok(video::Entity::find().filter(condition).all(conn).await?)
}
/// 使用 condition 筛选视频，返回视频列表和相关的分 P 列表
pub(super) async fn filter_videos_with_pages(
condition: Condition,
conn: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
Ok(video::Entity::find()
.filter(condition)
.find_with_related(page::Entity)
.all(conn)
.await?)
}
/// 返回设置了 path 的视频
pub(super) fn video_with_path(
mut video_model: video::ActiveModel,
base_path: &str,
video_info: &VideoInfo,
) -> video::ActiveModel {
if let Some(fmt_args) = &video_info.to_fmt_args() {
video_model.path = Set(Path::new(base_path)
.join(
TEMPLATE
.path_safe_render("video", fmt_args)
.expect("template render failed"),
)
.to_string_lossy()
.to_string());
}
video_model
}
/// 处理获取视频详细信息失败的情况
pub(super) async fn error_fetch_video_detail(
e: anyhow::Error,
video_model: bili_sync_entity::video::Model,
connection: &DatabaseConnection,
) -> Result<()> {
error!(
"获取视频 {} - {} 的详细信息失败，错误为：{}",
&video_model.bvid, &video_model.name, e
);
if let Some(BiliError::RequestFailed(-404, _)) = e.downcast_ref::<BiliError>() {
let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into();
video_active_model.valid = Set(false);
video_active_model.save(connection).await?;
}
Ok(())
}
/// 创建视频的所有分 P
pub(crate) async fn create_video_pages(
pages_info: &[PageInfo],
video_model: &video::Model,
connection: &DatabaseTransaction,
) -> Result<()> {
let page_models = pages_info
.iter()
.map(move |p| {
let (width, height) = match &p.dimension {
Some(d) => {
if d.rotate == 0 {
(Some(d.width), Some(d.height))
} else {
(Some(d.height), Some(d.width))
}
}
None => (None, None),
};
page::ActiveModel {
video_id: Set(video_model.id),
cid: Set(p.cid),
pid: Set(p.page),
name: Set(p.name.clone()),
width: Set(width),
height: Set(height),
duration: Set(p.duration),
image: Set(p.first_frame.clone()),
download_status: Set(0),
..Default::default()
}
})
.collect::<Vec<page::ActiveModel>>();
for page_chunk in page_models.chunks(50) {
page::Entity::insert_many(page_chunk.to_vec())
.on_conflict(
OnConflict::columns([page::Column::VideoId, page::Column::Pid])
.do_nothing()
.to_owned(),
)
.do_nothing()
.exec(connection)
.await?;
}
Ok(())
}
crates/bili_sync/src/adapter/mod.rs
+64 -52
View File
@@ -1,6 +1,5 @@
mod collection;
mod favorite;
mod helper;
mod submission;
mod watch_later;
@@ -8,16 +7,53 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::Result;
use async_trait::async_trait;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::SimpleExpr;
use sea_orm::DatabaseConnection;
use crate::adapter::collection::collection_from;
use crate::adapter::favorite::favorite_from;
use crate::adapter::submission::submission_from;
use crate::adapter::watch_later::watch_later_from;
use crate::bilibili::{self, BiliClient, CollectionItem, VideoInfo};
use crate::bilibili::{BiliClient, CollectionItem, VideoInfo};
pub trait VideoListModel {
/// 获取特定视频列表的筛选条件
fn filter_expr(&self) -> SimpleExpr;
// 为 video_model 设置该视频列表的关联 id
fn set_relation_id(&self, video_model: &mut bili_sync_entity::video::ActiveModel);
// 获取视频列表的保存路径
fn path(&self) -> &Path;
/// 获取视频 model 中记录的最新时间
fn get_latest_row_at(&self) -> DateTime;
/// 更新视频 model 中记录的最新时间，此处返回需要更新的 ActiveModel，接着调用 save 方法执行保存
/// 不同 VideoListModel 返回的类型不同，为了 VideoListModel 的 object safety 不能使用 impl Trait
/// Box<dyn ActiveModelTrait> 又提示 ActiveModelTrait 没有 object safety，因此手写一个 Enum 静态分发
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel;
/// 开始获取视频
fn log_fetch_video_start(&self);
/// 结束获取视频
fn log_fetch_video_end(&self);
/// 开始下载视频
fn log_download_video_start(&self);
/// 结束下载视频
fn log_download_video_end(&self);
/// 开始刷新视频
fn log_refresh_video_start(&self);
/// 结束刷新视频
fn log_refresh_video_end(&self, count: usize);
}
pub enum Args<'a> {
Favorite { fid: &'a str },
@@ -40,53 +76,29 @@ pub async fn video_list_from<'a>(
}
}
#[async_trait]
pub trait VideoListModel {
/// 未填充的视频
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<bili_sync_entity::video::Model>>;
/// 未处理的视频和分页
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(bili_sync_entity::video::Model, Vec<bili_sync_entity::page::Model>)>>;
/// 视频信息对应的视频 model
fn video_model_by_info(
&self,
video_info: &VideoInfo,
base_model: Option<bili_sync_entity::video::Model>,
) -> bili_sync_entity::video::ActiveModel;
/// 视频 model 中缺失的信息
async fn fetch_videos_detail(
&self,
video: bilibili::Video<'_>,
video_model: bili_sync_entity::video::Model,
connection: &DatabaseConnection,
) -> Result<()>;
/// 获取视频 model 中记录的最新时间
fn get_latest_row_at(&self) -> DateTime;
/// 更新视频 model 中记录的最新时间
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()>;
/// 开始获取视频
fn log_fetch_video_start(&self);
/// 结束获取视频
fn log_fetch_video_end(&self);
/// 开始下载视频
fn log_download_video_start(&self);
/// 结束下载视频
fn log_download_video_end(&self);
/// 开始刷新视频
fn log_refresh_video_start(&self);
/// 结束刷新视频
fn log_refresh_video_end(&self, count: usize);
pub enum _ActiveModel {
Favorite(bili_sync_entity::favorite::ActiveModel),
Collection(bili_sync_entity::collection::ActiveModel),
Submission(bili_sync_entity::submission::ActiveModel),
WatchLater(bili_sync_entity::watch_later::ActiveModel),
}
impl _ActiveModel {
pub async fn save(self, connection: &DatabaseConnection) -> Result<()> {
match self {
_ActiveModel::Favorite(model) => {
model.save(connection).await?;
}
_ActiveModel::Collection(model) => {
model.save(connection).await?;
}
_ActiveModel::Submission(model) => {
model.save(connection).await?;
}
_ActiveModel::WatchLater(model) => {
model.save(connection).await?;
}
}
Ok(())
}
}
crates/bili_sync/src/adapter/submission.rs
+12 -73
View File
@@ -2,100 +2,39 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result};
use async_trait::async_trait;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::helper::video_with_path;
use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, Submission, VideoInfo};
use crate::utils::status::STATUS_COMPLETED;
use crate::adapter::{VideoListModel, _ActiveModel};
use crate::bilibili::{BiliClient, Submission, VideoInfo};
#[async_trait]
impl VideoListModel for submission::Model {
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::SubmissionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null())
.into_condition(),
connection,
)
.await
fn filter_expr(&self) -> SimpleExpr {
video::Column::SubmissionId.eq(self.id)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
helper::filter_videos_with_pages(
video::Column::SubmissionId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.into_condition(),
connection,
)
.await
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
video_model.submission_id = Set(Some(self.id));
video_with_path(video_model, &self.path, video_info)
}
async fn fetch_videos_detail(
&self,
video: bilibili::Video<'_>,
video_model: video::Model,
connection: &DatabaseConnection,
) -> Result<()> {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
match info {
Ok((tags, view_info)) => {
let VideoInfo::Detail { pages, .. } = &view_info else {
unreachable!("view_info must be VideoInfo::View")
};
let txn = connection.begin().await?;
// 将分页信息写入数据库
helper::create_video_pages(pages, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model = self.video_model_by_info(&view_info, Some(video_model));
video_active_model.single_page = Set(Some(pages.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
helper::error_fetch_video_detail(e, video_model, connection).await?;
}
};
Ok(())
fn path(&self) -> &Path {
Path::new(self.path.as_str())
}
fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
submission::ActiveModel {
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
_ActiveModel::Submission(submission::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
})
}
fn log_fetch_video_start(&self) {
crates/bili_sync/src/adapter/watch_later.rs
+12 -70
View File
@@ -2,97 +2,39 @@ use std::path::Path;
use std::pin::Pin;
use anyhow::{Context, Result};
use async_trait::async_trait;
use bili_sync_entity::*;
use futures::Stream;
use sea_orm::entity::prelude::*;
use sea_orm::sea_query::{IntoCondition, OnConflict};
use sea_orm::sea_query::{OnConflict, SimpleExpr};
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
use sea_orm::{DatabaseConnection, Unchanged};
use crate::adapter::helper::video_with_path;
use crate::adapter::{helper, VideoListModel};
use crate::bilibili::{self, BiliClient, VideoInfo, WatchLater};
use crate::utils::status::STATUS_COMPLETED;
use crate::adapter::{VideoListModel, _ActiveModel};
use crate::bilibili::{BiliClient, VideoInfo, WatchLater};
#[async_trait]
impl VideoListModel for watch_later::Model {
async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
helper::filter_videos(
video::Column::WatchLaterId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.eq(0))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_null())
.into_condition(),
connection,
)
.await
fn filter_expr(&self) -> SimpleExpr {
video::Column::WatchLaterId.eq(self.id)
}
async fn unhandled_video_pages(
&self,
connection: &DatabaseConnection,
) -> Result<Vec<(video::Model, Vec<page::Model>)>> {
helper::filter_videos_with_pages(
video::Column::WatchLaterId
.eq(self.id)
.and(video::Column::Valid.eq(true))
.and(video::Column::DownloadStatus.lt(STATUS_COMPLETED))
.and(video::Column::Category.eq(2))
.and(video::Column::SinglePage.is_not_null())
.into_condition(),
connection,
)
.await
}
fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
let mut video_model = video_info.to_model(base_model);
fn set_relation_id(&self, video_model: &mut video::ActiveModel) {
video_model.watch_later_id = Set(Some(self.id));
video_with_path(video_model, &self.path, video_info)
}
async fn fetch_videos_detail(
&self,
video: bilibili::Video<'_>,
video_model: video::Model,
connection: &DatabaseConnection,
) -> Result<()> {
let info: Result<_> = async { Ok((video.get_tags().await?, video.get_pages().await?)) }.await;
match info {
Ok((tags, pages_info)) => {
let txn = connection.begin().await?;
// 将分页信息写入数据库
helper::create_video_pages(&pages_info, &video_model, &txn).await?;
// 将页标记和 tag 写入数据库
let mut video_active_model: video::ActiveModel = video_model.into();
video_active_model.single_page = Set(Some(pages_info.len() == 1));
video_active_model.tags = Set(Some(serde_json::to_value(tags)?));
video_active_model.save(&txn).await?;
txn.commit().await?;
}
Err(e) => {
helper::error_fetch_video_detail(e, video_model, connection).await?;
}
};
Ok(())
fn path(&self) -> &Path {
Path::new(self.path.as_str())
}
fn get_latest_row_at(&self) -> DateTime {
self.latest_row_at
}
async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
watch_later::ActiveModel {
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel {
_ActiveModel::WatchLater(watch_later::ActiveModel {
id: Unchanged(self.id),
latest_row_at: Set(datetime),
..Default::default()
}
.update(connection)
.await?;
Ok(())
})
}
fn log_fetch_video_start(&self) {