feat: 引入更健壮的新视频检测方法 (#228)

* feat: 为各个 video list 表添加 latest_row_at 字段

* chore: 为 model 引入新增的字段

* feat: 实现新版中断条件（待测试）

* test: 更新测试

crates/bili_sync/src/adapter/collection.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::path::Path;
 use std::pin::Pin;
 
@@ -9,7 +8,7 @@ use futures::Stream;
 use sea_orm::entity::prelude::*;
 use sea_orm::sea_query::{IntoCondition, OnConflict};
 use sea_orm::ActiveValue::Set;
-use sea_orm::{DatabaseConnection, TransactionTrait};
+use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
 
 use crate::adapter::{helper, VideoListModel};
 use crate::bilibili::{self, BiliClient, Collection, CollectionItem, CollectionType, VideoInfo};
@@ -17,10 +16,6 @@ use crate::utils::status::Status;
 
 #[async_trait]
 impl VideoListModel for collection::Model {
-    async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
-        helper::count_videos(video::Column::CollectionId.eq(self.id).into_condition(), connection).await
-    }
-
     async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
         helper::filter_videos(
             video::Column::CollectionId
@@ -52,20 +47,6 @@ impl VideoListModel for collection::Model {
         .await
     }
 
-    async fn exist_labels(
-        &self,
-        videos_info: &[VideoInfo],
-        connection: &DatabaseConnection,
-    ) -> Result<HashSet<String>> {
-        helper::video_keys(
-            video::Column::CollectionId.eq(self.id),
-            videos_info,
-            [video::Column::Bvid, video::Column::Pubtime],
-            connection,
-        )
-        .await
-    }
-
     fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
         let mut video_model = video_info.to_model(base_model);
         video_model.collection_id = Set(Some(self.id));
@@ -81,7 +62,7 @@ impl VideoListModel for collection::Model {
         let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
         match info {
             Ok((tags, view_info)) => {
-                let VideoInfo::View { pages, .. } = &view_info else {
+                let VideoInfo::Detail { pages, .. } = &view_info else {
                     unreachable!("view_info must be VideoInfo::View")
                 };
                 let txn = connection.begin().await?;
@@ -101,6 +82,21 @@ impl VideoListModel for collection::Model {
         Ok(())
     }
 
+    fn get_latest_row_at(&self) -> DateTime {
+        self.latest_row_at
+    }
+
+    async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
+        collection::ActiveModel {
+            id: Unchanged(self.id),
+            latest_row_at: Set(datetime),
+            ..Default::default()
+        }
+        .update(connection)
+        .await?;
+        Ok(())
+    }
+
     fn log_fetch_video_start(&self) {
         info!(
             "开始获取{} {} - {} 的视频与分页信息...",
@@ -146,14 +142,13 @@ impl VideoListModel for collection::Model {
         );
     }
 
-    fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
+    fn log_refresh_video_end(&self, count: usize) {
         info!(
-            "扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
+            "扫描{}: {} - {} 的新视频完成，获取了 {} 条新视频",
             CollectionType::from(self.r#type),
             self.s_id,
             self.name,
-            got_count,
-            new_count,
+            count,
         );
     }
 }

crates/bili_sync/src/adapter/favorite.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::path::Path;
 use std::pin::Pin;
 
@@ -9,7 +8,7 @@ use futures::Stream;
 use sea_orm::entity::prelude::*;
 use sea_orm::sea_query::{IntoCondition, OnConflict};
 use sea_orm::ActiveValue::Set;
-use sea_orm::{DatabaseConnection, TransactionTrait};
+use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
 
 use crate::adapter::{helper, VideoListModel};
 use crate::bilibili::{self, BiliClient, FavoriteList, VideoInfo};
@@ -17,10 +16,6 @@ use crate::utils::status::Status;
 
 #[async_trait]
 impl VideoListModel for favorite::Model {
-    async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
-        helper::count_videos(video::Column::FavoriteId.eq(self.id).into_condition(), connection).await
-    }
-
     async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
         helper::filter_videos(
             video::Column::FavoriteId
@@ -52,20 +47,6 @@ impl VideoListModel for favorite::Model {
         .await
     }
 
-    async fn exist_labels(
-        &self,
-        videos_info: &[VideoInfo],
-        connection: &DatabaseConnection,
-    ) -> Result<HashSet<String>> {
-        helper::video_keys(
-            video::Column::FavoriteId.eq(self.id),
-            videos_info,
-            [video::Column::Bvid, video::Column::Favtime],
-            connection,
-        )
-        .await
-    }
-
     fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
         let mut video_model = video_info.to_model(base_model);
         video_model.favorite_id = Set(Some(self.id));
@@ -98,6 +79,21 @@ impl VideoListModel for favorite::Model {
         Ok(())
     }
 
+    fn get_latest_row_at(&self) -> DateTime {
+        self.latest_row_at
+    }
+
+    async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
+        favorite::ActiveModel {
+            id: Unchanged(self.id),
+            latest_row_at: Set(datetime),
+            ..Default::default()
+        }
+        .update(connection)
+        .await?;
+        Ok(())
+    }
+
     fn log_fetch_video_start(&self) {
         info!("开始获取收藏夹 {} - {} 的视频与分页信息...", self.f_id, self.name);
     }
@@ -118,10 +114,10 @@ impl VideoListModel for favorite::Model {
         info!("开始扫描收藏夹: {} - {} 的新视频...", self.f_id, self.name);
     }
 
-    fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
+    fn log_refresh_video_end(&self, count: usize) {
         info!(
-            "扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
-            self.f_id, self.name, got_count, new_count
+            "扫描收藏夹: {} - {} 的新视频完成，获取了 {} 条新视频",
+            self.f_id, self.name, count
         );
     }
 }

crates/bili_sync/src/adapter/helper/mod.rs

@@ -1,21 +1,14 @@
-use std::collections::HashSet;
 use std::path::Path;
 
 use anyhow::Result;
 use bili_sync_entity::*;
 use sea_orm::entity::prelude::*;
-use sea_orm::sea_query::{OnConflict, SimpleExpr};
+use sea_orm::sea_query::OnConflict;
 use sea_orm::ActiveValue::Set;
-use sea_orm::{Condition, DatabaseTransaction, QuerySelect};
+use sea_orm::{Condition, DatabaseTransaction};
 
 use crate::bilibili::{BiliError, PageInfo, VideoInfo};
 use crate::config::{PathSafeTemplate, TEMPLATE};
-use crate::utils::id_time_key;
-
-/// 使用 condition 筛选视频，返回视频数量
-pub(super) async fn count_videos(condition: Condition, conn: &DatabaseConnection) -> Result<u64> {
-    Ok(video::Entity::find().filter(condition).count(conn).await?)
-}
 
 /// 使用 condition 筛选视频，返回视频列表
 pub(super) async fn filter_videos(condition: Condition, conn: &DatabaseConnection) -> Result<Vec<video::Model>> {
@@ -34,29 +27,6 @@ pub(super) async fn filter_videos_with_pages(
         .await?)
 }
 
-/// 返回 videos_info 存在于视频表里那部分对应的 key
-pub(super) async fn video_keys(
-    expr: SimpleExpr,
-    videos_info: &[VideoInfo],
-    columns: [video::Column; 2],
-    conn: &DatabaseConnection,
-) -> Result<HashSet<String>> {
-    Ok(video::Entity::find()
-        .filter(
-            video::Column::Bvid
-                .is_in(videos_info.iter().map(|v| v.bvid().to_string()))
-                .and(expr),
-        )
-        .select_only()
-        .columns(columns)
-        .into_tuple()
-        .all(conn)
-        .await?
-        .into_iter()
-        .map(|(bvid, time)| id_time_key(&bvid, &time))
-        .collect())
-}
-
 /// 返回设置了 path 的视频
 pub(super) fn video_with_path(
     mut video_model: video::ActiveModel,

crates/bili_sync/src/adapter/mod.rs

@@ -4,7 +4,6 @@ mod helper;
 mod submission;
 mod watch_later;
 
-use std::collections::HashSet;
 use std::path::Path;
 use std::pin::Pin;
 
@@ -43,9 +42,6 @@ pub async fn video_list_from<'a>(
 
 #[async_trait]
 pub trait VideoListModel {
-    /// 与视频列表关联的视频总数
-    async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64>;
-
     /// 未填充的视频
     async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<bili_sync_entity::video::Model>>;
 
@@ -55,10 +51,6 @@ pub trait VideoListModel {
         connection: &DatabaseConnection,
     ) -> Result<Vec<(bili_sync_entity::video::Model, Vec<bili_sync_entity::page::Model>)>>;
 
-    /// 该批次视频的存在标记
-    async fn exist_labels(&self, videos_info: &[VideoInfo], connection: &DatabaseConnection)
-        -> Result<HashSet<String>>;
-
     /// 视频信息对应的视频 model
     fn video_model_by_info(
         &self,
@@ -74,6 +66,12 @@ pub trait VideoListModel {
         connection: &DatabaseConnection,
     ) -> Result<()>;
 
+    /// 获取视频 model 中记录的最新时间
+    fn get_latest_row_at(&self) -> DateTime;
+
+    /// 更新视频 model 中记录的最新时间
+    async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()>;
+
     /// 开始获取视频
     fn log_fetch_video_start(&self);
 
@@ -90,5 +88,5 @@ pub trait VideoListModel {
     fn log_refresh_video_start(&self);
 
     /// 结束刷新视频
-    fn log_refresh_video_end(&self, got_count: usize, new_count: u64);
+    fn log_refresh_video_end(&self, count: usize);
 }

crates/bili_sync/src/adapter/submission.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::path::Path;
 use std::pin::Pin;
 
@@ -9,7 +8,7 @@ use futures::Stream;
 use sea_orm::entity::prelude::*;
 use sea_orm::sea_query::{IntoCondition, OnConflict};
 use sea_orm::ActiveValue::Set;
-use sea_orm::{DatabaseConnection, TransactionTrait};
+use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
 
 use crate::adapter::helper::video_with_path;
 use crate::adapter::{helper, VideoListModel};
@@ -18,10 +17,6 @@ use crate::utils::status::Status;
 
 #[async_trait]
 impl VideoListModel for submission::Model {
-    async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
-        helper::count_videos(video::Column::SubmissionId.eq(self.id).into_condition(), connection).await
-    }
-
     async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
         helper::filter_videos(
             video::Column::SubmissionId
@@ -53,20 +48,6 @@ impl VideoListModel for submission::Model {
         .await
     }
 
-    async fn exist_labels(
-        &self,
-        videos_info: &[VideoInfo],
-        connection: &DatabaseConnection,
-    ) -> Result<HashSet<String>> {
-        helper::video_keys(
-            video::Column::SubmissionId.eq(self.id),
-            videos_info,
-            [video::Column::Bvid, video::Column::Ctime],
-            connection,
-        )
-        .await
-    }
-
     fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
         let mut video_model = video_info.to_model(base_model);
         video_model.submission_id = Set(Some(self.id));
@@ -82,7 +63,7 @@ impl VideoListModel for submission::Model {
         let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
         match info {
             Ok((tags, view_info)) => {
-                let VideoInfo::View { pages, .. } = &view_info else {
+                let VideoInfo::Detail { pages, .. } = &view_info else {
                     unreachable!("view_info must be VideoInfo::View")
                 };
                 let txn = connection.begin().await?;
@@ -102,6 +83,21 @@ impl VideoListModel for submission::Model {
         Ok(())
     }
 
+    fn get_latest_row_at(&self) -> DateTime {
+        self.latest_row_at
+    }
+
+    async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
+        submission::ActiveModel {
+            id: Unchanged(self.id),
+            latest_row_at: Set(datetime),
+            ..Default::default()
+        }
+        .update(connection)
+        .await?;
+        Ok(())
+    }
+
     fn log_fetch_video_start(&self) {
         info!(
             "开始获取 UP 主 {} - {} 投稿的视频与分页信息...",
@@ -134,10 +130,10 @@ impl VideoListModel for submission::Model {
         info!("开始扫描 UP 主 {} - {} 投稿的新视频...", self.upper_id, self.upper_name);
     }
 
-    fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
+    fn log_refresh_video_end(&self, count: usize) {
         info!(
-            "扫描 UP 主 {} - {} 投稿的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
-            self.upper_id, self.upper_name, got_count, new_count,
+            "扫描 UP 主 {} - {} 投稿的新视频完成，获取了 {} 条新视频",
+            self.upper_id, self.upper_name, count,
         );
     }
 }

crates/bili_sync/src/adapter/watch_later.rs

@@ -1,4 +1,3 @@
-use std::collections::HashSet;
 use std::path::Path;
 use std::pin::Pin;
 
@@ -9,7 +8,7 @@ use futures::Stream;
 use sea_orm::entity::prelude::*;
 use sea_orm::sea_query::{IntoCondition, OnConflict};
 use sea_orm::ActiveValue::Set;
-use sea_orm::{DatabaseConnection, TransactionTrait};
+use sea_orm::{DatabaseConnection, TransactionTrait, Unchanged};
 
 use crate::adapter::helper::video_with_path;
 use crate::adapter::{helper, VideoListModel};
@@ -18,10 +17,6 @@ use crate::utils::status::Status;
 
 #[async_trait]
 impl VideoListModel for watch_later::Model {
-    async fn video_count(&self, connection: &DatabaseConnection) -> Result<u64> {
-        helper::count_videos(video::Column::WatchLaterId.eq(self.id).into_condition(), connection).await
-    }
-
     async fn unfilled_videos(&self, connection: &DatabaseConnection) -> Result<Vec<video::Model>> {
         helper::filter_videos(
             video::Column::WatchLaterId
@@ -53,20 +48,6 @@ impl VideoListModel for watch_later::Model {
         .await
     }
 
-    async fn exist_labels(
-        &self,
-        videos_info: &[VideoInfo],
-        connection: &DatabaseConnection,
-    ) -> Result<HashSet<String>> {
-        helper::video_keys(
-            video::Column::WatchLaterId.eq(self.id),
-            videos_info,
-            [video::Column::Bvid, video::Column::Favtime],
-            connection,
-        )
-        .await
-    }
-
     fn video_model_by_info(&self, video_info: &VideoInfo, base_model: Option<video::Model>) -> video::ActiveModel {
         let mut video_model = video_info.to_model(base_model);
         video_model.watch_later_id = Set(Some(self.id));
@@ -99,6 +80,21 @@ impl VideoListModel for watch_later::Model {
         Ok(())
     }
 
+    fn get_latest_row_at(&self) -> DateTime {
+        self.latest_row_at
+    }
+
+    async fn update_latest_row_at(&self, datetime: DateTime, connection: &DatabaseConnection) -> Result<()> {
+        watch_later::ActiveModel {
+            id: Unchanged(self.id),
+            latest_row_at: Set(datetime),
+            ..Default::default()
+        }
+        .update(connection)
+        .await?;
+        Ok(())
+    }
+
     fn log_fetch_video_start(&self) {
         info!("开始获取稍后再看的视频与分页信息...");
     }
@@ -119,11 +115,8 @@ impl VideoListModel for watch_later::Model {
         info!("开始扫描稍后再看的新视频...");
     }
 
-    fn log_refresh_video_end(&self, got_count: usize, new_count: u64) {
-        info!(
-            "扫描稍后再看的新视频完成，获取了 {} 条新视频，其中有 {} 条新视频",
-            got_count, new_count,
-        );
+    fn log_refresh_video_end(&self, count: usize) {
+        info!("扫描稍后再看的新视频完成，获取了 {} 条新视频", count);
     }
 }