feat: 支持手动触发全量更新，清除本地多余的视频条目与文件 (#678)

crates/bili_sync/src/api/request.rs

@@ -150,3 +150,8 @@ pub struct DefaultPathRequest {
 pub struct PollQrcodeRequest {
     pub qrcode_key: String,
 }
+
+#[derive(Debug, Deserialize)]
+pub struct FullSyncVideoSourceRequest {
+    pub delete_local: bool,
+}

crates/bili_sync/src/api/response.rs

@@ -229,3 +229,9 @@ pub struct UpdateVideoSourceResponse {
 pub type GenerateQrcodeResponse = Qrcode;
 
 pub type PollQrcodeResponse = PollStatus;
+
+#[derive(Serialize)]
+pub struct FullSyncVideoSourceResponse {
+    pub removed_count: usize,
+    pub warnings: Option<Vec<String>>,
+}

crates/bili_sync/src/api/routes/video_sources/mod.rs

@@ -1,12 +1,15 @@
+use std::collections::HashSet;
 use std::sync::Arc;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
-use axum::Router;
 use axum::extract::{Extension, Path, Query};
 use axum::routing::{get, post, put};
+use axum::{Json, Router};
 use bili_sync_entity::rule::Rule;
 use bili_sync_entity::*;
 use bili_sync_migration::Expr;
+use futures::stream::FuturesUnordered;
+use futures::{StreamExt, TryStreamExt};
 use itertools::Itertools;
 use sea_orm::ActiveValue::Set;
 use sea_orm::entity::prelude::*;
@@ -15,11 +18,12 @@ use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QuerySelect, QueryTr
 use crate::adapter::{_ActiveModel, VideoSource as _, VideoSourceEnum};
 use crate::api::error::InnerApiError;
 use crate::api::request::{
-    DefaultPathRequest, InsertCollectionRequest, InsertFavoriteRequest, InsertSubmissionRequest,
+    DefaultPathRequest, FullSyncVideoSourceRequest, InsertCollectionRequest, InsertFavoriteRequest,
-    UpdateVideoSourceRequest,
+    InsertSubmissionRequest, UpdateVideoSourceRequest,
 };
 use crate::api::response::{
-    UpdateVideoSourceResponse, VideoSource, VideoSourceDetail, VideoSourcesDetailsResponse, VideoSourcesResponse,
+    FullSyncVideoSourceResponse, UpdateVideoSourceResponse, VideoSource, VideoSourceDetail,
+    VideoSourcesDetailsResponse, VideoSourcesResponse,
 };
 use crate::api::wrapper::{ApiError, ApiResponse, ValidatedJson};
 use crate::bilibili::{BiliClient, Collection, CollectionItem, FavoriteList, Submission};
@@ -39,6 +43,7 @@ pub(super) fn router() -> Router {
             put(update_video_source).delete(remove_video_source),
         )
         .route("/video-sources/{type}/{id}/evaluate", post(evaluate_video_source))
+        .route("/video-sources/{type}/{id}/full-sync", post(full_sync_video_source))
         .route("/video-sources/favorites", post(insert_favorite))
         .route("/video-sources/collections", post(insert_collection))
         .route("/video-sources/submissions", post(insert_submission))
@@ -356,6 +361,80 @@ pub async fn evaluate_video_source(
     Ok(ApiResponse::ok(true))
 }
 
+pub async fn full_sync_video_source(
+    Path((source_type, id)): Path<(String, i32)>,
+    Extension(db): Extension<DatabaseConnection>,
+    Extension(bili_client): Extension<Arc<BiliClient>>,
+    Json(request): Json<FullSyncVideoSourceRequest>,
+) -> Result<ApiResponse<FullSyncVideoSourceResponse>, ApiError> {
+    let video_source: Option<VideoSourceEnum> = match source_type.as_str() {
+        "collections" => collection::Entity::find_by_id(id).one(&db).await?.map(Into::into),
+        "favorites" => favorite::Entity::find_by_id(id).one(&db).await?.map(Into::into),
+        "submissions" => submission::Entity::find_by_id(id).one(&db).await?.map(Into::into),
+        "watch_later" => watch_later::Entity::find_by_id(id).one(&db).await?.map(Into::into),
+        _ => return Err(InnerApiError::BadRequest("Invalid video source type".to_string()).into()),
+    };
+    let Some(video_source) = video_source else {
+        return Err(InnerApiError::NotFound(id).into());
+    };
+    let credential = &VersionedConfig::get().read().credential;
+    let filter_expr = video_source.filter_expr();
+    let (_, video_streams) = video_source.refresh(&bili_client, credential, &db).await?;
+    let all_videos = video_streams
+        .try_collect::<Vec<_>>()
+        .await
+        .context("failed to read all videos from video stream")?;
+    let all_bvids = all_videos.into_iter().map(|v| v.bvid_owned()).collect::<HashSet<_>>();
+    let videos_to_remove = video::Entity::find()
+        .filter(video::Column::Bvid.is_not_in(all_bvids).and(filter_expr))
+        .select_only()
+        .columns([video::Column::Id, video::Column::Path])
+        .into_tuple::<(i32, String)>()
+        .all(&db)
+        .await?;
+    if videos_to_remove.is_empty() {
+        return Ok(ApiResponse::ok(FullSyncVideoSourceResponse {
+            removed_count: 0,
+            warnings: None,
+        }));
+    }
+    let remove_count = videos_to_remove.len();
+    let (video_ids, video_paths): (Vec<i32>, Vec<String>) = videos_to_remove.into_iter().unzip();
+    let txn = db.begin().await?;
+    page::Entity::delete_many()
+        .filter(page::Column::VideoId.is_in(video_ids.iter().copied()))
+        .exec(&txn)
+        .await?;
+    video::Entity::delete_many()
+        .filter(video::Column::Id.is_in(video_ids))
+        .exec(&txn)
+        .await?;
+    txn.commit().await?;
+    let warnings = if request.delete_local {
+        let tasks = video_paths
+            .into_iter()
+            .map(|path| async move {
+                tokio::fs::remove_dir_all(&path)
+                    .await
+                    .with_context(|| format!("failed to remove {path}"))?;
+                Result::<_, anyhow::Error>::Ok(())
+            })
+            .collect::<FuturesUnordered<_>>();
+        Some(
+            tasks
+                .filter_map(|res| futures::future::ready(res.err().map(|e| format!("{:#}", e))))
+                .collect::<Vec<_>>()
+                .await,
+        )
+    } else {
+        None
+    };
+    Ok(ApiResponse::ok(FullSyncVideoSourceResponse {
+        removed_count: remove_count,
+        warnings,
+    }))
+}
+
 /// 新增收藏夹订阅
 pub async fn insert_favorite(
     Extension(db): Extension<DatabaseConnection>,

crates/bili_sync/src/utils/convert.rs

@@ -182,6 +182,17 @@ impl VideoInfo {
             VideoInfo::Detail { .. } => unreachable!(),
         }
     }
+
+    pub fn bvid_owned(self) -> String {
+        match self {
+            VideoInfo::Collection { bvid, .. }
+            | VideoInfo::Favorite { bvid, .. }
+            | VideoInfo::WatchLater { bvid, .. }
+            | VideoInfo::Submission { bvid, .. }
+            | VideoInfo::Dynamic { bvid, .. }
+            | VideoInfo::Detail { bvid, .. } => bvid,
+        }
+    }
 }
 
 impl PageInfo {

web/src/lib/api.ts

@@ -6,6 +6,8 @@ import type {
 	Config,
 	DashBoardResponse,
 	FavoritesResponse,
+	FullSyncVideoSourceRequest,
+	FullSyncVideoSourceResponse,
 	QrcodeGenerateResponse as GenerateQrcodeResponse,
 	InsertCollectionRequest,
 	InsertFavoriteRequest,
@@ -253,6 +255,14 @@ class ApiClient {
 		return this.post<boolean>(`/video-sources/${type}/${id}/evaluate`, null);
 	}
 
+	async fullSyncVideoSource(
+		type: string,
+		id: number,
+		data: FullSyncVideoSourceRequest
+	): Promise<ApiResponse<FullSyncVideoSourceResponse>> {
+		return this.post<FullSyncVideoSourceResponse>(`/video-sources/${type}/${id}/full-sync`, data);
+	}
+
 	async getDefaultPath(type: string, name: string): Promise<ApiResponse<string>> {
 		return this.get<string>(`/video-sources/${type}/default-path`, { name });
 	}
@@ -327,6 +337,8 @@ const api = {
 	removeVideoSource: (type: string, id: number) => apiClient.removeVideoSource(type, id),
 	evaluateVideoSourceRules: (type: string, id: number) =>
 		apiClient.evaluateVideoSourceRules(type, id),
+	fullSyncVideoSource: (type: string, id: number, data: { delete_local: boolean }) =>
+		apiClient.fullSyncVideoSource(type, id, data),
 	getDefaultPath: (type: string, name: string) => apiClient.getDefaultPath(type, name),
 	testNotifier: (notifier: Notifier) => apiClient.testNotifier(notifier),
 	getConfig: () => apiClient.getConfig(),

web/src/lib/types.ts

@@ -89,7 +89,16 @@ export interface UpdateFilteredVideoStatusResponse {
 
 export interface ApiError {
 	message: string;
-	status?: number;
+	status: number;
+}
+
+export interface FullSyncVideoSourceRequest {
+	delete_local: boolean;
+}
+
+export interface FullSyncVideoSourceResponse {
+	removed_count: number;
+	warnings?: string[];
 }
 
 export interface StatusUpdate {

web/src/routes/video-sources/+page.svelte

@@ -18,7 +18,8 @@
 		InfoIcon,
 		Trash2Icon,
 		CircleCheckBigIcon,
-		CircleXIcon
+		CircleXIcon,
+		RefreshCwIcon
 	} from '@lucide/svelte/icons';
 	import * as Tooltip from '$lib/components/ui/tooltip/index.js';
 	import { toast } from 'svelte-sonner';
@@ -58,6 +59,13 @@
 	let removeIdx: number = 0;
 	let removing = false;
 
+	// 全量更新对话框状态
+	let showFullSyncDialog = false;
+	let fullSyncSource: VideoSourceDetail | null = null;
+	let fullSyncType = '';
+	let fullSyncDeleteLocal = false;
+	let fullSyncing = false;
+
 	// 编辑表单数据
 	let editForm = {
 		path: '',
@@ -120,6 +128,44 @@
 		showRemoveDialog = true;
 	}
 
+	function openFullSyncDialog(type: string, source: VideoSourceDetail) {
+		fullSyncSource = source;
+		fullSyncType = type;
+		fullSyncDeleteLocal = false;
+		showFullSyncDialog = true;
+	}
+
+	async function fullSyncVideoSource() {
+		if (!fullSyncSource) return;
+		fullSyncing = true;
+		try {
+			let response = await api.fullSyncVideoSource(fullSyncType, fullSyncSource.id, {
+				delete_local: fullSyncDeleteLocal
+			});
+			if (response && response.data) {
+				showFullSyncDialog = false;
+				toast.success('全量更新成功', {
+					description: `已移除 ${response.data.removed_count} 个不存在的视频`
+				});
+				if (response.data.warnings && response.data.warnings.length > 0) {
+					toast.warning('部分本地文件夹删除失败', {
+						description: response.data.warnings.join('\n'),
+						duration: 10000,
+						descriptionClass: 'whitespace-pre-line'
+					});
+				}
+			} else {
+				toast.error('全量更新失败');
+			}
+		} catch (error) {
+			toast.error('全量更新失败', {
+				description: (error as ApiError).message
+			});
+		} finally {
+			fullSyncing = false;
+		}
+	}
+
 	// 保存编辑
 	async function saveEdit() {
 		if (!editingSource) return;
@@ -410,6 +456,21 @@
 														<p class="text-xs">重新评估规则</p>
 													</Tooltip.Content>
 												</Tooltip.Root>
+												<Tooltip.Root disableHoverableContent={true}>
+													<Tooltip.Trigger>
+														<Button
+															size="sm"
+															variant="outline"
+															onclick={() => openFullSyncDialog(key, source)}
+															class="h-8 w-8 p-0"
+														>
+															<RefreshCwIcon class="h-3 w-3" />
+														</Button>
+													</Tooltip.Trigger>
+													<Tooltip.Content>
+														<p class="text-xs">全量更新视频</p>
+													</Tooltip.Content>
+												</Tooltip.Root>
 												{#if activeTab !== 'watch_later'}
 													<Tooltip.Root disableHoverableContent={true}>
 														<Tooltip.Trigger>
@@ -581,6 +642,44 @@
 		</AlertDialog.Content>
 	</AlertDialog.Root>
 
+	<AlertDialog.Root bind:open={showFullSyncDialog}>
+		<AlertDialog.Content>
+			<AlertDialog.Header>
+				<AlertDialog.Title>全量更新视频</AlertDialog.Title>
+				<AlertDialog.Description>
+					确定要全量更新视频源 <strong>"{fullSyncSource?.name}"</strong> 吗？<br />
+					该操作会拉取该视频源下所有当前存在的视频，移除数据库中已不存在于该源的视频及其分页数据，<span
+						class="text-destructive font-medium">无法撤销</span
+					>。<br />
+				</AlertDialog.Description>
+			</AlertDialog.Header>
+			<div class="flex items-center space-x-2 py-2">
+				<input
+					type="checkbox"
+					id="delete-local"
+					bind:checked={fullSyncDeleteLocal}
+					class="text-primary focus:ring-primary h-4 w-4 rounded border-gray-300"
+				/>
+				<label for="delete-local" class="text-sm font-medium"> 同时删除本地视频文件夹 </label>
+			</div>
+			<AlertDialog.Footer>
+				<AlertDialog.Cancel
+					disabled={fullSyncing}
+					onclick={() => {
+						showFullSyncDialog = false;
+					}}>取消</AlertDialog.Cancel
+				>
+				<AlertDialog.Action
+					onclick={fullSyncVideoSource}
+					disabled={fullSyncing}
+					class="bg-amber-600 hover:bg-amber-700"
+				>
+					{fullSyncing ? '全量更新中' : '确认全量更新'}
+				</AlertDialog.Action>
+			</AlertDialog.Footer>
+		</AlertDialog.Content>
+	</AlertDialog.Root>
+
 	<!-- 添加对话框 -->
 	<Dialog.Root bind:open={showAddDialog}>
 		<Dialog.Content>