chore: 避免请求中包含多行 cookie，从 cookie 中移除 ac_time_value

Cargo.toml

@@ -4,7 +4,7 @@ default-members = ["crates/bili_sync"]
 resolver = "2"
 
 [workspace.package]
-version = "2.11.0"
+version = "2.11.1"
 authors = ["amtoaer <amtoaer@gmail.com>"]
 license = "MIT"
 description = "由 Rust & Tokio 驱动的哔哩哔哩同步工具"
@@ -15,40 +15,40 @@ publish = false
 bili_sync_entity = { path = "crates/bili_sync_entity" }
 bili_sync_migration = { path = "crates/bili_sync_migration" }
 
-anyhow = { version = "1.0.100", features = ["backtrace"] }
-arc-swap = { version = "1.8.0", features = ["serde"] }
+anyhow = { version = "1.0.102", features = ["backtrace"] }
+arc-swap = { version = "1.9.1", features = ["serde"] }
 async-stream = "0.3.6"
-async-tempfile = { version = "0.7.0", features = ["uuid"] }
+async-tempfile = { version = "0.8.0", features = ["uuid"] }
 async-trait = "0.1.89"
-axum = { version = "0.8.8", features = ["macros", "ws"] }
+axum = { version = "0.8.9", features = ["macros", "ws"] }
 base64 = "0.22.1"
-built = { version = "0.7.7", features = ["git2", "chrono"] }
-chrono = { version = "0.4.42", features = ["serde"] }
-clap = { version = "4.5.54", features = ["env", "string"] }
+built = { version = "0.8.1", features = ["git2", "chrono"] }
+chrono = { version = "0.4.45", features = ["serde"] }
+clap = { version = "4.6.1", features = ["env", "string"] }
 cookie = "0.18.1"
 croner = "3.0.1"
-dashmap = "6.1.0"
+dashmap = "6.2.1"
 derivative = "2.2.0"
 dirs = "6.0.0"
 dunce = "1.0.5"
-either = "1.15.0"
+either = "1.16.0"
 enum_dispatch = "0.3.13"
 float-ord = "0.3.2"
-futures = "0.3.31"
-git2 = { version = "0.20.3", features = [], default-features = false }
-handlebars = "6.4.0"
+futures = "0.3.32"
+git2 = { version = "0.21.0", features = [], default-features = false }
+handlebars = "6.4.1"
 hex = "0.4.3"
 itertools = "0.14.0"
 leaky-bucket = "1.1.2"
 md5 = "0.8.0"
-memchr = "2.7.6"
-once_cell = "1.21.3"
+memchr = "2.8.1"
+once_cell = "1.21.4"
 parking_lot = "0.12.5"
-prost = "0.14.1"
-quick-xml = { version = "0.38.4", features = ["async-tokio"] }
-rand = "0.9.2"
-regex = "1.12.2"
-reqwest = { version = "0.13.1", features = [
+prost = "0.14.4"
+quick-xml = { version = "0.40.1", features = ["async-tokio"] }
+rand = "0.10.1"
+regex = "1.12.3"
+reqwest = { version = "0.13.4", features = [
     "query",
     "form",
     "charset",
@@ -59,32 +59,32 @@ reqwest = { version = "0.13.1", features = [
     "rustls-no-provider",
     "stream",
 ], default-features = false }
-rsa = { version = "0.10.0-rc.9", features = ["sha2"] }
+rsa = { version = "0.10.0-rc.18", features = ["sha2"] }
 rust-embed-for-web = { git = "https://github.com/amtoaer/rust-embed-for-web", tag = "v1.0.0" }
-rustls = { version = "0.23.36", default-features = false, features = ["ring"] }
-sea-orm = { version = "1.1.19", features = [
+rustls = { version = "0.23.40", default-features = false, features = ["ring"] }
+sea-orm = { version = "1.1.20", features = [
     "macros",
     "runtime-tokio",
     "sqlx-sqlite",
     "sqlite-use-returning-for-3_35",
 ] }
-sea-orm-migration = { version = "1.1.19", features = [] }
+sea-orm-migration = { version = "1.1.20", features = [] }
 serde = { version = "1.0.228", features = ["derive"] }
-serde_json = "1.0.148"
+serde_json = "1.0.150"
 serde_urlencoded = "0.7.1"
-strum = { version = "0.27.2", features = ["derive"] }
-sysinfo = "0.37.2"
-thiserror = "2.0.17"
-tokio = { version = "1.49.0", features = ["full"] }
+strum = { version = "0.28.0", features = ["derive"] }
+sysinfo = "0.39.3"
+thiserror = "2.0.18"
+tokio = { version = "1.52.3", features = ["full"] }
 tokio-cron-scheduler = "0.15.1"
 tokio-stream = { version = "0.1.18", features = ["sync"] }
 tokio-util = { version = "0.7.18", features = ["io", "rt"] }
-toml = "0.9.10"
-tower = "0.5.2"
+toml = "1.1.2"
+tower = "0.5.3"
 tracing = "0.1.44"
-tracing-subscriber = { version = "0.3.22", features = ["chrono", "json"] }
-ua_generator = { version = "0.5.42", default-features = false }
-uuid = { version = "1.19.0", features = ["v4"] }
+tracing-subscriber = { version = "0.3.23", features = ["chrono", "json"] }
+ua_generator = { version = "0.5.65", default-features = false }
+uuid = { version = "1.23.2", features = ["v4"] }
 validator = { version = "0.20.0", features = ["derive"] }
 
 [workspace.metadata.release]

crates/bili_sync/src/api/response.rs

@@ -1,5 +1,6 @@
 use bili_sync_entity::rule::Rule;
 use bili_sync_entity::*;
+use sea_orm::prelude::DateTime;
 use sea_orm::{DerivePartialModel, FromQueryResult};
 use serde::Serialize;
 
@@ -218,6 +219,7 @@ pub struct VideoSourceDetail {
     #[serde(default)]
     pub use_dynamic_api: Option<bool>,
     pub enabled: bool,
+    pub latest_row_at: Option<DateTime>,
 }
 
 #[derive(Serialize)]

crates/bili_sync/src/api/routes/config/mod.rs

@@ -9,7 +9,7 @@ use sea_orm::DatabaseConnection;
 use crate::api::wrapper::{ApiError, ApiResponse, ValidatedJson};
 use crate::bilibili::BiliClient;
 use crate::config::{Config, VersionedConfig};
-use crate::notifier::Notifier;
+use crate::notifier::{Message, Notifier};
 
 pub(super) fn router() -> Router {
     Router::new()
@@ -41,7 +41,10 @@ pub async fn ping_notifiers(
         *ignore_cache = Some(());
     }
     notifier
-        .notify(bili_client.inner_client(), "This is a test notification from BiliSync.")
+        .notify(bili_client.inner_client(), Message{
+            message: "This is a test notification from BiliSync.".into(),
+            image_url: Some("https://socialify.git.ci/amtoaer/bili-sync/image?description=1&font=KoHo&issues=1&language=1&logo=https%3A%2F%2Fs2.loli.net%2F2023%2F12%2F02%2F9EwT2yInOu1d3zm.png&name=1&owner=1&pattern=Signal&pulls=1&stargazers=1&theme=Light".to_owned()),
+        })
         .await?;
     Ok(ApiResponse::ok(()))
 }

crates/bili_sync/src/api/routes/video_sources/mod.rs

@@ -104,7 +104,8 @@ pub async fn get_video_sources_details(
                 collection::Column::Name,
                 collection::Column::Path,
                 collection::Column::Rule,
-                collection::Column::Enabled
+                collection::Column::Enabled,
+                collection::Column::LatestRowAt
             ])
             .into_model::<VideoSourceDetail>()
             .all(&db),
@@ -115,7 +116,8 @@ pub async fn get_video_sources_details(
                 favorite::Column::Name,
                 favorite::Column::Path,
                 favorite::Column::Rule,
-                favorite::Column::Enabled
+                favorite::Column::Enabled,
+                favorite::Column::LatestRowAt
             ])
             .into_model::<VideoSourceDetail>()
             .all(&db),
@@ -127,7 +129,8 @@ pub async fn get_video_sources_details(
                 submission::Column::Path,
                 submission::Column::Enabled,
                 submission::Column::Rule,
-                submission::Column::UseDynamicApi
+                submission::Column::UseDynamicApi,
+                submission::Column::LatestRowAt
             ])
             .into_model::<VideoSourceDetail>()
             .all(&db),
@@ -138,7 +141,8 @@ pub async fn get_video_sources_details(
                 watch_later::Column::Id,
                 watch_later::Column::Path,
                 watch_later::Column::Enabled,
-                watch_later::Column::Rule
+                watch_later::Column::Rule,
+                watch_later::Column::LatestRowAt
             ])
             .into_model::<VideoSourceDetail>()
             .all(&db)
@@ -152,6 +156,7 @@ pub async fn get_video_sources_details(
             rule_display: None,
             use_dynamic_api: None,
             enabled: false,
+            latest_row_at: None,
         })
     }
     for sources in [&mut collections, &mut favorites, &mut submissions, &mut watch_later] {
@@ -159,6 +164,7 @@ pub async fn get_video_sources_details(
             if let Some(rule) = &item.rule {
                 item.rule_display = Some(rule.to_string());
             }
+            item.latest_row_at = item.latest_row_at.filter(|dt| dt.and_utc().timestamp() != 0);
         });
     }
     Ok(ApiResponse::ok(VideoSourcesDetailsResponse {

crates/bili_sync/src/bilibili/client.rs

@@ -49,12 +49,13 @@ impl Client {
         let mut req = self.0.request(method, url);
         // 如果有 credential，会将其转换成 cookie 添加到请求的 header 中
         if let Some(credential) = credential {
-            req = req
-                .header(header::COOKIE, format!("SESSDATA={}", credential.sessdata))
-                .header(header::COOKIE, format!("bili_jct={}", credential.bili_jct))
-                .header(header::COOKIE, format!("buvid3={}", credential.buvid3))
-                .header(header::COOKIE, format!("DedeUserID={}", credential.dedeuserid))
-                .header(header::COOKIE, format!("ac_time_value={}", credential.ac_time_value));
+            req = req.header(
+                header::COOKIE,
+                format!(
+                    "buvid3={}; SESSDATA={}; bili_jct={}; DedeUserID={}",
+                    credential.buvid3, credential.sessdata, credential.bili_jct, credential.dedeuserid
+                ),
+            );
         }
         req
     }

crates/bili_sync/src/bilibili/credential.rs

@@ -205,7 +205,7 @@ JNrRuoEUXpabUzGB8QIDAQAB
         let ts = chrono::Local::now().timestamp_millis() - 20000;
         let data = format!("refresh_{}", ts).into_bytes();
         let encrypted = key
-            .encrypt(&mut rand::rng(), Oaep::new::<Sha256>(), &data)
+            .encrypt(&mut rand::rng(), Oaep::<Sha256>::new(), &data)
             .expect("fail to encrypt");
         hex::encode(encrypted)
     }

crates/bili_sync/src/bilibili/error.rs

@@ -2,10 +2,14 @@ use thiserror::Error;
 
 #[derive(Error, Debug, Clone)]
 pub enum BiliError {
-    #[error("response missing 'code' or 'message' field, full response: {0}")]
+    #[error("response missing 'code' field, full response: {0}")]
     InvalidResponse(String),
-    #[error("API returned error code {0}, full response: {1}")]
-    ErrorResponse(i64, String),
+    #[error("API returned error code {code}, full response: {response}")]
+    ErrorResponse {
+        code: i64,
+        message: Option<String>,
+        response: String,
+    },
     #[error("risk control triggered by server, full response: {0}")]
     RiskControlOccurred(String),
     #[error("invalid HTTP response code {0}, reason: {1}")]
@@ -21,4 +25,15 @@ impl BiliError {
             BiliError::RiskControlOccurred(_) | BiliError::VideoStreamsEmpty | BiliError::InvalidStatusCode(_, _)
         )
     }
+
+    pub fn is_common_error(&self) -> bool {
+        if let BiliError::ErrorResponse { code, message, .. } = self {
+            for pair in [(-503, "服务暂不可用"), (-504, "服务调用超时")] {
+                if *code == pair.0 && message.as_ref().is_some_and(|m| m == pair.1) {
+                    return true;
+                }
+            }
+        }
+        false
+    }
 }

crates/bili_sync/src/bilibili/mod.rs

@@ -60,10 +60,18 @@ impl Validate for serde_json::Value {
         let code = self["code"]
             .as_i64()
             .with_context(|| BiliError::InvalidResponse(self.to_string()))?;
+        let message = self["message"].as_str().map(ToOwned::to_owned);
         if code == -352 || !self["data"]["v_voucher"].is_null() {
             bail!(BiliError::RiskControlOccurred(self.to_string()));
         }
-        ensure!(code == 0, BiliError::ErrorResponse(code, self.to_string()));
+        ensure!(
+            code == 0,
+            BiliError::ErrorResponse {
+                code,
+                message,
+                response: self.to_string(),
+            }
+        );
         Ok(self)
     }
 }

crates/bili_sync/src/config/current.rs

@@ -38,6 +38,8 @@ pub struct Config {
     pub page_name: String,
     #[serde(default)]
     pub notifiers: Option<Arc<Vec<Notifier>>>,
+    #[serde(default)]
+    pub ignore_common_errors: bool,
     #[serde(default = "default_favorite_path")]
     pub favorite_default_path: String,
     #[serde(default = "default_collection_path")]
@@ -124,6 +126,7 @@ impl Default for Config {
             video_name: "{{title}}".to_owned(),
             page_name: "{{bvid}}".to_owned(),
             notifiers: None,
+            ignore_common_errors: false,
             favorite_default_path: default_favorite_path(),
             collection_default_path: default_collection_path(),
             submission_default_path: default_submission_path(),

crates/bili_sync/src/downloader.rs

@@ -8,7 +8,7 @@ use async_tempfile::TempFile;
 use futures::TryStreamExt;
 use reqwest::{Method, StatusCode, header};
 use tokio::fs::{self};
-use tokio::io::{AsyncSeekExt, AsyncWriteExt};
+use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter};
 use tokio::process::Command;
 use tokio::task::JoinSet;
 use tokio_util::io::StreamReader;
@@ -155,8 +155,9 @@ impl Downloader {
             .error_for_status_ext()?;
         let expected = resp.header_content_length();
         let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
-        let received = tokio::io::copy(&mut stream_reader, file).await?;
-        file.flush().await?;
+        let mut file_writer = BufWriter::with_capacity(2 * 1024 * 1024, file);
+        let received = tokio::io::copy(&mut stream_reader, &mut file_writer).await?;
+        file_writer.flush().await?;
         if let Some(expected) = expected {
             ensure!(
                 received == expected,
@@ -244,8 +245,9 @@ impl Downloader {
                     );
                 }
                 let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
-                let received = tokio::io::copy(&mut stream_reader, &mut file_clone).await?;
-                file_clone.flush().await?;
+                let mut file_writer = BufWriter::with_capacity(2 * 1024 * 1024, file_clone);
+                let received = tokio::io::copy(&mut stream_reader, &mut file_writer).await?;
+                file_writer.flush().await?;
                 ensure!(
                     received == end - start + 1,
                     "downloaded bytes mismatch: expected {}, got {}",

crates/bili_sync/src/notifier/mod.rs

@@ -1,6 +1,8 @@
 mod info;
 mod message;
 
+use std::collections::HashMap;
+
 use anyhow::Result;
 use futures::future;
 pub use info::DownloadNotifyInfo;
@@ -16,10 +18,14 @@ pub enum Notifier {
     Telegram {
         bot_token: String,
         chat_id: String,
+        #[serde(default)]
+        skip_image: bool,
     },
     Webhook {
         url: String,
         template: Option<String>,
+        #[serde(default)]
+        headers: Option<HashMap<String, String>>,
         #[serde(skip)]
         // 一个内部辅助字段，用于决定是否强制渲染当前模板，在测试时使用
         ignore_cache: Option<()>,
@@ -56,8 +62,14 @@ impl Notifier {
 
     async fn notify_internal<'a>(&self, client: &reqwest::Client, message: &Message<'a>) -> Result<()> {
         match self {
-            Notifier::Telegram { bot_token, chat_id } => {
-                if let Some(img_url) = &message.image_url {
+            Notifier::Telegram {
+                bot_token,
+                chat_id,
+                skip_image,
+            } => {
+                if let Some(img_url) = &message.image_url
+                    && !*skip_image
+                {
                     let url = format!("https://api.telegram.org/bot{}/sendPhoto", bot_token);
                     let params = [
                         ("chat_id", chat_id.as_str()),
@@ -74,6 +86,7 @@ impl Notifier {
             Notifier::Webhook {
                 url,
                 template,
+                headers,
                 ignore_cache,
             } => {
                 let key = webhook_template_key(url);
@@ -82,12 +95,20 @@ impl Notifier {
                     Some(_) => handlebar.render_template(webhook_template_content(template), &message)?,
                     None => handlebar.render(&key, &message)?,
                 };
-                client
-                    .post(url)
-                    .header(header::CONTENT_TYPE, "application/json")
-                    .body(payload)
-                    .send()
-                    .await?;
+                let mut headers_map = header::HeaderMap::new();
+                headers_map.insert(header::CONTENT_TYPE, "application/json".try_into()?);
+
+                if let Some(custom_headers) = headers {
+                    for (key, value) in custom_headers {
+                        if let (Ok(key), Ok(value)) =
+                            (header::HeaderName::try_from(key), header::HeaderValue::try_from(value))
+                        {
+                            headers_map.insert(key, value);
+                        }
+                    }
+                }
+
+                client.post(url).headers(headers_map).body(payload).send().await?;
             }
         }
         Ok(())

crates/bili_sync/src/task/video_downloader.rs

@@ -140,6 +140,7 @@ impl DownloadTaskManager {
                     &initial_config,
                     &cx.bili_client,
                     format!("初始化视频下载任务失败：{:#}", err),
+                    &err,
                 );
                 None
             }
@@ -193,6 +194,7 @@ impl DownloadTaskManager {
                                 &initial_config,
                                 &cx.bili_client,
                                 format!("重载视频下载任务失败：{:#}", err),
+                                &err,
                             );
                             None
                         }
@@ -234,6 +236,7 @@ impl DownloadTaskManager {
                             &config,
                             &cx.bili_client,
                             format!("本轮凭据检查与刷新任务执行遇到错误：{:#}", e),
+                            &e,
                         );
                     }
                 }
@@ -285,6 +288,7 @@ impl DownloadTaskManager {
                             &config,
                             &cx.bili_client,
                             format!("本轮视频下载任务执行遇到错误：{:#}", e),
+                            &e,
                         );
                     }
                 }
@@ -360,6 +364,7 @@ async fn download_video(
                 config,
                 &bili_client,
                 format!("处理 {} 时遇到错误：{:#}，跳过该视频源", display_name, e),
+                &e,
             );
             if let Ok(e) = e.downcast::<BiliError>()
                 && e.is_risk_control_related()

crates/bili_sync/src/utils/model.rs

@@ -1,10 +1,11 @@
 use anyhow::{Context, Result, anyhow};
 use bili_sync_entity::*;
+use itertools::Itertools;
 use rand::seq::SliceRandom;
 use sea_orm::ActiveValue::Set;
-use sea_orm::DatabaseTransaction;
 use sea_orm::entity::prelude::*;
-use sea_orm::sea_query::{OnConflict, SimpleExpr};
+use sea_orm::sea_query::{Expr, OnConflict, SimpleExpr};
+use sea_orm::{ConnectionTrait, DatabaseTransaction, IdenStatic, Statement};
 
 use crate::adapter::{VideoSource, VideoSourceEnum};
 use crate::bilibili::VideoInfo;
@@ -76,8 +77,16 @@ pub async fn create_videos(
 
 /// 尝试创建 Page Model，如果发生冲突则忽略
 pub async fn create_pages(pages_model: Vec<page::ActiveModel>, connection: &DatabaseTransaction) -> Result<()> {
-    for page_chunk in pages_model.chunks(200) {
-        page::Entity::insert_many(page_chunk.to_vec())
+    let mut pages = pages_model.into_iter();
+    loop {
+        // 这里 insert_many 要求 IntoIterator，vec 上调用 chunks 返回的类型不匹配，需要 to_vec 做 clone
+        // itertools 的 into_iter().chunks() 由于 !Send 也无法直接使用
+        // 暂时手写 take + collect 作为避免 clone 的折中方案
+        let page_chunk = pages.by_ref().take(200).collect::<Vec<_>>();
+        if page_chunk.is_empty() {
+            break;
+        }
+        page::Entity::insert_many(page_chunk)
             .on_conflict(
                 OnConflict::columns([page::Column::VideoId, page::Column::Pid])
                     .do_nothing()
@@ -90,6 +99,90 @@ pub async fn create_pages(pages_model: Vec<page::ActiveModel>, connection: &Data
     Ok(())
 }
 
+/// 更新视频 model 的详情字段
+pub async fn update_video_detail_models(
+    videos: Vec<video::ActiveModel>,
+    connection: &DatabaseTransaction,
+) -> Result<()> {
+    if videos.is_empty() {
+        return Ok(());
+    }
+    let columns = [
+        video::Column::Id,
+        video::Column::CollectionId,
+        video::Column::FavoriteId,
+        video::Column::WatchLaterId,
+        video::Column::SubmissionId,
+        video::Column::UpperId,
+        video::Column::UpperName,
+        video::Column::UpperFace,
+        video::Column::Staff,
+        video::Column::Name,
+        video::Column::Bvid,
+        video::Column::Intro,
+        video::Column::Cover,
+        video::Column::Ctime,
+        video::Column::Pubtime,
+        video::Column::Favtime,
+        video::Column::DownloadStatus,
+        video::Column::Valid,
+        video::Column::ShouldDownload,
+        video::Column::Tags,
+        video::Column::SinglePage,
+    ];
+    let row = format!("({})", std::iter::repeat_n("?", columns.len()).join(", "));
+    let rows = std::iter::repeat_n(row.as_str(), videos.len()).join(", ");
+    let mut values = Vec::with_capacity(videos.len() * columns.len());
+    for video in videos {
+        for column in columns {
+            values.push(
+                video
+                    .get(column)
+                    .into_value()
+                    .ok_or_else(|| anyhow!("video column {} is not set", column.as_str()))?,
+            );
+        }
+    }
+    let sql = format!(
+        "WITH tempdata({}) AS (VALUES {}) \
+        UPDATE video \
+        SET {} \
+        FROM tempdata \
+        WHERE video.id = tempdata.id",
+        columns.iter().map(IdenStatic::as_str).join(", "),
+        rows,
+        columns
+            .iter()
+            .skip(1)
+            .map(|column| {
+                let column = column.as_str();
+                format!("{} = tempdata.{}", column, column)
+            })
+            .join(", ")
+    );
+    connection
+        .execute(Statement::from_sql_and_values(
+            connection.get_database_backend(),
+            sql,
+            values,
+        ))
+        .await?;
+    Ok(())
+}
+
+/// 将视频标记为失效
+pub async fn set_video_models_invalid(video_ids: Vec<i32>, connection: &DatabaseTransaction) -> Result<()> {
+    if video_ids.is_empty() {
+        return Ok(());
+    }
+    video::Entity::update_many()
+        .filter(video::Column::Id.is_in(video_ids))
+        .col_expr(video::Column::Valid, Expr::value(false))
+        .exec(connection)
+        .await?;
+    Ok(())
+}
+
 /// 更新视频 model 的下载状态
 pub async fn update_videos_model(videos: Vec<video::ActiveModel>, connection: &DatabaseConnection) -> Result<()> {
     video::Entity::insert_many(videos)

crates/bili_sync/src/utils/notify.rs

@@ -1,4 +1,6 @@
-use crate::bilibili::BiliClient;
+use anyhow::Error;
+
+use crate::bilibili::{BiliClient, BiliError};
 use crate::config::Config;
 use crate::notifier::{Message, NotifierAllExt};
 
@@ -12,8 +14,17 @@ pub fn notify(config: &Config, bili_client: &BiliClient, msg: impl Into<Message<
     }
 }
 
-pub fn error_and_notify(config: &Config, bili_client: &BiliClient, msg: String) {
+pub fn error_and_notify(config: &Config, bili_client: &BiliClient, msg: String, error: &Error) {
     error!("{msg}");
+    if config.ignore_common_errors
+        && error.chain().any(|cause| {
+            cause
+                .downcast_ref::<BiliError>()
+                .is_some_and(BiliError::is_common_error)
+        })
+    {
+        return;
+    }
     if let Some(notifiers) = &config.notifiers
         && !notifiers.is_empty()
     {

crates/bili_sync/src/workflow.rs

@@ -5,13 +5,12 @@ use std::pin::Pin;
 use anyhow::{Context, Result, anyhow, bail};
 use bili_sync_entity::upper_vec::Upper;
 use bili_sync_entity::*;
-use futures::stream::FuturesUnordered;
+use futures::stream::{self, FuturesUnordered};
 use futures::{Stream, StreamExt, TryStreamExt};
 use sea_orm::ActiveValue::Set;
 use sea_orm::TransactionTrait;
 use sea_orm::entity::prelude::*;
 use tokio::fs;
-use tokio::sync::Semaphore;
 
 use crate::adapter::{VideoSource, VideoSourceEnum};
 use crate::bilibili::{BestStream, BiliClient, BiliError, Dimension, PageInfo, Video, VideoInfo};
@@ -22,14 +21,20 @@ use crate::notifier::DownloadNotifyInfo;
 use crate::utils::download_context::DownloadContext;
 use crate::utils::format_arg::{page_format_args, video_format_args};
 use crate::utils::model::{
-    create_pages, create_videos, filter_unfilled_videos, filter_unhandled_video_pages, update_pages_model,
-    update_videos_model,
+    create_pages, create_videos, filter_unfilled_videos, filter_unhandled_video_pages, set_video_models_invalid,
+    update_pages_model, update_video_detail_models, update_videos_model,
 };
 use crate::utils::nfo::{NFO, ToNFO};
 use crate::utils::notify::notify;
 use crate::utils::rule::FieldEvaluatable;
 use crate::utils::status::{PageStatus, STATUS_OK, VideoStatus};
 
+#[allow(clippy::large_enum_variant)]
+enum VideoDetailUpdate {
+    Invalid(i32),
+    Detail(Vec<page::ActiveModel>, video::ActiveModel),
+}
+
 /// 完整地处理某个视频来源
 pub async fn process_video_source(
     video_source: VideoSourceEnum,
@@ -99,7 +104,7 @@ pub async fn refresh_video_source<'a>(
             }
         })
         .filter_map(|(idx, res)| futures::future::ready(video_source.should_filter(idx, res, &latest_row_at)))
-        .chunks(10);
+        .chunks(30);
     let mut count = 0;
     while let Some(videos_info) = video_streams.next().await {
         count += videos_info.len();
@@ -126,12 +131,8 @@ pub async fn fetch_video_details(
 ) -> Result<()> {
     video_source.log_fetch_video_start();
     let videos_model = filter_unfilled_videos(video_source.filter_expr(), connection).await?;
-    let semaphore = Semaphore::new(config.concurrent_limit.video);
-    let semaphore_ref = &semaphore;
-    let tasks = videos_model
-        .into_iter()
+    let mut tasks = stream::iter(videos_model)
         .map(|video_model| async move {
-            let _permit = semaphore_ref.acquire().await.context("acquire semaphore failed")?;
             let video = Video::new(bili_client, video_model.bvid.as_str(), &config.credential);
             let info: Result<_> = async { Ok((video.get_tags().await?, video.get_view_info().await?)) }.await;
             match info {
@@ -140,10 +141,10 @@ pub async fn fetch_video_details(
                         "获取视频 {} - {} 的详细信息失败，错误为：{:#}",
                         &video_model.bvid, &video_model.name, e
                     );
-                    if let Some(BiliError::ErrorResponse(-404, _)) = e.downcast_ref::<BiliError>() {
-                        let mut video_active_model: bili_sync_entity::video::ActiveModel = video_model.into();
-                        video_active_model.valid = Set(false);
-                        video_active_model.save(connection).await?;
+                    if let Some(BiliError::ErrorResponse { code: -404, .. }) = e.downcast_ref::<BiliError>() {
+                        Some(VideoDetailUpdate::Invalid(video_model.id))
+                    } else {
+                        None
                     }
                 }
                 Ok((tags, mut view_info)) => {
@@ -162,16 +163,30 @@ pub async fn fetch_video_details(
                     video_active_model.single_page = Set(Some(pages.len() == 1));
                     video_active_model.tags = Set(Some(tags.into()));
                     video_active_model.should_download = Set(video_source.rule().evaluate(&video_active_model, &pages));
-                    let txn = connection.begin().await?;
-                    create_pages(pages, &txn).await?;
-                    video_active_model.save(&txn).await?;
-                    txn.commit().await?;
+                    Some(VideoDetailUpdate::Detail(pages, video_active_model))
                 }
-            };
-            Ok::<_, anyhow::Error>(())
+            }
         })
-        .collect::<FuturesUnordered<_>>();
-    tasks.try_collect::<()>().await?;
+        .buffer_unordered(config.concurrent_limit.video)
+        .filter_map(futures::future::ready)
+        .chunks(15);
+    while let Some(details) = tasks.next().await {
+        let mut invalid_video_ids = Vec::new();
+        let mut pages = Vec::new();
+        let mut videos = Vec::new();
+        details.into_iter().for_each(|detail| match detail {
+            VideoDetailUpdate::Invalid(video_id) => invalid_video_ids.push(video_id),
+            VideoDetailUpdate::Detail(video_pages, video) => {
+                pages.extend(video_pages);
+                videos.push(video);
+            }
+        });
+        let txn = connection.begin().await?;
+        update_video_detail_models(videos, &txn).await?;
+        set_video_models_invalid(invalid_video_ids, &txn).await?;
+        create_pages(pages, &txn).await?;
+        txn.commit().await?;
+    }
     video_source.log_fetch_video_end();
     Ok(())
 }
@@ -185,13 +200,11 @@ pub async fn download_unprocessed_videos(
     config: &Config,
 ) -> Result<DownloadNotifyInfo> {
     video_source.log_download_video_start();
-    let semaphore = Semaphore::new(config.concurrent_limit.video);
     let downloader = Downloader::new(bili_client.client.clone());
     let cx = DownloadContext::new(bili_client, video_source, template, connection, &downloader, config);
     let unhandled_videos_pages = filter_unhandled_video_pages(video_source.filter_expr(), connection).await?;
     let mut assigned_upper_ids = HashSet::new();
-    let tasks = unhandled_videos_pages
-        .into_iter()
+    let tasks = stream::iter(unhandled_videos_pages)
         .map(|(video_model, pages_model)| {
             // 这里按理说是可以直接拿到 assigned_uppers 的，但rust 会错误地认为它引用了 local variable
             // 导致编译出错，暂时先这样单独提取出一个 owned 的 upper id 列表，再在任务内部筛选
@@ -200,9 +213,9 @@ pub async fn download_unprocessed_videos(
                 .map(|u| u.mid)
                 .filter(|uid| assigned_upper_ids.insert(*uid))
                 .collect::<Vec<_>>();
-            download_video_pages(video_model, pages_model, &semaphore, task_uids, cx)
+            download_video_pages(video_model, pages_model, task_uids, cx)
         })
-        .collect::<FuturesUnordered<_>>();
+        .buffer_unordered(config.concurrent_limit.video);
     let mut risk_control_related_error = None;
     let mut stream = tasks
         // 触发风控时设置 download_aborted 标记并终止流
@@ -234,11 +247,9 @@ pub async fn download_unprocessed_videos(
 pub async fn download_video_pages(
     video_model: video::Model,
     page_models: Vec<page::Model>,
-    semaphore: &Semaphore,
     upper_uids: Vec<i64>,
     cx: DownloadContext<'_>,
 ) -> Result<video::ActiveModel> {
-    let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
     let mut status = VideoStatus::from(video_model.download_status);
     let separate_status = status.should_run();
     // 未记录路径时填充，已经填充过路径时使用现有的
@@ -350,11 +361,9 @@ pub async fn dispatch_download_page(
     if !should_run {
         return Ok(ExecutionStatus::Skipped);
     }
-    let child_semaphore = Semaphore::new(cx.config.concurrent_limit.page);
-    let tasks = page_models
-        .into_iter()
-        .map(|page_model| download_page(video_model, page_model, &child_semaphore, base_path, cx))
-        .collect::<FuturesUnordered<_>>();
+    let tasks = stream::iter(page_models)
+        .map(|page_model| download_page(video_model, page_model, base_path, cx))
+        .buffer_unordered(cx.config.concurrent_limit.page);
     let (mut risk_control_related_error, mut target_status) = (None, STATUS_OK);
     let mut stream = tasks
         .take_while(|res| {
@@ -397,11 +406,9 @@ pub async fn dispatch_download_page(
 pub async fn download_page(
     video_model: &video::Model,
     page_model: page::Model,
-    semaphore: &Semaphore,
     base_path: &Path,
     cx: DownloadContext<'_>,
 ) -> Result<page::ActiveModel> {
-    let _permit = semaphore.acquire().await.context("acquire semaphore failed")?;
     let mut status = PageStatus::from(page_model.download_status);
     let separate_status = status.should_run();
     let is_single_page = video_model.single_page.context("single_page is null")?;

docs/.vitepress/config.mts

@@ -21,7 +21,7 @@ export default defineConfig({
 		nav: [
 			{ text: "主页", link: "/" },
 			{
-				text: "v2.11.0",
+				text: "v2.11.1",
 				items: [
 					{
 						text: "程序更新",

docs/introduction.md

@@ -1,7 +1,7 @@
 # bili-sync 是什么？
 
 > [!TIP]
-> 当前最新程序版本为 v2.11.0，文档将始终与最新程序版本保持一致。
+> 当前最新程序版本为 v2.11.1，文档将始终与最新程序版本保持一致。
 
 bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具。
 

rust-toolchain.toml

@@ -1,3 +1,3 @@
 [toolchain]
-channel = "1.94.0"
+channel = "1.96.0"
 components = ["clippy"]

web/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "bili-sync-web",
-	"version": "2.11.0",
+	"version": "2.11.1",
 	"devDependencies": {
 		"@eslint/compat": "^1.4.1",
 		"@eslint/js": "^9.39.2",

web/src/lib/types.ts

@@ -224,6 +224,7 @@ export interface VideoSourceDetail {
 	ruleDisplay: string | null;
 	useDynamicApi: boolean | null;
 	enabled: boolean;
+	latestRowAt: string | null;
 }
 
 export interface VideoSourcesDetailsResponse {
@@ -305,12 +306,14 @@ export interface TelegramNotifier {
 	type: 'telegram';
 	bot_token: string;
 	chat_id: string;
+	skip_image: boolean;
 }
 
 export interface WebhookNotifier {
 	type: 'webhook';
 	url: string;
 	template?: string | null;
+	headers?: Record<string, string> | null;
 }
 
 export type Notifier = TelegramNotifier | WebhookNotifier;
@@ -327,6 +330,7 @@ export interface Config {
 	video_name: string;
 	page_name: string;
 	notifiers: Notifier[] | null;
+	ignore_common_errors: boolean;
 	favorite_default_path: string;
 	collection_default_path: string;
 	submission_default_path: string;

web/src/routes/settings/+page.svelte

@@ -792,6 +792,17 @@
 				<!-- 通知设置 -->
 				<Tabs.Content value="notifiers" class="mt-6 space-y-6">
 					<div class="space-y-4">
+						<div class="flex items-center justify-between rounded-lg border p-4">
+							<div class="space-y-1">
+								<Label for="ignore-common-errors">忽略常见错误</Label>
+								<p class="text-muted-foreground text-sm">
+									b 站接口频繁出现 -503
+									服务暂不可用，在通知器内通知容易刷屏，开启该选项以忽略此错误通知
+								</p>
+							</div>
+							<Switch id="ignore-common-errors" bind:checked={formData.ignore_common_errors} />
+						</div>
+
 						<div class="flex items-center justify-between">
 							<div>
 								<h3 class="text-lg font-semibold">通知器管理</h3>

web/src/routes/settings/NotifierDialog.svelte

@@ -1,12 +1,11 @@
 <script lang="ts">
 	import { Button } from '$lib/components/ui/button/index.js';
+	import { Checkbox } from '$lib/components/ui/checkbox/index.js';
 	import { Input } from '$lib/components/ui/input/index.js';
 	import { Label } from '$lib/components/ui/label/index.js';
 	import { toast } from 'svelte-sonner';
 	import type { Notifier } from '$lib/types';
 
-	const jsonExample = '{"text": "您的消息内容"}';
-
 	export let notifier: Notifier | null = null;
 	export let onSave: (notifier: Notifier) => void;
 	export let onCancel: () => void;
@@ -14,8 +13,10 @@
 	let type: 'telegram' | 'webhook' = 'telegram';
 	let botToken = '';
 	let chatId = '';
+	let skipImage = false;
 	let webhookUrl = '';
 	let webhookTemplate = '';
+	let webhookHeaders: { key: string; value: string }[] = [];
 
 	// 初始化表单
 	$: {
@@ -24,22 +25,29 @@
 				type = 'telegram';
 				botToken = notifier.bot_token;
 				chatId = notifier.chat_id;
+				skipImage = notifier.skip_image;
 			} else {
 				type = 'webhook';
 				webhookUrl = notifier.url;
 				webhookTemplate = notifier.template || '';
+				if (notifier.headers) {
+					webhookHeaders = Object.entries(notifier.headers).map(([key, value]) => ({ key, value }));
+				} else {
+					webhookHeaders = [];
+				}
 			}
 		} else {
 			type = 'telegram';
 			botToken = '';
 			chatId = '';
+			skipImage = false;
 			webhookUrl = '';
 			webhookTemplate = '';
+			webhookHeaders = [];
 		}
 	}
 
 	function handleSave() {
-		// 验证表单
 		if (type === 'telegram') {
 			if (!botToken.trim()) {
 				toast.error('请输入 Bot Token');
@@ -53,7 +61,8 @@
 			const newNotifier: Notifier = {
 				type: 'telegram',
 				bot_token: botToken.trim(),
-				chat_id: chatId.trim()
+				chat_id: chatId.trim(),
+				skip_image: skipImage
 			};
 			onSave(newNotifier);
 		} else {
@@ -62,7 +71,6 @@
 				return;
 			}
 
-			// 简单的 URL 验证
 			try {
 				new URL(webhookUrl.trim());
 			} catch {
@@ -70,10 +78,20 @@
 				return;
 			}
 
+			const headers: Record<string, string> = {};
+			for (const { key, value } of webhookHeaders) {
+				const trimmedKey = key.trim();
+				const trimmedValue = value.trim();
+				if (trimmedKey && trimmedValue) {
+					headers[trimmedKey] = trimmedValue;
+				}
+			}
+
 			const newNotifier: Notifier = {
 				type: 'webhook',
 				url: webhookUrl.trim(),
-				template: webhookTemplate.trim() || null
+				template: webhookTemplate.trim() || null,
+				headers: Object.keys(headers).length > 0 ? headers : null
 			};
 			onSave(newNotifier);
 		}
@@ -108,14 +126,14 @@
 			<Input id="chat-id" placeholder="-1001234567890" bind:value={chatId} />
 			<p class="text-muted-foreground text-xs">目标聊天室的 ID（个人用户、群组或频道）</p>
 		</div>
+		<div class="flex items-center gap-2">
+			<Checkbox id="skip-image" bind:checked={skipImage} />
+			<Label for="skip-image" class="text-sm font-normal">仅发送文字</Label>
+		</div>
 	{:else if type === 'webhook'}
 		<div class="space-y-2">
 			<Label for="webhook-url">Webhook URL</Label>
-			<Input id="webhook-url" placeholder="https://example.com/webhook" bind:value={webhookUrl} />
-			<p class="text-muted-foreground text-xs">
-				接收通知的 Webhook 地址<br />
-				格式示例：{jsonExample}
-			</p>
+			<Input id="webhook-url" placeholder="请输入 Webhook 地址" bind:value={webhookUrl} />
 		</div>
 		<div class="space-y-2">
 			<Label for="webhook-template">模板（可选）</Label>
@@ -127,7 +145,48 @@
 			></textarea>
 			<p class="text-muted-foreground text-xs">
 				用于渲染 Webhook 的 Handlebars 模板。如果不填写，将使用默认模板。<br />
-				可用变量：<code class="text-xs">message</code>（通知内容）
+				可用变量：<code class="text-xs">message</code>（通知内容）、<code class="text-xs"
+					>image_url</code
+				>（封面图片地址，无图时为 null）
+			</p>
+		</div>
+
+		<div class="space-y-2">
+			<div class="flex items-center justify-between">
+				<Label>自定义请求头（可选）</Label>
+				<Button
+					variant="ghost"
+					size="sm"
+					onclick={() => (webhookHeaders = [...webhookHeaders, { key: '', value: '' }])}
+				>
+					+ 添加请求头
+				</Button>
+			</div>
+			{#each webhookHeaders as header, index (index)}
+				<div class="flex items-center gap-2">
+					<Input
+						placeholder="Header 名称（例如 Authorization）"
+						bind:value={header.key}
+						class="flex-1"
+					/>
+					<Input
+						placeholder="Header 值"
+						bind:value={header.value}
+						class="flex-1"
+						type={header.key.toLowerCase() === 'authorization' ? 'password' : 'text'}
+					/>
+					<Button
+						variant="ghost"
+						size="sm"
+						onclick={() => (webhookHeaders = webhookHeaders.filter((_, i) => i !== index))}
+						class="h-10 px-2"
+					>
+						×
+					</Button>
+				</div>
+			{/each}
+			<p class="text-muted-foreground text-xs">
+				添加自定义请求头，例如：Authorization: Bearer your_token
 			</p>
 		</div>
 	{/if}

web/src/routes/video-sources/+page.svelte

@@ -366,8 +366,9 @@
 									<Table.Row>
 										<Table.Head class="w-[20%]">名称</Table.Head>
 										<Table.Head class="w-[30%]">下载路径</Table.Head>
+										<Table.Head class="w-[15%]">最新视频时间</Table.Head>
 										<Table.Head class="w-[15%]">过滤规则</Table.Head>
-										<Table.Head class="w-[15%]">启用状态</Table.Head>
+										<Table.Head class="w-[10%]">启用状态</Table.Head>
 										<Table.Head class="w-[10%] text-right">操作</Table.Head>
 									</Table.Row>
 								</Table.Header>
@@ -387,6 +388,17 @@
 													</span>
 												</div>
 											</Table.Cell>
+											<Table.Cell>
+												{#if source.latestRowAt}
+													<Badge variant="secondary" class="flex w-fit items-center gap-1.5">
+														{new Date(source.latestRowAt).toLocaleString('zh-CN')}
+													</Badge>
+												{:else}
+													<Badge variant="secondary" class="flex w-fit items-center gap-1.5">
+														-
+													</Badge>
+												{/if}
+											</Table.Cell>
 											<Table.Cell>
 												{#if source.rule && source.rule.length > 0}
 													<Tooltip.Root disableHoverableContent={true}>