feat: 扩大风控检测，当 http 返回 403 或 412 时认为是风控 (#640)

crates/bili_sync/src/bilibili/client.rs

@@ -31,7 +31,7 @@ impl Client {
         );
         headers.insert(
             header::REFERER,
-            header::HeaderValue::from_static("https://www.bilibili.com"),
+            header::HeaderValue::from_static("https://www.bilibili.com/"),
         );
         Self(
             reqwest::Client::builder()

crates/bili_sync/src/bilibili/collection.rs

@@ -7,7 +7,7 @@ use reqwest::Method;
 use serde::Deserialize;
 use serde_json::Value;
 
-use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
+use crate::bilibili::{BiliClient, Credential, ErrorForStatusExt, Validate, VideoInfo};
 
 #[derive(PartialEq, Eq, Hash, Clone, Debug, Default, Copy)]
 pub enum CollectionType {
@@ -136,7 +136,7 @@ impl<'a> Collection<'a> {
             .query(&[("series_id", self.collection.sid.as_str())])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<Value>()
             .await?
             .validate()
@@ -176,7 +176,12 @@ impl<'a> Collection<'a> {
                     ("page_size", "30"),
                 ]),
         };
-        req.send().await?.error_for_status()?.json::<Value>().await?.validate()
+        req.send()
+            .await?
+            .error_for_status_ext()?
+            .json::<Value>()
+            .await?
+            .validate()
     }
 
     pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {

crates/bili_sync/src/bilibili/credential.rs

@@ -9,7 +9,7 @@ use rsa::sha2::Sha256;
 use rsa::{Oaep, RsaPublicKey};
 use serde::{Deserialize, Serialize};
 
-use crate::bilibili::{BiliError, Client, Validate};
+use crate::bilibili::{BiliError, Client, ErrorForStatusExt, Validate};
 
 const MIXIN_KEY_ENC_TAB: [usize; 64] = [
     46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38,
@@ -78,7 +78,7 @@ impl Credential {
             .request(Method::GET, "https://api.bilibili.com/x/web-interface/nav", Some(self))
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -94,7 +94,7 @@ impl Credential {
             )
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -111,7 +111,7 @@ impl Credential {
             .query(&[("qrcode_key", qrcode_key)])
             .send()
             .await?
-            .error_for_status()?;
+            .error_for_status_ext()?;
         let headers = std::mem::take(resp.headers_mut());
         let json = resp.json::<serde_json::Value>().await?.validate()?;
         let code = json["data"]["code"].as_i64().context("missing 'code' field in data")?;
@@ -147,7 +147,7 @@ impl Credential {
             .request(Method::GET, "https://api.bilibili.com/x/web-frontend/getbuvid", None)
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -167,7 +167,7 @@ impl Credential {
             )
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -220,7 +220,7 @@ JNrRuoEUXpabUzGB8QIDAQAB
             .header(header::COOKIE, "Domain=.bilibili.com")
             .send()
             .await?
-            .error_for_status()?;
+            .error_for_status_ext()?;
         regex_find(r#"<div id="1-name">(.+?)</div>"#, res.text().await?.as_str())
     }
 
@@ -241,7 +241,7 @@ JNrRuoEUXpabUzGB8QIDAQAB
             ])
             .send()
             .await?
-            .error_for_status()?;
+            .error_for_status_ext()?;
         let headers = std::mem::take(resp.headers_mut());
         let json = resp.json::<serde_json::Value>().await?.validate()?;
         let mut credential = Self::extract(headers, json)?;
@@ -263,7 +263,7 @@ JNrRuoEUXpabUzGB8QIDAQAB
             ])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;

crates/bili_sync/src/bilibili/dynamic.rs

@@ -5,7 +5,7 @@ use futures::Stream;
 use reqwest::Method;
 use serde_json::Value;
 
-use crate::bilibili::{BiliClient, Credential, MIXIN_KEY, Validate, VideoInfo, WbiSign};
+use crate::bilibili::{BiliClient, Credential, ErrorForStatusExt, MIXIN_KEY, Validate, VideoInfo, WbiSign};
 
 pub struct Dynamic<'a> {
     client: &'a BiliClient,
@@ -38,7 +38,7 @@ impl<'a> Dynamic<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()

crates/bili_sync/src/bilibili/error.rs

@@ -8,12 +8,17 @@ pub enum BiliError {
     ErrorResponse(i64, String),
     #[error("risk control triggered by server, full response: {0}")]
     RiskControlOccurred(String),
+    #[error("invalid HTTP response code {0}, reason: {1}")]
+    InvalidStatusCode(u16, &'static str),
     #[error("no video streams available (may indicate risk control)")]
     VideoStreamsEmpty,
 }
 
 impl BiliError {
     pub fn is_risk_control_related(&self) -> bool {
-        matches!(self, BiliError::RiskControlOccurred(_) | BiliError::VideoStreamsEmpty)
+        matches!(
+            self,
+            BiliError::RiskControlOccurred(_) | BiliError::VideoStreamsEmpty | BiliError::InvalidStatusCode(_, _)
+        )
     }
 }

crates/bili_sync/src/bilibili/favorite_list.rs

@@ -3,7 +3,7 @@ use async_stream::try_stream;
 use futures::Stream;
 use serde_json::Value;
 
-use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
+use crate::bilibili::{BiliClient, Credential, ErrorForStatusExt, Validate, VideoInfo};
 pub struct FavoriteList<'a> {
     client: &'a BiliClient,
     fid: String,
@@ -43,7 +43,7 @@ impl<'a> FavoriteList<'a> {
             .query(&[("media_id", &self.fid)])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -68,7 +68,7 @@ impl<'a> FavoriteList<'a> {
             ])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()

crates/bili_sync/src/bilibili/me.rs

@@ -1,7 +1,7 @@
 use anyhow::{Result, ensure};
 use reqwest::Method;
 
-use crate::bilibili::{BiliClient, Credential, Validate};
+use crate::bilibili::{BiliClient, Credential, ErrorForStatusExt, Validate};
 
 pub struct Me<'a> {
     client: &'a BiliClient,
@@ -29,7 +29,7 @@ impl<'a> Me<'a> {
             .query(&[("up_mid", &self.mid())])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -53,7 +53,7 @@ impl<'a> Me<'a> {
             .query(&[("pn", page_num), ("ps", page_size)])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -87,7 +87,7 @@ impl<'a> Me<'a> {
         let mut resp = request
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;

crates/bili_sync/src/bilibili/mod.rs

@@ -16,7 +16,7 @@ pub use favorite_list::FavoriteList;
 use favorite_list::Upper;
 pub use me::Me;
 use once_cell::sync::Lazy;
-use reqwest::RequestBuilder;
+use reqwest::{RequestBuilder, StatusCode};
 pub use submission::Submission;
 pub use video::{Dimension, PageInfo, Video};
 pub use watch_later::WatchLater;
@@ -47,6 +47,12 @@ pub(crate) trait Validate {
     fn validate(self) -> Result<Self::Output>;
 }
 
+pub(crate) trait ErrorForStatusExt {
+    type Output;
+
+    fn error_for_status_ext(self) -> Result<Self::Output>;
+}
+
 impl Validate for serde_json::Value {
     type Output = serde_json::Value;
 
@@ -62,6 +68,23 @@ impl Validate for serde_json::Value {
     }
 }
 
+impl ErrorForStatusExt for reqwest::Response {
+    type Output = reqwest::Response;
+
+    fn error_for_status_ext(self) -> Result<Self::Output> {
+        let status = self.status();
+        // 412 是由于请求频率过高导致的，确定是风控问题
+        // 403 目前偶尔出现在下载视频音频流时，由于是偶尔出现且过一段时间消失，暂时也当成风控问题处理
+        if status == StatusCode::PRECONDITION_FAILED || status == StatusCode::FORBIDDEN {
+            bail!(BiliError::InvalidStatusCode(
+                status.as_u16(),
+                status.canonical_reason().unwrap_or("Unknown")
+            ));
+        }
+        Ok(self.error_for_status()?)
+    }
+}
+
 pub(crate) trait WbiSign {
     type Output;
 

crates/bili_sync/src/bilibili/submission.rs

@@ -5,7 +5,7 @@ use reqwest::Method;
 use serde_json::Value;
 
 use crate::bilibili::favorite_list::Upper;
-use crate::bilibili::{BiliClient, Credential, Dynamic, MIXIN_KEY, Validate, VideoInfo, WbiSign};
+use crate::bilibili::{BiliClient, Credential, Dynamic, ErrorForStatusExt, MIXIN_KEY, Validate, VideoInfo, WbiSign};
 pub struct Submission<'a> {
     client: &'a BiliClient,
     pub upper_id: String,
@@ -39,7 +39,7 @@ impl<'a> Submission<'a> {
             .query(&[("mid", self.upper_id.as_str())])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -66,7 +66,7 @@ impl<'a> Submission<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()

crates/bili_sync/src/bilibili/video.rs

@@ -8,7 +8,7 @@ use crate::bilibili::analyzer::PageAnalyzer;
 use crate::bilibili::client::BiliClient;
 use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply};
 use crate::bilibili::subtitle::{SubTitle, SubTitleBody, SubTitleInfo, SubTitlesInfo};
-use crate::bilibili::{Credential, MIXIN_KEY, Validate, VideoInfo, WbiSign};
+use crate::bilibili::{Credential, ErrorForStatusExt, MIXIN_KEY, Validate, VideoInfo, WbiSign};
 
 pub struct Video<'a> {
     client: &'a BiliClient,
@@ -57,7 +57,7 @@ impl<'a> Video<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -77,7 +77,7 @@ impl<'a> Video<'a> {
             .query(&[("bvid", &self.bvid)])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -96,7 +96,7 @@ impl<'a> Video<'a> {
             .query(&[("bvid", &self.bvid)])
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -132,7 +132,7 @@ impl<'a> Video<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?;
+            .error_for_status_ext()?;
         let headers = std::mem::take(res.headers_mut());
         let content_type = headers.get("content-type");
         ensure!(
@@ -164,7 +164,7 @@ impl<'a> Video<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -181,7 +181,7 @@ impl<'a> Video<'a> {
             .wbi_sign(MIXIN_KEY.load().as_deref())?
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()?;
@@ -207,7 +207,7 @@ impl<'a> Video<'a> {
             .request(Method::GET, format!("https:{}", &info.subtitle_url).as_str(), None)
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?;
         let body: SubTitleBody = serde_json::from_value(res["body"].take())?;

crates/bili_sync/src/bilibili/watch_later.rs

@@ -3,7 +3,7 @@ use async_stream::try_stream;
 use futures::Stream;
 use serde_json::Value;
 
-use crate::bilibili::{BiliClient, Credential, Validate, VideoInfo};
+use crate::bilibili::{BiliClient, Credential, ErrorForStatusExt, Validate, VideoInfo};
 pub struct WatchLater<'a> {
     client: &'a BiliClient,
     credential: &'a Credential,
@@ -24,7 +24,7 @@ impl<'a> WatchLater<'a> {
             .await
             .send()
             .await?
-            .error_for_status()?
+            .error_for_status_ext()?
             .json::<serde_json::Value>()
             .await?
             .validate()

crates/bili_sync/src/downloader.rs

@@ -13,7 +13,7 @@ use tokio::process::Command;
 use tokio::task::JoinSet;
 use tokio_util::io::StreamReader;
 
-use crate::bilibili::Client;
+use crate::bilibili::{Client, ErrorForStatusExt};
 use crate::config::{ARGS, ConcurrentDownloadLimit};
 
 pub struct Downloader {
@@ -152,7 +152,7 @@ impl Downloader {
             .request(Method::GET, url, None)
             .send()
             .await?
-            .error_for_status()?;
+            .error_for_status_ext()?;
         let expected = resp.header_content_length();
         let mut stream_reader = StreamReader::new(resp.bytes_stream().map_err(std::io::Error::other));
         let received = tokio::io::copy(&mut stream_reader, file).await?;
@@ -184,7 +184,7 @@ impl Downloader {
                 .header(header::RANGE, "bytes=0-0")
                 .send()
                 .await?
-                .error_for_status()?;
+                .error_for_status_ext()?;
             if resp.status() != StatusCode::PARTIAL_CONTENT {
                 return self.fetch_serial(url, file).await;
             }
@@ -196,7 +196,7 @@ impl Downloader {
                 .request(Method::HEAD, url, None)
                 .send()
                 .await?
-                .error_for_status()?;
+                .error_for_status_ext()?;
             if resp
                 .headers()
                 .get(header::ACCEPT_RANGES)
@@ -234,7 +234,7 @@ impl Downloader {
                     .header(header::RANGE, &range_header)
                     .send()
                     .await?
-                    .error_for_status()?;
+                    .error_for_status_ext()?;
                 if let Some(content_length) = resp.header_content_length() {
                     ensure!(
                         content_length == end - start + 1,