feat: 下载时考虑 backup_url，支持按照 cdn 优先级排序 (#288)

crates/bili_sync/src/bilibili/analyzer.rs

@@ -2,6 +2,7 @@ use anyhow::{Context, Result, bail};
 use serde::{Deserialize, Serialize};
 
 use crate::bilibili::error::BiliError;
+use crate::config::CONFIG;
 
 pub struct PageAnalyzer {
     info: serde_json::Value,
@@ -101,24 +102,43 @@ pub enum Stream {
     EpisodeTryMp4(String),
     DashVideo {
         url: String,
+        backup_url: Vec<String>,
         quality: VideoQuality,
         codecs: VideoCodecs,
     },
     DashAudio {
         url: String,
+        backup_url: Vec<String>,
         quality: AudioQuality,
     },
 }
 
 // 通用的获取流链接的方法，交由 Downloader 使用
 impl Stream {
-    pub fn url(&self) -> &str {
+    pub fn urls(&self) -> Vec<&str> {
         match self {
-            Self::Flv(url) => url,
+            Self::Flv(url) | Self::Html5Mp4(url) | Self::EpisodeTryMp4(url) => vec![url],
-            Self::Html5Mp4(url) => url,
+            Self::DashVideo { url, backup_url, .. } | Self::DashAudio { url, backup_url, .. } => {
-            Self::EpisodeTryMp4(url) => url,
+                let mut urls = std::iter::once(url.as_str())
-            Self::DashVideo { url, .. } => url,
+                    .chain(backup_url.iter().map(|s| s.as_str()))
-            Self::DashAudio { url, .. } => url,
+                    .collect();
+                if !CONFIG.cdn_sorting {
+                    urls
+                } else {
+                    urls.sort_by_key(|u| {
+                        if u.contains("upos-") {
+                            0 // 服务商 cdn
+                        } else if u.contains("cn-") {
+                            1 // 自建 cdn
+                        } else if u.contains("mcdn") {
+                            2 // mcdn
+                        } else {
+                            3 // pcdn 或者其它
+                        }
+                    });
+                    urls
+                }
+            }
         }
     }
 }
@@ -180,10 +200,12 @@ impl PageAnalyzer {
             )]);
         }
         let mut streams: Vec<Stream> = Vec::new();
-        for video in self.info["dash"]["video"]
+        for video in self
-            .as_array()
+            .info
+            .pointer_mut("/dash/video")
+            .and_then(|v| v.as_array_mut())
             .ok_or(BiliError::RiskControlOccurred)?
-            .iter()
+            .iter_mut()
         {
             let (Some(url), Some(quality), Some(codecs)) = (
                 video["baseUrl"].as_str(),
@@ -211,12 +233,13 @@ impl PageAnalyzer {
             }
             streams.push(Stream::DashVideo {
                 url: url.to_string(),
+                backup_url: serde_json::from_value(video["backupUrl"].take()).unwrap_or_default(),
                 quality,
                 codecs,
             });
         }
-        if let Some(audios) = self.info["dash"]["audio"].as_array() {
+        if let Some(audios) = self.info.pointer_mut("/dash/audio").and_then(|a| a.as_array_mut()) {
-            for audio in audios.iter() {
+            for audio in audios.iter_mut() {
                 let (Some(url), Some(quality)) = (audio["baseUrl"].as_str(), audio["id"].as_u64()) else {
                     continue;
                 };
@@ -226,34 +249,44 @@ impl PageAnalyzer {
                 }
                 streams.push(Stream::DashAudio {
                     url: url.to_string(),
+                    backup_url: serde_json::from_value(audio["backupUrl"].take()).unwrap_or_default(),
                     quality,
                 });
             }
         }
-        let flac = &self.info["dash"]["flac"]["audio"];
+        if !filter_option.no_hires {
-        if !(filter_option.no_hires || flac.is_null()) {
+            if let Some(flac) = self.info.pointer_mut("/dash/flac/audio") {
-            let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else {
+                let (Some(url), Some(quality)) = (flac["baseUrl"].as_str(), flac["id"].as_u64()) else {
-                bail!("invalid flac stream");
+                    bail!("invalid flac stream");
-            };
+                };
-            let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?;
+                let quality = AudioQuality::from_repr(quality as usize).context("invalid flac stream quality")?;
-            if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
+                if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
-                streams.push(Stream::DashAudio {
+                    streams.push(Stream::DashAudio {
-                    url: url.to_string(),
+                        url: url.to_string(),
-                    quality,
+                        backup_url: serde_json::from_value(flac["backupUrl"].take()).unwrap_or_default(),
-                });
+                        quality,
+                    });
+                }
             }
         }
-        let dolby_audio = &self.info["dash"]["dolby"]["audio"][0];
+        if !filter_option.no_dolby_audio {
-        if !(filter_option.no_dolby_audio || dolby_audio.is_null()) {
+            if let Some(dolby_audio) = self
-            let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else {
+                .info
-                bail!("invalid dolby audio stream");
+                .pointer_mut("/dash/dolby/audio/0")
-            };
+                .and_then(|a| a.as_object_mut())
-            let quality = AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?;
+            {
-            if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
+                let (Some(url), Some(quality)) = (dolby_audio["baseUrl"].as_str(), dolby_audio["id"].as_u64()) else {
-                streams.push(Stream::DashAudio {
+                    bail!("invalid dolby audio stream");
-                    url: url.to_string(),
+                };
-                    quality,
+                let quality =
-                });
+                    AudioQuality::from_repr(quality as usize).context("invalid dolby audio stream quality")?;
+                if quality >= filter_option.audio_min_quality && quality <= filter_option.audio_max_quality {
+                    streams.push(Stream::DashAudio {
+                        url: url.to_string(),
+                        backup_url: serde_json::from_value(dolby_audio["backupUrl"].take()).unwrap_or_default(),
+                        quality,
+                    });
+                }
             }
         }
         Ok(streams)
@@ -270,32 +303,34 @@ impl PageAnalyzer {
         let (videos, audios): (Vec<Stream>, Vec<Stream>) =
             streams.into_iter().partition(|s| matches!(s, Stream::DashVideo { .. }));
         Ok(BestStream::VideoAudio {
-            video: Iterator::max_by(videos.into_iter(), |a, b| match (a, b) {
+            video: videos
-                (
+                .into_iter()
-                    Stream::DashVideo {
+                .max_by(|a, b| match (a, b) {
-                        quality: a_quality,
+                    (
-                        codecs: a_codecs,
+                        Stream::DashVideo {
-                        ..
+                            quality: a_quality,
-                    },
+                            codecs: a_codecs,
-                    Stream::DashVideo {
+                            ..
-                        quality: b_quality,
+                        },
-                        codecs: b_codecs,
+                        Stream::DashVideo {
-                        ..
+                            quality: b_quality,
-                    },
+                            codecs: b_codecs,
-                ) => {
+                            ..
-                    if a_quality != b_quality {
+                        },
-                        return a_quality.cmp(b_quality);
+                    ) => {
-                    };
+                        if a_quality != b_quality {
-                    filter_option
+                            return a_quality.cmp(b_quality);
-                        .codecs
+                        };
-                        .iter()
+                        filter_option
-                        .position(|c| c == b_codecs)
+                            .codecs
-                        .cmp(&filter_option.codecs.iter().position(|c| c == a_codecs))
+                            .iter()
-                }
+                            .position(|c| c == b_codecs)
-                _ => unreachable!(),
+                            .cmp(&filter_option.codecs.iter().position(|c| c == a_codecs))
-            })
+                    }
-            .context("no video stream found")?,
+                    _ => unreachable!(),
-            audio: Iterator::max_by(audios.into_iter(), |a, b| match (a, b) {
+                })
+                .context("no video stream found")?,
+            audio: audios.into_iter().max_by(|a, b| match (a, b) {
                 (Stream::DashAudio { quality: a_quality, .. }, Stream::DashAudio { quality: b_quality, .. }) => {
                     a_quality.cmp(b_quality)
                 }
@@ -389,4 +424,27 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn test_url_sort() {
+        let stream = Stream::DashVideo {
+            url: "https://xy116x207x155x163xy240ey95dy1010y700yy8dxy.mcdn.bilivideo.cn:4483".to_owned(),
+            backup_url: vec![
+                "https://upos-sz-mirrorcos.bilivideo.com".to_owned(),
+                "https://cn-tj-cu-01-11.bilivideo.com".to_owned(),
+                "https://xxx.v1d.szbdys.com".to_owned(),
+            ],
+            quality: VideoQuality::Quality1080p,
+            codecs: VideoCodecs::AVC,
+        };
+        assert_eq!(
+            stream.urls(),
+            vec![
+                "https://upos-sz-mirrorcos.bilivideo.com",
+                "https://cn-tj-cu-01-11.bilivideo.com",
+                "https://xy116x207x155x163xy240ey95dy1010y700yy8dxy.mcdn.bilivideo.cn:4483",
+                "https://xxx.v1d.szbdys.com"
+            ]
+        );
+    }
 }

crates/bili_sync/src/config/global.rs

@@ -81,6 +81,7 @@ fn load_config() -> Config {
     };
     Config {
         credential: arc_swap::ArcSwapOption::from(credential),
+        cdn_sorting: true,
         ..Default::default()
     }
 }

crates/bili_sync/src/config/mod.rs

@@ -69,6 +69,8 @@ pub struct Config {
     pub concurrent_limit: ConcurrentLimit,
     #[serde(default = "default_time_format")]
     pub time_format: String,
+    #[serde(default)]
+    pub cdn_sorting: bool,
 }
 
 impl Default for Config {
@@ -90,6 +92,7 @@ impl Default for Config {
             nfo_time_type: NFOTimeType::FavTime,
             concurrent_limit: ConcurrentLimit::default(),
             time_format: default_time_format(),
+            cdn_sorting: false,
         }
     }
 }

crates/bili_sync/src/downloader.rs

@@ -1,7 +1,7 @@
 use core::str;
 use std::path::Path;
 
-use anyhow::{Result, bail, ensure};
+use anyhow::{Context, Result, bail, ensure};
 use futures::TryStreamExt;
 use reqwest::Method;
 use tokio::fs::{self, File};
@@ -45,6 +45,22 @@ impl Downloader {
         Ok(())
     }
 
+    pub async fn fetch_with_fallback(&self, urls: &[&str], path: &Path) -> Result<()> {
+        if urls.is_empty() {
+            bail!("no urls provided");
+        }
+        let mut res = Ok(());
+        for url in urls {
+            match self.fetch(url, path).await {
+                Ok(_) => return Ok(()),
+                Err(err) => {
+                    res = Err(err);
+                }
+            }
+        }
+        res.with_context(|| format!("failed to download from {:?}", urls))
+    }
+
     pub async fn merge(&self, video_path: &Path, audio_path: &Path, output_path: &Path) -> Result<()> {
         let output = tokio::process::Command::new("ffmpeg")
             .args([

crates/bili_sync/src/workflow.rs

@@ -535,11 +535,11 @@ pub async fn fetch_page_video(
         .await?
         .best_stream(&CONFIG.filter_option)?;
     match streams {
-        BestStream::Mixed(mix_stream) => downloader.fetch(mix_stream.url(), page_path).await?,
+        BestStream::Mixed(mix_stream) => downloader.fetch_with_fallback(&mix_stream.urls(), page_path).await?,
         BestStream::VideoAudio {
             video: video_stream,
             audio: None,
-        } => downloader.fetch(video_stream.url(), page_path).await?,
+        } => downloader.fetch_with_fallback(&video_stream.urls(), page_path).await?,
         BestStream::VideoAudio {
             video: video_stream,
             audio: Some(audio_stream),
@@ -549,8 +549,12 @@ pub async fn fetch_page_video(
                 page_path.with_extension("tmp_audio"),
             );
             let res = async {
-                downloader.fetch(video_stream.url(), &tmp_video_path).await?;
+                downloader
-                downloader.fetch(audio_stream.url(), &tmp_audio_path).await?;
+                    .fetch_with_fallback(&video_stream.urls(), &tmp_video_path)
+                    .await?;
+                downloader
+                    .fetch_with_fallback(&audio_stream.urls(), &tmp_audio_path)
+                    .await?;
                 downloader.merge(&tmp_video_path, &tmp_audio_path, page_path).await
             }
             .await;