Compare commits

..

8 Commits

Author SHA1 Message Date
amtoaer
991ce3ea3c chore: 发布 bili-sync 2.1.2 2024-07-21 23:40:30 +08:00
amtoaer
e4fb096d0c build: 更新项目依赖 2024-07-21 22:51:56 +08:00
ᴀᴍᴛᴏᴀᴇʀ
28070aa7d8 docs: 添加"工作原理"小节 (#135) 2024-07-21 21:34:52 +08:00
ᴀᴍᴛᴏᴀᴇʀ
33e758bd91 refactor: 移除不必要的标记和代码块,统一 use 格式 (#144) 2024-07-21 19:16:52 +08:00
ᴀᴍᴛᴏᴀᴇʀ
86e858082d feat: 为下载视频接口加入 wbi 签名 (#143) 2024-07-21 18:47:09 +08:00
ᴀᴍᴛᴏᴀᴇʀ
2ffe432f37 feat: 为合集接口实现 wbi 签名 (#140) 2024-07-21 16:49:53 +08:00
A1ca7raz
6ef9ecaee0 chore: 更正许可证文件名错误 (#141) 2024-07-19 20:33:36 +08:00
amtoaer
9ef88e1b2b docs: 更新部分表述,更新当前的功能列表 2024-07-11 19:27:02 +08:00
25 changed files with 389 additions and 189 deletions

93
Cargo.lock generated
View File

@@ -342,9 +342,9 @@ checksum = "fbb36e985947064623dbd357f727af08ffd077f93d696782f3c56365fa2e2799"
[[package]]
name = "async-trait"
version = "0.1.80"
version = "0.1.81"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca"
checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107"
dependencies = [
"proc-macro2",
"quote",
@@ -418,7 +418,7 @@ dependencies = [
[[package]]
name = "bili_sync"
version = "2.1.1"
version = "2.1.2"
dependencies = [
"anyhow",
"arc-swap",
@@ -435,6 +435,7 @@ dependencies = [
"futures",
"handlebars",
"hex",
"md5",
"memchr",
"once_cell",
"prost",
@@ -446,6 +447,7 @@ dependencies = [
"sea-orm",
"serde",
"serde_json",
"serde_urlencoded",
"strum 0.26.3",
"thiserror",
"tokio",
@@ -456,7 +458,7 @@ dependencies = [
[[package]]
name = "bili_sync_entity"
version = "2.1.1"
version = "2.1.2"
dependencies = [
"sea-orm",
"serde_json",
@@ -464,7 +466,7 @@ dependencies = [
[[package]]
name = "bili_sync_migration"
version = "2.1.1"
version = "2.1.2"
dependencies = [
"async-std",
"sea-orm-migration",
@@ -621,9 +623,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.8"
version = "4.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d"
checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462"
dependencies = [
"clap_builder",
"clap_derive",
@@ -631,9 +633,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.8"
version = "4.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708"
checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942"
dependencies = [
"anstream",
"anstyle",
@@ -953,10 +955,11 @@ checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984"
[[package]]
name = "filenamify"
version = "0.1.0"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b781e8974b2cc71ac3c587c881c11ee5fe9a379f43503674e1e1052647593b4c"
checksum = "da9be5c5c7738f71d690d14e021cc0bd91888b07b015c5c95df22463e78ca09e"
dependencies = [
"lazy_static",
"regex",
]
@@ -990,7 +993,7 @@ checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
dependencies = [
"futures-core",
"futures-sink",
"spin 0.9.8",
"spin",
]
[[package]]
@@ -1208,9 +1211,9 @@ dependencies = [
[[package]]
name = "handlebars"
version = "5.1.2"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d08485b96a0e6393e9e4d1b8d48cf74ad6c063cd905eb33f42c1ce3f0377539b"
checksum = "5226a0e122dc74917f3a701484482bed3ee86d016c7356836abbaa033133a157"
dependencies = [
"log",
"pest",
@@ -1525,11 +1528,11 @@ dependencies = [
[[package]]
name = "lazy_static"
version = "1.4.0"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
dependencies = [
"spin 0.5.2",
"spin",
]
[[package]]
@@ -1615,6 +1618,12 @@ dependencies = [
"digest",
]
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.7.4"
@@ -2059,9 +2068,9 @@ dependencies = [
[[package]]
name = "prost"
version = "0.12.6"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29"
checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc"
dependencies = [
"bytes",
"prost-derive",
@@ -2069,9 +2078,9 @@ dependencies = [
[[package]]
name = "prost-derive"
version = "0.12.6"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1"
checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca"
dependencies = [
"anyhow",
"itertools",
@@ -2118,9 +2127,9 @@ dependencies = [
[[package]]
name = "quick-xml"
version = "0.35.0"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86e446ed58cef1bbfe847bc2fda0e2e4ea9f0e57b90c507d4781292590d72a4e"
checksum = "4091e032efecb09d7b1f711f487b85ab925632a842627e3200fb088382cde32c"
dependencies = [
"memchr",
"tokio",
@@ -2350,7 +2359,7 @@ dependencies = [
"cfg-if",
"getrandom",
"libc",
"spin 0.9.8",
"spin",
"untrusted",
"windows-sys 0.52.0",
]
@@ -2726,18 +2735,18 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "serde"
version = "1.0.203"
version = "1.0.204"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094"
checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.203"
version = "1.0.204"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba"
checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
dependencies = [
"proc-macro2",
"quote",
@@ -2867,12 +2876,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "spin"
version = "0.9.8"
@@ -3236,18 +3239,18 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.61"
version = "1.0.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709"
checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.61"
version = "1.0.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533"
checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
dependencies = [
"proc-macro2",
"quote",
@@ -3312,9 +3315,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.38.0"
version = "1.38.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a"
checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df"
dependencies = [
"backtrace",
"bytes",
@@ -3377,14 +3380,14 @@ dependencies = [
[[package]]
name = "toml"
version = "0.8.14"
version = "0.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335"
checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit 0.22.14",
"toml_edit 0.22.16",
]
[[package]]
@@ -3409,9 +3412,9 @@ dependencies = [
[[package]]
name = "toml_edit"
version = "0.22.14"
version = "0.22.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38"
checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788"
dependencies = [
"indexmap",
"serde",

View File

@@ -4,7 +4,7 @@ default-members = ["crates/bili_sync"]
resolver = "2"
[workspace.package]
version = "2.1.1"
version = "2.1.2"
authors = ["amtoaer <amtoaer@gmail.com>"]
license = "MIT"
description = "由 Rust & Tokio 驱动的哔哩哔哩同步工具"
@@ -19,20 +19,21 @@ anyhow = { version = "1.0.86", features = ["backtrace"] }
arc-swap = { version = "1.7.1", features = ["serde"] }
async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
async-stream = "0.3.5"
async-trait = "0.1.80"
async-trait = "0.1.81"
chrono = { version = "0.4.38", features = ["serde"] }
clap = { version = "4.5.8", features = ["env"] }
clap = { version = "4.5.9", features = ["env"] }
cookie = "0.18.1"
dirs = "5.0.1"
filenamify = "0.1.0"
filenamify = "0.1.1"
float-ord = "0.3.2"
futures = "0.3.30"
handlebars = "5.1.2"
handlebars = "6.0.0"
hex = "0.4.3"
md5 = "0.7.0"
memchr = "2.7.4"
once_cell = "1.19.0"
prost = "0.12.6"
quick-xml = { version = "0.35.0", features = ["async-tokio"] }
prost = "0.13.1"
quick-xml = { version = "0.36.0", features = ["async-tokio"] }
rand = "0.8.5"
regex = "1.10.5"
reqwest = { version = "0.12.5", features = [
@@ -51,12 +52,13 @@ sea-orm = { version = "0.12.15", features = [
"sqlx-sqlite",
] }
sea-orm-migration = { version = "0.12.15", features = [] }
serde = { version = "1.0.203", features = ["derive"] }
serde = { version = "1.0.204", features = ["derive"] }
serde_json = "1.0.120"
serde_urlencoded = "0.7.1"
strum = { version = "0.26.3", features = ["derive"] }
thiserror = "1.0.61"
tokio = { version = "1.38.0", features = ["full"] }
toml = "0.8.14"
thiserror = "1.0.63"
tokio = { version = "1.38.1", features = ["full"] }
toml = "0.8.15"
tracing = "0.1.40"
tracing-subscriber = { version = "0.3.18", features = ["chrono"] }

View File

View File

@@ -30,7 +30,8 @@ bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具,由 Rust
- [x] 使用数据库保存媒体信息,避免对同个视频的多次请求
- [x] 打印日志,并在请求出现风控时自动终止,等待下一轮执行
- [x] 提供多平台的二进制可执行文件,为 Linux 平台提供了立即可用的 Docker 镜像
- [ ] 支持对“稍后再看”内视频的自动扫描与下载
- [x] 支持对“稍后再看”内视频的自动扫描与下载
- [ ] 支持对 UP 主投稿视频的自动扫描与下载
- [ ] 下载单个文件时支持断点续传与并发下载

View File

@@ -24,6 +24,7 @@ float-ord = { workspace = true }
futures = { workspace = true }
handlebars = { workspace = true }
hex = { workspace = true }
md5 = { workspace = true }
memchr = { workspace = true }
once_cell = { workspace = true }
prost = { workspace = true }
@@ -35,6 +36,7 @@ rsa = { workspace = true }
sea-orm = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
serde_urlencoded = { workspace = true }
strum = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }

View File

@@ -11,7 +11,7 @@ use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
use super::VideoListModel;
use crate::adapter::VideoListModel;
use crate::bilibili::{BiliClient, BiliError, Collection, CollectionItem, CollectionType, Video, VideoInfo};
use crate::config::TEMPLATE;
use crate::utils::id_time_key;

View File

@@ -11,7 +11,7 @@ use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
use super::VideoListModel;
use crate::adapter::VideoListModel;
use crate::bilibili::{BiliClient, BiliError, FavoriteList, Video, VideoInfo};
use crate::config::TEMPLATE;
use crate::utils::id_time_key;

View File

@@ -11,7 +11,7 @@ use sea_orm::entity::prelude::*;
use sea_orm::ActiveValue::Set;
use sea_orm::{DatabaseConnection, QuerySelect, TransactionTrait};
use super::VideoListModel;
use crate::adapter::VideoListModel;
use crate::bilibili::{BiliClient, BiliError, Video, VideoInfo, WatchLater};
use crate::config::TEMPLATE;
use crate::utils::id_time_key;

View File

@@ -3,6 +3,7 @@ use std::sync::Arc;
use anyhow::{bail, Result};
use reqwest::{header, Method};
use crate::bilibili::credential::WbiImg;
use crate::bilibili::Credential;
use crate::config::CONFIG;
@@ -86,12 +87,12 @@ impl BiliClient {
CONFIG.save()
}
/// 检查凭据是否已设置且有效
pub async fn is_login(&self) -> Result<()> {
/// 获取 wbi img用于生成请求签名
pub async fn wbi_img(&self) -> Result<WbiImg> {
let credential = CONFIG.credential.load();
let Some(credential) = credential.as_deref() else {
bail!("no credential found");
};
credential.is_login(&self.client).await
credential.wbi_img(&self.client).await
}
}

View File

@@ -9,7 +9,8 @@ use reqwest::Method;
use serde::Deserialize;
use serde_json::Value;
use crate::bilibili::{BiliClient, Validate, VideoInfo};
use crate::bilibili::credential::encoded_query;
use crate::bilibili::{BiliClient, Validate, VideoInfo, MIXIN_KEY};
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
pub enum CollectionType {
@@ -108,10 +109,6 @@ impl<'a> Collection<'a> {
}
async fn get_series_info(&self) -> Result<Value> {
assert!(
self.collection.collection_type == CollectionType::Series,
"collection type is not series"
);
self.client
.request(Method::GET, "https://api.bilibili.com/x/series/series")
.query(&[("series_id", self.collection.sid.as_str())])
@@ -128,24 +125,30 @@ impl<'a> Collection<'a> {
let (url, query) = match self.collection.collection_type {
CollectionType::Series => (
"https://api.bilibili.com/x/series/archives",
vec![
("mid", self.collection.mid.as_str()),
("series_id", self.collection.sid.as_str()),
("only_normal", "true"),
("sort", "desc"),
("pn", page.as_str()),
("ps", "30"),
],
encoded_query(
vec![
("mid", self.collection.mid.as_str()),
("series_id", self.collection.sid.as_str()),
("only_normal", "true"),
("sort", "desc"),
("pn", page.as_str()),
("ps", "30"),
],
MIXIN_KEY.load().as_ref().unwrap(),
),
),
CollectionType::Season => (
"https://api.bilibili.com/x/polymer/web-space/seasons_archives_list",
vec![
("mid", self.collection.mid.as_str()),
("season_id", self.collection.sid.as_str()),
("sort_reverse", "true"),
("page_num", page.as_str()),
("page_size", "30"),
],
encoded_query(
vec![
("mid", self.collection.mid.as_str()),
("season_id", self.collection.sid.as_str()),
("sort_reverse", "true"),
("page_num", page.as_str()),
("page_size", "30"),
],
MIXIN_KEY.load().as_ref().unwrap(),
),
),
};
self.client

View File

@@ -11,6 +11,12 @@ use serde::{Deserialize, Serialize};
use crate::bilibili::{Client, Validate};
const MIXIN_KEY_ENC_TAB: [usize; 64] = [
46, 47, 18, 2, 53, 8, 23, 32, 15, 50, 10, 31, 58, 3, 45, 35, 27, 43, 5, 49, 33, 9, 42, 19, 29, 28, 14, 39, 12, 38,
41, 13, 37, 48, 7, 16, 24, 55, 40, 61, 26, 17, 0, 1, 60, 51, 30, 4, 22, 25, 54, 21, 56, 59, 6, 63, 57, 62, 11, 36,
20, 34, 44, 52,
];
#[derive(Default, Debug, Clone, Serialize, Deserialize)]
pub struct Credential {
pub sessdata: String,
@@ -20,7 +26,30 @@ pub struct Credential {
pub ac_time_value: String,
}
#[derive(Debug, Deserialize)]
pub struct WbiImg {
img_url: String,
sub_url: String,
}
impl WbiImg {
pub fn into_mixin_key(self) -> Option<String> {
get_mixin_key(self)
}
}
impl Credential {
pub async fn wbi_img(&self, client: &Client) -> Result<WbiImg> {
let mut res = client
.request(Method::GET, "https://api.bilibili.com/x/web-interface/nav", Some(self))
.send()
.await?
.json::<serde_json::Value>()
.await?
.validate()?;
Ok(serde_json::from_value(res["data"]["wbi_img"].take())?)
}
/// 检查凭据是否有效
pub async fn need_refresh(&self, client: &Client) -> Result<bool> {
let res = client
@@ -38,24 +67,6 @@ impl Credential {
res["data"]["refresh"].as_bool().ok_or(anyhow!("check refresh failed"))
}
/// 需要使用一个需要鉴权的接口来检查是否登录
/// 此处使用查看用户状态数的接口,该接口返回内容少,请求成本低
pub async fn is_login(&self, client: &Client) -> Result<()> {
client
.request(
Method::GET,
"https://api.bilibili.com/x/web-interface/nav/stat",
Some(self),
)
.send()
.await?
.error_for_status()?
.json::<serde_json::Value>()
.await?
.validate()?;
Ok(())
}
pub async fn refresh(&self, client: &Client) -> Result<Self> {
let correspond_path = Self::get_correspond_path();
let csrf = self.get_refresh_csrf(client, correspond_path).await?;
@@ -181,6 +192,41 @@ fn regex_find(pattern: &str, doc: &str) -> Result<String> {
.to_string())
}
fn get_filename(url: &str) -> Option<&str> {
url.rsplit_once('/')
.and_then(|(_, s)| s.rsplit_once('.'))
.map(|(s, _)| s)
}
fn get_mixin_key(wbi_img: WbiImg) -> Option<String> {
let key = match (
get_filename(wbi_img.img_url.as_str()),
get_filename(wbi_img.sub_url.as_str()),
) {
(Some(img_key), Some(sub_key)) => img_key.to_string() + sub_key,
_ => return None,
};
let key = key.as_bytes();
Some(MIXIN_KEY_ENC_TAB.iter().take(32).map(|&x| key[x] as char).collect())
}
pub fn encoded_query<'a>(params: Vec<(&'a str, impl Into<String>)>, mixin_key: &str) -> Vec<(&'a str, String)> {
let params = params.into_iter().map(|(k, v)| (k, v.into())).collect();
_encoded_query(params, mixin_key, chrono::Local::now().timestamp().to_string())
}
fn _encoded_query<'a>(params: Vec<(&'a str, String)>, mixin_key: &str, timestamp: String) -> Vec<(&'a str, String)> {
let mut params: Vec<(&'a str, String)> = params
.into_iter()
.map(|(k, v)| (k, v.chars().filter(|&x| !"!'()*".contains(x)).collect::<String>()))
.collect();
params.push(("wts", timestamp));
params.sort_by(|a, b| a.0.cmp(b.0));
let query = serde_urlencoded::to_string(&params).unwrap().replace('+', "%20");
params.push(("w_rid", format!("{:x}", md5::compute(query.clone() + mixin_key))));
params
}
#[cfg(test)]
mod tests {
use super::*;
@@ -199,4 +245,45 @@ mod tests {
"b0cc8411ded2f9db2cff2edb3123acac",
);
}
#[test]
fn test_encode_query() {
let query = vec![
("bar", "五一四".to_string()),
("baz", "1919810".to_string()),
("foo", "one one four".to_string()),
];
assert_eq!(
serde_urlencoded::to_string(query).unwrap().replace('+', "%20"),
"bar=%E4%BA%94%E4%B8%80%E5%9B%9B&baz=1919810&foo=one%20one%20four"
);
}
#[test]
fn test_wbi_key() {
let key = WbiImg {
img_url: "https://i0.hdslb.com/bfs/wbi/7cd084941338484aae1ad9425b84077c.png".to_string(),
sub_url: "https://i0.hdslb.com/bfs/wbi/4932caff0ff746eab6f01bf08b70ac45.png".to_string(),
};
let mixin_key = get_mixin_key(key);
assert_eq!(mixin_key, Some("ea1db124af3c7062474693fa704f4ff8".to_string()));
assert_eq!(
_encoded_query(
vec![
("foo", "114".to_string()),
("bar", "514".to_string()),
("zab", "1919810".to_string())
],
&mixin_key.unwrap(),
"1702204169".to_string(),
),
vec![
("bar", "514".to_string()),
("foo", "114".to_string()),
("wts", "1702204169".to_string()),
("zab", "1919810".to_string()),
("w_rid", "8f6f2b5b3d485fe1886cec6a0be8c5d4".to_string()),
]
)
}
}

View File

@@ -1,5 +1,8 @@
use std::sync::Arc;
pub use analyzer::{BestStream, FilterOption};
use anyhow::{bail, Result};
use arc_swap::ArcSwapOption;
use chrono::serde::ts_seconds;
use chrono::{DateTime, Utc};
pub use client::{BiliClient, Client};
@@ -9,6 +12,7 @@ pub use danmaku::DanmakuOption;
pub use error::BiliError;
pub use favorite_list::FavoriteList;
use favorite_list::Upper;
use once_cell::sync::Lazy;
pub use video::{Dimension, PageInfo, Video};
pub use watch_later::WatchLater;
@@ -22,6 +26,12 @@ mod favorite_list;
mod video;
mod watch_later;
static MIXIN_KEY: Lazy<ArcSwapOption<String>> = Lazy::new(Default::default);
pub(crate) fn set_global_mixin_key(key: String) {
MIXIN_KEY.store(Some(Arc::new(key)));
}
pub(crate) trait Validate {
type Output;
@@ -121,8 +131,16 @@ mod tests {
#[ignore = "only for manual test"]
#[tokio::test]
async fn assert_video_info_type() {
async fn test_video_info_type() {
let bili_client = BiliClient::new();
set_global_mixin_key(
bili_client
.wbi_img()
.await
.map(|x| x.into_mixin_key())
.unwrap()
.unwrap(),
);
let video = Video::new(&bili_client, "BV1Z54y1C7ZB".to_string());
assert!(matches!(video.get_view_info().await, Ok(VideoInfo::View { .. })));
let collection_item = CollectionItem {

View File

@@ -6,8 +6,9 @@ use reqwest::Method;
use crate::bilibili::analyzer::PageAnalyzer;
use crate::bilibili::client::BiliClient;
use crate::bilibili::credential::encoded_query;
use crate::bilibili::danmaku::{DanmakuElem, DanmakuWriter, DmSegMobileReply};
use crate::bilibili::{Validate, VideoInfo};
use crate::bilibili::{Validate, VideoInfo, MIXIN_KEY};
static MASK_CODE: u64 = 2251799813685247;
static XOR_CODE: u64 = 23442827791579;
@@ -61,7 +62,6 @@ impl<'a> Video<'a> {
Self { client, aid, bvid }
}
#[allow(dead_code)]
/// 直接调用视频信息接口获取详细的视频信息
pub async fn get_view_info(&self) -> Result<VideoInfo> {
let mut res = self
@@ -140,14 +140,17 @@ impl<'a> Video<'a> {
let mut res = self
.client
.request(Method::GET, "https://api.bilibili.com/x/player/wbi/playurl")
.query(&[
("avid", self.aid.as_str()),
("cid", page.cid.to_string().as_str()),
("qn", "127"),
("otype", "json"),
("fnval", "4048"),
("fourk", "1"),
])
.query(&encoded_query(
vec![
("avid", self.aid.as_str()),
("cid", page.cid.to_string().as_str()),
("qn", "127"),
("otype", "json"),
("fnval", "4048"),
("fourk", "1"),
],
MIXIN_KEY.load().as_ref().unwrap(),
))
.send()
.await?
.error_for_status()?

View File

@@ -31,42 +31,49 @@ async fn main() {
let bili_client = BiliClient::new();
let watch_later_config = &CONFIG.watch_later;
loop {
if let Err(e) = bili_client.is_login().await {
error!("检查登录状态时遇到错误:{e},等待下一轮执行");
time::sleep(Duration::from_secs(CONFIG.interval)).await;
continue;
}
if anchor != chrono::Local::now().date_naive() {
if let Err(e) = bili_client.check_refresh().await {
error!("检查刷新 Credential 遇到错误:{e},等待下一轮执行");
time::sleep(Duration::from_secs(CONFIG.interval)).await;
continue;
'inner: {
match bili_client.wbi_img().await.map(|wbi_img| wbi_img.into_mixin_key()) {
Ok(Some(mixin_key)) => bilibili::set_global_mixin_key(mixin_key),
Ok(_) => {
error!("获取 mixin key 失败,无法进行 wbi 签名,等待下一轮执行");
break 'inner;
}
Err(e) => {
error!("获取 mixin key 时遇到错误:{e},等待下一轮执行");
break 'inner;
}
};
if anchor != chrono::Local::now().date_naive() {
if let Err(e) = bili_client.check_refresh().await {
error!("检查刷新 Credential 遇到错误:{e},等待下一轮执行");
break 'inner;
}
anchor = chrono::Local::now().date_naive();
}
anchor = chrono::Local::now().date_naive();
}
for (fid, path) in &CONFIG.favorite_list {
if let Err(e) = process_video_list(Args::Favorite { fid }, &bili_client, path, &connection).await {
error!("处理收藏夹 {fid} 时遇到非预期的错误:{e}");
for (fid, path) in &CONFIG.favorite_list {
if let Err(e) = process_video_list(Args::Favorite { fid }, &bili_client, path, &connection).await {
error!("处理收藏夹 {fid} 时遇到非预期的错误:{e}");
}
}
}
info!("所有收藏夹处理完毕");
for (collection_item, path) in &CONFIG.collection_list {
if let Err(e) =
process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await
{
error!("处理合集 {collection_item:?} 时遇到非预期的错误:{e}");
info!("所有收藏夹处理完毕");
for (collection_item, path) in &CONFIG.collection_list {
if let Err(e) =
process_video_list(Args::Collection { collection_item }, &bili_client, path, &connection).await
{
error!("处理合集 {collection_item:?} 时遇到非预期的错误:{e}");
}
}
}
info!("所有合集处理完毕");
if watch_later_config.enabled {
if let Err(e) =
process_video_list(Args::WatchLater, &bili_client, &watch_later_config.path, &connection).await
{
error!("处理稍后再看时遇到非预期的错误:{e}");
info!("所有合集处理完毕");
if watch_later_config.enabled {
if let Err(e) =
process_video_list(Args::WatchLater, &bili_client, &watch_later_config.path, &connection).await
{
error!("处理稍后再看时遇到非预期的错误:{e}");
}
}
info!("稍后再看处理完毕");
info!("本轮任务执行完毕,等待下一轮执行");
}
info!("稍后再看处理完毕");
info!("本轮任务执行完毕,等待下一轮执行");
tokio::time::sleep(std::time::Duration::from_secs(CONFIG.interval)).await;
time::sleep(Duration::from_secs(CONFIG.interval)).await;
}
}

View File

@@ -13,7 +13,7 @@ pub struct Status(u32);
impl Status {
/// 如果 status 整体大于等于 1 << 31则表示任务已经被处理过不再需要重试。
/// 数据库可以使用 status < Status::handled() 来筛选需要处理的内容。
pub fn handled() -> u32 {
pub const fn handled() -> u32 {
1 << 31
}
@@ -32,19 +32,16 @@ impl Status {
/// 从低到高检查状态,如果该位置的任务应该继续尝试执行,则返回 true否则返回 false
fn should_run(&self, size: usize) -> Vec<bool> {
assert!(size < 10, "u32 can only store 10 status");
(0..size).map(|x| self.check_continue(x)).collect()
}
/// 如果任务的执行次数小于 STATUS_MAX_RETRY说明可以继续运行
fn check_continue(&self, offset: usize) -> bool {
assert!(offset < 10, "u32 can only store 10 status");
self.get_status(offset) < STATUS_MAX_RETRY
}
/// 根据任务结果更新状态,如果任务成功,设置为 STATUS_OK否则加一
fn update_status(&mut self, result: &[Result<()>]) {
assert!(result.len() < 10, "u32 can only store 10 status");
for (i, res) in result.iter().enumerate() {
self.set_result(res, i);
}
@@ -65,17 +62,6 @@ impl Status {
}
}
/// 根据 mask 设置状态,如果 mask 为 false则清除对应的状态
fn set_mask(&mut self, mask: &[bool]) {
assert!(mask.len() < 10, "u32 can only store 10 status");
for (i, &m) in mask.iter().enumerate() {
if !m {
self.clear(i);
self.set_flag(false);
}
}
}
fn plus_one(&mut self, offset: usize) {
self.0 += 1 << (3 * offset);
}
@@ -84,10 +70,6 @@ impl Status {
self.0 |= STATUS_OK << (3 * offset);
}
fn clear(&mut self, offset: usize) {
self.0 &= !(STATUS_OK << (3 * offset));
}
fn get_status(&self, offset: usize) -> u32 {
let helper = !0u32;
(self.0 & (helper << (offset * 3)) & (helper >> (32 - 3 * offset - 3))) >> (offset * 3)
@@ -109,11 +91,6 @@ impl VideoStatus {
Self(Status::new(status))
}
pub fn set_mask(&mut self, clear: &[bool]) {
assert!(clear.len() == 5, "VideoStatus should have 5 status");
self.0.set_mask(clear)
}
pub fn should_run(&self) -> Vec<bool> {
self.0.should_run(5)
}
@@ -139,11 +116,6 @@ impl PageStatus {
Self(Status::new(status))
}
pub fn set_mask(&mut self, clear: &[bool]) {
assert!(clear.len() == 4, "PageStatus should have 4 status");
self.0.set_mask(clear)
}
pub fn should_run(&self) -> Vec<bool> {
self.0.should_run(4)
}

View File

@@ -1,5 +1,3 @@
#![allow(dead_code, unused_variables)]
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::pin::Pin;
@@ -31,7 +29,7 @@ pub async fn process_video_list(
connection: &DatabaseConnection,
) -> Result<()> {
let (video_list_model, video_streams) = video_list_from(args, path, bili_client, connection).await?;
let video_list_model = refresh_video_list(bili_client, video_list_model, video_streams, connection).await?;
let video_list_model = refresh_video_list(video_list_model, video_streams, connection).await?;
let video_list_model = fetch_video_details(bili_client, video_list_model, connection).await?;
if ARGS.scan_only {
warn!("已开启仅扫描模式,跳过视频下载...");
@@ -42,7 +40,6 @@ pub async fn process_video_list(
/// 请求接口,获取视频列表中所有新添加的视频信息,将其写入数据库
pub async fn refresh_video_list<'a>(
bili_client: &'a BiliClient,
video_list_model: Box<dyn VideoListModel>,
video_streams: Pin<Box<dyn Stream<Item = VideoInfo> + 'a>>,
connection: &DatabaseConnection,

View File

@@ -21,7 +21,7 @@ export default defineConfig({
nav: [
{ text: "主页", link: "/" },
{
text: "v2.1.1",
text: "v2.1.2",
items: [
{
text: "程序更新",
@@ -47,6 +47,7 @@ export default defineConfig({
items: [
{ text: "配置文件", link: "/configuration" },
{ text: "命令行参数", link: "/args" },
{ text: "工作原理", link: "/design" },
],
},
{

Binary file not shown.

After

Width:  |  Height:  |  Size: 173 KiB

BIN
docs/assets/bili_video.jpg Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 172 KiB

BIN
docs/assets/multi_page.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 MiB

BIN
docs/assets/single_page.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 MiB

110
docs/design.md Normal file
View File

@@ -0,0 +1,110 @@
# 工作原理
本节会尽可能简单明了地介绍 `bili-sync` 的工作原理,让用户了解程序的整体执行过程。
## b 站的视频结构
在了解程序工作原理之前,我们需要先对 b 站视频的组织结构有一个大概的了解。简单来说:
- 收藏夹、稍后再看、视频合集、视频列表等结构都是由一系列视频构成的列表;
- 每个视频都有唯一的 bvid包含了封面、描述和标签信息并包含了一个或多个分页
- 每个分页都有一个唯一的 cid包含了封面、视频、音频、弹幕。
为了描述方便,在后文会将收藏夹、稍后再看这类结构统称为 video list将视频称为 video将分页称为 page。不难看出这三者有着很明显的层级关系**video list 包含若干 videovideo 包含若干 page**。
一个非常容易混淆的点是视频合集/视频列表与多页视频的区别:
> [!NOTE]
> ![bili_collection](./assets/bili_collection.jpg)
>
>![bili_video](./assets/bili_video.jpg)
这两张图中,上图是视频合集,下图是多页视频。这两者在展示上区别较小,但在结构上有相当大的不同。结合上面对 b 站视频结构的介绍,这个区别可以简单总结为:
+ **视频合集是由多个仅包含单个 page 的 video 组成的 video list**
+ **多页视频是由多个 page 组成的 video**
这说明它们是处于两个不同层级的结构,因此程序对其的处理方式也有着相当大的不同。
## 与 EMBY 媒体库的对应关系
EMBY 的一般结构是: `媒体库 - 文件夹 - 电影/电视剧 - 分季/分集`,方便起见,我采用了如下的对应关系:
1. **文件夹**:对应 b 站的 video list
2. **电视剧** 对应 b 站的 video
3. **第一季的所有分集**:对应 b 站的 page。
特别的,当 video 仅有一个 page 时为了避免过多的层级bili-sync 会将 page 展开到第二层级,变成与电视剧同级的电影。
因此,**需要将媒体库类型设置为“混合内容”以支持在同个媒体库中同时显示电视剧与电影**。
### 单 page 的 video
![single_page](./assets/single_page.png)
### 多 page 的 video
![multi_page](./assets/multi_page.png)
![multi_page_detail](./assets/multi_page_detail.png)
## 数据库设计
> [!NOTE]
> 可以[前往此处](https://github.com/amtoaer/bili-sync/tree/main/crates/bili_sync_entity/src/entities)实时查看当前版本的数据库表结构。
既然拥有着明显的层级关系,那数据库表就很容易设计了。为了简化实现,程序没有额外考虑单个 video 被多个 video list 引用的情况(如一个视频同时在收藏夹和稍后再看中)。而是简单的将其设计为了不交叉的层级结构。
### video list 表
从上面的介绍可以看出video list 并不是一个具体的结构,而是拥有多种实现的抽象概念。我选择将其特化实现为多张表:
1. favorite收藏夹
2. watch_later稍后再看
3. collection: 视频合集/视频列表;
4. ....
### video 表
video 表包含了 video 的基本信息,如 bvid、标题、封面、描述、标签等。此外video 表还包含了与 video list 的关联。
具体来说,每一种 video list 都在 video 表中有一个对应的列,指向 video list 表中的 id如 favorite_id、collection_id 等。接下来将这些键与 video 的 bvid 绑在一起建立唯一索引,就可以保证在同一个 video list 中不会有重复的 video。
### page 表
page 表包含了 page 的基本信息,如 cid、标题、封面等。与 video 类似但更简单page 表仅包含了与 video 的关联。
## 执行过程
### 初始化
程序启动时会读取配置文件、迁移数据库、初始化日志等操作。如果发现需要的文件不存在,程序会自动创建。
### 扫描 video list
> [!WARNING]
> b 站实现接口时为了节省资源,通过 video list 获取到的 video 列表通常是分页且不包含详细信息的。
程序会扫描所有配置文件中包含的 video list获取其中包含的 video 的简单信息并填充到数据库。在实现时需要避免频繁的全量扫描。
具体到 bili-sync 的实现中,程序在请求接口时会设置按时间顺序排序的参数,确保新发布的视频位于前面。拉取过程会逐页请求,使用视频的 bvid 与 time 字段来检验视频是否已经存在于数据库中。一旦发现 bvid 与 time 均相同的记录则认为已经到达扫描过的位置,停止拉取。
### 填充 video 详情
将新增视频的简单信息写入数据库后,下一步会填充 video 详情。正如上文所述:**通过 video list 获取到的 video 列表通常是不包含详细信息的**,因此需要额外的请求来填充这些信息。
这一步会筛选出所有未完全填充信息的 video逐个获取 video 的详细信息(如标签、视频分页等)并填充到数据库中。
在这个过程中,如果遇到 -404 错误码则说明视频无法被正常访问,程序会将该视频标记为无效并跳过。
### 下载未处理的视频
经过上面处理后,数据库中已经包含了所有需要的 video 信息,接下来只需要筛选其中“未完全下载”、“成功填充详细信息”的所有视频,并发下载即可。程序在 video 层级最多允许 3 个任务同时下载page 层级最多允许 2 个任务同时下载。
数据库中的 status 字段用于标记 video 和 page 的下载状态视频的各个部分封面、视频、nfo 等)包含在 status 的不同位中。程序会根据 status 的不同位来判断视频的下载状态,以此来决定是否需要下载。
如果某些部分下载失败status 字段会记录这些部分的失败次数,程序会在下次下载时重试。如果重试次数超过了设定的阈值,那么视频会被标记为下载失败,后续直接忽略。
此处程序对风控做了额外的处理,一般风控发生时接下来的所有请求都会失败,因此程序检测到风控时不会认为是某个视频下载失败,而是直接终止 video list 的全部下载任务,等待下次扫描时重试。

View File

@@ -1,7 +1,7 @@
# bili-sync 是什么?
> [!TIP]
> 当前最新程序版本为 v2.1.1,文档将始终与最新程序版本保持一致。
> 当前最新程序版本为 v2.1.2,文档将始终与最新程序版本保持一致。
bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具。
@@ -36,5 +36,6 @@ bili-sync 是一款专为 NAS 用户编写的哔哩哔哩同步工具。
- [x] 使用数据库保存媒体信息,避免对同个视频的多次请求
- [x] 打印日志,并在请求出现风控时自动终止,等待下一轮执行
- [x] 提供多平台的二进制可执行文件,为 Linux 平台提供了立即可用的 Docker 镜像
- [ ] 支持对“稍后再看”内视频的自动扫描与下载
- [x] 支持对“稍后再看”内视频的自动扫描与下载
- [ ] 支持对 UP 主投稿视频的自动扫描与下载
- [ ] 下载单个文件时支持断点续传与并发下载

View File

@@ -111,15 +111,7 @@ enabled = false
path = ""
```
看起来很长,但绝大部分选项是不需要做修改的。正常情况下,我们只需要关注
+ `interval`
+ `upper_path`
+ `credential`
+ `codecs`
+ `favorite_list`
+ `collection_list`
以下逐条说明。
虽然配置文件看起来很长,但绝大部分选项是不需要做修改的。一般来说,我们只需要关注其中的少数几个,以下逐条说明。
### `interval`