mirror of
https://github.com/amtoaer/bili-sync.git
synced 2026-09-05 23:49:53 +08:00
feat: 支持使用动态 api 获取投稿,该 api 会返回动态视频 (#485)
This commit is contained in:
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
**/target
|
**/target
|
||||||
auth_data
|
auth_data
|
||||||
*.sqlite
|
*.sqlite*
|
||||||
debug*
|
debug*
|
||||||
node_modules
|
node_modules
|
||||||
docs/.vitepress/cache
|
docs/.vitepress/cache
|
||||||
|
|||||||
@@ -44,7 +44,12 @@ impl VideoSource for collection::Model {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_take(&self, _release_datetime: &chrono::DateTime<Utc>, _latest_row_at: &chrono::DateTime<Utc>) -> bool {
|
fn should_take(
|
||||||
|
&self,
|
||||||
|
_idx: usize,
|
||||||
|
_release_datetime: &chrono::DateTime<Utc>,
|
||||||
|
_latest_row_at: &chrono::DateTime<Utc>,
|
||||||
|
) -> bool {
|
||||||
// collection(视频合集/视频列表)返回的内容似乎并非严格按照时间排序,并且不同 collection 的排序方式也不同
|
// collection(视频合集/视频列表)返回的内容似乎并非严格按照时间排序,并且不同 collection 的排序方式也不同
|
||||||
// 为了保证程序正确性,collection 不根据时间提前 break,而是每次都全量拉取
|
// 为了保证程序正确性,collection 不根据时间提前 break,而是每次都全量拉取
|
||||||
true
|
true
|
||||||
@@ -52,6 +57,7 @@ impl VideoSource for collection::Model {
|
|||||||
|
|
||||||
fn should_filter(
|
fn should_filter(
|
||||||
&self,
|
&self,
|
||||||
|
_idx: usize,
|
||||||
video_info: Result<VideoInfo, anyhow::Error>,
|
video_info: Result<VideoInfo, anyhow::Error>,
|
||||||
latest_row_at: &chrono::DateTime<Utc>,
|
latest_row_at: &chrono::DateTime<Utc>,
|
||||||
) -> Option<VideoInfo> {
|
) -> Option<VideoInfo> {
|
||||||
@@ -61,7 +67,6 @@ impl VideoSource for collection::Model {
|
|||||||
{
|
{
|
||||||
return Some(video_info);
|
return Some(video_info);
|
||||||
}
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -56,12 +56,18 @@ pub trait VideoSource {
|
|||||||
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel;
|
fn update_latest_row_at(&self, datetime: DateTime) -> _ActiveModel;
|
||||||
|
|
||||||
// 判断是否应该继续拉取视频
|
// 判断是否应该继续拉取视频
|
||||||
fn should_take(&self, release_datetime: &chrono::DateTime<Utc>, latest_row_at: &chrono::DateTime<Utc>) -> bool {
|
fn should_take(
|
||||||
|
&self,
|
||||||
|
_idx: usize,
|
||||||
|
release_datetime: &chrono::DateTime<Utc>,
|
||||||
|
latest_row_at: &chrono::DateTime<Utc>,
|
||||||
|
) -> bool {
|
||||||
release_datetime > latest_row_at
|
release_datetime > latest_row_at
|
||||||
}
|
}
|
||||||
|
|
||||||
fn should_filter(
|
fn should_filter(
|
||||||
&self,
|
&self,
|
||||||
|
_idx: usize,
|
||||||
video_info: Result<VideoInfo, anyhow::Error>,
|
video_info: Result<VideoInfo, anyhow::Error>,
|
||||||
_latest_row_at: &chrono::DateTime<Utc>,
|
_latest_row_at: &chrono::DateTime<Utc>,
|
||||||
) -> Option<VideoInfo> {
|
) -> Option<VideoInfo> {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use sea_orm::sea_query::SimpleExpr;
|
|||||||
use sea_orm::{DatabaseConnection, Unchanged};
|
use sea_orm::{DatabaseConnection, Unchanged};
|
||||||
|
|
||||||
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
use crate::adapter::{_ActiveModel, VideoSource, VideoSourceEnum};
|
||||||
use crate::bilibili::{BiliClient, Submission, VideoInfo};
|
use crate::bilibili::{BiliClient, Dynamic, Submission, VideoInfo};
|
||||||
|
|
||||||
impl VideoSource for submission::Model {
|
impl VideoSource for submission::Model {
|
||||||
fn display_name(&self) -> std::borrow::Cow<'static, str> {
|
fn display_name(&self) -> std::borrow::Cow<'static, str> {
|
||||||
@@ -42,6 +42,42 @@ impl VideoSource for submission::Model {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn should_take(
|
||||||
|
&self,
|
||||||
|
idx: usize,
|
||||||
|
release_datetime: &chrono::DateTime<chrono::Utc>,
|
||||||
|
latest_row_at: &chrono::DateTime<chrono::Utc>,
|
||||||
|
) -> bool {
|
||||||
|
// 如果使用动态 API,那么可能出现用户置顶了一个很久以前的视频在动态顶部的情况
|
||||||
|
// 这种情况应该继续拉取下去,不能因为第一条不满足条件就停止
|
||||||
|
// 后续的非置顶内容是正常由新到旧排序的,可以继续使用常规方式处理
|
||||||
|
if idx == 0 && self.use_dynamic_api {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
release_datetime > latest_row_at
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_filter(
|
||||||
|
&self,
|
||||||
|
idx: usize,
|
||||||
|
video_info: Result<VideoInfo, anyhow::Error>,
|
||||||
|
latest_row_at: &chrono::DateTime<chrono::Utc>,
|
||||||
|
) -> Option<VideoInfo> {
|
||||||
|
if idx == 0 && self.use_dynamic_api {
|
||||||
|
// 同理,动态 API 的第一条内容可能是置顶的老视频,单独做个过滤
|
||||||
|
// 其实不过滤也不影响逻辑正确性,因为后续 insert 发生冲突仍然会忽略掉
|
||||||
|
// 此处主要是出于性能考虑,减少不必要的数据库操作
|
||||||
|
if let Ok(video_info) = video_info
|
||||||
|
&& video_info.release_datetime() > latest_row_at
|
||||||
|
{
|
||||||
|
return Some(video_info);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
video_info.ok()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn rule(&self) -> &Option<Rule> {
|
fn rule(&self) -> &Option<Rule> {
|
||||||
&self.rule
|
&self.rule
|
||||||
}
|
}
|
||||||
@@ -69,6 +105,12 @@ impl VideoSource for submission::Model {
|
|||||||
}
|
}
|
||||||
.update(connection)
|
.update(connection)
|
||||||
.await?;
|
.await?;
|
||||||
Ok((updated_model.into(), Box::pin(submission.into_video_stream())))
|
let video_stream = if self.use_dynamic_api {
|
||||||
|
// 必须显式写出 dyn,否则 rust 会自动推导到 impl 从而认为 if else 返回类型不一致
|
||||||
|
Box::pin(Dynamic::from(submission).into_video_stream()) as Pin<Box<dyn Stream<Item = _> + Send + 'a>>
|
||||||
|
} else {
|
||||||
|
Box::pin(submission.into_video_stream())
|
||||||
|
};
|
||||||
|
Ok((updated_model.into(), video_stream))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -83,9 +83,11 @@ pub struct InsertSubmissionRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Validate)]
|
#[derive(Deserialize, Validate)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
pub struct UpdateVideoSourceRequest {
|
pub struct UpdateVideoSourceRequest {
|
||||||
#[validate(custom(function = "crate::utils::validation::validate_path"))]
|
#[validate(custom(function = "crate::utils::validation::validate_path"))]
|
||||||
pub path: String,
|
pub path: String,
|
||||||
pub enabled: bool,
|
pub enabled: bool,
|
||||||
pub rule: Option<Rule>,
|
pub rule: Option<Rule>,
|
||||||
|
pub use_dynamic_api: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -179,6 +179,8 @@ pub struct VideoSourceDetail {
|
|||||||
pub rule: Option<Rule>,
|
pub rule: Option<Rule>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub rule_display: Option<String>,
|
pub rule_display: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub use_dynamic_api: Option<bool>,
|
||||||
pub enabled: bool,
|
pub enabled: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -111,7 +111,8 @@ pub async fn get_video_sources_details(
|
|||||||
submission::Column::Id,
|
submission::Column::Id,
|
||||||
submission::Column::Path,
|
submission::Column::Path,
|
||||||
submission::Column::Enabled,
|
submission::Column::Enabled,
|
||||||
submission::Column::Rule
|
submission::Column::Rule,
|
||||||
|
submission::Column::UseDynamicApi
|
||||||
])
|
])
|
||||||
.into_model::<VideoSourceDetail>()
|
.into_model::<VideoSourceDetail>()
|
||||||
.all(&db),
|
.all(&db),
|
||||||
@@ -134,6 +135,7 @@ pub async fn get_video_sources_details(
|
|||||||
path: String::new(),
|
path: String::new(),
|
||||||
rule: None,
|
rule: None,
|
||||||
rule_display: None,
|
rule_display: None,
|
||||||
|
use_dynamic_api: None,
|
||||||
enabled: false,
|
enabled: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -179,6 +181,9 @@ pub async fn update_video_source(
|
|||||||
active_model.path = Set(request.path);
|
active_model.path = Set(request.path);
|
||||||
active_model.enabled = Set(request.enabled);
|
active_model.enabled = Set(request.enabled);
|
||||||
active_model.rule = Set(request.rule);
|
active_model.rule = Set(request.rule);
|
||||||
|
if let Some(use_dynamic_api) = request.use_dynamic_api {
|
||||||
|
active_model.use_dynamic_api = Set(use_dynamic_api);
|
||||||
|
}
|
||||||
_ActiveModel::Submission(active_model)
|
_ActiveModel::Submission(active_model)
|
||||||
}),
|
}),
|
||||||
"watch_later" => match watch_later::Entity::find_by_id(id).one(&db).await? {
|
"watch_later" => match watch_later::Entity::find_by_id(id).one(&db).await? {
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
use anyhow::{Context, Result, anyhow};
|
||||||
|
use async_stream::try_stream;
|
||||||
|
use chrono::serde::ts_seconds;
|
||||||
|
use chrono::{DateTime, Utc};
|
||||||
|
use futures::Stream;
|
||||||
|
use reqwest::Method;
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use crate::bilibili::credential::encoded_query;
|
||||||
|
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
|
||||||
|
|
||||||
|
pub struct Dynamic<'a> {
|
||||||
|
client: &'a BiliClient,
|
||||||
|
pub upper_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, serde::Deserialize)]
|
||||||
|
pub struct DynamicItemPublished {
|
||||||
|
#[serde(with = "ts_seconds")]
|
||||||
|
pub_ts: DateTime<Utc>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'a> Dynamic<'a> {
|
||||||
|
pub fn new(client: &'a BiliClient, upper_id: String) -> Self {
|
||||||
|
Self { client, upper_id }
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn get_dynamics(&self, offset: Option<String>) -> Result<Value> {
|
||||||
|
self.client
|
||||||
|
.request(
|
||||||
|
Method::GET,
|
||||||
|
"https://api.bilibili.com/x/polymer/web-dynamic/v1/feed/all",
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.query(&encoded_query(
|
||||||
|
vec![
|
||||||
|
("host_mid", self.upper_id.as_str()),
|
||||||
|
("offset", offset.as_deref().unwrap_or("")),
|
||||||
|
],
|
||||||
|
MIXIN_KEY.load().as_deref(),
|
||||||
|
))
|
||||||
|
.send()
|
||||||
|
.await?
|
||||||
|
.error_for_status()?
|
||||||
|
.json::<serde_json::Value>()
|
||||||
|
.await?
|
||||||
|
.validate()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn into_video_stream(self) -> impl Stream<Item = Result<VideoInfo>> + 'a {
|
||||||
|
try_stream! {
|
||||||
|
let mut offset = None;
|
||||||
|
loop {
|
||||||
|
let mut res = self
|
||||||
|
.get_dynamics(offset.take())
|
||||||
|
.await
|
||||||
|
.with_context(|| "failed to get dynamics")?;
|
||||||
|
let items = res["data"]["items"].as_array_mut().context("items not exist")?;
|
||||||
|
for item in items.iter_mut() {
|
||||||
|
if item["type"].as_str().is_none_or(|t| t != "DYNAMIC_TYPE_AV") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let published: DynamicItemPublished = serde_json::from_value(item["modules"]["module_author"].take())
|
||||||
|
.with_context(|| "failed to parse published time")?;
|
||||||
|
let mut video_info: VideoInfo =
|
||||||
|
serde_json::from_value(item["modules"]["module_dynamic"]["major"]["archive"].take())?;
|
||||||
|
// 这些地方不使用 let else 是因为 try_stream! 宏不支持
|
||||||
|
if let VideoInfo::Dynamic { ref mut pubtime, .. } = video_info {
|
||||||
|
*pubtime = published.pub_ts;
|
||||||
|
yield video_info;
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("video info is not dynamic"))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let (Some(has_more), Some(new_offset)) =
|
||||||
|
(res["data"]["has_more"].as_bool(), res["data"]["offset"].as_str())
|
||||||
|
{
|
||||||
|
if !has_more {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
offset = Some(new_offset.to_string());
|
||||||
|
} else {
|
||||||
|
Err(anyhow!("no has_more or offset found"))?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,6 +9,7 @@ pub use client::{BiliClient, Client};
|
|||||||
pub use collection::{Collection, CollectionItem, CollectionType};
|
pub use collection::{Collection, CollectionItem, CollectionType};
|
||||||
pub use credential::Credential;
|
pub use credential::Credential;
|
||||||
pub use danmaku::DanmakuOption;
|
pub use danmaku::DanmakuOption;
|
||||||
|
pub use dynamic::Dynamic;
|
||||||
pub use error::BiliError;
|
pub use error::BiliError;
|
||||||
pub use favorite_list::FavoriteList;
|
pub use favorite_list::FavoriteList;
|
||||||
use favorite_list::Upper;
|
use favorite_list::Upper;
|
||||||
@@ -23,6 +24,7 @@ mod client;
|
|||||||
mod collection;
|
mod collection;
|
||||||
mod credential;
|
mod credential;
|
||||||
mod danmaku;
|
mod danmaku;
|
||||||
|
mod dynamic;
|
||||||
mod error;
|
mod error;
|
||||||
mod favorite_list;
|
mod favorite_list;
|
||||||
mod me;
|
mod me;
|
||||||
@@ -135,18 +137,32 @@ pub enum VideoInfo {
|
|||||||
#[serde(rename = "created", with = "ts_seconds")]
|
#[serde(rename = "created", with = "ts_seconds")]
|
||||||
ctime: DateTime<Utc>,
|
ctime: DateTime<Utc>,
|
||||||
},
|
},
|
||||||
|
// 从动态获取的视频信息(此处 pubtime 未在结构中,因此使用 default + 手动赋值)
|
||||||
|
Dynamic {
|
||||||
|
title: String,
|
||||||
|
bvid: String,
|
||||||
|
desc: String,
|
||||||
|
cover: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pubtime: DateTime<Utc>,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
use futures::StreamExt;
|
use futures::StreamExt;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use crate::config::VersionedConfig;
|
||||||
|
use crate::database::setup_database;
|
||||||
use crate::utils::init_logger;
|
use crate::utils::init_logger;
|
||||||
|
|
||||||
#[ignore = "only for manual test"]
|
#[ignore = "only for manual test"]
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_video_info_type() {
|
async fn test_video_info_type() -> Result<()> {
|
||||||
|
VersionedConfig::init_for_test(&setup_database(Path::new("./test.sqlite")).await?).await?;
|
||||||
init_logger("None,bili_sync=debug", None);
|
init_logger("None,bili_sync=debug", None);
|
||||||
let bili_client = BiliClient::new();
|
let bili_client = BiliClient::new();
|
||||||
// 请求 UP 主视频必须要获取 mixin key,使用 key 计算请求参数的签名,否则直接提示权限不足返回空
|
// 请求 UP 主视频必须要获取 mixin key,使用 key 计算请求参数的签名,否则直接提示权限不足返回空
|
||||||
@@ -200,6 +216,17 @@ mod tests {
|
|||||||
.await;
|
.await;
|
||||||
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Submission { .. })));
|
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Submission { .. })));
|
||||||
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
assert!(videos.iter().rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||||
|
// 测试动态
|
||||||
|
let dynamic = Dynamic::new(&bili_client, "659898".to_string());
|
||||||
|
let videos = dynamic
|
||||||
|
.into_video_stream()
|
||||||
|
.take(20)
|
||||||
|
.filter_map(|v| futures::future::ready(v.ok()))
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.await;
|
||||||
|
assert!(videos.iter().all(|v| matches!(v, VideoInfo::Dynamic { .. })));
|
||||||
|
assert!(videos.iter().skip(1).rev().is_sorted_by_key(|v| v.release_datetime()));
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[ignore = "only for manual test"]
|
#[ignore = "only for manual test"]
|
||||||
|
|||||||
@@ -6,12 +6,18 @@ use serde_json::Value;
|
|||||||
|
|
||||||
use crate::bilibili::credential::encoded_query;
|
use crate::bilibili::credential::encoded_query;
|
||||||
use crate::bilibili::favorite_list::Upper;
|
use crate::bilibili::favorite_list::Upper;
|
||||||
use crate::bilibili::{BiliClient, MIXIN_KEY, Validate, VideoInfo};
|
use crate::bilibili::{BiliClient, Dynamic, MIXIN_KEY, Validate, VideoInfo};
|
||||||
pub struct Submission<'a> {
|
pub struct Submission<'a> {
|
||||||
client: &'a BiliClient,
|
client: &'a BiliClient,
|
||||||
pub upper_id: String,
|
pub upper_id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl<'a> From<Submission<'a>> for Dynamic<'a> {
|
||||||
|
fn from(submission: Submission<'a>) -> Self {
|
||||||
|
Dynamic::new(submission.client, submission.upper_id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl<'a> Submission<'a> {
|
impl<'a> Submission<'a> {
|
||||||
pub fn new(client: &'a BiliClient, upper_id: String) -> Self {
|
pub fn new(client: &'a BiliClient, upper_id: String) -> Self {
|
||||||
Self { client, upper_id }
|
Self { client, upper_id }
|
||||||
|
|||||||
@@ -53,11 +53,16 @@ impl VersionedConfig {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
/// 单元测试直接使用测试专用的配置即可
|
/// 仅在测试环境使用,该方法会尝试从测试数据库中加载配置并写入到全局的 VERSIONED_CONFIG
|
||||||
pub fn get() -> &'static VersionedConfig {
|
pub async fn init_for_test(connection: &DatabaseConnection) -> Result<()> {
|
||||||
use std::sync::LazyLock;
|
let Some(Ok(config)) = Config::load_from_database(&connection).await? else {
|
||||||
static TEST_CONFIG: LazyLock<VersionedConfig> = LazyLock::new(|| VersionedConfig::new(Config::test_default()));
|
bail!("no config found in test database");
|
||||||
return &TEST_CONFIG;
|
};
|
||||||
|
let versioned_config = VersionedConfig::new(config);
|
||||||
|
VERSIONED_CONFIG
|
||||||
|
.set(versioned_config)
|
||||||
|
.map_err(|e| anyhow!("VERSIONED_CONFIG has already been initialized: {}", e))?;
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(test))]
|
#[cfg(not(test))]
|
||||||
@@ -66,6 +71,16 @@ impl VersionedConfig {
|
|||||||
VERSIONED_CONFIG.get().expect("VERSIONED_CONFIG is not initialized")
|
VERSIONED_CONFIG.get().expect("VERSIONED_CONFIG is not initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
/// 尝试获取全局的 `VersionedConfig`,如果未初始化则退回测试环境的默认配置
|
||||||
|
pub fn get() -> &'static VersionedConfig {
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
static FALLBACK_CONFIG: LazyLock<VersionedConfig> =
|
||||||
|
LazyLock::new(|| VersionedConfig::new(Config::test_default()));
|
||||||
|
// 优先从全局变量获取,未初始化则返回测试环境的默认配置
|
||||||
|
return VERSIONED_CONFIG.get().unwrap_or_else(|| &FALLBACK_CONFIG);
|
||||||
|
}
|
||||||
|
|
||||||
pub fn new(config: Config) -> Self {
|
pub fn new(config: Config) -> Self {
|
||||||
Self {
|
Self {
|
||||||
inner: ArcSwap::from_pointee(config),
|
inner: ArcSwap::from_pointee(config),
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::path::Path;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
@@ -6,14 +7,12 @@ use sea_orm::sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqliteSynch
|
|||||||
use sea_orm::sqlx::{ConnectOptions as SqlxConnectOptions, Sqlite};
|
use sea_orm::sqlx::{ConnectOptions as SqlxConnectOptions, Sqlite};
|
||||||
use sea_orm::{ConnectOptions, Database, DatabaseConnection, SqlxSqliteConnector};
|
use sea_orm::{ConnectOptions, Database, DatabaseConnection, SqlxSqliteConnector};
|
||||||
|
|
||||||
use crate::config::CONFIG_DIR;
|
fn database_url(path: &Path) -> String {
|
||||||
|
format!("sqlite://{}?mode=rwc", path.to_string_lossy())
|
||||||
fn database_url() -> String {
|
|
||||||
format!("sqlite://{}?mode=rwc", CONFIG_DIR.join("data.sqlite").to_string_lossy())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn database_connection() -> Result<DatabaseConnection> {
|
async fn database_connection(database_url: &str) -> Result<DatabaseConnection> {
|
||||||
let mut option = ConnectOptions::new(database_url());
|
let mut option = ConnectOptions::new(database_url);
|
||||||
option
|
option
|
||||||
.max_connections(50)
|
.max_connections(50)
|
||||||
.min_connections(5)
|
.min_connections(5)
|
||||||
@@ -35,18 +34,25 @@ async fn database_connection() -> Result<DatabaseConnection> {
|
|||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn migrate_database() -> Result<()> {
|
async fn migrate_database(database_url: &str) -> Result<()> {
|
||||||
// 注意此处使用内部构造的 DatabaseConnection,而不是通过 database_connection() 获取
|
// 注意此处使用内部构造的 DatabaseConnection,而不是通过 database_connection() 获取
|
||||||
// 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题,而使用默认的连接选项不会
|
// 这是因为使用多个连接的 Connection 会导致奇怪的迁移顺序问题,而使用默认的连接选项不会
|
||||||
let connection = Database::connect(database_url()).await?;
|
let connection = Database::connect(database_url).await?;
|
||||||
Ok(Migrator::up(&connection, None).await?)
|
Ok(Migrator::up(&connection, None).await?)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// 进行数据库迁移并获取数据库连接,供外部使用
|
/// 进行数据库迁移并获取数据库连接,供外部使用
|
||||||
pub async fn setup_database() -> Result<DatabaseConnection> {
|
pub async fn setup_database(path: &Path) -> Result<DatabaseConnection> {
|
||||||
tokio::fs::create_dir_all(CONFIG_DIR.as_path()).await.context(
|
if let Some(parent) = path.parent() {
|
||||||
"Failed to create config directory. Please check if you have granted necessary permissions to your folder.",
|
tokio::fs::create_dir_all(parent).await.context(
|
||||||
)?;
|
"Failed to create config directory. Please check if you have granted necessary permissions to your folder.",
|
||||||
migrate_database().await.context("Failed to migrate database")?;
|
)?;
|
||||||
database_connection().await.context("Failed to connect to database")
|
}
|
||||||
|
let database_url = database_url(path);
|
||||||
|
migrate_database(&database_url)
|
||||||
|
.await
|
||||||
|
.context("Failed to migrate database")?;
|
||||||
|
database_connection(&database_url)
|
||||||
|
.await
|
||||||
|
.context("Failed to connect to database")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use tokio_util::task::TaskTracker;
|
use tokio_util::task::TaskTracker;
|
||||||
|
|
||||||
use crate::api::{LogHelper, MAX_HISTORY_LOGS};
|
use crate::api::{LogHelper, MAX_HISTORY_LOGS};
|
||||||
use crate::config::{ARGS, VersionedConfig};
|
use crate::config::{ARGS, CONFIG_DIR, VersionedConfig};
|
||||||
use crate::database::setup_database;
|
use crate::database::setup_database;
|
||||||
use crate::utils::init_logger;
|
use crate::utils::init_logger;
|
||||||
use crate::utils::signal::terminate;
|
use crate::utils::signal::terminate;
|
||||||
@@ -85,7 +85,9 @@ async fn init() -> (DatabaseConnection, LogHelper) {
|
|||||||
init_logger(&ARGS.log_level, Some(log_writer.clone()));
|
init_logger(&ARGS.log_level, Some(log_writer.clone()));
|
||||||
info!("欢迎使用 Bili-Sync,当前程序版本:{}", config::version());
|
info!("欢迎使用 Bili-Sync,当前程序版本:{}", config::version());
|
||||||
info!("项目地址:https://github.com/amtoaer/bili-sync");
|
info!("项目地址:https://github.com/amtoaer/bili-sync");
|
||||||
let connection = setup_database().await.expect("数据库初始化失败");
|
let connection = setup_database(&CONFIG_DIR.join("data.sqlite"))
|
||||||
|
.await
|
||||||
|
.expect("数据库初始化失败");
|
||||||
info!("数据库初始化完成");
|
info!("数据库初始化完成");
|
||||||
VersionedConfig::init(&connection).await.expect("配置初始化失败");
|
VersionedConfig::init(&connection).await.expect("配置初始化失败");
|
||||||
info!("配置初始化完成");
|
info!("配置初始化完成");
|
||||||
|
|||||||
@@ -97,7 +97,23 @@ impl VideoInfo {
|
|||||||
valid: Set(true),
|
valid: Set(true),
|
||||||
..default
|
..default
|
||||||
},
|
},
|
||||||
_ => unreachable!(),
|
VideoInfo::Dynamic {
|
||||||
|
title,
|
||||||
|
bvid,
|
||||||
|
desc,
|
||||||
|
cover,
|
||||||
|
pubtime,
|
||||||
|
} => bili_sync_entity::video::ActiveModel {
|
||||||
|
bvid: Set(bvid),
|
||||||
|
name: Set(title),
|
||||||
|
intro: Set(desc),
|
||||||
|
cover: Set(cover),
|
||||||
|
pubtime: Set(pubtime.naive_utc()),
|
||||||
|
category: Set(2), // 动态里的视频内容类型肯定是视频
|
||||||
|
valid: Set(true),
|
||||||
|
..default
|
||||||
|
},
|
||||||
|
VideoInfo::Detail { .. } => unreachable!(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,8 +161,9 @@ impl VideoInfo {
|
|||||||
VideoInfo::Collection { pubtime: time, .. }
|
VideoInfo::Collection { pubtime: time, .. }
|
||||||
| VideoInfo::Favorite { fav_time: time, .. }
|
| VideoInfo::Favorite { fav_time: time, .. }
|
||||||
| VideoInfo::WatchLater { fav_time: time, .. }
|
| VideoInfo::WatchLater { fav_time: time, .. }
|
||||||
| VideoInfo::Submission { ctime: time, .. } => time,
|
| VideoInfo::Submission { ctime: time, .. }
|
||||||
_ => unreachable!(),
|
| VideoInfo::Dynamic { pubtime: time, .. } => time,
|
||||||
|
VideoInfo::Detail { .. } => unreachable!(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -58,7 +58,8 @@ pub async fn refresh_video_source<'a>(
|
|||||||
let mut max_datetime = latest_row_at;
|
let mut max_datetime = latest_row_at;
|
||||||
let mut error = Ok(());
|
let mut error = Ok(());
|
||||||
let mut video_streams = video_streams
|
let mut video_streams = video_streams
|
||||||
.take_while(|res| {
|
.enumerate()
|
||||||
|
.take_while(|(idx, res)| {
|
||||||
match res {
|
match res {
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error = Err(anyhow!(e.to_string()));
|
error = Err(anyhow!(e.to_string()));
|
||||||
@@ -72,11 +73,11 @@ pub async fn refresh_video_source<'a>(
|
|||||||
if release_datetime > &max_datetime {
|
if release_datetime > &max_datetime {
|
||||||
max_datetime = *release_datetime;
|
max_datetime = *release_datetime;
|
||||||
}
|
}
|
||||||
futures::future::ready(video_source.should_take(release_datetime, &latest_row_at))
|
futures::future::ready(video_source.should_take(*idx, release_datetime, &latest_row_at))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.filter_map(|res| futures::future::ready(video_source.should_filter(res, &latest_row_at)))
|
.filter_map(|(idx, res)| futures::future::ready(video_source.should_filter(idx, res, &latest_row_at)))
|
||||||
.chunks(10);
|
.chunks(10);
|
||||||
let mut count = 0;
|
let mut count = 0;
|
||||||
while let Some(videos_info) = video_streams.next().await {
|
while let Some(videos_info) = video_streams.next().await {
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ pub struct Model {
|
|||||||
pub upper_name: String,
|
pub upper_name: String,
|
||||||
pub path: String,
|
pub path: String,
|
||||||
pub created_at: String,
|
pub created_at: String,
|
||||||
|
pub use_dynamic_api: bool,
|
||||||
pub latest_row_at: DateTime,
|
pub latest_row_at: DateTime,
|
||||||
pub rule: Option<Rule>,
|
pub rule: Option<Rule>,
|
||||||
pub enabled: bool,
|
pub enabled: bool,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ mod m20250612_090826_add_enabled;
|
|||||||
mod m20250613_043257_add_config;
|
mod m20250613_043257_add_config;
|
||||||
mod m20250712_080013_add_video_created_at_index;
|
mod m20250712_080013_add_video_created_at_index;
|
||||||
mod m20250903_094454_add_rule_and_should_download;
|
mod m20250903_094454_add_rule_and_should_download;
|
||||||
|
mod m20251009_123713_add_use_dynamic_api;
|
||||||
|
|
||||||
pub struct Migrator;
|
pub struct Migrator;
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ impl MigratorTrait for Migrator {
|
|||||||
Box::new(m20250613_043257_add_config::Migration),
|
Box::new(m20250613_043257_add_config::Migration),
|
||||||
Box::new(m20250712_080013_add_video_created_at_index::Migration),
|
Box::new(m20250712_080013_add_video_created_at_index::Migration),
|
||||||
Box::new(m20250903_094454_add_rule_and_should_download::Migration),
|
Box::new(m20250903_094454_add_rule_and_should_download::Migration),
|
||||||
|
Box::new(m20251009_123713_add_use_dynamic_api::Migration),
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
use sea_orm_migration::prelude::*;
|
||||||
|
use sea_orm_migration::schema::*;
|
||||||
|
|
||||||
|
#[derive(DeriveMigrationName)]
|
||||||
|
pub struct Migration;
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl MigrationTrait for Migration {
|
||||||
|
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
manager
|
||||||
|
.alter_table(
|
||||||
|
Table::alter()
|
||||||
|
.table(Submission::Table)
|
||||||
|
.add_column(boolean(Submission::UseDynamicApi).default(false))
|
||||||
|
.to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||||
|
manager
|
||||||
|
.alter_table(
|
||||||
|
Table::alter()
|
||||||
|
.table(Submission::Table)
|
||||||
|
.drop_column(Submission::UseDynamicApi)
|
||||||
|
.to_owned(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(DeriveIden)]
|
||||||
|
enum Submission {
|
||||||
|
Table,
|
||||||
|
UseDynamicApi,
|
||||||
|
}
|
||||||
@@ -188,8 +188,9 @@ export interface VideoSourceDetail {
|
|||||||
id: number;
|
id: number;
|
||||||
name: string;
|
name: string;
|
||||||
path: string;
|
path: string;
|
||||||
rule?: Rule | null;
|
rule: Rule | null;
|
||||||
ruleDisplay?: string | null;
|
ruleDisplay: string | null;
|
||||||
|
useDynamicApi: boolean | null;
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -206,6 +207,7 @@ export interface UpdateVideoSourceRequest {
|
|||||||
path: string;
|
path: string;
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
rule?: Rule | null;
|
rule?: Rule | null;
|
||||||
|
useDynamicApi?: boolean | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 配置相关类型
|
// 配置相关类型
|
||||||
@@ -315,5 +317,5 @@ export interface TaskStatus {
|
|||||||
}
|
}
|
||||||
|
|
||||||
export interface UpdateVideoSourceResponse {
|
export interface UpdateVideoSourceResponse {
|
||||||
ruleDisplay?: string;
|
ruleDisplay: string;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@
|
|||||||
import UserIcon from '@lucide/svelte/icons/user';
|
import UserIcon from '@lucide/svelte/icons/user';
|
||||||
import ClockIcon from '@lucide/svelte/icons/clock';
|
import ClockIcon from '@lucide/svelte/icons/clock';
|
||||||
import PlusIcon from '@lucide/svelte/icons/plus';
|
import PlusIcon from '@lucide/svelte/icons/plus';
|
||||||
|
import InfoIcon from '@lucide/svelte/icons/info';
|
||||||
import * as Tooltip from '$lib/components/ui/tooltip/index.js';
|
import * as Tooltip from '$lib/components/ui/tooltip/index.js';
|
||||||
import { toast } from 'svelte-sonner';
|
import { toast } from 'svelte-sonner';
|
||||||
import { setBreadcrumb } from '$lib/stores/breadcrumb';
|
import { setBreadcrumb } from '$lib/stores/breadcrumb';
|
||||||
@@ -48,7 +49,8 @@
|
|||||||
let editForm = {
|
let editForm = {
|
||||||
path: '',
|
path: '',
|
||||||
enabled: false,
|
enabled: false,
|
||||||
rule: null as Rule | null
|
rule: null as Rule | null,
|
||||||
|
useDynamicApi: null as boolean | null
|
||||||
};
|
};
|
||||||
|
|
||||||
// 表单数据
|
// 表单数据
|
||||||
@@ -86,7 +88,8 @@
|
|||||||
editForm = {
|
editForm = {
|
||||||
path: source.path,
|
path: source.path,
|
||||||
enabled: source.enabled,
|
enabled: source.enabled,
|
||||||
rule: source.rule || null
|
useDynamicApi: source.useDynamicApi,
|
||||||
|
rule: source.rule
|
||||||
};
|
};
|
||||||
showEditDialog = true;
|
showEditDialog = true;
|
||||||
}
|
}
|
||||||
@@ -110,7 +113,8 @@
|
|||||||
let response = await api.updateVideoSource(editingType, editingSource.id, {
|
let response = await api.updateVideoSource(editingType, editingSource.id, {
|
||||||
path: editForm.path,
|
path: editForm.path,
|
||||||
enabled: editForm.enabled,
|
enabled: editForm.enabled,
|
||||||
rule: editForm.rule
|
rule: editForm.rule,
|
||||||
|
useDynamicApi: editForm.useDynamicApi
|
||||||
});
|
});
|
||||||
// 更新本地数据
|
// 更新本地数据
|
||||||
if (videoSourcesData && editingSource) {
|
if (videoSourcesData && editingSource) {
|
||||||
@@ -122,6 +126,7 @@
|
|||||||
path: editForm.path,
|
path: editForm.path,
|
||||||
enabled: editForm.enabled,
|
enabled: editForm.enabled,
|
||||||
rule: editForm.rule,
|
rule: editForm.rule,
|
||||||
|
useDynamicApi: editForm.useDynamicApi,
|
||||||
ruleDisplay: response.data.ruleDisplay
|
ruleDisplay: response.data.ruleDisplay
|
||||||
};
|
};
|
||||||
videoSourcesData = { ...videoSourcesData };
|
videoSourcesData = { ...videoSourcesData };
|
||||||
@@ -266,9 +271,12 @@
|
|||||||
<Table.Header>
|
<Table.Header>
|
||||||
<Table.Row>
|
<Table.Row>
|
||||||
<Table.Head class="w-[20%]">名称</Table.Head>
|
<Table.Head class="w-[20%]">名称</Table.Head>
|
||||||
<Table.Head class="w-[40%]">下载路径</Table.Head>
|
<Table.Head class="w-[30%]">下载路径</Table.Head>
|
||||||
<Table.Head class="w-[15%]">过滤规则</Table.Head>
|
<Table.Head class="w-[15%]">过滤规则</Table.Head>
|
||||||
<Table.Head class="w-[15%]">状态</Table.Head>
|
<Table.Head class="w-[10%]">启用状态</Table.Head>
|
||||||
|
{#if key === 'submissions'}
|
||||||
|
<Table.Head class="w-[10%]">使用动态 API</Table.Head>
|
||||||
|
{/if}
|
||||||
<Table.Head class="w-[10%] text-right">操作</Table.Head>
|
<Table.Head class="w-[10%] text-right">操作</Table.Head>
|
||||||
</Table.Row>
|
</Table.Row>
|
||||||
</Table.Header>
|
</Table.Header>
|
||||||
@@ -304,11 +312,17 @@
|
|||||||
<Table.Cell>
|
<Table.Cell>
|
||||||
<div class="flex h-8 items-center gap-2">
|
<div class="flex h-8 items-center gap-2">
|
||||||
<Switch checked={source.enabled} disabled />
|
<Switch checked={source.enabled} disabled />
|
||||||
<span class="text-muted-foreground text-sm whitespace-nowrap">
|
|
||||||
{source.enabled ? '已启用' : '已禁用'}
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
</Table.Cell>
|
</Table.Cell>
|
||||||
|
{#if key === 'submissions'}
|
||||||
|
<Table.Cell>
|
||||||
|
<div class="flex h-8 items-center gap-2">
|
||||||
|
{#if source.useDynamicApi !== null}
|
||||||
|
<Switch checked={source.useDynamicApi} disabled />
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
</Table.Cell>
|
||||||
|
{/if}
|
||||||
<Table.Cell class="text-right">
|
<Table.Cell class="text-right">
|
||||||
<Button
|
<Button
|
||||||
size="sm"
|
size="sm"
|
||||||
@@ -395,6 +409,28 @@
|
|||||||
<Label class="text-sm font-medium">启用此视频源</Label>
|
<Label class="text-sm font-medium">启用此视频源</Label>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{#if editingType === 'submissions' && editForm.useDynamicApi !== null}
|
||||||
|
<div class="flex items-center space-x-2">
|
||||||
|
<Switch bind:checked={editForm.useDynamicApi} />
|
||||||
|
<div class="flex items-center gap-1">
|
||||||
|
<Label class="text-sm font-medium">使用动态 API 获取视频</Label>
|
||||||
|
<Tooltip.Root>
|
||||||
|
<Tooltip.Trigger>
|
||||||
|
<InfoIcon class="text-muted-foreground h-3.5 w-3.5" />
|
||||||
|
</Tooltip.Trigger>
|
||||||
|
<Tooltip.Content>
|
||||||
|
<p class="text-xs">
|
||||||
|
只有使用动态 API
|
||||||
|
才能拉取到动态视频,但该接口不提供筛选参数,需要拉取全部类型的动态后在本地筛选出视频。<br
|
||||||
|
/>这在扫描时会获取到较多无效数据并增加请求次数,可根据实际情况酌情选择,推荐仅在
|
||||||
|
UP 主有较多动态视频时开启。
|
||||||
|
</p>
|
||||||
|
</Tooltip.Content>
|
||||||
|
</Tooltip.Root>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
<!-- 规则编辑器 -->
|
<!-- 规则编辑器 -->
|
||||||
<div>
|
<div>
|
||||||
<RuleEditor rule={editForm.rule} onRuleChange={(rule) => (editForm.rule = rule)} />
|
<RuleEditor rule={editForm.rule} onRuleChange={(rule) => (editForm.rule = rule)} />
|
||||||
|
|||||||
Reference in New Issue
Block a user