Compare commits

...

17 Commits
1.0.7 ... 1.1.2

Author SHA1 Message Date
amtoaer
961913c4fb doc: 加入字幕相关文档 2023-12-07 22:11:37 +08:00
amtoaer
fa20e5efee feat: 开放弹幕的各项设置 2023-12-07 21:45:18 +08:00
amtoaer
38fb0a4560 fix: 安全地移除配置项 2023-12-07 21:29:57 +08:00
amtoaer
9e94e3b73e chore: try except 按块分割,移除无用的设置项 2023-12-07 21:15:40 +08:00
amtoaer
b955a9fe45 chore: 替换掉被标记 deprecated 的方法 2023-12-06 18:17:17 +08:00
amtoaer
9d151b4731 feat: 命令默认不覆盖现有内容,更新文档 2023-12-06 01:19:08 +08:00
amtoaer
1686c1a8df feat: 支持弹幕下载 2023-12-06 00:39:46 +08:00
amtoaer
de6eaeb4a6 chore: 整理代码逻辑,留出下载字幕的入口 2023-12-06 00:00:42 +08:00
amtoaer
46d1810e7c chore: 处理成功的日志往外提一级 2023-12-04 01:50:31 +08:00
amtoaer
89e2567fef feat: tag 获取失败不影响主流程 2023-12-04 01:35:29 +08:00
amtoaer
38caf1f0d6 fix: 修复运行错误 2023-12-04 01:02:04 +08:00
amtoaer
6877171f4d fix: 修复参数错误 2023-12-04 00:54:04 +08:00
amtoaer
29d06a040b fix: 注册命令 2023-12-04 00:50:22 +08:00
amtoaer
ceec5d6780 feat: 尝试支持视频标签 2023-12-04 00:39:42 +08:00
amtoaer
650498d4a1 fix: 每天应仅检查一次 credential 2023-12-03 23:14:15 +08:00
amtoaer
96ff84391d doc: 修复图片路径过长的问题 2023-12-02 01:29:33 +08:00
amtoaer
44e8a2c97d doc: 加入对额外命令的描述,添加图片 2023-12-02 01:26:55 +08:00
12 changed files with 419 additions and 164 deletions

2
.gitignore vendored
View File

@@ -4,7 +4,7 @@ debug.py
videos
config.test.json
database.test.db*
example.json
example*.json
thumbs.test
config
data

View File

@@ -1,4 +1,4 @@
.PHONY: install fmt start-daemon start-once
.PHONY: install fmt start-daemon start-once db-init db-migrate db-upgrade sync-conf
install:
@echo "Installing dependencies..."
@@ -22,4 +22,10 @@ db-migrate:
@poetry run aerich migrate
db-upgrade:
@poetry run aerich upgrade
@poetry run aerich upgrade
sync-conf:
@echo "Syncing config..."
@cp ${CONFIG_SRC} ./config/
@cp ${DB_SRC} ./data/
@echo "Done."

View File

@@ -1,4 +1,6 @@
# bili-sync
![bili-sync](https://socialify.git.ci/amtoaer/bili-sync/image?description=1&font=KoHo&issues=1&language=1&logo=https%3A%2F%2Fs2.loli.net%2F2023%2F12%2F02%2F9EwT2yInOu1d3zm.png&name=1&owner=1&pattern=Signal&pulls=1&stargazers=1&theme=Light)
## 简介
为 NAS 用户编写的 BILIBILI 收藏夹同步工具,可方便导入 EMBY 等媒体库工具浏览。
@@ -20,6 +22,14 @@
对于配置文件的前五项,请参考[凭据获取流程](https://nemo2011.github.io/bilibili-api/#/get-credential)。
```python
@dataclass
class SubtitleConfig(DataClassJsonMixin):
font_name: str = "微软雅黑,黑体" # 字体
font_size: float = 40 # 字号
alpha: float = 0.8 # 透明度
fly_time: float = 5 # 滚动弹幕持续时间
static_time: float = 10 # 静态弹幕持续时间
class Config(DataClassJsonMixin):
sessdata: str = ""
bili_jct: str = ""
@@ -27,8 +37,8 @@ class Config(DataClassJsonMixin):
dedeuserid: str = ""
ac_time_value: str = ""
interval: int = 20 # 任务执行的间隔时间
favorite_ids: list[int] = field(default_factory=list) # 收藏夹的 id
path_mapper: dict[int, str] = field(default_factory=dict) # 收藏夹的 id 到存储目录的映射
subtitle: SubtitleConfig = field(default_factory=SubtitleConfig) # 字幕相关设置
```
程序默认会将配置文件存储于 `${程序路径}/config/config.json`,数据库文件存储于 `${程序路径}/data/data.db`,如果发现不存在则新建并写入初始配置。
@@ -73,15 +83,47 @@ services:
"dedeuserid": "xxxxxxxxxxxxxxxxxx",
"ac_time_value": "xxxxxxxxxxxxxxxxxx",
"interval": 20,
"favorite_ids": [
711322958
],
"path_mapper": {
"711322958": "/Videos/Bilibilis/Bilibili-711322958/"
},
"subtitle": {
"font_name": "微软雅黑,黑体",
"font_size": 40.0,
"alpha": 0.8,
"fly_time": 5.0,
"static_time": 10.0
}
}
```
## 支持的额外命令
为满足需要,该应用包含几个单独的命令,可在程序目录下使用 `python entry.py ${command name}` 运行。
1. `once`
处理收藏夹,和一般定时任务触发时执行的操作完全相同,但仅运行一次。
2. `recheck`
将本地不存在的视频文件标记成未下载,下次定时任务触发时将一并下载。
3. `refresh_refresh_poster`
更新本地视频的封面。
3. `refresh_upper`
更新本地up的头像和元数据。
3. `refresh_nfo`
更新本地视频的元数据。(如标签、标题等信息)
3. `refresh_video`
更新本地的视频源文件。
3. `refresh_subtitle`
更新本地的弹幕文件。
**对于以 refresh 开头的命令,均支持 --force 参数,如果有 --force 参数,将全量覆盖对应内容,否则默认仅更新缺失的部分。**
## 路线图
- [x] 凭证认证

View File

@@ -1,11 +1,14 @@
import asyncio
import functools
from pathlib import Path
from typing import Callable
from loguru import logger
from constants import MediaStatus, MediaType
from models import FavoriteItem, Upper
from processor import download_content
from utils import aexists, amakedirs
from models import FavoriteItem
from processor import process_favorite_item
from utils import aexists, aremove
async def recheck():
@@ -37,26 +40,62 @@ async def recheck():
logger.info("Database updated.")
async def upper_thumb():
makedir_tasks = []
other_tasks = []
for upper in await Upper.all():
if all(
await asyncio.gather(
aexists(upper.thumb_path), aexists(upper.meta_path)
)
):
logger.info(
"Upper {} {} already exists, skipped.", upper.mid, upper.name
)
makedir_tasks.append(amakedirs(upper.thumb_path.parent, exist_ok=True))
logger.info("Saving metadata for upper {} {}...", upper.mid, upper.name)
other_tasks.extend(
[
upper.save_metadata(),
download_content(upper.thumb, upper.thumb_path),
]
async def _refresh_favorite_item_info(
path_getter: Callable[[FavoriteItem], list[Path]],
process_poster: bool = False,
process_video: bool = False,
process_nfo: bool = False,
process_upper: bool = False,
process_subtitle: bool = False,
force: bool = False,
):
items = await FavoriteItem.filter(downloaded=True).prefetch_related("upper")
if force:
# 如果强制刷新,那么就先把现存的所有内容删除
await asyncio.gather(
*[aremove(path) for item in items for path in path_getter(item)],
return_exceptions=True,
)
await asyncio.gather(*makedir_tasks)
await asyncio.gather(*other_tasks)
logger.info("All done.")
await asyncio.gather(
*[
process_favorite_item(
item,
process_poster=process_poster,
process_video=process_video,
process_nfo=process_nfo,
process_upper=process_upper,
process_subtitle=process_subtitle,
)
for item in items
],
return_exceptions=True,
)
refresh_nfo = functools.partial(
_refresh_favorite_item_info, lambda item: [item.nfo_path], process_nfo=True
)
refresh_poster = functools.partial(
_refresh_favorite_item_info,
lambda item: [item.poster_path],
process_poster=True,
)
refresh_video = functools.partial(
_refresh_favorite_item_info,
lambda item: [item.video_path],
process_video=True,
)
refresh_upper = functools.partial(
_refresh_favorite_item_info,
lambda item: item.upper_path,
process_upper=True,
)
refresh_subtitle = functools.partial(
_refresh_favorite_item_info,
lambda item: [item.subtitle_path],
process_subtitle=True,
)

View File

@@ -4,7 +4,14 @@ import sys
import uvloop
from loguru import logger
from commands import recheck, upper_thumb
from commands import (
recheck,
refresh_nfo,
refresh_poster,
refresh_subtitle,
refresh_upper,
refresh_video,
)
from models import init_model
from processor import cleanup, process
from settings import settings
@@ -14,14 +21,22 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
async def entry() -> None:
await init_model()
for command, func in [
force = any("force" in _ for _ in sys.argv)
for command, func in (
("once", process),
("recheck", recheck),
("upper_thumb", upper_thumb),
]:
("refresh_poster", refresh_poster),
("refresh_upper", refresh_upper),
("refresh_nfo", refresh_nfo),
("refresh_video", refresh_video),
("refresh_subtitle", refresh_subtitle),
):
if any(command in _ for _ in sys.argv):
logger.info("Running {}...", command)
await func()
if command.startswith("refresh"):
await func(force=force)
else:
await func()
return
logger.info("Running daemon...")
while True:

View File

@@ -0,0 +1,11 @@
from tortoise import BaseDBAsyncClient
async def upgrade(db: BaseDBAsyncClient) -> str:
return """
ALTER TABLE "favoriteitem" ADD "tags" JSON;"""
async def downgrade(db: BaseDBAsyncClient) -> str:
return """
ALTER TABLE "favoriteitem" DROP COLUMN "tags";"""

View File

@@ -79,6 +79,7 @@ class FavoriteItem(Model):
bvid = fields.CharField(max_length=255)
desc = fields.TextField()
cover = fields.TextField()
tags = fields.JSONField(null=True)
favorite_list = fields.ForeignKeyField(
"models.FavoriteList", related_name="items"
)
@@ -132,6 +133,20 @@ class FavoriteItem(Model):
/ f"{self.bvid}-poster.jpg"
)
@property
def upper_path(self) -> list[Path]:
return [
self.upper.thumb_path,
self.upper.meta_path,
]
@property
def subtitle_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"{self.bvid}.zh-CN.default.ass"
)
async def init_model() -> None:
await Tortoise.init(config=TORTOISE_ORM)

7
nfo.py
View File

@@ -25,6 +25,7 @@ class Actor:
class EpisodeInfo:
title: str
plot: str
tags: list[str]
actor: list[Actor]
bvid: str
aired: datetime.datetime
@@ -35,6 +36,11 @@ class EpisodeInfo:
def to_xml(self) -> str:
actor = "\n".join(_.to_xml() for _ in self.actor)
tags = (
"\n".join(f" <genre>{_}</genre>" for _ in self.tags)
if isinstance(self.tags, list)
else ""
)
return f"""
<?xml version="1.0" encoding="utf-8" standalone="yes"?>
<episodedetails>
@@ -43,6 +49,7 @@ class EpisodeInfo:
<title>{self.title}</title>
{actor}
<year>{self.aired.year}</year>
{tags}
<uniqueid type="bilibili">{self.bvid}</uniqueid>
<aired>{self.aired.strftime("%Y-%m-%d")}</aired>
</episodedetails>

6
poetry.lock generated
View File

@@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
[[package]]
name = "aerich"
@@ -301,7 +301,7 @@ pillow = ">=10.1.0,<10.2.0"
pycryptodomex = ">=3.19.0,<3.20.0"
pyyaml = ">=6.0,<7.0"
qrcode = ">=7.4.2,<7.5.0"
qrcode-terminal = ">=0.8,<1.0"
qrcode_terminal = ">=0.8,<1.0"
requests = ">=2.31.0,<2.32.0"
rsa = ">=4.9,<5.0"
tqdm = ">=4.66.1,<4.67.0"
@@ -311,7 +311,7 @@ yarl = ">=1.9.2,<1.10.0"
type = "git"
url = "https://github.com/amtoaer/bilibili-api.git"
reference = "dev"
resolved_reference = "30db504b410787bdc8a5cd1ff63bacf6b365f393"
resolved_reference = "0281d7c58e8a26706cefc96e7e427cb05a26d866"
[[package]]
name = "black"

View File

@@ -3,10 +3,10 @@ import datetime
from asyncio import Semaphore, create_subprocess_exec
from asyncio.subprocess import DEVNULL
from bilibili_api import favorite_list, video
from bilibili_api import ass, favorite_list, video
from bilibili_api.exceptions import ResponseCodeException
from loguru import logger
from tortoise import Tortoise
from tortoise.connection import connections
from constants import FFMPEG_COMMAND, MediaStatus, MediaType
from credential import credential
@@ -20,7 +20,7 @@ anchor = datetime.date.today()
async def cleanup() -> None:
await client.aclose()
await Tortoise.close_connections()
await connections.close_all()
def concurrent_decorator(concurrency: int) -> callable:
@@ -81,22 +81,17 @@ async def manage_model(medias: list[dict], fav_list: FavoriteList) -> None:
async def process() -> None:
global anchor
if (
today := datetime.date.today()
) > anchor and await credential.check_refresh():
try:
await credential.refresh()
anchor = today
logger.info("Credential refreshed.")
except Exception:
logger.exception("Failed to refresh credential.")
return
for favorite_id in settings.favorite_ids:
if favorite_id not in settings.path_mapper:
logger.warning(
f"Favorite {favorite_id} not in path mapper, ignored."
)
continue
if (today := datetime.date.today()) > anchor:
anchor = today
logger.info("Check credential.")
if await credential.check_refresh():
try:
await credential.refresh()
logger.info("Credential refreshed.")
except Exception:
logger.exception("Failed to refresh credential.")
return
for favorite_id in settings.path_mapper:
await process_favorite(favorite_id)
@@ -149,118 +144,226 @@ async def process_favorite(favorite_id: int) -> None:
downloaded=False,
).prefetch_related("upper")
await asyncio.gather(
*[process_video(item) for item in all_unprocessed_items],
*[process_favorite_item(item) for item in all_unprocessed_items],
return_exceptions=True,
)
logger.info("Favorite {} {} processed successfully.", favorite_id, title)
@concurrent_decorator(4)
async def process_video(fav_item: FavoriteItem) -> None:
async def process_favorite_item(
fav_item: FavoriteItem,
process_poster=True,
process_video=True,
process_nfo=True,
process_upper=True,
process_subtitle=True,
) -> None:
logger.info("Start to process video {} {}", fav_item.bvid, fav_item.name)
if fav_item.type != MediaType.VIDEO:
logger.warning("Media {} is not a video, skipped.", fav_item.name)
return
v = video.Video(fav_item.bvid, credential=credential)
# 如果没有获取过 tags那么尝试获取一下
try:
if await aexists(fav_item.video_path):
fav_item.downloaded = True
await fav_item.save()
logger.info(
"{} {} already exists, skipped.", fav_item.bvid, fav_item.name
)
return
# 写入 up 主头像
if not all(
await asyncio.gather(
aexists(fav_item.upper.thumb_path),
aexists(fav_item.upper.meta_path),
)
):
await amakedirs(fav_item.upper.thumb_path.parent, exist_ok=True)
await fav_item.upper.save_metadata()
await download_content(
fav_item.upper.thumb, fav_item.upper.thumb_path
)
# 写入 nfo
await EpisodeInfo(
title=fav_item.name,
plot=fav_item.desc,
actor=[
Actor(
name=fav_item.upper.mid,
role=fav_item.upper.name,
)
],
bvid=fav_item.bvid,
aired=fav_item.ctime,
).write_nfo(fav_item.nfo_path)
# 写入 poster
await download_content(fav_item.cover, fav_item.poster_path)
# 开始处理视频内容
v = video.Video(fav_item.bvid, credential=credential)
detector = video.VideoDownloadURLDataDetecter(
await v.get_download_url(page_index=0)
)
streams = detector.detect_best_streams()
if detector.check_flv_stream():
await download_content(streams[0].url, fav_item.tmp_video_path)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(fav_item.tmp_video_path),
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
fav_item.tmp_video_path.unlink()
else:
await asyncio.gather(
download_content(streams[0].url, fav_item.tmp_video_path),
download_content(streams[1].url, fav_item.tmp_audio_path),
)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(fav_item.tmp_video_path),
"-i",
str(fav_item.tmp_audio_path),
"-c",
"copy",
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
fav_item.tmp_video_path.unlink()
fav_item.tmp_audio_path.unlink()
fav_item.downloaded = True
await fav_item.save()
logger.info(
"{} {} processed successfully.", fav_item.bvid, fav_item.name
)
except ResponseCodeException as e:
match e.code:
case 62002:
fav_item.status = MediaStatus.INVISIBLE
case -404:
fav_item.status = MediaStatus.DELETED
case _:
logger.exception(
"Failed to process video {} {}, error_code: {}",
fav_item.bvid,
fav_item.name,
e.code,
)
return
await fav_item.save()
logger.error(
"Video {} {} is not available, marked as {}",
fav_item.bvid,
fav_item.name,
fav_item.status.text,
)
if fav_item.tags is None:
fav_item.tags = [_["tag_name"] for _ in await v.get_tags()]
except Exception:
logger.exception(
"Failed to process video {} {}", fav_item.bvid, fav_item.name
"Failed to get tags of video {} {}",
fav_item.bvid,
fav_item.name,
)
if process_upper:
try:
if not all(
await asyncio.gather(
aexists(fav_item.upper.thumb_path),
aexists(fav_item.upper.meta_path),
)
):
await amakedirs(fav_item.upper.thumb_path.parent, exist_ok=True)
await asyncio.gather(
fav_item.upper.save_metadata(),
download_content(
fav_item.upper.thumb, fav_item.upper.thumb_path
),
return_exceptions=True,
)
else:
logger.info(
"Upper {} {} already exists, skipped.",
fav_item.upper.mid,
fav_item.upper.name,
)
except Exception:
logger.exception(
"Failed to process upper {} {}",
fav_item.upper.mid,
fav_item.upper.name,
)
if process_nfo:
try:
if not await aexists(fav_item.nfo_path):
await EpisodeInfo(
title=fav_item.name,
plot=fav_item.desc,
actor=[
Actor(
name=fav_item.upper.mid,
role=fav_item.upper.name,
)
],
tags=fav_item.tags,
bvid=fav_item.bvid,
aired=fav_item.ctime,
).write_nfo(fav_item.nfo_path)
else:
logger.info(
"NFO of {} {} already exists, skipped.",
fav_item.bvid,
fav_item.name,
)
except Exception:
logger.exception(
"Failed to process nfo of video {} {}",
fav_item.bvid,
fav_item.name,
)
if process_poster:
try:
if not await aexists(fav_item.poster_path):
try:
await download_content(fav_item.cover, fav_item.poster_path)
except Exception:
logger.exception(
"Failed to download poster of video {} {}",
fav_item.bvid,
fav_item.name,
)
else:
logger.info(
"Poster of {} {} already exists, skipped.",
fav_item.bvid,
fav_item.name,
)
except Exception:
logger.exception(
"Failed to process poster of video {} {}",
fav_item.bvid,
fav_item.name,
)
if process_subtitle:
try:
if not await aexists(fav_item.subtitle_path):
await ass.make_ass_file_danmakus_protobuf(
v,
0,
str(fav_item.subtitle_path.resolve()),
credential=credential,
font_name=settings.subtitle.font_name,
font_size=settings.subtitle.font_size,
alpha=settings.subtitle.alpha,
fly_time=settings.subtitle.fly_time,
static_time=settings.subtitle.static_time,
)
else:
logger.info(
"Subtitle of {} {} already exists, skipped.",
fav_item.bvid,
fav_item.name,
)
except Exception:
logger.exception(
"Failed to process subtitle of video {} {}",
fav_item.bvid,
fav_item.name,
)
if process_video:
try:
if await aexists(fav_item.video_path):
fav_item.downloaded = True
logger.info(
"Video {} {} already exists, skipped.",
fav_item.bvid,
fav_item.name,
)
else:
# 开始处理视频内容
detector = video.VideoDownloadURLDataDetecter(
await v.get_download_url(page_index=0)
)
streams = detector.detect_best_streams()
if detector.check_flv_stream():
await download_content(
streams[0].url, fav_item.tmp_video_path
)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(fav_item.tmp_video_path),
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
fav_item.tmp_video_path.unlink()
else:
await asyncio.gather(
download_content(
streams[0].url, fav_item.tmp_video_path
),
download_content(
streams[1].url, fav_item.tmp_audio_path
),
)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(fav_item.tmp_video_path),
"-i",
str(fav_item.tmp_audio_path),
"-c",
"copy",
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
fav_item.tmp_video_path.unlink()
fav_item.tmp_audio_path.unlink()
fav_item.downloaded = True
except ResponseCodeException as e:
match e.code:
case 62002:
fav_item.status = MediaStatus.INVISIBLE
case -404:
fav_item.status = MediaStatus.DELETED
case _:
logger.exception(
"Failed to process video {} {}, error_code: {}",
fav_item.bvid,
fav_item.name,
e.code,
)
if fav_item.status != MediaStatus.NORMAL:
logger.error(
"Video {} {} is not available, marked as {}",
fav_item.bvid,
fav_item.name,
fav_item.status.text,
)
except Exception:
logger.exception(
"Failed to process video {} {}", fav_item.bvid, fav_item.name
)
await fav_item.save()
logger.info(
"{} {} is processed successfully.",
fav_item.bvid,
fav_item.name,
)

View File

@@ -2,21 +2,34 @@ from dataclasses import dataclass, field, fields
from pathlib import Path
from typing import Self
from dataclasses_json import DataClassJsonMixin
from dataclasses_json import DataClassJsonMixin, Undefined
from constants import DEFAULT_CONFIG_PATH
@dataclass
class SubtitleConfig(DataClassJsonMixin):
dataclass_json_config = {"undefined": Undefined.EXCLUDE}
font_name: str = "微软雅黑,黑体" # 字体
font_size: float = 40 # 字号
alpha: float = 0.8 # 透明度
fly_time: float = 5 # 滚动弹幕持续时间
static_time: float = 10 # 静态弹幕持续时间
@dataclass
class Config(DataClassJsonMixin):
dataclass_json_config = {"undefined": Undefined.EXCLUDE}
sessdata: str = ""
bili_jct: str = ""
buvid3: str = ""
dedeuserid: str = ""
ac_time_value: str = ""
interval: int = 20
favorite_ids: list[int] = field(default_factory=list)
path_mapper: dict[int, str] = field(default_factory=dict)
subtitle: SubtitleConfig = field(default_factory=SubtitleConfig)
def validate(self) -> Self:
"""所有值必须被设置"""

View File

@@ -3,7 +3,7 @@ from pathlib import Path
import aiofiles
import httpx
from aiofiles.base import AiofilesContextManager
from aiofiles.os import makedirs
from aiofiles.os import makedirs, remove
from aiofiles.ospath import exists
from aiofiles.threadpool.text import AsyncTextIOWrapper
from bilibili_api import HEADERS
@@ -31,3 +31,7 @@ def aopen(
path: Path, mode: str = "r", **kwargs
) -> AiofilesContextManager[None, None, AsyncTextIOWrapper]:
return aiofiles.open(path, mode, **kwargs)
async def aremove(path: Path) -> None:
await remove(path)