feat: 支持弹幕下载

.gitignore

@@ -4,7 +4,7 @@ debug.py
 videos
 config.test.json
 database.test.db*
-example.json
+example*.json
 thumbs.test
 config
 data

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install fmt start-daemon start-once
+.PHONY: install fmt start-daemon start-once db-init db-migrate db-upgrade sync-conf
 
 install:
 	@echo "Installing dependencies..."
@@ -22,4 +22,10 @@ db-migrate:
 	@poetry run aerich migrate
 
 db-upgrade:
-	@poetry run aerich upgrade
+	@poetry run aerich upgrade
+
+sync-conf:
+	@echo "Syncing config..."
+	@cp ${CONFIG_SRC} ./config/
+	@cp ${DB_SRC} ./data/
+	@echo "Done."

commands.py

@@ -1,11 +1,14 @@
 import asyncio
+import functools
+from pathlib import Path
+from typing import Callable
 
 from loguru import logger
 
 from constants import MediaStatus, MediaType
-from models import FavoriteItem, Upper
-from processor import download_content
-from utils import aexists, amakedirs
+from models import FavoriteItem
+from processor import process_favorite_item
+from utils import aexists, aremove
 
 
 async def recheck():
@@ -37,26 +40,59 @@ async def recheck():
     logger.info("Database updated.")
 
 
-async def upper_thumb():
-    makedir_tasks = []
-    other_tasks = []
-    for upper in await Upper.all():
-        if all(
-            await asyncio.gather(
-                aexists(upper.thumb_path), aexists(upper.meta_path)
+async def _refresh_favorite_item_info(
+    path_getter: Callable[[FavoriteItem], list[Path]],
+    process_poster: bool = False,
+    process_video: bool = False,
+    process_nfo: bool = False,
+    process_upper: bool = False,
+    process_subtitle: bool = False,
+):
+    items = await FavoriteItem.filter(downloaded=True).prefetch_related("upper")
+    await asyncio.gather(
+        *[aremove(path) for item in items for path in path_getter(item)],
+        return_exceptions=True,
+    )
+    await asyncio.gather(
+        *[
+            process_favorite_item(
+                item,
+                process_poster=process_poster,
+                process_video=process_video,
+                process_nfo=process_nfo,
+                process_upper=process_upper,
+                process_subtitle=process_subtitle,
             )
-        ):
-            logger.info(
-                "Upper {} {} already exists, skipped.", upper.mid, upper.name
-            )
-        makedir_tasks.append(amakedirs(upper.thumb_path.parent, exist_ok=True))
-        logger.info("Saving metadata for upper {} {}...", upper.mid, upper.name)
-        other_tasks.extend(
-            [
-                upper.save_metadata(),
-                download_content(upper.thumb, upper.thumb_path),
-            ]
-        )
-    await asyncio.gather(*makedir_tasks)
-    await asyncio.gather(*other_tasks)
-    logger.info("All done.")
+            for item in items
+        ],
+        return_exceptions=True,
+    )
+
+
+refresh_nfo = functools.partial(
+    _refresh_favorite_item_info, lambda item: [item.nfo_path], process_nfo=True
+)
+
+refresh_poster = functools.partial(
+    _refresh_favorite_item_info,
+    lambda item: [item.poster_path],
+    process_poster=True,
+)
+
+refresh_video = functools.partial(
+    _refresh_favorite_item_info,
+    lambda item: [item.video_path],
+    process_video=True,
+)
+
+refresh_upper = functools.partial(
+    _refresh_favorite_item_info,
+    lambda item: item.upper_path,
+    process_upper=True,
+)
+
+refresh_subtitle = functools.partial(
+    _refresh_favorite_item_info,
+    lambda item: [item.subtitle_path],
+    process_subtitle=True,
+)

entry.py

@@ -4,7 +4,14 @@ import sys
 import uvloop
 from loguru import logger
 
-from commands import recheck, upper_thumb
+from commands import (
+    recheck,
+    refresh_nfo,
+    refresh_poster,
+    refresh_subtitle,
+    refresh_upper,
+    refresh_video,
+)
 from models import init_model
 from processor import cleanup, process
 from settings import settings
@@ -14,11 +21,15 @@ asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
 
 async def entry() -> None:
     await init_model()
-    for command, func in [
+    for command, func in (
         ("once", process),
         ("recheck", recheck),
-        ("upper_thumb", upper_thumb),
-    ]:
+        ("refresh_poster", refresh_poster),
+        ("refresh_upper", refresh_upper),
+        ("refresh_nfo", refresh_nfo),
+        ("refresh_video", refresh_video),
+        ("refresh_subtitle", refresh_subtitle),
+    ):
         if any(command in _ for _ in sys.argv):
             logger.info("Running {}...", command)
             await func()

migrations/models/2_20231204003326_update.py

@@ -0,0 +1,11 @@
+from tortoise import BaseDBAsyncClient
+
+
+async def upgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        ALTER TABLE "favoriteitem" ADD "tags" JSON;"""
+
+
+async def downgrade(db: BaseDBAsyncClient) -> str:
+    return """
+        ALTER TABLE "favoriteitem" DROP COLUMN "tags";"""

models.py

@@ -79,6 +79,7 @@ class FavoriteItem(Model):
     bvid = fields.CharField(max_length=255)
     desc = fields.TextField()
     cover = fields.TextField()
+    tags = fields.JSONField(null=True)
     favorite_list = fields.ForeignKeyField(
         "models.FavoriteList", related_name="items"
     )
@@ -132,6 +133,20 @@ class FavoriteItem(Model):
             / f"{self.bvid}-poster.jpg"
         )
 
+    @property
+    def upper_path(self) -> list[Path]:
+        return [
+            self.upper.thumb_path,
+            self.upper.meta_path,
+        ]
+
+    @property
+    def subtitle_path(self) -> Path:
+        return (
+            Path(settings.path_mapper[self.favorite_list_id])
+            / f"{self.bvid}.zh-CN.default.ass"
+        )
+
 
 async def init_model() -> None:
     await Tortoise.init(config=TORTOISE_ORM)

nfo.py

@@ -25,6 +25,7 @@ class Actor:
 class EpisodeInfo:
     title: str
     plot: str
+    tags: list[str]
     actor: list[Actor]
     bvid: str
     aired: datetime.datetime
@@ -35,6 +36,11 @@ class EpisodeInfo:
 
     def to_xml(self) -> str:
         actor = "\n".join(_.to_xml() for _ in self.actor)
+        tags = (
+            "\n".join(f"    <genre>{_}</genre>" for _ in self.tags)
+            if isinstance(self.tags, list)
+            else ""
+        )
         return f"""
 <?xml version="1.0" encoding="utf-8" standalone="yes"?>
 <episodedetails>
@@ -43,6 +49,7 @@ class EpisodeInfo:
     <title>{self.title}</title>
 {actor}
     <year>{self.aired.year}</year>
+{tags}
     <uniqueid type="bilibili">{self.bvid}</uniqueid>
     <aired>{self.aired.strftime("%Y-%m-%d")}</aired>
 </episodedetails>

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
 
 [[package]]
 name = "aerich"
@@ -301,7 +301,7 @@ pillow = ">=10.1.0,<10.2.0"
 pycryptodomex = ">=3.19.0,<3.20.0"
 pyyaml = ">=6.0,<7.0"
 qrcode = ">=7.4.2,<7.5.0"
-qrcode-terminal = ">=0.8,<1.0"
+qrcode_terminal = ">=0.8,<1.0"
 requests = ">=2.31.0,<2.32.0"
 rsa = ">=4.9,<5.0"
 tqdm = ">=4.66.1,<4.67.0"
@@ -311,7 +311,7 @@ yarl = ">=1.9.2,<1.10.0"
 type = "git"
 url = "https://github.com/amtoaer/bilibili-api.git"
 reference = "dev"
-resolved_reference = "30db504b410787bdc8a5cd1ff63bacf6b365f393"
+resolved_reference = "0281d7c58e8a26706cefc96e7e427cb05a26d866"
 
 [[package]]
 name = "black"

processor.py

@@ -3,7 +3,7 @@ import datetime
 from asyncio import Semaphore, create_subprocess_exec
 from asyncio.subprocess import DEVNULL
 
-from bilibili_api import favorite_list, video
+from bilibili_api import ass, favorite_list, video
 from bilibili_api.exceptions import ResponseCodeException
 from loguru import logger
 from tortoise import Tortoise
@@ -149,95 +149,160 @@ async def process_favorite(favorite_id: int) -> None:
         downloaded=False,
     ).prefetch_related("upper")
     await asyncio.gather(
-        *[process_video(item) for item in all_unprocessed_items],
+        *[process_favorite_item(item) for item in all_unprocessed_items],
         return_exceptions=True,
     )
     logger.info("Favorite {} {} processed successfully.", favorite_id, title)
 
 
 @concurrent_decorator(4)
-async def process_video(fav_item: FavoriteItem) -> None:
+async def process_favorite_item(
+    fav_item: FavoriteItem,
+    process_poster=True,
+    process_video=True,
+    process_nfo=True,
+    process_upper=True,
+    process_subtitle=True,
+) -> None:
     logger.info("Start to process video {} {}", fav_item.bvid, fav_item.name)
     if fav_item.type != MediaType.VIDEO:
         logger.warning("Media {} is not a video, skipped.", fav_item.name)
         return
+    v = video.Video(fav_item.bvid, credential=credential)
     try:
-        if await aexists(fav_item.video_path):
-            fav_item.downloaded = True
-            await fav_item.save()
-            logger.info(
-                "{} {} already exists, skipped.", fav_item.bvid, fav_item.name
-            )
-            return
-        # 写入 up 主头像
-        if not all(
-            await asyncio.gather(
-                aexists(fav_item.upper.thumb_path),
-                aexists(fav_item.upper.meta_path),
-            )
-        ):
-            await amakedirs(fav_item.upper.thumb_path.parent, exist_ok=True)
-            await fav_item.upper.save_metadata()
-            await download_content(
-                fav_item.upper.thumb, fav_item.upper.thumb_path
-            )
-        # 写入 nfo
-        await EpisodeInfo(
-            title=fav_item.name,
-            plot=fav_item.desc,
-            actor=[
-                Actor(
-                    name=fav_item.upper.mid,
-                    role=fav_item.upper.name,
+        if process_upper:
+            # 写入 up 主头像
+            if not all(
+                await asyncio.gather(
+                    aexists(fav_item.upper.thumb_path),
+                    aexists(fav_item.upper.meta_path),
                 )
-            ],
-            bvid=fav_item.bvid,
-            aired=fav_item.ctime,
-        ).write_nfo(fav_item.nfo_path)
-        # 写入 poster
-        await download_content(fav_item.cover, fav_item.poster_path)
-        # 开始处理视频内容
-        v = video.Video(fav_item.bvid, credential=credential)
-        detector = video.VideoDownloadURLDataDetecter(
-            await v.get_download_url(page_index=0)
-        )
-        streams = detector.detect_best_streams()
-        if detector.check_flv_stream():
-            await download_content(streams[0].url, fav_item.tmp_video_path)
-            process = await create_subprocess_exec(
-                FFMPEG_COMMAND,
-                "-i",
-                str(fav_item.tmp_video_path),
-                str(fav_item.video_path),
-                stdout=DEVNULL,
-                stderr=DEVNULL,
-            )
-            await process.communicate()
-            fav_item.tmp_video_path.unlink()
-        else:
-            await asyncio.gather(
-                download_content(streams[0].url, fav_item.tmp_video_path),
-                download_content(streams[1].url, fav_item.tmp_audio_path),
-            )
-            process = await create_subprocess_exec(
-                FFMPEG_COMMAND,
-                "-i",
-                str(fav_item.tmp_video_path),
-                "-i",
-                str(fav_item.tmp_audio_path),
-                "-c",
-                "copy",
-                str(fav_item.video_path),
-                stdout=DEVNULL,
-                stderr=DEVNULL,
-            )
-            await process.communicate()
-            fav_item.tmp_video_path.unlink()
-            fav_item.tmp_audio_path.unlink()
-        fav_item.downloaded = True
-        await fav_item.save()
+            ):
+                await amakedirs(fav_item.upper.thumb_path.parent, exist_ok=True)
+                await asyncio.gather(
+                    fav_item.upper.save_metadata(),
+                    download_content(
+                        fav_item.upper.thumb, fav_item.upper.thumb_path
+                    ),
+                    return_exceptions=True,
+                )
+            else:
+                logger.info(
+                    "Upper {} {} already exists, skipped.",
+                    fav_item.upper.mid,
+                    fav_item.upper.name,
+                )
+        if process_nfo:
+            if not await aexists(fav_item.nfo_path):
+                if fav_item.tags is None:
+                    try:
+                        fav_item.tags = [
+                            _["tag_name"] for _ in await v.get_tags()
+                        ]
+                    except Exception:
+                        logger.exception(
+                            "Failed to get tags of video {} {}",
+                            fav_item.bvid,
+                            fav_item.name,
+                        )
+                # 写入 nfo
+                await EpisodeInfo(
+                    title=fav_item.name,
+                    plot=fav_item.desc,
+                    actor=[
+                        Actor(
+                            name=fav_item.upper.mid,
+                            role=fav_item.upper.name,
+                        )
+                    ],
+                    tags=fav_item.tags,
+                    bvid=fav_item.bvid,
+                    aired=fav_item.ctime,
+                ).write_nfo(fav_item.nfo_path)
+            else:
+                logger.info(
+                    "NFO of {} {} already exists, skipped.",
+                    fav_item.bvid,
+                    fav_item.name,
+                )
+        if process_poster:
+            # 写入 poster
+            if not await aexists(fav_item.poster_path):
+                await download_content(fav_item.cover, fav_item.poster_path)
+            else:
+                logger.info(
+                    "Poster of {} {} already exists, skipped.",
+                    fav_item.bvid,
+                    fav_item.name,
+                )
+        if process_subtitle:
+            if not await aexists(fav_item.subtitle_path):
+                await ass.make_ass_file_danmakus_protobuf(
+                    v, 0, str(fav_item.subtitle_path.resolve())
+                )
+            else:
+                logger.info(
+                    "Subtitle of {} {} already exists, skipped.",
+                    fav_item.bvid,
+                    fav_item.name,
+                )
+        if process_video:
+            if await aexists(fav_item.video_path):
+                fav_item.downloaded = True
+                logger.info(
+                    "Video {} {} already exists, skipped.",
+                    fav_item.bvid,
+                    fav_item.name,
+                )
+            else:
+                # 开始处理视频内容
+                detector = video.VideoDownloadURLDataDetecter(
+                    await v.get_download_url(page_index=0)
+                )
+                streams = detector.detect_best_streams()
+                if detector.check_flv_stream():
+                    await download_content(
+                        streams[0].url, fav_item.tmp_video_path
+                    )
+                    process = await create_subprocess_exec(
+                        FFMPEG_COMMAND,
+                        "-i",
+                        str(fav_item.tmp_video_path),
+                        str(fav_item.video_path),
+                        stdout=DEVNULL,
+                        stderr=DEVNULL,
+                    )
+                    await process.communicate()
+                    fav_item.tmp_video_path.unlink()
+                else:
+                    await asyncio.gather(
+                        download_content(
+                            streams[0].url, fav_item.tmp_video_path
+                        ),
+                        download_content(
+                            streams[1].url, fav_item.tmp_audio_path
+                        ),
+                    )
+                    process = await create_subprocess_exec(
+                        FFMPEG_COMMAND,
+                        "-i",
+                        str(fav_item.tmp_video_path),
+                        "-i",
+                        str(fav_item.tmp_audio_path),
+                        "-c",
+                        "copy",
+                        str(fav_item.video_path),
+                        stdout=DEVNULL,
+                        stderr=DEVNULL,
+                    )
+                    await process.communicate()
+                    fav_item.tmp_video_path.unlink()
+                    fav_item.tmp_audio_path.unlink()
+                fav_item.downloaded = True
         logger.info(
-            "{} {} processed successfully.", fav_item.bvid, fav_item.name
+            "{} {} processed successfully.",
+            fav_item.bvid,
+            fav_item.name,
         )
     except ResponseCodeException as e:
         match e.code:
@@ -253,7 +318,6 @@ async def process_video(fav_item: FavoriteItem) -> None:
                     e.code,
                 )
                 return
-        await fav_item.save()
         logger.error(
             "Video {} {} is not available, marked as {}",
             fav_item.bvid,
@@ -264,3 +328,5 @@ async def process_video(fav_item: FavoriteItem) -> None:
         logger.exception(
             "Failed to process video {} {}", fav_item.bvid, fav_item.name
         )
+    finally:
+        await fav_item.save()

utils.py

@@ -3,7 +3,7 @@ from pathlib import Path
 import aiofiles
 import httpx
 from aiofiles.base import AiofilesContextManager
-from aiofiles.os import makedirs
+from aiofiles.os import makedirs, remove
 from aiofiles.ospath import exists
 from aiofiles.threadpool.text import AsyncTextIOWrapper
 from bilibili_api import HEADERS
@@ -31,3 +31,7 @@ def aopen(
     path: Path, mode: str = "r", **kwargs
 ) -> AiofilesContextManager[None, None, AsyncTextIOWrapper]:
     return aiofiles.open(path, mode, **kwargs)
+
+
+async def aremove(path: Path) -> None:
+    await remove(path)