GithubBackup/bili-sync
1
0
Fork 0
mirror of https://github.com/amtoaer/bili-sync.git synced 2026-05-07 08:02:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: 接入数据库，功能基本完成

Browse Source
This commit is contained in:
amtoaer
2023-11-25 01:59:15 +08:00
parent 4dee129d38
commit 994209ddee
9 changed files with 510 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@@ -4,4 +4,5 @@ debug.py
videos
config.test.json
database.test.db*
example.json
example.json
thumbs.test

11
Makefile
View File

@@ -13,4 +13,13 @@ start-daemon:
@poetry run python entry.py
start-once:
@poetry run python entry.py --once
@poetry run python entry.py --once
db-init:
@poetry run aerich init-db
db-migrate:
@poetry run aerich migrate
db-upgrade:
@poetry run aerich upgrade

20
constants.py
View File

@@ -14,10 +14,30 @@ DEFAULT_DATABASE_PATH = (
else Path(__file__).parent / "database.test.db"
)
DEFAULT_THUMB_PATH = (
Path(__file__).parent / "thumbs"
if not os.getenv("TESTING")
else Path(__file__).parent / "thumbs.test"
)
FFMPEG_COMMAND = "ffmpeg"
MIGRATE_COMMAND = "aerich"
class MediaType(IntEnum):
VIDEO = 2
AUDIO = 12
VIDEO_COLLECTION = 21
TORTOISE_ORM = {
"connections": {"default": f"sqlite://{DEFAULT_DATABASE_PATH}"},
"apps": {
"models": {
"models": ["models", "aerich.models"],
"default_connection": "default",
},
},
"use_tz": True,
}

44
migrations/models/0_20231125010440_init.py Normal file
View File

@@ -0,0 +1,44 @@
from tortoise import BaseDBAsyncClient
async def upgrade(db: BaseDBAsyncClient) -> str:
return """
CREATE TABLE IF NOT EXISTS "favoritelist" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"name" VARCHAR(255) NOT NULL,
"created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
) /* 收藏列表 */;
CREATE TABLE IF NOT EXISTS "upper" (
"mid" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"name" VARCHAR(255) NOT NULL,
"thumb" TEXT NOT NULL,
"created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
"updated_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
) /* up主 */;
CREATE TABLE IF NOT EXISTS "favoriteitem" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"name" VARCHAR(255) NOT NULL,
"type" SMALLINT NOT NULL /* VIDEO: 2\nAUDIO: 12\nVIDEO_COLLECTION: 21 */,
"bvid" VARCHAR(255) NOT NULL,
"desc" TEXT NOT NULL,
"cover" TEXT NOT NULL,
"ctime" TIMESTAMP NOT NULL,
"pubtime" TIMESTAMP NOT NULL,
"fav_time" TIMESTAMP NOT NULL,
"downloaded" INT NOT NULL DEFAULT 0,
"favorite_list_id" INT NOT NULL REFERENCES "favoritelist" ("id") ON DELETE CASCADE,
"upper_id" INT NOT NULL REFERENCES "upper" ("mid") ON DELETE CASCADE,
CONSTRAINT "uid_favoriteite_bvid_d7b8ea" UNIQUE ("bvid", "favorite_list_id")
) /* 收藏条目 */;
CREATE TABLE IF NOT EXISTS "aerich" (
"id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
"version" VARCHAR(255) NOT NULL,
"app" VARCHAR(100) NOT NULL,
"content" JSON NOT NULL
);"""
async def downgrade(db: BaseDBAsyncClient) -> str:
return """
"""

86
models.py
View File

@@ -1,7 +1,16 @@
from asyncio import create_subprocess_exec
from pathlib import Path
from tortoise import Tortoise, fields
from tortoise.models import Model
from constants import DEFAULT_DATABASE_PATH
from constants import (
DEFAULT_THUMB_PATH,
MIGRATE_COMMAND,
TORTOISE_ORM,
MediaType,
)
from settings import settings
class FavoriteList(Model):
@@ -12,33 +21,92 @@ class FavoriteList(Model):
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_now=True)
@property
def video_list_path(self) -> Path:
return Path(settings.path_mapper[self.id])
class Upper(Model):
"""up主"""
id = fields.IntField(pk=True)
mid = fields.IntField(pk=True)
name = fields.CharField(max_length=255)
thumb = fields.TextField()
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_now=True)
@property
def thumb_path(self) -> Path:
return DEFAULT_THUMB_PATH / f"{self.mid}.jpg"
class FavoriteItem(Model):
"""收藏条目"""
id = fields.IntField(pk=True)
name = fields.CharField(max_length=255)
type = fields.IntEnumField(enum_type=MediaType)
bvid = fields.CharField(max_length=255)
desc = fields.TextField()
cover = fields.TextField()
favorite_list = fields.ForeignKeyField(
"models.FavoriteList", related_name="items"
)
upper = fields.ForeignKeyField("models.Upper", related_name="uploads")
created_at = fields.DatetimeField(auto_now_add=True)
updated_at = fields.DatetimeField(auto_now=True)
ctime = fields.DatetimeField()
pubtime = fields.DatetimeField()
fav_time = fields.DatetimeField()
downloaded = fields.BooleanField(default=False)
class Meta:
unique_together = (("bvid", "favorite_list_id"),)
@property
def safe_name(self) -> str:
return self.name.replace("/", "_")
@property
def tmp_video_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"tmp_{self.bvid}_video"
)
@property
def tmp_audio_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"tmp_{self.bvid}_audio"
)
@property
def video_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"{self.bvid}.mp4"
)
@property
def nfo_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"{self.bvid}.nfo"
)
@property
def poster_path(self) -> Path:
return (
Path(settings.path_mapper[self.favorite_list_id])
/ f"{self.bvid}-poster.jpg"
)
async def init_model():
await Tortoise.init(
db_url=f"sqlite://{DEFAULT_DATABASE_PATH}",
modules={"models": ["models"]},
async def init_model() -> None:
await Tortoise.init(config=TORTOISE_ORM)
process = await create_subprocess_exec(
"poetry",
"run",
MIGRATE_COMMAND,
"upgrade",
)
await Tortoise.generate_schemas()
await process.communicate()

4
nfo.py
View File

@@ -6,11 +6,15 @@ from pathlib import Path
@dataclass
class Actor:
name: str
role: str
thumb: Path
def to_xml(self) -> str:
return f"""
<actor>
<name>{self.name}</name>
<role>{self.role}</role>
<thumb>{self.thumb.resolve()}</thumb>
</actor>
""".strip(
"\n"

199
poetry.lock generated
View File

@@ -1,5 +1,27 @@
# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.
[[package]]
name = "aerich"
version = "0.7.2"
description = "A database migrations tool for Tortoise ORM."
optional = false
python-versions = ">=3.7,<4.0"
files = [
{file = "aerich-0.7.2-py3-none-any.whl", hash = "sha256:84c78c07d45436b89ca4db5411eca4e9292a591fb7d6fd4282fa4a7d0c6d2af1"},
{file = "aerich-0.7.2.tar.gz", hash = "sha256:31d67de7b96184636b89de99062e059e5e6204b6251d24c33eb21fc9cf982e09"},
]
[package.dependencies]
click = "*"
dictdiffer = "*"
pydantic = "*"
tomlkit = "*"
tortoise-orm = "*"
[package.extras]
asyncmy = ["asyncmy (>=0.2.8rc1,<0.3.0)"]
asyncpg = ["asyncpg"]
[[package]]
name = "aiofiles"
version = "23.2.1"
@@ -134,6 +156,17 @@ files = [
[package.dependencies]
typing_extensions = ">=3.7.2"
[[package]]
name = "annotated-types"
version = "0.6.0"
description = "Reusable constraint types to use with typing.Annotated"
optional = false
python-versions = ">=3.8"
files = [
{file = "annotated_types-0.6.0-py3-none-any.whl", hash = "sha256:0641064de18ba7a25dee8f96403ebc39113d0cb953a01429249d5c7564666a43"},
{file = "annotated_types-0.6.0.tar.gz", hash = "sha256:563339e807e53ffd9c267e99fc6d9ea23eb8443c08f112651963e24e22f84a5d"},
]
[[package]]
name = "anyio"
version = "4.0.0"
@@ -573,6 +606,23 @@ files = [
{file = "decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330"},
]
[[package]]
name = "dictdiffer"
version = "0.9.0"
description = "Dictdiffer is a library that helps you to diff and patch dictionaries."
optional = false
python-versions = "*"
files = [
{file = "dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595"},
{file = "dictdiffer-0.9.0.tar.gz", hash = "sha256:17bacf5fbfe613ccf1b6d512bd766e6b21fb798822a133aa86098b8ac9997578"},
]
[package.extras]
all = ["Sphinx (>=3)", "check-manifest (>=0.42)", "mock (>=1.3.0)", "numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "numpy (>=1.20.0)", "pytest (==5.4.3)", "pytest (>=6)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "pytest-pycodestyle (>=2)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2)", "pytest-pydocstyle (>=2.2.0)", "sphinx (>=3)", "sphinx-rtd-theme (>=0.2)", "tox (>=3.7.0)"]
docs = ["Sphinx (>=3)", "sphinx-rtd-theme (>=0.2)"]
numpy = ["numpy (>=1.13.0)", "numpy (>=1.15.0)", "numpy (>=1.18.0)", "numpy (>=1.20.0)"]
tests = ["check-manifest (>=0.42)", "mock (>=1.3.0)", "pytest (==5.4.3)", "pytest (>=6)", "pytest-cov (>=2.10.1)", "pytest-isort (>=1.2.0)", "pytest-pycodestyle (>=2)", "pytest-pycodestyle (>=2.2.0)", "pytest-pydocstyle (>=2)", "pytest-pydocstyle (>=2.2.0)", "sphinx (>=3)", "tox (>=3.7.0)"]
[[package]]
name = "executing"
version = "2.0.1"
@@ -1266,6 +1316,142 @@ files = [
{file = "pycryptodomex-3.19.0.tar.gz", hash = "sha256:af83a554b3f077564229865c45af0791be008ac6469ef0098152139e6bd4b5b6"},
]
[[package]]
name = "pydantic"
version = "2.5.2"
description = "Data validation using Python type hints"
optional = false
python-versions = ">=3.7"
files = [
{file = "pydantic-2.5.2-py3-none-any.whl", hash = "sha256:80c50fb8e3dcecfddae1adbcc00ec5822918490c99ab31f6cf6140ca1c1429f0"},
{file = "pydantic-2.5.2.tar.gz", hash = "sha256:ff177ba64c6faf73d7afa2e8cad38fd456c0dbe01c9954e71038001cd15a6edd"},
]
[package.dependencies]
annotated-types = ">=0.4.0"
pydantic-core = "2.14.5"
typing-extensions = ">=4.6.1"
[package.extras]
email = ["email-validator (>=2.0.0)"]
[[package]]
name = "pydantic-core"
version = "2.14.5"
description = ""
optional = false
python-versions = ">=3.7"
files = [
{file = "pydantic_core-2.14.5-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:7e88f5696153dc516ba6e79f82cc4747e87027205f0e02390c21f7cb3bd8abfd"},
{file = "pydantic_core-2.14.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4641e8ad4efb697f38a9b64ca0523b557c7931c5f84e0fd377a9a3b05121f0de"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:774de879d212db5ce02dfbf5b0da9a0ea386aeba12b0b95674a4ce0593df3d07"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ebb4e035e28f49b6f1a7032920bb9a0c064aedbbabe52c543343d39341a5b2a3"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b53e9ad053cd064f7e473a5f29b37fc4cc9dc6d35f341e6afc0155ea257fc911"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8aa1768c151cf562a9992462239dfc356b3d1037cc5a3ac829bb7f3bda7cc1f9"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac5c82fc632c599f4639a5886f96867ffced74458c7db61bc9a66ccb8ee3113"},
{file = "pydantic_core-2.14.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d2ae91f50ccc5810b2f1b6b858257c9ad2e08da70bf890dee02de1775a387c66"},
{file = "pydantic_core-2.14.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6b9ff467ffbab9110e80e8c8de3bcfce8e8b0fd5661ac44a09ae5901668ba997"},
{file = "pydantic_core-2.14.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61ea96a78378e3bd5a0be99b0e5ed00057b71f66115f5404d0dae4819f495093"},
{file = "pydantic_core-2.14.5-cp310-none-win32.whl", hash = "sha256:bb4c2eda937a5e74c38a41b33d8c77220380a388d689bcdb9b187cf6224c9720"},
{file = "pydantic_core-2.14.5-cp310-none-win_amd64.whl", hash = "sha256:b7851992faf25eac90bfcb7bfd19e1f5ffa00afd57daec8a0042e63c74a4551b"},
{file = "pydantic_core-2.14.5-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:4e40f2bd0d57dac3feb3a3aed50f17d83436c9e6b09b16af271b6230a2915459"},
{file = "pydantic_core-2.14.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ab1cdb0f14dc161ebc268c09db04d2c9e6f70027f3b42446fa11c153521c0e88"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aae7ea3a1c5bb40c93cad361b3e869b180ac174656120c42b9fadebf685d121b"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:60b7607753ba62cf0739177913b858140f11b8af72f22860c28eabb2f0a61937"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2248485b0322c75aee7565d95ad0e16f1c67403a470d02f94da7344184be770f"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:823fcc638f67035137a5cd3f1584a4542d35a951c3cc68c6ead1df7dac825c26"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96581cfefa9123accc465a5fd0cc833ac4d75d55cc30b633b402e00e7ced00a6"},
{file = "pydantic_core-2.14.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a33324437018bf6ba1bb0f921788788641439e0ed654b233285b9c69704c27b4"},
{file = "pydantic_core-2.14.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9bd18fee0923ca10f9a3ff67d4851c9d3e22b7bc63d1eddc12f439f436f2aada"},
{file = "pydantic_core-2.14.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:853a2295c00f1d4429db4c0fb9475958543ee80cfd310814b5c0ef502de24dda"},
{file = "pydantic_core-2.14.5-cp311-none-win32.whl", hash = "sha256:cb774298da62aea5c80a89bd58c40205ab4c2abf4834453b5de207d59d2e1651"},
{file = "pydantic_core-2.14.5-cp311-none-win_amd64.whl", hash = "sha256:e87fc540c6cac7f29ede02e0f989d4233f88ad439c5cdee56f693cc9c1c78077"},
{file = "pydantic_core-2.14.5-cp311-none-win_arm64.whl", hash = "sha256:57d52fa717ff445cb0a5ab5237db502e6be50809b43a596fb569630c665abddf"},
{file = "pydantic_core-2.14.5-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:e60f112ac88db9261ad3a52032ea46388378034f3279c643499edb982536a093"},
{file = "pydantic_core-2.14.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6e227c40c02fd873c2a73a98c1280c10315cbebe26734c196ef4514776120aeb"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0cbc7fff06a90bbd875cc201f94ef0ee3929dfbd5c55a06674b60857b8b85ed"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:103ef8d5b58596a731b690112819501ba1db7a36f4ee99f7892c40da02c3e189"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c949f04ecad823f81b1ba94e7d189d9dfb81edbb94ed3f8acfce41e682e48cef"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1452a1acdf914d194159439eb21e56b89aa903f2e1c65c60b9d874f9b950e5d"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb4679d4c2b089e5ef89756bc73e1926745e995d76e11925e3e96a76d5fa51fc"},
{file = "pydantic_core-2.14.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cf9d3fe53b1ee360e2421be95e62ca9b3296bf3f2fb2d3b83ca49ad3f925835e"},
{file = "pydantic_core-2.14.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:70f4b4851dbb500129681d04cc955be2a90b2248d69273a787dda120d5cf1f69"},
{file = "pydantic_core-2.14.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:59986de5710ad9613ff61dd9b02bdd2f615f1a7052304b79cc8fa2eb4e336d2d"},
{file = "pydantic_core-2.14.5-cp312-none-win32.whl", hash = "sha256:699156034181e2ce106c89ddb4b6504c30db8caa86e0c30de47b3e0654543260"},
{file = "pydantic_core-2.14.5-cp312-none-win_amd64.whl", hash = "sha256:5baab5455c7a538ac7e8bf1feec4278a66436197592a9bed538160a2e7d11e36"},
{file = "pydantic_core-2.14.5-cp312-none-win_arm64.whl", hash = "sha256:e47e9a08bcc04d20975b6434cc50bf82665fbc751bcce739d04a3120428f3e27"},
{file = "pydantic_core-2.14.5-cp37-cp37m-macosx_10_7_x86_64.whl", hash = "sha256:af36f36538418f3806048f3b242a1777e2540ff9efaa667c27da63d2749dbce0"},
{file = "pydantic_core-2.14.5-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:45e95333b8418ded64745f14574aa9bfc212cb4fbeed7a687b0c6e53b5e188cd"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e47a76848f92529879ecfc417ff88a2806438f57be4a6a8bf2961e8f9ca9ec7"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d81e6987b27bc7d101c8597e1cd2bcaa2fee5e8e0f356735c7ed34368c471550"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:34708cc82c330e303f4ce87758828ef6e457681b58ce0e921b6e97937dd1e2a3"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:652c1988019752138b974c28f43751528116bcceadad85f33a258869e641d753"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e4d090e73e0725b2904fdbdd8d73b8802ddd691ef9254577b708d413bf3006e"},
{file = "pydantic_core-2.14.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c7d5b5005f177764e96bd584d7bf28d6e26e96f2a541fdddb934c486e36fd59"},
{file = "pydantic_core-2.14.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a71891847f0a73b1b9eb86d089baee301477abef45f7eaf303495cd1473613e4"},
{file = "pydantic_core-2.14.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a717aef6971208f0851a2420b075338e33083111d92041157bbe0e2713b37325"},
{file = "pydantic_core-2.14.5-cp37-none-win32.whl", hash = "sha256:de790a3b5aa2124b8b78ae5faa033937a72da8efe74b9231698b5a1dd9be3405"},
{file = "pydantic_core-2.14.5-cp37-none-win_amd64.whl", hash = "sha256:6c327e9cd849b564b234da821236e6bcbe4f359a42ee05050dc79d8ed2a91588"},
{file = "pydantic_core-2.14.5-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:ef98ca7d5995a82f43ec0ab39c4caf6a9b994cb0b53648ff61716370eadc43cf"},
{file = "pydantic_core-2.14.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c6eae413494a1c3f89055da7a5515f32e05ebc1a234c27674a6956755fb2236f"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcf4e6d85614f7a4956c2de5a56531f44efb973d2fe4a444d7251df5d5c4dcfd"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6637560562134b0e17de333d18e69e312e0458ee4455bdad12c37100b7cad706"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:77fa384d8e118b3077cccfcaf91bf83c31fe4dc850b5e6ee3dc14dc3d61bdba1"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16e29bad40bcf97aac682a58861249ca9dcc57c3f6be22f506501833ddb8939c"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:531f4b4252fac6ca476fbe0e6f60f16f5b65d3e6b583bc4d87645e4e5ddde331"},
{file = "pydantic_core-2.14.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:074f3d86f081ce61414d2dc44901f4f83617329c6f3ab49d2bc6c96948b2c26b"},
{file = "pydantic_core-2.14.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c2adbe22ab4babbca99c75c5d07aaf74f43c3195384ec07ccbd2f9e3bddaecec"},
{file = "pydantic_core-2.14.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0f6116a558fd06d1b7c2902d1c4cf64a5bd49d67c3540e61eccca93f41418124"},
{file = "pydantic_core-2.14.5-cp38-none-win32.whl", hash = "sha256:fe0a5a1025eb797752136ac8b4fa21aa891e3d74fd340f864ff982d649691867"},
{file = "pydantic_core-2.14.5-cp38-none-win_amd64.whl", hash = "sha256:079206491c435b60778cf2b0ee5fd645e61ffd6e70c47806c9ed51fc75af078d"},
{file = "pydantic_core-2.14.5-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:a6a16f4a527aae4f49c875da3cdc9508ac7eef26e7977952608610104244e1b7"},
{file = "pydantic_core-2.14.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:abf058be9517dc877227ec3223f0300034bd0e9f53aebd63cf4456c8cb1e0863"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:49b08aae5013640a3bfa25a8eebbd95638ec3f4b2eaf6ed82cf0c7047133f03b"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c2d97e906b4ff36eb464d52a3bc7d720bd6261f64bc4bcdbcd2c557c02081ed2"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3128e0bbc8c091ec4375a1828d6118bc20404883169ac95ffa8d983b293611e6"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:88e74ab0cdd84ad0614e2750f903bb0d610cc8af2cc17f72c28163acfcf372a4"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c339dabd8ee15f8259ee0f202679b6324926e5bc9e9a40bf981ce77c038553db"},
{file = "pydantic_core-2.14.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3387277f1bf659caf1724e1afe8ee7dbc9952a82d90f858ebb931880216ea955"},
{file = "pydantic_core-2.14.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ba6b6b3846cfc10fdb4c971980a954e49d447cd215ed5a77ec8190bc93dd7bc5"},
{file = "pydantic_core-2.14.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca61d858e4107ce5e1330a74724fe757fc7135190eb5ce5c9d0191729f033209"},
{file = "pydantic_core-2.14.5-cp39-none-win32.whl", hash = "sha256:ec1e72d6412f7126eb7b2e3bfca42b15e6e389e1bc88ea0069d0cc1742f477c6"},
{file = "pydantic_core-2.14.5-cp39-none-win_amd64.whl", hash = "sha256:c0b97ec434041827935044bbbe52b03d6018c2897349670ff8fe11ed24d1d4ab"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79e0a2cdbdc7af3f4aee3210b1172ab53d7ddb6a2d8c24119b5706e622b346d0"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:678265f7b14e138d9a541ddabbe033012a2953315739f8cfa6d754cc8063e8ca"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95b15e855ae44f0c6341ceb74df61b606e11f1087e87dcb7482377374aac6abe"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09b0e985fbaf13e6b06a56d21694d12ebca6ce5414b9211edf6f17738d82b0f8"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ad873900297bb36e4b6b3f7029d88ff9829ecdc15d5cf20161775ce12306f8a"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:2d0ae0d8670164e10accbeb31d5ad45adb71292032d0fdb9079912907f0085f4"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d37f8ec982ead9ba0a22a996129594938138a1503237b87318392a48882d50b7"},
{file = "pydantic_core-2.14.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:35613015f0ba7e14c29ac6c2483a657ec740e5ac5758d993fdd5870b07a61d8b"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab4ea451082e684198636565224bbb179575efc1658c48281b2c866bfd4ddf04"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ce601907e99ea5b4adb807ded3570ea62186b17f88e271569144e8cca4409c7"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb2ed8b3fe4bf4506d6dab3b93b83bbc22237e230cba03866d561c3577517d18"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70f947628e074bb2526ba1b151cee10e4c3b9670af4dbb4d73bc8a89445916b5"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:4bc536201426451f06f044dfbf341c09f540b4ebdb9fd8d2c6164d733de5e634"},
{file = "pydantic_core-2.14.5-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f4791cf0f8c3104ac668797d8c514afb3431bc3305f5638add0ba1a5a37e0d88"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:038c9f763e650712b899f983076ce783175397c848da04985658e7628cbe873b"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:27548e16c79702f1e03f5628589c6057c9ae17c95b4c449de3c66b589ead0520"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c97bee68898f3f4344eb02fec316db93d9700fb1e6a5b760ffa20d71d9a46ce3"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b9b759b77f5337b4ea024f03abc6464c9f35d9718de01cfe6bae9f2e139c397e"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:439c9afe34638ace43a49bf72d201e0ffc1a800295bed8420c2a9ca8d5e3dbb3"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:ba39688799094c75ea8a16a6b544eb57b5b0f3328697084f3f2790892510d144"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:ccd4d5702bb90b84df13bd491be8d900b92016c5a455b7e14630ad7449eb03f8"},
{file = "pydantic_core-2.14.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:81982d78a45d1e5396819bbb4ece1fadfe5f079335dd28c4ab3427cd95389944"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:7f8210297b04e53bc3da35db08b7302a6a1f4889c79173af69b72ec9754796b8"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:8c8a8812fe6f43a3a5b054af6ac2d7b8605c7bcab2804a8a7d68b53f3cd86e00"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:206ed23aecd67c71daf5c02c3cd19c0501b01ef3cbf7782db9e4e051426b3d0d"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c2027d05c8aebe61d898d4cffd774840a9cb82ed356ba47a90d99ad768f39789"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:40180930807ce806aa71eda5a5a5447abb6b6a3c0b4b3b1b1962651906484d68"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:615a0a4bff11c45eb3c1996ceed5bdaa2f7b432425253a7c2eed33bb86d80abc"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:f5e412d717366e0677ef767eac93566582518fe8be923361a5c204c1a62eaafe"},
{file = "pydantic_core-2.14.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:513b07e99c0a267b1d954243845d8a833758a6726a3b5d8948306e3fe14675e3"},
{file = "pydantic_core-2.14.5.tar.gz", hash = "sha256:6d30226dfc816dd0fdf120cae611dd2215117e4f9b124af8c60ab9093b6e8e71"},
]
[package.dependencies]
typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
[[package]]
name = "pygments"
version = "2.17.1"
@@ -1523,6 +1709,17 @@ pure-eval = "*"
[package.extras]
tests = ["cython", "littleutils", "pygments", "pytest", "typeguard"]
[[package]]
name = "tomlkit"
version = "0.12.3"
description = "Style preserving TOML library"
optional = false
python-versions = ">=3.7"
files = [
{file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"},
{file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"},
]
[[package]]
name = "tortoise-orm"
version = "0.20.0"
@@ -1812,4 +2009,4 @@ multidict = ">=4.0"
[metadata]
lock-version = "2.0"
python-versions = "^3.11"
content-hash = "aa8b8172538f633284fb81f9cb6d1ca6644c3c104859738237965f70ccb2fb59"
content-hash = "8a2ecbf196e4f1d2cef3b98ee986a2c812ecd8b2411da4948b9f2d9fe996d3ff"

207
processor.py
View File

@@ -8,9 +8,11 @@ import aiofiles
import httpx
from bilibili_api import HEADERS, favorite_list, video
from loguru import logger
from tortoise import Tortoise
from constants import FFMPEG_COMMAND, MediaType
from constants import DEFAULT_THUMB_PATH, FFMPEG_COMMAND, MediaType
from credential import credential
from models import FavoriteItem, FavoriteList, Upper
from nfo import Actor, EpisodeInfo
from settings import settings
@@ -21,6 +23,7 @@ client = httpx.AsyncClient(headers=HEADERS)
async def cleanup() -> None:
await client.aclose()
await Tortoise.close_connections()
def concurrent_decorator(concurrency: int) -> callable:
@@ -36,7 +39,7 @@ def concurrent_decorator(concurrency: int) -> callable:
return decorator
async def download_content(url: str, path: Path):
async def download_content(url: str, path: Path) -> None:
async with client.stream("GET", url) as resp, aiofiles.open(
path, "wb"
) as f:
@@ -46,11 +49,53 @@ async def download_content(url: str, path: Path):
await f.write(chunk)
async def process():
async def manage_model(medias: list[dict], fav_list: FavoriteList) -> None:
uppers = [
Upper(
mid=media["upper"]["mid"],
name=media["upper"]["name"],
thumb=media["upper"]["face"],
)
for media in medias
]
await Upper.bulk_create(
uppers, on_conflict=["mid"], update_fields=["name", "thumb"]
)
items = [
FavoriteItem(
name=media["title"],
type=media["type"],
bvid=media["bvid"],
desc=media["intro"],
cover=media["cover"],
favorite_list=fav_list,
upper_id=media["upper"]["mid"],
ctime=datetime.datetime.utcfromtimestamp(media["ctime"]),
pubtime=datetime.datetime.utcfromtimestamp(media["pubtime"]),
fav_time=datetime.datetime.utcfromtimestamp(media["fav_time"]),
downloaded=False,
)
for media in medias
]
await FavoriteItem.bulk_create(
items,
on_conflict=["bvid", "favorite_list_id"],
update_fields=[
"name",
"type",
"desc",
"cover",
"ctime",
"pubtime",
"fav_time",
],
)
async def process() -> None:
global anchor
if (
datetime.datetime.now() - anchor
).days >= 1 and await credential.check_refresh():
if (datetime.datetime.now() - anchor).days >= 3:
# 暂定三天刷新一次凭据，具体看情况调整
try:
credential.refresh()
anchor = datetime.datetime.today()
@@ -68,98 +113,138 @@ async def process():
async def process_favorite(favorite_id: int) -> None:
save_path = Path(settings.path_mapper[favorite_id])
save_path.mkdir(parents=True, exist_ok=True)
page, tasks = 0, []
# 预先请求第一页内容以获取收藏夹标题
favorite_video_list = await favorite_list.get_video_favorite_list_content(
favorite_id, page=1, credential=credential
)
logger.info(
"start to process favorite {}: {}",
favorite_id,
favorite_video_list["info"]["title"],
)
fav_list, _ = await FavoriteList.get_or_create(
id=favorite_id, defaults={"name": favorite_video_list["info"]["title"]}
)
fav_list.video_list_path.mkdir(parents=True, exist_ok=True)
DEFAULT_THUMB_PATH.mkdir(parents=True, exist_ok=True)
page = 0
while True:
page += 1
favorite_video_list = (
await favorite_list.get_video_favorite_list_content(
favorite_id, page=page, credential=credential
if page > 1:
favorite_video_list = (
await favorite_list.get_video_favorite_list_content(
favorite_id, page=page, credential=credential
)
)
# 先看看对应 bvid 的记录是否存在
existed_items = await FavoriteItem.filter(
favorite_list=fav_list,
bvid__in=[media["bvid"] for media in favorite_video_list["medias"]],
)
if page == 1:
logger.info(
"start to process favorite {}: {}",
favorite_id,
favorite_video_list["info"]["title"],
)
tasks.extend(
[
process_video(save_path, media)
for media in favorite_video_list["medias"]
]
logger.info("len of existed_items: {}", len(existed_items))
# 记录一下获得的列表中的 bvid 和 fav_time
media_info = {
(media["bvid"], media["fav_time"])
for media in favorite_video_list["medias"]
}
logger.error("media_info: {}", media_info)
logger.error(
"existed_items: {}",
{
(item.bvid, int(item.fav_time.timestamp()))
for item in existed_items
},
)
if not favorite_video_list["has_more"]:
# 如果有 bvid 和 fav_time 都相同的记录，说明已经到达了上次处理到的位置
continue_flag = not media_info & {
(item.bvid, int(item.fav_time.timestamp()))
for item in existed_items
}
if not continue_flag:
logger.info("已经处理到上次的位置，跳过。")
await manage_model(favorite_video_list["medias"], fav_list)
if not (continue_flag and favorite_video_list["has_more"]):
break
await asyncio.gather(*tasks, return_exceptions=True)
all_unprocessed_items = await FavoriteItem.filter(
favorite_list=fav_list, downloaded=False
).prefetch_related("upper")
result = await asyncio.gather(
*[process_video(item) for item in list(all_unprocessed_items)[:5]],
return_exceptions=True,
)
logger.error(result)
@concurrent_decorator(4)
async def process_video(save_path: Path, media: dict) -> None:
title = media["title"]
safe_title = media["title"].replace("/", "_")
logger.info("start to process video {}", title)
if media["type"] != MediaType.VIDEO:
logger.warning("Media {} is not a video, skipped.", title)
async def process_video(fav_item: FavoriteItem) -> None:
logger.info("start to process video {}", fav_item.name)
if fav_item.type != MediaType.VIDEO:
logger.warning("Media {} is not a video, skipped.", fav_item.name)
return
final_path = save_path / f"{safe_title}.mp4"
if final_path.exists():
logger.info(f"{final_path} already exists, skipped.")
if fav_item.video_path.exists():
fav_item.downloaded = True
await fav_item.save()
logger.info(
"{} {} already exists, skipped.", fav_item.bvid, fav_item.name
)
return
# 写入 up 主头像
if not fav_item.upper.thumb_path.exists():
await download_content(fav_item.upper.thumb, fav_item.upper.thumb_path)
# 写入 nfo
nfo_path = save_path / f"{safe_title}.nfo"
EpisodeInfo(
title=title,
plot=media["intro"],
actor=[Actor(f"{media['upper']['mid']} - {media['upper']['name']}")],
bvid=media["bvid"],
aired=datetime.datetime.fromtimestamp(media["ctime"]),
).write_nfo(nfo_path)
title=fav_item.name,
plot=fav_item.desc,
actor=[
Actor(
name=fav_item.upper.mid,
role=fav_item.upper.name,
thumb=fav_item.upper.thumb_path,
)
],
bvid=fav_item.bvid,
aired=fav_item.ctime,
).write_nfo(fav_item.nfo_path)
# 写入 poster
cover_path = save_path / f"{safe_title}-poster.jpg"
await download_content(media["cover"], cover_path)
await download_content(fav_item.cover, fav_item.poster_path)
# 开始处理视频内容
v = video.Video(media["bvid"], credential=credential)
v = video.Video(fav_item.bvid, credential=credential)
detector = video.VideoDownloadURLDataDetecter(
await v.get_download_url(page_index=0)
)
streams = detector.detect_best_streams()
if detector.check_flv_stream():
tmp_path = save_path / f"{safe_title}.flv"
await download_content(streams[0].url, tmp_path)
await download_content(streams[0].url, fav_item.tmp_video_path)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(tmp_path),
str(final_path),
str(fav_item.tmp_video_path),
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
tmp_path.unlink()
fav_item.tmp_video_path.unlink()
else:
tmp_video_path, tmp_audio_path = (
save_path / f"{safe_title}_video.m4s",
save_path / f"{safe_title}_audio.m4s",
)
await asyncio.gather(
download_content(streams[0].url, tmp_video_path),
download_content(streams[1].url, tmp_audio_path),
download_content(streams[0].url, fav_item.tmp_video_path),
download_content(streams[1].url, fav_item.tmp_audio_path),
)
process = await create_subprocess_exec(
FFMPEG_COMMAND,
"-i",
str(tmp_video_path),
str(fav_item.tmp_video_path),
"-i",
str(tmp_audio_path),
str(fav_item.tmp_audio_path),
"-c",
"copy",
str(final_path),
str(fav_item.video_path),
stdout=DEVNULL,
stderr=DEVNULL,
)
await process.communicate()
tmp_video_path.unlink()
tmp_audio_path.unlink()
logger.info(f"{title} downloaded successfully.")
fav_item.tmp_video_path.unlink()
fav_item.tmp_audio_path.unlink()
fav_item.downloaded = True
await fav_item.save()
logger.info("{} {} processed successfully.", fav_item.bvid, fav_item.name)

14
pyproject.toml
View File

@@ -8,13 +8,13 @@ readme = "README.md"
[tool.poetry.dependencies]
python = "^3.11"
bilibili-api-python = {git = "https://github.com/amtoaer/bilibili-api.git", rev = "dev"}
bilibili-api-python = { git = "https://github.com/amtoaer/bilibili-api.git", rev = "dev" }
dataclasses-json = "0.6.2"
tortoise-orm = "0.20.0"
loguru = "0.7.2"
uvloop = "0.19.0"
aiofiles = "23.2.1"
aerich = "0.7.2"
[tool.poetry.group.dev.dependencies]
black = "23.11.0"
@@ -50,12 +50,16 @@ select = [
"NPY", # https://beta.ruff.rs/docs/rules/#numpy-specific-rules-npy
"RUF100", # https://beta.ruff.rs/docs/configuration/#automatic-noqa-management
]
ignore = [
"A003", # Class attribute `id` is shadowing a Python builtin
"A003", # Class attribute `id` is shadowing a Python builtin
]
exclude = ["migrations"]
[tool.aerich]
tortoise_orm = "constants.TORTOISE_ORM"
location = "./migrations"
src_folder = "./."
[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"