From ec5776a0ed494f8bbe20bfd23b6b1d22381a187e Mon Sep 17 00:00:00 2001
From: amtoaer <amtoaer@gmail.com>
Date: Sat, 24 Feb 2024 21:37:34 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20recheck=20=E5=AF=B9=E5=88=86=20p=20?=
 =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=81=9A=E9=80=82=E9=85=8D=EF=BC=8C=E4=B8=BA?=
 =?UTF-8?q?=E6=89=80=E6=9C=89=E7=9A=84=E6=95=B0=E6=8D=AE=E5=BA=93=E6=89=B9?=
 =?UTF-8?q?=E9=87=8F=E6=93=8D=E4=BD=9C=E6=8C=87=E5=AE=9A=20batch=5Fsize?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 commands.py  | 31 +++++++++++++++++++++++--------
 processor.py |  8 ++++++--
 2 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/commands.py b/commands.py
index 0947574..8b4d8e5 100644
--- a/commands.py
+++ b/commands.py
@@ -13,17 +13,32 @@ from utils import aexists, aremove
 
 async def recheck():
     """刷新数据库中视频的状态，如果发现文件不存在则标记未下载，以便在下次任务重新下载，在自己手动删除文件后调用"""
-    items = await FavoriteItem.filter(type=MediaType.VIDEO, status=MediaStatus.NORMAL, downloaded=True)
-    exists = await asyncio.gather(*[aexists(item.video_path) for item in items])
-    for item, exist in zip(items, exists):
-        if isinstance(exist, Exception):
-            logger.error("Error when checking file {} {}: {}.", item.bvid, item.name, exist)
+
+    async def is_ok(item: FavoriteItem) -> bool:
+        if len(item.pages):
+            # 多 p 视频全部存在才算存在
+            return all(await asyncio.gather(*[aexists(page.video_path) for page in item.pages]))
+        return await aexists(item.video_path)
+
+    items = await FavoriteItem.filter(
+        type=MediaType.VIDEO, status=MediaStatus.NORMAL, downloaded=True
+    ).prefetch_related("pages")
+    items_to_update = []
+    for item in items:
+        for page in item.pages:
+            # 疑似 tortoise 的 bug，prefetch_related 不会更新反向引用的字段，这里手动更新一下
+            page.favorite_item = item
+    items_ok = await asyncio.gather(*[is_ok(item) for item in items], return_exceptions=True)
+    for item, ok in zip(items, items_ok):
+        if isinstance(ok, Exception):
+            logger.error("Error when checking file {} {}: {}.", item.bvid, item.name, ok)
             continue
-        if not exist:
-            logger.info("File {} {} not exists, mark as not downloaded.", item.bvid, item.name)
+        if not ok:
+            logger.info("Lack of file detected for {} {}, mark as not downloaded.", item.bvid, item.name)
             item.downloaded = False
+            items_to_update.append(item)
     logger.info("Updating database...")
-    await FavoriteItem.bulk_update(items, fields=["downloaded"])
+    await FavoriteItem.bulk_update(items_to_update, fields=["downloaded"], batch_size=300)
     logger.info("Database updated.")
 
 
diff --git a/processor.py b/processor.py
index a1adde8..c89ee8b 100644
--- a/processor.py
+++ b/processor.py
@@ -46,7 +46,7 @@ async def update_favorite_item(medias: list[dict], fav_list: FavoriteList) -> No
     uppers = [
         Upper(mid=media["upper"]["mid"], name=media["upper"]["name"], thumb=media["upper"]["face"]) for media in medias
     ]
-    await Upper.bulk_create(uppers, on_conflict=["mid"], update_fields=["name", "thumb"])
+    await Upper.bulk_create(uppers, on_conflict=["mid"], update_fields=["name", "thumb"], batch_size=300)
     items = [
         FavoriteItem(
             name=media["title"],
@@ -67,6 +67,7 @@ async def update_favorite_item(medias: list[dict], fav_list: FavoriteList) -> No
         items,
         on_conflict=["bvid", "favorite_list_id"],
         update_fields=["name", "type", "desc", "cover", "ctime", "pubtime", "fav_time"],
+        batch_size=300,
     )
 
 
@@ -173,7 +174,10 @@ async def process_favorite_item(
                 single_page = True
             else:
                 pages = await FavoriteItemPage.bulk_create(
-                    pages, on_conflict=["favorite_item_id", "page"], update_fields=["cid", "name", "image"]
+                    pages,
+                    on_conflict=["favorite_item_id", "page"],
+                    update_fields=["cid", "name", "image"],
+                    batch_size=300,
                 )
                 if process_nfo:
                     try: