fix(mediaserver): sync library data incrementally

app/chain/mediaserver.py

@@ -1,4 +1,5 @@
 import threading
+from datetime import datetime
 from typing import List, Union, Optional, Generator, Any
 
 from app.chain import ChainBase
@@ -134,9 +135,10 @@ class MediaServerChain(ChainBase):
         with lock:
             # 汇总统计
             total_count = 0
-            # 清空登记薄
             dboper = MediaServerOper()
-            dboper.empty()
+            enabled_servers = [mediaserver.name for mediaserver in mediaservers
+                               if mediaserver and mediaserver.enabled and mediaserver.name]
+            dboper.delete_excluded_servers(enabled_servers)
             # 遍历媒体服务器
             for mediaserver in mediaservers:
                 if not mediaserver:
@@ -152,6 +154,7 @@ class MediaServerChain(ChainBase):
                 if not libraries:
                     logger.info(f"没有获取到媒体服务器 {server_name} 的媒体库，跳过")
                     continue
+                sync_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")
                 for library in libraries:
                     if sync_libraries \
                             and "all" not in sync_libraries \
@@ -180,8 +183,11 @@ class MediaServerChain(ChainBase):
                         item_dict = item.model_dump()
                         item_dict["seasoninfo"] = seasoninfo
                         item_dict["item_type"] = item_type
-                        dboper.add(**item_dict)
+                        item_dict["lst_mod_date"] = sync_time
+                        dboper.upsert(**item_dict)
                     logger.info(f"{server_name} 媒体库 {library.name} 同步完成，共同步数量：{library_count}")
                     # 总数累加
                     total_count += library_count
+                stale_count = dboper.delete_stale(server=server_name, sync_time=sync_time)
+                logger.info(f"媒体服务器 {server_name} 清理陈旧数据完成，删除数量：{stale_count}")
                 logger.info(f"媒体服务器 {server_name} 数据同步完成，总同步数量：{total_count}")

app/db/mediaserver_oper.py

@@ -14,24 +14,67 @@ class MediaServerOper(DbOper):
     def __init__(self, db: Session = None):
         super().__init__(db)
 
+    @staticmethod
+    def __prepare_payload(kwargs: dict) -> dict:
+        """
+        过滤数据库模型不存在或不应由远端覆盖的字段
+        """
+        return {
+            k: v for k, v in kwargs.items()
+            if hasattr(MediaServerItem, k) and k != "id"
+        }
+
     def add(self, **kwargs) -> bool:
         """
         新增媒体服务器数据
         """
-        # MediaServerItem中没有的属性剔除
-        kwargs = {k: v for k, v in kwargs.items() if hasattr(MediaServerItem, k)}
+        kwargs = self.__prepare_payload(kwargs)
+        server = kwargs.get("server")
+        item_id = kwargs.get("item_id")
+        if not server or not item_id:
+            return False
         item = MediaServerItem(**kwargs)
-        if not item.get_by_itemid(self._db, kwargs.get("item_id")):
+        if not item.get_by_server_itemid(self._db, server, item_id):
             item.create(self._db)
             return True
         return False
 
+    def upsert(self, **kwargs) -> bool:
+        """
+        按媒体服务器和条目ID新增或更新数据
+        """
+        kwargs = self.__prepare_payload(kwargs)
+        server = kwargs.get("server")
+        item_id = kwargs.get("item_id")
+        if not server or not item_id:
+            return False
+
+        item = MediaServerItem.get_by_server_itemid(self._db, server, item_id)
+        if item:
+            item.update(self._db, kwargs)
+            return False
+
+        MediaServerItem(**kwargs).create(self._db)
+        return True
+
     def empty(self, server: Optional[str] = None):
         """
         清空媒体服务器数据
         """
         MediaServerItem.empty(self._db, server)
 
+    def delete_stale(self, server: str, sync_time: str) -> int:
+        """
+        删除本轮同步未更新的旧数据
+        """
+        return MediaServerItem.delete_stale(self._db, server, sync_time)
+
+    def delete_excluded_servers(self, servers: list[str]) -> int:
+        """
+        删除未启用或已移除媒体服务器的数据
+        """
+        return MediaServerItem.delete_excluded_servers(self._db, servers)
+
     def exists(self, **kwargs) -> Optional[MediaServerItem]:
         """
         判断媒体服务器数据是否存在

app/db/models/mediaserver.py

@@ -1,7 +1,7 @@
 from datetime import datetime
-from typing import Optional
+from typing import Optional, List
 
-from sqlalchemy import Column, Integer, String, JSON, Index
+from sqlalchemy import Column, Integer, String, JSON, Index, or_
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session
@@ -44,6 +44,7 @@ class MediaServerItem(Base):
     lst_mod_date = Column(String, default=datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
 
     __table_args__ = (
+        Index('ux_mediaserveritem_server_item_id', 'server', 'item_id', unique=True),
         Index('ix_mediaserveritem_tmdbid_item_type', 'tmdbid', 'item_type'),
     )
 
@@ -52,13 +53,34 @@ class MediaServerItem(Base):
     def get_by_itemid(cls, db: Session, item_id: str):
         return db.query(cls).filter(cls.item_id == item_id).first()
 
+    @classmethod
+    @db_query
+    def get_by_server_itemid(cls, db: Session, server: str, item_id: str):
+        return db.query(cls).filter(cls.server == server,
+                                    cls.item_id == item_id).first()
+
     @classmethod
     @db_update
     def empty(cls, db: Session, server: Optional[str] = None):
         if server is None:
-            db.query(cls).delete()
+            db.query(cls).delete(synchronize_session=False)
         else:
-            db.query(cls).filter(cls.server == server).delete()
+            db.query(cls).filter(cls.server == server).delete(synchronize_session=False)
+
+    @classmethod
+    @db_update
+    def delete_stale(cls, db: Session, server: str, sync_time: str):
+        return db.query(cls).filter(cls.server == server,
+                                    or_(cls.lst_mod_date.is_(None),
+                                        cls.lst_mod_date != sync_time)).delete(synchronize_session=False)
+
+    @classmethod
+    @db_update
+    def delete_excluded_servers(cls, db: Session, servers: List[str]):
+        if not servers:
+            return db.query(cls).delete(synchronize_session=False)
+        return db.query(cls).filter(or_(cls.server.is_(None),
+                                        ~cls.server.in_(servers))).delete(synchronize_session=False)
 
     @classmethod
     @db_query