mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-21 00:07:32 +08:00
refactor(cache): simplify recognition cache persistence
This commit is contained in:
jxxghp
@@ -11,7 +11,6 @@ from app.core.metainfo import MetaInfo
|
||||
from app.log import logger
|
||||
from app.modules import _ModuleBase
|
||||
from app.modules.douban.apiv2 import DoubanApi
|
||||
from app.modules.douban.douban_cache import DoubanCache
|
||||
from app.modules.douban.scraper import DoubanScraper
|
||||
from app.schemas import MediaPerson, APIRateLimitException
|
||||
from app.schemas.types import MediaType, ModuleType, MediaRecognizeType
|
||||
@@ -24,12 +23,10 @@ from app.utils.zhconv import convert as zhconv_convert
|
||||
class DoubanModule(_ModuleBase):
|
||||
doubanapi: DoubanApi = None
|
||||
scraper: DoubanScraper = None
|
||||
cache: DoubanCache = None
|
||||
|
||||
def init_module(self) -> None:
|
||||
self.doubanapi = DoubanApi()
|
||||
self.scraper = DoubanScraper()
|
||||
self.cache = DoubanCache()
|
||||
|
||||
def stop(self):
|
||||
self.doubanapi.close()
|
||||
@@ -110,7 +107,6 @@ class DoubanModule(_ModuleBase):
|
||||
def _recognize_media_core(self, meta: MetaBase = None,
|
||||
mtype: MediaType = None,
|
||||
doubanid: Optional[str] = None,
|
||||
cache: Optional[bool] = True,
|
||||
douban_info_func=None,
|
||||
match_doubaninfo_func=None,
|
||||
**kwargs) -> Optional[MediaInfo]:
|
||||
@@ -119,7 +115,6 @@ class DoubanModule(_ModuleBase):
|
||||
:param meta: 识别的元数据
|
||||
:param mtype: 识别的媒体类型，与doubanid配套
|
||||
:param doubanid: 豆瓣ID
|
||||
:param cache: 是否使用缓存
|
||||
:param douban_info_func: 获取豆瓣信息的函数
|
||||
:param match_doubaninfo_func: 匹配豆瓣信息的函数
|
||||
:return: 识别的媒体信息，包括剧集信息
|
||||
@@ -134,69 +129,39 @@ class DoubanModule(_ModuleBase):
|
||||
):
|
||||
return None
|
||||
|
||||
if not meta:
|
||||
# 未提供元数据时，直接查询豆瓣信息，不使用缓存
|
||||
cache_info = {}
|
||||
if doubanid:
|
||||
info = douban_info_func(
|
||||
doubanid=doubanid,
|
||||
mtype=mtype or (meta.type if meta else None),
|
||||
)
|
||||
elif not meta.name:
|
||||
logger.error("识别媒体信息时未提供元数据名称")
|
||||
return None
|
||||
else:
|
||||
# 读取缓存
|
||||
if mtype:
|
||||
meta.type = mtype
|
||||
if doubanid:
|
||||
meta.doubanid = doubanid
|
||||
cache_info = self.cache.get(meta) if cache else {}
|
||||
cache_hit = False
|
||||
|
||||
# 识别豆瓣信息
|
||||
if not cache_info or not cache:
|
||||
# 缓存没有或者强制不使用缓存
|
||||
if doubanid:
|
||||
# 直接查询详情
|
||||
info = douban_info_func(doubanid=doubanid, mtype=mtype or meta.type)
|
||||
elif meta:
|
||||
info = {}
|
||||
for name in self._prepare_search_names(meta):
|
||||
if meta.begin_season is not None:
|
||||
logger.info(f"正在识别 {name} 第{meta.begin_season}季 ...")
|
||||
else:
|
||||
logger.info(f"正在识别 {name} ...")
|
||||
# 匹配豆瓣信息
|
||||
match_info = match_doubaninfo_func(name=name,
|
||||
mtype=mtype or meta.type,
|
||||
year=meta.year,
|
||||
season=meta.begin_season)
|
||||
if match_info:
|
||||
# 匹配到豆瓣信息
|
||||
info = douban_info_func(
|
||||
doubanid=match_info.get("id"),
|
||||
mtype=mtype or meta.type
|
||||
)
|
||||
if info:
|
||||
break
|
||||
else:
|
||||
logger.error("识别媒体信息时未提供元数据或豆瓣ID")
|
||||
return None
|
||||
|
||||
# 保存到缓存
|
||||
if meta and cache:
|
||||
self.cache.update(meta, info)
|
||||
else:
|
||||
# 使用缓存信息
|
||||
cache_hit = True
|
||||
if cache_info.get("title"):
|
||||
logger.info(f"{meta.name} 使用豆瓣识别缓存：{cache_info.get('title')}")
|
||||
info = douban_info_func(mtype=cache_info.get("type"),
|
||||
doubanid=cache_info.get("id"))
|
||||
else:
|
||||
logger.info(f"{meta.name} 使用豆瓣识别缓存：无法识别")
|
||||
info = None
|
||||
info = {}
|
||||
for name in self._prepare_search_names(meta):
|
||||
if meta.begin_season is not None:
|
||||
logger.info(f"正在识别 {name} 第{meta.begin_season}季 ...")
|
||||
else:
|
||||
logger.info(f"正在识别 {name} ...")
|
||||
match_info = match_doubaninfo_func(
|
||||
name=name,
|
||||
mtype=mtype or meta.type,
|
||||
year=meta.year,
|
||||
season=meta.begin_season,
|
||||
)
|
||||
if match_info:
|
||||
info = douban_info_func(
|
||||
doubanid=match_info.get("id"),
|
||||
mtype=mtype or meta.type,
|
||||
)
|
||||
if info:
|
||||
break
|
||||
|
||||
if info:
|
||||
# 赋值TMDB信息并返回
|
||||
mediainfo = MediaInfo(douban_info=info)
|
||||
mediainfo.recognize_cache_hit = cache_hit
|
||||
if meta:
|
||||
logger.info(f"{meta.name} 豆瓣识别结果：{mediainfo.type.value} "
|
||||
f"{mediainfo.title_year} "
|
||||
@@ -213,7 +178,6 @@ class DoubanModule(_ModuleBase):
|
||||
async def _async_recognize_media_core(self, meta: MetaBase = None,
|
||||
mtype: MediaType = None,
|
||||
doubanid: Optional[str] = None,
|
||||
cache: Optional[bool] = True,
|
||||
async_douban_info_func=None,
|
||||
async_match_doubaninfo_func=None,
|
||||
**kwargs) -> Optional[MediaInfo]:
|
||||
@@ -222,7 +186,6 @@ class DoubanModule(_ModuleBase):
|
||||
:param meta: 识别的元数据
|
||||
:param mtype: 识别的媒体类型，与doubanid配套
|
||||
:param doubanid: 豆瓣ID
|
||||
:param cache: 是否使用缓存
|
||||
:param async_douban_info_func: 获取豆瓣信息的异步函数
|
||||
:param async_match_doubaninfo_func: 匹配豆瓣信息的异步函数
|
||||
:return: 识别的媒体信息，包括剧集信息
|
||||
@@ -237,69 +200,39 @@ class DoubanModule(_ModuleBase):
|
||||
):
|
||||
return None
|
||||
|
||||
if not meta:
|
||||
# 未提供元数据时，直接查询豆瓣信息，不使用缓存
|
||||
cache_info = {}
|
||||
if doubanid:
|
||||
info = await async_douban_info_func(
|
||||
doubanid=doubanid,
|
||||
mtype=mtype or (meta.type if meta else None),
|
||||
)
|
||||
elif not meta.name:
|
||||
logger.error("识别媒体信息时未提供元数据名称")
|
||||
return None
|
||||
else:
|
||||
# 读取缓存
|
||||
if mtype:
|
||||
meta.type = mtype
|
||||
if doubanid:
|
||||
meta.doubanid = doubanid
|
||||
cache_info = self.cache.get(meta) if cache else {}
|
||||
cache_hit = False
|
||||
|
||||
# 识别豆瓣信息
|
||||
if not cache_info or not cache:
|
||||
# 缓存没有或者强制不使用缓存
|
||||
if doubanid:
|
||||
# 直接查询详情
|
||||
info = await async_douban_info_func(doubanid=doubanid, mtype=mtype or meta.type)
|
||||
elif meta:
|
||||
info = {}
|
||||
for name in self._prepare_search_names(meta):
|
||||
if meta.begin_season is not None:
|
||||
logger.info(f"正在识别 {name} 第{meta.begin_season}季 ...")
|
||||
else:
|
||||
logger.info(f"正在识别 {name} ...")
|
||||
# 匹配豆瓣信息
|
||||
match_info = await async_match_doubaninfo_func(name=name,
|
||||
mtype=mtype or meta.type,
|
||||
year=meta.year,
|
||||
season=meta.begin_season)
|
||||
if match_info:
|
||||
# 匹配到豆瓣信息
|
||||
info = await async_douban_info_func(
|
||||
doubanid=match_info.get("id"),
|
||||
mtype=mtype or meta.type
|
||||
)
|
||||
if info:
|
||||
break
|
||||
else:
|
||||
logger.error("识别媒体信息时未提供元数据或豆瓣ID")
|
||||
return None
|
||||
|
||||
# 保存到缓存
|
||||
if meta and cache:
|
||||
self.cache.update(meta, info)
|
||||
else:
|
||||
# 使用缓存信息
|
||||
cache_hit = True
|
||||
if cache_info.get("title"):
|
||||
logger.info(f"{meta.name} 使用豆瓣识别缓存：{cache_info.get('title')}")
|
||||
info = await async_douban_info_func(mtype=cache_info.get("type"),
|
||||
doubanid=cache_info.get("id"))
|
||||
else:
|
||||
logger.info(f"{meta.name} 使用豆瓣识别缓存：无法识别")
|
||||
info = None
|
||||
info = {}
|
||||
for name in self._prepare_search_names(meta):
|
||||
if meta.begin_season is not None:
|
||||
logger.info(f"正在识别 {name} 第{meta.begin_season}季 ...")
|
||||
else:
|
||||
logger.info(f"正在识别 {name} ...")
|
||||
match_info = await async_match_doubaninfo_func(
|
||||
name=name,
|
||||
mtype=mtype or meta.type,
|
||||
year=meta.year,
|
||||
season=meta.begin_season,
|
||||
)
|
||||
if match_info:
|
||||
info = await async_douban_info_func(
|
||||
doubanid=match_info.get("id"),
|
||||
mtype=mtype or meta.type,
|
||||
)
|
||||
if info:
|
||||
break
|
||||
|
||||
if info:
|
||||
# 赋值TMDB信息并返回
|
||||
mediainfo = MediaInfo(douban_info=info)
|
||||
mediainfo.recognize_cache_hit = cache_hit
|
||||
if meta:
|
||||
logger.info(f"{meta.name} 豆瓣识别结果：{mediainfo.type.value} "
|
||||
f"{mediainfo.title_year} "
|
||||
@@ -316,21 +249,18 @@ class DoubanModule(_ModuleBase):
|
||||
def recognize_media(self, meta: MetaBase = None,
|
||||
mtype: MediaType = None,
|
||||
doubanid: Optional[str] = None,
|
||||
cache: Optional[bool] = True,
|
||||
**kwargs) -> Optional[MediaInfo]:
|
||||
"""
|
||||
识别媒体信息
|
||||
:param meta: 识别的元数据
|
||||
:param mtype: 识别的媒体类型，与doubanid配套
|
||||
:param doubanid: 豆瓣ID
|
||||
:param cache: 是否使用缓存
|
||||
:return: 识别的媒体信息，包括剧集信息
|
||||
"""
|
||||
return self._recognize_media_core(
|
||||
meta=meta,
|
||||
mtype=mtype,
|
||||
doubanid=doubanid,
|
||||
cache=cache,
|
||||
douban_info_func=self.douban_info,
|
||||
match_doubaninfo_func=self.match_doubaninfo,
|
||||
**kwargs
|
||||
@@ -339,51 +269,23 @@ class DoubanModule(_ModuleBase):
|
||||
async def async_recognize_media(self, meta: MetaBase = None,
|
||||
mtype: MediaType = None,
|
||||
doubanid: Optional[str] = None,
|
||||
cache: Optional[bool] = True,
|
||||
**kwargs) -> Optional[MediaInfo]:
|
||||
"""
|
||||
识别媒体信息（异步版本）
|
||||
:param meta: 识别的元数据
|
||||
:param mtype: 识别的媒体类型，与doubanid配套
|
||||
:param doubanid: 豆瓣ID
|
||||
:param cache: 是否使用缓存
|
||||
:return: 识别的媒体信息，包括剧集信息
|
||||
"""
|
||||
return await self._async_recognize_media_core(
|
||||
meta=meta,
|
||||
mtype=mtype,
|
||||
doubanid=doubanid,
|
||||
cache=cache,
|
||||
async_douban_info_func=self.async_douban_info,
|
||||
async_match_doubaninfo_func=self.async_match_doubaninfo,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
def update_recognize_cache(
|
||||
self,
|
||||
meta: MetaBase,
|
||||
mediainfo: MediaInfo,
|
||||
) -> Optional[bool]:
|
||||
"""
|
||||
回填豆瓣本地识别缓存，覆盖名称负缓存，避免共享识别后重复回查。
|
||||
"""
|
||||
if not meta or not mediainfo:
|
||||
return None
|
||||
if mediainfo.source != "douban" or not mediainfo.douban_info:
|
||||
return None
|
||||
self.cache.update(meta, mediainfo.douban_info)
|
||||
return True
|
||||
|
||||
async def async_update_recognize_cache(
|
||||
self,
|
||||
meta: MetaBase,
|
||||
mediainfo: MediaInfo,
|
||||
) -> Optional[bool]:
|
||||
"""
|
||||
异步回填豆瓣本地识别缓存。
|
||||
"""
|
||||
return self.update_recognize_cache(meta=meta, mediainfo=mediainfo)
|
||||
|
||||
@rate_limit_exponential(source="douban_info")
|
||||
def douban_info(self, doubanid: str, mtype: MediaType = None, raise_exception: bool = True) -> Optional[dict]:
|
||||
"""
|
||||
@@ -1272,7 +1174,6 @@ class DoubanModule(_ModuleBase):
|
||||
"""
|
||||
logger.info("开始清除豆瓣缓存 ...")
|
||||
self.doubanapi.clear_cache()
|
||||
self.cache.clear()
|
||||
logger.info("豆瓣缓存清除完成")
|
||||
|
||||
def douban_movie_credits(self, doubanid: str) -> List[schemas.MediaPerson]:
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
import pickle
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Optional
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.config import settings
|
||||
from app.core.meta import MetaBase
|
||||
from app.core.metainfo import MetaInfo
|
||||
from app.log import logger
|
||||
from app.schemas.types import MediaType
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
lock = RLock()
|
||||
|
||||
|
||||
class DoubanCache(metaclass=WeakSingleton):
|
||||
"""
|
||||
豆瓣缓存数据
|
||||
{
|
||||
"id": '',
|
||||
"title": '',
|
||||
"year": '',
|
||||
"type": MediaType
|
||||
}
|
||||
"""
|
||||
# 豆瓣缓存过期
|
||||
_douban_cache_expire: bool = True
|
||||
|
||||
def __init__(self):
|
||||
"""初始化豆瓣识别缓存并恢复本地持久化数据。"""
|
||||
self.maxsize = settings.CONF.douban
|
||||
self.ttl = settings.CONF.meta
|
||||
self.region = "__douban_cache__"
|
||||
self._meta_filepath = settings.TEMP_PATH / self.region
|
||||
# 初始化缓存
|
||||
self._cache = TTLCache(region=self.region, maxsize=self.maxsize, ttl=self.ttl)
|
||||
# 非Redis加载本地缓存数据
|
||||
if not self._cache.is_redis():
|
||||
for key, value in self.__load(self._meta_filepath).items():
|
||||
self._cache.set(key, value)
|
||||
|
||||
def clear(self):
|
||||
"""
|
||||
清空所有豆瓣缓存
|
||||
"""
|
||||
with lock:
|
||||
self._cache.clear()
|
||||
self.save(force=True)
|
||||
|
||||
def list_items(self) -> list[dict]:
|
||||
"""返回可供管理界面展示的豆瓣识别缓存列表。"""
|
||||
with lock:
|
||||
cache_items = []
|
||||
for key, value in self._cache.items():
|
||||
if not isinstance(value, dict):
|
||||
continue
|
||||
media_type = value.get("type")
|
||||
if not isinstance(media_type, MediaType):
|
||||
try:
|
||||
media_type = MediaType(media_type)
|
||||
except (TypeError, ValueError):
|
||||
media_type = None
|
||||
cache_items.append({
|
||||
"key": key,
|
||||
"douban_id": value.get("id") or 0,
|
||||
"title": value.get("title") or "",
|
||||
"year": value.get("year") or "",
|
||||
"media_type": media_type.to_agent() if media_type else "unknown",
|
||||
"poster_path": value.get("poster_path") or "",
|
||||
})
|
||||
return sorted(cache_items, key=lambda item: item["key"])
|
||||
|
||||
@staticmethod
|
||||
def __get_key(meta: MetaBase) -> str:
|
||||
"""
|
||||
获取缓存KEY
|
||||
"""
|
||||
return f"[{meta.type.value if meta.type else '未知'}]" \
|
||||
f"{meta.doubanid or meta.name}-{meta.year}-{meta.begin_season}"
|
||||
|
||||
def get(self, meta: MetaBase):
|
||||
"""
|
||||
根据KEY值获取缓存值
|
||||
"""
|
||||
key = self.__get_key(meta)
|
||||
with lock:
|
||||
return self._cache.get(key) or {}
|
||||
|
||||
def delete(self, key: str) -> dict:
|
||||
"""
|
||||
删除缓存信息
|
||||
@param key: 缓存key
|
||||
@return: 被删除的缓存内容
|
||||
"""
|
||||
with lock:
|
||||
redis_data = self._cache.get(key)
|
||||
if redis_data:
|
||||
self._cache.delete(key)
|
||||
self.save(force=True)
|
||||
return redis_data
|
||||
return {}
|
||||
|
||||
def modify(self, key: str, title: str) -> dict:
|
||||
"""
|
||||
修改缓存信息
|
||||
@param key: 缓存key
|
||||
@param title: 标题
|
||||
@return: 被修改后缓存内容
|
||||
"""
|
||||
with lock:
|
||||
redis_data = self._cache.get(key)
|
||||
if redis_data:
|
||||
redis_data["title"] = title
|
||||
self._cache.set(key, redis_data)
|
||||
return redis_data
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def __load(path: Path) -> dict:
|
||||
"""
|
||||
从文件中加载缓存
|
||||
"""
|
||||
try:
|
||||
if path.exists():
|
||||
with open(path, 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.error(f"加载缓存失败: {str(e)} - {traceback.format_exc()}")
|
||||
return {}
|
||||
|
||||
def update(self, meta: MetaBase, info: dict) -> None:
|
||||
"""
|
||||
新增或更新缓存条目
|
||||
"""
|
||||
if info:
|
||||
# 缓存标题
|
||||
cache_title = info.get("title")
|
||||
# 缓存年份
|
||||
cache_year = info.get('year')
|
||||
# 类型
|
||||
if isinstance(info.get('media_type'), MediaType):
|
||||
mtype = info.get('media_type')
|
||||
elif info.get("type"):
|
||||
mtype = MediaType.MOVIE if info.get("type") == "movie" else MediaType.TV
|
||||
else:
|
||||
meta = MetaInfo(cache_title)
|
||||
if meta.begin_season is not None:
|
||||
mtype = MediaType.TV
|
||||
else:
|
||||
mtype = MediaType.MOVIE
|
||||
# 海报
|
||||
poster_path = info.get("pic", {}).get("large")
|
||||
if not poster_path and info.get("cover_url"):
|
||||
poster_path = info.get("cover_url")
|
||||
if not poster_path and info.get("cover"):
|
||||
poster_path = info.get("cover").get("url")
|
||||
|
||||
with lock:
|
||||
self._cache.set(self.__get_key(meta), {
|
||||
"id": info.get("id"),
|
||||
"type": mtype,
|
||||
"year": cache_year,
|
||||
"title": cache_title,
|
||||
"poster_path": poster_path
|
||||
})
|
||||
|
||||
elif info is not None:
|
||||
# None时不缓存，此时代表网络错误，允许重复请求
|
||||
with lock:
|
||||
self._cache.set(self.__get_key(meta), {
|
||||
"id": 0
|
||||
})
|
||||
|
||||
def save(self, force: Optional[bool] = False) -> None:
|
||||
"""
|
||||
保存缓存数据到文件
|
||||
"""
|
||||
# Redis不需要保存到本地文件
|
||||
if self._cache.is_redis():
|
||||
return
|
||||
|
||||
# 本地文件
|
||||
meta_data = self.__load(self._meta_filepath)
|
||||
# 当前缓存数据（去除无法识别）
|
||||
new_meta_data = {k: v for k, v in self._cache.items() if v.get("id")}
|
||||
|
||||
if not force \
|
||||
and meta_data.keys() == new_meta_data.keys():
|
||||
return
|
||||
# 写入本地
|
||||
with open(self._meta_filepath, 'wb') as f:
|
||||
pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # noqa
|
||||
|
||||
def __del__(self):
|
||||
"""实例释放前保存非 Redis 缓存。"""
|
||||
self.save()
|
||||
@@ -1,9 +1,10 @@
|
||||
import pickle
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
from math import ceil
|
||||
from threading import RLock
|
||||
from time import time
|
||||
|
||||
from app.core.cache import TTLCache
|
||||
from app.core.cache import FileCache, TTLCache
|
||||
from app.core.config import settings
|
||||
from app.core.meta import MetaBase
|
||||
from app.log import logger
|
||||
@@ -11,6 +12,9 @@ from app.schemas.types import MediaType
|
||||
from app.utils.singleton import WeakSingleton
|
||||
|
||||
lock = RLock()
|
||||
PERSISTENCE_VERSION = 1
|
||||
PERSISTENCE_REGION = "recognize"
|
||||
PERSISTENCE_KEY = "tmdb"
|
||||
|
||||
|
||||
class TmdbCache(metaclass=WeakSingleton):
|
||||
@@ -23,21 +27,78 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
"type": MediaType
|
||||
}
|
||||
"""
|
||||
# TMDB缓存过期
|
||||
_tmdb_cache_expire: bool = True
|
||||
|
||||
def __init__(self):
|
||||
"""初始化 TMDB 识别缓存并恢复本地持久化数据。"""
|
||||
self.maxsize = settings.CONF.douban
|
||||
"""初始化 TMDB 识别缓存并恢复未过期的持久化数据。"""
|
||||
self.maxsize = settings.CONF.tmdb
|
||||
self.ttl = settings.CONF.meta
|
||||
self.region = "__tmdb_cache__"
|
||||
self._meta_filepath = settings.TEMP_PATH / self.region
|
||||
# 初始化缓存
|
||||
self._cache = TTLCache(region=self.region, maxsize=self.maxsize, ttl=self.ttl)
|
||||
# 非Redis加载本地缓存数据
|
||||
self._expires_at: dict[str, float] = {}
|
||||
self._dirty = False
|
||||
self._file_cache = None
|
||||
self._legacy_file_cache = None
|
||||
self._legacy_cache_found = False
|
||||
if not self._cache.is_redis():
|
||||
for key, value in self.__load(self._meta_filepath).items():
|
||||
self._cache.set(key, value)
|
||||
self._file_cache = FileCache(base=settings.CACHE_PATH, ttl=self.ttl)
|
||||
self._legacy_file_cache = FileCache(base=settings.TEMP_PATH.parent, ttl=self.ttl)
|
||||
self._restore()
|
||||
|
||||
def _restore(self) -> None:
|
||||
"""从统一文件缓存恢复仍在有效期内的 TMDB 识别数据。"""
|
||||
try:
|
||||
content = self._file_cache.get(PERSISTENCE_KEY, region=PERSISTENCE_REGION)
|
||||
if not content:
|
||||
content = self._legacy_file_cache.get(
|
||||
self.region,
|
||||
region=settings.TEMP_PATH.name,
|
||||
)
|
||||
if content:
|
||||
self._legacy_cache_found = True
|
||||
self._dirty = True
|
||||
if not content:
|
||||
return
|
||||
payload = pickle.loads(content)
|
||||
now = time()
|
||||
if (
|
||||
isinstance(payload, dict)
|
||||
and payload.get("version") == PERSISTENCE_VERSION
|
||||
and isinstance(payload.get("items"), dict)
|
||||
):
|
||||
items = payload["items"]
|
||||
elif isinstance(payload, dict):
|
||||
# 旧版缓存没有保存过期时间，迁移时从当前时刻重新计算一次有效期。
|
||||
items = {
|
||||
key: {"value": value, "expires_at": now + self.ttl}
|
||||
for key, value in payload.items()
|
||||
}
|
||||
self._dirty = True
|
||||
else:
|
||||
return
|
||||
|
||||
for key, item in items.items():
|
||||
if not isinstance(item, dict):
|
||||
self._dirty = True
|
||||
continue
|
||||
value = item.get("value")
|
||||
expires_at = item.get("expires_at")
|
||||
if not isinstance(value, dict) or not isinstance(expires_at, (int, float)):
|
||||
self._dirty = True
|
||||
continue
|
||||
remaining_ttl = expires_at - now
|
||||
if remaining_ttl <= 0:
|
||||
self._dirty = True
|
||||
continue
|
||||
self._cache.set(key, value, ttl=ceil(remaining_ttl))
|
||||
self._expires_at[key] = expires_at
|
||||
except Exception as err:
|
||||
logger.error(f"加载TMDB识别缓存失败：{str(err)} - {traceback.format_exc()}")
|
||||
|
||||
def _set(self, key: str, value: dict) -> None:
|
||||
"""写入单条 TMDB 识别缓存并记录其独立过期时间。"""
|
||||
self._cache.set(key, value)
|
||||
if not self._cache.is_redis():
|
||||
self._expires_at[key] = time() + self.ttl
|
||||
self._dirty = True
|
||||
|
||||
def clear(self):
|
||||
"""
|
||||
@@ -45,6 +106,8 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
"""
|
||||
with lock:
|
||||
self._cache.clear()
|
||||
self._expires_at.clear()
|
||||
self._dirty = True
|
||||
self.save(force=True)
|
||||
|
||||
def list_items(self) -> list[dict]:
|
||||
@@ -87,7 +150,10 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
key = self.__get_key(meta)
|
||||
|
||||
with lock:
|
||||
return self._cache.get(key) or {}
|
||||
cache_data = self._cache.get(key)
|
||||
if not cache_data and self._expires_at.pop(key, None) is not None:
|
||||
self._dirty = True
|
||||
return cache_data or {}
|
||||
|
||||
def delete(self, key: str) -> dict:
|
||||
"""
|
||||
@@ -99,6 +165,8 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
redis_data = self._cache.get(key)
|
||||
if redis_data:
|
||||
self._cache.delete(key)
|
||||
self._expires_at.pop(key, None)
|
||||
self._dirty = True
|
||||
self.save(force=True)
|
||||
return redis_data
|
||||
return {}
|
||||
@@ -114,24 +182,10 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
redis_data = self._cache.get(key)
|
||||
if redis_data:
|
||||
redis_data['title'] = title
|
||||
self._cache.set(key, redis_data)
|
||||
self._set(key, redis_data)
|
||||
return redis_data
|
||||
return {}
|
||||
|
||||
@staticmethod
|
||||
def __load(path: Path) -> dict:
|
||||
"""
|
||||
从文件中加载缓存
|
||||
"""
|
||||
try:
|
||||
if path.exists():
|
||||
with open(path, 'rb') as f:
|
||||
data = pickle.load(f)
|
||||
return data
|
||||
except Exception as e:
|
||||
logger.error(f'加载缓存失败：{str(e)} - {traceback.format_exc()}')
|
||||
return {}
|
||||
|
||||
def update(self, meta: MetaBase, info: dict) -> None:
|
||||
"""
|
||||
新增或更新缓存条目
|
||||
@@ -157,32 +211,68 @@ class TmdbCache(metaclass=WeakSingleton):
|
||||
"poster_path": info.get("poster_path"),
|
||||
"backdrop_path": info.get("backdrop_path")
|
||||
}
|
||||
self._cache.set(key, cache_data)
|
||||
self._set(key, cache_data)
|
||||
|
||||
elif info is not None:
|
||||
# None时不缓存，此时代表网络错误，允许重复请求
|
||||
with lock:
|
||||
self._cache.set(key, {"id": 0})
|
||||
self._set(key, {"id": 0})
|
||||
|
||||
def save(self, force: bool = False) -> None:
|
||||
"""
|
||||
保存缓存数据到文件
|
||||
使用统一文件缓存保存未过期的 TMDB 识别数据。
|
||||
"""
|
||||
# Redis不需要保存到本地文件
|
||||
if self._cache.is_redis():
|
||||
return
|
||||
with lock:
|
||||
now = time()
|
||||
cache_items = dict(self._cache.items())
|
||||
active_keys = set(cache_items)
|
||||
stale_keys = set(self._expires_at) - active_keys
|
||||
if stale_keys:
|
||||
for key in stale_keys:
|
||||
self._expires_at.pop(key, None)
|
||||
self._dirty = True
|
||||
|
||||
# Redis不可用时，保存到本地文件
|
||||
meta_data = self.__load(self._meta_filepath)
|
||||
# 当前缓存，去除无法识别
|
||||
new_meta_data = {k: v for k, v in self._cache.items() if v.get("id")}
|
||||
persisted_items = {}
|
||||
for key, value in cache_items.items():
|
||||
expires_at = self._expires_at.get(key)
|
||||
if expires_at is None:
|
||||
expires_at = now + self.ttl
|
||||
self._expires_at[key] = expires_at
|
||||
self._dirty = True
|
||||
if expires_at <= now or not value.get("id"):
|
||||
continue
|
||||
persisted_items[key] = {
|
||||
"value": value,
|
||||
"expires_at": expires_at,
|
||||
}
|
||||
|
||||
if not force \
|
||||
and meta_data.keys() == new_meta_data.keys():
|
||||
return
|
||||
if not force and not self._dirty:
|
||||
return
|
||||
|
||||
with open(self._meta_filepath, 'wb') as f:
|
||||
pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # type: ignore
|
||||
try:
|
||||
if persisted_items:
|
||||
payload = {
|
||||
"version": PERSISTENCE_VERSION,
|
||||
"items": persisted_items,
|
||||
}
|
||||
self._file_cache.set(
|
||||
PERSISTENCE_KEY,
|
||||
pickle.dumps(payload, pickle.HIGHEST_PROTOCOL),
|
||||
region=PERSISTENCE_REGION,
|
||||
)
|
||||
else:
|
||||
self._file_cache.delete(PERSISTENCE_KEY, region=PERSISTENCE_REGION)
|
||||
if self._legacy_cache_found:
|
||||
self._legacy_file_cache.delete(
|
||||
self.region,
|
||||
region=settings.TEMP_PATH.name,
|
||||
)
|
||||
self._legacy_cache_found = False
|
||||
self._dirty = False
|
||||
except Exception as err:
|
||||
logger.error(f"保存TMDB识别缓存失败：{str(err)} - {traceback.format_exc()}")
|
||||
|
||||
def __del__(self):
|
||||
"""实例释放前保存非 Redis 缓存。"""
|
||||
|
||||
Reference in New Issue
Block a user