From a29f9876492b80e01574e82e6c7ec1825b56ed47 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sat, 18 Jan 2025 02:10:17 +0800 Subject: [PATCH 1/5] feat(cache): add cache backend implementations and decorator support --- app/core/cache.py | 320 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 320 insertions(+) create mode 100644 app/core/cache.py diff --git a/app/core/cache.py b/app/core/cache.py new file mode 100644 index 00000000..817d5065 --- /dev/null +++ b/app/core/cache.py @@ -0,0 +1,320 @@ +import inspect +import os +from abc import ABC, abstractmethod +from collections import defaultdict +from functools import wraps +from typing import Any, Dict, Optional + +import redis +from cachetools import TTLCache +from cachetools.keys import hashkey + + +class CacheBackend(ABC): + """ + 缓存后端基类,定义通用的缓存接口 + """ + + @abstractmethod + def set(self, key: str, value: Any, ttl: int, region: str = "default", **kwargs) -> None: + """ + 设置缓存 + + :param key: 缓存的键 + :param value: 缓存的值 + :param ttl: 缓存的存活时间,单位秒 + :param region: 缓存的区 + :param kwargs: 其他参数 + """ + pass + + @abstractmethod + def get(self, key: str, region: str = "default") -> Any: + """ + 获取缓存 + + :param key: 缓存的键 + :param region: 缓存的区 + :return: 返回缓存的值,如果缓存不存在返回 None + """ + pass + + @abstractmethod + def delete(self, key: str, region: str = "default") -> None: + """ + 删除缓存 + + :param key: 缓存的键 + :param region: 缓存的区 + """ + pass + + @abstractmethod + def clear(self, region: Optional[str] = None) -> None: + """ + 清除指定区域的缓存或全部缓存 + + :param region: 缓存的区 + """ + pass + + @staticmethod + def get_region(region: str = "default"): + """ + 获取缓存的区 + """ + return f"region:{region}" if region else "region:default" + + +class CacheToolsBackend(CacheBackend): + """ + 基于 `cachetools.TTLCache` 实现的缓存后端,支持动态 TTL 和 Maxsize + """ + + def __init__(self, maxsize: int = 1000, ttl: int = 1800): + """ + 初始化缓存实例 + + :param maxsize: 缓存的最大条目数 + :param ttl: 默认缓存存活时间,单位秒 + """ + self.maxsize = maxsize + self.ttl = ttl + # 存储各个 region 的缓存实例,region -> {key -> TTLCache} + self._region_caches: Dict[str, Dict[str, TTLCache]] = defaultdict(dict) + + def set(self, key: str, value: Any, ttl: int = None, region: str = "default", **kwargs) -> None: + """ + 设置缓存值支持每个 key 独立配置 TTL 和 Maxsize + + :param key: 缓存的键 + :param value: 缓存的值 + :param ttl: 缓存的存活时间,单位秒如果未传入则使用默认值 + :param region: 缓存的区 + :param kwargs: maxsize: 缓存的最大条目数如果未传入则使用默认值 + """ + ttl = ttl or self.ttl + maxsize = kwargs.get("maxsize", self.maxsize) + region = self.get_region(region) + # 如果该 key 尚未有缓存实例,则创建一个新的 TTLCache 实例 + region_cache = self._region_caches[region] + if key not in region_cache: + region_cache[key] = TTLCache(maxsize=maxsize, ttl=ttl) + # 为每个 key 获取独立的缓存实例 + cache = region_cache[key] + # 设置缓存值 + cache[key] = value + + def get(self, key: str, region: str = "default") -> Any: + """ + 获取缓存的值 + + :param key: 缓存的键 + :param region: 缓存的区 + :return: 返回缓存的值,如果缓存不存在返回 None + """ + region = self.get_region(region) + region_cache = self._region_caches[region] + if key not in region_cache: + return None + # 获取缓存实例并返回缓存值 + cache = region_cache[key] + return cache.get(key) + + def delete(self, key: str, region: str = "default") -> None: + """ + 删除缓存 + + :param key: 缓存的键 + :param region: 缓存的区 + """ + region = self.get_region(region) + region_cache = self._region_caches[region] + if key not in region_cache: + return None + # 获取缓存实例并删除指定的缓存 + cache = region_cache[key] + del cache[key] + + def clear(self, region: Optional[str] = None) -> None: + """ + 清除指定区域的缓存或全部缓存 + + :param region: 缓存的区 + """ + if region: + region_cache = self._region_caches[region] + for cache in region_cache.values(): + cache.clear() + else: + for region_cache in self._region_caches.values(): + for cache in region_cache.values(): + cache.clear() + + +class RedisBackend(CacheBackend): + """ + 基于 Redis 实现的缓存后端,支持通过 Redis 存储缓存 + """ + + def __init__(self, redis_url: str = "redis://localhost", ttl: int = 1800): + """ + 初始化 Redis 缓存实例 + + :param redis_url: Redis 服务的 URL + :param ttl: 缓存的存活时间,单位秒 + """ + self.redis_url = redis_url + self.ttl = ttl + self.client = redis.StrictRedis.from_url(redis_url) + + @staticmethod + def get_redis_key(region, key): + """ + 获取缓存 Key + """ + # 使用 region 作为缓存键的一部分 + return f"region:{region}:key:{key}" + + def set(self, key: str, value: Any, ttl: int = None, region: str = "default", **kwargs) -> None: + """ + 设置缓存 + + :param key: 缓存的键 + :param value: 缓存的值 + :param ttl: 缓存的存活时间,单位秒如果未传入则使用默认值 + :param region: 缓存的区 + :param kwargs: kwargs + """ + ttl = ttl or self.ttl + redis_key = self.get_redis_key(region, key) + self.client.setex(redis_key, ttl, value) + + def get(self, key: str, region: str = "default") -> Any: + """ + 获取缓存的值 + + :param key: 缓存的键 + :param region: 缓存的区 + :return: 返回缓存的值,如果缓存不存在返回 None + """ + redis_key = self.get_redis_key(region, key) + value = self.client.get(redis_key) + return value + + def delete(self, key: str, region: str = "default") -> None: + """ + 删除缓存 + + :param key: 缓存的键 + :param region: 缓存的区 + """ + redis_key = self.get_redis_key(region, key) + self.client.delete(redis_key) + + def clear(self, region: Optional[str] = None) -> None: + """ + 清除 Redis 中指定区域的缓存或全部缓存 + + :param region: 缓存的区 + """ + if region: + # 清除指定区域的所有键 + pattern = f"{region}:*" + keys = list(self.client.keys(pattern)) + if keys: + self.client.delete(*keys) + else: + # 清除所有缓存 + self.client.flushdb() + + +def get_cache_backend(maxsize: int = 1000, ttl: int = 1800) -> CacheBackend: + """ + 根据配置获取缓存后端实例 + + :param maxsize: 缓存的最大条目数 + :param ttl: 缓存的默认存活时间,单位秒 + :return: 返回缓存后端实例 + """ + cache_type = os.getenv("CACHE_TYPE", "cachetools").lower() + + if cache_type == "redis": + return RedisBackend(redis_url=os.getenv("REDIS_URL", "redis://localhost")) + return CacheToolsBackend(maxsize=maxsize, ttl=ttl) + + +# 缓存后端实例 +cache_backend = get_cache_backend() + + +def cached(region: str = "default", maxsize: int = 1000, ttl: int = 1800, + skip_none: bool = True, skip_empty: bool = True): + """ + 自定义缓存装饰器,支持为每个 key 动态传递 maxsize 和 ttl + + :param region: 缓存的区 + :param maxsize: 缓存的最大条目数,默认值为 1000 + :param ttl: 缓存的存活时间,单位秒,默认值为 1800 + :param skip_none: 跳过 None 缓存,默认为 True + :param skip_empty: 跳过空值缓存(如 [], {}, "", set()),默认为 True + :return: 装饰器函数 + """ + + def should_cache(value: Any) -> bool: + """ + 判断是否应该缓存结果,如果返回值是 None 或空值则不缓存 + + :param value: 要判断的缓存值 + :return: 是否缓存结果 + """ + if skip_none and value is None: + return False + # if disable_empty and value in [[], {}, "", set()]: + if skip_empty and not value: + return False + return True + + def get_cache_key(func, args, kwargs): + """ + 获取缓存的键,通过哈希函数对函数的参数进行处理 + :param func: 被装饰的函数 + :param args: 位置参数 + :param kwargs: 关键字参数 + :return: 缓存键 + """ + # 获取方法签名 + signature = inspect.signature(func) + resolved_kwargs = {} + # 获取默认值并结合传递的参数(如果有) + for param, value in signature.parameters.items(): + if param in kwargs: + # 使用显式传递的参数 + resolved_kwargs[param] = kwargs[param] + elif value.default is not inspect.Parameter.empty: + # 没有传递参数时使用默认值 + resolved_kwargs[param] = value.default + # 构造缓存键 + return f"{func.__name__}_{hashkey(*args, **resolved_kwargs)}" + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # 获取缓存键 + cache_key = get_cache_key(func, args, kwargs) + # 尝试获取缓存 + cached_value = cache_backend.get(cache_key, region=region) + if should_cache(cached_value): + return cached_value + # 执行函数并缓存结果 + result = func(*args, **kwargs) + # 判断是否需要缓存 + if not should_cache(result): + return result + # 设置缓存(如果有传入的 maxsize 和 ttl,则覆盖默认值) + cache_backend.set(cache_key, result, ttl=ttl, maxsize=maxsize, region=region) + return result + + return wrapper + + return decorator From 11d4f272686387d78d1b9d6e7dca77b45a80829c Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sat, 18 Jan 2025 02:12:20 +0800 Subject: [PATCH 2/5] feat(cache): migrate cachetools usage to unified cache backend --- app/chain/mediaserver.py | 5 ++--- app/chain/tmdb.py | 7 +++---- app/modules/bangumi/bangumi.py | 7 ++++--- app/modules/douban/apiv2.py | 8 ++++---- app/modules/fanart/__init__.py | 8 +++----- app/modules/filemanager/storages/alist.py | 4 ++-- app/modules/plex/plex.py | 6 +++--- app/modules/themoviedb/tmdbapi.py | 8 ++++---- app/modules/themoviedb/tmdbv3api/objs/discover.py | 6 +++--- app/modules/themoviedb/tmdbv3api/objs/trending.py | 4 ++-- app/modules/themoviedb/tmdbv3api/tmdb.py | 4 ++-- app/utils/web.py | 6 +++--- requirements.in | 3 ++- 13 files changed, 37 insertions(+), 39 deletions(-) diff --git a/app/chain/mediaserver.py b/app/chain/mediaserver.py index 07dba726..ead78f89 100644 --- a/app/chain/mediaserver.py +++ b/app/chain/mediaserver.py @@ -1,9 +1,8 @@ import threading from typing import List, Union, Optional, Generator -from cachetools import cached, TTLCache - from app.chain import ChainBase +from app.core.cache import cached from app.core.config import global_vars from app.db.mediaserver_oper import MediaServerOper from app.helper.service import ServiceConfigHelper @@ -94,7 +93,7 @@ class MediaServerChain(ChainBase): """ return self.run_module("mediaserver_latest", count=count, server=server, username=username) - @cached(cache=TTLCache(maxsize=1, ttl=3600)) + @cached(maxsize=1, ttl=3600) def get_latest_wallpapers(self, server: str = None, count: int = 10, remote: bool = True, username: str = None) -> List[str]: """ diff --git a/app/chain/tmdb.py b/app/chain/tmdb.py index 6699b5b4..bd84aa94 100644 --- a/app/chain/tmdb.py +++ b/app/chain/tmdb.py @@ -1,10 +1,9 @@ import random from typing import Optional, List -from cachetools import cached, TTLCache - from app import schemas from app.chain import ChainBase +from app.core.cache import cached from app.core.context import MediaInfo from app.schemas import MediaType from app.utils.singleton import Singleton @@ -119,7 +118,7 @@ class TmdbChain(ChainBase, metaclass=Singleton): """ return self.run_module("tmdb_person_credits", person_id=person_id, page=page) - @cached(cache=TTLCache(maxsize=1, ttl=3600)) + @cached(maxsize=1, ttl=3600) def get_random_wallpager(self) -> Optional[str]: """ 获取随机壁纸,缓存1个小时 @@ -133,7 +132,7 @@ class TmdbChain(ChainBase, metaclass=Singleton): return info.backdrop_path return None - @cached(cache=TTLCache(maxsize=1, ttl=3600)) + @cached(maxsize=1, ttl=3600) def get_trending_wallpapers(self, num: int = 10) -> List[str]: """ 获取所有流行壁纸 diff --git a/app/modules/bangumi/bangumi.py b/app/modules/bangumi/bangumi.py index abf9330b..fc1db92b 100644 --- a/app/modules/bangumi/bangumi.py +++ b/app/modules/bangumi/bangumi.py @@ -1,8 +1,8 @@ from datetime import datetime import requests -from cachetools import TTLCache, cached +from app.core.cache import cached from app.core.config import settings from app.utils.http import RequestUtils @@ -29,7 +29,7 @@ class BangumiApi(object): pass @classmethod - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["bangumi"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["bangumi"], ttl=settings.CACHE_CONF["meta"]) def __invoke(cls, url, **kwargs): req_url = cls._base_url + url params = {} @@ -188,7 +188,8 @@ class BangumiApi(object): 获取人物参演作品 """ ret_list = [] - result = self.__invoke(self._urls["person_credits"] % person_id, _ts=datetime.strftime(datetime.now(), '%Y%m%d')) + result = self.__invoke(self._urls["person_credits"] % person_id, + _ts=datetime.strftime(datetime.now(), '%Y%m%d')) if result: for item in result: ret_list.append(item) diff --git a/app/modules/douban/apiv2.py b/app/modules/douban/apiv2.py index 56a21b2e..478c625e 100644 --- a/app/modules/douban/apiv2.py +++ b/app/modules/douban/apiv2.py @@ -7,8 +7,8 @@ from random import choice from urllib import parse import requests -from cachetools import TTLCache, cached +from app.core.cache import cached from app.core.config import settings from app.utils.http import RequestUtils from app.utils.singleton import Singleton @@ -174,14 +174,14 @@ class DoubanApi(metaclass=Singleton): ).digest() ).decode() - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]) def __invoke_recommend(self, url: str, **kwargs) -> dict: """ 推荐/发现类API """ return self.__invoke(url, **kwargs) - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]) def __invoke_search(self, url: str, **kwargs) -> dict: """ 搜索类API @@ -216,7 +216,7 @@ class DoubanApi(metaclass=Singleton): return resp.json() return resp.json() if resp else {} - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]) def __post(self, url: str, **kwargs) -> dict: """ POST请求 diff --git a/app/modules/fanart/__init__.py b/app/modules/fanart/__init__.py index abfe4ab9..93d13ecc 100644 --- a/app/modules/fanart/__init__.py +++ b/app/modules/fanart/__init__.py @@ -1,8 +1,7 @@ import re from typing import Optional, Tuple, Union -from cachetools import TTLCache, cached - +from app.core.cache import cached from app.core.context import MediaInfo, settings from app.log import logger from app.modules import _ModuleBase @@ -11,7 +10,6 @@ from app.utils.http import RequestUtils class FanartModule(_ModuleBase): - """ { "name": "The Wheel of Time", @@ -384,7 +382,7 @@ class FanartModule(_ModuleBase): continue if not isinstance(images, list): continue - + # 图片属性xx_path image_name = self.__name(name) if image_name.startswith("season"): @@ -422,7 +420,7 @@ class FanartModule(_ModuleBase): return result @classmethod - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"]) def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]: if media_type == MediaType.MOVIE: image_url = cls._movie_url % queryid diff --git a/app/modules/filemanager/storages/alist.py b/app/modules/filemanager/storages/alist.py index cc4eadfb..023c7a20 100644 --- a/app/modules/filemanager/storages/alist.py +++ b/app/modules/filemanager/storages/alist.py @@ -4,10 +4,10 @@ from datetime import datetime from pathlib import Path from typing import Optional, List, Dict -from cachetools import cached, TTLCache from requests import Response from app import schemas +from app.core.cache import cached from app.core.config import settings from app.log import logger from app.modules.filemanager.storages import StorageBase @@ -67,7 +67,7 @@ class Alist(StorageBase, metaclass=Singleton): return self.__generate_token @property - @cached(cache=TTLCache(maxsize=1, ttl=60 * 60 * 24 * 2 - 60 * 5)) + @cached(maxsize=1, ttl=60 * 60 * 24 * 2 - 60 * 5) def __generate_token(self) -> str: """ 使用账号密码生成一个临时token diff --git a/app/modules/plex/plex.py b/app/modules/plex/plex.py index b04d2753..48e4be62 100644 --- a/app/modules/plex/plex.py +++ b/app/modules/plex/plex.py @@ -3,13 +3,13 @@ from pathlib import Path from typing import List, Optional, Dict, Tuple, Generator, Any, Union from urllib.parse import quote_plus -from cachetools import TTLCache, cached from plexapi import media from plexapi.myplex import MyPlexAccount from plexapi.server import PlexServer from requests import Response, Session from app import schemas +from app.core.cache import cached from app.log import logger from app.schemas import MediaType from app.utils.http import RequestUtils @@ -83,7 +83,7 @@ class Plex: logger.error(f"Authentication failed: {e}") return None - @cached(cache=TTLCache(maxsize=100, ttl=86400)) + @cached(maxsize=100, ttl=86400) def __get_library_images(self, library_key: str, mtype: int) -> Optional[List[str]]: """ 获取媒体服务器最近添加的媒体的图片列表 @@ -293,7 +293,7 @@ class Plex: season_episodes[episode.seasonNumber].append(episode.index) return videos.key, season_episodes - def get_remote_image_by_id(self, + def get_remote_image_by_id(self, item_id: str, image_type: str, depth: int = 0, diff --git a/app/modules/themoviedb/tmdbapi.py b/app/modules/themoviedb/tmdbapi.py index b27ca808..178bd4f1 100644 --- a/app/modules/themoviedb/tmdbapi.py +++ b/app/modules/themoviedb/tmdbapi.py @@ -3,9 +3,9 @@ from typing import Optional, List from urllib.parse import quote import zhconv -from cachetools import TTLCache, cached from lxml import etree +from app.core.cache import cached from app.core.config import settings from app.log import logger from app.schemas.types import MediaType @@ -491,7 +491,7 @@ class TmdbApi: return ret_info - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]) def match_web(self, name: str, mtype: MediaType) -> Optional[dict]: """ 搜索TMDB网站,直接抓取结果,结果只有一条时才返回 @@ -678,14 +678,14 @@ class TmdbApi: else: en_title = __get_tmdb_lang_title(tmdb_info, "US") tmdb_info['en_title'] = en_title or org_title - + # 查找香港台湾译名 tmdb_info['hk_title'] = __get_tmdb_lang_title(tmdb_info, "HK") tmdb_info['tw_title'] = __get_tmdb_lang_title(tmdb_info, "TW") # 查找新加坡名(用于替代中文名) tmdb_info['sg_title'] = __get_tmdb_lang_title(tmdb_info, "SG") or org_title - + def __get_movie_detail(self, tmdbid: int, append_to_response: str = "images," diff --git a/app/modules/themoviedb/tmdbv3api/objs/discover.py b/app/modules/themoviedb/tmdbv3api/objs/discover.py index a79891a5..70f1ff41 100644 --- a/app/modules/themoviedb/tmdbv3api/objs/discover.py +++ b/app/modules/themoviedb/tmdbv3api/objs/discover.py @@ -1,5 +1,5 @@ +from app.core.cache import cached from ..tmdb import TMDb -from cachetools import cached, TTLCache try: from urllib import urlencode @@ -13,7 +13,7 @@ class Discover(TMDb): "tv": "/discover/tv" } - @cached(cache=TTLCache(maxsize=1, ttl=43200)) + @cached(maxsize=1, ttl=43200) def discover_movies(self, params_tuple): """ Discover movies by different types of data like average rating, number of votes, genres and certifications. @@ -23,7 +23,7 @@ class Discover(TMDb): params = dict(params_tuple) return self._request_obj(self._urls["movies"], urlencode(params), key="results", call_cached=False) - @cached(cache=TTLCache(maxsize=1, ttl=43200)) + @cached(maxsize=1, ttl=43200) def discover_tv_shows(self, params_tuple): """ Discover TV shows by different types of data like average rating, number of votes, genres, diff --git a/app/modules/themoviedb/tmdbv3api/objs/trending.py b/app/modules/themoviedb/tmdbv3api/objs/trending.py index 7338792b..49ba53bd 100644 --- a/app/modules/themoviedb/tmdbv3api/objs/trending.py +++ b/app/modules/themoviedb/tmdbv3api/objs/trending.py @@ -1,4 +1,4 @@ -from cachetools import cached, TTLCache +from app.core.cache import cached from ..tmdb import TMDb @@ -6,7 +6,7 @@ from ..tmdb import TMDb class Trending(TMDb): _urls = {"trending": "/trending/%s/%s"} - @cached(cache=TTLCache(maxsize=1, ttl=43200)) + @cached(maxsize=1, ttl=43200) def _trending(self, media_type="all", time_window="day", page=1): """ Get trending, TTLCache 12 hours diff --git a/app/modules/themoviedb/tmdbv3api/tmdb.py b/app/modules/themoviedb/tmdbv3api/tmdb.py index e6fb1bb0..02e164cd 100644 --- a/app/modules/themoviedb/tmdbv3api/tmdb.py +++ b/app/modules/themoviedb/tmdbv3api/tmdb.py @@ -7,8 +7,8 @@ from datetime import datetime import requests import requests.exceptions -from cachetools import TTLCache, cached +from app.core.cache import cached from app.core.config import settings from app.utils.http import RequestUtils from .exceptions import TMDbException @@ -137,7 +137,7 @@ class TMDb(object): def cache(self, cache): os.environ[self.TMDB_CACHE_ENABLED] = str(cache) - @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])) + @cached(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]) def cached_request(self, method, url, data, json, _ts=datetime.strftime(datetime.now(), '%Y%m%d')): """ diff --git a/app/utils/web.py b/app/utils/web.py index 6145adb7..9e83070c 100644 --- a/app/utils/web.py +++ b/app/utils/web.py @@ -1,6 +1,6 @@ from typing import Optional, List -from cachetools import TTLCache, cached +from app.core.cache import cached from app.utils.http import RequestUtils @@ -75,7 +75,7 @@ class WebUtils: return "" @staticmethod - @cached(cache=TTLCache(maxsize=1, ttl=3600)) + @cached(maxsize=1, ttl=3600) def get_bing_wallpaper() -> Optional[str]: """ 获取Bing每日壁纸 @@ -93,7 +93,7 @@ class WebUtils: return None @staticmethod - @cached(cache=TTLCache(maxsize=1, ttl=3600)) + @cached(maxsize=1, ttl=3600) def get_bing_wallpapers(num: int = 7) -> List[str]: """ 获取7天的Bing每日壁纸 diff --git a/requirements.in b/requirements.in index deea47a6..3601052d 100644 --- a/requirements.in +++ b/requirements.in @@ -64,4 +64,5 @@ python-cookietools==0.0.2.1 aligo~=6.2.4 aiofiles~=24.1.0 jieba~=0.42.1 -rsa~=4.9 \ No newline at end of file +rsa~=4.9 +redis~=5.2.1 \ No newline at end of file From 6d2059447ee3070cbcd7b19e81994e2782089f06 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sat, 18 Jan 2025 02:14:01 +0800 Subject: [PATCH 3/5] feat(cache): enhance get_plugins to skip empty during network errors --- app/helper/plugin.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/helper/plugin.py b/app/helper/plugin.py index bb8a0ecf..487392e6 100644 --- a/app/helper/plugin.py +++ b/app/helper/plugin.py @@ -4,11 +4,11 @@ import traceback from pathlib import Path from typing import Any, Dict, List, Optional, Tuple, Set -from cachetools import TTLCache, cached from packaging.specifiers import SpecifierSet, InvalidSpecifier from packaging.version import Version, InvalidVersion from pkg_resources import Requirement, working_set +from app.core.cache import cached from app.core.config import settings from app.db.systemconfig_oper import SystemConfigOper from app.log import logger @@ -38,24 +38,26 @@ class PluginHelper(metaclass=Singleton): if self.install_report(): self.systemconfig.set(SystemConfigKey.PluginInstallReport, "1") - @cached(cache=TTLCache(maxsize=1000, ttl=1800)) - def get_plugins(self, repo_url: str, package_version: str = None) -> Dict[str, dict]: + @cached(maxsize=1000, ttl=1800, skip_empty=False) + def get_plugins(self, repo_url: str, package_version: str = None) -> Optional[Dict[str, dict]]: """ 获取Github所有最新插件列表 :param repo_url: Github仓库地址 :param package_version: 首选插件版本 (如 "v2", "v3"),如果不指定则获取 v1 版本 """ if not repo_url: - return {} + return None user, repo = self.get_repo_info(repo_url) if not user or not repo: - return {} + return None raw_url = self._base_url.format(user=user, repo=repo) package_url = f"{raw_url}package.{package_version}.json" if package_version else f"{raw_url}package.json" res = self.__request_with_fallback(package_url, headers=settings.REPO_GITHUB_HEADERS(repo=f"{user}/{repo}")) + if res is None: + return None if res: try: return json.loads(res.text) @@ -113,7 +115,7 @@ class PluginHelper(metaclass=Singleton): return None, None return user, repo - @cached(cache=TTLCache(maxsize=1, ttl=1800)) + @cached(maxsize=1, ttl=1800) def get_statistic(self) -> Dict: """ 获取插件安装统计 From d9508533e16e19b3e1a5683e45cba107e53d30f3 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sat, 18 Jan 2025 02:32:08 +0800 Subject: [PATCH 4/5] feat(cache): add cache region support --- app/chain/recommend.py | 72 ++++++++++------------------------------- app/core/cache.py | 34 ++++++++++--------- app/helper/subscribe.py | 14 ++++---- 3 files changed, 43 insertions(+), 77 deletions(-) diff --git a/app/chain/recommend.py b/app/chain/recommend.py index d0290625..e42b3c3c 100644 --- a/app/chain/recommend.py +++ b/app/chain/recommend.py @@ -1,18 +1,15 @@ -import inspect import io import tempfile -from functools import wraps from pathlib import Path -from typing import Any, Callable, List +from typing import Any, List from PIL import Image -from cachetools import TTLCache -from cachetools.keys import hashkey from app.chain import ChainBase from app.chain.bangumi import BangumiChain from app.chain.douban import DoubanChain from app.chain.tmdb import TmdbChain +from app.core.cache import cache_backend, cached from app.core.config import settings, global_vars from app.log import logger from app.schemas import MediaType @@ -23,42 +20,7 @@ from app.utils.singleton import Singleton # 推荐相关的专用缓存 recommend_ttl = 24 * 3600 -recommend_cache = TTLCache(maxsize=256, ttl=recommend_ttl) - - -# 推荐缓存装饰器,避免偶发网络获取数据为空时,页面由于缓存为空长时间渲染异常问题 -def cached_with_empty_check(func: Callable): - """ - 缓存装饰器,用于缓存函数的返回结果,仅在结果非空时进行缓存 - - :param func: 被装饰的函数 - :return: 包装后的函数 - """ - - @wraps(func) - def wrapper(*args, **kwargs): - signature = inspect.signature(func) - resolved_kwargs = {} - # 获取默认值并结合传递的参数(如果有) - for param, value in signature.parameters.items(): - if param in kwargs: - # 使用显式传递的参数 - resolved_kwargs[param] = kwargs[param] - elif value.default is not inspect.Parameter.empty: - # 没有传递参数时使用默认值 - resolved_kwargs[param] = value.default - # 使用 cachetools 缓存,构造缓存键 - cache_key = f"{func.__name__}_{hashkey(*args, **resolved_kwargs)}" - if cache_key in recommend_cache: - return recommend_cache[cache_key] - result = func(*args, **kwargs) - # 如果返回值为空,则不缓存 - if result in [None, [], {}]: - return result - recommend_cache[cache_key] = result - return result - - return wrapper +recommend_cache_region = "recommend" class RecommendChain(ChainBase, metaclass=Singleton): @@ -78,7 +40,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): 刷新推荐 """ logger.debug("Starting to refresh Recommend data.") - recommend_cache.clear() + cache_backend.clear(region=recommend_cache_region) logger.debug("Recommend Cache has been cleared.") # 推荐来源方法 @@ -194,7 +156,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): logger.debug(f"Failed to write cache file {cache_path} for URL {url}: {e}") @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def tmdb_movies(self, sort_by: str = "popularity.desc", with_genres: str = "", with_original_language: str = "", page: int = 1) -> Any: """ @@ -208,7 +170,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [movie.to_dict() for movie in movies] if movies else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def tmdb_tvs(self, sort_by: str = "popularity.desc", with_genres: str = "", with_original_language: str = "zh|en|ja|ko", page: int = 1) -> Any: """ @@ -222,7 +184,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [tv.to_dict() for tv in tvs] if tvs else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def tmdb_trending(self, page: int = 1) -> Any: """ TMDB流行趋势 @@ -231,7 +193,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [info.to_dict() for info in infos] if infos else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def bangumi_calendar(self, page: int = 1, count: int = 30) -> Any: """ Bangumi每日放送 @@ -240,7 +202,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in medias[(page - 1) * count: page * count]] if medias else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_movie_showing(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣正在热映 @@ -249,7 +211,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in movies] if movies else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_movies(self, sort: str = "R", tags: str = "", page: int = 1, count: int = 30) -> Any: """ 豆瓣最新电影 @@ -259,7 +221,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in movies] if movies else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_tvs(self, sort: str = "R", tags: str = "", page: int = 1, count: int = 30) -> Any: """ 豆瓣最新电视剧 @@ -269,7 +231,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in tvs] if tvs else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_movie_top250(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣电影TOP250 @@ -278,7 +240,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in movies] if movies else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_tv_weekly_chinese(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣国产剧集榜 @@ -287,7 +249,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in tvs] if tvs else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_tv_weekly_global(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣全球剧集榜 @@ -296,7 +258,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in tvs] if tvs else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_tv_animation(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣热门动漫 @@ -305,7 +267,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in tvs] if tvs else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_movie_hot(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣热门电影 @@ -314,7 +276,7 @@ class RecommendChain(ChainBase, metaclass=Singleton): return [media.to_dict() for media in movies] if movies else [] @log_execution_time(logger=logger) - @cached_with_empty_check + @cached(maxsize=16, ttl=recommend_ttl, region=recommend_cache_region) def douban_tv_hot(self, page: int = 1, count: int = 30) -> Any: """ 豆瓣热门电视剧 diff --git a/app/core/cache.py b/app/core/cache.py index 817d5065..86897cf5 100644 --- a/app/core/cache.py +++ b/app/core/cache.py @@ -9,6 +9,9 @@ import redis from cachetools import TTLCache from cachetools.keys import hashkey +# 默认缓存区 +DEFAULT_CACHE_REGION = "DEFAULT" + class CacheBackend(ABC): """ @@ -16,7 +19,7 @@ class CacheBackend(ABC): """ @abstractmethod - def set(self, key: str, value: Any, ttl: int, region: str = "default", **kwargs) -> None: + def set(self, key: str, value: Any, ttl: int, region: str = DEFAULT_CACHE_REGION, **kwargs) -> None: """ 设置缓存 @@ -29,7 +32,7 @@ class CacheBackend(ABC): pass @abstractmethod - def get(self, key: str, region: str = "default") -> Any: + def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Any: """ 获取缓存 @@ -40,7 +43,7 @@ class CacheBackend(ABC): pass @abstractmethod - def delete(self, key: str, region: str = "default") -> None: + def delete(self, key: str, region: str = DEFAULT_CACHE_REGION) -> None: """ 删除缓存 @@ -59,7 +62,7 @@ class CacheBackend(ABC): pass @staticmethod - def get_region(region: str = "default"): + def get_region(region: str = DEFAULT_CACHE_REGION): """ 获取缓存的区 """ @@ -83,7 +86,7 @@ class CacheToolsBackend(CacheBackend): # 存储各个 region 的缓存实例,region -> {key -> TTLCache} self._region_caches: Dict[str, Dict[str, TTLCache]] = defaultdict(dict) - def set(self, key: str, value: Any, ttl: int = None, region: str = "default", **kwargs) -> None: + def set(self, key: str, value: Any, ttl: int = None, region: str = DEFAULT_CACHE_REGION, **kwargs) -> None: """ 设置缓存值支持每个 key 独立配置 TTL 和 Maxsize @@ -105,7 +108,7 @@ class CacheToolsBackend(CacheBackend): # 设置缓存值 cache[key] = value - def get(self, key: str, region: str = "default") -> Any: + def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Any: """ 获取缓存的值 @@ -121,7 +124,7 @@ class CacheToolsBackend(CacheBackend): cache = region_cache[key] return cache.get(key) - def delete(self, key: str, region: str = "default") -> None: + def delete(self, key: str, region: str = DEFAULT_CACHE_REGION) -> None: """ 删除缓存 @@ -143,6 +146,7 @@ class CacheToolsBackend(CacheBackend): :param region: 缓存的区 """ if region: + region = self.get_region(region) region_cache = self._region_caches[region] for cache in region_cache.values(): cache.clear() @@ -176,7 +180,7 @@ class RedisBackend(CacheBackend): # 使用 region 作为缓存键的一部分 return f"region:{region}:key:{key}" - def set(self, key: str, value: Any, ttl: int = None, region: str = "default", **kwargs) -> None: + def set(self, key: str, value: Any, ttl: int = None, region: str = DEFAULT_CACHE_REGION, **kwargs) -> None: """ 设置缓存 @@ -190,7 +194,7 @@ class RedisBackend(CacheBackend): redis_key = self.get_redis_key(region, key) self.client.setex(redis_key, ttl, value) - def get(self, key: str, region: str = "default") -> Any: + def get(self, key: str, region: str = DEFAULT_CACHE_REGION) -> Any: """ 获取缓存的值 @@ -202,7 +206,7 @@ class RedisBackend(CacheBackend): value = self.client.get(redis_key) return value - def delete(self, key: str, region: str = "default") -> None: + def delete(self, key: str, region: str = DEFAULT_CACHE_REGION) -> None: """ 删除缓存 @@ -244,11 +248,7 @@ def get_cache_backend(maxsize: int = 1000, ttl: int = 1800) -> CacheBackend: return CacheToolsBackend(maxsize=maxsize, ttl=ttl) -# 缓存后端实例 -cache_backend = get_cache_backend() - - -def cached(region: str = "default", maxsize: int = 1000, ttl: int = 1800, +def cached(region: str = DEFAULT_CACHE_REGION, maxsize: int = 1000, ttl: int = 1800, skip_none: bool = True, skip_empty: bool = True): """ 自定义缓存装饰器,支持为每个 key 动态传递 maxsize 和 ttl @@ -318,3 +318,7 @@ def cached(region: str = "default", maxsize: int = 1000, ttl: int = 1800, return wrapper return decorator + + +# 缓存后端实例 +cache_backend = get_cache_backend() diff --git a/app/helper/subscribe.py b/app/helper/subscribe.py index 9ac6377a..4942712e 100644 --- a/app/helper/subscribe.py +++ b/app/helper/subscribe.py @@ -1,8 +1,7 @@ from threading import Thread from typing import List, Tuple -from cachetools import TTLCache, cached - +from app.core.cache import cached, cache_backend from app.core.config import settings from app.db.subscribe_oper import SubscribeOper from app.db.systemconfig_oper import SystemConfigOper @@ -31,7 +30,7 @@ class SubscribeHelper(metaclass=Singleton): _sub_fork = f"{settings.MP_SERVER_HOST}/subscribe/fork/%s" - _shares_cache = TTLCache(maxsize=20, ttl=1800) + _shares_cache_region = "subscribe_share" def __init__(self): self.systemconfig = SystemConfigOper() @@ -41,7 +40,7 @@ class SubscribeHelper(metaclass=Singleton): if self.sub_report(): self.systemconfig.set(SystemConfigKey.SubscribeReport, "1") - @cached(cache=TTLCache(maxsize=20, ttl=1800)) + @cached(maxsize=20, ttl=1800) def get_statistic(self, stype: str, page: int = 1, count: int = 30) -> List[dict]: """ 获取订阅统计数据 @@ -129,6 +128,7 @@ class SubscribeHelper(metaclass=Singleton): return False, "订阅不存在" subscribe_dict = subscribe.to_dict() subscribe_dict.pop("id") + cache_backend.clear(region=self._shares_cache_region) res = RequestUtils(proxies=settings.PROXY, content_type="application/json", timeout=10).post(self._sub_share, json={ @@ -142,7 +142,7 @@ class SubscribeHelper(metaclass=Singleton): return False, "连接MoviePilot服务器失败" if res.ok: # 清除 get_shares 的缓存,以便实时看到结果 - self._shares_cache.clear() + cache_backend.clear(region=self._shares_cache_region) return True, "" else: return False, res.json().get("message") @@ -160,7 +160,7 @@ class SubscribeHelper(metaclass=Singleton): return False, "连接MoviePilot服务器失败" if res.ok: # 清除 get_shares 的缓存,以便实时看到结果 - self._shares_cache.clear() + cache_backend.clear(region=self._shares_cache_region) return True, "" else: return False, res.json().get("message") @@ -181,7 +181,7 @@ class SubscribeHelper(metaclass=Singleton): else: return False, res.json().get("message") - @cached(cache=_shares_cache) + @cached(region=_shares_cache_region) def get_shares(self, name: str, page: int = 1, count: int = 30) -> List[dict]: """ 获取订阅分享数据 From ad0241b7f105097a6bff085bc23e841e99dcea15 Mon Sep 17 00:00:00 2001 From: InfinityPacer <160988576+InfinityPacer@users.noreply.github.com> Date: Sat, 18 Jan 2025 02:44:56 +0800 Subject: [PATCH 5/5] feat(cache): set default skip_empty to False --- app/core/cache.py | 4 ++-- app/helper/plugin.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/core/cache.py b/app/core/cache.py index 86897cf5..2b8a85cb 100644 --- a/app/core/cache.py +++ b/app/core/cache.py @@ -249,7 +249,7 @@ def get_cache_backend(maxsize: int = 1000, ttl: int = 1800) -> CacheBackend: def cached(region: str = DEFAULT_CACHE_REGION, maxsize: int = 1000, ttl: int = 1800, - skip_none: bool = True, skip_empty: bool = True): + skip_none: bool = True, skip_empty: bool = False): """ 自定义缓存装饰器,支持为每个 key 动态传递 maxsize 和 ttl @@ -257,7 +257,7 @@ def cached(region: str = DEFAULT_CACHE_REGION, maxsize: int = 1000, ttl: int = 1 :param maxsize: 缓存的最大条目数,默认值为 1000 :param ttl: 缓存的存活时间,单位秒,默认值为 1800 :param skip_none: 跳过 None 缓存,默认为 True - :param skip_empty: 跳过空值缓存(如 [], {}, "", set()),默认为 True + :param skip_empty: 跳过空值缓存(如 [], {}, "", set()),默认为 False :return: 装饰器函数 """ diff --git a/app/helper/plugin.py b/app/helper/plugin.py index 487392e6..616e41b1 100644 --- a/app/helper/plugin.py +++ b/app/helper/plugin.py @@ -38,7 +38,7 @@ class PluginHelper(metaclass=Singleton): if self.install_report(): self.systemconfig.set(SystemConfigKey.PluginInstallReport, "1") - @cached(maxsize=1000, ttl=1800, skip_empty=False) + @cached(maxsize=1000, ttl=1800) def get_plugins(self, repo_url: str, package_version: str = None) -> Optional[Dict[str, dict]]: """ 获取Github所有最新插件列表