GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-05-22 16:50:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 优化缓存策略

Browse Source
This commit is contained in:
jxxghp
2024-11-21 15:52:08 +08:00
parent acd9e38477
commit ebffdebdb2
9 changed files with 90 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
app/modules/douban/apiv2.py
View File

@@ -3,11 +3,11 @@ import base64
import hashlib
import hmac
from datetime import datetime
from functools import lru_cache
from random import choice
from urllib import parse
import requests
from cachetools import TTLCache, cached
from app.core.config import settings
from app.utils.http import RequestUtils
@@ -160,12 +160,12 @@ class DoubanApi(metaclass=Singleton):
self._session = requests.Session()
@classmethod
def __sign(cls, url: str, ts: int, method='GET') -> str:
def __sign(cls, url: str, ts: str, method='GET') -> str:
"""
签名
"""
url_path = parse.urlparse(url).path
raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), str(ts)])
raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), ts])
return base64.b64encode(
hmac.new(
cls._api_secret_key.encode(),
@@ -174,7 +174,7 @@ class DoubanApi(metaclass=Singleton):
).digest()
).decode()
@lru_cache(maxsize=settings.CACHE_CONF.get('douban'))
@cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
def __invoke(self, url: str, **kwargs) -> dict:
"""
GET请求
@@ -203,7 +203,7 @@ class DoubanApi(metaclass=Singleton):
return resp.json()
return resp.json() if resp else {}
@lru_cache(maxsize=settings.CACHE_CONF.get('douban'))
@cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
def __post(self, url: str, **kwargs) -> dict:
"""
POST请求

4
app/modules/douban/douban_cache.py
View File

@@ -16,7 +16,7 @@ from app.schemas.types import MediaType
lock = RLock()
CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta')
EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]
class DoubanCache(metaclass=Singleton):
@@ -77,7 +77,7 @@ class DoubanCache(metaclass=Singleton):
@return: 被删除的缓存内容
"""
with lock:
return self._meta_data.pop(key, None)
return self._meta_data.pop(key, {})
def delete_by_doubanid(self, doubanid: str) -> None:
"""

7
app/modules/fanart/__init__.py
View File

@@ -1,12 +1,13 @@
import re
from functools import lru_cache
from typing import Optional, Tuple, Union
from cachetools import TTLCache, cached
from app.core.context import MediaInfo, settings
from app.log import logger
from app.modules import _ModuleBase
from app.utils.http import RequestUtils
from app.schemas.types import MediaType, ModuleType
from app.utils.http import RequestUtils
class FanartModule(_ModuleBase):
@@ -404,7 +405,7 @@ class FanartModule(_ModuleBase):
return result
@classmethod
@lru_cache(maxsize=settings.CACHE_CONF.get('fanart'))
@cached(cache=TTLCache(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"]))
def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]:
if media_type == MediaType.MOVIE:
image_url = cls._movie_url % queryid

22
app/modules/themoviedb/tmdb_cache.py
View File

@@ -15,7 +15,7 @@ from app.schemas.types import MediaType
lock = RLock()
CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta')
EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]
class TmdbCache(metaclass=Singleton):
@@ -75,7 +75,7 @@ class TmdbCache(metaclass=Singleton):
@return: 被删除的缓存内容
"""
with lock:
return self._meta_data.pop(key, None)
return self._meta_data.pop(key, {})
def delete_by_tmdbid(self, tmdbid: int) -> None:
"""
@@ -138,14 +138,14 @@ class TmdbCache(metaclass=Singleton):
if cache_year:
cache_year = cache_year[:4]
self._meta_data[self.__get_key(meta)] = {
"id": info.get("id"),
"type": info.get("media_type"),
"year": cache_year,
"title": cache_title,
"poster_path": info.get("poster_path"),
"backdrop_path": info.get("backdrop_path"),
CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP
}
"id": info.get("id"),
"type": info.get("media_type"),
"year": cache_year,
"title": cache_title,
"poster_path": info.get("poster_path"),
"backdrop_path": info.get("backdrop_path"),
CACHE_EXPIRE_TIMESTAMP_STR: int(time.time()) + EXPIRE_TIMESTAMP
}
elif info is not None:
# None时不缓存，此时代表网络错误，允许重复请求
self._meta_data[self.__get_key(meta)] = {'id': 0}
@@ -164,7 +164,7 @@ class TmdbCache(metaclass=Singleton):
return
with open(self._meta_path, 'wb') as f:
pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL)
pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # type: ignore
def _random_sample(self, new_meta_data: dict) -> bool:
"""

6
app/modules/themoviedb/tmdbapi.py
View File

@@ -1,9 +1,9 @@
import traceback
from functools import lru_cache
from typing import Optional, List
from urllib.parse import quote
import zhconv
from cachetools import TTLCache, cached
from lxml import etree
from app.core.config import settings
@@ -27,8 +27,6 @@ class TmdbApi:
self.tmdb.domain = settings.TMDB_API_DOMAIN
# 开启缓存
self.tmdb.cache = True
# 缓存大小
self.tmdb.REQUEST_CACHE_MAXSIZE = settings.CACHE_CONF.get('tmdb')
# APIKEY
self.tmdb.api_key = settings.TMDB_API_KEY
# 语种
@@ -466,7 +464,7 @@ class TmdbApi:
return ret_info
@lru_cache(maxsize=settings.CACHE_CONF.get('tmdb'))
@cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
def match_web(self, name: str, mtype: MediaType) -> Optional[dict]:
"""
搜索TMDB网站，直接抓取结果，结果只有一条时才返回

6
app/modules/themoviedb/tmdbv3api/tmdb.py
View File

@@ -4,11 +4,12 @@ import logging
import os
import time
from datetime import datetime
from functools import lru_cache
import requests
import requests.exceptions
from cachetools import TTLCache, cached
from app.core.config import settings
from app.utils.http import RequestUtils
from .exceptions import TMDbException
@@ -24,7 +25,6 @@ class TMDb(object):
TMDB_CACHE_ENABLED = "TMDB_CACHE_ENABLED"
TMDB_PROXIES = "TMDB_PROXIES"
TMDB_DOMAIN = "TMDB_DOMAIN"
REQUEST_CACHE_MAXSIZE = None
_req = None
_session = None
@@ -137,7 +137,7 @@ class TMDb(object):
def cache(self, cache):
os.environ[self.TMDB_CACHE_ENABLED] = str(cache)
@lru_cache(maxsize=REQUEST_CACHE_MAXSIZE)
@cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
def cached_request(self, method, url, data, json,
_ts=datetime.strftime(datetime.now(), '%Y%m%d')):
"""