GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-07 08:26:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: 优化缓存策略

Browse Source
This commit is contained in:
jxxghp
2024-11-21 15:52:08 +08:00
parent acd9e38477
commit ebffdebdb2
9 changed files with 90 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/chain/torrents.py
+3 -3
View File
@@ -175,7 +175,7 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
# 按pubdate降序排列 # 按pubdate降序排列
torrents.sort(key=lambda x: x.pubdate or '', reverse=True) torrents.sort(key=lambda x: x.pubdate or '', reverse=True)
# 取前N条 # 取前N条
torrents = torrents[:settings.CACHE_CONF.get('refresh')] torrents = torrents[:settings.CACHE_CONF["refresh"]]
if torrents: if torrents:
# 过滤出没有处理过的种子 # 过滤出没有处理过的种子
torrents = [torrent for torrent in torrents torrents = [torrent for torrent in torrents
@@ -215,8 +215,8 @@ class TorrentsChain(ChainBase, metaclass=Singleton):
else: else:
torrents_cache[domain].append(context) torrents_cache[domain].append(context)
# 如果超过了限制条数则移除掉前面的 # 如果超过了限制条数则移除掉前面的
if len(torrents_cache[domain]) > settings.CACHE_CONF.get('torrents'): if len(torrents_cache[domain]) > settings.CACHE_CONF["torrents"]:
torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF.get('torrents'):] torrents_cache[domain] = torrents_cache[domain][-settings.CACHE_CONF["torrents"]:]
# 回收资源 # 回收资源
del torrents del torrents
else: else:
app/core/config.py
+49 -18
View File
@@ -8,7 +8,7 @@ from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Type from typing import Any, Dict, List, Optional, Tuple, Type
from dotenv import set_key from dotenv import set_key
from pydantic import BaseModel, BaseSettings, validator from pydantic import BaseModel, BaseSettings, validator, Field
from app.log import logger from app.log import logger
from app.utils.system import SystemUtils from app.utils.system import SystemUtils
@@ -36,7 +36,7 @@ class ConfigModel(BaseModel):
# RESOURCE密钥 # RESOURCE密钥
RESOURCE_SECRET_KEY: str = secrets.token_urlsafe(32) RESOURCE_SECRET_KEY: str = secrets.token_urlsafe(32)
# 允许的域名 # 允许的域名
ALLOWED_HOSTS: list = ["*"] ALLOWED_HOSTS: list = Field(default_factory=lambda: ["*"])
# TOKEN过期时间 # TOKEN过期时间
ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8 ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
# RESOURCE_TOKEN过期时间 # RESOURCE_TOKEN过期时间
@@ -114,29 +114,39 @@ class ConfigModel(BaseModel):
# 是否启用DOH解析域名 # 是否启用DOH解析域名
DOH_ENABLE: bool = True DOH_ENABLE: bool = True
# 使用 DOH 解析的域名列表 # 使用 DOH 解析的域名列表
DOH_DOMAINS: str = "api.themoviedb.org,api.tmdb.org,webservice.fanart.tv,api.github.com,github.com,raw.githubusercontent.com,api.telegram.org" DOH_DOMAINS: str = ("api.themoviedb.org,"
"api.tmdb.org,"
"webservice.fanart.tv,"
"api.github.com,"
"github.com,"
"raw.githubusercontent.com,"
"api.telegram.org")
# DOH 解析服务器列表 # DOH 解析服务器列表
DOH_RESOLVERS: str = "1.0.0.1,1.1.1.1,9.9.9.9,149.112.112.112" DOH_RESOLVERS: str = "1.0.0.1,1.1.1.1,9.9.9.9,149.112.112.112"
# 支持的后缀格式 # 支持的后缀格式
RMT_MEDIAEXT: list = ['.mp4', '.mkv', '.ts', '.iso', RMT_MEDIAEXT: list = Field(
default_factory=lambda: ['.mp4', '.mkv', '.ts', '.iso',
'.rmvb', '.avi', '.mov', '.mpeg', '.rmvb', '.avi', '.mov', '.mpeg',
'.mpg', '.wmv', '.3gp', '.asf', '.mpg', '.wmv', '.3gp', '.asf',
'.m4v', '.flv', '.m2ts', '.strm', '.m4v', '.flv', '.m2ts', '.strm',
'.tp', '.f4v'] '.tp', '.f4v']
)
# 支持的字幕文件后缀格式 # 支持的字幕文件后缀格式
RMT_SUBEXT: list = ['.srt', '.ass', '.ssa', '.sup'] RMT_SUBEXT: list = Field(default_factory=lambda: ['.srt', '.ass', '.ssa', '.sup'])
# 支持的音轨文件后缀格式 # 支持的音轨文件后缀格式
RMT_AUDIO_TRACK_EXT: list = ['.mka'] RMT_AUDIO_TRACK_EXT: list = Field(default_factory=lambda: ['.mka'])
# 音轨文件后缀格式 # 音轨文件后缀格式
RMT_AUDIOEXT: list = ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf', RMT_AUDIOEXT: list = Field(
default_factory=lambda: ['.aac', '.ac3', '.amr', '.caf', '.cda', '.dsf',
'.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3', '.dff', '.kar', '.m4a', '.mp1', '.mp2', '.mp3',
'.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg', '.mid', '.mod', '.mka', '.mpc', '.nsf', '.ogg',
'.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak', '.pcm', '.rmi', '.s3m', '.snd', '.spx', '.tak',
'.tta', '.vqf', '.wav', '.wma', '.tta', '.vqf', '.wav', '.wma',
'.aifc', '.aiff', '.alac', '.adif', '.adts', '.aifc', '.aiff', '.alac', '.adif', '.adts',
'.flac', '.midi', '.opus', '.sfalc'] '.flac', '.midi', '.opus', '.sfalc']
)
# 下载器临时文件后缀 # 下载器临时文件后缀
DOWNLOAD_TMPEXT: list = ['.!qb', '.part'] DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: ['.!qb', '.part'])
# 媒体服务器同步间隔（小时） # 媒体服务器同步间隔（小时）
MEDIASERVER_SYNC_INTERVAL: int = 6 MEDIASERVER_SYNC_INTERVAL: int = 6
# 订阅模式 # 订阅模式
@@ -189,7 +199,10 @@ class ConfigModel(BaseModel):
# 服务器地址，对应 https://github.com/jxxghp/MoviePilot-Server 项目 # 服务器地址，对应 https://github.com/jxxghp/MoviePilot-Server 项目
MP_SERVER_HOST: str = "https://movie-pilot.org" MP_SERVER_HOST: str = "https://movie-pilot.org"
# 插件市场仓库地址，多个地址使用,分隔，地址以/结尾 # 插件市场仓库地址，多个地址使用,分隔，地址以/结尾
PLUGIN_MARKET: str = "https://github.com/jxxghp/MoviePilot-Plugins,https://github.com/thsrite/MoviePilot-Plugins,https://github.com/honue/MoviePilot-Plugins,https://github.com/InfinityPacer/MoviePilot-Plugins" PLUGIN_MARKET: str = ("https://github.com/jxxghp/MoviePilot-Plugins,"
"https://github.com/thsrite/MoviePilot-Plugins,"
"https://github.com/honue/MoviePilot-Plugins,"
"https://github.com/InfinityPacer/MoviePilot-Plugins")
# 插件安装数据共享 # 插件安装数据共享
PLUGIN_STATISTIC_SHARE: bool = True PLUGIN_STATISTIC_SHARE: bool = True
# 是否开启插件热加载 # 是否开启插件热加载
@@ -207,10 +220,18 @@ class ConfigModel(BaseModel):
# 全局图片缓存，将媒体图片缓存到本地 # 全局图片缓存，将媒体图片缓存到本地
GLOBAL_IMAGE_CACHE: bool = False GLOBAL_IMAGE_CACHE: bool = False
# 允许的图片缓存域名 # 允许的图片缓存域名
SECURITY_IMAGE_DOMAINS: List[str] = ["image.tmdb.org", "static-mdb.v.geilijiasu.com", "doubanio.com", "lain.bgm.tv", SECURITY_IMAGE_DOMAINS: List[str] = Field(
"raw.githubusercontent.com", "github.com"] default_factory=lambda: ["image.tmdb.org",
"static-mdb.v.geilijiasu.com",
"doubanio.com",
"lain.bgm.tv",
"raw.githubusercontent.com",
"github.com"]
)
# 允许的图片文件后缀格式 # 允许的图片文件后缀格式
SECURITY_IMAGE_SUFFIXES: List[str] = [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"] SECURITY_IMAGE_SUFFIXES: List[str] = Field(
default_factory=lambda: [".jpg", ".jpeg", ".png", ".webp", ".gif", ".svg"]
)
class Settings(BaseSettings, ConfigModel): class Settings(BaseSettings, ConfigModel):
@@ -437,22 +458,32 @@ class Settings(BaseSettings, ConfigModel):
@property @property
def CACHE_CONF(self): def CACHE_CONF(self):
"""
{
"torrents": "缓存种子数量",
"refresh": "订阅刷新处理数量",
"tmdb": "TMDB请求缓存数量",
"douban": "豆瓣请求缓存数量",
"fanart": "Fanart请求缓存数量",
"meta": "元数据缓存过期时间（秒）"
}
"""
if self.BIG_MEMORY_MODE: if self.BIG_MEMORY_MODE:
return { return {
"torrents": 200,
"refresh": 100,
"tmdb": 1024, "tmdb": 1024,
"refresh": 50,
"torrents": 100,
"douban": 512, "douban": 512,
"fanart": 512, "fanart": 512,
"meta": (self.META_CACHE_EXPIRE or 168) * 3600 "meta": (self.META_CACHE_EXPIRE or 24) * 3600
} }
return { return {
"torrents": 100,
"refresh": 50,
"tmdb": 256, "tmdb": 256,
"refresh": 30,
"torrents": 50,
"douban": 256, "douban": 256,
"fanart": 128, "fanart": 128,
"meta": (self.META_CACHE_EXPIRE or 72) * 3600 "meta": (self.META_CACHE_EXPIRE or 2) * 3600
} }
@property @property
app/modules/douban/apiv2.py
+5 -5
View File
@@ -3,11 +3,11 @@ import base64
import hashlib import hashlib
import hmac import hmac
from datetime import datetime from datetime import datetime
from functools import lru_cache
from random import choice from random import choice
from urllib import parse from urllib import parse
import requests import requests
from cachetools import TTLCache, cached
from app.core.config import settings from app.core.config import settings
from app.utils.http import RequestUtils from app.utils.http import RequestUtils
@@ -160,12 +160,12 @@ class DoubanApi(metaclass=Singleton):
self._session = requests.Session() self._session = requests.Session()
@classmethod @classmethod
def __sign(cls, url: str, ts: int, method='GET') -> str: def __sign(cls, url: str, ts: str, method='GET') -> str:
""" """
签名 签名
""" """
url_path = parse.urlparse(url).path url_path = parse.urlparse(url).path
raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), str(ts)]) raw_sign = '&'.join([method.upper(), parse.quote(url_path, safe=''), ts])
return base64.b64encode( return base64.b64encode(
hmac.new( hmac.new(
cls._api_secret_key.encode(), cls._api_secret_key.encode(),
@@ -174,7 +174,7 @@ class DoubanApi(metaclass=Singleton):
).digest() ).digest()
).decode() ).decode()
@lru_cache(maxsize=settings.CACHE_CONF.get('douban')) @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
def __invoke(self, url: str, **kwargs) -> dict: def __invoke(self, url: str, **kwargs) -> dict:
""" """
GET请求 GET请求
@@ -203,7 +203,7 @@ class DoubanApi(metaclass=Singleton):
return resp.json() return resp.json()
return resp.json() if resp else {} return resp.json() if resp else {}
@lru_cache(maxsize=settings.CACHE_CONF.get('douban')) @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["douban"], ttl=settings.CACHE_CONF["meta"]))
def __post(self, url: str, **kwargs) -> dict: def __post(self, url: str, **kwargs) -> dict:
""" """
POST请求 POST请求
app/modules/douban/douban_cache.py
+2 -2
View File
@@ -16,7 +16,7 @@ from app.schemas.types import MediaType
lock = RLock() lock = RLock()
CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp" CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta') EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]
class DoubanCache(metaclass=Singleton): class DoubanCache(metaclass=Singleton):
@@ -77,7 +77,7 @@ class DoubanCache(metaclass=Singleton):
@return: 被删除的缓存内容 @return: 被删除的缓存内容
""" """
with lock: with lock:
return self._meta_data.pop(key, None) return self._meta_data.pop(key, {})
def delete_by_doubanid(self, doubanid: str) -> None: def delete_by_doubanid(self, doubanid: str) -> None:
""" """
app/modules/fanart/__init__.py
+4 -3
View File
@@ -1,12 +1,13 @@
import re import re
from functools import lru_cache
from typing import Optional, Tuple, Union from typing import Optional, Tuple, Union
from cachetools import TTLCache, cached
from app.core.context import MediaInfo, settings from app.core.context import MediaInfo, settings
from app.log import logger from app.log import logger
from app.modules import _ModuleBase from app.modules import _ModuleBase
from app.utils.http import RequestUtils
from app.schemas.types import MediaType, ModuleType from app.schemas.types import MediaType, ModuleType
from app.utils.http import RequestUtils
class FanartModule(_ModuleBase): class FanartModule(_ModuleBase):
@@ -404,7 +405,7 @@ class FanartModule(_ModuleBase):
return result return result
@classmethod @classmethod
@lru_cache(maxsize=settings.CACHE_CONF.get('fanart')) @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["fanart"], ttl=settings.CACHE_CONF["meta"]))
def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]: def __request_fanart(cls, media_type: MediaType, queryid: Union[str, int]) -> Optional[dict]:
if media_type == MediaType.MOVIE: if media_type == MediaType.MOVIE:
image_url = cls._movie_url % queryid image_url = cls._movie_url % queryid
app/modules/themoviedb/tmdb_cache.py
+3 -3
View File
@@ -15,7 +15,7 @@ from app.schemas.types import MediaType
lock = RLock() lock = RLock()
CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp" CACHE_EXPIRE_TIMESTAMP_STR = "cache_expire_timestamp"
EXPIRE_TIMESTAMP = settings.CACHE_CONF.get('meta') EXPIRE_TIMESTAMP = settings.CACHE_CONF["meta"]
class TmdbCache(metaclass=Singleton): class TmdbCache(metaclass=Singleton):
@@ -75,7 +75,7 @@ class TmdbCache(metaclass=Singleton):
@return: 被删除的缓存内容 @return: 被删除的缓存内容
""" """
with lock: with lock:
return self._meta_data.pop(key, None) return self._meta_data.pop(key, {})
def delete_by_tmdbid(self, tmdbid: int) -> None: def delete_by_tmdbid(self, tmdbid: int) -> None:
""" """
@@ -164,7 +164,7 @@ class TmdbCache(metaclass=Singleton):
return return
with open(self._meta_path, 'wb') as f: with open(self._meta_path, 'wb') as f:
pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) pickle.dump(new_meta_data, f, pickle.HIGHEST_PROTOCOL) # type: ignore
def _random_sample(self, new_meta_data: dict) -> bool: def _random_sample(self, new_meta_data: dict) -> bool:
""" """
app/modules/themoviedb/tmdbapi.py
+2 -4
View File
@@ -1,9 +1,9 @@
import traceback import traceback
from functools import lru_cache
from typing import Optional, List from typing import Optional, List
from urllib.parse import quote from urllib.parse import quote
import zhconv import zhconv
from cachetools import TTLCache, cached
from lxml import etree from lxml import etree
from app.core.config import settings from app.core.config import settings
@@ -27,8 +27,6 @@ class TmdbApi:
self.tmdb.domain = settings.TMDB_API_DOMAIN self.tmdb.domain = settings.TMDB_API_DOMAIN
# 开启缓存 # 开启缓存
self.tmdb.cache = True self.tmdb.cache = True
# 缓存大小
self.tmdb.REQUEST_CACHE_MAXSIZE = settings.CACHE_CONF.get('tmdb')
# APIKEY # APIKEY
self.tmdb.api_key = settings.TMDB_API_KEY self.tmdb.api_key = settings.TMDB_API_KEY
# 语种 # 语种
@@ -466,7 +464,7 @@ class TmdbApi:
return ret_info return ret_info
@lru_cache(maxsize=settings.CACHE_CONF.get('tmdb')) @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
def match_web(self, name: str, mtype: MediaType) -> Optional[dict]: def match_web(self, name: str, mtype: MediaType) -> Optional[dict]:
""" """
搜索TMDB网站，直接抓取结果，结果只有一条时才返回 搜索TMDB网站，直接抓取结果，结果只有一条时才返回
app/modules/themoviedb/tmdbv3api/tmdb.py
+3 -3
View File
@@ -4,11 +4,12 @@ import logging
import os import os
import time import time
from datetime import datetime from datetime import datetime
from functools import lru_cache
import requests import requests
import requests.exceptions import requests.exceptions
from cachetools import TTLCache, cached
from app.core.config import settings
from app.utils.http import RequestUtils from app.utils.http import RequestUtils
from .exceptions import TMDbException from .exceptions import TMDbException
@@ -24,7 +25,6 @@ class TMDb(object):
TMDB_CACHE_ENABLED = "TMDB_CACHE_ENABLED" TMDB_CACHE_ENABLED = "TMDB_CACHE_ENABLED"
TMDB_PROXIES = "TMDB_PROXIES" TMDB_PROXIES = "TMDB_PROXIES"
TMDB_DOMAIN = "TMDB_DOMAIN" TMDB_DOMAIN = "TMDB_DOMAIN"
REQUEST_CACHE_MAXSIZE = None
_req = None _req = None
_session = None _session = None
@@ -137,7 +137,7 @@ class TMDb(object):
def cache(self, cache): def cache(self, cache):
os.environ[self.TMDB_CACHE_ENABLED] = str(cache) os.environ[self.TMDB_CACHE_ENABLED] = str(cache)
@lru_cache(maxsize=REQUEST_CACHE_MAXSIZE) @cached(cache=TTLCache(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"]))
def cached_request(self, method, url, data, json, def cached_request(self, method, url, data, json,
_ts=datetime.strftime(datetime.now(), '%Y%m%d')): _ts=datetime.strftime(datetime.now(), '%Y%m%d')):
""" """
app/scheduler.py
+1 -1
View File
@@ -330,7 +330,7 @@ class Scheduler(metaclass=Singleton):
"interval", "interval",
id="clear_cache", id="clear_cache",
name="缓存清理", name="缓存清理",
hours=settings.CACHE_CONF.get("meta") / 3600, hours=settings.CACHE_CONF["meta"] / 3600,
kwargs={ kwargs={
'job_id': 'clear_cache' 'job_id': 'clear_cache'
} }