mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-06 07:56:52 +08:00
refactor(tmdb): 空结果缓存过期时间收敛为设置项 EMPTY_RESULT_CACHE_TTL
- ConfigModel TMDB 配置区新增 EMPTY_RESULT_CACHE_TTL(默认 30 分钟),支持环境变量覆盖 - request/async_request/discover/TmdbCache 统一改读 settings,移除模块级常量与跨模块导入
This commit is contained in:
@@ -11,7 +11,6 @@ from app.domain.meta.metabase import MetaBase
|
|||||||
from app.runtime.log import logger
|
from app.runtime.log import logger
|
||||||
from app.schemas.types import MediaSource, MediaType
|
from app.schemas.types import MediaSource, MediaType
|
||||||
from app.foundation.singleton import WeakSingleton
|
from app.foundation.singleton import WeakSingleton
|
||||||
from app.modules.themoviedb.tmdbv3api.tmdb import EMPTY_RESULT_CACHE_TTL
|
|
||||||
|
|
||||||
lock = RLock()
|
lock = RLock()
|
||||||
PERSISTENCE_VERSION = 1
|
PERSISTENCE_VERSION = 1
|
||||||
@@ -262,7 +261,7 @@ class TmdbCache(metaclass=WeakSingleton):
|
|||||||
# 负识别缓存使用独立的短 TTL:故障期间「合法 JSON 但结果为空」会被
|
# 负识别缓存使用独立的短 TTL:故障期间「合法 JSON 但结果为空」会被
|
||||||
# 记为未识别,若按完整有效期固化,故障自愈后同名仍会被判无法识别;
|
# 记为未识别,若按完整有效期固化,故障自愈后同名仍会被判无法识别;
|
||||||
# 短过期让恢复后可重新识别,真不存在的条目过期后重新确认一次即可
|
# 短过期让恢复后可重新识别,真不存在的条目过期后重新确认一次即可
|
||||||
self._set(key, {"id": 0}, ttl=EMPTY_RESULT_CACHE_TTL)
|
self._set(key, {"id": 0}, ttl=settings.EMPTY_RESULT_CACHE_TTL)
|
||||||
|
|
||||||
def save(self, force: bool = False) -> None:
|
def save(self, force: bool = False) -> None:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from app.runtime.cache import cached
|
from app.runtime.cache import cached
|
||||||
from ..tmdb import TMDb, EMPTY_RESULT_CACHE_TTL
|
from app.runtime.config import settings
|
||||||
|
from ..tmdb import TMDb
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from urllib import urlencode
|
from urllib import urlencode
|
||||||
@@ -13,7 +14,7 @@ class Discover(TMDb):
|
|||||||
"tv": "/discover/tv"
|
"tv": "/discover/tv"
|
||||||
}
|
}
|
||||||
|
|
||||||
@cached(maxsize=1, ttl=43200, empty_ttl=EMPTY_RESULT_CACHE_TTL)
|
@cached(maxsize=1, ttl=43200, empty_ttl=settings.EMPTY_RESULT_CACHE_TTL)
|
||||||
def discover_movies(self, params_tuple):
|
def discover_movies(self, params_tuple):
|
||||||
"""
|
"""
|
||||||
Discover movies by different types of data like average rating, number of votes, genres and certifications.
|
Discover movies by different types of data like average rating, number of votes, genres and certifications.
|
||||||
@@ -23,7 +24,7 @@ class Discover(TMDb):
|
|||||||
params = dict(params_tuple)
|
params = dict(params_tuple)
|
||||||
return self._request_obj(self._urls["movies"], urlencode(params), key="results", call_cached=False)
|
return self._request_obj(self._urls["movies"], urlencode(params), key="results", call_cached=False)
|
||||||
|
|
||||||
@cached(maxsize=1, ttl=43200, empty_ttl=EMPTY_RESULT_CACHE_TTL)
|
@cached(maxsize=1, ttl=43200, empty_ttl=settings.EMPTY_RESULT_CACHE_TTL)
|
||||||
def discover_tv_shows(self, params_tuple):
|
def discover_tv_shows(self, params_tuple):
|
||||||
"""
|
"""
|
||||||
Discover TV shows by different types of data like average rating, number of votes, genres,
|
Discover TV shows by different types of data like average rating, number of votes, genres,
|
||||||
@@ -33,7 +34,7 @@ class Discover(TMDb):
|
|||||||
"""
|
"""
|
||||||
return self._request_obj(self._urls["tv"], urlencode(params_tuple), key="results", call_cached=False)
|
return self._request_obj(self._urls["tv"], urlencode(params_tuple), key="results", call_cached=False)
|
||||||
|
|
||||||
@cached(maxsize=1, ttl=43200, empty_ttl=EMPTY_RESULT_CACHE_TTL)
|
@cached(maxsize=1, ttl=43200, empty_ttl=settings.EMPTY_RESULT_CACHE_TTL)
|
||||||
async def async_discover_movies(self, params_tuple):
|
async def async_discover_movies(self, params_tuple):
|
||||||
"""
|
"""
|
||||||
Discover movies by different types of data like average rating, number of votes, genres and certifications.(异步版本)
|
Discover movies by different types of data like average rating, number of votes, genres and certifications.(异步版本)
|
||||||
@@ -43,7 +44,7 @@ class Discover(TMDb):
|
|||||||
params = dict(params_tuple)
|
params = dict(params_tuple)
|
||||||
return await self._async_request_obj(self._urls["movies"], urlencode(params), key="results", call_cached=False)
|
return await self._async_request_obj(self._urls["movies"], urlencode(params), key="results", call_cached=False)
|
||||||
|
|
||||||
@cached(maxsize=1, ttl=43200, empty_ttl=EMPTY_RESULT_CACHE_TTL)
|
@cached(maxsize=1, ttl=43200, empty_ttl=settings.EMPTY_RESULT_CACHE_TTL)
|
||||||
async def async_discover_tv_shows(self, params_tuple):
|
async def async_discover_tv_shows(self, params_tuple):
|
||||||
"""
|
"""
|
||||||
Discover TV shows by different types of data like average rating, number of votes, genres,
|
Discover TV shows by different types of data like average rating, number of votes, genres,
|
||||||
|
|||||||
@@ -22,11 +22,6 @@ logger = logging.getLogger(__name__)
|
|||||||
# 故取区间内的经验值。
|
# 故取区间内的经验值。
|
||||||
RETRY_BACKOFF_SECONDS = 2
|
RETRY_BACKOFF_SECONDS = 2
|
||||||
|
|
||||||
# 空结果缓存的独立过期时间(秒)。TMDB 代理故障期间「合法 JSON 但 results 为空」的
|
|
||||||
# 响应会随默认 TTL(可达数十小时)固化,故障自愈后同 key 仍持续命中空结果;空结果
|
|
||||||
# 改用 30 分钟短 TTL,既能拦住故障窗口内的重复回源,又能在故障恢复后较快自然失效。
|
|
||||||
EMPTY_RESULT_CACHE_TTL = 30 * 60
|
|
||||||
|
|
||||||
|
|
||||||
def _is_business_failure_snapshot(snapshot) -> bool:
|
def _is_business_failure_snapshot(snapshot) -> bool:
|
||||||
"""
|
"""
|
||||||
@@ -47,8 +42,8 @@ def _is_empty_result_snapshot(snapshot) -> bool:
|
|||||||
判断响应快照是否为空结果(列表/搜索类接口的 results 为空列表)。
|
判断响应快照是否为空结果(列表/搜索类接口的 results 为空列表)。
|
||||||
|
|
||||||
这类快照结构合法但无业务内容,常由代理瞬时故障产生;不能靠 skip_none/skip_empty
|
这类快照结构合法但无业务内容,常由代理瞬时故障产生;不能靠 skip_none/skip_empty
|
||||||
识别(快照本身是非空字典),需单独谓词判定后按 EMPTY_RESULT_CACHE_TTL 短 TTL
|
识别(快照本身是非空字典),需单独谓词判定后按 settings.EMPTY_RESULT_CACHE_TTL
|
||||||
缓存。详情类接口无 results 字段,不属于空结果。
|
短 TTL 缓存。详情类接口无 results 字段,不属于空结果。
|
||||||
"""
|
"""
|
||||||
if not isinstance(snapshot, dict):
|
if not isinstance(snapshot, dict):
|
||||||
return False
|
return False
|
||||||
@@ -180,7 +175,7 @@ class TMDb(object):
|
|||||||
|
|
||||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||||
skip_if=_is_business_failure_snapshot,
|
skip_if=_is_business_failure_snapshot,
|
||||||
empty_ttl=EMPTY_RESULT_CACHE_TTL, empty_if=_is_empty_result_snapshot)
|
empty_ttl=settings.EMPTY_RESULT_CACHE_TTL, empty_if=_is_empty_result_snapshot)
|
||||||
def request(self, method, url, data, json, **kwargs):
|
def request(self, method, url, data, json, **kwargs):
|
||||||
req = self._request_once(method, url, data, json)
|
req = self._request_once(method, url, data, json)
|
||||||
if req is None and method == "GET" and self._owns_session:
|
if req is None and method == "GET" and self._owns_session:
|
||||||
@@ -206,7 +201,7 @@ class TMDb(object):
|
|||||||
|
|
||||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||||
skip_if=_is_business_failure_snapshot,
|
skip_if=_is_business_failure_snapshot,
|
||||||
empty_ttl=EMPTY_RESULT_CACHE_TTL, empty_if=_is_empty_result_snapshot)
|
empty_ttl=settings.EMPTY_RESULT_CACHE_TTL, empty_if=_is_empty_result_snapshot)
|
||||||
async def async_request(self, method, url, data, json, **kwargs):
|
async def async_request(self, method, url, data, json, **kwargs):
|
||||||
req = await self._async_request_once(method, url, data, json)
|
req = await self._async_request_once(method, url, data, json)
|
||||||
if req is None:
|
if req is None:
|
||||||
|
|||||||
@@ -255,6 +255,8 @@ class ConfigModel(BaseModel):
|
|||||||
TMDB_API_DOMAIN: str = "api.themoviedb.org"
|
TMDB_API_DOMAIN: str = "api.themoviedb.org"
|
||||||
# TMDB元数据语言
|
# TMDB元数据语言
|
||||||
TMDB_LOCALE: str = "zh"
|
TMDB_LOCALE: str = "zh"
|
||||||
|
# TMDB空结果缓存独立过期时间(秒),故障期间产生的空响应快速过期,故障自愈后可自然恢复
|
||||||
|
EMPTY_RESULT_CACHE_TTL: int = 30 * 60
|
||||||
# 刮削使用TMDB原始语种图片
|
# 刮削使用TMDB原始语种图片
|
||||||
TMDB_SCRAP_ORIGINAL_IMAGE: bool = False
|
TMDB_SCRAP_ORIGINAL_IMAGE: bool = False
|
||||||
# TMDB API Key
|
# TMDB API Key
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
from app.runtime.config import settings
|
from app.runtime.config import settings
|
||||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||||
from app.modules.themoviedb.tmdbv3api.tmdb import EMPTY_RESULT_CACHE_TTL
|
|
||||||
from app.schemas.types import MediaSource, MediaType
|
from app.schemas.types import MediaSource, MediaType
|
||||||
|
|
||||||
|
|
||||||
@@ -133,9 +132,9 @@ def test_update_negative_cache_uses_short_ttl():
|
|||||||
|
|
||||||
key = _key("电视剧")
|
key = _key("电视剧")
|
||||||
assert cache._cache.data[key] == {"id": 0}
|
assert cache._cache.data[key] == {"id": 0}
|
||||||
assert cache._cache.ttls[key] == EMPTY_RESULT_CACHE_TTL
|
assert cache._cache.ttls[key] == settings.EMPTY_RESULT_CACHE_TTL
|
||||||
# 持久化用的过期时间也应随短 TTL 计算,不能沿用默认有效期
|
# 持久化用的过期时间也应随短 TTL 计算,不能沿用默认有效期
|
||||||
assert cache._expires_at[key] <= time() + EMPTY_RESULT_CACHE_TTL + 5
|
assert cache._expires_at[key] <= time() + settings.EMPTY_RESULT_CACHE_TTL + 5
|
||||||
|
|
||||||
|
|
||||||
def test_update_positive_cache_keeps_default_ttl():
|
def test_update_positive_cache_keeps_default_ttl():
|
||||||
|
|||||||
@@ -9,11 +9,11 @@ import asyncio
|
|||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
|
|
||||||
from app.modules.themoviedb.tmdbv3api.tmdb import (
|
from app.modules.themoviedb.tmdbv3api.tmdb import (
|
||||||
EMPTY_RESULT_CACHE_TTL,
|
|
||||||
TMDb,
|
TMDb,
|
||||||
_is_empty_result_snapshot,
|
_is_empty_result_snapshot,
|
||||||
)
|
)
|
||||||
from app.runtime.cache import MemoryBackend
|
from app.runtime.cache import MemoryBackend
|
||||||
|
from app.runtime.config import settings
|
||||||
|
|
||||||
from tests.test_tmdb_response_cache import _FakeResponse
|
from tests.test_tmdb_response_cache import _FakeResponse
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ def _make_tmdb() -> TMDb:
|
|||||||
|
|
||||||
def test_empty_result_cache_ttl_is_thirty_minutes():
|
def test_empty_result_cache_ttl_is_thirty_minutes():
|
||||||
"""空结果缓存的独立过期时间应为 30 分钟。"""
|
"""空结果缓存的独立过期时间应为 30 分钟。"""
|
||||||
assert EMPTY_RESULT_CACHE_TTL == 30 * 60
|
assert settings.EMPTY_RESULT_CACHE_TTL == 30 * 60
|
||||||
|
|
||||||
|
|
||||||
def test_empty_result_snapshot_predicate():
|
def test_empty_result_snapshot_predicate():
|
||||||
@@ -67,7 +67,7 @@ def test_empty_result_is_cached_but_expires_with_short_ttl():
|
|||||||
|
|
||||||
region_cache = _request_region_cache()
|
region_cache = _request_region_cache()
|
||||||
started_at = region_cache.timer()
|
started_at = region_cache.timer()
|
||||||
region_cache.expire(time=started_at + EMPTY_RESULT_CACHE_TTL + 1)
|
region_cache.expire(time=started_at + settings.EMPTY_RESULT_CACHE_TTL + 1)
|
||||||
|
|
||||||
with patch.object(TMDb, "_request_once", return_value=fake) as req:
|
with patch.object(TMDb, "_request_once", return_value=fake) as req:
|
||||||
tmdb.request("GET", url, None, None)
|
tmdb.request("GET", url, None, None)
|
||||||
@@ -87,7 +87,7 @@ def test_non_empty_result_keeps_default_ttl():
|
|||||||
region_cache = _request_region_cache()
|
region_cache = _request_region_cache()
|
||||||
started_at = region_cache.timer()
|
started_at = region_cache.timer()
|
||||||
# 推进到短 TTL 之后:非空结果不应在此刻过期
|
# 推进到短 TTL 之后:非空结果不应在此刻过期
|
||||||
region_cache.expire(time=started_at + EMPTY_RESULT_CACHE_TTL + 1)
|
region_cache.expire(time=started_at + settings.EMPTY_RESULT_CACHE_TTL + 1)
|
||||||
|
|
||||||
tmdb.request("GET", url, None, None)
|
tmdb.request("GET", url, None, None)
|
||||||
assert req.call_count == 1
|
assert req.call_count == 1
|
||||||
@@ -106,7 +106,7 @@ def test_async_empty_result_is_cached_with_short_ttl():
|
|||||||
|
|
||||||
region_cache = _request_region_cache()
|
region_cache = _request_region_cache()
|
||||||
started_at = region_cache.timer()
|
started_at = region_cache.timer()
|
||||||
region_cache.expire(time=started_at + EMPTY_RESULT_CACHE_TTL + 1)
|
region_cache.expire(time=started_at + settings.EMPTY_RESULT_CACHE_TTL + 1)
|
||||||
|
|
||||||
with patch.object(TMDb, "_async_request_once", return_value=fake) as req:
|
with patch.object(TMDb, "_async_request_once", return_value=fake) as req:
|
||||||
asyncio.run(tmdb.async_request("GET", url, None, None))
|
asyncio.run(tmdb.async_request("GET", url, None, None))
|
||||||
|
|||||||
Reference in New Issue
Block a user