fix: handle tmdb gzip json responses

This commit is contained in:
jxxghp
2026-05-20 16:54:01 +08:00
parent e00aa42f94
commit 47c4e84fdd
2 changed files with 73 additions and 6 deletions

View File

@@ -22,6 +22,7 @@ logger = logging.getLogger(__name__)
class TMDb(object):
_RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__"
_JSON_DECODE_FAILED = object()
_MAX_GZIP_DECODE_DEPTH = 3
def __init__(self, session=None, language=None):
self._api_key = settings.TMDB_API_KEY
@@ -37,14 +38,27 @@ class TMDb(object):
if not self._session:
self._session = requests.Session()
self._req = RequestUtils(ua=settings.NORMAL_USER_AGENT, session=self._session, proxies=self.proxies)
request_headers = self._build_request_headers()
self._req = RequestUtils(headers=request_headers, session=self._session, proxies=self.proxies)
self._async_req = AsyncRequestUtils(ua=settings.NORMAL_USER_AGENT, proxies=self.proxies)
self._async_req = AsyncRequestUtils(headers=request_headers, proxies=self.proxies)
self._remaining = 40
self._reset = None
self._timeout = 15
@staticmethod
def _build_request_headers():
"""
构造TMDB JSON请求头避免小体积JSON被代理重复压缩。
"""
return {
"User-Agent": settings.NORMAL_USER_AGENT,
"Content-Type": "application/x-www-form-urlencoded; charset=UTF-8",
"Accept": "application/json",
"Accept-Encoding": "identity",
}
@property
def page(self):
return self._page
@@ -197,10 +211,27 @@ class TMDb(object):
if "gzip" not in encodings and not content_bytes.startswith(b"\x1f\x8b"):
return cls._JSON_DECODE_FAILED
try:
return jsonlib.loads(gzip.decompress(content_bytes))
except (OSError, EOFError, ValueError, UnicodeDecodeError):
return cls._JSON_DECODE_FAILED
for json_payload in cls._iter_gzip_decoded_payloads(content_bytes):
try:
return jsonlib.loads(json_payload)
except (ValueError, UnicodeDecodeError):
continue
return cls._JSON_DECODE_FAILED
@classmethod
def _iter_gzip_decoded_payloads(cls, content_bytes: bytes):
"""
逐层解开gzip响应体兼容客户端或代理只解压了部分层级的情况。
"""
current_payload = content_bytes
for _ in range(cls._MAX_GZIP_DECODE_DEPTH):
if not current_payload.startswith(b"\x1f\x8b"):
return
try:
current_payload = gzip.decompress(current_payload)
except (OSError, EOFError):
return
yield current_payload
@staticmethod
def _get_header_value(headers, name):