fix(cache): enhance tmdb match_web rate-limiting and caching

app/modules/themoviedb/tmdbapi.py

@@ -8,8 +8,10 @@ from lxml import etree
 from app.core.cache import cached
 from app.core.config import settings
 from app.log import logger
+from app.schemas import APIRateLimitException
 from app.schemas.types import MediaType
 from app.utils.http import RequestUtils
+from app.utils.limit import rate_limit_exponential
 from app.utils.string import StringUtils
 from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
 from .tmdbv3api.exceptions import TMDbException
@@ -492,6 +494,7 @@ class TmdbApi:
             return ret_info
 
     @cached(maxsize=settings.CACHE_CONF["tmdb"], ttl=settings.CACHE_CONF["meta"])
+    @rate_limit_exponential(source="match_tmdb_web", max_wait=1800, enable_logging=True)
     def match_web(self, name: str, mtype: MediaType) -> Optional[dict]:
         """
         搜索TMDB网站，直接抓取结果，结果只有一条时才返回
@@ -504,51 +507,56 @@ class TmdbApi:
             return {}
         logger.info("正在从TheDbMovie网站查询：%s ..." % name)
         tmdb_url = "https://www.themoviedb.org/search?query=%s" % quote(name)
-        res = RequestUtils(timeout=5, ua=settings.USER_AGENT).get_res(url=tmdb_url)
+        res = RequestUtils(timeout=5, ua=settings.USER_AGENT, proxies=settings.PROXY).get_res(url=tmdb_url)
-        if res and res.status_code == 200:
+        if res is None:
-            html_text = res.text
+            return None
-            if not html_text:
+        if res.status_code == 429:
-                return None
+            raise APIRateLimitException("触发TheDbMovie网站限流，获取媒体信息失败")
-            try:
+        if res.status_code != 200:
-                tmdb_links = []
+            return {}
-                html = etree.HTML(html_text)
+        html_text = res.text
-                if mtype == MediaType.TV:
+        if not html_text:
-                    links = html.xpath("//a[@data-id and @data-media-type='tv']/@href")
+            return {}
-                else:
+        try:
-                    links = html.xpath("//a[@data-id]/@href")
+            tmdb_links = []
-                for link in links:
+            html = etree.HTML(html_text)
-                    if not link or (not link.startswith("/tv") and not link.startswith("/movie")):
+            if mtype == MediaType.TV:
-                        continue
+                links = html.xpath("//a[@data-id and @data-media-type='tv']/@href")
-                    if link not in tmdb_links:
+            else:
-                        tmdb_links.append(link)
+                links = html.xpath("//a[@data-id]/@href")
-                if len(tmdb_links) == 1:
+            for link in links:
-                    tmdbinfo = self.get_info(
+                if not link or (not link.startswith("/tv") and not link.startswith("/movie")):
-                        mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
+                    continue
-                        tmdbid=tmdb_links[0].split("/")[-1])
+                if link not in tmdb_links:
-                    if tmdbinfo:
+                    tmdb_links.append(link)
-                        if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
+            if len(tmdb_links) == 1:
-                            return {}
+                tmdbinfo = self.get_info(
-                        if tmdbinfo.get('media_type') == MediaType.MOVIE:
+                    mtype=MediaType.TV if tmdb_links[0].startswith("/tv") else MediaType.MOVIE,
-                            logger.info("%s 从WEB识别到 电影：TMDBID=%s, 名称=%s, 上映日期=%s" % (
+                    tmdbid=tmdb_links[0].split("/")[-1])
-                                name,
+                if tmdbinfo:
-                                tmdbinfo.get('id'),
+                    if mtype == MediaType.TV and tmdbinfo.get('media_type') != MediaType.TV:
-                                tmdbinfo.get('title'),
+                        return {}
-                                tmdbinfo.get('release_date')))
+                    if tmdbinfo.get('media_type') == MediaType.MOVIE:
-                        else:
+                        logger.info("%s 从WEB识别到 电影：TMDBID=%s, 名称=%s, 上映日期=%s" % (
-                            logger.info("%s 从WEB识别到 电视剧：TMDBID=%s, 名称=%s, 首播日期=%s" % (
+                            name,
-                                name,
+                            tmdbinfo.get('id'),
-                                tmdbinfo.get('id'),
+                            tmdbinfo.get('title'),
-                                tmdbinfo.get('name'),
+                            tmdbinfo.get('release_date')))
-                                tmdbinfo.get('first_air_date')))
+                    else:
-                    return tmdbinfo
+                        logger.info("%s 从WEB识别到 电视剧：TMDBID=%s, 名称=%s, 首播日期=%s" % (
-                elif len(tmdb_links) > 1:
+                            name,
-                    logger.info("%s TMDB网站返回数据过多：%s" % (name, len(tmdb_links)))
+                            tmdbinfo.get('id'),
-                else:
+                            tmdbinfo.get('name'),
-                    logger.info("%s TMDB网站未查询到媒体信息！" % name)
+                            tmdbinfo.get('first_air_date')))
-            except Exception as err:
+                return tmdbinfo
-                logger.error(f"从TheDbMovie网站查询出错：{str(err)}")
+            elif len(tmdb_links) > 1:
-                return None
+                logger.info("%s TMDB网站返回数据过多：%s" % (name, len(tmdb_links)))
-        return None
+            else:
+                logger.info("%s TMDB网站未查询到媒体信息！" % name)
+        except Exception as err:
+            logger.error(f"从TheDbMovie网站查询出错：{str(err)}")
+            return {}
+        return {}
 
     def get_info(self,
                  mtype: MediaType,