fix indexer async

app/modules/indexer/__init__.py

@@ -214,7 +214,8 @@ class IndexerModule(_ModuleBase):
             logger.warn(f"{site.get('name')} 未搜索到数据，共搜索 {search_count} 次，耗时 {seconds} 秒")
             return []
         else:
-            logger.info(f"{site.get('name')} 搜索完成，共搜索 {search_count} 次，耗时 {seconds} 秒，返回数据：{len(result_array)}")
+            logger.info(
+                f"{site.get('name')} 搜索完成，共搜索 {search_count} 次，耗时 {seconds} 秒，返回数据：{len(result_array)}")
             # TorrentInfo
             torrents = [TorrentInfo(site=site.get("id"),
                                     site_name=site.get("name"),
@@ -252,11 +253,40 @@ class IndexerModule(_ModuleBase):
         try:
             return _spider.is_error, _spider.get_torrents()
         finally:
-            # 显式清理SiteSpider对象
+            del _spider
+
+    @staticmethod
+    async def __async_spider_search(indexer: dict,
+                                    search_word: Optional[str] = None,
+                                    mtype: MediaType = None,
+                                    cat: Optional[str] = None,
+                                    page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        异步根据关键字搜索单个站点
+        :param: indexer: 站点配置
+        :param: search_word: 关键字
+        :param: cat: 分类
+        :param: page: 页码
+        :param: mtype: 媒体类型
+        :param: timeout: 超时时间
+        :return: 是否发生错误, 种子列表
+        """
+        _spider = SiteSpider(indexer=indexer,
+                             keyword=search_word,
+                             mtype=mtype,
+                             cat=cat,
+                             page=page)
+
+        try:
+            result = await _spider.async_get_torrents()
+            return _spider.is_error, result
+        finally:
             del _spider
 
     def refresh_torrents(self, site: dict,
-                         keyword: Optional[str] = None, cat: Optional[str] = None, page: Optional[int] = 0) -> Optional[List[TorrentInfo]]:
+                         keyword: Optional[str] = None,
+                         cat: Optional[str] = None,
+                         page: Optional[int] = 0) -> Optional[List[TorrentInfo]]:
         """
         获取站点最新一页的种子，多个站点需要多线程处理
         :param site:  站点

app/modules/indexer/spider/__init__.py

@@ -5,13 +5,14 @@ from typing import Any, Optional
 from typing import List
 from urllib.parse import quote, urlencode, urlparse, parse_qs
 
+from fastapi.concurrency import run_in_threadpool
 from jinja2 import Template
 from pyquery import PyQuery
 
 from app.core.config import settings
 from app.log import logger
 from app.schemas.types import MediaType
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -80,13 +81,10 @@ class SiteSpider:
         self.torrents_info = {}
         self.torrents_info_array = []
 
-    def get_torrents(self) -> List[dict]:
+    def __get_search_url(self):
         """
-        开始请求
+        获取搜索URL
         """
-        if not self.search or not self.domain:
-            return []
-
         # 种子搜索相对路径
         paths = self.search.get('paths', [])
         torrentspath = ""
@@ -200,6 +198,18 @@ class SiteSpider:
             # 搜索Url
             searchurl = self.domain + str(torrentspath).format(**inputs_dict)
 
+        return searchurl
+
+    def get_torrents(self) -> List[dict]:
+        """
+        开始请求
+        """
+        if not self.search or not self.domain:
+            return []
+
+        # 获取搜索URL
+        searchurl = self.__get_search_url()
+
         logger.info(f"开始请求：{searchurl}")
 
         # requests请求
@@ -219,6 +229,36 @@ class SiteSpider:
             )
         )
 
+    async def async_get_torrents(self) -> List[dict]:
+        """
+        异步请求
+        """
+        if not self.search or not self.domain:
+            return []
+
+        # 获取搜索URL
+        searchurl = self.__get_search_url()
+
+        logger.info(f"开始异步请求：{searchurl}")
+
+        # httpx请求
+        ret = await AsyncRequestUtils(
+            ua=self.ua,
+            cookies=self.cookie,
+            timeout=self._timeout,
+            referer=self.referer,
+            proxies=self.proxies
+        ).get_res(searchurl, allow_redirects=True)
+        # 解析返回
+        return await run_in_threadpool(
+            self.parse,
+            RequestUtils.get_decoded_html_content(
+                ret,
+                performance_mode=settings.ENCODING_DETECTION_PERFORMANCE_MODE,
+                confidence_threshold=settings.ENCODING_DETECTION_MIN_CONFIDENCE
+            )
+        )
+
     def __get_title(self, torrent: Any):
         # title default text
         if 'title' not in self.fields:

app/modules/indexer/spider/haidan.py

@@ -5,7 +5,7 @@ from app.core.config import settings
 from app.db.systemconfig_oper import SystemConfigOper
 from app.log import logger
 from app.schemas import MediaType
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -63,9 +63,9 @@ class HaiDanSpider:
             self._ua = indexer.get('ua')
             self._timeout = indexer.get('timeout') or 15
 
-    def search(self, keyword: str, mtype: MediaType = None) -> Tuple[bool, List[dict]]:
+    def __get_params(self, keyword: str, mtype: MediaType = None) -> dict:
         """
-        搜索
+        获取请求参数
         """
 
         def __dict_to_query(_params: dict):
@@ -75,11 +75,7 @@ class HaiDanSpider:
             for key, value in _params.items():
                 if isinstance(value, list):
                     _params[key] = ','.join(map(str, value))
-            return urllib.parse.urlencode(params)
+            return urllib.parse.urlencode(_params)
-
-        # 检查cookie
-        if not self._cookie:
-            return True, []
 
         if not mtype:
             categories = []
@@ -94,26 +90,19 @@ class HaiDanSpider:
         else:
             search_area = '0'
 
-        params = {
+        return __dict_to_query({
             "isapi": "1",
             "search_area": search_area,  # 0-标题 1-简介（较慢）3-发种用户名 4-IMDb
             "search": keyword,
             "search_mode": "0",  # 0-与 1-或 2-精准
             "cat": categories
-        }
+        })
-        res = RequestUtils(
+
-            cookies=self._cookie,
+    def __parse_result(self, result: dict):
-            ua=self._ua,
+        """
-            proxies=self._proxy,
+        解析结果
-            timeout=self._timeout
+        """
-        ).get_res(url=f"{self._searchurl}?{__dict_to_query(params)}")
         torrents = []
-        if res and res.status_code == 200:
-            result = res.json()
-            code = result.get('code')
-            if code != 0:
-                logger.warn(f"{self._name} 搜索失败：{result.get('msg')}")
-                return True, []
         data = result.get('data') or {}
         for tid, item in data.items():
             category_value = result.get('category')
@@ -140,13 +129,73 @@ class HaiDanSpider:
                 'category': category
             }
             torrents.append(torrent)
+        return torrents
+
+    def search(self, keyword: str, mtype: MediaType = None) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+
+        # 检查cookie
+        if not self._cookie:
+            return True, []
+
+        # 获取参数
+        params_str = self.__get_params(keyword, mtype)
+
+        # 发送请求
+        res = RequestUtils(
+            cookies=self._cookie,
+            ua=self._ua,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url=f"{self._searchurl}?{params_str}")
+        if res and res.status_code == 200:
+            result = res.json()
+            code = result.get('code')
+            if code != 0:
+                logger.warn(f"{self._name} 搜索失败：{result.get('msg')}")
+                return True, []
+            return False, self.__parse_result(result)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+
+    async def async_search(self, keyword: str, mtype: MediaType = None) -> Tuple[bool, List[dict]]:
+        """
+        异步搜索
+        """
+        # 检查cookie
+        if not self._cookie:
+            return True, []
+
+        # 获取参数
+        params_str = self.__get_params(keyword, mtype)
+
+        # 发送请求
+        res = await AsyncRequestUtils(
+            cookies=self._cookie,
+            ua=self._ua,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url=f"{self._searchurl}?{params_str}")
+
+        if res and res.status_code == 200:
+            result = res.json()
+            code = result.get('code')
+            if code != 0:
+                logger.warn(f"{self._name} 搜索失败：{result.get('msg')}")
+                return True, []
+            return False, self.__parse_result(result)
         elif res is not None:
             logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
             return True, []
         else:
             logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
             return True, []
-        return False, torrents
 
     def __get_downloadvolumefactor(self, discount: str) -> float:
         """

app/modules/indexer/spider/hddolby.py

@@ -4,7 +4,7 @@ from app.core.config import settings
 from app.db.systemconfig_oper import SystemConfigOper
 from app.log import logger
 from app.schemas import MediaType
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -73,11 +73,10 @@ class HddolbySpider:
             self._searchurl = f"https://api.{self._domain_host}/api/v1/torrent/search"
             self._pageurl = f"{self._domain}details.php?id=%s&hit=1"
 
-    def search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+    def __get_params(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> dict:
         """
-        搜索
+        获取请求参数
         """
-
         if mtype == MediaType.TV:
             categories = self._tv_category
         elif mtype == MediaType.MOVIE:
@@ -86,7 +85,7 @@ class HddolbySpider:
             categories = list(set(self._movie_category + self._tv_category))
 
         # 输入参数
-        params = {
+        return {
             "keyword": keyword,
             "page_number": page,
             "page_size": 100,
@@ -94,20 +93,14 @@ class HddolbySpider:
             "visible": 1,
         }
 
-        res = RequestUtils(
+    def __parse_result(self, results: List[dict]) -> List[dict]:
-            headers={
+        """
-                "Content-Type": "application/json",
+        解析搜索结果
-                "Accept": "application/json, text/plain, */*",
+        """
-                "x-api-key": self._apikey
-            },
-            cookies=self._cookie,
-            proxies=self._proxy,
-            referer=f"{self._domain}",
-            timeout=self._timeout
-        ).post_res(url=self._searchurl, json=params)
         torrents = []
-        if res and res.status_code == 200:
+        if not results:
-            results = res.json().get('data', []) or []
+            return []
+
         for result in results:
             """
             {
@@ -167,13 +160,66 @@ class HddolbySpider:
                 'category': category
             }
             torrents.append(torrent)
+        return torrents
+
+    def search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+
+        # 准备参数
+        params = self.__get_params(keyword, mtype, page)
+
+        # 发送请求
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "application/json, text/plain, */*",
+                "x-api-key": self._apikey
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            referer=f"{self._domain}",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', []) or []
+            return False, self.__parse_result(results)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+
+    async def async_search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        异步搜索
+        """
+        # 准备参数
+        params = self.__get_params(keyword, mtype, page)
+
+        # 发送请求
+        res = await AsyncRequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "Accept": "application/json, text/plain, */*",
+                "x-api-key": self._apikey
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            referer=f"{self._domain}",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', []) or []
+            return False, self.__parse_result(results)
         elif res is not None:
             logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
             return True, []
         else:
             logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
             return True, []
-        return False, torrents
 
     @staticmethod
     def __get_downloadvolumefactor(discount: int) -> float:

app/modules/indexer/spider/mtorrent.py

@@ -7,7 +7,7 @@ from app.core.config import settings
 from app.db.systemconfig_oper import SystemConfigOper
 from app.log import logger
 from app.schemas import MediaType
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -65,40 +65,32 @@ class MTorrentSpider:
             self._token = indexer.get('token')
             self._timeout = indexer.get('timeout') or 15
 
-    def search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+    def __get_params(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> dict:
         """
-        搜索
+        获取请求参数
         """
-        # 检查ApiKey
-        if not self._apikey:
-            return True, []
-
         if not mtype:
             categories = []
         elif mtype == MediaType.TV:
             categories = self._tv_category
         else:
             categories = self._movie_category
-        params = {
+        return {
             "keyword": keyword,
             "categories": categories,
             "pageNumber": int(page) + 1,
             "pageSize": self._size,
             "visible": 1
         }
-        res = RequestUtils(
+
-            headers={
+    def __parse_result(self, results: List[dict]):
-                "Content-Type": "application/json",
+        """
-                "User-Agent": f"{self._ua}",
+        解析搜索结果
-                "x-api-key": self._apikey
+        """
-            },
-            proxies=self._proxy,
-            referer=f"{self._domain}browse",
-            timeout=self._timeout
-        ).post_res(url=self._searchurl, json=params)
         torrents = []
-        if res and res.status_code == 200:
+        if not results:
-            results = res.json().get('data', {}).get("data") or []
+            return torrents
+
         for result in results:
             category_value = result.get('category')
             if category_value in self._tv_category \
@@ -136,13 +128,71 @@ class MTorrentSpider:
                 'category': category
             }
             torrents.append(torrent)
+        return torrents
+
+    def search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+        # 检查ApiKey
+        if not self._apikey:
+            return True, []
+
+        # 获取请求参数
+        params = self.__get_params(keyword, mtype, page)
+
+        # 发送请求
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "x-api-key": self._apikey
+            },
+            proxies=self._proxy,
+            referer=f"{self._domain}browse",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("data") or []
+            return False, self.__parse_result(results)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+
+    async def async_search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+        # 检查ApiKey
+        if not self._apikey:
+            return True, []
+
+        # 获取请求参数
+        params = self.__get_params(keyword, mtype, page)
+
+        # 发送请求
+        res = await AsyncRequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "x-api-key": self._apikey
+            },
+            proxies=self._proxy,
+            referer=f"{self._domain}browse",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("data") or []
+            return False, self.__parse_result(results)
         elif res is not None:
             logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
             return True, []
         else:
             logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
             return True, []
-        return False, torrents
 
     @staticmethod
     def __find_imdbid(imdb: str) -> str:

app/modules/indexer/spider/tnode.py

@@ -1,23 +1,18 @@
 import re
 from typing import Tuple, List, Optional
 
+from app.core.cache import cached
 from app.core.config import settings
 from app.log import logger
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
+from app.utils.singleton import Singleton
 from app.utils.string import StringUtils
 
 
-class TNodeSpider:
+class TNodeSpider(metaclass=Singleton):
-    _indexerid = None
-    _domain = None
-    _name = ""
-    _proxy = None
-    _cookie = None
-    _ua = None
-    _token = None
     _size = 100
     _timeout = 15
-    _searchurl = "%sapi/torrent/advancedSearch"
+    _baseurl = "%sapi/torrent/advancedSearch"
     _downloadurl = "%sapi/torrent/download/%s"
     _pageurl = "%storrent/info/%s"
 
@@ -25,19 +20,16 @@ class TNodeSpider:
         if indexer:
             self._indexerid = indexer.get('id')
             self._domain = indexer.get('domain')
-            self._searchurl = self._searchurl % self._domain
+            self._searchurl = self._baseurl % self._domain
             self._name = indexer.get('name')
             if indexer.get('proxy'):
                 self._proxy = settings.PROXY
             self._cookie = indexer.get('cookie')
             self._ua = indexer.get('ua')
             self._timeout = indexer.get('timeout') or 15
-        self.init_config()
 
-    def init_config(self):
+    @cached(region="indexer_spider", maxsize=1, ttl=60 * 60 * 24, skip_empty=True)
-        self.__get_token()
+    def __get_token(self) -> Optional[str]:
-
-    def __get_token(self):
         if not self._domain:
             return
         res = RequestUtils(ua=self._ua,
@@ -47,14 +39,29 @@ class TNodeSpider:
         if res and res.status_code == 200:
             csrf_token = re.search(r'<meta name="x-csrf-token" content="(.+?)">', res.text)
             if csrf_token:
-                self._token = csrf_token.group(1)
+                return csrf_token.group(1)
+        return None
 
-    def search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+    @cached(region="indexer_spider", maxsize=1, ttl=60 * 60 * 24, skip_empty=True)
-        if not self._token:
+    async def __async_get_token(self) -> Optional[str]:
-            logger.warn(f"{self._name} 未获取到token，无法搜索")
+        if not self._domain:
-            return True, []
+            return
+        res = await AsyncRequestUtils(ua=self._ua,
+                                      cookies=self._cookie,
+                                      proxies=self._proxy,
+                                      timeout=self._timeout).get_res(url=self._domain)
+        if res and res.status_code == 200:
+            csrf_token = re.search(r'<meta name="x-csrf-token" content="(.+?)">', res.text)
+            if csrf_token:
+                _token = csrf_token.group(1)
+        return None
+
+    def __get_params(self, keyword: str = None, page: Optional[int] = 0) -> dict:
+        """
+        获取搜索参数
+        """
         search_type = "imdbid" if (keyword and keyword.startswith('tt')) else "title"
-        params = {
+        return {
             "page": int(page) + 1,
             "size": self._size,
             "type": search_type,
@@ -69,19 +76,15 @@ class TNodeSpider:
             "resolution": [],
             "group": []
         }
-        res = RequestUtils(
+
-            headers={
+    def __parse_result(self, results: List[dict]) -> List[dict]:
-                'X-CSRF-TOKEN': self._token,
+        """
-                "Content-Type": "application/json; charset=utf-8",
+        解析搜索结果
-                "User-Agent": f"{self._ua}"
+        """
-            },
-            cookies=self._cookie,
-            proxies=self._proxy,
-            timeout=self._timeout
-        ).post_res(url=self._searchurl, json=params)
         torrents = []
-        if res and res.status_code == 200:
+        if not results:
-            results = res.json().get('data', {}).get("torrents") or []
+            return torrents
+
         for result in results:
             torrent = {
                 'title': result.get('title'),
@@ -98,10 +101,73 @@ class TNodeSpider:
                 'imdbid': result.get('imdb')
             }
             torrents.append(torrent)
+
+        return torrents
+
+    def search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+        # 获取token
+        _token = self.__get_token()
+        if not _token:
+            logger.warn(f"{self._name} 未获取到token，无法搜索")
+            return True, []
+
+        # 获取请求参数
+        params = self.__get_params(keyword, page)
+
+        # 发送请求
+        res = RequestUtils(
+            headers={
+                'X-CSRF-TOKEN': _token,
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._ua}"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("torrents") or []
+            return False, self.__parse_result(results)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+        
+    async def async_search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        异步搜索
+        """
+        # 获取token
+        _token = await self.__async_get_token()
+        if not _token:
+            logger.warn(f"{self._name} 未获取到token，无法搜索")
+            return True, []
+
+        # 获取请求参数
+        params = self.__get_params(keyword, page)
+
+        # 发送请求
+        res = await AsyncRequestUtils(
+            headers={
+                'X-CSRF-TOKEN': _token,
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._ua}"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("torrents") or []
+            return False, self.__parse_result(results)
         elif res is not None:
             logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
             return True, []
         else:
             logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
             return True, []
-        return False, torrents

app/modules/indexer/spider/torrentleech.py

@@ -3,7 +3,7 @@ from urllib.parse import quote
 
 from app.core.config import settings
 from app.log import logger
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -23,32 +23,20 @@ class TorrentLeech:
             self._proxy = settings.PROXY
             self._timeout = indexer.get('timeout') or 15
 
-    def search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+    def __parse_result(self, results: List[dict]) -> List[dict]:
-
+        """
-        if StringUtils.is_chinese(keyword):
+        解析搜索结果
-            # 不支持中文
+        """
-            return True, []
-
-        if keyword:
-            url = self._searchurl % (self._indexer.get('domain'), quote(keyword))
-        else:
-            url = self._browseurl % (self._indexer.get('domain'), int(page) + 1)
-        res = RequestUtils(
-            headers={
-                "Content-Type": "application/json; charset=utf-8",
-                "User-Agent": f"{self._indexer.get('ua')}",
-            },
-            cookies=self._indexer.get('cookie'),
-            proxies=self._proxy,
-            timeout=self._timeout
-        ).get_res(url)
         torrents = []
-        if res and res.status_code == 200:
+        if not results:
-            results = res.json().get('torrentList') or []
+            return torrents
+
         for result in results:
             torrent = {
                 'title': result.get('name'),
-                    'enclosure': self._downloadurl % (self._indexer.get('domain'), result.get('fid'), result.get('filename')),
+                'enclosure': self._downloadurl % (self._indexer.get('domain'),
+                                                  result.get('fid'),
+                                                  result.get('filename')),
                 'pubdate': StringUtils.format_timestamp(result.get('addedTimestamp')),
                 'size': result.get('size'),
                 'seeders': result.get('seeders'),
@@ -60,6 +48,33 @@ class TorrentLeech:
                 'imdbid': result.get('imdbID')
             }
             torrents.append(torrent)
+        return torrents
+
+    def search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索种子
+        """
+        if StringUtils.is_chinese(keyword):
+            # 不支持中文
+            return True, []
+
+        if keyword:
+            url = self._searchurl % (self._indexer.get('domain'), quote(keyword))
+        else:
+            url = self._browseurl % (self._indexer.get('domain'), int(page) + 1)
+
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._indexer.get('ua')}",
+            },
+            cookies=self._indexer.get('cookie'),
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url)
+        if res and res.status_code == 200:
+            results = res.json().get('torrentList') or []
+            return False, self.__parse_result(results)
         elif res is not None:
             logger.warn(f"{self._indexer.get('name')} 搜索失败，错误码：{res.status_code}")
             return True, []
@@ -67,4 +82,34 @@ class TorrentLeech:
             logger.warn(f"{self._indexer.get('name')} 搜索失败，无法连接 {self._indexer.get('domain')}")
             return True, []
 
-        return False, torrents
+    async def async_search(self, keyword: str, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        异步搜索种子
+        """
+        if StringUtils.is_chinese(keyword):
+            # 不支持中文
+            return True, []
+
+        if keyword:
+            url = self._searchurl % (self._indexer.get('domain'), quote(keyword))
+        else:
+            url = self._browseurl % (self._indexer.get('domain'), int(page) + 1)
+
+        res = await AsyncRequestUtils(
+            headers={
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._indexer.get('ua')}",
+            },
+            cookies=self._indexer.get('cookie'),
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url)
+        if res and res.status_code == 200:
+            results = res.json().get('torrentList') or []
+            return False, self.__parse_result(results)
+        elif res is not None:
+            logger.warn(f"{self._indexer.get('name')} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._indexer.get('name')} 搜索失败，无法连接 {self._indexer.get('domain')}")
+            return True, []

app/modules/indexer/spider/yema.py

@@ -4,7 +4,7 @@ from app.core.config import settings
 from app.db.systemconfig_oper import SystemConfigOper
 from app.log import logger
 from app.schemas import MediaType
-from app.utils.http import RequestUtils
+from app.utils.http import RequestUtils, AsyncRequestUtils
 from app.utils.string import StringUtils
 
 
@@ -57,9 +57,9 @@ class YemaSpider:
             self._ua = indexer.get('ua')
             self._timeout = indexer.get('timeout') or 15
 
-    def search(self, keyword: str, mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+    def __get_params(self, keyword: str = None, page: Optional[int] = 0) -> dict:
         """
-        搜索
+        获取搜索参数
         """
         params = {
             "pageParam": {
@@ -69,30 +69,20 @@ class YemaSpider:
             },
             "sorter": {}
         }
-        # 新接口可不传 categoryId 参数
-        # if mtype == MediaType.MOVIE:
-        #     params.update({
-        #         "categoryId": self._movie_category,
-        #     })
-        #     pass
         if keyword:
             params.update({
                 "keyword": keyword,
             })
-        res = RequestUtils(
+        return params
-            headers={
+
-                "Content-Type": "application/json",
+    def __parse_result(self, results: List[dict]) -> List[dict]:
-                "User-Agent": f"{self._ua}",
+        """
-                "Accept": "application/json, text/plain, */*"
+        解析搜索结果
-            },
+        """
-            cookies=self._cookie,
-            proxies=self._proxy,
-            referer=f"{self._domain}",
-            timeout=self._timeout
-        ).post_res(url=self._searchurl, json=params)
         torrents = []
-        if res and res.status_code == 200:
+        if not results:
-            results = res.json().get('data', []) or []
+            return torrents
+
         for result in results:
             category_value = result.get('categoryId')
             if category_value in self._tv_category:
@@ -114,7 +104,6 @@ class YemaSpider:
                 'title': result.get('showName'),
                 'description': result.get('shortDesc'),
                 'enclosure': self.__get_download_url(result.get('id')),
-                    # 使用上架时间，而不是用户发布时间，上架时间即其他用户可见时间
                 'pubdate': StringUtils.unify_datetime_str(result.get('listingTime')),
                 'size': result.get('fileSize'),
                 'seeders': result.get('seedNum'),
@@ -128,13 +117,62 @@ class YemaSpider:
                 'category': category
             }
             torrents.append(torrent)
+
+        return torrents
+
+    def search(self, keyword: str,
+               mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "Accept": "application/json, text/plain, */*"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            referer=f"{self._domain}",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=self.__get_params(keyword, page))
+        if res and res.status_code == 200:
+            results = res.json().get('data', []) or []
+            return False, self.__parse_result(results)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+
+    async def async_search(self, keyword: str,
+                           mtype: MediaType = None, page: Optional[int] = 0) -> Tuple[bool, List[dict]]:
+        """
+        异步搜索
+        """
+        res = await AsyncRequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "Accept": "application/json, text/plain, */*"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            referer=f"{self._domain}",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=self.__get_params(keyword, page))
+
+        if res and res.status_code == 200:
+            results = res.json().get('data', []) or []
+            return False, self.__parse_result(results)
         elif res is not None:
             logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
             return True, []
         else:
             logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
             return True, []
-        return False, torrents
 
     @staticmethod
     def __get_downloadvolumefactor(discount: str) -> float: