add site parsers

app/modules/indexer/spider/__init__.py

@@ -0,0 +1,761 @@
+import copy
+import datetime
+import re
+import traceback
+from typing import List
+from urllib.parse import quote, urlencode, urlparse, parse_qs
+
+import chardet
+from jinja2 import Template
+from pyquery import PyQuery
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.helper.browser import PlaywrightHelper
+from app.log import logger
+from app.schemas.types import MediaType
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class TorrentSpider:
+    # 是否出现错误
+    is_error: bool = False
+    # 索引器ID
+    indexerid: int = None
+    # 索引器名称
+    indexername: str = None
+    # 站点域名
+    domain: str = None
+    # 站点Cookie
+    cookie: str = None
+    # 站点UA
+    ua: str = None
+    # Requests 代理
+    proxies: dict = None
+    # playwright 代理
+    proxy_server: dict = None
+    # 是否渲染
+    render: bool = False
+    # Referer
+    referer: str = None
+    # 搜索关键字
+    keyword: str = None
+    # 媒体类型
+    mtype: MediaType = None
+    # 搜索路径、方式配置
+    search: dict = {}
+    # 批量搜索配置
+    batch: dict = {}
+    # 浏览配置
+    browse: dict = {}
+    # 站点分类配置
+    category: dict = {}
+    # 站点种子列表配置
+    list: dict = {}
+    # 站点种子字段配置
+    fields: dict = {}
+    # 页码
+    page: int = 0
+    # 搜索条数, 默认: 100条
+    result_num: int = 100
+    # 单个种子信息
+    torrents_info: dict = {}
+    # 种子列表
+    torrents_info_array: list = []
+    # 搜索超时, 默认: 15秒
+    _timeout = 15
+
+    def __init__(self,
+                 indexer: CommentedMap,
+                 keyword: [str, list] = None,
+                 page: int = 0,
+                 referer: str = None,
+                 mtype: MediaType = None):
+        """
+        设置查询参数
+        :param indexer: 索引器
+        :param keyword: 搜索关键字，如果数组则为批量搜索
+        :param page: 页码
+        :param referer: Referer
+        :param mtype: 媒体类型
+        """
+        if not indexer:
+            return
+        self.keyword = keyword
+        self.mtype = mtype
+        self.indexerid = indexer.get('id')
+        self.indexername = indexer.get('name')
+        self.search = indexer.get('search')
+        self.batch = indexer.get('batch')
+        self.browse = indexer.get('browse')
+        self.category = indexer.get('category')
+        self.list = indexer.get('torrents').get('list', {})
+        self.fields = indexer.get('torrents').get('fields')
+        self.render = indexer.get('render')
+        self.domain = indexer.get('domain')
+        self.result_num = int(indexer.get('result_num') or 100)
+        self._timeout = int(indexer.get('timeout') or 15)
+        self.page = page
+        if self.domain and not str(self.domain).endswith("/"):
+            self.domain = self.domain + "/"
+        if indexer.get('ua'):
+            self.ua = indexer.get('ua') or settings.USER_AGENT
+        else:
+            self.ua = settings.USER_AGENT
+        if indexer.get('proxy'):
+            self.proxies = settings.PROXY
+            self.proxy_server = settings.PROXY_SERVER
+        if indexer.get('cookie'):
+            self.cookie = indexer.get('cookie')
+        if referer:
+            self.referer = referer
+        self.torrents_info_array = []
+
+    def get_torrents(self) -> List[dict]:
+        """
+        开始请求
+        """
+        if not self.search or not self.domain:
+            return []
+
+        # 种子搜索相对路径
+        paths = self.search.get('paths', [])
+        torrentspath = ""
+        if len(paths) == 1:
+            torrentspath = paths[0].get('path', '')
+        else:
+            for path in paths:
+                if path.get("type") == "all" and not self.mtype:
+                    torrentspath = path.get('path')
+                    break
+                elif path.get("type") == "movie" and self.mtype == MediaType.MOVIE:
+                    torrentspath = path.get('path')
+                    break
+                elif path.get("type") == "tv" and self.mtype == MediaType.TV:
+                    torrentspath = path.get('path')
+                    break
+
+        # 精确搜索
+        if self.keyword:
+
+            if isinstance(self.keyword, list):
+                # 批量查询
+                if self.batch:
+                    delimiter = self.batch.get('delimiter') or ' '
+                    space_replace = self.batch.get('space_replace') or ' '
+                    search_word = delimiter.join([str(k).replace(' ',
+                                                                 space_replace) for k in self.keyword])
+                else:
+                    search_word = " ".join(self.keyword)
+                # 查询模式：或
+                search_mode = "1"
+            else:
+                # 单个查询
+                search_word = self.keyword
+                # 查询模式与
+                search_mode = "0"
+
+            # 搜索URL
+            indexer_params = self.search.get("params", {}).copy()
+            if indexer_params:
+                search_area = indexer_params.get('search_area')
+                # search_area非0表示支持imdbid搜索
+                if (search_area and
+                        (not self.keyword or not self.keyword.startswith('tt'))):
+                    # 支持imdbid搜索，但关键字不是imdbid时，不启用imdbid搜索
+                    indexer_params.pop('search_area')
+                # 变量字典
+                inputs_dict = {
+                    "keyword": search_word
+                }
+                # 查询参数，默认查询标题
+                params = {
+                    "search_mode": search_mode,
+                    "search_area": 0,
+                    "page": self.page or 0,
+                    "notnewword": 1
+                }
+                # 额外参数
+                for key, value in indexer_params.items():
+                    params.update({
+                        "%s" % key: str(value).format(**inputs_dict)
+                    })
+                # 分类条件
+                if self.category:
+                    if self.mtype == MediaType.TV:
+                        cats = self.category.get("tv") or []
+                    elif self.mtype == MediaType.MOVIE:
+                        cats = self.category.get("movie") or []
+                    else:
+                        cats = (self.category.get("movie") or []) + (self.category.get("tv") or [])
+                    for cat in cats:
+                        if self.category.get("field"):
+                            value = params.get(self.category.get("field"), "")
+                            params.update({
+                                "%s" % self.category.get("field"): value + self.category.get("delimiter",
+                                                                                             ' ') + cat.get("id")
+                            })
+                        else:
+                            params.update({
+                                "cat%s" % cat.get("id"): 1
+                            })
+                searchurl = self.domain + torrentspath + "?" + urlencode(params)
+            else:
+                # 变量字典
+                inputs_dict = {
+                    "keyword": quote(search_word),
+                    "page": self.page or 0
+                }
+                # 无额外参数
+                searchurl = self.domain + str(torrentspath).format(**inputs_dict)
+
+        # 列表浏览
+        else:
+            # 变量字典
+            inputs_dict = {
+                "page": self.page or 0,
+                "keyword": ""
+            }
+            # 有单独浏览路径
+            if self.browse:
+                torrentspath = self.browse.get("path")
+                if self.browse.get("start"):
+                    start_page = int(self.browse.get("start")) + int(self.page or 0)
+                    inputs_dict.update({
+                        "page": start_page
+                    })
+            elif self.page:
+                torrentspath = torrentspath + f"?page={self.page}"
+            # 搜索Url
+            searchurl = self.domain + str(torrentspath).format(**inputs_dict)
+
+        logger.info(f"开始请求：{searchurl}")
+
+        if self.render:
+            # 浏览器仿真
+            page_source = PlaywrightHelper().get_page_source(
+                url=searchurl,
+                cookies=self.cookie,
+                ua=self.ua,
+                proxies=self.proxy_server,
+                timeout=self._timeout
+            )
+        else:
+            # requests请求
+            ret = RequestUtils(
+                ua=self.ua,
+                cookies=self.cookie,
+                timeout=self._timeout,
+                referer=self.referer,
+                proxies=self.proxies
+            ).get_res(searchurl, allow_redirects=True)
+            if ret is not None:
+                # 使用chardet检测字符编码
+                raw_data = ret.content
+                if raw_data:
+                    try:
+                        result = chardet.detect(raw_data)
+                        encoding = result['encoding']
+                        # 解码为字符串
+                        page_source = raw_data.decode(encoding)
+                    except Exception as e:
+                        logger.debug(f"chardet解码失败：{str(e)}")
+                        # 探测utf-8解码
+                        if re.search(r"charset=\"?utf-8\"?", ret.text, re.IGNORECASE):
+                            ret.encoding = "utf-8"
+                        else:
+                            ret.encoding = ret.apparent_encoding
+                        page_source = ret.text
+                else:
+                    page_source = ret.text
+            else:
+                page_source = ""
+
+        # 解析
+        return self.parse(page_source)
+
+    def __get_title(self, torrent):
+        # title default text
+        if 'title' not in self.fields:
+            return
+        selector = self.fields.get('title', {})
+        if 'selector' in selector:
+            title = torrent(selector.get('selector', '')).clone()
+            self.__remove(title, selector)
+            items = self.__attribute_or_text(title, selector)
+            self.torrents_info['title'] = self.__index(items, selector)
+        elif 'text' in selector:
+            render_dict = {}
+            if "title_default" in self.fields:
+                title_default_selector = self.fields.get('title_default', {})
+                title_default_item = torrent(title_default_selector.get('selector', '')).clone()
+                self.__remove(title_default_item, title_default_selector)
+                items = self.__attribute_or_text(title_default_item, selector)
+                title_default = self.__index(items, title_default_selector)
+                render_dict.update({'title_default': title_default})
+            if "title_optional" in self.fields:
+                title_optional_selector = self.fields.get('title_optional', {})
+                title_optional_item = torrent(title_optional_selector.get('selector', '')).clone()
+                self.__remove(title_optional_item, title_optional_selector)
+                items = self.__attribute_or_text(title_optional_item, title_optional_selector)
+                title_optional = self.__index(items, title_optional_selector)
+                render_dict.update({'title_optional': title_optional})
+            self.torrents_info['title'] = Template(selector.get('text')).render(fields=render_dict)
+        self.torrents_info['title'] = self.__filter_text(self.torrents_info.get('title'),
+                                                         selector.get('filters'))
+
+    def __get_description(self, torrent):
+        # title optional text
+        if 'description' not in self.fields:
+            return
+        selector = self.fields.get('description', {})
+        if "selector" in selector \
+                or "selectors" in selector:
+            description = torrent(selector.get('selector', selector.get('selectors', ''))).clone()
+            if description:
+                self.__remove(description, selector)
+                items = self.__attribute_or_text(description, selector)
+                self.torrents_info['description'] = self.__index(items, selector)
+        elif "text" in selector:
+            render_dict = {}
+            if "tags" in self.fields:
+                tags_selector = self.fields.get('tags', {})
+                tags_item = torrent(tags_selector.get('selector', '')).clone()
+                self.__remove(tags_item, tags_selector)
+                items = self.__attribute_or_text(tags_item, tags_selector)
+                tag = self.__index(items, tags_selector)
+                render_dict.update({'tags': tag})
+            if "subject" in self.fields:
+                subject_selector = self.fields.get('subject', {})
+                subject_item = torrent(subject_selector.get('selector', '')).clone()
+                self.__remove(subject_item, subject_selector)
+                items = self.__attribute_or_text(subject_item, subject_selector)
+                subject = self.__index(items, subject_selector)
+                render_dict.update({'subject': subject})
+            if "description_free_forever" in self.fields:
+                description_free_forever_selector = self.fields.get("description_free_forever", {})
+                description_free_forever_item = torrent(description_free_forever_selector.get("selector", '')).clone()
+                self.__remove(description_free_forever_item, description_free_forever_selector)
+                items = self.__attribute_or_text(description_free_forever_item, description_free_forever_selector)
+                description_free_forever = self.__index(items, description_free_forever_selector)
+                render_dict.update({"description_free_forever": description_free_forever})
+            if "description_normal" in self.fields:
+                description_normal_selector = self.fields.get("description_normal", {})
+                description_normal_item = torrent(description_normal_selector.get("selector", '')).clone()
+                self.__remove(description_normal_item, description_normal_selector)
+                items = self.__attribute_or_text(description_normal_item, description_normal_selector)
+                description_normal = self.__index(items, description_normal_selector)
+                render_dict.update({"description_normal": description_normal})
+            self.torrents_info['description'] = Template(selector.get('text')).render(fields=render_dict)
+        self.torrents_info['description'] = self.__filter_text(self.torrents_info.get('description'),
+                                                               selector.get('filters'))
+
+    def __get_detail(self, torrent):
+        # details page text
+        if 'details' not in self.fields:
+            return
+        selector = self.fields.get('details', {})
+        details = torrent(selector.get('selector', '')).clone()
+        self.__remove(details, selector)
+        items = self.__attribute_or_text(details, selector)
+        item = self.__index(items, selector)
+        detail_link = self.__filter_text(item, selector.get('filters'))
+        if detail_link:
+            if not detail_link.startswith("http"):
+                if detail_link.startswith("//"):
+                    self.torrents_info['page_url'] = self.domain.split(":")[0] + ":" + detail_link
+                elif detail_link.startswith("/"):
+                    self.torrents_info['page_url'] = self.domain + detail_link[1:]
+                else:
+                    self.torrents_info['page_url'] = self.domain + detail_link
+            else:
+                self.torrents_info['page_url'] = detail_link
+
+    def __get_download(self, torrent):
+        # download link text
+        if 'download' not in self.fields:
+            return
+        selector = self.fields.get('download', {})
+        download = torrent(selector.get('selector', '')).clone()
+        self.__remove(download, selector)
+        items = self.__attribute_or_text(download, selector)
+        item = self.__index(items, selector)
+        download_link = self.__filter_text(item, selector.get('filters'))
+        if download_link:
+            if not download_link.startswith("http") \
+                    and not download_link.startswith("magnet"):
+                _scheme, _domain = StringUtils.get_url_netloc(self.domain)
+                if _domain in download_link:
+                    if download_link.startswith("/"):
+                        self.torrents_info['enclosure'] = f"{_scheme}:{download_link}"
+                    else:
+                        self.torrents_info['enclosure'] = f"{_scheme}://{download_link}"
+                else:
+                    if download_link.startswith("/"):
+                        self.torrents_info['enclosure'] = f"{self.domain}{download_link[1:]}"
+                    else:
+                        self.torrents_info['enclosure'] = f"{self.domain}{download_link}"
+            else:
+                self.torrents_info['enclosure'] = download_link
+
+    def __get_imdbid(self, torrent):
+        # imdbid
+        if "imdbid" not in self.fields:
+            return
+        selector = self.fields.get('imdbid', {})
+        imdbid = torrent(selector.get('selector', '')).clone()
+        self.__remove(imdbid, selector)
+        items = self.__attribute_or_text(imdbid, selector)
+        item = self.__index(items, selector)
+        self.torrents_info['imdbid'] = item
+        self.torrents_info['imdbid'] = self.__filter_text(self.torrents_info.get('imdbid'),
+                                                          selector.get('filters'))
+
+    def __get_size(self, torrent):
+        # torrent size int
+        if 'size' not in self.fields:
+            return
+        selector = self.fields.get('size', {})
+        size = torrent(selector.get('selector', selector.get("selectors", ''))).clone()
+        self.__remove(size, selector)
+        items = self.__attribute_or_text(size, selector)
+        item = self.__index(items, selector)
+        if item:
+            size_val = item.replace("\n", "").strip()
+            size_val = self.__filter_text(size_val,
+                                          selector.get('filters'))
+            self.torrents_info['size'] = StringUtils.num_filesize(size_val)
+        else:
+            self.torrents_info['size'] = 0
+
+    def __get_leechers(self, torrent):
+        # torrent leechers int
+        if 'leechers' not in self.fields:
+            return
+        selector = self.fields.get('leechers', {})
+        leechers = torrent(selector.get('selector', '')).clone()
+        self.__remove(leechers, selector)
+        items = self.__attribute_or_text(leechers, selector)
+        item = self.__index(items, selector)
+        if item:
+            peers_val = item.split("/")[0]
+            peers_val = peers_val.replace(",", "")
+            peers_val = self.__filter_text(peers_val,
+                                           selector.get('filters'))
+            self.torrents_info['peers'] = int(peers_val) if peers_val and peers_val.isdigit() else 0
+        else:
+            self.torrents_info['peers'] = 0
+
+    def __get_seeders(self, torrent):
+        # torrent leechers int
+        if 'seeders' not in self.fields:
+            return
+        selector = self.fields.get('seeders', {})
+        seeders = torrent(selector.get('selector', '')).clone()
+        self.__remove(seeders, selector)
+        items = self.__attribute_or_text(seeders, selector)
+        item = self.__index(items, selector)
+        if item:
+            seeders_val = item.split("/")[0]
+            seeders_val = seeders_val.replace(",", "")
+            seeders_val = self.__filter_text(seeders_val,
+                                             selector.get('filters'))
+            self.torrents_info['seeders'] = int(seeders_val) if seeders_val and seeders_val.isdigit() else 0
+        else:
+            self.torrents_info['seeders'] = 0
+
+    def __get_grabs(self, torrent):
+        # torrent grabs int
+        if 'grabs' not in self.fields:
+            return
+        selector = self.fields.get('grabs', {})
+        grabs = torrent(selector.get('selector', '')).clone()
+        self.__remove(grabs, selector)
+        items = self.__attribute_or_text(grabs, selector)
+        item = self.__index(items, selector)
+        if item:
+            grabs_val = item.split("/")[0]
+            grabs_val = grabs_val.replace(",", "")
+            grabs_val = self.__filter_text(grabs_val,
+                                           selector.get('filters'))
+            self.torrents_info['grabs'] = int(grabs_val) if grabs_val and grabs_val.isdigit() else 0
+        else:
+            self.torrents_info['grabs'] = 0
+
+    def __get_pubdate(self, torrent):
+        # torrent pubdate yyyy-mm-dd hh:mm:ss
+        if 'date_added' not in self.fields:
+            return
+        selector = self.fields.get('date_added', {})
+        pubdate = torrent(selector.get('selector', '')).clone()
+        self.__remove(pubdate, selector)
+        items = self.__attribute_or_text(pubdate, selector)
+        pubdate_str = self.__index(items, selector)
+        if pubdate_str:
+            pubdate_str = pubdate_str.replace('\n', ' ').strip()
+        self.torrents_info['pubdate'] = self.__filter_text(pubdate_str,
+                                                           selector.get('filters'))
+
+    def __get_date_elapsed(self, torrent):
+        # torrent data elaspsed text
+        if 'date_elapsed' not in self.fields:
+            return
+        selector = self.fields.get('date_elapsed', {})
+        date_elapsed = torrent(selector.get('selector', '')).clone()
+        self.__remove(date_elapsed, selector)
+        items = self.__attribute_or_text(date_elapsed, selector)
+        self.torrents_info['date_elapsed'] = self.__index(items, selector)
+        self.torrents_info['date_elapsed'] = self.__filter_text(self.torrents_info.get('date_elapsed'),
+                                                                selector.get('filters'))
+
+    def __get_downloadvolumefactor(self, torrent):
+        # downloadvolumefactor int
+        selector = self.fields.get('downloadvolumefactor', {})
+        if not selector:
+            return
+        self.torrents_info['downloadvolumefactor'] = 1
+        if 'case' in selector:
+            for downloadvolumefactorselector in list(selector.get('case', {}).keys()):
+                downloadvolumefactor = torrent(downloadvolumefactorselector)
+                if len(downloadvolumefactor) > 0:
+                    self.torrents_info['downloadvolumefactor'] = selector.get('case', {}).get(
+                        downloadvolumefactorselector)
+                    break
+        elif "selector" in selector:
+            downloadvolume = torrent(selector.get('selector', '')).clone()
+            self.__remove(downloadvolume, selector)
+            items = self.__attribute_or_text(downloadvolume, selector)
+            item = self.__index(items, selector)
+            if item:
+                downloadvolumefactor = re.search(r'(\d+\.?\d*)', item)
+                if downloadvolumefactor:
+                    self.torrents_info['downloadvolumefactor'] = int(downloadvolumefactor.group(1))
+
+    def __get_uploadvolumefactor(self, torrent):
+        # uploadvolumefactor int
+        selector = self.fields.get('uploadvolumefactor', {})
+        if not selector:
+            return
+        self.torrents_info['uploadvolumefactor'] = 1
+        if 'case' in selector:
+            for uploadvolumefactorselector in list(selector.get('case', {}).keys()):
+                uploadvolumefactor = torrent(uploadvolumefactorselector)
+                if len(uploadvolumefactor) > 0:
+                    self.torrents_info['uploadvolumefactor'] = selector.get('case', {}).get(
+                        uploadvolumefactorselector)
+                    break
+        elif "selector" in selector:
+            uploadvolume = torrent(selector.get('selector', '')).clone()
+            self.__remove(uploadvolume, selector)
+            items = self.__attribute_or_text(uploadvolume, selector)
+            item = self.__index(items, selector)
+            if item:
+                uploadvolumefactor = re.search(r'(\d+\.?\d*)', item)
+                if uploadvolumefactor:
+                    self.torrents_info['uploadvolumefactor'] = int(uploadvolumefactor.group(1))
+
+    def __get_labels(self, torrent):
+        # labels ['label1', 'label2']
+        if 'labels' not in self.fields:
+            return
+        selector = self.fields.get('labels', {})
+        labels = torrent(selector.get("selector", "")).clone()
+        self.__remove(labels, selector)
+        items = self.__attribute_or_text(labels, selector)
+        if items:
+            self.torrents_info['labels'] = [item for item in items if item]
+        else:
+            self.torrents_info['labels'] = []
+
+    def __get_free_date(self, torrent):
+        # free date yyyy-mm-dd hh:mm:ss
+        if 'freedate' not in self.fields:
+            return
+        selector = self.fields.get('freedate', {})
+        freedate = torrent(selector.get('selector', '')).clone()
+        self.__remove(freedate, selector)
+        items = self.__attribute_or_text(freedate, selector)
+        self.torrents_info['freedate'] = self.__index(items, selector)
+        self.torrents_info['freedate'] = self.__filter_text(self.torrents_info.get('freedate'),
+                                                            selector.get('filters'))
+
+    def __get_hit_and_run(self, torrent):
+        # hitandrun True/False
+        if 'hr' not in self.fields:
+            return
+        selector = self.fields.get('hr', {})
+        hit_and_run = torrent(selector.get('selector', ''))
+        if hit_and_run:
+            self.torrents_info['hit_and_run'] = True
+        else:
+            self.torrents_info['hit_and_run'] = False
+
+    def __get_category(self, torrent):
+        # category 电影/电视剧
+        if 'category' not in self.fields:
+            return
+        selector = self.fields.get('category', {})
+        category = torrent(selector.get('selector', '')).clone()
+        self.__remove(category, selector)
+        items = self.__attribute_or_text(category, selector)
+        category_value = self.__index(items, selector)
+        category_value = self.__filter_text(category_value,
+                                            selector.get('filters'))
+        if category_value and self.category:
+            tv_cats = [str(cat.get("id")) for cat in self.category.get("tv") or []]
+            movie_cats = [str(cat.get("id")) for cat in self.category.get("movie") or []]
+            if category_value in tv_cats \
+                    and category_value not in movie_cats:
+                self.torrents_info['category'] = MediaType.TV.value
+            elif category_value in movie_cats:
+                self.torrents_info['category'] = MediaType.MOVIE.value
+            else:
+                self.torrents_info['category'] = MediaType.UNKNOWN.value
+        else:
+            self.torrents_info['category'] = MediaType.UNKNOWN.value
+
+    def get_info(self, torrent) -> dict:
+        """
+        解析单条种子数据
+        """
+        self.torrents_info = {}
+        try:
+            # 标题
+            self.__get_title(torrent)
+            # 描述
+            self.__get_description(torrent)
+            # 详情页面
+            self.__get_detail(torrent)
+            # 下载链接
+            self.__get_download(torrent)
+            # 完成数
+            self.__get_grabs(torrent)
+            # 下载数
+            self.__get_leechers(torrent)
+            # 做种数
+            self.__get_seeders(torrent)
+            # 大小
+            self.__get_size(torrent)
+            # IMDBID
+            self.__get_imdbid(torrent)
+            # 下载系数
+            self.__get_downloadvolumefactor(torrent)
+            # 上传系数
+            self.__get_uploadvolumefactor(torrent)
+            # 发布时间
+            self.__get_pubdate(torrent)
+            # 已发布时间
+            self.__get_date_elapsed(torrent)
+            # 免费载止时间
+            self.__get_free_date(torrent)
+            # 标签
+            self.__get_labels(torrent)
+            # HR
+            self.__get_hit_and_run(torrent)
+            # 分类
+            self.__get_category(torrent)
+
+        except Exception as err:
+            logger.error("%s 搜索出现错误：%s" % (self.indexername, str(err)))
+        return self.torrents_info
+
+    @staticmethod
+    def __filter_text(text: str, filters: list):
+        """
+        对文件进行处理
+        """
+        if not text or not filters or not isinstance(filters, list):
+            return text
+        if not isinstance(text, str):
+            text = str(text)
+        for filter_item in filters:
+            if not text:
+                break
+            method_name = filter_item.get("name")
+            try:
+                args = filter_item.get("args")
+                if method_name == "re_search" and isinstance(args, list):
+                    rematch = re.search(r"%s" % args[0], text)
+                    if rematch:
+                        text = rematch.group(args[-1])
+                elif method_name == "split" and isinstance(args, list):
+                    text = text.split(r"%s" % args[0])[args[-1]]
+                elif method_name == "replace" and isinstance(args, list):
+                    text = text.replace(r"%s" % args[0], r"%s" % args[-1])
+                elif method_name == "dateparse" and isinstance(args, str):
+                    text = text.replace("\n", " ").strip()
+                    text = datetime.datetime.strptime(text, r"%s" % args)
+                elif method_name == "strip":
+                    text = text.strip()
+                elif method_name == "appendleft":
+                    text = f"{args}{text}"
+                elif method_name == "querystring":
+                    parsed_url = urlparse(text)
+                    query_params = parse_qs(parsed_url.query)
+                    param_value = query_params.get(args)
+                    text = param_value[0] if param_value else ''
+            except Exception as err:
+                logger.debug(f'过滤器 {method_name} 处理失败：{str(err)} - {traceback.format_exc()}')
+        return text.strip()
+
+    @staticmethod
+    def __remove(item, selector):
+        """
+        移除元素
+        """
+        if selector and "remove" in selector:
+            removelist = selector.get('remove', '').split(', ')
+            for v in removelist:
+                item.remove(v)
+
+    @staticmethod
+    def __attribute_or_text(item, selector: dict):
+        if not selector:
+            return item
+        if not item:
+            return []
+        if 'attribute' in selector:
+            items = [i.attr(selector.get('attribute')) for i in item.items() if i]
+        else:
+            items = [i.text() for i in item.items() if i]
+        return items
+
+    @staticmethod
+    def __index(items: list, selector: dict):
+        if not items:
+            return None
+        if selector:
+            if "contents" in selector \
+                    and len(items) > int(selector.get("contents")):
+                items = items[0].split("\n")[selector.get("contents")]
+            elif "index" in selector \
+                    and len(items) > int(selector.get("index")):
+                items = items[int(selector.get("index"))]
+        if isinstance(items, list):
+            items = items[0]
+        return items
+
+    def parse(self, html_text: str) -> List[dict]:
+        """
+        解析整个页面
+        """
+        if not html_text:
+            self.is_error = True
+            return []
+        # 清空旧结果
+        self.torrents_info_array = []
+        try:
+            # 解析站点文本对象
+            html_doc = PyQuery(html_text)
+            # 种子筛选器
+            torrents_selector = self.list.get('selector', '')
+            # 遍历种子html列表
+            for torn in html_doc(torrents_selector):
+                self.torrents_info_array.append(copy.deepcopy(self.get_info(PyQuery(torn))))
+                if len(self.torrents_info_array) >= int(self.result_num):
+                    break
+            return self.torrents_info_array
+        except Exception as err:
+            self.is_error = True
+            logger.warn(f"错误：{self.indexername} {str(err)}")

app/modules/indexer/spider/haidan.py

@@ -0,0 +1,167 @@
+import urllib.parse
+from typing import Tuple, List
+
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.db.systemconfig_oper import SystemConfigOper
+from app.log import logger
+from app.schemas import MediaType
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class HaiDanSpider:
+    """
+    haidan.video API
+    """
+    _indexerid = None
+    _domain = None
+    _url = None
+    _name = ""
+    _proxy = None
+    _cookie = None
+    _ua = None
+    _size = 100
+    _searchurl = "%storrents.php"
+    _detailurl = "%sdetails.php?group_id=%s&torrent_id=%s"
+    _timeout = 15
+
+    # 电影分类
+    _movie_category = ['401', '404', '405']
+    _tv_category = ['402', '403', '404', '405']
+
+    # 足销状态 1-普通，2-免费，3-2X，4-2X免费，5-50%，6-2X50%，7-30%
+    _dl_state = {
+        "1": 1,
+        "2": 0,
+        "3": 1,
+        "4": 0,
+        "5": 0.5,
+        "6": 0.5,
+        "7": 0.3
+    }
+    _up_state = {
+        "1": 1,
+        "2": 1,
+        "3": 2,
+        "4": 2,
+        "5": 1,
+        "6": 2,
+        "7": 1
+    }
+
+    def __init__(self, indexer: CommentedMap):
+        self.systemconfig = SystemConfigOper()
+        if indexer:
+            self._indexerid = indexer.get('id')
+            self._url = indexer.get('domain')
+            self._domain = StringUtils.get_url_domain(self._url)
+            self._searchurl = self._searchurl % self._url
+            self._name = indexer.get('name')
+            if indexer.get('proxy'):
+                self._proxy = settings.PROXY
+            self._cookie = indexer.get('cookie')
+            self._ua = indexer.get('ua')
+            self._timeout = indexer.get('timeout') or 15
+
+    def search(self, keyword: str, mtype: MediaType = None) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+
+        def __dict_to_query(_params: dict):
+            """
+            将数组转换为逗号分隔的字符串
+            """
+            for key, value in _params.items():
+                if isinstance(value, list):
+                    _params[key] = ','.join(map(str, value))
+            return urllib.parse.urlencode(params)
+
+        # 检查cookie
+        if not self._cookie:
+            return True, []
+
+        if not mtype:
+            categories = []
+        elif mtype == MediaType.TV:
+            categories = self._tv_category
+        else:
+            categories = self._movie_category
+
+        # 搜索类型
+        if keyword.startswith('tt'):
+            search_area = '4'
+        else:
+            search_area = '0'
+
+        params = {
+            "isapi": "1",
+            "search_area": search_area,  # 0-标题 1-简介（较慢）3-发种用户名 4-IMDb
+            "search": keyword,
+            "search_mode": "0",  # 0-与 1-或 2-精准
+            "cat": categories
+        }
+        res = RequestUtils(
+            cookies=self._cookie,
+            ua=self._ua,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url=f"{self._searchurl}?{__dict_to_query(params)}")
+        torrents = []
+        if res and res.status_code == 200:
+            result = res.json()
+            code = result.get('code')
+            if code != 0:
+                logger.warn(f"{self._name} 搜索失败：{result.get('msg')}")
+                return True, []
+            data = result.get('data') or {}
+            for tid, item in data.items():
+                category_value = result.get('category')
+                if category_value in self._tv_category \
+                        and category_value not in self._movie_category:
+                    category = MediaType.TV.value
+                elif category_value in self._movie_category:
+                    category = MediaType.MOVIE.value
+                else:
+                    category = MediaType.UNKNOWN.value
+                torrent = {
+                    'title': item.get('name'),
+                    'description': item.get('small_descr'),
+                    'enclosure': item.get('url'),
+                    'pubdate': StringUtils.format_timestamp(item.get('added')),
+                    'size': int(item.get('size') or '0'),
+                    'seeders': int(item.get('seeders') or '0'),
+                    'peers': int(item.get("leechers") or '0'),
+                    'grabs': int(item.get("times_completed") or '0'),
+                    'downloadvolumefactor': self.__get_downloadvolumefactor(item.get('sp_state')),
+                    'uploadvolumefactor': self.__get_uploadvolumefactor(item.get('sp_state')),
+                    'page_url': self._detailurl % (self._url, item.get('group_id'), tid),
+                    'labels': [],
+                    'category': category
+                }
+                torrents.append(torrent)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+        return False, torrents
+
+    def __get_downloadvolumefactor(self, discount: str) -> float:
+        """
+        获取下载系数
+        """
+        if discount:
+            return self._dl_state.get(discount, 1)
+        return 1
+
+    def __get_uploadvolumefactor(self, discount: str) -> float:
+        """
+        获取上传系数
+        """
+        if discount:
+            return self._up_state.get(discount, 1)
+        return 1

app/modules/indexer/spider/mtorrent.py

@@ -0,0 +1,205 @@
+import base64
+import json
+import re
+from typing import Tuple, List
+
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.db.systemconfig_oper import SystemConfigOper
+from app.log import logger
+from app.schemas import MediaType
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class MTorrentSpider:
+    """
+    mTorrent API
+    """
+    _indexerid = None
+    _domain = None
+    _url = None
+    _name = ""
+    _proxy = None
+    _cookie = None
+    _ua = None
+    _size = 100
+    _searchurl = "https://api.%s/api/torrent/search"
+    _downloadurl = "https://api.%s/api/torrent/genDlToken"
+    _pageurl = "%sdetail/%s"
+    _timeout = 15
+
+    # 电影分类
+    _movie_category = ['401', '419', '420', '421', '439', '405', '404']
+    _tv_category = ['403', '402', '435', '438', '404', '405']
+
+    # API KEY
+    _apikey = None
+    # JWT Token
+    _token = None
+
+    # 标签
+    _labels = {
+        "0": "",
+        "1": "DIY",
+        "2": "国配",
+        "3": "DIY 国配",
+        "4": "中字",
+        "5": "DIY 中字",
+        "6": "国配 中字",
+        "7": "DIY 国配 中字"
+    }
+
+    def __init__(self, indexer: CommentedMap):
+        self.systemconfig = SystemConfigOper()
+        if indexer:
+            self._indexerid = indexer.get('id')
+            self._url = indexer.get('domain')
+            self._domain = StringUtils.get_url_domain(self._url)
+            self._searchurl = self._searchurl % self._domain
+            self._name = indexer.get('name')
+            if indexer.get('proxy'):
+                self._proxy = settings.PROXY
+            self._cookie = indexer.get('cookie')
+            self._ua = indexer.get('ua')
+            self._apikey = indexer.get('apikey')
+            self._token = indexer.get('token')
+            self._timeout = indexer.get('timeout') or 15
+
+    def search(self, keyword: str, mtype: MediaType = None, page: int = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+        # 检查ApiKey
+        if not self._apikey:
+            return True, []
+
+        if not mtype:
+            categories = []
+        elif mtype == MediaType.TV:
+            categories = self._tv_category
+        else:
+            categories = self._movie_category
+        params = {
+            "keyword": keyword,
+            "categories": categories,
+            "pageNumber": int(page) + 1,
+            "pageSize": self._size,
+            "visible": 1
+        }
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "x-api-key": self._apikey
+            },
+            proxies=self._proxy,
+            referer=f"{self._domain}browse",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        torrents = []
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("data") or []
+            for result in results:
+                category_value = result.get('category')
+                if category_value in self._tv_category \
+                        and category_value not in self._movie_category:
+                    category = MediaType.TV.value
+                elif category_value in self._movie_category:
+                    category = MediaType.MOVIE.value
+                else:
+                    category = MediaType.UNKNOWN.value
+                labels_value = self._labels.get(result.get('labels') or "0") or ""
+                if labels_value:
+                    labels = labels_value.split()
+                else:
+                    labels = []
+                torrent = {
+                    'title': result.get('name'),
+                    'description': result.get('smallDescr'),
+                    'enclosure': self.__get_download_url(result.get('id')),
+                    'pubdate': StringUtils.format_timestamp(result.get('createdDate')),
+                    'size': int(result.get('size') or '0'),
+                    'seeders': int(result.get('status', {}).get("seeders") or '0'),
+                    'peers': int(result.get('status', {}).get("leechers") or '0'),
+                    'grabs': int(result.get('status', {}).get("timesCompleted") or '0'),
+                    'downloadvolumefactor': self.__get_downloadvolumefactor(result.get('status', {}).get("discount")),
+                    'uploadvolumefactor': self.__get_uploadvolumefactor(result.get('status', {}).get("discount")),
+                    'page_url': self._pageurl % (self._url, result.get('id')),
+                    'imdbid': self.__find_imdbid(result.get('imdb')),
+                    'labels': labels,
+                    'category': category
+                }
+                torrents.append(torrent)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+        return False, torrents
+
+    @staticmethod
+    def __find_imdbid(imdb: str) -> str:
+        """
+        从imdb链接中提取imdbid
+        """
+        if imdb:
+            m = re.search(r"tt\d+", imdb)
+            if m:
+                return m.group(0)
+        return ""
+
+    @staticmethod
+    def __get_downloadvolumefactor(discount: str) -> float:
+        """
+        获取下载系数
+        """
+        discount_dict = {
+            "FREE": 0,
+            "PERCENT_50": 0.5,
+            "PERCENT_70": 0.3,
+            "_2X_FREE": 0,
+            "_2X_PERCENT_50": 0.5
+        }
+        if discount:
+            return discount_dict.get(discount, 1)
+        return 1
+
+    @staticmethod
+    def __get_uploadvolumefactor(discount: str) -> float:
+        """
+        获取上传系数
+        """
+        uploadvolumefactor_dict = {
+            "_2X": 2.0,
+            "_2X_FREE": 2.0,
+            "_2X_PERCENT_50": 2.0
+        }
+        if discount:
+            return uploadvolumefactor_dict.get(discount, 1)
+        return 1
+
+    def __get_download_url(self, torrent_id: str) -> str:
+        """
+        获取下载链接，返回base64编码的json字符串及URL
+        """
+        url = self._downloadurl % self._domain
+        params = {
+            'method': 'post',
+            'cookie': False,
+            'params': {
+                'id': torrent_id
+            },
+            'header': {
+                'Content-Type': 'application/json',
+                'User-Agent': f'{self._ua}',
+                'Accept': 'application/json, text/plain, */*',
+                'x-api-key': self._apikey
+            },
+            'result': 'data'
+        }
+        # base64编码
+        base64_str = base64.b64encode(json.dumps(params).encode('utf-8')).decode('utf-8')
+        return f"[{base64_str}]{url}"

app/modules/indexer/spider/tnode.py

@@ -0,0 +1,109 @@
+import re
+from typing import Tuple, List
+
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.log import logger
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class TNodeSpider:
+    _indexerid = None
+    _domain = None
+    _name = ""
+    _proxy = None
+    _cookie = None
+    _ua = None
+    _token = None
+    _size = 100
+    _timeout = 15
+    _searchurl = "%sapi/torrent/advancedSearch"
+    _downloadurl = "%sapi/torrent/download/%s"
+    _pageurl = "%storrent/info/%s"
+
+    def __init__(self, indexer: CommentedMap):
+        if indexer:
+            self._indexerid = indexer.get('id')
+            self._domain = indexer.get('domain')
+            self._searchurl = self._searchurl % self._domain
+            self._name = indexer.get('name')
+            if indexer.get('proxy'):
+                self._proxy = settings.PROXY
+            self._cookie = indexer.get('cookie')
+            self._ua = indexer.get('ua')
+            self._timeout = indexer.get('timeout') or 15
+        self.init_config()
+
+    def init_config(self):
+        self.__get_token()
+
+    def __get_token(self):
+        if not self._domain:
+            return
+        res = RequestUtils(ua=self._ua,
+                           cookies=self._cookie,
+                           proxies=self._proxy,
+                           timeout=self._timeout).get_res(url=self._domain)
+        if res and res.status_code == 200:
+            csrf_token = re.search(r'<meta name="x-csrf-token" content="(.+?)">', res.text)
+            if csrf_token:
+                self._token = csrf_token.group(1)
+
+    def search(self, keyword: str, page: int = 0) -> Tuple[bool, List[dict]]:
+        if not self._token:
+            logger.warn(f"{self._name} 未获取到token，无法搜索")
+            return True, []
+        search_type = "imdbid" if (keyword and keyword.startswith('tt')) else "title"
+        params = {
+            "page": int(page) + 1,
+            "size": self._size,
+            "type": search_type,
+            "keyword": keyword or "",
+            "sorter": "id",
+            "order": "desc",
+            "tags": [],
+            "category": [501, 502, 503, 504],
+            "medium": [],
+            "videoCoding": [],
+            "audioCoding": [],
+            "resolution": [],
+            "group": []
+        }
+        res = RequestUtils(
+            headers={
+                'X-CSRF-TOKEN': self._token,
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._ua}"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        torrents = []
+        if res and res.status_code == 200:
+            results = res.json().get('data', {}).get("torrents") or []
+            for result in results:
+                torrent = {
+                    'title': result.get('title'),
+                    'description': result.get('subtitle'),
+                    'enclosure': self._downloadurl % (self._domain, result.get('id')),
+                    'pubdate': StringUtils.format_timestamp(result.get('upload_time')),
+                    'size': result.get('size'),
+                    'seeders': result.get('seeding'),
+                    'peers': result.get('leeching'),
+                    'grabs': result.get('complete'),
+                    'downloadvolumefactor': result.get('downloadRate'),
+                    'uploadvolumefactor': result.get('uploadRate'),
+                    'page_url': self._pageurl % (self._domain, result.get('id')),
+                    'imdbid': result.get('imdb')
+                }
+                torrents.append(torrent)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+        return False, torrents

app/modules/indexer/spider/torrentleech.py

@@ -0,0 +1,72 @@
+from typing import List, Tuple
+from urllib.parse import quote
+
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.log import logger
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class TorrentLeech:
+    _indexer = None
+    _proxy = None
+    _size = 100
+    _searchurl = "%storrents/browse/list/query/%s"
+    _browseurl = "%storrents/browse/list/page/2%s"
+    _downloadurl = "%sdownload/%s/%s"
+    _pageurl = "%storrent/%s"
+    _timeout = 15
+
+    def __init__(self, indexer: CommentedMap):
+        self._indexer = indexer
+        if indexer.get('proxy'):
+            self._proxy = settings.PROXY
+            self._timeout = indexer.get('timeout') or 15
+
+    def search(self, keyword: str, page: int = 0) -> Tuple[bool, List[dict]]:
+
+        if StringUtils.is_chinese(keyword):
+            # 不支持中文
+            return True, []
+
+        if keyword:
+            url = self._searchurl % (self._indexer.get('domain'), quote(keyword))
+        else:
+            url = self._browseurl % (self._indexer.get('domain'), int(page) + 1)
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json; charset=utf-8",
+                "User-Agent": f"{self._indexer.get('ua')}",
+            },
+            cookies=self._indexer.get('cookie'),
+            proxies=self._proxy,
+            timeout=self._timeout
+        ).get_res(url)
+        torrents = []
+        if res and res.status_code == 200:
+            results = res.json().get('torrentList') or []
+            for result in results:
+                torrent = {
+                    'title': result.get('name'),
+                    'enclosure': self._downloadurl % (self._indexer.get('domain'), result.get('fid'), result.get('filename')),
+                    'pubdate': StringUtils.format_timestamp(result.get('addedTimestamp')),
+                    'size': result.get('size'),
+                    'seeders': result.get('seeders'),
+                    'peers': result.get('leechers'),
+                    'grabs': result.get('completed'),
+                    'downloadvolumefactor': result.get('download_multiplier'),
+                    'uploadvolumefactor': 1,
+                    'page_url': self._pageurl % (self._indexer.get('domain'), result.get('fid')),
+                    'imdbid': result.get('imdbID')
+                }
+                torrents.append(torrent)
+        elif res is not None:
+            logger.warn(f"{self._indexer.get('name')} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._indexer.get('name')} 搜索失败，无法连接 {self._indexer.get('domain')}")
+            return True, []
+
+        return False, torrents

app/modules/indexer/spider/yema.py

@@ -0,0 +1,148 @@
+from typing import Tuple, List
+
+from ruamel.yaml import CommentedMap
+
+from app.core.config import settings
+from app.db.systemconfig_oper import SystemConfigOper
+from app.log import logger
+from app.schemas import MediaType
+from app.utils.http import RequestUtils
+from app.utils.string import StringUtils
+
+
+class YemaSpider:
+    """
+    YemaPT API
+    """
+    _indexerid = None
+    _domain = None
+    _name = ""
+    _proxy = None
+    _cookie = None
+    _ua = None
+    _size = 40
+    _searchurl = "%sapi/torrent/fetchCategoryOpenTorrentList"
+    _downloadurl = "%sapi/torrent/download?id=%s"
+    _pageurl = "%s#/torrent/detail/%s/"
+    _timeout = 15
+
+    # 分类
+    _movie_category = 4
+    _tv_category = 5
+
+    def __init__(self, indexer: CommentedMap):
+        self.systemconfig = SystemConfigOper()
+        if indexer:
+            self._indexerid = indexer.get('id')
+            self._domain = indexer.get('domain')
+            self._searchurl = self._searchurl % self._domain
+            self._name = indexer.get('name')
+            if indexer.get('proxy'):
+                self._proxy = settings.PROXY
+            self._cookie = indexer.get('cookie')
+            self._ua = indexer.get('ua')
+            self._timeout = indexer.get('timeout') or 15
+
+    def search(self, keyword: str, mtype: MediaType = None, page: int = 0) -> Tuple[bool, List[dict]]:
+        """
+        搜索
+        """
+        if not mtype:
+            categoryId = self._movie_category
+        elif mtype == MediaType.TV:
+            categoryId = self._tv_category
+        else:
+            categoryId = self._movie_category
+        params = {
+            "categoryId": categoryId,
+            "pageParam": {
+                "current": page + 1,
+                "pageSize": self._size,
+                "total": self._size
+            },
+            "sorter": {}
+        }
+        if keyword:
+            params.update({
+                "keyword": keyword,
+            })
+        res = RequestUtils(
+            headers={
+                "Content-Type": "application/json",
+                "User-Agent": f"{self._ua}",
+                "Accept": "application/json, text/plain, */*"
+            },
+            cookies=self._cookie,
+            proxies=self._proxy,
+            referer=f"{self._domain}",
+            timeout=self._timeout
+        ).post_res(url=self._searchurl, json=params)
+        torrents = []
+        if res and res.status_code == 200:
+            results = res.json().get('data', []) or []
+            for result in results:
+                category_value = result.get('categoryId')
+                if category_value == self._tv_category:
+                    category = MediaType.TV.value
+                elif category_value == self._movie_category:
+                    category = MediaType.MOVIE.value
+                else:
+                    category = MediaType.UNKNOWN.value
+                torrent = {
+                    'title': result.get('showName'),
+                    'description': result.get('shortDesc'),
+                    'enclosure': self.__get_download_url(result.get('id')),
+                    'pubdate': StringUtils.unify_datetime_str(result.get('gmtCreate')),
+                    'size': result.get('fileSize'),
+                    'seeders': result.get('seedNum'),
+                    'peers': result.get('leechNum'),
+                    'grabs': result.get('completedNum'),
+                    'downloadvolumefactor': self.__get_downloadvolumefactor(result.get('downloadPromotion')),
+                    'uploadvolumefactor': self.__get_uploadvolumefactor(result.get('uploadPromotion')),
+                    'freedate': StringUtils.unify_datetime_str(result.get('downloadPromotionEndTime')),
+                    'page_url': self._pageurl % (self._domain, result.get('id')),
+                    'labels': [],
+                    'category': category
+                }
+                torrents.append(torrent)
+        elif res is not None:
+            logger.warn(f"{self._name} 搜索失败，错误码：{res.status_code}")
+            return True, []
+        else:
+            logger.warn(f"{self._name} 搜索失败，无法连接 {self._domain}")
+            return True, []
+        return False, torrents
+
+    @staticmethod
+    def __get_downloadvolumefactor(discount: str) -> float:
+        """
+        获取下载系数
+        """
+        discount_dict = {
+            "free": 0,
+            "half": 0.5,
+            "none": 1
+        }
+        if discount:
+            return discount_dict.get(discount, 1)
+        return 1
+
+    @staticmethod
+    def __get_uploadvolumefactor(discount: str) -> float:
+        """
+        获取上传系数
+        """
+        discount_dict = {
+            "none": 1,
+            "one_half": 1.5,
+            "double_upload": 2
+        }
+        if discount:
+            return discount_dict.get(discount, 1)
+        return 1
+
+    def __get_download_url(self, torrent_id: str) -> str:
+        """
+        获取下载链接
+        """
+        return self._downloadurl % (self._domain, torrent_id)