# -*- coding: utf-8 -*- import json import re from abc import ABCMeta, abstractmethod from enum import Enum from typing import Optional from urllib.parse import urljoin, urlsplit from requests import Session from app.adapters.network.cloudflare import under_challenge from app.adapters.network.http import RequestUtils from app.domain.site import SiteUtils from app.foundation import size as size_tools from app.runtime.log import logger from app.runtime.settings import get_runtime_setting # 站点框架 class SiteSchema(Enum): """站点用户数据解析框架类型""" DiscuzX = "DiscuzX" Gazelle = "Gazelle" Ipt = "IPTorrents" NexusPhp = "NexusPhp" NexusProject = "NexusProject" NexusRabbit = "NexusRabbit" NexusHhanclub = "NexusHhanclub" NexusAudiences = "NexusAudiences" SmallHorse = "Small Horse" Unit3d = "Unit3d" TorrentLeech = "TorrentLeech" FileList = "FileList" TNode = "TNode" MTorrent = "MTorrent" Yema = "Yema" HDDolby = "HDDolby" Zhixing = "Zhixing" Bitpt = "Bitpt" RousiPro = "RousiPro" SunnyPT = "SunnyPT" class SiteParserBase(metaclass=ABCMeta): """站点用户数据解析器基类""" # 站点模版 schema = None # 请求模式 cookie/apikey request_mode = "cookie" def __init__(self, site_name: str, url: str, site_cookie: str, apikey: str, token: str, session: Session = None, ua: Optional[str] = None, emulate: bool = False, proxy: bool = None, api_url: Optional[str] = None): """ 初始化站点用户数据解析器 :param site_name: 站点名称 :param url: 站点前端地址 :param site_cookie: 站点 Cookie :param apikey: 站点 API Key :param token: 站点 Token :param session: 可复用的 HTTP 会话 :param ua: 请求 User-Agent :param emulate: 是否使用浏览器仿真 :param proxy: 是否使用系统代理 :param api_url: 站点独立 API Base URL """ super().__init__() # 站点信息 self.apikey = apikey self.token = token self._site_name = site_name self._site_url = url __split_url = urlsplit(url) self._site_domain = __split_url.netloc self._base_url = f"{__split_url.scheme}://{__split_url.netloc}" self._site_cookie = site_cookie self._api_url = api_url self._session = session if session else None self._ua = ua self._emulate = emulate self._proxy = proxy self._index_html = "" # 用户信息 self.username = None self.userid = None self.user_level = None self.join_at = None self.bonus = 0.0 # 流量信息 self.upload = 0 self.download = 0 self.ratio = 0 # 做种信息 self.seeding = 0 self.leeching = 0 self.seeding_size = 0 self.leeching_size = 0 self.uploaded = 0 self.completed = 0 self.incomplete = 0 self.uploaded_size = 0 self.completed_size = 0 self.incomplete_size = 0 # 做种人数, 种子大小 self.seeding_info = [] # 未读消息 self.message_unread = 0 self.message_unread_contents = [] self.message_read_force = False # 全局附加请求头 self._addition_headers = None # 用户基础信息页面 self._user_basic_page = None # 用户基础信息参数 self._user_basic_params = None # 用户基础信息请求方法 self._user_basic_method = None # 用户基础信息请求头 self._user_basic_headers = None # 用户详情信息页面 self._user_detail_page = "userdetails.php?id=" # 用户详情信息参数 self._user_detail_params = None # 用户详情信息请求头 self._user_detail_headers = None # 用户流量信息页面 self._user_traffic_page = "index.php" # 用户流量信息参数 self._user_traffic_params = None # 用户流量信息请求头 self._user_traffic_headers = None # 用户未读消息页面 self._user_mail_unread_page = "messages.php?action=viewmailbox&box=1&unread=yes" # 系统未读消息页面 self._sys_mail_unread_page = "messages.php?action=viewmailbox&box=-2&unread=yes" # 未读消息数参数 self._mail_unread_params = None # 未读消息数请求头 self._mail_unread_headers = None # 未读消息内容参数 self._mail_content_params = None # 未读消息内容请求头 self._mail_content_headers = None # 用户做种信息页面 self._torrent_seeding_page = "getusertorrentlistajax.php?userid=" # 用户做种信息参数 self._torrent_seeding_params = None # 用户做种信息请求头 self._torrent_seeding_headers = None # 错误信息 self.err_msg: Optional[str] = None def site_schema(self) -> SiteSchema: """ 站点解析模型 :return: 站点解析模型 """ return self.schema @staticmethod def num_filesize(text) -> int: """ 将站点页面中的文件大小文本转换为字节。 """ return size_tools.parse_size(text) def parse(self): """ 解析站点信息 :return: """ try: # Cookie模式时，获取站点首页html if self.request_mode == "apikey": if not self.apikey and not self.token: logger.warn(f"{self._site_name} 未设置cookie 或 apikey/token，跳过后续操作") return self._index_html = {} else: # 检查是否已经登录 self._index_html = self._get_page_content(url=self._site_url) if not self._parse_logged_in(self._index_html): return # 解析站点页面 self._parse_site_page(self._index_html) # 解析用户基础信息 if self._user_basic_page: self._parse_user_base_info( self._get_page_content( url=urljoin(self._base_url, self._user_basic_page), params=self._user_basic_params, headers=self._user_basic_headers, **({"method": self._user_basic_method} if self._user_basic_method else {}), ) ) else: self._parse_user_base_info(self._index_html) # 解析用户详细信息 if self._user_detail_page: self._parse_user_detail_info( self._get_page_content( url=urljoin(self._base_url, self._user_detail_page), params=self._user_detail_params, headers=self._user_detail_headers ) ) # 解析用户未读消息 if get_runtime_setting('SITE_MESSAGE'): self._pase_unread_msgs() # 解析用户上传、下载、分享率等信息 if self._user_traffic_page: self._parse_user_traffic_info( self._get_page_content( url=urljoin(self._base_url, self._user_traffic_page), params=self._user_traffic_params, headers=self._user_traffic_headers ) ) # 解析用户做种信息 self._parse_seeding_pages() finally: # 关闭连接 self.close() def _pase_unread_msgs(self): """ 解析所有未读消息标题和内容 :return: """ unread_msg_links = [] if self.message_unread > 0 or self.message_read_force: links = {self._user_mail_unread_page, self._sys_mail_unread_page} for link in links: if not link: continue msg_links = [] next_page = self._parse_message_unread_links( self._get_page_content( url=urljoin(self._base_url, link), params=self._mail_unread_params, headers=self._mail_unread_headers ), msg_links) while next_page: next_page = self._parse_message_unread_links( self._get_page_content( url=urljoin(self._base_url, next_page), params=self._mail_unread_params, headers=self._mail_unread_headers ), msg_links ) unread_msg_links.extend(msg_links) # 重新更新未读消息数（99999表示有消息但数量未知） if unread_msg_links and not self.message_unread: self.message_unread = len(unread_msg_links) # 解析未读消息内容 for msg_link in unread_msg_links: logger.debug(f"{self._site_name} 信息链接 {msg_link}") head, date, content = self._parse_message_content( self._get_page_content( urljoin(self._base_url, msg_link), params=self._mail_content_params, headers=self._mail_content_headers ) ) logger.debug(f"{self._site_name} 标题 {head} 时间 {date} 内容 {content}") self.message_unread_contents.append((head, date, content)) def _parse_seeding_pages(self): """ 解析做种页面 """ if self._torrent_seeding_page: # 第一页 next_page = self._parse_user_torrent_seeding_info( self._get_page_content( url=urljoin(self._base_url, self._torrent_seeding_page), params=self._torrent_seeding_params, headers=self._torrent_seeding_headers ) ) # 其他页处理 while next_page is not None and next_page is not False: next_page = self._parse_user_torrent_seeding_info( self._get_page_content( url=urljoin(urljoin(self._base_url, self._torrent_seeding_page), next_page), params=self._torrent_seeding_params, headers=self._torrent_seeding_headers ), multi_page=True) @staticmethod def _prepare_html_text(html_text): """ 处理掉HTML中的干扰部分 """ return re.sub(r"#\d+", "", re.sub(r"\d+px", "", html_text)) @abstractmethod def _parse_message_unread_links(self, html_text: str, msg_links: list) -> Optional[str]: """ 获取未阅读消息链接 :param html_text: :return: """ pass def _get_page_content( self, url: str, params: dict = None, headers: dict = None, method: Optional[str] = None, ): """ 获取页面内容 :param url: 网页地址 :param params: post参数 :param headers: 额外的请求头 :param method: 强制使用的 HTTP 请求方法 :return: """ req_headers = None proxies = get_runtime_setting('PROXY') if self._proxy else None if self._ua or headers or self._addition_headers: if self.request_mode == "apikey": req_headers = {} else: req_headers = { "User-Agent": f"{self._ua}" } if headers: req_headers.update(headers) else: req_headers.update({ "Content-Type": "application/x-www-form-urlencoded; charset=UTF-8", }) if self._addition_headers: req_headers.update(self._addition_headers) if self.request_mode == "apikey": # 使用apikey请求，通过请求头传递 cookie = None session = None else: # 使用cookie请求 cookie = self._site_cookie session = self._session if method == "post" or params: if (req_headers or {}).get("Content-Type") == "application/json": res = RequestUtils(cookies=cookie, session=session, timeout=60, proxies=proxies, headers=req_headers).post_res(url=url, json=params or {}) else: res = RequestUtils(cookies=cookie, session=session, timeout=60, proxies=proxies, headers=req_headers).post_res(url=url, data=params or {}) else: res = RequestUtils(cookies=cookie, session=session, timeout=60, proxies=proxies, headers=req_headers).get_res(url=url) if res is not None and res.status_code in (200, 500, 403): if req_headers and "application/json" in str(req_headers.get("Accept")): try: return json.dumps(res.json()) except (json.JSONDecodeError, ValueError) as e: logger.error(f"{self._site_name} API响应JSON解析失败: {e}") return "" else: # 如果cloudflare 有防护，尝试使用浏览器仿真 if under_challenge(res.text): logger.warn( f"{self._site_name} 检测到Cloudflare，请更新Cookie和UA") return "" return RequestUtils.get_decoded_html_content(res, get_runtime_setting('ENCODING_DETECTION_PERFORMANCE_MODE'), get_runtime_setting('ENCODING_DETECTION_MIN_CONFIDENCE')) return "" @abstractmethod def _parse_site_page(self, html_text: str): """ 解析站点相关信息页面 :param html_text: :return: """ pass @abstractmethod def _parse_user_base_info(self, html_text: str): """ 解析用户基础信息 :param html_text: :return: """ pass def _parse_logged_in(self, html_text: str) -> bool: """ 解析用户是否已经登陆 :param html_text: :return: True/False """ logged_in = SiteUtils.is_logged_in(html_text) if not logged_in: self.err_msg = "未检测到已登陆，请检查cookies是否过期" logger.warn(f"{self._site_name} 未登录，跳过后续操作") return logged_in @abstractmethod def _parse_user_traffic_info(self, html_text: str): """ 解析用户的上传，下载，分享率等信息 :param html_text: :return: """ pass @abstractmethod def _parse_user_torrent_seeding_info(self, html_text: str, multi_page: bool = False) -> Optional[str]: """ 解析用户的做种相关信息 :param html_text: :param multi_page: 是否多页数据 :return: 下页地址 """ pass @abstractmethod def _parse_user_detail_info(self, html_text: str): """ 解析用户的详细信息 加入时间/等级/魔力值等 :param html_text: :return: """ pass @abstractmethod def _parse_message_content(self, html_text): """ 解析短消息内容 :param html_text: :return: head: message, date: time, content: message content """ pass def close(self): """ 关闭会话 """ if self._session: self._session.close() self._session = None def clear(self): """ 清除当前解析器的所有信息 """ self._index_html = "" self.seeding_info.clear() self.message_unread_contents.clear() def to_dict(self): """ 转化为字典 """ attributes = [ attr for attr in dir(self) if not callable(getattr(self, attr)) and not attr.startswith("_") ] return { attr: getattr(self, attr).value if isinstance(getattr(self, attr), SiteSchema) else getattr(self, attr) for attr in attributes }