mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-29 03:56:43 +08:00
536 lines
21 KiB
Python
536 lines
21 KiB
Python
import traceback
|
|
from typing import List, Tuple, Union, Optional
|
|
from urllib.parse import urljoin, urlparse
|
|
|
|
import dateutil.parser
|
|
from lxml import etree
|
|
|
|
from app.application.configuration import get_chain_runtime_config_snapshot
|
|
from app.adapters.network.browser import PlaywrightHelper
|
|
from app.runtime.log import logger
|
|
from app.adapters.system import rust as rust_accel
|
|
from app.adapters.network.http import RequestUtils
|
|
|
|
|
|
class RssHelper:
|
|
"""
|
|
RSS帮助类，解析RSS报文、获取RSS地址等
|
|
"""
|
|
|
|
# RSS解析限制配置
|
|
MAX_RSS_SIZE = 50 * 1024 * 1024 # 50MB最大RSS文件大小
|
|
MAX_RSS_ITEMS = 1000 # 最大解析条目数
|
|
|
|
# 各站点RSS链接获取配置
|
|
rss_link_conf = {
|
|
"default": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"hares.top": {
|
|
"xpath": "//*[@id='layui-layer100001']/div[2]/div/p[4]/a/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"et8.org": {
|
|
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/a[2]/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"pttime.org": {
|
|
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/text()[5]",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"showrows": 10,
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1
|
|
}
|
|
},
|
|
"ourbits.club": {
|
|
"xpath": "//a[@class='gen_rsslink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"totheglory.im": {
|
|
"xpath": "//textarea/text()",
|
|
"url": "rsstools.php?c51=51&c52=52&c53=53&c54=54&c108=108&c109=109&c62=62&c63=63&c67=67&c69=69&c70=70&c73=73&c76=76&c75=75&c74=74&c87=87&c88=88&c99=99&c90=90&c58=58&c103=103&c101=101&c60=60",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"monikadesign.uk": {
|
|
"xpath": "//a/@href",
|
|
"url": "rss",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"zhuque.in": {
|
|
"xpath": "//a/@href",
|
|
"url": "user/rss",
|
|
"render": True,
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
}
|
|
},
|
|
"hdchina.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"rsscart": 0
|
|
}
|
|
},
|
|
"audiences.me": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"torrent_type": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"shadowflow.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"paid": 0,
|
|
"search_mode": 0,
|
|
"showrows": 30
|
|
}
|
|
},
|
|
"hddolby.com": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"hdhome.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"pthome.net": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"exp": 180
|
|
}
|
|
},
|
|
"ptsbao.club": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"size": 0
|
|
}
|
|
},
|
|
"leaves.red": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 0,
|
|
"paid": 2
|
|
}
|
|
},
|
|
"hdtime.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 0,
|
|
}
|
|
},
|
|
"m-team.io": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"showrows": 50,
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"https": 1
|
|
}
|
|
},
|
|
"u2.dmhy.org": {
|
|
"xpath": "//a[@class='faqlink']/@href",
|
|
"url": "getrss.php",
|
|
"params": {
|
|
"inclbookmarked": 0,
|
|
"itemsmalldescr": 1,
|
|
"showrows": 50,
|
|
"search_mode": 1,
|
|
"inclautochecked": 1,
|
|
"trackerssl": 1
|
|
}
|
|
},
|
|
}
|
|
|
|
@classmethod
|
|
def _get_site_domain(cls, url: str) -> str:
|
|
"""按 RSS 站点配置匹配域名，未命中时回退到最后两级域名。"""
|
|
hostname = (urlparse(url).hostname or "").lower()
|
|
for domain in cls.rss_link_conf:
|
|
if domain == "default":
|
|
continue
|
|
if hostname == domain or hostname.endswith(f".{domain}"):
|
|
return domain
|
|
parts = hostname.split(".")
|
|
return ".".join(parts[-2:]) if len(parts) >= 2 else hostname
|
|
|
|
@staticmethod
|
|
def _parse_publish_time(value: str):
|
|
"""将 RSS 常见日期表达解析为 datetime，无法解析时返回 None。"""
|
|
try:
|
|
return dateutil.parser.parse(value)
|
|
except dateutil.parser.ParserError:
|
|
return None
|
|
|
|
def __parse_with_rust(self, ret_xml: Optional[str]) -> Optional[list]:
|
|
"""
|
|
调用 Rust RSS 解析器，并统一处理基础 XML 校验和最大条目限制。
|
|
"""
|
|
if not ret_xml or not ret_xml.strip():
|
|
return None
|
|
ret_xml_stripped = ret_xml.strip()
|
|
if not ret_xml_stripped.startswith('<'):
|
|
return None
|
|
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
|
|
if rust_items is None:
|
|
return None
|
|
if len(rust_items) > self.MAX_RSS_ITEMS:
|
|
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS}，仅处理前{self.MAX_RSS_ITEMS}个")
|
|
return rust_items[:self.MAX_RSS_ITEMS]
|
|
|
|
def parse(self, url, proxy: bool = False,
|
|
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
|
|
"""解析 RSS 地址并保留插件兼容的返回约定。"""
|
|
return self._parse_impl(url, proxy=proxy, timeout=timeout, headers=headers, ua=ua)
|
|
|
|
def _parse_impl(self, url, proxy: bool = False,
|
|
timeout: Optional[int] = 15, headers: dict = None,
|
|
ua: str = None) -> Union[List[dict], None, bool]:
|
|
"""
|
|
解析RSS订阅URL，获取RSS中的种子信息
|
|
:param url: RSS地址
|
|
:param proxy: 是否使用代理
|
|
:param timeout: 请求超时
|
|
:param headers: 自定义请求头
|
|
:param ua: 自定义User-Agent
|
|
:return: 种子信息列表，如为None代表Rss过期，如果为False则为错误
|
|
"""
|
|
# 开始处理
|
|
ret_array: list = []
|
|
if not url:
|
|
return False
|
|
|
|
try:
|
|
ret = RequestUtils(ua=ua,
|
|
proxies=get_chain_runtime_config_snapshot().proxy if proxy else None,
|
|
timeout=timeout or 30, headers=headers).get_res(url)
|
|
if not ret:
|
|
logger.error(f"获取RSS失败：请求返回空值，URL: {url}")
|
|
return False
|
|
except Exception as err:
|
|
logger.error(f"获取RSS失败：{str(err)} - {traceback.format_exc()}")
|
|
return False
|
|
|
|
if ret:
|
|
# 检查HTTP状态码
|
|
if ret.status_code != 200:
|
|
logger.error(f"RSS请求失败，状态码: {ret.status_code}, URL: {url}")
|
|
return False
|
|
ret_xml = None
|
|
root = None
|
|
try:
|
|
# 检查响应大小，避免处理过大的RSS文件
|
|
raw_data = ret.content
|
|
if raw_data and len(raw_data) > self.MAX_RSS_SIZE:
|
|
logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB，跳过解析")
|
|
return False
|
|
|
|
if raw_data:
|
|
ret_xml = RequestUtils.get_decoded_xml_content(
|
|
ret,
|
|
performance_mode=get_chain_runtime_config_snapshot().encoding_detection_performance_mode,
|
|
confidence_threshold=get_chain_runtime_config_snapshot().encoding_detection_min_confidence
|
|
)
|
|
rust_items = self.__parse_with_rust(ret_xml)
|
|
if rust_items is not None:
|
|
return rust_items
|
|
if not ret_xml:
|
|
ret_xml = ret.text
|
|
|
|
# 验证RSS内容是否有效
|
|
if not ret_xml or not ret_xml.strip():
|
|
logger.error("RSS内容为空")
|
|
return False
|
|
|
|
# 检查是否包含基本的RSS/XML结构
|
|
ret_xml_stripped = ret_xml.strip()
|
|
if not ret_xml_stripped.startswith('<'):
|
|
logger.error("RSS内容不是有效的XML格式")
|
|
return False
|
|
|
|
rust_items = self.__parse_with_rust(ret_xml)
|
|
if rust_items is not None:
|
|
return rust_items
|
|
|
|
# 使用lxml.etree解析XML
|
|
parser = None
|
|
try:
|
|
# 创建解析器，禁用网络访问以提高安全性和性能
|
|
parser = etree.XMLParser(
|
|
recover=True, # 容错模式
|
|
strip_cdata=False, # 保留CDATA
|
|
resolve_entities=False, # 禁用外部实体解析
|
|
no_network=True, # 禁用网络访问
|
|
huge_tree=False # 禁用大文档解析，避免内存问题
|
|
)
|
|
root = etree.fromstring(ret_xml.encode('utf-8'), parser=parser)
|
|
except etree.XMLSyntaxError as xml_error:
|
|
logger.debug(f"XML解析失败：{str(xml_error)}，尝试HTML解析")
|
|
# 如果XML解析失败，尝试作为HTML解析
|
|
try:
|
|
root = etree.HTML(ret_xml)
|
|
if root is not None:
|
|
# 查找RSS根节点
|
|
rss_root = root.xpath('//rss | //feed')
|
|
if rss_root:
|
|
root = rss_root[0]
|
|
except Exception as e:
|
|
logger.error(f"HTML解析也失败：{str(e)}")
|
|
return False
|
|
except Exception as general_error:
|
|
logger.error(f"解析RSS时发生未预期错误：{str(general_error)}")
|
|
return False
|
|
finally:
|
|
if parser is not None:
|
|
try:
|
|
parser.close()
|
|
except Exception as close_error:
|
|
logger.debug(f"关闭解析器时出错：{str(close_error)}")
|
|
del parser
|
|
|
|
if root is None:
|
|
logger.error("无法解析RSS内容")
|
|
return False
|
|
|
|
# 查找所有item或entry节点
|
|
items = root.xpath('.//item | .//entry')
|
|
|
|
# 限制处理的条目数量
|
|
items_count = min(len(items), self.MAX_RSS_ITEMS)
|
|
if len(items) > self.MAX_RSS_ITEMS:
|
|
logger.warning(f"RSS条目过多: {len(items)}，仅处理前{self.MAX_RSS_ITEMS}个")
|
|
try:
|
|
for item in items[:items_count]:
|
|
try:
|
|
# 使用xpath提取信息，更高效
|
|
title_nodes = item.xpath('.//title')
|
|
title = title_nodes[0].text if title_nodes and title_nodes[0].text else ""
|
|
if not title:
|
|
continue
|
|
|
|
# 描述
|
|
desc_nodes = item.xpath('.//description | .//summary')
|
|
description = desc_nodes[0].text if desc_nodes and desc_nodes[0].text else ""
|
|
|
|
# 种子页面
|
|
link_nodes = item.xpath('.//link')
|
|
if link_nodes:
|
|
link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else link_nodes[0].get('href', '')
|
|
else:
|
|
link = ""
|
|
|
|
# 种子链接
|
|
enclosure_nodes = item.xpath('.//enclosure')
|
|
enclosure = enclosure_nodes[0].get('url', '') if enclosure_nodes else ""
|
|
if not enclosure and not link:
|
|
continue
|
|
# 部分RSS只有link没有enclosure
|
|
if not enclosure and link:
|
|
enclosure = link
|
|
|
|
# 大小
|
|
size = 0
|
|
if enclosure_nodes:
|
|
size_attr = enclosure_nodes[0].get('length', '0')
|
|
if size_attr and str(size_attr).isdigit():
|
|
size = int(size_attr)
|
|
|
|
# 发布日期
|
|
pubdate_nodes = item.xpath('./pubDate | ./published | ./updated')
|
|
if not pubdate_nodes:
|
|
pubdate_nodes = item.xpath('.//*[local-name()="pubDate"] | .//*[local-name()="published"] | .//*[local-name()="updated"]')
|
|
|
|
pubdate = ""
|
|
if pubdate_nodes and pubdate_nodes[0].text:
|
|
pubdate = self._parse_publish_time(pubdate_nodes[0].text)
|
|
if pubdate is not None:
|
|
# 转为本地时区
|
|
pubdate = pubdate.astimezone(tz=None)
|
|
|
|
# 获取豆瓣昵称
|
|
nickname_nodes = item.xpath('.//*[local-name()="creator"]')
|
|
nickname = nickname_nodes[0].text if nickname_nodes and nickname_nodes[0].text else ""
|
|
|
|
# 返回对象
|
|
tmp_dict = {
|
|
'title': title,
|
|
'enclosure': enclosure,
|
|
'size': size,
|
|
'description': description,
|
|
'link': link,
|
|
'pubdate': pubdate
|
|
}
|
|
# 如果豆瓣昵称不为空，返回数据增加豆瓣昵称，供doubansync插件获取
|
|
if nickname:
|
|
tmp_dict['nickname'] = nickname
|
|
ret_array.append(tmp_dict)
|
|
|
|
except Exception as e1:
|
|
logger.debug(f"解析RSS条目失败：{str(e1)} - {traceback.format_exc()}")
|
|
continue
|
|
finally:
|
|
items.clear()
|
|
del items
|
|
|
|
except Exception as e2:
|
|
logger.error(f"解析RSS失败：{str(e2)} - {traceback.format_exc()}")
|
|
# RSS过期检查
|
|
_rss_expired_msg = [
|
|
"RSS 链接已过期, 您需要获得一个新的!",
|
|
"RSS Link has expired, You need to get a new one!",
|
|
"RSS Link has expired, You need to get new!"
|
|
]
|
|
if ret_xml in _rss_expired_msg:
|
|
return None
|
|
return False
|
|
finally:
|
|
if root is not None:
|
|
del root
|
|
if ret_xml is not None:
|
|
del ret_xml
|
|
|
|
return ret_array
|
|
|
|
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False, timeout: int = None) -> Tuple[str, str]:
|
|
"""
|
|
获取站点rss地址
|
|
:param url: 站点地址
|
|
:param cookie: 站点cookie
|
|
:param ua: 站点ua
|
|
:param proxy: 是否使用代理
|
|
:param timeout: 请求超时时间
|
|
:return: rss地址、错误信息
|
|
"""
|
|
try:
|
|
# 获取站点域名
|
|
domain = self._get_site_domain(url)
|
|
# 获取配置
|
|
site_conf = self.rss_link_conf.get(domain) or self.rss_link_conf.get("default")
|
|
# RSS地址
|
|
rss_url = urljoin(url, site_conf.get("url"))
|
|
# RSS请求参数
|
|
rss_params = site_conf.get("params")
|
|
# 请求RSS页面
|
|
if site_conf.get("render"):
|
|
html_text = PlaywrightHelper().get_page_source(
|
|
url=rss_url,
|
|
cookies=cookie,
|
|
ua=ua,
|
|
proxies=get_chain_runtime_config_snapshot().proxy_server if proxy else None,
|
|
timeout=timeout or 60
|
|
)
|
|
else:
|
|
res = RequestUtils(
|
|
cookies=cookie,
|
|
timeout=timeout or 30,
|
|
ua=ua,
|
|
proxies=get_chain_runtime_config_snapshot().proxy if proxy else None
|
|
).post_res(url=rss_url, data=rss_params)
|
|
if res:
|
|
html_text = res.text
|
|
elif res is not None:
|
|
return "", f"获取 {url} RSS链接失败，错误码：{res.status_code}，错误原因：{res.reason}"
|
|
else:
|
|
return "", f"获取RSS链接失败：无法连接 {url} "
|
|
|
|
# 解析HTML
|
|
if html_text:
|
|
html = None
|
|
try:
|
|
html = etree.HTML(html_text)
|
|
if html is not None and len(html) > 0:
|
|
rss_link = html.xpath(site_conf.get("xpath"))
|
|
if rss_link:
|
|
return str(rss_link[-1]), ""
|
|
finally:
|
|
if html is not None:
|
|
del html
|
|
|
|
return "", f"获取RSS链接失败：{url}"
|
|
except Exception as e:
|
|
return "", f"获取 {url} RSS链接失败：{str(e)}"
|